From 9f376701b46a6d1f3928ab54df7949c80ddaf0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Fri, 10 Feb 2023 13:30:08 +0100 Subject: [PATCH 01/28] added geolocation check to filter.py and geolocation functions to server.py --- fail2ban/server/filter.py | 42 +++++++++++++++++++++++++++++++++++++++ fail2ban/server/server.py | 9 +++++++++ 2 files changed, 51 insertions(+) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 6896828477..933f2730d1 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -29,6 +29,9 @@ import re import sys import time +import json + +from urllib.request import urlopen, Request from .actions import Actions from .failmanager import FailManagerEmpty, FailManager @@ -83,6 +86,8 @@ def __init__(self, jail, useDns='warn'): ## The ignore IP list. self.__ignoreIpSet = set() self.__ignoreIpList = [] + ## The ignore GEO ip list. + self.__ignoreGeoSet = set() ## External command self.__ignoreCommand = False ## Cache for ignoreip: @@ -541,6 +546,33 @@ def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): def getIgnoreIP(self): return self.__ignoreIpList + list(self.__ignoreIpSet) + def addIgnoreGEO(self, geo): + # An empty string is always false + if geo == "": + return + + # Avoid exact duplicates + if geo in self.__ignoreGeoSet: + logSys.log(logging.MSG, " Ignore duplicate %r, already in geo ignore list", geo) + return + + # log and append to ignore list + logSys.debug(" Add %r to geo ignore list", geo) + self.__ignoreGeoSet.add(geo) + + def delIgnoreGEO(self, geo=None): + # clear all: + if geo is None: + self.__ignoreGeoSet.clear() + return + # delete by ip: + logSys.debug(" Remove %r from geo ignore list", geo) + if geo in self.__ignoreGeoSet: + self.__ignoreGeoSet.remove(geo) + + def getIgnoreGEO(self): + return list(self.__ignoreGeoSet) + ## # Check if IP address/DNS is in the ignore list. # @@ -579,6 +611,16 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): if self.__ignoreCache: c.set(key, True) return True + # check if the IP's geolocation is not on geo ignore list + httprequest = Request("http://ip-api.com/json/" + str(ip), headers={"Accept": "application/json"}) + + response = urlopen(httprequest) + geoipcc = json.loads(response.read().decode())['countryCode'] + + if response == 200 and geoipcc in self.__ignoreGeoSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) + return True + # check if the IP is covered by ignore IP (in set or in subnet/dns): if ip in self.__ignoreIpSet: self.logIgnoreIp(ip, log_ignore, ignore_source="ip") diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 660f791899..ce201b5e7b 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -375,6 +375,15 @@ def delIgnoreIP(self, name, ip): def getIgnoreIP(self, name): return self.__jails[name].filter.getIgnoreIP() + + def addIgnoreGEO(self, name, geo): + self.__jails[name].filter.addIgnoreGEO(geo) + + def delIgnoreGEO(self, name, geo): + self.__jails[name].filter.delIgnoreGEO(geo) + + def getIgnoreGEO(self, name): + return self.__jails[name].filter.getIgnoreGEO() def addLogPath(self, name, fileName, tail=False): filter_ = self.__jails[name].filter From 6c446ebfc00554c5007feb073b983ea597fed0ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Fri, 10 Feb 2023 13:37:21 +0100 Subject: [PATCH 02/28] added geolocation function calls to transmitter.py --- fail2ban/server/filter.py | 1 - fail2ban/server/transmitter.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 933f2730d1..811ecc8493 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -613,7 +613,6 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): # check if the IP's geolocation is not on geo ignore list httprequest = Request("http://ip-api.com/json/" + str(ip), headers={"Accept": "application/json"}) - response = urlopen(httprequest) geoipcc = json.loads(response.read().decode())['countryCode'] diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 6de60f94e1..416a032880 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -235,6 +235,16 @@ def __commandSet(self, command, multiple=False): self.__server.delIgnoreIP(name, value) if self.__quiet: return return self.__server.getIgnoreIP(name) + elif command[1] == "addignoregeo": + for value in command[2:]: + self.__server.addIgnoreGEO(name, value) + if self.__quiet: return + return self.__server.getIgnoreGEO(name) + elif command[1] == "delignoregeo": + value = command[2] + self.__server.delIgnoreGEO(name, value) + if self.__quiet: return + return self.__server.getIgnoreGEO(name) elif command[1] == "ignorecommand": value = command[2] self.__server.setIgnoreCommand(name, value) From 0466e9e00d2e5d6a504de4690354d81681fe1c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 08:30:36 +0100 Subject: [PATCH 03/28] added addignoregeo to manpage and protocol.py --- fail2ban/protocol.py | 2 ++ man/fail2ban-client.1 | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index a81c665724..0e61f5948c 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -88,6 +88,8 @@ def __getattr__(self, name): ["set ignoreself true|false", "allows the ignoring of own IP addresses"], ["set addignoreip ", "adds to the ignore list of "], ["set delignoreip ", "removes from the ignore list of "], +["set addignoregeo ", "adds to the ignore list of "], +["set delignoregeo ", "removes from the ignore list of "], ["set ignorecommand ", "sets ignorecommand of "], ["set ignorecache ", "sets ignorecache of "], ["set addlogpath ['tail']", "adds to the monitoring list of , optionally starting at the 'tail' of the file (default 'head')."], diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 9fa0b8033d..7ea96a4191 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -229,6 +229,14 @@ adds to the ignore list of removes from the ignore list of .TP +\fBset addignoregeo \fR +adds to the ignore list of + +.TP +\fBset delignoreigeo \fR +removes from the ignore list +of +.TP \fBset ignorecommand \fR sets ignorecommand of .TP From 0b21d7c87f6795a07d59cbb72594e1b86def09a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 08:38:36 +0100 Subject: [PATCH 04/28] added ignoregeo to jailreader --- fail2ban/client/jailreader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 37746d4c31..9706c53054 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -108,6 +108,7 @@ def _glob(path): "ignorecommand": ["string", None], "ignoreself": ["bool", None], "ignoreip": ["string", None], + "ignoregeo": ["string", None], "ignorecache": ["string", None], "filter": ["string", ""], "logtimezone": ["string", None], @@ -264,6 +265,8 @@ def convert(self, allow_no_files=False): backend = value elif opt == "ignoreip": stream.append(["set", self.__name, "addignoreip"] + splitwords(value)) + elif opt == "ignoregeo": + stream.append(["set", self.__name, "addignoregeo"] + splitwords(value)) elif opt not in JailReader._ignoreOpts: stream.append(["set", self.__name, opt, value]) # consider options order (after other options): From 41dbb824ed277ae6cee616fe6a53eb7dc8ed7f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 08:55:31 +0100 Subject: [PATCH 05/28] added ignoregeo commands to client and manpages --- fail2ban/protocol.py | 1 + man/fail2ban-client.1 | 4 ++++ man/jail.conf.5 | 3 +++ 3 files changed, 8 insertions(+) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 0e61f5948c..d0cb1ed8a4 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -131,6 +131,7 @@ def __getattr__(self, name): ["get journalmatch", "gets the journal filter match for "], ["get ignoreself", "gets the current value of the ignoring the own IP addresses"], ["get ignoreip", "gets the list of ignored IP addresses for "], +["get ignoregeo", "gets the list of ignored GEO locations for "], ["get ignorecommand", "gets ignorecommand of "], ["get failregex", "gets the list of regular expressions which matches the failures for "], ["get ignoreregex", "gets the list of regular expressions which matches patterns to ignore for "], diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 7ea96a4191..b8e0b37b5c 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -400,6 +400,10 @@ ignoring the own IP addresses gets the list of ignored IP addresses for .TP +\fBget ignoregeo\fR +gets the list of ignored GEO +addresses for +.TP \fBget ignorecommand\fR gets ignorecommand of .TP diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 052fce80bf..5f8a1ce0d0 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -246,6 +246,9 @@ boolean value (default true) indicates the banning of own IP addresses should be .B ignoreip list of IPs not to ban. They can include a DNS resp. CIDR mask too. The option affects additionally to \fBignoreself\fR (if true) and don't need to contain own DNS resp. IPs of the running host. .TP +.B ignoregeo +list of GEO location codes of IPs not to ban. The IPs will be queried on specified GEOIP database server. +.TP .B ignorecommand command that is executed to determine if the current candidate IP for banning (or failure-ID for raw IDs) should not be banned. The option affects additionally to \fBignoreself\fR and \fBignoreip\fR and will be first executed if both don't hit. .br From 6513bc244e4910c1b58d6f7bfd92a61b8d6d8671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 08:57:08 +0100 Subject: [PATCH 06/28] added getignoregeo to transmitter.py --- fail2ban/server/transmitter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 416a032880..c25ae637a1 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -463,6 +463,8 @@ def __commandGet(self, command): return self.__server.getIgnoreSelf(name) elif command[1] == "ignoreip": return self.__server.getIgnoreIP(name) + elif command[1] == "ignoregeo": + return self.__server.getIgnoreGEO(name) elif command[1] == "ignorecommand": return self.__server.getIgnoreCommand(name) elif command[1] == "ignorecache": From 777da5d742b6d4cf677399cfd2e2348230bd4bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 09:01:09 +0100 Subject: [PATCH 07/28] added ignoregeo to client/beautifier.py --- fail2ban/client/beautifier.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 97cd38b21c..935c627451 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -147,6 +147,14 @@ def beautify(self, response): for ip in response[:-1]: msg += "|- " + ip + "\n" msg += "`- " + response[-1] + elif inC[2] in ("ignoregeo", "addignoregeo", "delignoregeo"): + if len(response) == 0: + msg = "No GEO location is ignored" + else: + msg = "These GEO locations are ignored:\n" + for geo in response[:-1]: + msg += "|- " + geo + "\n" + msg += "`- " + response[-1] elif inC[2] in ("failregex", "addfailregex", "delfailregex", "ignoreregex", "addignoreregex", "delignoreregex"): if len(response) == 0: From 7e7138307ea827b466e46c621aeb02560d184d9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 09:29:12 +0100 Subject: [PATCH 08/28] fixed bug with response.getcode() --- fail2ban/server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 811ecc8493..1f2eecb08d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -616,7 +616,7 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): response = urlopen(httprequest) geoipcc = json.loads(response.read().decode())['countryCode'] - if response == 200 and geoipcc in self.__ignoreGeoSet: + if response.getcode() == 200 and geoipcc in self.__ignoreGeoSet: self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) return True From 2c7120aea26c7df1ee552c9a8bd7d8a72fcfc913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 10:51:49 +0100 Subject: [PATCH 09/28] added setters and getters for geoip api to server, filter and transmitter --- fail2ban/server/filter.py | 15 +++++++++++++++ fail2ban/server/server.py | 6 ++++++ fail2ban/server/transmitter.py | 5 +++++ 3 files changed, 26 insertions(+) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 1f2eecb08d..d4186b8ea2 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -86,6 +86,8 @@ def __init__(self, jail, useDns='warn'): ## The ignore IP list. self.__ignoreIpSet = set() self.__ignoreIpList = [] + # GEO-IP database api endpoint + self.__ignoreGeoApi = None ## The ignore GEO ip list. self.__ignoreGeoSet = set() ## External command @@ -546,6 +548,19 @@ def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): def getIgnoreIP(self): return self.__ignoreIpList + list(self.__ignoreIpSet) + def setIgnoreGEOAPI(self, api): + if api == "": + return + + if self.__ignoreGeoApi: + logSys.log(logging.MSG, " GEO API was already set to %r and will be changed.", self.__ignoreGeoApi) + + logSys.debug(" Set %r as GEO API endpoint", api) + self.__ignoreGeoApi = api + + def getIgnoreGEOAPI(self): + return self.__ignoreGeoApi + def addIgnoreGEO(self, geo): # An empty string is always false if geo == "": diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index ce201b5e7b..e94b2e0eb8 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -376,6 +376,12 @@ def delIgnoreIP(self, name, ip): def getIgnoreIP(self, name): return self.__jails[name].filter.getIgnoreIP() + def setIgnoreGEOAPI(self, name, api): + self.__jails[name].filter.setIgnoreGEOAPI(api) + + def getIgnoreGEOAPI(self, name): + self.__jails[name].filter.getIgnoreGEOAPI() + def addIgnoreGEO(self, name, geo): self.__jails[name].filter.addIgnoreGEO(geo) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index c25ae637a1..fc50d059c6 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -235,6 +235,11 @@ def __commandSet(self, command, multiple=False): self.__server.delIgnoreIP(name, value) if self.__quiet: return return self.__server.getIgnoreIP(name) + elif command[1] == "setignoregeoapi": + for value in command[2:]: + self.__server.setIgnoreGEOAPI(name, value) + if self.__quiet: return + return self.__server.getIgnoreGEOAPI(name) elif command[1] == "addignoregeo": for value in command[2:]: self.__server.addIgnoreGEO(name, value) From d3c18540ade543f8bb9cfb12751c145cb268c2ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 12:53:49 +0100 Subject: [PATCH 10/28] added ignoregeoapi to jaireader --- fail2ban/client/jailreader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 9706c53054..792f1af717 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -108,6 +108,7 @@ def _glob(path): "ignorecommand": ["string", None], "ignoreself": ["bool", None], "ignoreip": ["string", None], + "ignoregeoapi": ["string", None], "ignoregeo": ["string", None], "ignorecache": ["string", None], "filter": ["string", ""], From 01a843684bf0c32f862f8c9c9e93a78477f4fdc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 15:13:51 +0100 Subject: [PATCH 11/28] Revert "fixed bug with response.getcode()" This reverts commit 7e7138307ea827b466e46c621aeb02560d184d9e. --- fail2ban/server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index d4186b8ea2..f2b860e5c9 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -631,7 +631,7 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): response = urlopen(httprequest) geoipcc = json.loads(response.read().decode())['countryCode'] - if response.getcode() == 200 and geoipcc in self.__ignoreGeoSet: + if response == 200 and geoipcc in self.__ignoreGeoSet: self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) return True From 32934ce863246985363ec0e10bbc72f0d16e0149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 15:15:46 +0100 Subject: [PATCH 12/28] Revert "added ignoregeoapi to jaireader" This reverts commit d3c18540ade543f8bb9cfb12751c145cb268c2ea. --- fail2ban/client/jailreader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 792f1af717..9706c53054 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -108,7 +108,6 @@ def _glob(path): "ignorecommand": ["string", None], "ignoreself": ["bool", None], "ignoreip": ["string", None], - "ignoregeoapi": ["string", None], "ignoregeo": ["string", None], "ignorecache": ["string", None], "filter": ["string", ""], From 4296f0ed6b32543e8ca8418d6afe28a25d57ab5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 15:15:54 +0100 Subject: [PATCH 13/28] Revert "added setters and getters for geoip api to server, filter and transmitter" This reverts commit 2c7120aea26c7df1ee552c9a8bd7d8a72fcfc913. --- fail2ban/server/filter.py | 15 --------------- fail2ban/server/server.py | 6 ------ fail2ban/server/transmitter.py | 5 ----- 3 files changed, 26 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index f2b860e5c9..811ecc8493 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -86,8 +86,6 @@ def __init__(self, jail, useDns='warn'): ## The ignore IP list. self.__ignoreIpSet = set() self.__ignoreIpList = [] - # GEO-IP database api endpoint - self.__ignoreGeoApi = None ## The ignore GEO ip list. self.__ignoreGeoSet = set() ## External command @@ -548,19 +546,6 @@ def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): def getIgnoreIP(self): return self.__ignoreIpList + list(self.__ignoreIpSet) - def setIgnoreGEOAPI(self, api): - if api == "": - return - - if self.__ignoreGeoApi: - logSys.log(logging.MSG, " GEO API was already set to %r and will be changed.", self.__ignoreGeoApi) - - logSys.debug(" Set %r as GEO API endpoint", api) - self.__ignoreGeoApi = api - - def getIgnoreGEOAPI(self): - return self.__ignoreGeoApi - def addIgnoreGEO(self, geo): # An empty string is always false if geo == "": diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index e94b2e0eb8..ce201b5e7b 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -376,12 +376,6 @@ def delIgnoreIP(self, name, ip): def getIgnoreIP(self, name): return self.__jails[name].filter.getIgnoreIP() - def setIgnoreGEOAPI(self, name, api): - self.__jails[name].filter.setIgnoreGEOAPI(api) - - def getIgnoreGEOAPI(self, name): - self.__jails[name].filter.getIgnoreGEOAPI() - def addIgnoreGEO(self, name, geo): self.__jails[name].filter.addIgnoreGEO(geo) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index fc50d059c6..c25ae637a1 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -235,11 +235,6 @@ def __commandSet(self, command, multiple=False): self.__server.delIgnoreIP(name, value) if self.__quiet: return return self.__server.getIgnoreIP(name) - elif command[1] == "setignoregeoapi": - for value in command[2:]: - self.__server.setIgnoreGEOAPI(name, value) - if self.__quiet: return - return self.__server.getIgnoreGEOAPI(name) elif command[1] == "addignoregeo": for value in command[2:]: self.__server.addIgnoreGEO(name, value) From 15f51945e7010de7c5db47b72338889448906fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 16 Feb 2023 15:23:09 +0100 Subject: [PATCH 14/28] replace API with geoiplookup --- fail2ban/server/filter.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 811ecc8493..6e8d4a7293 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -29,9 +29,9 @@ import re import sys import time -import json +import subprocess +import shutil -from urllib.request import urlopen, Request from .actions import Actions from .failmanager import FailManagerEmpty, FailManager @@ -612,13 +612,12 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): return True # check if the IP's geolocation is not on geo ignore list - httprequest = Request("http://ip-api.com/json/" + str(ip), headers={"Accept": "application/json"}) - response = urlopen(httprequest) - geoipcc = json.loads(response.read().decode())['countryCode'] + if shutil.which('geoiplookup'): + geoipcc = str(subprocess.check_output(['geoiplookup',str(ip)])).split(" ")[3].replace(",","") - if response == 200 and geoipcc in self.__ignoreGeoSet: - self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) - return True + if geoipcc in self.__ignoreGeoSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) + return True # check if the IP is covered by ignore IP (in set or in subnet/dns): if ip in self.__ignoreIpSet: From aadae7c046f96b2d232343ef6a5f4600ecf4c25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 08:25:52 +0100 Subject: [PATCH 15/28] updated manual page --- man/jail.conf.5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 5f8a1ce0d0..5e319de678 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -247,7 +247,7 @@ boolean value (default true) indicates the banning of own IP addresses should be list of IPs not to ban. They can include a DNS resp. CIDR mask too. The option affects additionally to \fBignoreself\fR (if true) and don't need to contain own DNS resp. IPs of the running host. .TP .B ignoregeo -list of GEO location codes of IPs not to ban. The IPs will be queried on specified GEOIP database server. +list of GEO location codes of IPs not to ban. For this to work, you should have a `geoiplookup` or other program with the same name installed. This feature was only tested with Maxmind's `geoiplookup`. .TP .B ignorecommand command that is executed to determine if the current candidate IP for banning (or failure-ID for raw IDs) should not be banned. The option affects additionally to \fBignoreself\fR and \fBignoreip\fR and will be first executed if both don't hit. From 73ee9f718e8a5582080d8705d7fecc388dfec32a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 08:45:41 +0100 Subject: [PATCH 16/28] added warning if geoiplookup is not present --- fail2ban/server/filter.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 6e8d4a7293..1a25f3c11b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -612,12 +612,15 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): return True # check if the IP's geolocation is not on geo ignore list - if shutil.which('geoiplookup'): - geoipcc = str(subprocess.check_output(['geoiplookup',str(ip)])).split(" ")[3].replace(",","") + if self.__ignoreGeoSet: + if shutil.which("geoiplookup"): + geoipcc = str(subprocess.check_output(["geoiplookup",str(ip)])).split(" ")[3].replace(",","") - if geoipcc in self.__ignoreGeoSet: - self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) - return True + if geoipcc in self.__ignoreGeoSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) + return True + else: + self._logWarnOnce("_next_geoByTimeWarn", ("Cannot find geoiplookup. Geolocation is unavailable.")) # check if the IP is covered by ignore IP (in set or in subnet/dns): if ip in self.__ignoreIpSet: From c25e8095d63eb9855398692409f96c8b960fd50b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 08:48:14 +0100 Subject: [PATCH 17/28] added geoiplookup as optional to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6bf94c25b7..280b4217d7 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Optional: - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) and python bindings: * [python-systemd package](https://www.freedesktop.org/software/systemd/python-systemd/index.html) - [dnspython](http://www.dnspython.org/) +- [geoiplookup](https://linux.die.net/man/1/geoiplookup) To install: From 5876032904c12dfb7a60b56132ae815009c750c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 10:54:40 +0100 Subject: [PATCH 18/28] added mmdblookup since geoiplookup DB is retired --- fail2ban/server/filter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 1a25f3c11b..546719de4e 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -612,10 +612,12 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): return True # check if the IP's geolocation is not on geo ignore list + geoipcc = '' if self.__ignoreGeoSet: - if shutil.which("geoiplookup"): + if shutil.which("mmdblookup"): + geoipcc = str(subprocess.check_output(["mmdblookup","--file","/usr/share/GeoIP/GeoIP2-Country.mmdb","--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") + elif shutil.which("geoiplookup"): geoipcc = str(subprocess.check_output(["geoiplookup",str(ip)])).split(" ")[3].replace(",","") - if geoipcc in self.__ignoreGeoSet: self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) return True From f0bb798ecc5812f5fab213dffd7ea57b534f50f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 11:02:48 +0100 Subject: [PATCH 19/28] added geoip2 country file check --- fail2ban/server/filter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 546719de4e..5f94619af9 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -612,9 +612,10 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): return True # check if the IP's geolocation is not on geo ignore list - geoipcc = '' + geoipcc = "" + geoip2cf = "/usr/share/GeoIP/GeoIP2-Country.mmdb" if self.__ignoreGeoSet: - if shutil.which("mmdblookup"): + if shutil.which("mmdblookup") and os.path.isfile(geoip2cf): geoipcc = str(subprocess.check_output(["mmdblookup","--file","/usr/share/GeoIP/GeoIP2-Country.mmdb","--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") elif shutil.which("geoiplookup"): geoipcc = str(subprocess.check_output(["geoiplookup",str(ip)])).split(" ")[3].replace(",","") From 8b609ff63924d32cc4e365e41ecc1324d98ef5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 11:16:05 +0100 Subject: [PATCH 20/28] added logs for geoiplookup --- fail2ban/server/filter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 5f94619af9..cc1434a961 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -616,14 +616,17 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): geoip2cf = "/usr/share/GeoIP/GeoIP2-Country.mmdb" if self.__ignoreGeoSet: if shutil.which("mmdblookup") and os.path.isfile(geoip2cf): - geoipcc = str(subprocess.check_output(["mmdblookup","--file","/usr/share/GeoIP/GeoIP2-Country.mmdb","--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") + geoipcc = str(subprocess.check_output(["mmdblookup","--file",geoip2cf,"--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") elif shutil.which("geoiplookup"): + if shutil.which("mmdblookup") and !os.path.isfile(geoip2cf): + self._logWarnOnce("_next_geocfByTimeWarn", ("Found mmdblookup but cannot find mmdb country file at /usr/share/GeoIP/GeoIP2-Country.mmdb, using geoiplookup instead.")) + geoipcc = str(subprocess.check_output(["geoiplookup",str(ip)])).split(" ")[3].replace(",","") if geoipcc in self.__ignoreGeoSet: self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) return True else: - self._logWarnOnce("_next_geoByTimeWarn", ("Cannot find geoiplookup. Geolocation is unavailable.")) + self._logWarnOnce("_next_geoByTimeWarn", ("Cannot find geoiplookup or correctly set mmdblookup. Geolocation is unavailable. If you have mmdblookup installed, please check if the mmdb country file is located at /usr/share/GeoIP/GeoIP2-Country.mmdb. If not run geoipupdate.")) # check if the IP is covered by ignore IP (in set or in subnet/dns): if ip in self.__ignoreIpSet: From 87e21b3d23c835f7b7a77f8a13e74f97d0f33453 Mon Sep 17 00:00:00 2001 From: Robert Belanec Date: Thu, 23 Feb 2023 11:18:36 +0100 Subject: [PATCH 21/28] fixed typo --- fail2ban/client/actionreader.py | 4 +- fail2ban/client/configparserinc.py | 12 +-- fail2ban/client/configreader.py | 4 +- fail2ban/client/csocket.py | 4 +- fail2ban/client/fail2banclient.py | 4 +- fail2ban/client/fail2banregex.py | 22 ++--- fail2ban/client/filterreader.py | 2 +- fail2ban/client/jailreader.py | 4 +- fail2ban/helpers.py | 15 +-- fail2ban/server/action.py | 19 ++-- fail2ban/server/actions.py | 28 +++--- fail2ban/server/asyncserver.py | 4 +- fail2ban/server/banmanager.py | 14 +-- fail2ban/server/database.py | 6 +- fail2ban/server/failmanager.py | 8 +- fail2ban/server/failregex.py | 8 +- fail2ban/server/filterpoll.py | 2 +- fail2ban/server/filterpyinotify.py | 4 +- fail2ban/server/filtersystemd.py | 2 +- fail2ban/server/ipdns.py | 18 ++-- fail2ban/server/jail.py | 14 +-- fail2ban/server/mytime.py | 2 +- fail2ban/server/server.py | 20 ++-- fail2ban/server/strptime.py | 8 +- fail2ban/server/ticket.py | 14 +-- fail2ban/server/transmitter.py | 2 +- fail2ban/server/utils.py | 4 +- fail2ban/tests/actiontestcase.py | 4 +- fail2ban/tests/banmanagertestcase.py | 2 +- fail2ban/tests/clientreadertestcase.py | 4 +- fail2ban/tests/databasetestcase.py | 16 +-- fail2ban/tests/datedetectortestcase.py | 6 +- fail2ban/tests/fail2banclienttestcase.py | 8 +- fail2ban/tests/fail2banregextestcase.py | 4 +- fail2ban/tests/failmanagertestcase.py | 10 +- .../tests/files/config/apache-auth/digest.py | 20 ++-- fail2ban/tests/filtertestcase.py | 98 +++++++++---------- fail2ban/tests/misctestcase.py | 22 ++--- fail2ban/tests/observertestcase.py | 34 +++---- fail2ban/tests/samplestestcase.py | 8 +- fail2ban/tests/servertestcase.py | 28 +++--- fail2ban/tests/sockettestcase.py | 2 +- fail2ban/tests/utils.py | 22 ++--- 43 files changed, 267 insertions(+), 269 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 88b0aca140..5a57338ba2 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -89,11 +89,11 @@ def convert(self): stream = list() stream.append(head + ["addaction", self._name]) multi = [] - for opt, optval in opts.iteritems(): + for opt, optval in opts.items(): if opt in self._configOpts and not opt.startswith('known/'): multi.append([opt, optval]) if self._initOpts: - for opt, optval in self._initOpts.iteritems(): + for opt, optval in self._initOpts.items(): if opt not in self._configOpts and not opt.startswith('known/'): multi.append([opt, optval]) if len(multi) > 1: diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index cc4ada0aed..e46379a179 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -62,7 +62,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, parser, option, accum, rest, section, map, *args, **kwargs) else: # pragma: 3.x no cover - from ConfigParser import SafeConfigParser, \ + from configparser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError # Interpolate missing known/option as option from default section @@ -327,7 +327,7 @@ def options(self, section, withDefault=True): # mix it with defaults: return set(opts.keys()) | set(self._defaults) # only own option names: - return opts.keys() + return list(opts.keys()) def read(self, filenames, get_includes=True): if not isinstance(filenames, list): @@ -356,7 +356,7 @@ def read(self, filenames, get_includes=True): ret += i # merge defaults and all sections to self: alld.update(cfg.get_defaults()) - for n, s in cfg.get_sections().iteritems(): + for n, s in cfg.get_sections().items(): # conditional sections cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n) if cond: @@ -366,14 +366,14 @@ def read(self, filenames, get_includes=True): del(s['__name__']) except KeyError: pass - for k in s.keys(): + for k in list(s.keys()): v = s.pop(k) s[k + cond] = v s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: self.merge_section('KNOWN/'+n, - dict(filter(lambda i: i[0] in s, s2.iteritems())), '') + dict([i for i in iter(s2.items()) if i[0] in s]), '') # merge section s2.update(s) else: @@ -400,7 +400,7 @@ def merge_section(self, section, options, pref=None): sec.update(options) return sk = {} - for k, v in options.iteritems(): + for k, v in options.items(): if not k.startswith(pref) and k != '__name__': sk[pref+k] = v sec.update(sk) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index c7f965ce52..8310ade139 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -26,7 +26,7 @@ import glob import os -from ConfigParser import NoOptionError, NoSectionError +from configparser import NoOptionError, NoSectionError from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags @@ -221,7 +221,7 @@ def read(self, filename): config_files += sorted(glob.glob('%s/*.local' % config_dir)) # choose only existing ones - config_files = filter(os.path.exists, config_files) + config_files = list(filter(os.path.exists, config_files)) if len(config_files): # at least one config exists and accessible diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index 88795674bd..ed2d251682 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -47,7 +47,7 @@ def __del__(self): def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string - obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) + obj = dumps(list(map(CSocket.convert, msg)), HIGHEST_PROTOCOL) self.__csock.send(obj) self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) @@ -72,7 +72,7 @@ def close(self): @staticmethod def convert(m): """Convert every "unexpected" member of message to string""" - if isinstance(m, (basestring, bool, int, float, list, dict, set)): + if isinstance(m, (str, bool, int, float, list, dict, set)): return m else: # pragma: no cover return str(m) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index f3b0f7b271..e8fa410d05 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -45,7 +45,7 @@ def _thread_name(): return threading.current_thread().__class__.__name__ def input_command(): # pragma: no cover - return raw_input(PROMPT) + return input(PROMPT) ## # @@ -456,7 +456,7 @@ def start(self, argv): return False finally: self._alive = False - for s, sh in _prev_signals.iteritems(): + for s, sh in _prev_signals.items(): signal.signal(s, sh) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index b1795588c9..aca262f082 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -40,10 +40,10 @@ import shlex import sys import time -import urllib +import urllib.request, urllib.parse, urllib.error from optparse import OptionParser, Option -from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError +from configparser import NoOptionError, NoSectionError, MissingSectionHeaderError try: # pragma: no cover from ..server.filtersystemd import FilterSystemd @@ -67,7 +67,7 @@ def debuggexURL(sample, regex, multiline=False, useDns="yes"): 'flavor': 'python' } if multiline: args['flags'] = 'm' - return 'https://www.debuggex.com/?' + urllib.urlencode(args) + return 'https://www.debuggex.com/?' + urllib.parse.urlencode(args) def output(args): # pragma: no cover (overriden in test-cases) print(args) @@ -246,7 +246,7 @@ class Fail2banRegex(object): def __init__(self, opts): # set local protected members from given options: - self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.iteritems())) + self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.items())) self._opts = opts self._maxlines_set = False # so we allow to override maxlines in cmdline self._datepattern_set = False @@ -313,7 +313,7 @@ def _dumpRealOptions(self, reader, fltOpt): realopts = {} combopts = reader.getCombined() # output all options that are specified in filter-argument as well as some special (mostly interested): - for k in ['logtype', 'datepattern'] + fltOpt.keys(): + for k in ['logtype', 'datepattern'] + list(fltOpt.keys()): # combined options win, but they contain only a sub-set in filter expected keys, # so get the rest from definition section: try: @@ -440,7 +440,7 @@ def readRegex(self, value, regextype): self.output( "Use %11s line : %s" % (regex, shortstr(value)) ) regex_values = {regextype: [RegexStat(value)]} - for regextype, regex_values in regex_values.iteritems(): + for regextype, regex_values in regex_values.items(): regex = regextype + 'regex' setattr(self, "_" + regex, regex_values) for regex in regex_values: @@ -532,13 +532,13 @@ def _out(ret): def _out(ret): for r in ret: for r in r[3].get('matches'): - if not isinstance(r, basestring): + if not isinstance(r, str): r = ''.join(r for r in r) output(r) elif ofmt == 'row': def _out(ret): for r in ret: - output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches'))) + output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].items() if k != 'matches'))) elif '<' not in ofmt: def _out(ret): for r in ret: @@ -573,7 +573,7 @@ def _get_msg(self): # wrap multiline tag (msg) interpolations to single line: for r, v in rows: for r in r[3].get('matches'): - if not isinstance(r, basestring): + if not isinstance(r, str): r = ''.join(r for r in r) r = v.replace("\x00msg\x00", r) output(r) @@ -639,9 +639,9 @@ def printLines(self, ltype): ans = [[]] for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] - b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + + b = [a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), - multiline, self._opts.usedns), ans) + multiline, self._opts.usedns) for a in ans] pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 24341014db..0b6c017259 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -71,7 +71,7 @@ def convert(self): @staticmethod def _fillStream(stream, opts, jailName): prio0idx = 0 - for opt, value in opts.iteritems(): + for opt, value in opts.items(): # Do not send a command if the value is not set (empty). if value is None: continue if opt in ("failregex", "ignoreregex"): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 9706c53054..51ad21c99e 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -118,7 +118,7 @@ def _glob(path): } _configOpts.update(FilterReader._configOpts) - _ignoreOpts = set(['action', 'filter', 'enabled'] + FilterReader._configOpts.keys()) + _ignoreOpts = set(['action', 'filter', 'enabled'] + list(FilterReader._configOpts.keys())) def getOptions(self): @@ -241,7 +241,7 @@ def convert(self, allow_no_files=False): stream.extend(self.__filter.convert()) # and using options from jail: FilterReader._fillStream(stream, self.__opts, self.__name) - for opt, value in self.__opts.iteritems(): + for opt, value in self.__opts.items(): if opt == "logpath": if self.__opts.get('backend', '').startswith("systemd"): continue found_files = 0 diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 5c1750a622..dee445fca7 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -31,6 +31,7 @@ from threading import Lock from .server.mytime import MyTime +import importlib try: import ctypes @@ -63,7 +64,7 @@ def __resetDefaultEncoding(encoding): from imp import load_dynamic as __ldm _sys = __ldm('_sys', 'sys') except ImportError: # pragma: no cover - only if load_dynamic fails - reload(sys) + importlib.reload(sys) _sys = sys if hasattr(_sys, "setdefaultencoding"): _sys.setdefaultencoding(encoding) @@ -101,7 +102,7 @@ def uni_string(x): else: # pragma: 3.x no cover def uni_decode(x, enc=PREFER_ENC, errors='strict'): try: - if isinstance(x, unicode): + if isinstance(x, str): return x.encode(enc, errors) return x except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable @@ -110,7 +111,7 @@ def uni_decode(x, enc=PREFER_ENC, errors='strict'): return x.encode(enc, 'replace') if sys.getdefaultencoding().upper() != 'UTF-8': # pragma: no cover - utf-8 is default encoding now def uni_string(x): - if not isinstance(x, unicode): + if not isinstance(x, str): return str(x) return x.encode(PREFER_ENC, 'replace') else: @@ -118,7 +119,7 @@ def uni_string(x): def _as_bool(val): - return bool(val) if not isinstance(val, basestring) \ + return bool(val) if not isinstance(val, str) \ else val.lower() in ('1', 'on', 'true', 'yes') @@ -327,7 +328,7 @@ def splitwords(s): """ if not s: return [] - return filter(bool, map(lambda v: v.strip(), re.split('[ ,\n]+', s))) + return list(filter(bool, [v.strip() for v in re.split('[ ,\n]+', s)])) if sys.version_info >= (3,5): eval(compile(r'''if 1: @@ -444,7 +445,7 @@ def substituteRecursiveTags(inptags, conditional='', while True: repFlag = False # substitute each value: - for tag in tags.iterkeys(): + for tag in tags.keys(): # ignore escaped or already done (or in ignore list): if tag in ignore or tag in done: continue # ignore replacing callable items from calling map - should be converted on demand only (by get): @@ -484,7 +485,7 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue # if calling map - be sure we've string: - if not isinstance(repl, basestring): repl = uni_string(repl) + if not isinstance(repl, str): repl = uni_string(repl) value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 16ff66212a..760a73d787 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -114,9 +114,9 @@ def _asrepr(self, calculated=False): def _asdict(self, calculated=False, checker=None): d = dict(self.data, **self.storage) if not calculated: - return dict((n,v) for n,v in d.iteritems() \ + return dict((n,v) for n,v in d.items() \ if not callable(v) or n in self.CM_REPR_ITEMS) - for n,v in d.items(): + for n,v in list(d.items()): if callable(v): try: # calculate: @@ -182,7 +182,7 @@ def copy(self): return self.__class__(_merge_copy_dicts(self.data, self.storage)) -class ActionBase(object): +class ActionBase(object, metaclass=ABCMeta): """An abstract base class for actions in Fail2Ban. Action Base is a base definition of what methods need to be in @@ -212,7 +212,6 @@ class ActionBase(object): Any additional arguments specified in `jail.conf` or passed via `fail2ban-client` will be passed as keyword arguments. """ - __metaclass__ = ABCMeta @classmethod def __subclasshook__(cls, C): @@ -423,7 +422,7 @@ def _operationExecuted(self, tag, family, *args): if not callable(family): # pragma: no cover return self.__substCache.get(key, {}).get(family) # family as expression - use it to filter values: - return [v for f, v in self.__substCache.get(key, {}).iteritems() if family(f)] + return [v for f, v in self.__substCache.get(key, {}).items() if family(f)] cmd = args[0] if cmd: # set: try: @@ -435,7 +434,7 @@ def _operationExecuted(self, tag, family, *args): try: famd = self.__substCache[key] cmd = famd.pop(family) - for family, v in famd.items(): + for family, v in list(famd.items()): if v == cmd: del famd[family] except KeyError: # pragma: no cover @@ -451,7 +450,7 @@ def _executeOperation(self, tag, operation, family=[], afterExec=None): res = True err = 'Script error' if not family: # all started: - family = [famoper for (famoper,v) in self.__started.iteritems() if v] + family = [famoper for (famoper,v) in self.__started.items() if v] for famoper in family: try: cmd = self._getOperation(tag, famoper) @@ -631,7 +630,7 @@ def flush(self): and executes the resulting command. """ # collect started families, may be started on demand (conditional): - family = [f for (f,v) in self.__started.iteritems() if v & 3 == 3]; # started and contains items + family = [f for (f,v) in self.__started.items() if v & 3 == 3]; # started and contains items # if nothing contains items: if not family: return True # flush: @@ -656,7 +655,7 @@ def _stop(self, family=None): """ # collect started families, if started on demand (conditional): if family is None: - family = [f for (f,v) in self.__started.iteritems() if v] + family = [f for (f,v) in self.__started.items() if v] # if no started (on demand) actions: if not family: return True self.__started = {} @@ -690,7 +689,7 @@ def consistencyCheck(self, beforeRepair=None): ret = True # for each started family: if self.actioncheck: - for (family, started) in self.__started.items(): + for (family, started) in list(self.__started.items()): if started and not self._invariantCheck(family, beforeRepair): # reset started flag and command of executed operation: self.__started[family] = 0 diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index fa045ab5ee..b052d342a9 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -156,11 +156,11 @@ def reload(self, begin=True): else: if hasattr(self, '_reload_actions'): # reload actions after all parameters set via stream: - for name, initOpts in self._reload_actions.iteritems(): + for name, initOpts in self._reload_actions.items(): if name in self._actions: self._actions[name].reload(**(initOpts if initOpts else {})) # remove obsolete actions (untouched by reload process): - delacts = OrderedDict((name, action) for name, action in self._actions.iteritems() + delacts = OrderedDict((name, action) for name, action in self._actions.items() if name not in self._reload_actions) if len(delacts): # unban all tickets using removed actions only: @@ -217,7 +217,7 @@ def getBanned(self, ids): return lst if len(ids) == 1: return 1 if ids[0] in lst else 0 - return map(lambda ip: 1 if ip in lst else 0, ids) + return [1 if ip in lst else 0 for ip in ids] def getBanList(self, withTime=False): """Returns the list of banned IP addresses. @@ -288,7 +288,7 @@ def removeBannedIP(self, ip=None, db=True, ifexists=False): if not isinstance(ip, IPAddr): ipa = IPAddr(ip) if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): - ips = filter(ipa.contains, self.banManager.getBanList()) + ips = list(filter(ipa.contains, self.banManager.getBanList())) if ips: return self.removeBannedIP(ips, db, ifexists) # not found: @@ -305,7 +305,7 @@ def stopActions(self, actions=None): """ if actions is None: actions = self._actions - for name, action in reversed(actions.items()): + for name, action in reversed(list(actions.items())): try: action.stop() except Exception as e: @@ -328,7 +328,7 @@ def run(self): True when the thread exits nicely. """ cnt = 0 - for name, action in self._actions.iteritems(): + for name, action in self._actions.items(): try: action.start() except Exception as e: @@ -505,7 +505,7 @@ def __checkBan(self, tickets=None): Observers.Main.add('banFound', bTicket, self._jail, btime) logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip) # do actions : - for name, action in self._actions.iteritems(): + for name, action in self._actions.items(): try: if bTicket.restored and getattr(action, 'norestored', False): continue @@ -543,13 +543,13 @@ def __checkBan(self, tickets=None): # avoid too often checks: if not rebanacts and MyTime.time() > self.__lastConsistencyCheckTM + 3: self.__lastConsistencyCheckTM = MyTime.time() - for action in self._actions.itervalues(): + for action in self._actions.values(): if hasattr(action, 'consistencyCheck'): action.consistencyCheck() # check epoch in order to reban it: if bTicket.banEpoch < self.banEpoch: if not rebanacts: rebanacts = dict( - (name, action) for name, action in self._actions.iteritems() + (name, action) for name, action in self._actions.items() if action.banEpoch > bTicket.banEpoch) cnt += self.__reBan(bTicket, actions=rebanacts) else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions: @@ -576,8 +576,8 @@ def __reBan(self, ticket, actions=None, log=True): ip = ticket.getID() aInfo = self._getActionInfo(ticket) if log: - logSys.notice("[%s] Reban %s%s", self._jail.name, ip, (', action %r' % actions.keys()[0] if len(actions) == 1 else '')) - for name, action in actions.iteritems(): + logSys.notice("[%s] Reban %s%s", self._jail.name, ip, (', action %r' % list(actions.keys())[0] if len(actions) == 1 else '')) + for name, action in actions.items(): try: logSys.debug("[%s] action %r: reban %s", self._jail.name, name, ip) if not aInfo.immutable: aInfo.reset() @@ -601,7 +601,7 @@ def _prolongBan(self, ticket): if not self.banManager._inBanList(ticket): return # do actions : aInfo = None - for name, action in self._actions.iteritems(): + for name, action in self._actions.items(): try: if ticket.restored and getattr(action, 'norestored', False): continue @@ -650,7 +650,7 @@ def __flushBan(self, db=False, actions=None, stop=False): cnt = 0 # first we'll execute flush for actions supporting this operation: unbactions = {} - for name, action in (actions if actions is not None else self._actions).iteritems(): + for name, action in (actions if actions is not None else self._actions).items(): try: if hasattr(action, 'flush') and (not isinstance(action, CommandAction) or action.actionflush): logSys.notice("[%s] Flush ticket(s) with %s", self._jail.name, name) @@ -705,7 +705,7 @@ def __unBan(self, ticket, actions=None, log=True): aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Unban %s", self._jail.name, ip) - for name, action in unbactions.iteritems(): + for name, action in unbactions.items(): try: logSys.debug("[%s] action %r: unban %s", self._jail.name, name, ip) if not aInfo.immutable: aInfo.reset() diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index e34007374e..f5f9740b2d 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -178,7 +178,7 @@ def loop(active, timeout=None, use_poll=False, err_count=None): elif err_count['listen'] > 100: # pragma: no cover - normally unreachable if ( e.args[0] == errno.EMFILE # [Errno 24] Too many open files - or sum(err_count.itervalues()) > 1000 + or sum(err_count.values()) > 1000 ): logSys.critical("Too many errors - critical count reached %r", err_count) break @@ -220,7 +220,7 @@ def handle_accept(self): elif self.__errCount['accept'] > 100: if ( (isinstance(e, socket.error) and e.args[0] == errno.EMFILE) # [Errno 24] Too many open files - or sum(self.__errCount.itervalues()) > 1000 + or sum(self.__errCount.values()) > 1000 ): logSys.critical("Too many errors - critical count reached %r", self.__errCount) self.stop() diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 9168d5b89b..d3e89820d3 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -103,7 +103,7 @@ def getBanList(self, ordered=False, withTime=False): return list(self.__banList.keys()) with self.__lock: lst = [] - for ticket in self.__banList.itervalues(): + for ticket in self.__banList.values(): eob = ticket.getEndOfBanTime(self.__banTime) lst.append((ticket,eob)) lst.sort(key=lambda t: t[1]) @@ -161,7 +161,7 @@ def getBanListExtendedCymruInfo(self, timeout=10): return return_dict # get ips in lock: with self.__lock: - banIPs = [banData.getIP() for banData in self.__banList.values()] + banIPs = [banData.getIP() for banData in list(self.__banList.values())] # get cymru info: try: for ip in banIPs: @@ -333,7 +333,7 @@ def unBanList(self, time, maxCount=0x7fffffff): # Gets the list of ticket to remove (thereby correct next unban time). unBanList = {} nextUnbanTime = BanTicket.MAX_TIME - for fid,ticket in self.__banList.iteritems(): + for fid,ticket in self.__banList.items(): # current time greater as end of ban - timed out: eob = ticket.getEndOfBanTime(self.__banTime) if time > eob: @@ -349,15 +349,15 @@ def unBanList(self, time, maxCount=0x7fffffff): if len(unBanList): if len(unBanList) / 2.0 <= len(self.__banList) / 3.0: # few as 2/3 should be removed - remove particular items: - for fid in unBanList.iterkeys(): + for fid in unBanList.keys(): del self.__banList[fid] else: # create new dictionary without items to be deleted: - self.__banList = dict((fid,ticket) for fid,ticket in self.__banList.iteritems() \ + self.__banList = dict((fid,ticket) for fid,ticket in self.__banList.items() \ if fid not in unBanList) # return list of tickets: - return unBanList.values() + return list(unBanList.values()) ## # Flush the ban list. @@ -367,7 +367,7 @@ def unBanList(self, time, maxCount=0x7fffffff): def flushBanList(self): with self.__lock: - uBList = self.__banList.values() + uBList = list(self.__banList.values()) self.__banList = dict() return uBList diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 877cbb93b7..59e376b711 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -67,13 +67,13 @@ def _json_loads_safe(x): else: # pragma: 3.x no cover def _normalize(x): if isinstance(x, dict): - return dict((_normalize(k), _normalize(v)) for k, v in x.iteritems()) + return dict((_normalize(k), _normalize(v)) for k, v in x.items()) elif isinstance(x, (list, set)): return [_normalize(element) for element in x] - elif isinstance(x, unicode): + elif isinstance(x, str): # in 2.x default text_factory is unicode - so return proper unicode here: return x.encode(PREFER_ENC, 'replace').decode(PREFER_ENC) - elif isinstance(x, basestring): + elif isinstance(x, str): return x.decode(PREFER_ENC, 'replace') return x diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 3c71d51a47..2effc45010 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -55,7 +55,7 @@ def getFailTotal(self): def getFailCount(self): # may be slow on large list of failures, should be used for test purposes only... with self.__lock: - return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) + return len(self.__failList), sum([f.getRetry() for f in list(self.__failList.values())]) def setMaxRetry(self, value): self.__maxRetry = value @@ -116,7 +116,7 @@ def addFailure(self, ticket, count=1, observed=False): # in case of having many active failures, it should be ran only # if debug level is "low" enough failures_summary = ', '.join(['%s:%d' % (k, v.getRetry()) - for k,v in self.__failList.iteritems()]) + for k,v in self.__failList.items()]) logSys.log(logLevel, "Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s" % (self.__failTotal, len(self.__failList), failures_summary)) @@ -129,7 +129,7 @@ def size(self): def cleanup(self, time): time -= self.__maxTime with self.__lock: - todelete = [fid for fid,item in self.__failList.iteritems() \ + todelete = [fid for fid,item in self.__failList.items() \ if item.getTime() <= time] if len(todelete) == len(self.__failList): # remove all: @@ -143,7 +143,7 @@ def cleanup(self, time): del self.__failList[fid] else: # create new dictionary without items to be deleted: - self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \ + self.__failList = dict((fid,item) for fid,item in self.__failList.items() \ if item.getTime() > time) self.__bgSvc.service() diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index a9b144af55..89c9438da9 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -143,9 +143,7 @@ def __init__(self, regex, multiline=False, **kwargs): self._regex = regex self._altValues = [] self._tupleValues = [] - for k in filter( - lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex - ): + for k in [k for k in self._regexObj.groupindex if len(k) > len(COMPLNAME_PRE[0])]: n = COMPLNAME_CRE.match(k) if n: g, n = n.group(1), mapTag2Opt(n.group(2)) @@ -235,7 +233,7 @@ def getRegex(self): # @staticmethod def _tupleLinesBuf(tupleLines): - return "\n".join(map(lambda v: "".join(v[::2]), tupleLines)) + "\n" + return "\n".join(["".join(v[::2]) for v in tupleLines]) + "\n" ## # Searches the regular expression. @@ -247,7 +245,7 @@ def _tupleLinesBuf(tupleLines): def search(self, tupleLines, orgLines=None): buf = tupleLines - if not isinstance(tupleLines, basestring): + if not isinstance(tupleLines, str): buf = Regex._tupleLinesBuf(tupleLines) self._matchCache = self._regexObj.search(buf) if self._matchCache: diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 196955e59f..8a289cea03 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -173,4 +173,4 @@ def isModified(self, filename): return False def getPendingPaths(self): - return self.__file404Cnt.keys() + return list(self.__file404Cnt.keys()) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 16b6cfd553..2ee49169e1 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -155,7 +155,7 @@ def _delPending(self, path): except KeyError: pass def getPendingPaths(self): - return self.__pending.keys() + return list(self.__pending.keys()) def _checkPending(self): if not self.__pending: @@ -181,7 +181,7 @@ def _checkPending(self): self.__pendingChkTime = time.time() self.__pendingMinTime = minTime # process now because we've missed it in monitoring: - for path, isDir in found.iteritems(): + for path, isDir in found.items(): self._delPending(path) # refresh monitoring of this: if isDir is not None: diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index a83b7a132d..2ab67a9a64 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -253,7 +253,7 @@ def formatJournalEntry(self, logentry): return ((logline[:0], date[0] + ' ', logline.replace('\n', '\\n')), date[1]) def seekToTime(self, date): - if isinstance(date, (int, long)): + if isinstance(date, int): date = float(date) self.__journal.seek_realtime(date) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index b435c6dfb3..0f7ccb016c 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -366,7 +366,7 @@ def __wrap_ipstr(ipstr): raise ValueError("invalid ipstr %r, too many plen representation" % (ipstr,)) if "." in s[1] or ":" in s[1]: # 255.255.255.0 resp. ffff:: style mask s[1] = IPAddr.masktoplen(s[1]) - s[1] = long(s[1]) + s[1] = int(s[1]) return s def __init(self, ipstr, cidr=CIDR_UNSPEC): @@ -400,7 +400,7 @@ def __init(self, ipstr, cidr=CIDR_UNSPEC): # mask out host portion if prefix length is supplied if cidr is not None and cidr >= 0: - mask = ~(0xFFFFFFFFL >> cidr) + mask = ~(0xFFFFFFFF >> cidr) self._addr &= mask self._plen = cidr @@ -412,13 +412,13 @@ def __init(self, ipstr, cidr=CIDR_UNSPEC): # mask out host portion if prefix length is supplied if cidr is not None and cidr >= 0: - mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr) + mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF >> cidr) self._addr &= mask self._plen = cidr # if IPv6 address is a IPv4-compatible, make instance a IPv4 elif self.isInNet(IPAddr.IP6_4COMPAT): - self._addr = lo & 0xFFFFFFFFL + self._addr = lo & 0xFFFFFFFF self._family = socket.AF_INET self._plen = 32 else: @@ -428,7 +428,7 @@ def __repr__(self): return repr(self.ntoa) def __str__(self): - return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) + return self.ntoa if isinstance(self.ntoa, str) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str @@ -542,7 +542,7 @@ def ntoa(self): elif self.isIPv6: # convert network to host byte order hi = self._addr >> 64 - lo = self._addr & 0xFFFFFFFFFFFFFFFFL + lo = self._addr & 0xFFFFFFFFFFFFFFFF binary = struct.pack("!QQ", hi, lo) if self._plen and self._plen < 128: add = "/%d" % self._plen @@ -600,9 +600,9 @@ def isInNet(self, net): if self.family != net.family: return False if self.isIPv4: - mask = ~(0xFFFFFFFFL >> net.plen) + mask = ~(0xFFFFFFFF >> net.plen) elif self.isIPv6: - mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen) + mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF >> net.plen) else: return False @@ -622,7 +622,7 @@ def __getMaskMap(): m4 = (1 << 32)-1 mmap = {m6: 128, m4: 32, 0: 0} m = 0 - for i in xrange(0, 128): + for i in range(0, 128): m |= 1 << i if i < 32: mmap[m ^ m4] = 32-1-i diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 2c84e47538..fe5c605aab 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -26,7 +26,7 @@ import logging import math import random -import Queue +import queue from .actions import Actions from ..helpers import getLogger, _as_bool, extractOptions, MyTime @@ -76,7 +76,7 @@ def __init__(self, name, backend = "auto", db=None): "might not function correctly. Please shorten" % name) self.__name = name - self.__queue = Queue.Queue() + self.__queue = queue.Queue() self.__filter = None # Extra parameters for increase ban time self._banExtra = {}; @@ -127,25 +127,25 @@ def _setBackend(self, backend): "Failed to initialize any backend for Jail %r" % self.name) def _initPolling(self, **kwargs): - from filterpoll import FilterPoll + from .filterpoll import FilterPoll logSys.info("Jail '%s' uses poller %r" % (self.name, kwargs)) self.__filter = FilterPoll(self, **kwargs) def _initGamin(self, **kwargs): # Try to import gamin - from filtergamin import FilterGamin + from .filtergamin import FilterGamin logSys.info("Jail '%s' uses Gamin %r" % (self.name, kwargs)) self.__filter = FilterGamin(self, **kwargs) def _initPyinotify(self, **kwargs): # Try to import pyinotify - from filterpyinotify import FilterPyinotify + from .filterpyinotify import FilterPyinotify logSys.info("Jail '%s' uses pyinotify %r" % (self.name, kwargs)) self.__filter = FilterPyinotify(self, **kwargs) def _initSystemd(self, **kwargs): # pragma: systemd no cover # Try to import systemd - from filtersystemd import FilterSystemd + from .filtersystemd import FilterSystemd logSys.info("Jail '%s' uses systemd %r" % (self.name, kwargs)) self.__filter = FilterSystemd(self, **kwargs) @@ -219,7 +219,7 @@ def getFailTicket(self): try: ticket = self.__queue.get(False) return ticket - except Queue.Empty: + except queue.Empty: return False def setBanTimeExtra(self, opt, value): diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 315d8a3004..ff46b7efeb 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -165,7 +165,7 @@ def str2seconds(val): @returns number (calculated seconds from expression "val") """ - if isinstance(val, (int, long, float, complex)): + if isinstance(val, (int, float, complex)): return val # replace together standing abbreviations, example '1d12h' -> '1d 12h': val = MyTime._str2sec_prep.sub(r" \1", val) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index ce201b5e7b..9de82fc907 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -209,7 +209,7 @@ def quit(self): # Restore default signal handlers: if _thread_name() == '_MainThread': - for s, sh in self.__prev_signals.iteritems(): + for s, sh in self.__prev_signals.items(): signal.signal(s, sh) # Give observer a small chance to complete its work before exit @@ -287,10 +287,10 @@ def stopAllJail(self): logSys.info("Stopping all jails") with self.__lock: # 1st stop all jails (signal and stop actions/filter thread): - for name in self.__jails.keys(): + for name in list(self.__jails.keys()): self.delJail(name, stop=True, join=False) # 2nd wait for end and delete jails: - for name in self.__jails.keys(): + for name in list(self.__jails.keys()): self.delJail(name, stop=False, join=True) def clearCaches(self): @@ -328,7 +328,7 @@ def reloadJails(self, name, opts, begin): if "--restart" in opts: self.stopAllJail() # first set all affected jail(s) to idle and reset filter regex and other lists/dicts: - for jn, jail in self.__jails.iteritems(): + for jn, jail in self.__jails.items(): if name == '--all' or jn == name: jail.idle = True self.__reload_state[jn] = jail @@ -339,7 +339,7 @@ def reloadJails(self, name, opts, begin): # end reload, all affected (or new) jails have already all new parameters (via stream) and (re)started: with self.__lock: deljails = [] - for jn, jail in self.__jails.iteritems(): + for jn, jail in self.__jails.items(): # still in reload state: if jn in self.__reload_state: # remove jails that are not reloaded (untouched, so not in new configuration) @@ -548,7 +548,7 @@ def setUnbanIP(self, name=None, value=None, ifexists=True): jails = [self.__jails[name]] else: # in all jails: - jails = self.__jails.values() + jails = list(self.__jails.values()) # unban given or all (if value is None): cnt = 0 ifexists |= (name is None) @@ -562,7 +562,7 @@ def banned(self, name=None, ids=None): jails = [self.__jails[name]] else: # in all jails: - jails = self.__jails.values() + jails = list(self.__jails.values()) # check banned ids: res = [] if name is None and ids: @@ -612,7 +612,7 @@ def isStarted(self): def isAlive(self, jailnum=None): if jailnum is not None and len(self.__jails) != jailnum: return 0 - for jail in self.__jails.values(): + for jail in list(self.__jails.values()): if not jail.isAlive(): return 0 return 1 @@ -827,7 +827,7 @@ def setIPv6IsAllowed(value): return DNSUtils.setIPv6IsAllowed(value) def setThreadOptions(self, value): - for o, v in value.iteritems(): + for o, v in value.items(): if o == 'stacksize': threading.stack_size(int(v)*1024) else: # pragma: no cover @@ -951,7 +951,7 @@ def __createDaemon(self): # pragma: no cover maxfd = os.sysconf("SC_OPEN_MAX") except (AttributeError, ValueError): maxfd = 256 # default maximum - fdlist = xrange(maxfd+1) + fdlist = range(maxfd+1) # urandom should not be closed in Python 3.4.0. Fixed in 3.4.1 # http://bugs.python.org/issue21207 diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 12be163a60..1f459f028b 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -99,7 +99,7 @@ def grp(exprset): if len(exprset) > 1 else "".join(exprset) exprset = set( cent(now[0].year + i) for i in (-1, distance) ) if len(now) > 1 and now[1]: - exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) + exprset |= set( cent(now[1].year + i) for i in range(-1, now[0].year-now[1].year+1, distance) ) return grp(sorted(list(exprset))) # more precise year patterns, within same century of last year and @@ -116,7 +116,7 @@ def grp(exprset): _updateTimeRE() def getTimePatternRE(): - keys = timeRE.keys() + keys = list(timeRE.keys()) patt = (r"%%(%%|%s|[%s])" % ( "|".join([k for k in keys if len(k) > 1]), "".join([k for k in keys if len(k) == 1]), @@ -171,7 +171,7 @@ def zone2offset(tz, dt): """ if isinstance(tz, int): return tz - if isinstance(tz, basestring): + if isinstance(tz, str): return validateTimeZone(tz) tz, tzo = tz if tzo is None or tzo == '': # without offset @@ -208,7 +208,7 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): year = month = day = tzoffset = \ weekday = julian = week_of_year = None hour = minute = second = fraction = 0 - for key, val in found_dict.iteritems(): + for key, val in found_dict.items(): if val is None: continue # Directives not explicitly handled below: # c, x, X diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 96e6777353..80b1469c58 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -55,7 +55,7 @@ def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None): self._time = time if time is not None else MyTime.time() self._data = {'matches': matches or [], 'failures': 0} if data is not None: - for k,v in data.iteritems(): + for k,v in data.items(): if v is not None: self._data[k] = v if ticket: @@ -88,7 +88,7 @@ def update(self, ticket): def setID(self, value): # guarantee using IPAddr instead of unicode, str for the IP - if isinstance(value, basestring): + if isinstance(value, str): value = IPAddr(value) self._id = value @@ -180,7 +180,7 @@ def setData(self, *args, **argv): if len(args) == 1: # todo: if support >= 2.7 only: # self._data = {k:v for k,v in args[0].iteritems() if v is not None} - self._data = dict([(k,v) for k,v in args[0].iteritems() if v is not None]) + self._data = dict([(k,v) for k,v in args[0].items() if v is not None]) # add k,v list or dict (merge): elif len(args) == 2: self._data.update((args,)) @@ -191,7 +191,7 @@ def setData(self, *args, **argv): # filter (delete) None values: # todo: if support >= 2.7 only: # self._data = {k:v for k,v in self._data.iteritems() if v is not None} - self._data = dict([(k,v) for k,v in self._data.iteritems() if v is not None]) + self._data = dict([(k,v) for k,v in self._data.items() if v is not None]) def getData(self, key=None, default=None): # return whole data dict: @@ -200,17 +200,17 @@ def getData(self, key=None, default=None): # return default if not exists: if not self._data: return default - if not isinstance(key,(str,unicode,type(None),int,float,bool,complex)): + if not isinstance(key,(str,type(None),int,float,bool,complex)): # return filtered by lambda/function: if callable(key): # todo: if support >= 2.7 only: # return {k:v for k,v in self._data.iteritems() if key(k)} - return dict([(k,v) for k,v in self._data.iteritems() if key(k)]) + return dict([(k,v) for k,v in self._data.items() if key(k)]) # return filtered by keys: if hasattr(key, '__iter__'): # todo: if support >= 2.7 only: # return {k:v for k,v in self._data.iteritems() if k in key} - return dict([(k,v) for k,v in self._data.iteritems() if k in key]) + return dict([(k,v) for k,v in self._data.items() if k in key]) # return single value of data: return self._data.get(key, default) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index c25ae637a1..660ed03aed 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -500,7 +500,7 @@ def __commandGet(self, command): opt = command[1][len("bantime."):] return self.__server.getBanTimeExtra(name, opt) elif command[1] == "actions": - return self.__server.getActions(name).keys() + return list(self.__server.getActions(name).keys()) elif command[1] == "action": actionname = command[2] actionvalue = command[3] diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 18073ea7d5..729ee43f20 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -53,7 +53,7 @@ # Dictionary to lookup signal name from number signame = dict((num, name) - for name, num in signal.__dict__.iteritems() if name.startswith("SIG")) + for name, num in signal.__dict__.items() if name.startswith("SIG")) class Utils(): """Utilities provide diverse static methods like executes OS shell commands, etc. @@ -140,7 +140,7 @@ def buildShellCmd(realCmd, varsDict): if not isinstance(realCmd, list): realCmd = [realCmd] i = len(realCmd)-1 - for k, v in varsDict.iteritems(): + for k, v in varsDict.items(): varsStat += "%s=$%s " % (k, i) realCmd.append(v) i += 1 diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index ce5de48334..772c687ff4 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -242,14 +242,14 @@ def testReplaceTagSelfRecursion(self): setattr(self.__action, 'ab', "") setattr(self.__action, 'x?family=inet6', "") # produce self-referencing properties except: - self.assertRaisesRegexp(ValueError, r"properties contain self referencing definitions", + self.assertRaisesRegex(ValueError, r"properties contain self referencing definitions", lambda: self.__action.replaceTag("", self.__action._properties, conditional="family=inet4") ) # remore self-referencing in props: delattr(self.__action, 'ac') # produce self-referencing query except: - self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", + self.assertRaisesRegex(ValueError, r"possible self referencing definitions in query", lambda: self.__action.replaceTag(""*30, self.__action._properties, conditional="family=inet6") ) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index cf25ac0fcd..2c0c4c4fc8 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -177,7 +177,7 @@ def setUp(self): super(StatusExtendedCymruInfo, self).setUp() unittest.F2B.SkipIfNoNetwork() setUpMyTime() - self.__ban_ip = iter(DNSUtils.dnsToIp("resolver1.opendns.com")).next() + self.__ban_ip = next(iter(DNSUtils.dnsToIp("resolver1.opendns.com"))) self.__asn = "36692" self.__country = "US" self.__rir = "arin" diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 37083a069e..a98f4e94cd 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -419,7 +419,7 @@ def testSplitOption(self): # And multiple groups (`][` instead of `,`) result = extractOptions(option.replace(',', '][')) expected2 = (expected[0], - dict((k, v.replace(',', '][')) for k, v in expected[1].iteritems()) + dict((k, v.replace(',', '][')) for k, v in expected[1].items()) ) self.assertEqual(expected2, result) @@ -1018,7 +1018,7 @@ def testMultipleSameAction(self, basedir): self.assertEqual(add_actions[-1][-1], "{}") def testLogPathFileFilterBackend(self): - self.assertRaisesRegexp(ValueError, r"Have not found any log file for .* jail", + self.assertRaisesRegex(ValueError, r"Have not found any log file for .* jail", self._testLogPath, backend='polling') def testLogPathSystemdBackend(self): diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 8cc394be73..6ac483c6cb 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -67,7 +67,7 @@ def setUp(self): @property def db(self): - if isinstance(self._db, basestring) and self._db == ':auto-create-in-memory:': + if isinstance(self._db, str) and self._db == ':auto-create-in-memory:': self._db = getFail2BanDb(self.dbFilename) return self._db @db.setter @@ -159,7 +159,7 @@ def testUpdateDb(self): self.db = Fail2BanDb(self.dbFilename) self.assertEqual(self.db.getJailNames(), set(['DummyJail #29162448 with 0 tickets'])) self.assertEqual(self.db.getLogPaths(), set(['/tmp/Fail2BanDb_pUlZJh.log'])) - ticket = FailTicket("127.0.0.1", 1388009242.26, [u"abc\n"]) + ticket = FailTicket("127.0.0.1", 1388009242.26, ["abc\n"]) self.assertEqual(self.db.getBans()[0], ticket) self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__) @@ -185,9 +185,9 @@ def testUpdateDb2(self): self.assertEqual(len(bans), 2) # compare first ticket completely: ticket = FailTicket("1.2.3.7", 1417595494, [ - u'Dec 3 09:31:08 f2btest test:auth[27658]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', - u'Dec 3 09:31:32 f2btest test:auth[27671]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', - u'Dec 3 09:31:34 f2btest test:auth[27673]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7' + 'Dec 3 09:31:08 f2btest test:auth[27658]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', + 'Dec 3 09:31:32 f2btest test:auth[27671]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', + 'Dec 3 09:31:34 f2btest test:auth[27673]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7' ]) ticket.setAttempt(3) self.assertEqual(bans[0], ticket) @@ -287,11 +287,11 @@ def testAddBanInvalidEncoded(self): # invalid + valid, invalid + valid unicode, invalid + valid dual converted (like in filter:readline by fallback) ... tickets = [ FailTicket("127.0.0.1", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), - FailTicket("127.0.0.2", 0, ['user "test"', u'user "\xd1\xe2\xe5\xf2\xe0"', u'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), + FailTicket("127.0.0.2", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), FailTicket("127.0.0.3", 0, ['user "test"', b'user "\xd1\xe2\xe5\xf2\xe0"', b'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), - FailTicket("127.0.0.4", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', u'user "\xe4\xf6\xfc\xdf"']), + FailTicket("127.0.0.4", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xe4\xf6\xfc\xdf"']), FailTicket("127.0.0.5", 0, ['user "test"', 'unterminated \xcf']), - FailTicket("127.0.0.6", 0, ['user "test"', u'unterminated \xcf']), + FailTicket("127.0.0.6", 0, ['user "test"', 'unterminated \xcf']), FailTicket("127.0.0.7", 0, ['user "test"', b'unterminated \xcf']) ] for ticket in tickets: diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index bc33bc0528..fc43677a1b 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -288,7 +288,7 @@ def testFullYearMatch_gh130(self): self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') # confuse it with year being at the end - for i in xrange(10): + for i in range(10): ( logTime, logMatch ) = self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '11/10/2012 02:37:17') @@ -538,7 +538,7 @@ def testAmbiguousDatePattern(self): date = dd.getTime(line) if matched: self.assertTrue(date) - if isinstance(matched, basestring): + if isinstance(matched, str): self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(matched, date[0]) @@ -573,7 +573,7 @@ def testVariousFormatSpecs(self): date = dd.getTime(line) if matched: self.assertTrue(date) - if isinstance(matched, basestring): # pragma: no cover + if isinstance(matched, str): # pragma: no cover self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(matched, date[0]) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index bc93dae7cc..9d3f382fcb 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -367,10 +367,10 @@ def _stopAndWaitForServerEnd(code=(SUCCESS, FAILED)): # several commands to server in body of decorated function: return f(self, tmp, startparams, *args, **kwargs) except Exception as e: # pragma: no cover - print('=== Catch an exception: %s' % e) + print(('=== Catch an exception: %s' % e)) log = self.getLog() if log: - print('=== Error of server, log: ===\n%s===' % log) + print(('=== Error of server, log: ===\n%s===' % log)) self.pruneLog() raise finally: @@ -440,7 +440,7 @@ def _wait_for_srv(self, tmp, ready=True, startparams=None, phase=None): ) except: # pragma: no cover if _inherited_log(startparams): - print('=== Error by wait fot server, log: ===\n%s===' % self.getLog()) + print(('=== Error by wait fot server, log: ===\n%s===' % self.getLog())) self.pruneLog() log = pjoin(tmp, "f2b.log") if isfile(log): @@ -1702,6 +1702,6 @@ def _testServerStartStop(self, tmp, startparams): self.stopAndWaitForServerEnd(SUCCESS) def testServerStartStop(self): - for i in xrange(2000): + for i in range(2000): self._testServerStartStop() diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 213ea89b8a..a71a17774d 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -589,8 +589,8 @@ def testNLCharAsPartOfUniChar(self): # test on unicode string containing \x0A as part of uni-char, # it must produce exactly 2 lines (both are failures): for l in ( - u'1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', - u'1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' + '1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', + '1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' ): fout.write(l.encode(enc)) fout.close() diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index 42b0fbd297..b07159fbb0 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -45,11 +45,11 @@ def tearDown(self): super(AddFailure, self).tearDown() def _addDefItems(self): - self.__items = [[u'193.168.0.128', 1167605999.0], - [u'193.168.0.128', 1167605999.0], - [u'193.168.0.128', 1167605999.0], - [u'193.168.0.128', 1167605999.0], - [u'193.168.0.128', 1167605999.0], + self.__items = [['193.168.0.128', 1167605999.0], + ['193.168.0.128', 1167605999.0], + ['193.168.0.128', 1167605999.0], + ['193.168.0.128', 1167605999.0], + ['193.168.0.128', 1167605999.0], ['87.142.124.10', 1167605999.0], ['87.142.124.10', 1167605999.0], ['87.142.124.10', 1167605999.0], diff --git a/fail2ban/tests/files/config/apache-auth/digest.py b/fail2ban/tests/files/config/apache-auth/digest.py index 0358859480..e2297ab312 100755 --- a/fail2ban/tests/files/config/apache-auth/digest.py +++ b/fail2ban/tests/files/config/apache-auth/digest.py @@ -41,7 +41,7 @@ def auth(v): response="%s" """ % ( username, algorithm, realm, url, nonce, qop, response ) # opaque="%s", - print(p.method, p.url, p.headers) + print((p.method, p.url, p.headers)) s = requests.Session() return s.send(p) @@ -76,18 +76,18 @@ def preauth(): # [Sun Jul 28 21:41:20 2013] [error] [client 127.0.0.1] Digest: unknown algorithm `super funky chicken' received: /digest/ -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) v['algorithm'] = algorithm r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) nonce = v['nonce'] v['nonce']=v['nonce'][5:-5] r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) # [Sun Jul 28 21:05:31.178340 2013] [auth_digest:error] [pid 24224:tid 139895539455744] [client 127.0.0.1:56906] AH01793: invalid qop `auth' received: /digest/qop_none/ @@ -95,7 +95,7 @@ def preauth(): v['nonce']=nonce[0:11] + 'ZZZ' + nonce[14:] r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) #[Sun Jul 28 21:18:11.769228 2013] [auth_digest:error] [pid 24752:tid 139895505884928] [client 127.0.0.1:56964] AH01776: invalid nonce b9YAiJDiBAZZZ1b1abe02d20063ea3b16b544ea1b0d981c1bafe received - hash is not d42d824dee7aaf50c3ba0a7c6290bd453e3dd35b @@ -107,7 +107,7 @@ def preauth(): time.sleep(1) r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) # Obtained by putting the following code in modules/aaa/mod_auth_digest.c # in the function initialize_secret @@ -137,7 +137,7 @@ def preauth(): v=preauth() -print(v['nonce']) +print((v['nonce'])) realm = v['Digest realm'][1:-1] (t,) = struct.unpack('l',base64.b64decode(v['nonce'][1:13])) @@ -156,13 +156,13 @@ def preauth(): r = auth(v) #[Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) url='/digest_onetime/' v=preauth() # Need opaque header handling in auth r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 4e308e38e6..e31e6492d2 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -22,7 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier; 2012 Yaroslav Halchenko" __license__ = "GPL" -from __builtin__ import open as fopen +from builtins import open as fopen import unittest import os import re @@ -209,7 +209,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line else: fin = in_ # Skip - for i in xrange(skip): + for i in range(skip): fin.readline() # Read i = 0 @@ -250,7 +250,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p # Required for filtering fields.update(TEST_JOURNAL_FIELDS) # Skip - for i in xrange(skip): + for i in range(skip): fin.readline() # Read/Write i = 0 @@ -312,18 +312,18 @@ def testAssertWrongTime(self): def testTest_tm(self): unittest.F2B.SkipIfFast() ## test function "_tm" works correct (returns the same as slow strftime): - for i in xrange(1417512352, (1417512352 // 3600 + 3) * 3600): + for i in range(1417512352, (1417512352 // 3600 + 3) * 3600): tm = MyTime.time2str(i) if _tm(i) != tm: # pragma: no cover - never reachable self.assertEqual((_tm(i), i), (tm, i)) def testWrongCharInTupleLine(self): ## line tuple has different types (ascii after ascii / unicode): - for a1 in ('', u'', b''): - for a2 in ('2016-09-05T20:18:56', u'2016-09-05T20:18:56', b'2016-09-05T20:18:56'): + for a1 in ('', '', b''): + for a2 in ('2016-09-05T20:18:56', '2016-09-05T20:18:56', b'2016-09-05T20:18:56'): for a3 in ( 'Fail for "g\xc3\xb6ran" from 192.0.2.1', - u'Fail for "g\xc3\xb6ran" from 192.0.2.1', + 'Fail for "g\xc3\xb6ran" from 192.0.2.1', b'Fail for "g\xc3\xb6ran" from 192.0.2.1' ): # join should work if all arguments have the same type: @@ -510,7 +510,7 @@ def testWrongTimeOrTZ(self): def testAddAttempt(self): self.filter.setMaxRetry(3) - for i in xrange(1, 1+3): + for i in range(1, 1+3): self.filter.addAttempt('192.0.2.1') self.assertLogged('Attempt 192.0.2.1', '192.0.2.1:%d' % i, all=True, wait=True) self.jail.actions._Actions__checkBan() @@ -547,7 +547,7 @@ def testIgnoreCache(self): # like both test-cases above, just cached (so once per key)... self.filter.ignoreCache = {"key":""} self.filter.ignoreCommand = 'if [ "" = "10.0.0.1" ]; then exit 0; fi; exit 1' - for i in xrange(5): + for i in range(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1")) self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0")) @@ -558,7 +558,7 @@ def testIgnoreCache(self): # by host of IP: self.filter.ignoreCache = {"key":""} self.filter.ignoreCommand = 'if [ "" = "test-host" ]; then exit 0; fi; exit 1' - for i in xrange(5): + for i in range(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList(FailTicket("2001:db8::1"))) self.assertFalse(self.filter.inIgnoreIPList(FailTicket("2001:db8::ffff"))) @@ -570,7 +570,7 @@ def testIgnoreCache(self): self.filter.ignoreCache = {"key":"", "max-count":"10", "max-time":"1h"} self.assertEqual(self.filter.ignoreCache, ["", 10, 60*60]) self.filter.ignoreCommand = 'if [ "" = "tester" ]; then exit 0; fi; exit 1' - for i in xrange(5): + for i in range(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList(FailTicket("tester", data={'user': 'tester'}))) self.assertFalse(self.filter.inIgnoreIPList(FailTicket("root", data={'user': 'root'}))) @@ -673,7 +673,7 @@ def testMissingLogFiles(self): def testDecodeLineWarn(self): # incomplete line (missing byte at end), warning is suppressed: - l = u"correct line\n" + l = "correct line\n" r = l.encode('utf-16le') self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r), l) self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r[0:-1]), l[0:-1]) @@ -733,7 +733,7 @@ def testSeekToTimeSmallFile(self): fc = FileContainer(fname, self.filter.getLogEncoding()) fc.open() # no time - nothing should be found : - for i in xrange(10): + for i in range(10): f.write("[sshd] error: PAM: failure len 1\n") f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) @@ -807,14 +807,14 @@ def testSeekToTimeLargeFile(self): # variable length of file (ca 45K or 450K before and hereafter): # write lines with smaller as search time: t = time - count - 1 - for i in xrange(count): + for i in range(count): f.write("%s [sshd] error: PAM: failure\n" % _tm(t)) t += 1 f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) self.assertEqual(fc.getPos(), 47*count) # write lines with exact search time: - for i in xrange(10): + for i in range(10): f.write("%s [sshd] error: PAM: failure\n" % _tm(time)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) @@ -823,8 +823,8 @@ def testSeekToTimeLargeFile(self): self.assertEqual(fc.getPos(), 47*count) # write lines with greater as search time: t = time+1 - for i in xrange(count//500): - for j in xrange(500): + for i in range(count//500): + for j in range(500): f.write("%s [sshd] error: PAM: failure\n" % _tm(t)) t += 1 f.flush() @@ -1634,10 +1634,10 @@ def test_WrongChar(self): # Add direct utf, unicode, blob: for l in ( "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", - u"error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", + "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", b"error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1".decode('utf-8', 'replace'), "error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", - u"error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", + "error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", b"error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2".decode('utf-8', 'replace') ): fields = self.journal_fields @@ -1666,7 +1666,7 @@ class GetFailures(LogCaptureTestCase): # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, - [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3) + ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3) def setUp(self): """Call before every test case.""" @@ -1752,8 +1752,8 @@ def testNLCharAsPartOfUniChar(self): # test on unicode string containing \x0A as part of uni-char, # it must produce exactly 2 lines (both are failures): for l in ( - u'%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, - u'%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm + '%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, + '%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm ): fout.write(l.encode(enc)) fout.close() @@ -1774,8 +1774,8 @@ def testNLCharAsPartOfUniChar(self): def testGetFailures02(self): output = ('141.3.81.106', 4, 1124013539.0, - [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' - % m for m in 53, 54, 57, 58]) + ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' + % m for m in (53, 54, 57, 58)]) self.filter.setMaxRetry(4) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) @@ -1886,19 +1886,19 @@ def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) output_yes = ( ('93.184.216.34', 1, 1124013299.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] ), ('93.184.216.34', 1, 1124013539.0, - [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ), ('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] ), ) output_no = ( ('93.184.216.34', 1, 1124013539.0, - [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ) ) @@ -2004,9 +2004,9 @@ def testCache(self): self.assertTrue(c.get('a') is None) self.assertEqual(c.get('a', 'test'), 'test') # exact 5 elements : - for i in xrange(5): + for i in range(5): c.set(i, i) - for i in xrange(5): + for i in range(5): self.assertEqual(c.get(i), i) # remove unavailable key: c.unset('a'); c.unset('a') @@ -2014,30 +2014,30 @@ def testCache(self): def testCacheMaxSize(self): c = Utils.Cache(maxCount=5, maxTime=60) # exact 5 elements : - for i in xrange(5): + for i in range(5): c.set(i, i) - self.assertEqual([c.get(i) for i in xrange(5)], [i for i in xrange(5)]) - self.assertNotIn(-1, (c.get(i, -1) for i in xrange(5))) + self.assertEqual([c.get(i) for i in range(5)], [i for i in range(5)]) + self.assertNotIn(-1, (c.get(i, -1) for i in range(5))) # add one - too many: c.set(10, i) # one element should be removed : - self.assertIn(-1, (c.get(i, -1) for i in xrange(5))) + self.assertIn(-1, (c.get(i, -1) for i in range(5))) # test max size (not expired): - for i in xrange(10): + for i in range(10): c.set(i, 1) self.assertEqual(len(c), 5) def testCacheMaxTime(self): # test max time (expired, timeout reached) : c = Utils.Cache(maxCount=5, maxTime=0.0005) - for i in xrange(10): + for i in range(10): c.set(i, 1) st = time.time() self.assertTrue(Utils.wait_for(lambda: time.time() >= st + 0.0005, 1)) # we have still 5 elements (or fewer if too slow test mashine): self.assertTrue(len(c) <= 5) # but all that are expiered also: - for i in xrange(10): + for i in range(10): self.assertTrue(c.get(i) is None) # here the whole cache should be empty: self.assertEqual(len(c), 0) @@ -2058,7 +2058,7 @@ def _TestCacheStr2IP(forw=True, result=[], random=False): c = count while c: c -= 1 - s = xrange(0, 256, 1) if forw else xrange(255, -1, -1) + s = range(0, 256, 1) if forw else range(255, -1, -1) if random: shuffle([i for i in s]) for i in s: IPAddr('192.0.2.'+str(i), IPAddr.FAM_IPv4) @@ -2184,16 +2184,16 @@ def testIpToName(self): def testAddr2bin(self): res = IPAddr('10.0.0.0') - self.assertEqual(res.addr, 167772160L) + self.assertEqual(res.addr, 167772160) res = IPAddr('10.0.0.0', cidr=None) - self.assertEqual(res.addr, 167772160L) - res = IPAddr('10.0.0.0', cidr=32L) - self.assertEqual(res.addr, 167772160L) - res = IPAddr('10.0.0.1', cidr=32L) - self.assertEqual(res.addr, 167772161L) + self.assertEqual(res.addr, 167772160) + res = IPAddr('10.0.0.0', cidr=32) + self.assertEqual(res.addr, 167772160) + res = IPAddr('10.0.0.1', cidr=32) + self.assertEqual(res.addr, 167772161) self.assertTrue(res.isSingle) - res = IPAddr('10.0.0.1', cidr=31L) - self.assertEqual(res.addr, 167772160L) + res = IPAddr('10.0.0.1', cidr=31) + self.assertEqual(res.addr, 167772160) self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') @@ -2272,9 +2272,9 @@ def testIPAddr_Compare(self): '93.184.216.34': 'ip4-test', '2606:2800:220:1:248:1893:25c8:1946': 'ip6-test' } - d2 = dict([(IPAddr(k), v) for k, v in d.iteritems()]) - self.assertTrue(isinstance(d.keys()[0], basestring)) - self.assertTrue(isinstance(d2.keys()[0], IPAddr)) + d2 = dict([(IPAddr(k), v) for k, v in d.items()]) + self.assertTrue(isinstance(list(d.keys())[0], str)) + self.assertTrue(isinstance(list(d2.keys())[0], IPAddr)) self.assertEqual(d.get(ip4[2], ''), 'ip4-test') self.assertEqual(d.get(ip6[2], ''), 'ip6-test') self.assertEqual(d2.get(str(ip4[2]), ''), 'ip4-test') diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 4b02637755..d91729fb93 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -29,9 +29,9 @@ import shutil import fnmatch from glob import glob -from StringIO import StringIO +from io import StringIO -from utils import LogCaptureTestCase, logSys as DefLogSys +from .utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, \ getVerbosityFormat, splitwords, uni_decode, uni_string @@ -67,7 +67,7 @@ def testsplitwords(self): self.assertEqual(splitwords(' 1\n 2'), ['1', '2']) self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) # string as unicode: - self.assertEqual(splitwords(u' 1\n 2, 3'), ['1', '2', '3']) + self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) def _sh_call(cmd): @@ -191,12 +191,12 @@ def testmbasename(self): def testUniConverters(self): self.assertRaises(Exception, uni_decode, - (b'test' if sys.version_info >= (3,) else u'test'), 'f2b-test::non-existing-encoding') - uni_decode((b'test\xcf' if sys.version_info >= (3,) else u'test\xcf')) + (b'test' if sys.version_info >= (3,) else 'test'), 'f2b-test::non-existing-encoding') + uni_decode((b'test\xcf' if sys.version_info >= (3,) else 'test\xcf')) uni_string(b'test\xcf') uni_string('test\xcf') if sys.version_info < (3,) and 'PyPy' not in sys.version: - uni_string(u'test\xcf') + uni_string('test\xcf') def testSafeLogging(self): # logging should be exception-safe, to avoid possible errors (concat, str. conversion, representation failures, etc) @@ -208,7 +208,7 @@ def __repr__(self): if self.err: raise Exception('no represenation for test!') else: - return u'conv-error (\xf2\xf0\xe5\xf2\xe8\xe9), unterminated utf \xcf' + return 'conv-error (\xf2\xf0\xe5\xf2\xe8\xe9), unterminated utf \xcf' test = Test() logSys.log(logging.NOTICE, "test 1a: %r", test) self.assertLogged("Traceback", "no represenation for test!") @@ -256,7 +256,7 @@ def deep_function(i): func_raise() try: - print deep_function(3) + print(deep_function(3)) except ValueError: s = tb() @@ -273,7 +273,7 @@ def deep_function(i): self.assertIn(':', s) def _testAssertionErrorRE(self, regexp, fun, *args, **kwargs): - self.assertRaisesRegexp(AssertionError, regexp, fun, *args, **kwargs) + self.assertRaisesRegex(AssertionError, regexp, fun, *args, **kwargs) def testExtendedAssertRaisesRE(self): ## test _testAssertionErrorRE several fail cases: @@ -311,13 +311,13 @@ def testExtendedAssertMethods(self): self._testAssertionErrorRE(r"'a' unexpectedly found in 'cba'", self.assertNotIn, 'a', 'cba') self._testAssertionErrorRE(r"1 unexpectedly found in \[0, 1, 2\]", - self.assertNotIn, 1, xrange(3)) + self.assertNotIn, 1, range(3)) self._testAssertionErrorRE(r"'A' unexpectedly found in \['C', 'A'\]", self.assertNotIn, 'A', (c.upper() for c in 'cba' if c != 'b')) self._testAssertionErrorRE(r"'a' was not found in 'xyz'", self.assertIn, 'a', 'xyz') self._testAssertionErrorRE(r"5 was not found in \[0, 1, 2\]", - self.assertIn, 5, xrange(3)) + self.assertIn, 5, range(3)) self._testAssertionErrorRE(r"'A' was not found in \['C', 'B'\]", self.assertIn, 'A', (c.upper() for c in 'cba' if c != 'a')) ## assertLogged, assertNotLogged positive case: diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index 9b44c6dded..ace1184c68 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -68,7 +68,7 @@ def testDefault(self, multipliers = None): a.setBanTimeExtra('multipliers', multipliers) # test algorithm and max time 24 hours : self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], + [a.calcBanTime(600, i) for i in range(1, 11)], [1200, 2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400] ) # with extra large max time (30 days): @@ -80,38 +80,38 @@ def testDefault(self, multipliers = None): if multcnt < 11: arr = arr[0:multcnt-1] + ([arr[multcnt-2]] * (11-multcnt)) self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], + [a.calcBanTime(600, i) for i in range(1, 11)], arr ) a.setBanTimeExtra('maxtime', '1d') # change factor : a.setBanTimeExtra('factor', '2'); self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], + [a.calcBanTime(600, i) for i in range(1, 11)], [2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400, 86400] ) # factor is float : a.setBanTimeExtra('factor', '1.33'); self.assertEqual( - [int(a.calcBanTime(600, i)) for i in xrange(1, 11)], + [int(a.calcBanTime(600, i)) for i in range(1, 11)], [1596, 3192, 6384, 12768, 25536, 51072, 86400, 86400, 86400, 86400] ) a.setBanTimeExtra('factor', None); # change max time : a.setBanTimeExtra('maxtime', '12h') self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], + [a.calcBanTime(600, i) for i in range(1, 11)], [1200, 2400, 4800, 9600, 19200, 38400, 43200, 43200, 43200, 43200] ) a.setBanTimeExtra('maxtime', '24h') ## test randomization - not possibe all 10 times we have random = 0: a.setBanTimeExtra('rndtime', '5m') self.assertTrue( - False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] + False in [1200 in [a.calcBanTime(600, 1) for i in range(10)] for c in range(10)] ) a.setBanTimeExtra('rndtime', None) self.assertFalse( - False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] + False in [1200 in [a.calcBanTime(600, 1) for i in range(10)] for c in range(10)] ) # restore default: a.setBanTimeExtra('multipliers', None) @@ -123,7 +123,7 @@ def testMultipliers(self): # this multipliers has the same values as default formula, we test stop growing after count 9: self.testDefault('1 2 4 8 16 32 64 128 256') # this multipliers has exactly the same values as default formula, test endless growing (stops by count 31 only): - self.testDefault(' '.join([str(1<= 0: - line1 = f.next() + line1 = next(f) self.assertTrue(line1.endswith("Before file moved\n")) - line2 = f.next() + line2 = next(f) self.assertTrue(line2.endswith("After file moved\n")) try: - n = f.next() + n = next(f) if n.find("Command: ['flushlogs']") >=0: - self.assertRaises(StopIteration, f.next) + self.assertRaises(StopIteration, f.__next__) else: self.fail("Exception StopIteration or Command: ['flushlogs'] expected. Got: %s" % n) except StopIteration: pass # on higher debugging levels this is expected with open(fn,'r') as f: - line1 = f.next() + line1 = next(f) if line1.find('rollover performed on') >= 0: - line1 = f.next() + line1 = next(f) self.assertTrue(line1.endswith("After flushlogs\n")) - self.assertRaises(StopIteration, f.next) + self.assertRaises(StopIteration, f.__next__) f.close() finally: os.remove(fn2) @@ -1185,7 +1185,7 @@ def testStartFailedSockExists(self): os.remove(f) -from clientreadertestcase import ActionReader, JailsReader, CONFIG_DIR +from .clientreadertestcase import ActionReader, JailsReader, CONFIG_DIR class ServerConfigReaderTests(LogCaptureTestCase): diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index e3a07998d6..15253ce097 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -153,7 +153,7 @@ def testSocketConnectBroken(self): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: self.close() - self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", + self.assertRaisesRegex(Exception, r"reset by peer|Broken pipe", lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10))) finally: RequestHandler.found_terminator = org_handler diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 8bcc1431d4..98ec581146 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -35,7 +35,7 @@ import threading import unittest -from cStringIO import StringIO +from io import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode @@ -171,8 +171,8 @@ def initProcess(opts): # Let know the version if opts.verbosity != 0: - print("Fail2ban %s test suite. Python %s. Please wait..." \ - % (version, str(sys.version).replace('\n', ''))) + print(("Fail2ban %s test suite. Python %s. Please wait..." \ + % (version, str(sys.version).replace('\n', '')))) return opts; @@ -303,7 +303,7 @@ def F2B_SkipIfNoNetwork(): c.clear = lambda: logSys.warn('clear CACHE_ipToName is disabled in test suite') # increase max count and max time (too many entries, long time testing): c.setOptions(maxCount=10000, maxTime=5*60) - for i in xrange(256): + for i in range(256): c.set('192.0.2.%s' % i, None) c.set('198.51.100.%s' % i, None) c.set('203.0.113.%s' % i, None) @@ -531,8 +531,8 @@ def addTest(self, suite): import difflib, pprint if not hasattr(unittest.TestCase, 'assertDictEqual'): def assertDictEqual(self, d1, d2, msg=None): - self.assert_(isinstance(d1, dict), 'First argument is not a dictionary') - self.assert_(isinstance(d2, dict), 'Second argument is not a dictionary') + self.assertTrue(isinstance(d1, dict), 'First argument is not a dictionary') + self.assertTrue(isinstance(d2, dict), 'Second argument is not a dictionary') if d1 != d2: standardMsg = '%r != %r' % (d1, d2) diff = ('\n' + '\n'.join(difflib.ndiff( @@ -550,7 +550,7 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None) # used to recognize having element as nested dict, list or tuple: def _is_nested(v): if isinstance(v, dict): - return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) + return any(isinstance(v, (dict, list, tuple)) for v in v.values()) return any(isinstance(v, (dict, list, tuple)) for v in v) if nestedOnly: _nest_sorted = sorted @@ -570,7 +570,7 @@ def _assertSortedEqual(a, b, level, nestedOnly, key): return raise ValueError('%r != %r' % (a, b)) if isinstance(a, dict) and isinstance(b, dict): # compare dict's: - for k, v1 in a.iteritems(): + for k, v1 in a.items(): v2 = b[k] if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key) @@ -605,14 +605,14 @@ def assertRaisesRegexp(self, exccls, regexp, fun, *args, **kwargs): self.fail('\"%s\" does not match \"%s\"' % (regexp, e)) else: self.fail('%s not raised' % getattr(exccls, '__name__')) - unittest.TestCase.assertRaisesRegexp = assertRaisesRegexp + unittest.TestCase.assertRaisesRegex = assertRaisesRegexp # always custom following methods, because we use atm better version of both (support generators) if True: ## if not hasattr(unittest.TestCase, 'assertIn'): def assertIn(self, a, b, msg=None): bb = b wrap = False - if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring): + if msg is None and hasattr(b, '__iter__') and not isinstance(b, str): b, bb = itertools.tee(b) wrap = True if a not in b: @@ -623,7 +623,7 @@ def assertIn(self, a, b, msg=None): def assertNotIn(self, a, b, msg=None): bb = b wrap = False - if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring): + if msg is None and hasattr(b, '__iter__') and not isinstance(b, str): b, bb = itertools.tee(b) wrap = True if a in b: From a439821173ee51b5042c8b2cb040077d0d4f57b0 Mon Sep 17 00:00:00 2001 From: Robert Belanec Date: Thu, 23 Feb 2023 11:20:52 +0100 Subject: [PATCH 22/28] revert a mistake commit after 2to3 --- fail2ban/client/actionreader.py | 4 +- fail2ban/client/configparserinc.py | 12 +-- fail2ban/client/configreader.py | 4 +- fail2ban/client/csocket.py | 4 +- fail2ban/client/fail2banclient.py | 4 +- fail2ban/client/fail2banregex.py | 22 ++--- fail2ban/client/filterreader.py | 2 +- fail2ban/client/jailreader.py | 4 +- fail2ban/helpers.py | 15 +-- fail2ban/server/action.py | 19 ++-- fail2ban/server/actions.py | 28 +++--- fail2ban/server/asyncserver.py | 4 +- fail2ban/server/banmanager.py | 14 +-- fail2ban/server/database.py | 6 +- fail2ban/server/failmanager.py | 8 +- fail2ban/server/failregex.py | 8 +- fail2ban/server/filterpoll.py | 2 +- fail2ban/server/filterpyinotify.py | 4 +- fail2ban/server/filtersystemd.py | 2 +- fail2ban/server/ipdns.py | 18 ++-- fail2ban/server/jail.py | 14 +-- fail2ban/server/mytime.py | 2 +- fail2ban/server/server.py | 20 ++-- fail2ban/server/strptime.py | 8 +- fail2ban/server/ticket.py | 14 +-- fail2ban/server/transmitter.py | 2 +- fail2ban/server/utils.py | 4 +- fail2ban/tests/actiontestcase.py | 4 +- fail2ban/tests/banmanagertestcase.py | 2 +- fail2ban/tests/clientreadertestcase.py | 4 +- fail2ban/tests/databasetestcase.py | 16 +-- fail2ban/tests/datedetectortestcase.py | 6 +- fail2ban/tests/fail2banclienttestcase.py | 8 +- fail2ban/tests/fail2banregextestcase.py | 4 +- fail2ban/tests/failmanagertestcase.py | 10 +- .../tests/files/config/apache-auth/digest.py | 20 ++-- fail2ban/tests/filtertestcase.py | 98 +++++++++---------- fail2ban/tests/misctestcase.py | 22 ++--- fail2ban/tests/observertestcase.py | 34 +++---- fail2ban/tests/samplestestcase.py | 8 +- fail2ban/tests/servertestcase.py | 28 +++--- fail2ban/tests/sockettestcase.py | 2 +- fail2ban/tests/utils.py | 22 ++--- 43 files changed, 267 insertions(+), 269 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 88b0aca140..5a57338ba2 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -89,11 +89,11 @@ def convert(self): stream = list() stream.append(head + ["addaction", self._name]) multi = [] - for opt, optval in opts.iteritems(): + for opt, optval in opts.items(): if opt in self._configOpts and not opt.startswith('known/'): multi.append([opt, optval]) if self._initOpts: - for opt, optval in self._initOpts.iteritems(): + for opt, optval in self._initOpts.items(): if opt not in self._configOpts and not opt.startswith('known/'): multi.append([opt, optval]) if len(multi) > 1: diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index cc4ada0aed..e46379a179 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -62,7 +62,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, parser, option, accum, rest, section, map, *args, **kwargs) else: # pragma: 3.x no cover - from ConfigParser import SafeConfigParser, \ + from configparser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError # Interpolate missing known/option as option from default section @@ -327,7 +327,7 @@ def options(self, section, withDefault=True): # mix it with defaults: return set(opts.keys()) | set(self._defaults) # only own option names: - return opts.keys() + return list(opts.keys()) def read(self, filenames, get_includes=True): if not isinstance(filenames, list): @@ -356,7 +356,7 @@ def read(self, filenames, get_includes=True): ret += i # merge defaults and all sections to self: alld.update(cfg.get_defaults()) - for n, s in cfg.get_sections().iteritems(): + for n, s in cfg.get_sections().items(): # conditional sections cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n) if cond: @@ -366,14 +366,14 @@ def read(self, filenames, get_includes=True): del(s['__name__']) except KeyError: pass - for k in s.keys(): + for k in list(s.keys()): v = s.pop(k) s[k + cond] = v s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: self.merge_section('KNOWN/'+n, - dict(filter(lambda i: i[0] in s, s2.iteritems())), '') + dict([i for i in iter(s2.items()) if i[0] in s]), '') # merge section s2.update(s) else: @@ -400,7 +400,7 @@ def merge_section(self, section, options, pref=None): sec.update(options) return sk = {} - for k, v in options.iteritems(): + for k, v in options.items(): if not k.startswith(pref) and k != '__name__': sk[pref+k] = v sec.update(sk) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index c7f965ce52..8310ade139 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -26,7 +26,7 @@ import glob import os -from ConfigParser import NoOptionError, NoSectionError +from configparser import NoOptionError, NoSectionError from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags @@ -221,7 +221,7 @@ def read(self, filename): config_files += sorted(glob.glob('%s/*.local' % config_dir)) # choose only existing ones - config_files = filter(os.path.exists, config_files) + config_files = list(filter(os.path.exists, config_files)) if len(config_files): # at least one config exists and accessible diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index 88795674bd..ed2d251682 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -47,7 +47,7 @@ def __del__(self): def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string - obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) + obj = dumps(list(map(CSocket.convert, msg)), HIGHEST_PROTOCOL) self.__csock.send(obj) self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) @@ -72,7 +72,7 @@ def close(self): @staticmethod def convert(m): """Convert every "unexpected" member of message to string""" - if isinstance(m, (basestring, bool, int, float, list, dict, set)): + if isinstance(m, (str, bool, int, float, list, dict, set)): return m else: # pragma: no cover return str(m) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index f3b0f7b271..e8fa410d05 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -45,7 +45,7 @@ def _thread_name(): return threading.current_thread().__class__.__name__ def input_command(): # pragma: no cover - return raw_input(PROMPT) + return input(PROMPT) ## # @@ -456,7 +456,7 @@ def start(self, argv): return False finally: self._alive = False - for s, sh in _prev_signals.iteritems(): + for s, sh in _prev_signals.items(): signal.signal(s, sh) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index b1795588c9..aca262f082 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -40,10 +40,10 @@ import shlex import sys import time -import urllib +import urllib.request, urllib.parse, urllib.error from optparse import OptionParser, Option -from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError +from configparser import NoOptionError, NoSectionError, MissingSectionHeaderError try: # pragma: no cover from ..server.filtersystemd import FilterSystemd @@ -67,7 +67,7 @@ def debuggexURL(sample, regex, multiline=False, useDns="yes"): 'flavor': 'python' } if multiline: args['flags'] = 'm' - return 'https://www.debuggex.com/?' + urllib.urlencode(args) + return 'https://www.debuggex.com/?' + urllib.parse.urlencode(args) def output(args): # pragma: no cover (overriden in test-cases) print(args) @@ -246,7 +246,7 @@ class Fail2banRegex(object): def __init__(self, opts): # set local protected members from given options: - self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.iteritems())) + self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.items())) self._opts = opts self._maxlines_set = False # so we allow to override maxlines in cmdline self._datepattern_set = False @@ -313,7 +313,7 @@ def _dumpRealOptions(self, reader, fltOpt): realopts = {} combopts = reader.getCombined() # output all options that are specified in filter-argument as well as some special (mostly interested): - for k in ['logtype', 'datepattern'] + fltOpt.keys(): + for k in ['logtype', 'datepattern'] + list(fltOpt.keys()): # combined options win, but they contain only a sub-set in filter expected keys, # so get the rest from definition section: try: @@ -440,7 +440,7 @@ def readRegex(self, value, regextype): self.output( "Use %11s line : %s" % (regex, shortstr(value)) ) regex_values = {regextype: [RegexStat(value)]} - for regextype, regex_values in regex_values.iteritems(): + for regextype, regex_values in regex_values.items(): regex = regextype + 'regex' setattr(self, "_" + regex, regex_values) for regex in regex_values: @@ -532,13 +532,13 @@ def _out(ret): def _out(ret): for r in ret: for r in r[3].get('matches'): - if not isinstance(r, basestring): + if not isinstance(r, str): r = ''.join(r for r in r) output(r) elif ofmt == 'row': def _out(ret): for r in ret: - output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches'))) + output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].items() if k != 'matches'))) elif '<' not in ofmt: def _out(ret): for r in ret: @@ -573,7 +573,7 @@ def _get_msg(self): # wrap multiline tag (msg) interpolations to single line: for r, v in rows: for r in r[3].get('matches'): - if not isinstance(r, basestring): + if not isinstance(r, str): r = ''.join(r for r in r) r = v.replace("\x00msg\x00", r) output(r) @@ -639,9 +639,9 @@ def printLines(self, ltype): ans = [[]] for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] - b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + + b = [a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), - multiline, self._opts.usedns), ans) + multiline, self._opts.usedns) for a in ans] pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 24341014db..0b6c017259 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -71,7 +71,7 @@ def convert(self): @staticmethod def _fillStream(stream, opts, jailName): prio0idx = 0 - for opt, value in opts.iteritems(): + for opt, value in opts.items(): # Do not send a command if the value is not set (empty). if value is None: continue if opt in ("failregex", "ignoreregex"): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 9706c53054..51ad21c99e 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -118,7 +118,7 @@ def _glob(path): } _configOpts.update(FilterReader._configOpts) - _ignoreOpts = set(['action', 'filter', 'enabled'] + FilterReader._configOpts.keys()) + _ignoreOpts = set(['action', 'filter', 'enabled'] + list(FilterReader._configOpts.keys())) def getOptions(self): @@ -241,7 +241,7 @@ def convert(self, allow_no_files=False): stream.extend(self.__filter.convert()) # and using options from jail: FilterReader._fillStream(stream, self.__opts, self.__name) - for opt, value in self.__opts.iteritems(): + for opt, value in self.__opts.items(): if opt == "logpath": if self.__opts.get('backend', '').startswith("systemd"): continue found_files = 0 diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 5c1750a622..dee445fca7 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -31,6 +31,7 @@ from threading import Lock from .server.mytime import MyTime +import importlib try: import ctypes @@ -63,7 +64,7 @@ def __resetDefaultEncoding(encoding): from imp import load_dynamic as __ldm _sys = __ldm('_sys', 'sys') except ImportError: # pragma: no cover - only if load_dynamic fails - reload(sys) + importlib.reload(sys) _sys = sys if hasattr(_sys, "setdefaultencoding"): _sys.setdefaultencoding(encoding) @@ -101,7 +102,7 @@ def uni_string(x): else: # pragma: 3.x no cover def uni_decode(x, enc=PREFER_ENC, errors='strict'): try: - if isinstance(x, unicode): + if isinstance(x, str): return x.encode(enc, errors) return x except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable @@ -110,7 +111,7 @@ def uni_decode(x, enc=PREFER_ENC, errors='strict'): return x.encode(enc, 'replace') if sys.getdefaultencoding().upper() != 'UTF-8': # pragma: no cover - utf-8 is default encoding now def uni_string(x): - if not isinstance(x, unicode): + if not isinstance(x, str): return str(x) return x.encode(PREFER_ENC, 'replace') else: @@ -118,7 +119,7 @@ def uni_string(x): def _as_bool(val): - return bool(val) if not isinstance(val, basestring) \ + return bool(val) if not isinstance(val, str) \ else val.lower() in ('1', 'on', 'true', 'yes') @@ -327,7 +328,7 @@ def splitwords(s): """ if not s: return [] - return filter(bool, map(lambda v: v.strip(), re.split('[ ,\n]+', s))) + return list(filter(bool, [v.strip() for v in re.split('[ ,\n]+', s)])) if sys.version_info >= (3,5): eval(compile(r'''if 1: @@ -444,7 +445,7 @@ def substituteRecursiveTags(inptags, conditional='', while True: repFlag = False # substitute each value: - for tag in tags.iterkeys(): + for tag in tags.keys(): # ignore escaped or already done (or in ignore list): if tag in ignore or tag in done: continue # ignore replacing callable items from calling map - should be converted on demand only (by get): @@ -484,7 +485,7 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue # if calling map - be sure we've string: - if not isinstance(repl, basestring): repl = uni_string(repl) + if not isinstance(repl, str): repl = uni_string(repl) value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 16ff66212a..760a73d787 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -114,9 +114,9 @@ def _asrepr(self, calculated=False): def _asdict(self, calculated=False, checker=None): d = dict(self.data, **self.storage) if not calculated: - return dict((n,v) for n,v in d.iteritems() \ + return dict((n,v) for n,v in d.items() \ if not callable(v) or n in self.CM_REPR_ITEMS) - for n,v in d.items(): + for n,v in list(d.items()): if callable(v): try: # calculate: @@ -182,7 +182,7 @@ def copy(self): return self.__class__(_merge_copy_dicts(self.data, self.storage)) -class ActionBase(object): +class ActionBase(object, metaclass=ABCMeta): """An abstract base class for actions in Fail2Ban. Action Base is a base definition of what methods need to be in @@ -212,7 +212,6 @@ class ActionBase(object): Any additional arguments specified in `jail.conf` or passed via `fail2ban-client` will be passed as keyword arguments. """ - __metaclass__ = ABCMeta @classmethod def __subclasshook__(cls, C): @@ -423,7 +422,7 @@ def _operationExecuted(self, tag, family, *args): if not callable(family): # pragma: no cover return self.__substCache.get(key, {}).get(family) # family as expression - use it to filter values: - return [v for f, v in self.__substCache.get(key, {}).iteritems() if family(f)] + return [v for f, v in self.__substCache.get(key, {}).items() if family(f)] cmd = args[0] if cmd: # set: try: @@ -435,7 +434,7 @@ def _operationExecuted(self, tag, family, *args): try: famd = self.__substCache[key] cmd = famd.pop(family) - for family, v in famd.items(): + for family, v in list(famd.items()): if v == cmd: del famd[family] except KeyError: # pragma: no cover @@ -451,7 +450,7 @@ def _executeOperation(self, tag, operation, family=[], afterExec=None): res = True err = 'Script error' if not family: # all started: - family = [famoper for (famoper,v) in self.__started.iteritems() if v] + family = [famoper for (famoper,v) in self.__started.items() if v] for famoper in family: try: cmd = self._getOperation(tag, famoper) @@ -631,7 +630,7 @@ def flush(self): and executes the resulting command. """ # collect started families, may be started on demand (conditional): - family = [f for (f,v) in self.__started.iteritems() if v & 3 == 3]; # started and contains items + family = [f for (f,v) in self.__started.items() if v & 3 == 3]; # started and contains items # if nothing contains items: if not family: return True # flush: @@ -656,7 +655,7 @@ def _stop(self, family=None): """ # collect started families, if started on demand (conditional): if family is None: - family = [f for (f,v) in self.__started.iteritems() if v] + family = [f for (f,v) in self.__started.items() if v] # if no started (on demand) actions: if not family: return True self.__started = {} @@ -690,7 +689,7 @@ def consistencyCheck(self, beforeRepair=None): ret = True # for each started family: if self.actioncheck: - for (family, started) in self.__started.items(): + for (family, started) in list(self.__started.items()): if started and not self._invariantCheck(family, beforeRepair): # reset started flag and command of executed operation: self.__started[family] = 0 diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index fa045ab5ee..b052d342a9 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -156,11 +156,11 @@ def reload(self, begin=True): else: if hasattr(self, '_reload_actions'): # reload actions after all parameters set via stream: - for name, initOpts in self._reload_actions.iteritems(): + for name, initOpts in self._reload_actions.items(): if name in self._actions: self._actions[name].reload(**(initOpts if initOpts else {})) # remove obsolete actions (untouched by reload process): - delacts = OrderedDict((name, action) for name, action in self._actions.iteritems() + delacts = OrderedDict((name, action) for name, action in self._actions.items() if name not in self._reload_actions) if len(delacts): # unban all tickets using removed actions only: @@ -217,7 +217,7 @@ def getBanned(self, ids): return lst if len(ids) == 1: return 1 if ids[0] in lst else 0 - return map(lambda ip: 1 if ip in lst else 0, ids) + return [1 if ip in lst else 0 for ip in ids] def getBanList(self, withTime=False): """Returns the list of banned IP addresses. @@ -288,7 +288,7 @@ def removeBannedIP(self, ip=None, db=True, ifexists=False): if not isinstance(ip, IPAddr): ipa = IPAddr(ip) if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): - ips = filter(ipa.contains, self.banManager.getBanList()) + ips = list(filter(ipa.contains, self.banManager.getBanList())) if ips: return self.removeBannedIP(ips, db, ifexists) # not found: @@ -305,7 +305,7 @@ def stopActions(self, actions=None): """ if actions is None: actions = self._actions - for name, action in reversed(actions.items()): + for name, action in reversed(list(actions.items())): try: action.stop() except Exception as e: @@ -328,7 +328,7 @@ def run(self): True when the thread exits nicely. """ cnt = 0 - for name, action in self._actions.iteritems(): + for name, action in self._actions.items(): try: action.start() except Exception as e: @@ -505,7 +505,7 @@ def __checkBan(self, tickets=None): Observers.Main.add('banFound', bTicket, self._jail, btime) logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip) # do actions : - for name, action in self._actions.iteritems(): + for name, action in self._actions.items(): try: if bTicket.restored and getattr(action, 'norestored', False): continue @@ -543,13 +543,13 @@ def __checkBan(self, tickets=None): # avoid too often checks: if not rebanacts and MyTime.time() > self.__lastConsistencyCheckTM + 3: self.__lastConsistencyCheckTM = MyTime.time() - for action in self._actions.itervalues(): + for action in self._actions.values(): if hasattr(action, 'consistencyCheck'): action.consistencyCheck() # check epoch in order to reban it: if bTicket.banEpoch < self.banEpoch: if not rebanacts: rebanacts = dict( - (name, action) for name, action in self._actions.iteritems() + (name, action) for name, action in self._actions.items() if action.banEpoch > bTicket.banEpoch) cnt += self.__reBan(bTicket, actions=rebanacts) else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions: @@ -576,8 +576,8 @@ def __reBan(self, ticket, actions=None, log=True): ip = ticket.getID() aInfo = self._getActionInfo(ticket) if log: - logSys.notice("[%s] Reban %s%s", self._jail.name, ip, (', action %r' % actions.keys()[0] if len(actions) == 1 else '')) - for name, action in actions.iteritems(): + logSys.notice("[%s] Reban %s%s", self._jail.name, ip, (', action %r' % list(actions.keys())[0] if len(actions) == 1 else '')) + for name, action in actions.items(): try: logSys.debug("[%s] action %r: reban %s", self._jail.name, name, ip) if not aInfo.immutable: aInfo.reset() @@ -601,7 +601,7 @@ def _prolongBan(self, ticket): if not self.banManager._inBanList(ticket): return # do actions : aInfo = None - for name, action in self._actions.iteritems(): + for name, action in self._actions.items(): try: if ticket.restored and getattr(action, 'norestored', False): continue @@ -650,7 +650,7 @@ def __flushBan(self, db=False, actions=None, stop=False): cnt = 0 # first we'll execute flush for actions supporting this operation: unbactions = {} - for name, action in (actions if actions is not None else self._actions).iteritems(): + for name, action in (actions if actions is not None else self._actions).items(): try: if hasattr(action, 'flush') and (not isinstance(action, CommandAction) or action.actionflush): logSys.notice("[%s] Flush ticket(s) with %s", self._jail.name, name) @@ -705,7 +705,7 @@ def __unBan(self, ticket, actions=None, log=True): aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Unban %s", self._jail.name, ip) - for name, action in unbactions.iteritems(): + for name, action in unbactions.items(): try: logSys.debug("[%s] action %r: unban %s", self._jail.name, name, ip) if not aInfo.immutable: aInfo.reset() diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index e34007374e..f5f9740b2d 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -178,7 +178,7 @@ def loop(active, timeout=None, use_poll=False, err_count=None): elif err_count['listen'] > 100: # pragma: no cover - normally unreachable if ( e.args[0] == errno.EMFILE # [Errno 24] Too many open files - or sum(err_count.itervalues()) > 1000 + or sum(err_count.values()) > 1000 ): logSys.critical("Too many errors - critical count reached %r", err_count) break @@ -220,7 +220,7 @@ def handle_accept(self): elif self.__errCount['accept'] > 100: if ( (isinstance(e, socket.error) and e.args[0] == errno.EMFILE) # [Errno 24] Too many open files - or sum(self.__errCount.itervalues()) > 1000 + or sum(self.__errCount.values()) > 1000 ): logSys.critical("Too many errors - critical count reached %r", self.__errCount) self.stop() diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 9168d5b89b..d3e89820d3 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -103,7 +103,7 @@ def getBanList(self, ordered=False, withTime=False): return list(self.__banList.keys()) with self.__lock: lst = [] - for ticket in self.__banList.itervalues(): + for ticket in self.__banList.values(): eob = ticket.getEndOfBanTime(self.__banTime) lst.append((ticket,eob)) lst.sort(key=lambda t: t[1]) @@ -161,7 +161,7 @@ def getBanListExtendedCymruInfo(self, timeout=10): return return_dict # get ips in lock: with self.__lock: - banIPs = [banData.getIP() for banData in self.__banList.values()] + banIPs = [banData.getIP() for banData in list(self.__banList.values())] # get cymru info: try: for ip in banIPs: @@ -333,7 +333,7 @@ def unBanList(self, time, maxCount=0x7fffffff): # Gets the list of ticket to remove (thereby correct next unban time). unBanList = {} nextUnbanTime = BanTicket.MAX_TIME - for fid,ticket in self.__banList.iteritems(): + for fid,ticket in self.__banList.items(): # current time greater as end of ban - timed out: eob = ticket.getEndOfBanTime(self.__banTime) if time > eob: @@ -349,15 +349,15 @@ def unBanList(self, time, maxCount=0x7fffffff): if len(unBanList): if len(unBanList) / 2.0 <= len(self.__banList) / 3.0: # few as 2/3 should be removed - remove particular items: - for fid in unBanList.iterkeys(): + for fid in unBanList.keys(): del self.__banList[fid] else: # create new dictionary without items to be deleted: - self.__banList = dict((fid,ticket) for fid,ticket in self.__banList.iteritems() \ + self.__banList = dict((fid,ticket) for fid,ticket in self.__banList.items() \ if fid not in unBanList) # return list of tickets: - return unBanList.values() + return list(unBanList.values()) ## # Flush the ban list. @@ -367,7 +367,7 @@ def unBanList(self, time, maxCount=0x7fffffff): def flushBanList(self): with self.__lock: - uBList = self.__banList.values() + uBList = list(self.__banList.values()) self.__banList = dict() return uBList diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 877cbb93b7..59e376b711 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -67,13 +67,13 @@ def _json_loads_safe(x): else: # pragma: 3.x no cover def _normalize(x): if isinstance(x, dict): - return dict((_normalize(k), _normalize(v)) for k, v in x.iteritems()) + return dict((_normalize(k), _normalize(v)) for k, v in x.items()) elif isinstance(x, (list, set)): return [_normalize(element) for element in x] - elif isinstance(x, unicode): + elif isinstance(x, str): # in 2.x default text_factory is unicode - so return proper unicode here: return x.encode(PREFER_ENC, 'replace').decode(PREFER_ENC) - elif isinstance(x, basestring): + elif isinstance(x, str): return x.decode(PREFER_ENC, 'replace') return x diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 3c71d51a47..2effc45010 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -55,7 +55,7 @@ def getFailTotal(self): def getFailCount(self): # may be slow on large list of failures, should be used for test purposes only... with self.__lock: - return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) + return len(self.__failList), sum([f.getRetry() for f in list(self.__failList.values())]) def setMaxRetry(self, value): self.__maxRetry = value @@ -116,7 +116,7 @@ def addFailure(self, ticket, count=1, observed=False): # in case of having many active failures, it should be ran only # if debug level is "low" enough failures_summary = ', '.join(['%s:%d' % (k, v.getRetry()) - for k,v in self.__failList.iteritems()]) + for k,v in self.__failList.items()]) logSys.log(logLevel, "Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s" % (self.__failTotal, len(self.__failList), failures_summary)) @@ -129,7 +129,7 @@ def size(self): def cleanup(self, time): time -= self.__maxTime with self.__lock: - todelete = [fid for fid,item in self.__failList.iteritems() \ + todelete = [fid for fid,item in self.__failList.items() \ if item.getTime() <= time] if len(todelete) == len(self.__failList): # remove all: @@ -143,7 +143,7 @@ def cleanup(self, time): del self.__failList[fid] else: # create new dictionary without items to be deleted: - self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \ + self.__failList = dict((fid,item) for fid,item in self.__failList.items() \ if item.getTime() > time) self.__bgSvc.service() diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index a9b144af55..89c9438da9 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -143,9 +143,7 @@ def __init__(self, regex, multiline=False, **kwargs): self._regex = regex self._altValues = [] self._tupleValues = [] - for k in filter( - lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex - ): + for k in [k for k in self._regexObj.groupindex if len(k) > len(COMPLNAME_PRE[0])]: n = COMPLNAME_CRE.match(k) if n: g, n = n.group(1), mapTag2Opt(n.group(2)) @@ -235,7 +233,7 @@ def getRegex(self): # @staticmethod def _tupleLinesBuf(tupleLines): - return "\n".join(map(lambda v: "".join(v[::2]), tupleLines)) + "\n" + return "\n".join(["".join(v[::2]) for v in tupleLines]) + "\n" ## # Searches the regular expression. @@ -247,7 +245,7 @@ def _tupleLinesBuf(tupleLines): def search(self, tupleLines, orgLines=None): buf = tupleLines - if not isinstance(tupleLines, basestring): + if not isinstance(tupleLines, str): buf = Regex._tupleLinesBuf(tupleLines) self._matchCache = self._regexObj.search(buf) if self._matchCache: diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 196955e59f..8a289cea03 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -173,4 +173,4 @@ def isModified(self, filename): return False def getPendingPaths(self): - return self.__file404Cnt.keys() + return list(self.__file404Cnt.keys()) diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 16b6cfd553..2ee49169e1 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -155,7 +155,7 @@ def _delPending(self, path): except KeyError: pass def getPendingPaths(self): - return self.__pending.keys() + return list(self.__pending.keys()) def _checkPending(self): if not self.__pending: @@ -181,7 +181,7 @@ def _checkPending(self): self.__pendingChkTime = time.time() self.__pendingMinTime = minTime # process now because we've missed it in monitoring: - for path, isDir in found.iteritems(): + for path, isDir in found.items(): self._delPending(path) # refresh monitoring of this: if isDir is not None: diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index a83b7a132d..2ab67a9a64 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -253,7 +253,7 @@ def formatJournalEntry(self, logentry): return ((logline[:0], date[0] + ' ', logline.replace('\n', '\\n')), date[1]) def seekToTime(self, date): - if isinstance(date, (int, long)): + if isinstance(date, int): date = float(date) self.__journal.seek_realtime(date) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index b435c6dfb3..0f7ccb016c 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -366,7 +366,7 @@ def __wrap_ipstr(ipstr): raise ValueError("invalid ipstr %r, too many plen representation" % (ipstr,)) if "." in s[1] or ":" in s[1]: # 255.255.255.0 resp. ffff:: style mask s[1] = IPAddr.masktoplen(s[1]) - s[1] = long(s[1]) + s[1] = int(s[1]) return s def __init(self, ipstr, cidr=CIDR_UNSPEC): @@ -400,7 +400,7 @@ def __init(self, ipstr, cidr=CIDR_UNSPEC): # mask out host portion if prefix length is supplied if cidr is not None and cidr >= 0: - mask = ~(0xFFFFFFFFL >> cidr) + mask = ~(0xFFFFFFFF >> cidr) self._addr &= mask self._plen = cidr @@ -412,13 +412,13 @@ def __init(self, ipstr, cidr=CIDR_UNSPEC): # mask out host portion if prefix length is supplied if cidr is not None and cidr >= 0: - mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr) + mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF >> cidr) self._addr &= mask self._plen = cidr # if IPv6 address is a IPv4-compatible, make instance a IPv4 elif self.isInNet(IPAddr.IP6_4COMPAT): - self._addr = lo & 0xFFFFFFFFL + self._addr = lo & 0xFFFFFFFF self._family = socket.AF_INET self._plen = 32 else: @@ -428,7 +428,7 @@ def __repr__(self): return repr(self.ntoa) def __str__(self): - return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) + return self.ntoa if isinstance(self.ntoa, str) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str @@ -542,7 +542,7 @@ def ntoa(self): elif self.isIPv6: # convert network to host byte order hi = self._addr >> 64 - lo = self._addr & 0xFFFFFFFFFFFFFFFFL + lo = self._addr & 0xFFFFFFFFFFFFFFFF binary = struct.pack("!QQ", hi, lo) if self._plen and self._plen < 128: add = "/%d" % self._plen @@ -600,9 +600,9 @@ def isInNet(self, net): if self.family != net.family: return False if self.isIPv4: - mask = ~(0xFFFFFFFFL >> net.plen) + mask = ~(0xFFFFFFFF >> net.plen) elif self.isIPv6: - mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen) + mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF >> net.plen) else: return False @@ -622,7 +622,7 @@ def __getMaskMap(): m4 = (1 << 32)-1 mmap = {m6: 128, m4: 32, 0: 0} m = 0 - for i in xrange(0, 128): + for i in range(0, 128): m |= 1 << i if i < 32: mmap[m ^ m4] = 32-1-i diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 2c84e47538..fe5c605aab 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -26,7 +26,7 @@ import logging import math import random -import Queue +import queue from .actions import Actions from ..helpers import getLogger, _as_bool, extractOptions, MyTime @@ -76,7 +76,7 @@ def __init__(self, name, backend = "auto", db=None): "might not function correctly. Please shorten" % name) self.__name = name - self.__queue = Queue.Queue() + self.__queue = queue.Queue() self.__filter = None # Extra parameters for increase ban time self._banExtra = {}; @@ -127,25 +127,25 @@ def _setBackend(self, backend): "Failed to initialize any backend for Jail %r" % self.name) def _initPolling(self, **kwargs): - from filterpoll import FilterPoll + from .filterpoll import FilterPoll logSys.info("Jail '%s' uses poller %r" % (self.name, kwargs)) self.__filter = FilterPoll(self, **kwargs) def _initGamin(self, **kwargs): # Try to import gamin - from filtergamin import FilterGamin + from .filtergamin import FilterGamin logSys.info("Jail '%s' uses Gamin %r" % (self.name, kwargs)) self.__filter = FilterGamin(self, **kwargs) def _initPyinotify(self, **kwargs): # Try to import pyinotify - from filterpyinotify import FilterPyinotify + from .filterpyinotify import FilterPyinotify logSys.info("Jail '%s' uses pyinotify %r" % (self.name, kwargs)) self.__filter = FilterPyinotify(self, **kwargs) def _initSystemd(self, **kwargs): # pragma: systemd no cover # Try to import systemd - from filtersystemd import FilterSystemd + from .filtersystemd import FilterSystemd logSys.info("Jail '%s' uses systemd %r" % (self.name, kwargs)) self.__filter = FilterSystemd(self, **kwargs) @@ -219,7 +219,7 @@ def getFailTicket(self): try: ticket = self.__queue.get(False) return ticket - except Queue.Empty: + except queue.Empty: return False def setBanTimeExtra(self, opt, value): diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 315d8a3004..ff46b7efeb 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -165,7 +165,7 @@ def str2seconds(val): @returns number (calculated seconds from expression "val") """ - if isinstance(val, (int, long, float, complex)): + if isinstance(val, (int, float, complex)): return val # replace together standing abbreviations, example '1d12h' -> '1d 12h': val = MyTime._str2sec_prep.sub(r" \1", val) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index ce201b5e7b..9de82fc907 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -209,7 +209,7 @@ def quit(self): # Restore default signal handlers: if _thread_name() == '_MainThread': - for s, sh in self.__prev_signals.iteritems(): + for s, sh in self.__prev_signals.items(): signal.signal(s, sh) # Give observer a small chance to complete its work before exit @@ -287,10 +287,10 @@ def stopAllJail(self): logSys.info("Stopping all jails") with self.__lock: # 1st stop all jails (signal and stop actions/filter thread): - for name in self.__jails.keys(): + for name in list(self.__jails.keys()): self.delJail(name, stop=True, join=False) # 2nd wait for end and delete jails: - for name in self.__jails.keys(): + for name in list(self.__jails.keys()): self.delJail(name, stop=False, join=True) def clearCaches(self): @@ -328,7 +328,7 @@ def reloadJails(self, name, opts, begin): if "--restart" in opts: self.stopAllJail() # first set all affected jail(s) to idle and reset filter regex and other lists/dicts: - for jn, jail in self.__jails.iteritems(): + for jn, jail in self.__jails.items(): if name == '--all' or jn == name: jail.idle = True self.__reload_state[jn] = jail @@ -339,7 +339,7 @@ def reloadJails(self, name, opts, begin): # end reload, all affected (or new) jails have already all new parameters (via stream) and (re)started: with self.__lock: deljails = [] - for jn, jail in self.__jails.iteritems(): + for jn, jail in self.__jails.items(): # still in reload state: if jn in self.__reload_state: # remove jails that are not reloaded (untouched, so not in new configuration) @@ -548,7 +548,7 @@ def setUnbanIP(self, name=None, value=None, ifexists=True): jails = [self.__jails[name]] else: # in all jails: - jails = self.__jails.values() + jails = list(self.__jails.values()) # unban given or all (if value is None): cnt = 0 ifexists |= (name is None) @@ -562,7 +562,7 @@ def banned(self, name=None, ids=None): jails = [self.__jails[name]] else: # in all jails: - jails = self.__jails.values() + jails = list(self.__jails.values()) # check banned ids: res = [] if name is None and ids: @@ -612,7 +612,7 @@ def isStarted(self): def isAlive(self, jailnum=None): if jailnum is not None and len(self.__jails) != jailnum: return 0 - for jail in self.__jails.values(): + for jail in list(self.__jails.values()): if not jail.isAlive(): return 0 return 1 @@ -827,7 +827,7 @@ def setIPv6IsAllowed(value): return DNSUtils.setIPv6IsAllowed(value) def setThreadOptions(self, value): - for o, v in value.iteritems(): + for o, v in value.items(): if o == 'stacksize': threading.stack_size(int(v)*1024) else: # pragma: no cover @@ -951,7 +951,7 @@ def __createDaemon(self): # pragma: no cover maxfd = os.sysconf("SC_OPEN_MAX") except (AttributeError, ValueError): maxfd = 256 # default maximum - fdlist = xrange(maxfd+1) + fdlist = range(maxfd+1) # urandom should not be closed in Python 3.4.0. Fixed in 3.4.1 # http://bugs.python.org/issue21207 diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 12be163a60..1f459f028b 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -99,7 +99,7 @@ def grp(exprset): if len(exprset) > 1 else "".join(exprset) exprset = set( cent(now[0].year + i) for i in (-1, distance) ) if len(now) > 1 and now[1]: - exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) + exprset |= set( cent(now[1].year + i) for i in range(-1, now[0].year-now[1].year+1, distance) ) return grp(sorted(list(exprset))) # more precise year patterns, within same century of last year and @@ -116,7 +116,7 @@ def grp(exprset): _updateTimeRE() def getTimePatternRE(): - keys = timeRE.keys() + keys = list(timeRE.keys()) patt = (r"%%(%%|%s|[%s])" % ( "|".join([k for k in keys if len(k) > 1]), "".join([k for k in keys if len(k) == 1]), @@ -171,7 +171,7 @@ def zone2offset(tz, dt): """ if isinstance(tz, int): return tz - if isinstance(tz, basestring): + if isinstance(tz, str): return validateTimeZone(tz) tz, tzo = tz if tzo is None or tzo == '': # without offset @@ -208,7 +208,7 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): year = month = day = tzoffset = \ weekday = julian = week_of_year = None hour = minute = second = fraction = 0 - for key, val in found_dict.iteritems(): + for key, val in found_dict.items(): if val is None: continue # Directives not explicitly handled below: # c, x, X diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 96e6777353..80b1469c58 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -55,7 +55,7 @@ def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None): self._time = time if time is not None else MyTime.time() self._data = {'matches': matches or [], 'failures': 0} if data is not None: - for k,v in data.iteritems(): + for k,v in data.items(): if v is not None: self._data[k] = v if ticket: @@ -88,7 +88,7 @@ def update(self, ticket): def setID(self, value): # guarantee using IPAddr instead of unicode, str for the IP - if isinstance(value, basestring): + if isinstance(value, str): value = IPAddr(value) self._id = value @@ -180,7 +180,7 @@ def setData(self, *args, **argv): if len(args) == 1: # todo: if support >= 2.7 only: # self._data = {k:v for k,v in args[0].iteritems() if v is not None} - self._data = dict([(k,v) for k,v in args[0].iteritems() if v is not None]) + self._data = dict([(k,v) for k,v in args[0].items() if v is not None]) # add k,v list or dict (merge): elif len(args) == 2: self._data.update((args,)) @@ -191,7 +191,7 @@ def setData(self, *args, **argv): # filter (delete) None values: # todo: if support >= 2.7 only: # self._data = {k:v for k,v in self._data.iteritems() if v is not None} - self._data = dict([(k,v) for k,v in self._data.iteritems() if v is not None]) + self._data = dict([(k,v) for k,v in self._data.items() if v is not None]) def getData(self, key=None, default=None): # return whole data dict: @@ -200,17 +200,17 @@ def getData(self, key=None, default=None): # return default if not exists: if not self._data: return default - if not isinstance(key,(str,unicode,type(None),int,float,bool,complex)): + if not isinstance(key,(str,type(None),int,float,bool,complex)): # return filtered by lambda/function: if callable(key): # todo: if support >= 2.7 only: # return {k:v for k,v in self._data.iteritems() if key(k)} - return dict([(k,v) for k,v in self._data.iteritems() if key(k)]) + return dict([(k,v) for k,v in self._data.items() if key(k)]) # return filtered by keys: if hasattr(key, '__iter__'): # todo: if support >= 2.7 only: # return {k:v for k,v in self._data.iteritems() if k in key} - return dict([(k,v) for k,v in self._data.iteritems() if k in key]) + return dict([(k,v) for k,v in self._data.items() if k in key]) # return single value of data: return self._data.get(key, default) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index c25ae637a1..660ed03aed 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -500,7 +500,7 @@ def __commandGet(self, command): opt = command[1][len("bantime."):] return self.__server.getBanTimeExtra(name, opt) elif command[1] == "actions": - return self.__server.getActions(name).keys() + return list(self.__server.getActions(name).keys()) elif command[1] == "action": actionname = command[2] actionvalue = command[3] diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 18073ea7d5..729ee43f20 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -53,7 +53,7 @@ # Dictionary to lookup signal name from number signame = dict((num, name) - for name, num in signal.__dict__.iteritems() if name.startswith("SIG")) + for name, num in signal.__dict__.items() if name.startswith("SIG")) class Utils(): """Utilities provide diverse static methods like executes OS shell commands, etc. @@ -140,7 +140,7 @@ def buildShellCmd(realCmd, varsDict): if not isinstance(realCmd, list): realCmd = [realCmd] i = len(realCmd)-1 - for k, v in varsDict.iteritems(): + for k, v in varsDict.items(): varsStat += "%s=$%s " % (k, i) realCmd.append(v) i += 1 diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index ce5de48334..772c687ff4 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -242,14 +242,14 @@ def testReplaceTagSelfRecursion(self): setattr(self.__action, 'ab', "") setattr(self.__action, 'x?family=inet6', "") # produce self-referencing properties except: - self.assertRaisesRegexp(ValueError, r"properties contain self referencing definitions", + self.assertRaisesRegex(ValueError, r"properties contain self referencing definitions", lambda: self.__action.replaceTag("", self.__action._properties, conditional="family=inet4") ) # remore self-referencing in props: delattr(self.__action, 'ac') # produce self-referencing query except: - self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", + self.assertRaisesRegex(ValueError, r"possible self referencing definitions in query", lambda: self.__action.replaceTag(""*30, self.__action._properties, conditional="family=inet6") ) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index cf25ac0fcd..2c0c4c4fc8 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -177,7 +177,7 @@ def setUp(self): super(StatusExtendedCymruInfo, self).setUp() unittest.F2B.SkipIfNoNetwork() setUpMyTime() - self.__ban_ip = iter(DNSUtils.dnsToIp("resolver1.opendns.com")).next() + self.__ban_ip = next(iter(DNSUtils.dnsToIp("resolver1.opendns.com"))) self.__asn = "36692" self.__country = "US" self.__rir = "arin" diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 37083a069e..a98f4e94cd 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -419,7 +419,7 @@ def testSplitOption(self): # And multiple groups (`][` instead of `,`) result = extractOptions(option.replace(',', '][')) expected2 = (expected[0], - dict((k, v.replace(',', '][')) for k, v in expected[1].iteritems()) + dict((k, v.replace(',', '][')) for k, v in expected[1].items()) ) self.assertEqual(expected2, result) @@ -1018,7 +1018,7 @@ def testMultipleSameAction(self, basedir): self.assertEqual(add_actions[-1][-1], "{}") def testLogPathFileFilterBackend(self): - self.assertRaisesRegexp(ValueError, r"Have not found any log file for .* jail", + self.assertRaisesRegex(ValueError, r"Have not found any log file for .* jail", self._testLogPath, backend='polling') def testLogPathSystemdBackend(self): diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 8cc394be73..6ac483c6cb 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -67,7 +67,7 @@ def setUp(self): @property def db(self): - if isinstance(self._db, basestring) and self._db == ':auto-create-in-memory:': + if isinstance(self._db, str) and self._db == ':auto-create-in-memory:': self._db = getFail2BanDb(self.dbFilename) return self._db @db.setter @@ -159,7 +159,7 @@ def testUpdateDb(self): self.db = Fail2BanDb(self.dbFilename) self.assertEqual(self.db.getJailNames(), set(['DummyJail #29162448 with 0 tickets'])) self.assertEqual(self.db.getLogPaths(), set(['/tmp/Fail2BanDb_pUlZJh.log'])) - ticket = FailTicket("127.0.0.1", 1388009242.26, [u"abc\n"]) + ticket = FailTicket("127.0.0.1", 1388009242.26, ["abc\n"]) self.assertEqual(self.db.getBans()[0], ticket) self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__) @@ -185,9 +185,9 @@ def testUpdateDb2(self): self.assertEqual(len(bans), 2) # compare first ticket completely: ticket = FailTicket("1.2.3.7", 1417595494, [ - u'Dec 3 09:31:08 f2btest test:auth[27658]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', - u'Dec 3 09:31:32 f2btest test:auth[27671]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', - u'Dec 3 09:31:34 f2btest test:auth[27673]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7' + 'Dec 3 09:31:08 f2btest test:auth[27658]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', + 'Dec 3 09:31:32 f2btest test:auth[27671]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', + 'Dec 3 09:31:34 f2btest test:auth[27673]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7' ]) ticket.setAttempt(3) self.assertEqual(bans[0], ticket) @@ -287,11 +287,11 @@ def testAddBanInvalidEncoded(self): # invalid + valid, invalid + valid unicode, invalid + valid dual converted (like in filter:readline by fallback) ... tickets = [ FailTicket("127.0.0.1", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), - FailTicket("127.0.0.2", 0, ['user "test"', u'user "\xd1\xe2\xe5\xf2\xe0"', u'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), + FailTicket("127.0.0.2", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), FailTicket("127.0.0.3", 0, ['user "test"', b'user "\xd1\xe2\xe5\xf2\xe0"', b'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), - FailTicket("127.0.0.4", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', u'user "\xe4\xf6\xfc\xdf"']), + FailTicket("127.0.0.4", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xe4\xf6\xfc\xdf"']), FailTicket("127.0.0.5", 0, ['user "test"', 'unterminated \xcf']), - FailTicket("127.0.0.6", 0, ['user "test"', u'unterminated \xcf']), + FailTicket("127.0.0.6", 0, ['user "test"', 'unterminated \xcf']), FailTicket("127.0.0.7", 0, ['user "test"', b'unterminated \xcf']) ] for ticket in tickets: diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index bc33bc0528..fc43677a1b 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -288,7 +288,7 @@ def testFullYearMatch_gh130(self): self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') # confuse it with year being at the end - for i in xrange(10): + for i in range(10): ( logTime, logMatch ) = self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '11/10/2012 02:37:17') @@ -538,7 +538,7 @@ def testAmbiguousDatePattern(self): date = dd.getTime(line) if matched: self.assertTrue(date) - if isinstance(matched, basestring): + if isinstance(matched, str): self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(matched, date[0]) @@ -573,7 +573,7 @@ def testVariousFormatSpecs(self): date = dd.getTime(line) if matched: self.assertTrue(date) - if isinstance(matched, basestring): # pragma: no cover + if isinstance(matched, str): # pragma: no cover self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(matched, date[0]) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index bc93dae7cc..9d3f382fcb 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -367,10 +367,10 @@ def _stopAndWaitForServerEnd(code=(SUCCESS, FAILED)): # several commands to server in body of decorated function: return f(self, tmp, startparams, *args, **kwargs) except Exception as e: # pragma: no cover - print('=== Catch an exception: %s' % e) + print(('=== Catch an exception: %s' % e)) log = self.getLog() if log: - print('=== Error of server, log: ===\n%s===' % log) + print(('=== Error of server, log: ===\n%s===' % log)) self.pruneLog() raise finally: @@ -440,7 +440,7 @@ def _wait_for_srv(self, tmp, ready=True, startparams=None, phase=None): ) except: # pragma: no cover if _inherited_log(startparams): - print('=== Error by wait fot server, log: ===\n%s===' % self.getLog()) + print(('=== Error by wait fot server, log: ===\n%s===' % self.getLog())) self.pruneLog() log = pjoin(tmp, "f2b.log") if isfile(log): @@ -1702,6 +1702,6 @@ def _testServerStartStop(self, tmp, startparams): self.stopAndWaitForServerEnd(SUCCESS) def testServerStartStop(self): - for i in xrange(2000): + for i in range(2000): self._testServerStartStop() diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 213ea89b8a..a71a17774d 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -589,8 +589,8 @@ def testNLCharAsPartOfUniChar(self): # test on unicode string containing \x0A as part of uni-char, # it must produce exactly 2 lines (both are failures): for l in ( - u'1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', - u'1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' + '1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', + '1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' ): fout.write(l.encode(enc)) fout.close() diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index 42b0fbd297..b07159fbb0 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -45,11 +45,11 @@ def tearDown(self): super(AddFailure, self).tearDown() def _addDefItems(self): - self.__items = [[u'193.168.0.128', 1167605999.0], - [u'193.168.0.128', 1167605999.0], - [u'193.168.0.128', 1167605999.0], - [u'193.168.0.128', 1167605999.0], - [u'193.168.0.128', 1167605999.0], + self.__items = [['193.168.0.128', 1167605999.0], + ['193.168.0.128', 1167605999.0], + ['193.168.0.128', 1167605999.0], + ['193.168.0.128', 1167605999.0], + ['193.168.0.128', 1167605999.0], ['87.142.124.10', 1167605999.0], ['87.142.124.10', 1167605999.0], ['87.142.124.10', 1167605999.0], diff --git a/fail2ban/tests/files/config/apache-auth/digest.py b/fail2ban/tests/files/config/apache-auth/digest.py index 0358859480..e2297ab312 100755 --- a/fail2ban/tests/files/config/apache-auth/digest.py +++ b/fail2ban/tests/files/config/apache-auth/digest.py @@ -41,7 +41,7 @@ def auth(v): response="%s" """ % ( username, algorithm, realm, url, nonce, qop, response ) # opaque="%s", - print(p.method, p.url, p.headers) + print((p.method, p.url, p.headers)) s = requests.Session() return s.send(p) @@ -76,18 +76,18 @@ def preauth(): # [Sun Jul 28 21:41:20 2013] [error] [client 127.0.0.1] Digest: unknown algorithm `super funky chicken' received: /digest/ -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) v['algorithm'] = algorithm r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) nonce = v['nonce'] v['nonce']=v['nonce'][5:-5] r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) # [Sun Jul 28 21:05:31.178340 2013] [auth_digest:error] [pid 24224:tid 139895539455744] [client 127.0.0.1:56906] AH01793: invalid qop `auth' received: /digest/qop_none/ @@ -95,7 +95,7 @@ def preauth(): v['nonce']=nonce[0:11] + 'ZZZ' + nonce[14:] r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) #[Sun Jul 28 21:18:11.769228 2013] [auth_digest:error] [pid 24752:tid 139895505884928] [client 127.0.0.1:56964] AH01776: invalid nonce b9YAiJDiBAZZZ1b1abe02d20063ea3b16b544ea1b0d981c1bafe received - hash is not d42d824dee7aaf50c3ba0a7c6290bd453e3dd35b @@ -107,7 +107,7 @@ def preauth(): time.sleep(1) r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) # Obtained by putting the following code in modules/aaa/mod_auth_digest.c # in the function initialize_secret @@ -137,7 +137,7 @@ def preauth(): v=preauth() -print(v['nonce']) +print((v['nonce'])) realm = v['Digest realm'][1:-1] (t,) = struct.unpack('l',base64.b64decode(v['nonce'][1:13])) @@ -156,13 +156,13 @@ def preauth(): r = auth(v) #[Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) url='/digest_onetime/' v=preauth() # Need opaque header handling in auth r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) r = auth(v) -print(r.status_code,r.headers, r.text) +print((r.status_code,r.headers, r.text)) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 4e308e38e6..e31e6492d2 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -22,7 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier; 2012 Yaroslav Halchenko" __license__ = "GPL" -from __builtin__ import open as fopen +from builtins import open as fopen import unittest import os import re @@ -209,7 +209,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line else: fin = in_ # Skip - for i in xrange(skip): + for i in range(skip): fin.readline() # Read i = 0 @@ -250,7 +250,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p # Required for filtering fields.update(TEST_JOURNAL_FIELDS) # Skip - for i in xrange(skip): + for i in range(skip): fin.readline() # Read/Write i = 0 @@ -312,18 +312,18 @@ def testAssertWrongTime(self): def testTest_tm(self): unittest.F2B.SkipIfFast() ## test function "_tm" works correct (returns the same as slow strftime): - for i in xrange(1417512352, (1417512352 // 3600 + 3) * 3600): + for i in range(1417512352, (1417512352 // 3600 + 3) * 3600): tm = MyTime.time2str(i) if _tm(i) != tm: # pragma: no cover - never reachable self.assertEqual((_tm(i), i), (tm, i)) def testWrongCharInTupleLine(self): ## line tuple has different types (ascii after ascii / unicode): - for a1 in ('', u'', b''): - for a2 in ('2016-09-05T20:18:56', u'2016-09-05T20:18:56', b'2016-09-05T20:18:56'): + for a1 in ('', '', b''): + for a2 in ('2016-09-05T20:18:56', '2016-09-05T20:18:56', b'2016-09-05T20:18:56'): for a3 in ( 'Fail for "g\xc3\xb6ran" from 192.0.2.1', - u'Fail for "g\xc3\xb6ran" from 192.0.2.1', + 'Fail for "g\xc3\xb6ran" from 192.0.2.1', b'Fail for "g\xc3\xb6ran" from 192.0.2.1' ): # join should work if all arguments have the same type: @@ -510,7 +510,7 @@ def testWrongTimeOrTZ(self): def testAddAttempt(self): self.filter.setMaxRetry(3) - for i in xrange(1, 1+3): + for i in range(1, 1+3): self.filter.addAttempt('192.0.2.1') self.assertLogged('Attempt 192.0.2.1', '192.0.2.1:%d' % i, all=True, wait=True) self.jail.actions._Actions__checkBan() @@ -547,7 +547,7 @@ def testIgnoreCache(self): # like both test-cases above, just cached (so once per key)... self.filter.ignoreCache = {"key":""} self.filter.ignoreCommand = 'if [ "" = "10.0.0.1" ]; then exit 0; fi; exit 1' - for i in xrange(5): + for i in range(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1")) self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0")) @@ -558,7 +558,7 @@ def testIgnoreCache(self): # by host of IP: self.filter.ignoreCache = {"key":""} self.filter.ignoreCommand = 'if [ "" = "test-host" ]; then exit 0; fi; exit 1' - for i in xrange(5): + for i in range(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList(FailTicket("2001:db8::1"))) self.assertFalse(self.filter.inIgnoreIPList(FailTicket("2001:db8::ffff"))) @@ -570,7 +570,7 @@ def testIgnoreCache(self): self.filter.ignoreCache = {"key":"", "max-count":"10", "max-time":"1h"} self.assertEqual(self.filter.ignoreCache, ["", 10, 60*60]) self.filter.ignoreCommand = 'if [ "" = "tester" ]; then exit 0; fi; exit 1' - for i in xrange(5): + for i in range(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList(FailTicket("tester", data={'user': 'tester'}))) self.assertFalse(self.filter.inIgnoreIPList(FailTicket("root", data={'user': 'root'}))) @@ -673,7 +673,7 @@ def testMissingLogFiles(self): def testDecodeLineWarn(self): # incomplete line (missing byte at end), warning is suppressed: - l = u"correct line\n" + l = "correct line\n" r = l.encode('utf-16le') self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r), l) self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r[0:-1]), l[0:-1]) @@ -733,7 +733,7 @@ def testSeekToTimeSmallFile(self): fc = FileContainer(fname, self.filter.getLogEncoding()) fc.open() # no time - nothing should be found : - for i in xrange(10): + for i in range(10): f.write("[sshd] error: PAM: failure len 1\n") f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) @@ -807,14 +807,14 @@ def testSeekToTimeLargeFile(self): # variable length of file (ca 45K or 450K before and hereafter): # write lines with smaller as search time: t = time - count - 1 - for i in xrange(count): + for i in range(count): f.write("%s [sshd] error: PAM: failure\n" % _tm(t)) t += 1 f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) self.assertEqual(fc.getPos(), 47*count) # write lines with exact search time: - for i in xrange(10): + for i in range(10): f.write("%s [sshd] error: PAM: failure\n" % _tm(time)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) @@ -823,8 +823,8 @@ def testSeekToTimeLargeFile(self): self.assertEqual(fc.getPos(), 47*count) # write lines with greater as search time: t = time+1 - for i in xrange(count//500): - for j in xrange(500): + for i in range(count//500): + for j in range(500): f.write("%s [sshd] error: PAM: failure\n" % _tm(t)) t += 1 f.flush() @@ -1634,10 +1634,10 @@ def test_WrongChar(self): # Add direct utf, unicode, blob: for l in ( "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", - u"error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", + "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", b"error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1".decode('utf-8', 'replace'), "error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", - u"error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", + "error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", b"error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2".decode('utf-8', 'replace') ): fields = self.journal_fields @@ -1666,7 +1666,7 @@ class GetFailures(LogCaptureTestCase): # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, - [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3) + ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3) def setUp(self): """Call before every test case.""" @@ -1752,8 +1752,8 @@ def testNLCharAsPartOfUniChar(self): # test on unicode string containing \x0A as part of uni-char, # it must produce exactly 2 lines (both are failures): for l in ( - u'%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, - u'%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm + '%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, + '%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm ): fout.write(l.encode(enc)) fout.close() @@ -1774,8 +1774,8 @@ def testNLCharAsPartOfUniChar(self): def testGetFailures02(self): output = ('141.3.81.106', 4, 1124013539.0, - [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' - % m for m in 53, 54, 57, 58]) + ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' + % m for m in (53, 54, 57, 58)]) self.filter.setMaxRetry(4) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) @@ -1886,19 +1886,19 @@ def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) output_yes = ( ('93.184.216.34', 1, 1124013299.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] ), ('93.184.216.34', 1, 1124013539.0, - [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ), ('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] ), ) output_no = ( ('93.184.216.34', 1, 1124013539.0, - [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ) ) @@ -2004,9 +2004,9 @@ def testCache(self): self.assertTrue(c.get('a') is None) self.assertEqual(c.get('a', 'test'), 'test') # exact 5 elements : - for i in xrange(5): + for i in range(5): c.set(i, i) - for i in xrange(5): + for i in range(5): self.assertEqual(c.get(i), i) # remove unavailable key: c.unset('a'); c.unset('a') @@ -2014,30 +2014,30 @@ def testCache(self): def testCacheMaxSize(self): c = Utils.Cache(maxCount=5, maxTime=60) # exact 5 elements : - for i in xrange(5): + for i in range(5): c.set(i, i) - self.assertEqual([c.get(i) for i in xrange(5)], [i for i in xrange(5)]) - self.assertNotIn(-1, (c.get(i, -1) for i in xrange(5))) + self.assertEqual([c.get(i) for i in range(5)], [i for i in range(5)]) + self.assertNotIn(-1, (c.get(i, -1) for i in range(5))) # add one - too many: c.set(10, i) # one element should be removed : - self.assertIn(-1, (c.get(i, -1) for i in xrange(5))) + self.assertIn(-1, (c.get(i, -1) for i in range(5))) # test max size (not expired): - for i in xrange(10): + for i in range(10): c.set(i, 1) self.assertEqual(len(c), 5) def testCacheMaxTime(self): # test max time (expired, timeout reached) : c = Utils.Cache(maxCount=5, maxTime=0.0005) - for i in xrange(10): + for i in range(10): c.set(i, 1) st = time.time() self.assertTrue(Utils.wait_for(lambda: time.time() >= st + 0.0005, 1)) # we have still 5 elements (or fewer if too slow test mashine): self.assertTrue(len(c) <= 5) # but all that are expiered also: - for i in xrange(10): + for i in range(10): self.assertTrue(c.get(i) is None) # here the whole cache should be empty: self.assertEqual(len(c), 0) @@ -2058,7 +2058,7 @@ def _TestCacheStr2IP(forw=True, result=[], random=False): c = count while c: c -= 1 - s = xrange(0, 256, 1) if forw else xrange(255, -1, -1) + s = range(0, 256, 1) if forw else range(255, -1, -1) if random: shuffle([i for i in s]) for i in s: IPAddr('192.0.2.'+str(i), IPAddr.FAM_IPv4) @@ -2184,16 +2184,16 @@ def testIpToName(self): def testAddr2bin(self): res = IPAddr('10.0.0.0') - self.assertEqual(res.addr, 167772160L) + self.assertEqual(res.addr, 167772160) res = IPAddr('10.0.0.0', cidr=None) - self.assertEqual(res.addr, 167772160L) - res = IPAddr('10.0.0.0', cidr=32L) - self.assertEqual(res.addr, 167772160L) - res = IPAddr('10.0.0.1', cidr=32L) - self.assertEqual(res.addr, 167772161L) + self.assertEqual(res.addr, 167772160) + res = IPAddr('10.0.0.0', cidr=32) + self.assertEqual(res.addr, 167772160) + res = IPAddr('10.0.0.1', cidr=32) + self.assertEqual(res.addr, 167772161) self.assertTrue(res.isSingle) - res = IPAddr('10.0.0.1', cidr=31L) - self.assertEqual(res.addr, 167772160L) + res = IPAddr('10.0.0.1', cidr=31) + self.assertEqual(res.addr, 167772160) self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') @@ -2272,9 +2272,9 @@ def testIPAddr_Compare(self): '93.184.216.34': 'ip4-test', '2606:2800:220:1:248:1893:25c8:1946': 'ip6-test' } - d2 = dict([(IPAddr(k), v) for k, v in d.iteritems()]) - self.assertTrue(isinstance(d.keys()[0], basestring)) - self.assertTrue(isinstance(d2.keys()[0], IPAddr)) + d2 = dict([(IPAddr(k), v) for k, v in d.items()]) + self.assertTrue(isinstance(list(d.keys())[0], str)) + self.assertTrue(isinstance(list(d2.keys())[0], IPAddr)) self.assertEqual(d.get(ip4[2], ''), 'ip4-test') self.assertEqual(d.get(ip6[2], ''), 'ip6-test') self.assertEqual(d2.get(str(ip4[2]), ''), 'ip4-test') diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 4b02637755..d91729fb93 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -29,9 +29,9 @@ import shutil import fnmatch from glob import glob -from StringIO import StringIO +from io import StringIO -from utils import LogCaptureTestCase, logSys as DefLogSys +from .utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, \ getVerbosityFormat, splitwords, uni_decode, uni_string @@ -67,7 +67,7 @@ def testsplitwords(self): self.assertEqual(splitwords(' 1\n 2'), ['1', '2']) self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) # string as unicode: - self.assertEqual(splitwords(u' 1\n 2, 3'), ['1', '2', '3']) + self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) def _sh_call(cmd): @@ -191,12 +191,12 @@ def testmbasename(self): def testUniConverters(self): self.assertRaises(Exception, uni_decode, - (b'test' if sys.version_info >= (3,) else u'test'), 'f2b-test::non-existing-encoding') - uni_decode((b'test\xcf' if sys.version_info >= (3,) else u'test\xcf')) + (b'test' if sys.version_info >= (3,) else 'test'), 'f2b-test::non-existing-encoding') + uni_decode((b'test\xcf' if sys.version_info >= (3,) else 'test\xcf')) uni_string(b'test\xcf') uni_string('test\xcf') if sys.version_info < (3,) and 'PyPy' not in sys.version: - uni_string(u'test\xcf') + uni_string('test\xcf') def testSafeLogging(self): # logging should be exception-safe, to avoid possible errors (concat, str. conversion, representation failures, etc) @@ -208,7 +208,7 @@ def __repr__(self): if self.err: raise Exception('no represenation for test!') else: - return u'conv-error (\xf2\xf0\xe5\xf2\xe8\xe9), unterminated utf \xcf' + return 'conv-error (\xf2\xf0\xe5\xf2\xe8\xe9), unterminated utf \xcf' test = Test() logSys.log(logging.NOTICE, "test 1a: %r", test) self.assertLogged("Traceback", "no represenation for test!") @@ -256,7 +256,7 @@ def deep_function(i): func_raise() try: - print deep_function(3) + print(deep_function(3)) except ValueError: s = tb() @@ -273,7 +273,7 @@ def deep_function(i): self.assertIn(':', s) def _testAssertionErrorRE(self, regexp, fun, *args, **kwargs): - self.assertRaisesRegexp(AssertionError, regexp, fun, *args, **kwargs) + self.assertRaisesRegex(AssertionError, regexp, fun, *args, **kwargs) def testExtendedAssertRaisesRE(self): ## test _testAssertionErrorRE several fail cases: @@ -311,13 +311,13 @@ def testExtendedAssertMethods(self): self._testAssertionErrorRE(r"'a' unexpectedly found in 'cba'", self.assertNotIn, 'a', 'cba') self._testAssertionErrorRE(r"1 unexpectedly found in \[0, 1, 2\]", - self.assertNotIn, 1, xrange(3)) + self.assertNotIn, 1, range(3)) self._testAssertionErrorRE(r"'A' unexpectedly found in \['C', 'A'\]", self.assertNotIn, 'A', (c.upper() for c in 'cba' if c != 'b')) self._testAssertionErrorRE(r"'a' was not found in 'xyz'", self.assertIn, 'a', 'xyz') self._testAssertionErrorRE(r"5 was not found in \[0, 1, 2\]", - self.assertIn, 5, xrange(3)) + self.assertIn, 5, range(3)) self._testAssertionErrorRE(r"'A' was not found in \['C', 'B'\]", self.assertIn, 'A', (c.upper() for c in 'cba' if c != 'a')) ## assertLogged, assertNotLogged positive case: diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index 9b44c6dded..ace1184c68 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -68,7 +68,7 @@ def testDefault(self, multipliers = None): a.setBanTimeExtra('multipliers', multipliers) # test algorithm and max time 24 hours : self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], + [a.calcBanTime(600, i) for i in range(1, 11)], [1200, 2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400] ) # with extra large max time (30 days): @@ -80,38 +80,38 @@ def testDefault(self, multipliers = None): if multcnt < 11: arr = arr[0:multcnt-1] + ([arr[multcnt-2]] * (11-multcnt)) self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], + [a.calcBanTime(600, i) for i in range(1, 11)], arr ) a.setBanTimeExtra('maxtime', '1d') # change factor : a.setBanTimeExtra('factor', '2'); self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], + [a.calcBanTime(600, i) for i in range(1, 11)], [2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400, 86400] ) # factor is float : a.setBanTimeExtra('factor', '1.33'); self.assertEqual( - [int(a.calcBanTime(600, i)) for i in xrange(1, 11)], + [int(a.calcBanTime(600, i)) for i in range(1, 11)], [1596, 3192, 6384, 12768, 25536, 51072, 86400, 86400, 86400, 86400] ) a.setBanTimeExtra('factor', None); # change max time : a.setBanTimeExtra('maxtime', '12h') self.assertEqual( - [a.calcBanTime(600, i) for i in xrange(1, 11)], + [a.calcBanTime(600, i) for i in range(1, 11)], [1200, 2400, 4800, 9600, 19200, 38400, 43200, 43200, 43200, 43200] ) a.setBanTimeExtra('maxtime', '24h') ## test randomization - not possibe all 10 times we have random = 0: a.setBanTimeExtra('rndtime', '5m') self.assertTrue( - False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] + False in [1200 in [a.calcBanTime(600, 1) for i in range(10)] for c in range(10)] ) a.setBanTimeExtra('rndtime', None) self.assertFalse( - False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] + False in [1200 in [a.calcBanTime(600, 1) for i in range(10)] for c in range(10)] ) # restore default: a.setBanTimeExtra('multipliers', None) @@ -123,7 +123,7 @@ def testMultipliers(self): # this multipliers has the same values as default formula, we test stop growing after count 9: self.testDefault('1 2 4 8 16 32 64 128 256') # this multipliers has exactly the same values as default formula, test endless growing (stops by count 31 only): - self.testDefault(' '.join([str(1<= 0: - line1 = f.next() + line1 = next(f) self.assertTrue(line1.endswith("Before file moved\n")) - line2 = f.next() + line2 = next(f) self.assertTrue(line2.endswith("After file moved\n")) try: - n = f.next() + n = next(f) if n.find("Command: ['flushlogs']") >=0: - self.assertRaises(StopIteration, f.next) + self.assertRaises(StopIteration, f.__next__) else: self.fail("Exception StopIteration or Command: ['flushlogs'] expected. Got: %s" % n) except StopIteration: pass # on higher debugging levels this is expected with open(fn,'r') as f: - line1 = f.next() + line1 = next(f) if line1.find('rollover performed on') >= 0: - line1 = f.next() + line1 = next(f) self.assertTrue(line1.endswith("After flushlogs\n")) - self.assertRaises(StopIteration, f.next) + self.assertRaises(StopIteration, f.__next__) f.close() finally: os.remove(fn2) @@ -1185,7 +1185,7 @@ def testStartFailedSockExists(self): os.remove(f) -from clientreadertestcase import ActionReader, JailsReader, CONFIG_DIR +from .clientreadertestcase import ActionReader, JailsReader, CONFIG_DIR class ServerConfigReaderTests(LogCaptureTestCase): diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index e3a07998d6..15253ce097 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -153,7 +153,7 @@ def testSocketConnectBroken(self): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: self.close() - self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", + self.assertRaisesRegex(Exception, r"reset by peer|Broken pipe", lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10))) finally: RequestHandler.found_terminator = org_handler diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 8bcc1431d4..98ec581146 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -35,7 +35,7 @@ import threading import unittest -from cStringIO import StringIO +from io import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode @@ -171,8 +171,8 @@ def initProcess(opts): # Let know the version if opts.verbosity != 0: - print("Fail2ban %s test suite. Python %s. Please wait..." \ - % (version, str(sys.version).replace('\n', ''))) + print(("Fail2ban %s test suite. Python %s. Please wait..." \ + % (version, str(sys.version).replace('\n', '')))) return opts; @@ -303,7 +303,7 @@ def F2B_SkipIfNoNetwork(): c.clear = lambda: logSys.warn('clear CACHE_ipToName is disabled in test suite') # increase max count and max time (too many entries, long time testing): c.setOptions(maxCount=10000, maxTime=5*60) - for i in xrange(256): + for i in range(256): c.set('192.0.2.%s' % i, None) c.set('198.51.100.%s' % i, None) c.set('203.0.113.%s' % i, None) @@ -531,8 +531,8 @@ def addTest(self, suite): import difflib, pprint if not hasattr(unittest.TestCase, 'assertDictEqual'): def assertDictEqual(self, d1, d2, msg=None): - self.assert_(isinstance(d1, dict), 'First argument is not a dictionary') - self.assert_(isinstance(d2, dict), 'Second argument is not a dictionary') + self.assertTrue(isinstance(d1, dict), 'First argument is not a dictionary') + self.assertTrue(isinstance(d2, dict), 'Second argument is not a dictionary') if d1 != d2: standardMsg = '%r != %r' % (d1, d2) diff = ('\n' + '\n'.join(difflib.ndiff( @@ -550,7 +550,7 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None) # used to recognize having element as nested dict, list or tuple: def _is_nested(v): if isinstance(v, dict): - return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) + return any(isinstance(v, (dict, list, tuple)) for v in v.values()) return any(isinstance(v, (dict, list, tuple)) for v in v) if nestedOnly: _nest_sorted = sorted @@ -570,7 +570,7 @@ def _assertSortedEqual(a, b, level, nestedOnly, key): return raise ValueError('%r != %r' % (a, b)) if isinstance(a, dict) and isinstance(b, dict): # compare dict's: - for k, v1 in a.iteritems(): + for k, v1 in a.items(): v2 = b[k] if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key) @@ -605,14 +605,14 @@ def assertRaisesRegexp(self, exccls, regexp, fun, *args, **kwargs): self.fail('\"%s\" does not match \"%s\"' % (regexp, e)) else: self.fail('%s not raised' % getattr(exccls, '__name__')) - unittest.TestCase.assertRaisesRegexp = assertRaisesRegexp + unittest.TestCase.assertRaisesRegex = assertRaisesRegexp # always custom following methods, because we use atm better version of both (support generators) if True: ## if not hasattr(unittest.TestCase, 'assertIn'): def assertIn(self, a, b, msg=None): bb = b wrap = False - if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring): + if msg is None and hasattr(b, '__iter__') and not isinstance(b, str): b, bb = itertools.tee(b) wrap = True if a not in b: @@ -623,7 +623,7 @@ def assertIn(self, a, b, msg=None): def assertNotIn(self, a, b, msg=None): bb = b wrap = False - if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring): + if msg is None and hasattr(b, '__iter__') and not isinstance(b, str): b, bb = itertools.tee(b) wrap = True if a in b: From 25f7a8a1f094477814ff33d18b39e5ec65714e07 Mon Sep 17 00:00:00 2001 From: Robert Belanec Date: Thu, 23 Feb 2023 11:24:58 +0100 Subject: [PATCH 23/28] Revert "added logs for geoiplookup" This reverts commit 8b609ff63924d32cc4e365e41ecc1324d98ef5f2. --- fail2ban/server/filter.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index cc1434a961..5f94619af9 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -616,17 +616,14 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): geoip2cf = "/usr/share/GeoIP/GeoIP2-Country.mmdb" if self.__ignoreGeoSet: if shutil.which("mmdblookup") and os.path.isfile(geoip2cf): - geoipcc = str(subprocess.check_output(["mmdblookup","--file",geoip2cf,"--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") + geoipcc = str(subprocess.check_output(["mmdblookup","--file","/usr/share/GeoIP/GeoIP2-Country.mmdb","--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") elif shutil.which("geoiplookup"): - if shutil.which("mmdblookup") and !os.path.isfile(geoip2cf): - self._logWarnOnce("_next_geocfByTimeWarn", ("Found mmdblookup but cannot find mmdb country file at /usr/share/GeoIP/GeoIP2-Country.mmdb, using geoiplookup instead.")) - geoipcc = str(subprocess.check_output(["geoiplookup",str(ip)])).split(" ")[3].replace(",","") if geoipcc in self.__ignoreGeoSet: self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) return True else: - self._logWarnOnce("_next_geoByTimeWarn", ("Cannot find geoiplookup or correctly set mmdblookup. Geolocation is unavailable. If you have mmdblookup installed, please check if the mmdb country file is located at /usr/share/GeoIP/GeoIP2-Country.mmdb. If not run geoipupdate.")) + self._logWarnOnce("_next_geoByTimeWarn", ("Cannot find geoiplookup. Geolocation is unavailable.")) # check if the IP is covered by ignore IP (in set or in subnet/dns): if ip in self.__ignoreIpSet: From 56dec6fadd6e9494d1ef35c97a2c4fb399628ff3 Mon Sep 17 00:00:00 2001 From: Robert Belanec Date: Thu, 23 Feb 2023 11:25:21 +0100 Subject: [PATCH 24/28] Revert "fixed typo" This reverts commit 87e21b3d23c835f7b7a77f8a13e74f97d0f33453. --- fail2ban/client/actionreader.py | 4 +- fail2ban/client/configparserinc.py | 12 +-- fail2ban/client/configreader.py | 4 +- fail2ban/client/csocket.py | 4 +- fail2ban/client/fail2banclient.py | 4 +- fail2ban/client/fail2banregex.py | 22 ++--- fail2ban/client/filterreader.py | 2 +- fail2ban/client/jailreader.py | 4 +- fail2ban/helpers.py | 15 ++- fail2ban/server/action.py | 19 ++-- fail2ban/server/actions.py | 28 +++--- fail2ban/server/asyncserver.py | 4 +- fail2ban/server/banmanager.py | 14 +-- fail2ban/server/database.py | 6 +- fail2ban/server/failmanager.py | 8 +- fail2ban/server/failregex.py | 8 +- fail2ban/server/filterpoll.py | 2 +- fail2ban/server/filterpyinotify.py | 4 +- fail2ban/server/filtersystemd.py | 2 +- fail2ban/server/ipdns.py | 18 ++-- fail2ban/server/jail.py | 14 +-- fail2ban/server/mytime.py | 2 +- fail2ban/server/server.py | 20 ++-- fail2ban/server/strptime.py | 8 +- fail2ban/server/ticket.py | 14 +-- fail2ban/server/transmitter.py | 2 +- fail2ban/server/utils.py | 4 +- fail2ban/tests/actiontestcase.py | 4 +- fail2ban/tests/banmanagertestcase.py | 2 +- fail2ban/tests/clientreadertestcase.py | 4 +- fail2ban/tests/databasetestcase.py | 16 +-- fail2ban/tests/datedetectortestcase.py | 6 +- fail2ban/tests/fail2banclienttestcase.py | 8 +- fail2ban/tests/fail2banregextestcase.py | 4 +- fail2ban/tests/failmanagertestcase.py | 10 +- .../tests/files/config/apache-auth/digest.py | 20 ++-- fail2ban/tests/filtertestcase.py | 98 +++++++++---------- fail2ban/tests/misctestcase.py | 22 ++--- fail2ban/tests/observertestcase.py | 34 +++---- fail2ban/tests/samplestestcase.py | 8 +- fail2ban/tests/servertestcase.py | 28 +++--- fail2ban/tests/sockettestcase.py | 2 +- fail2ban/tests/utils.py | 22 ++--- 43 files changed, 269 insertions(+), 267 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 5a57338ba2..88b0aca140 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -89,11 +89,11 @@ def convert(self): stream = list() stream.append(head + ["addaction", self._name]) multi = [] - for opt, optval in opts.items(): + for opt, optval in opts.iteritems(): if opt in self._configOpts and not opt.startswith('known/'): multi.append([opt, optval]) if self._initOpts: - for opt, optval in self._initOpts.items(): + for opt, optval in self._initOpts.iteritems(): if opt not in self._configOpts and not opt.startswith('known/'): multi.append([opt, optval]) if len(multi) > 1: diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index e46379a179..cc4ada0aed 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -62,7 +62,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, parser, option, accum, rest, section, map, *args, **kwargs) else: # pragma: 3.x no cover - from configparser import SafeConfigParser, \ + from ConfigParser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError # Interpolate missing known/option as option from default section @@ -327,7 +327,7 @@ def options(self, section, withDefault=True): # mix it with defaults: return set(opts.keys()) | set(self._defaults) # only own option names: - return list(opts.keys()) + return opts.keys() def read(self, filenames, get_includes=True): if not isinstance(filenames, list): @@ -356,7 +356,7 @@ def read(self, filenames, get_includes=True): ret += i # merge defaults and all sections to self: alld.update(cfg.get_defaults()) - for n, s in cfg.get_sections().items(): + for n, s in cfg.get_sections().iteritems(): # conditional sections cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n) if cond: @@ -366,14 +366,14 @@ def read(self, filenames, get_includes=True): del(s['__name__']) except KeyError: pass - for k in list(s.keys()): + for k in s.keys(): v = s.pop(k) s[k + cond] = v s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: self.merge_section('KNOWN/'+n, - dict([i for i in iter(s2.items()) if i[0] in s]), '') + dict(filter(lambda i: i[0] in s, s2.iteritems())), '') # merge section s2.update(s) else: @@ -400,7 +400,7 @@ def merge_section(self, section, options, pref=None): sec.update(options) return sk = {} - for k, v in options.items(): + for k, v in options.iteritems(): if not k.startswith(pref) and k != '__name__': sk[pref+k] = v sec.update(sk) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 8310ade139..c7f965ce52 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -26,7 +26,7 @@ import glob import os -from configparser import NoOptionError, NoSectionError +from ConfigParser import NoOptionError, NoSectionError from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags @@ -221,7 +221,7 @@ def read(self, filename): config_files += sorted(glob.glob('%s/*.local' % config_dir)) # choose only existing ones - config_files = list(filter(os.path.exists, config_files)) + config_files = filter(os.path.exists, config_files) if len(config_files): # at least one config exists and accessible diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index ed2d251682..88795674bd 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -47,7 +47,7 @@ def __del__(self): def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string - obj = dumps(list(map(CSocket.convert, msg)), HIGHEST_PROTOCOL) + obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) self.__csock.send(obj) self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) @@ -72,7 +72,7 @@ def close(self): @staticmethod def convert(m): """Convert every "unexpected" member of message to string""" - if isinstance(m, (str, bool, int, float, list, dict, set)): + if isinstance(m, (basestring, bool, int, float, list, dict, set)): return m else: # pragma: no cover return str(m) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index e8fa410d05..f3b0f7b271 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -45,7 +45,7 @@ def _thread_name(): return threading.current_thread().__class__.__name__ def input_command(): # pragma: no cover - return input(PROMPT) + return raw_input(PROMPT) ## # @@ -456,7 +456,7 @@ def start(self, argv): return False finally: self._alive = False - for s, sh in _prev_signals.items(): + for s, sh in _prev_signals.iteritems(): signal.signal(s, sh) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index aca262f082..b1795588c9 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -40,10 +40,10 @@ import shlex import sys import time -import urllib.request, urllib.parse, urllib.error +import urllib from optparse import OptionParser, Option -from configparser import NoOptionError, NoSectionError, MissingSectionHeaderError +from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError try: # pragma: no cover from ..server.filtersystemd import FilterSystemd @@ -67,7 +67,7 @@ def debuggexURL(sample, regex, multiline=False, useDns="yes"): 'flavor': 'python' } if multiline: args['flags'] = 'm' - return 'https://www.debuggex.com/?' + urllib.parse.urlencode(args) + return 'https://www.debuggex.com/?' + urllib.urlencode(args) def output(args): # pragma: no cover (overriden in test-cases) print(args) @@ -246,7 +246,7 @@ class Fail2banRegex(object): def __init__(self, opts): # set local protected members from given options: - self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.items())) + self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.iteritems())) self._opts = opts self._maxlines_set = False # so we allow to override maxlines in cmdline self._datepattern_set = False @@ -313,7 +313,7 @@ def _dumpRealOptions(self, reader, fltOpt): realopts = {} combopts = reader.getCombined() # output all options that are specified in filter-argument as well as some special (mostly interested): - for k in ['logtype', 'datepattern'] + list(fltOpt.keys()): + for k in ['logtype', 'datepattern'] + fltOpt.keys(): # combined options win, but they contain only a sub-set in filter expected keys, # so get the rest from definition section: try: @@ -440,7 +440,7 @@ def readRegex(self, value, regextype): self.output( "Use %11s line : %s" % (regex, shortstr(value)) ) regex_values = {regextype: [RegexStat(value)]} - for regextype, regex_values in regex_values.items(): + for regextype, regex_values in regex_values.iteritems(): regex = regextype + 'regex' setattr(self, "_" + regex, regex_values) for regex in regex_values: @@ -532,13 +532,13 @@ def _out(ret): def _out(ret): for r in ret: for r in r[3].get('matches'): - if not isinstance(r, str): + if not isinstance(r, basestring): r = ''.join(r for r in r) output(r) elif ofmt == 'row': def _out(ret): for r in ret: - output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].items() if k != 'matches'))) + output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches'))) elif '<' not in ofmt: def _out(ret): for r in ret: @@ -573,7 +573,7 @@ def _get_msg(self): # wrap multiline tag (msg) interpolations to single line: for r, v in rows: for r in r[3].get('matches'): - if not isinstance(r, str): + if not isinstance(r, basestring): r = ''.join(r for r in r) r = v.replace("\x00msg\x00", r) output(r) @@ -639,9 +639,9 @@ def printLines(self, ltype): ans = [[]] for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] - b = [a[0] + ' | ' + a[1].getFailRegex() + ' | ' + + b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex(), - multiline, self._opts.usedns) for a in ans] + multiline, self._opts.usedns), ans) pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 0b6c017259..24341014db 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -71,7 +71,7 @@ def convert(self): @staticmethod def _fillStream(stream, opts, jailName): prio0idx = 0 - for opt, value in opts.items(): + for opt, value in opts.iteritems(): # Do not send a command if the value is not set (empty). if value is None: continue if opt in ("failregex", "ignoreregex"): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 51ad21c99e..9706c53054 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -118,7 +118,7 @@ def _glob(path): } _configOpts.update(FilterReader._configOpts) - _ignoreOpts = set(['action', 'filter', 'enabled'] + list(FilterReader._configOpts.keys())) + _ignoreOpts = set(['action', 'filter', 'enabled'] + FilterReader._configOpts.keys()) def getOptions(self): @@ -241,7 +241,7 @@ def convert(self, allow_no_files=False): stream.extend(self.__filter.convert()) # and using options from jail: FilterReader._fillStream(stream, self.__opts, self.__name) - for opt, value in self.__opts.items(): + for opt, value in self.__opts.iteritems(): if opt == "logpath": if self.__opts.get('backend', '').startswith("systemd"): continue found_files = 0 diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index dee445fca7..5c1750a622 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -31,7 +31,6 @@ from threading import Lock from .server.mytime import MyTime -import importlib try: import ctypes @@ -64,7 +63,7 @@ def __resetDefaultEncoding(encoding): from imp import load_dynamic as __ldm _sys = __ldm('_sys', 'sys') except ImportError: # pragma: no cover - only if load_dynamic fails - importlib.reload(sys) + reload(sys) _sys = sys if hasattr(_sys, "setdefaultencoding"): _sys.setdefaultencoding(encoding) @@ -102,7 +101,7 @@ def uni_string(x): else: # pragma: 3.x no cover def uni_decode(x, enc=PREFER_ENC, errors='strict'): try: - if isinstance(x, str): + if isinstance(x, unicode): return x.encode(enc, errors) return x except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable @@ -111,7 +110,7 @@ def uni_decode(x, enc=PREFER_ENC, errors='strict'): return x.encode(enc, 'replace') if sys.getdefaultencoding().upper() != 'UTF-8': # pragma: no cover - utf-8 is default encoding now def uni_string(x): - if not isinstance(x, str): + if not isinstance(x, unicode): return str(x) return x.encode(PREFER_ENC, 'replace') else: @@ -119,7 +118,7 @@ def uni_string(x): def _as_bool(val): - return bool(val) if not isinstance(val, str) \ + return bool(val) if not isinstance(val, basestring) \ else val.lower() in ('1', 'on', 'true', 'yes') @@ -328,7 +327,7 @@ def splitwords(s): """ if not s: return [] - return list(filter(bool, [v.strip() for v in re.split('[ ,\n]+', s)])) + return filter(bool, map(lambda v: v.strip(), re.split('[ ,\n]+', s))) if sys.version_info >= (3,5): eval(compile(r'''if 1: @@ -445,7 +444,7 @@ def substituteRecursiveTags(inptags, conditional='', while True: repFlag = False # substitute each value: - for tag in tags.keys(): + for tag in tags.iterkeys(): # ignore escaped or already done (or in ignore list): if tag in ignore or tag in done: continue # ignore replacing callable items from calling map - should be converted on demand only (by get): @@ -485,7 +484,7 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue # if calling map - be sure we've string: - if not isinstance(repl, str): repl = uni_string(repl) + if not isinstance(repl, basestring): repl = uni_string(repl) value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 760a73d787..16ff66212a 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -114,9 +114,9 @@ def _asrepr(self, calculated=False): def _asdict(self, calculated=False, checker=None): d = dict(self.data, **self.storage) if not calculated: - return dict((n,v) for n,v in d.items() \ + return dict((n,v) for n,v in d.iteritems() \ if not callable(v) or n in self.CM_REPR_ITEMS) - for n,v in list(d.items()): + for n,v in d.items(): if callable(v): try: # calculate: @@ -182,7 +182,7 @@ def copy(self): return self.__class__(_merge_copy_dicts(self.data, self.storage)) -class ActionBase(object, metaclass=ABCMeta): +class ActionBase(object): """An abstract base class for actions in Fail2Ban. Action Base is a base definition of what methods need to be in @@ -212,6 +212,7 @@ class ActionBase(object, metaclass=ABCMeta): Any additional arguments specified in `jail.conf` or passed via `fail2ban-client` will be passed as keyword arguments. """ + __metaclass__ = ABCMeta @classmethod def __subclasshook__(cls, C): @@ -422,7 +423,7 @@ def _operationExecuted(self, tag, family, *args): if not callable(family): # pragma: no cover return self.__substCache.get(key, {}).get(family) # family as expression - use it to filter values: - return [v for f, v in self.__substCache.get(key, {}).items() if family(f)] + return [v for f, v in self.__substCache.get(key, {}).iteritems() if family(f)] cmd = args[0] if cmd: # set: try: @@ -434,7 +435,7 @@ def _operationExecuted(self, tag, family, *args): try: famd = self.__substCache[key] cmd = famd.pop(family) - for family, v in list(famd.items()): + for family, v in famd.items(): if v == cmd: del famd[family] except KeyError: # pragma: no cover @@ -450,7 +451,7 @@ def _executeOperation(self, tag, operation, family=[], afterExec=None): res = True err = 'Script error' if not family: # all started: - family = [famoper for (famoper,v) in self.__started.items() if v] + family = [famoper for (famoper,v) in self.__started.iteritems() if v] for famoper in family: try: cmd = self._getOperation(tag, famoper) @@ -630,7 +631,7 @@ def flush(self): and executes the resulting command. """ # collect started families, may be started on demand (conditional): - family = [f for (f,v) in self.__started.items() if v & 3 == 3]; # started and contains items + family = [f for (f,v) in self.__started.iteritems() if v & 3 == 3]; # started and contains items # if nothing contains items: if not family: return True # flush: @@ -655,7 +656,7 @@ def _stop(self, family=None): """ # collect started families, if started on demand (conditional): if family is None: - family = [f for (f,v) in self.__started.items() if v] + family = [f for (f,v) in self.__started.iteritems() if v] # if no started (on demand) actions: if not family: return True self.__started = {} @@ -689,7 +690,7 @@ def consistencyCheck(self, beforeRepair=None): ret = True # for each started family: if self.actioncheck: - for (family, started) in list(self.__started.items()): + for (family, started) in self.__started.items(): if started and not self._invariantCheck(family, beforeRepair): # reset started flag and command of executed operation: self.__started[family] = 0 diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index b052d342a9..fa045ab5ee 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -156,11 +156,11 @@ def reload(self, begin=True): else: if hasattr(self, '_reload_actions'): # reload actions after all parameters set via stream: - for name, initOpts in self._reload_actions.items(): + for name, initOpts in self._reload_actions.iteritems(): if name in self._actions: self._actions[name].reload(**(initOpts if initOpts else {})) # remove obsolete actions (untouched by reload process): - delacts = OrderedDict((name, action) for name, action in self._actions.items() + delacts = OrderedDict((name, action) for name, action in self._actions.iteritems() if name not in self._reload_actions) if len(delacts): # unban all tickets using removed actions only: @@ -217,7 +217,7 @@ def getBanned(self, ids): return lst if len(ids) == 1: return 1 if ids[0] in lst else 0 - return [1 if ip in lst else 0 for ip in ids] + return map(lambda ip: 1 if ip in lst else 0, ids) def getBanList(self, withTime=False): """Returns the list of banned IP addresses. @@ -288,7 +288,7 @@ def removeBannedIP(self, ip=None, db=True, ifexists=False): if not isinstance(ip, IPAddr): ipa = IPAddr(ip) if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): - ips = list(filter(ipa.contains, self.banManager.getBanList())) + ips = filter(ipa.contains, self.banManager.getBanList()) if ips: return self.removeBannedIP(ips, db, ifexists) # not found: @@ -305,7 +305,7 @@ def stopActions(self, actions=None): """ if actions is None: actions = self._actions - for name, action in reversed(list(actions.items())): + for name, action in reversed(actions.items()): try: action.stop() except Exception as e: @@ -328,7 +328,7 @@ def run(self): True when the thread exits nicely. """ cnt = 0 - for name, action in self._actions.items(): + for name, action in self._actions.iteritems(): try: action.start() except Exception as e: @@ -505,7 +505,7 @@ def __checkBan(self, tickets=None): Observers.Main.add('banFound', bTicket, self._jail, btime) logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip) # do actions : - for name, action in self._actions.items(): + for name, action in self._actions.iteritems(): try: if bTicket.restored and getattr(action, 'norestored', False): continue @@ -543,13 +543,13 @@ def __checkBan(self, tickets=None): # avoid too often checks: if not rebanacts and MyTime.time() > self.__lastConsistencyCheckTM + 3: self.__lastConsistencyCheckTM = MyTime.time() - for action in self._actions.values(): + for action in self._actions.itervalues(): if hasattr(action, 'consistencyCheck'): action.consistencyCheck() # check epoch in order to reban it: if bTicket.banEpoch < self.banEpoch: if not rebanacts: rebanacts = dict( - (name, action) for name, action in self._actions.items() + (name, action) for name, action in self._actions.iteritems() if action.banEpoch > bTicket.banEpoch) cnt += self.__reBan(bTicket, actions=rebanacts) else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions: @@ -576,8 +576,8 @@ def __reBan(self, ticket, actions=None, log=True): ip = ticket.getID() aInfo = self._getActionInfo(ticket) if log: - logSys.notice("[%s] Reban %s%s", self._jail.name, ip, (', action %r' % list(actions.keys())[0] if len(actions) == 1 else '')) - for name, action in actions.items(): + logSys.notice("[%s] Reban %s%s", self._jail.name, ip, (', action %r' % actions.keys()[0] if len(actions) == 1 else '')) + for name, action in actions.iteritems(): try: logSys.debug("[%s] action %r: reban %s", self._jail.name, name, ip) if not aInfo.immutable: aInfo.reset() @@ -601,7 +601,7 @@ def _prolongBan(self, ticket): if not self.banManager._inBanList(ticket): return # do actions : aInfo = None - for name, action in self._actions.items(): + for name, action in self._actions.iteritems(): try: if ticket.restored and getattr(action, 'norestored', False): continue @@ -650,7 +650,7 @@ def __flushBan(self, db=False, actions=None, stop=False): cnt = 0 # first we'll execute flush for actions supporting this operation: unbactions = {} - for name, action in (actions if actions is not None else self._actions).items(): + for name, action in (actions if actions is not None else self._actions).iteritems(): try: if hasattr(action, 'flush') and (not isinstance(action, CommandAction) or action.actionflush): logSys.notice("[%s] Flush ticket(s) with %s", self._jail.name, name) @@ -705,7 +705,7 @@ def __unBan(self, ticket, actions=None, log=True): aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Unban %s", self._jail.name, ip) - for name, action in unbactions.items(): + for name, action in unbactions.iteritems(): try: logSys.debug("[%s] action %r: unban %s", self._jail.name, name, ip) if not aInfo.immutable: aInfo.reset() diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index f5f9740b2d..e34007374e 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -178,7 +178,7 @@ def loop(active, timeout=None, use_poll=False, err_count=None): elif err_count['listen'] > 100: # pragma: no cover - normally unreachable if ( e.args[0] == errno.EMFILE # [Errno 24] Too many open files - or sum(err_count.values()) > 1000 + or sum(err_count.itervalues()) > 1000 ): logSys.critical("Too many errors - critical count reached %r", err_count) break @@ -220,7 +220,7 @@ def handle_accept(self): elif self.__errCount['accept'] > 100: if ( (isinstance(e, socket.error) and e.args[0] == errno.EMFILE) # [Errno 24] Too many open files - or sum(self.__errCount.values()) > 1000 + or sum(self.__errCount.itervalues()) > 1000 ): logSys.critical("Too many errors - critical count reached %r", self.__errCount) self.stop() diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index d3e89820d3..9168d5b89b 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -103,7 +103,7 @@ def getBanList(self, ordered=False, withTime=False): return list(self.__banList.keys()) with self.__lock: lst = [] - for ticket in self.__banList.values(): + for ticket in self.__banList.itervalues(): eob = ticket.getEndOfBanTime(self.__banTime) lst.append((ticket,eob)) lst.sort(key=lambda t: t[1]) @@ -161,7 +161,7 @@ def getBanListExtendedCymruInfo(self, timeout=10): return return_dict # get ips in lock: with self.__lock: - banIPs = [banData.getIP() for banData in list(self.__banList.values())] + banIPs = [banData.getIP() for banData in self.__banList.values()] # get cymru info: try: for ip in banIPs: @@ -333,7 +333,7 @@ def unBanList(self, time, maxCount=0x7fffffff): # Gets the list of ticket to remove (thereby correct next unban time). unBanList = {} nextUnbanTime = BanTicket.MAX_TIME - for fid,ticket in self.__banList.items(): + for fid,ticket in self.__banList.iteritems(): # current time greater as end of ban - timed out: eob = ticket.getEndOfBanTime(self.__banTime) if time > eob: @@ -349,15 +349,15 @@ def unBanList(self, time, maxCount=0x7fffffff): if len(unBanList): if len(unBanList) / 2.0 <= len(self.__banList) / 3.0: # few as 2/3 should be removed - remove particular items: - for fid in unBanList.keys(): + for fid in unBanList.iterkeys(): del self.__banList[fid] else: # create new dictionary without items to be deleted: - self.__banList = dict((fid,ticket) for fid,ticket in self.__banList.items() \ + self.__banList = dict((fid,ticket) for fid,ticket in self.__banList.iteritems() \ if fid not in unBanList) # return list of tickets: - return list(unBanList.values()) + return unBanList.values() ## # Flush the ban list. @@ -367,7 +367,7 @@ def unBanList(self, time, maxCount=0x7fffffff): def flushBanList(self): with self.__lock: - uBList = list(self.__banList.values()) + uBList = self.__banList.values() self.__banList = dict() return uBList diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 59e376b711..877cbb93b7 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -67,13 +67,13 @@ def _json_loads_safe(x): else: # pragma: 3.x no cover def _normalize(x): if isinstance(x, dict): - return dict((_normalize(k), _normalize(v)) for k, v in x.items()) + return dict((_normalize(k), _normalize(v)) for k, v in x.iteritems()) elif isinstance(x, (list, set)): return [_normalize(element) for element in x] - elif isinstance(x, str): + elif isinstance(x, unicode): # in 2.x default text_factory is unicode - so return proper unicode here: return x.encode(PREFER_ENC, 'replace').decode(PREFER_ENC) - elif isinstance(x, str): + elif isinstance(x, basestring): return x.decode(PREFER_ENC, 'replace') return x diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 2effc45010..3c71d51a47 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -55,7 +55,7 @@ def getFailTotal(self): def getFailCount(self): # may be slow on large list of failures, should be used for test purposes only... with self.__lock: - return len(self.__failList), sum([f.getRetry() for f in list(self.__failList.values())]) + return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) def setMaxRetry(self, value): self.__maxRetry = value @@ -116,7 +116,7 @@ def addFailure(self, ticket, count=1, observed=False): # in case of having many active failures, it should be ran only # if debug level is "low" enough failures_summary = ', '.join(['%s:%d' % (k, v.getRetry()) - for k,v in self.__failList.items()]) + for k,v in self.__failList.iteritems()]) logSys.log(logLevel, "Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s" % (self.__failTotal, len(self.__failList), failures_summary)) @@ -129,7 +129,7 @@ def size(self): def cleanup(self, time): time -= self.__maxTime with self.__lock: - todelete = [fid for fid,item in self.__failList.items() \ + todelete = [fid for fid,item in self.__failList.iteritems() \ if item.getTime() <= time] if len(todelete) == len(self.__failList): # remove all: @@ -143,7 +143,7 @@ def cleanup(self, time): del self.__failList[fid] else: # create new dictionary without items to be deleted: - self.__failList = dict((fid,item) for fid,item in self.__failList.items() \ + self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \ if item.getTime() > time) self.__bgSvc.service() diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 89c9438da9..a9b144af55 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -143,7 +143,9 @@ def __init__(self, regex, multiline=False, **kwargs): self._regex = regex self._altValues = [] self._tupleValues = [] - for k in [k for k in self._regexObj.groupindex if len(k) > len(COMPLNAME_PRE[0])]: + for k in filter( + lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex + ): n = COMPLNAME_CRE.match(k) if n: g, n = n.group(1), mapTag2Opt(n.group(2)) @@ -233,7 +235,7 @@ def getRegex(self): # @staticmethod def _tupleLinesBuf(tupleLines): - return "\n".join(["".join(v[::2]) for v in tupleLines]) + "\n" + return "\n".join(map(lambda v: "".join(v[::2]), tupleLines)) + "\n" ## # Searches the regular expression. @@ -245,7 +247,7 @@ def _tupleLinesBuf(tupleLines): def search(self, tupleLines, orgLines=None): buf = tupleLines - if not isinstance(tupleLines, str): + if not isinstance(tupleLines, basestring): buf = Regex._tupleLinesBuf(tupleLines) self._matchCache = self._regexObj.search(buf) if self._matchCache: diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 8a289cea03..196955e59f 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -173,4 +173,4 @@ def isModified(self, filename): return False def getPendingPaths(self): - return list(self.__file404Cnt.keys()) + return self.__file404Cnt.keys() diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 2ee49169e1..16b6cfd553 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -155,7 +155,7 @@ def _delPending(self, path): except KeyError: pass def getPendingPaths(self): - return list(self.__pending.keys()) + return self.__pending.keys() def _checkPending(self): if not self.__pending: @@ -181,7 +181,7 @@ def _checkPending(self): self.__pendingChkTime = time.time() self.__pendingMinTime = minTime # process now because we've missed it in monitoring: - for path, isDir in found.items(): + for path, isDir in found.iteritems(): self._delPending(path) # refresh monitoring of this: if isDir is not None: diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 2ab67a9a64..a83b7a132d 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -253,7 +253,7 @@ def formatJournalEntry(self, logentry): return ((logline[:0], date[0] + ' ', logline.replace('\n', '\\n')), date[1]) def seekToTime(self, date): - if isinstance(date, int): + if isinstance(date, (int, long)): date = float(date) self.__journal.seek_realtime(date) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 0f7ccb016c..b435c6dfb3 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -366,7 +366,7 @@ def __wrap_ipstr(ipstr): raise ValueError("invalid ipstr %r, too many plen representation" % (ipstr,)) if "." in s[1] or ":" in s[1]: # 255.255.255.0 resp. ffff:: style mask s[1] = IPAddr.masktoplen(s[1]) - s[1] = int(s[1]) + s[1] = long(s[1]) return s def __init(self, ipstr, cidr=CIDR_UNSPEC): @@ -400,7 +400,7 @@ def __init(self, ipstr, cidr=CIDR_UNSPEC): # mask out host portion if prefix length is supplied if cidr is not None and cidr >= 0: - mask = ~(0xFFFFFFFF >> cidr) + mask = ~(0xFFFFFFFFL >> cidr) self._addr &= mask self._plen = cidr @@ -412,13 +412,13 @@ def __init(self, ipstr, cidr=CIDR_UNSPEC): # mask out host portion if prefix length is supplied if cidr is not None and cidr >= 0: - mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF >> cidr) + mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> cidr) self._addr &= mask self._plen = cidr # if IPv6 address is a IPv4-compatible, make instance a IPv4 elif self.isInNet(IPAddr.IP6_4COMPAT): - self._addr = lo & 0xFFFFFFFF + self._addr = lo & 0xFFFFFFFFL self._family = socket.AF_INET self._plen = 32 else: @@ -428,7 +428,7 @@ def __repr__(self): return repr(self.ntoa) def __str__(self): - return self.ntoa if isinstance(self.ntoa, str) else str(self.ntoa) + return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str @@ -542,7 +542,7 @@ def ntoa(self): elif self.isIPv6: # convert network to host byte order hi = self._addr >> 64 - lo = self._addr & 0xFFFFFFFFFFFFFFFF + lo = self._addr & 0xFFFFFFFFFFFFFFFFL binary = struct.pack("!QQ", hi, lo) if self._plen and self._plen < 128: add = "/%d" % self._plen @@ -600,9 +600,9 @@ def isInNet(self, net): if self.family != net.family: return False if self.isIPv4: - mask = ~(0xFFFFFFFF >> net.plen) + mask = ~(0xFFFFFFFFL >> net.plen) elif self.isIPv6: - mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF >> net.plen) + mask = ~(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFL >> net.plen) else: return False @@ -622,7 +622,7 @@ def __getMaskMap(): m4 = (1 << 32)-1 mmap = {m6: 128, m4: 32, 0: 0} m = 0 - for i in range(0, 128): + for i in xrange(0, 128): m |= 1 << i if i < 32: mmap[m ^ m4] = 32-1-i diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index fe5c605aab..2c84e47538 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -26,7 +26,7 @@ import logging import math import random -import queue +import Queue from .actions import Actions from ..helpers import getLogger, _as_bool, extractOptions, MyTime @@ -76,7 +76,7 @@ def __init__(self, name, backend = "auto", db=None): "might not function correctly. Please shorten" % name) self.__name = name - self.__queue = queue.Queue() + self.__queue = Queue.Queue() self.__filter = None # Extra parameters for increase ban time self._banExtra = {}; @@ -127,25 +127,25 @@ def _setBackend(self, backend): "Failed to initialize any backend for Jail %r" % self.name) def _initPolling(self, **kwargs): - from .filterpoll import FilterPoll + from filterpoll import FilterPoll logSys.info("Jail '%s' uses poller %r" % (self.name, kwargs)) self.__filter = FilterPoll(self, **kwargs) def _initGamin(self, **kwargs): # Try to import gamin - from .filtergamin import FilterGamin + from filtergamin import FilterGamin logSys.info("Jail '%s' uses Gamin %r" % (self.name, kwargs)) self.__filter = FilterGamin(self, **kwargs) def _initPyinotify(self, **kwargs): # Try to import pyinotify - from .filterpyinotify import FilterPyinotify + from filterpyinotify import FilterPyinotify logSys.info("Jail '%s' uses pyinotify %r" % (self.name, kwargs)) self.__filter = FilterPyinotify(self, **kwargs) def _initSystemd(self, **kwargs): # pragma: systemd no cover # Try to import systemd - from .filtersystemd import FilterSystemd + from filtersystemd import FilterSystemd logSys.info("Jail '%s' uses systemd %r" % (self.name, kwargs)) self.__filter = FilterSystemd(self, **kwargs) @@ -219,7 +219,7 @@ def getFailTicket(self): try: ticket = self.__queue.get(False) return ticket - except queue.Empty: + except Queue.Empty: return False def setBanTimeExtra(self, opt, value): diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index ff46b7efeb..315d8a3004 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -165,7 +165,7 @@ def str2seconds(val): @returns number (calculated seconds from expression "val") """ - if isinstance(val, (int, float, complex)): + if isinstance(val, (int, long, float, complex)): return val # replace together standing abbreviations, example '1d12h' -> '1d 12h': val = MyTime._str2sec_prep.sub(r" \1", val) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 9de82fc907..ce201b5e7b 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -209,7 +209,7 @@ def quit(self): # Restore default signal handlers: if _thread_name() == '_MainThread': - for s, sh in self.__prev_signals.items(): + for s, sh in self.__prev_signals.iteritems(): signal.signal(s, sh) # Give observer a small chance to complete its work before exit @@ -287,10 +287,10 @@ def stopAllJail(self): logSys.info("Stopping all jails") with self.__lock: # 1st stop all jails (signal and stop actions/filter thread): - for name in list(self.__jails.keys()): + for name in self.__jails.keys(): self.delJail(name, stop=True, join=False) # 2nd wait for end and delete jails: - for name in list(self.__jails.keys()): + for name in self.__jails.keys(): self.delJail(name, stop=False, join=True) def clearCaches(self): @@ -328,7 +328,7 @@ def reloadJails(self, name, opts, begin): if "--restart" in opts: self.stopAllJail() # first set all affected jail(s) to idle and reset filter regex and other lists/dicts: - for jn, jail in self.__jails.items(): + for jn, jail in self.__jails.iteritems(): if name == '--all' or jn == name: jail.idle = True self.__reload_state[jn] = jail @@ -339,7 +339,7 @@ def reloadJails(self, name, opts, begin): # end reload, all affected (or new) jails have already all new parameters (via stream) and (re)started: with self.__lock: deljails = [] - for jn, jail in self.__jails.items(): + for jn, jail in self.__jails.iteritems(): # still in reload state: if jn in self.__reload_state: # remove jails that are not reloaded (untouched, so not in new configuration) @@ -548,7 +548,7 @@ def setUnbanIP(self, name=None, value=None, ifexists=True): jails = [self.__jails[name]] else: # in all jails: - jails = list(self.__jails.values()) + jails = self.__jails.values() # unban given or all (if value is None): cnt = 0 ifexists |= (name is None) @@ -562,7 +562,7 @@ def banned(self, name=None, ids=None): jails = [self.__jails[name]] else: # in all jails: - jails = list(self.__jails.values()) + jails = self.__jails.values() # check banned ids: res = [] if name is None and ids: @@ -612,7 +612,7 @@ def isStarted(self): def isAlive(self, jailnum=None): if jailnum is not None and len(self.__jails) != jailnum: return 0 - for jail in list(self.__jails.values()): + for jail in self.__jails.values(): if not jail.isAlive(): return 0 return 1 @@ -827,7 +827,7 @@ def setIPv6IsAllowed(value): return DNSUtils.setIPv6IsAllowed(value) def setThreadOptions(self, value): - for o, v in value.items(): + for o, v in value.iteritems(): if o == 'stacksize': threading.stack_size(int(v)*1024) else: # pragma: no cover @@ -951,7 +951,7 @@ def __createDaemon(self): # pragma: no cover maxfd = os.sysconf("SC_OPEN_MAX") except (AttributeError, ValueError): maxfd = 256 # default maximum - fdlist = range(maxfd+1) + fdlist = xrange(maxfd+1) # urandom should not be closed in Python 3.4.0. Fixed in 3.4.1 # http://bugs.python.org/issue21207 diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 1f459f028b..12be163a60 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -99,7 +99,7 @@ def grp(exprset): if len(exprset) > 1 else "".join(exprset) exprset = set( cent(now[0].year + i) for i in (-1, distance) ) if len(now) > 1 and now[1]: - exprset |= set( cent(now[1].year + i) for i in range(-1, now[0].year-now[1].year+1, distance) ) + exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) return grp(sorted(list(exprset))) # more precise year patterns, within same century of last year and @@ -116,7 +116,7 @@ def grp(exprset): _updateTimeRE() def getTimePatternRE(): - keys = list(timeRE.keys()) + keys = timeRE.keys() patt = (r"%%(%%|%s|[%s])" % ( "|".join([k for k in keys if len(k) > 1]), "".join([k for k in keys if len(k) == 1]), @@ -171,7 +171,7 @@ def zone2offset(tz, dt): """ if isinstance(tz, int): return tz - if isinstance(tz, str): + if isinstance(tz, basestring): return validateTimeZone(tz) tz, tzo = tz if tzo is None or tzo == '': # without offset @@ -208,7 +208,7 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): year = month = day = tzoffset = \ weekday = julian = week_of_year = None hour = minute = second = fraction = 0 - for key, val in found_dict.items(): + for key, val in found_dict.iteritems(): if val is None: continue # Directives not explicitly handled below: # c, x, X diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 80b1469c58..96e6777353 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -55,7 +55,7 @@ def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None): self._time = time if time is not None else MyTime.time() self._data = {'matches': matches or [], 'failures': 0} if data is not None: - for k,v in data.items(): + for k,v in data.iteritems(): if v is not None: self._data[k] = v if ticket: @@ -88,7 +88,7 @@ def update(self, ticket): def setID(self, value): # guarantee using IPAddr instead of unicode, str for the IP - if isinstance(value, str): + if isinstance(value, basestring): value = IPAddr(value) self._id = value @@ -180,7 +180,7 @@ def setData(self, *args, **argv): if len(args) == 1: # todo: if support >= 2.7 only: # self._data = {k:v for k,v in args[0].iteritems() if v is not None} - self._data = dict([(k,v) for k,v in args[0].items() if v is not None]) + self._data = dict([(k,v) for k,v in args[0].iteritems() if v is not None]) # add k,v list or dict (merge): elif len(args) == 2: self._data.update((args,)) @@ -191,7 +191,7 @@ def setData(self, *args, **argv): # filter (delete) None values: # todo: if support >= 2.7 only: # self._data = {k:v for k,v in self._data.iteritems() if v is not None} - self._data = dict([(k,v) for k,v in self._data.items() if v is not None]) + self._data = dict([(k,v) for k,v in self._data.iteritems() if v is not None]) def getData(self, key=None, default=None): # return whole data dict: @@ -200,17 +200,17 @@ def getData(self, key=None, default=None): # return default if not exists: if not self._data: return default - if not isinstance(key,(str,type(None),int,float,bool,complex)): + if not isinstance(key,(str,unicode,type(None),int,float,bool,complex)): # return filtered by lambda/function: if callable(key): # todo: if support >= 2.7 only: # return {k:v for k,v in self._data.iteritems() if key(k)} - return dict([(k,v) for k,v in self._data.items() if key(k)]) + return dict([(k,v) for k,v in self._data.iteritems() if key(k)]) # return filtered by keys: if hasattr(key, '__iter__'): # todo: if support >= 2.7 only: # return {k:v for k,v in self._data.iteritems() if k in key} - return dict([(k,v) for k,v in self._data.items() if k in key]) + return dict([(k,v) for k,v in self._data.iteritems() if k in key]) # return single value of data: return self._data.get(key, default) diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 660ed03aed..c25ae637a1 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -500,7 +500,7 @@ def __commandGet(self, command): opt = command[1][len("bantime."):] return self.__server.getBanTimeExtra(name, opt) elif command[1] == "actions": - return list(self.__server.getActions(name).keys()) + return self.__server.getActions(name).keys() elif command[1] == "action": actionname = command[2] actionvalue = command[3] diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 729ee43f20..18073ea7d5 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -53,7 +53,7 @@ # Dictionary to lookup signal name from number signame = dict((num, name) - for name, num in signal.__dict__.items() if name.startswith("SIG")) + for name, num in signal.__dict__.iteritems() if name.startswith("SIG")) class Utils(): """Utilities provide diverse static methods like executes OS shell commands, etc. @@ -140,7 +140,7 @@ def buildShellCmd(realCmd, varsDict): if not isinstance(realCmd, list): realCmd = [realCmd] i = len(realCmd)-1 - for k, v in varsDict.items(): + for k, v in varsDict.iteritems(): varsStat += "%s=$%s " % (k, i) realCmd.append(v) i += 1 diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 772c687ff4..ce5de48334 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -242,14 +242,14 @@ def testReplaceTagSelfRecursion(self): setattr(self.__action, 'ab', "") setattr(self.__action, 'x?family=inet6', "") # produce self-referencing properties except: - self.assertRaisesRegex(ValueError, r"properties contain self referencing definitions", + self.assertRaisesRegexp(ValueError, r"properties contain self referencing definitions", lambda: self.__action.replaceTag("", self.__action._properties, conditional="family=inet4") ) # remore self-referencing in props: delattr(self.__action, 'ac') # produce self-referencing query except: - self.assertRaisesRegex(ValueError, r"possible self referencing definitions in query", + self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", lambda: self.__action.replaceTag(""*30, self.__action._properties, conditional="family=inet6") ) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 2c0c4c4fc8..cf25ac0fcd 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -177,7 +177,7 @@ def setUp(self): super(StatusExtendedCymruInfo, self).setUp() unittest.F2B.SkipIfNoNetwork() setUpMyTime() - self.__ban_ip = next(iter(DNSUtils.dnsToIp("resolver1.opendns.com"))) + self.__ban_ip = iter(DNSUtils.dnsToIp("resolver1.opendns.com")).next() self.__asn = "36692" self.__country = "US" self.__rir = "arin" diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index a98f4e94cd..37083a069e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -419,7 +419,7 @@ def testSplitOption(self): # And multiple groups (`][` instead of `,`) result = extractOptions(option.replace(',', '][')) expected2 = (expected[0], - dict((k, v.replace(',', '][')) for k, v in expected[1].items()) + dict((k, v.replace(',', '][')) for k, v in expected[1].iteritems()) ) self.assertEqual(expected2, result) @@ -1018,7 +1018,7 @@ def testMultipleSameAction(self, basedir): self.assertEqual(add_actions[-1][-1], "{}") def testLogPathFileFilterBackend(self): - self.assertRaisesRegex(ValueError, r"Have not found any log file for .* jail", + self.assertRaisesRegexp(ValueError, r"Have not found any log file for .* jail", self._testLogPath, backend='polling') def testLogPathSystemdBackend(self): diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 6ac483c6cb..8cc394be73 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -67,7 +67,7 @@ def setUp(self): @property def db(self): - if isinstance(self._db, str) and self._db == ':auto-create-in-memory:': + if isinstance(self._db, basestring) and self._db == ':auto-create-in-memory:': self._db = getFail2BanDb(self.dbFilename) return self._db @db.setter @@ -159,7 +159,7 @@ def testUpdateDb(self): self.db = Fail2BanDb(self.dbFilename) self.assertEqual(self.db.getJailNames(), set(['DummyJail #29162448 with 0 tickets'])) self.assertEqual(self.db.getLogPaths(), set(['/tmp/Fail2BanDb_pUlZJh.log'])) - ticket = FailTicket("127.0.0.1", 1388009242.26, ["abc\n"]) + ticket = FailTicket("127.0.0.1", 1388009242.26, [u"abc\n"]) self.assertEqual(self.db.getBans()[0], ticket) self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__) @@ -185,9 +185,9 @@ def testUpdateDb2(self): self.assertEqual(len(bans), 2) # compare first ticket completely: ticket = FailTicket("1.2.3.7", 1417595494, [ - 'Dec 3 09:31:08 f2btest test:auth[27658]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', - 'Dec 3 09:31:32 f2btest test:auth[27671]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', - 'Dec 3 09:31:34 f2btest test:auth[27673]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7' + u'Dec 3 09:31:08 f2btest test:auth[27658]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', + u'Dec 3 09:31:32 f2btest test:auth[27671]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7', + u'Dec 3 09:31:34 f2btest test:auth[27673]: pam_unix(test:auth): authentication failure; logname= uid=0 euid=0 tty=test ruser= rhost=1.2.3.7' ]) ticket.setAttempt(3) self.assertEqual(bans[0], ticket) @@ -287,11 +287,11 @@ def testAddBanInvalidEncoded(self): # invalid + valid, invalid + valid unicode, invalid + valid dual converted (like in filter:readline by fallback) ... tickets = [ FailTicket("127.0.0.1", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), - FailTicket("127.0.0.2", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), + FailTicket("127.0.0.2", 0, ['user "test"', u'user "\xd1\xe2\xe5\xf2\xe0"', u'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), FailTicket("127.0.0.3", 0, ['user "test"', b'user "\xd1\xe2\xe5\xf2\xe0"', b'user "\xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f"']), - FailTicket("127.0.0.4", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', 'user "\xe4\xf6\xfc\xdf"']), + FailTicket("127.0.0.4", 0, ['user "test"', 'user "\xd1\xe2\xe5\xf2\xe0"', u'user "\xe4\xf6\xfc\xdf"']), FailTicket("127.0.0.5", 0, ['user "test"', 'unterminated \xcf']), - FailTicket("127.0.0.6", 0, ['user "test"', 'unterminated \xcf']), + FailTicket("127.0.0.6", 0, ['user "test"', u'unterminated \xcf']), FailTicket("127.0.0.7", 0, ['user "test"', b'unterminated \xcf']) ] for ticket in tickets: diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index fc43677a1b..bc33bc0528 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -288,7 +288,7 @@ def testFullYearMatch_gh130(self): self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') # confuse it with year being at the end - for i in range(10): + for i in xrange(10): ( logTime, logMatch ) = self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '11/10/2012 02:37:17') @@ -538,7 +538,7 @@ def testAmbiguousDatePattern(self): date = dd.getTime(line) if matched: self.assertTrue(date) - if isinstance(matched, str): + if isinstance(matched, basestring): self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(matched, date[0]) @@ -573,7 +573,7 @@ def testVariousFormatSpecs(self): date = dd.getTime(line) if matched: self.assertTrue(date) - if isinstance(matched, str): # pragma: no cover + if isinstance(matched, basestring): # pragma: no cover self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(matched, date[0]) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 9d3f382fcb..bc93dae7cc 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -367,10 +367,10 @@ def _stopAndWaitForServerEnd(code=(SUCCESS, FAILED)): # several commands to server in body of decorated function: return f(self, tmp, startparams, *args, **kwargs) except Exception as e: # pragma: no cover - print(('=== Catch an exception: %s' % e)) + print('=== Catch an exception: %s' % e) log = self.getLog() if log: - print(('=== Error of server, log: ===\n%s===' % log)) + print('=== Error of server, log: ===\n%s===' % log) self.pruneLog() raise finally: @@ -440,7 +440,7 @@ def _wait_for_srv(self, tmp, ready=True, startparams=None, phase=None): ) except: # pragma: no cover if _inherited_log(startparams): - print(('=== Error by wait fot server, log: ===\n%s===' % self.getLog())) + print('=== Error by wait fot server, log: ===\n%s===' % self.getLog()) self.pruneLog() log = pjoin(tmp, "f2b.log") if isfile(log): @@ -1702,6 +1702,6 @@ def _testServerStartStop(self, tmp, startparams): self.stopAndWaitForServerEnd(SUCCESS) def testServerStartStop(self): - for i in range(2000): + for i in xrange(2000): self._testServerStartStop() diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index a71a17774d..213ea89b8a 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -589,8 +589,8 @@ def testNLCharAsPartOfUniChar(self): # test on unicode string containing \x0A as part of uni-char, # it must produce exactly 2 lines (both are failures): for l in ( - '1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', - '1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' + u'1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', + u'1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' ): fout.write(l.encode(enc)) fout.close() diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index b07159fbb0..42b0fbd297 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -45,11 +45,11 @@ def tearDown(self): super(AddFailure, self).tearDown() def _addDefItems(self): - self.__items = [['193.168.0.128', 1167605999.0], - ['193.168.0.128', 1167605999.0], - ['193.168.0.128', 1167605999.0], - ['193.168.0.128', 1167605999.0], - ['193.168.0.128', 1167605999.0], + self.__items = [[u'193.168.0.128', 1167605999.0], + [u'193.168.0.128', 1167605999.0], + [u'193.168.0.128', 1167605999.0], + [u'193.168.0.128', 1167605999.0], + [u'193.168.0.128', 1167605999.0], ['87.142.124.10', 1167605999.0], ['87.142.124.10', 1167605999.0], ['87.142.124.10', 1167605999.0], diff --git a/fail2ban/tests/files/config/apache-auth/digest.py b/fail2ban/tests/files/config/apache-auth/digest.py index e2297ab312..0358859480 100755 --- a/fail2ban/tests/files/config/apache-auth/digest.py +++ b/fail2ban/tests/files/config/apache-auth/digest.py @@ -41,7 +41,7 @@ def auth(v): response="%s" """ % ( username, algorithm, realm, url, nonce, qop, response ) # opaque="%s", - print((p.method, p.url, p.headers)) + print(p.method, p.url, p.headers) s = requests.Session() return s.send(p) @@ -76,18 +76,18 @@ def preauth(): # [Sun Jul 28 21:41:20 2013] [error] [client 127.0.0.1] Digest: unknown algorithm `super funky chicken' received: /digest/ -print((r.status_code,r.headers, r.text)) +print(r.status_code,r.headers, r.text) v['algorithm'] = algorithm r = auth(v) -print((r.status_code,r.headers, r.text)) +print(r.status_code,r.headers, r.text) nonce = v['nonce'] v['nonce']=v['nonce'][5:-5] r = auth(v) -print((r.status_code,r.headers, r.text)) +print(r.status_code,r.headers, r.text) # [Sun Jul 28 21:05:31.178340 2013] [auth_digest:error] [pid 24224:tid 139895539455744] [client 127.0.0.1:56906] AH01793: invalid qop `auth' received: /digest/qop_none/ @@ -95,7 +95,7 @@ def preauth(): v['nonce']=nonce[0:11] + 'ZZZ' + nonce[14:] r = auth(v) -print((r.status_code,r.headers, r.text)) +print(r.status_code,r.headers, r.text) #[Sun Jul 28 21:18:11.769228 2013] [auth_digest:error] [pid 24752:tid 139895505884928] [client 127.0.0.1:56964] AH01776: invalid nonce b9YAiJDiBAZZZ1b1abe02d20063ea3b16b544ea1b0d981c1bafe received - hash is not d42d824dee7aaf50c3ba0a7c6290bd453e3dd35b @@ -107,7 +107,7 @@ def preauth(): time.sleep(1) r = auth(v) -print((r.status_code,r.headers, r.text)) +print(r.status_code,r.headers, r.text) # Obtained by putting the following code in modules/aaa/mod_auth_digest.c # in the function initialize_secret @@ -137,7 +137,7 @@ def preauth(): v=preauth() -print((v['nonce'])) +print(v['nonce']) realm = v['Digest realm'][1:-1] (t,) = struct.unpack('l',base64.b64decode(v['nonce'][1:13])) @@ -156,13 +156,13 @@ def preauth(): r = auth(v) #[Mon Jul 29 02:12:55.539813 2013] [auth_digest:error] [pid 9647:tid 139895522670336] [client 127.0.0.1:58474] AH01777: invalid nonce 59QJppTiBAA=b08983fd166ade9840407df1b0f75b9e6e07d88d received - user attempted time travel -print((r.status_code,r.headers, r.text)) +print(r.status_code,r.headers, r.text) url='/digest_onetime/' v=preauth() # Need opaque header handling in auth r = auth(v) -print((r.status_code,r.headers, r.text)) +print(r.status_code,r.headers, r.text) r = auth(v) -print((r.status_code,r.headers, r.text)) +print(r.status_code,r.headers, r.text) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index e31e6492d2..4e308e38e6 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -22,7 +22,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier; 2012 Yaroslav Halchenko" __license__ = "GPL" -from builtins import open as fopen +from __builtin__ import open as fopen import unittest import os import re @@ -209,7 +209,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line else: fin = in_ # Skip - for i in range(skip): + for i in xrange(skip): fin.readline() # Read i = 0 @@ -250,7 +250,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p # Required for filtering fields.update(TEST_JOURNAL_FIELDS) # Skip - for i in range(skip): + for i in xrange(skip): fin.readline() # Read/Write i = 0 @@ -312,18 +312,18 @@ def testAssertWrongTime(self): def testTest_tm(self): unittest.F2B.SkipIfFast() ## test function "_tm" works correct (returns the same as slow strftime): - for i in range(1417512352, (1417512352 // 3600 + 3) * 3600): + for i in xrange(1417512352, (1417512352 // 3600 + 3) * 3600): tm = MyTime.time2str(i) if _tm(i) != tm: # pragma: no cover - never reachable self.assertEqual((_tm(i), i), (tm, i)) def testWrongCharInTupleLine(self): ## line tuple has different types (ascii after ascii / unicode): - for a1 in ('', '', b''): - for a2 in ('2016-09-05T20:18:56', '2016-09-05T20:18:56', b'2016-09-05T20:18:56'): + for a1 in ('', u'', b''): + for a2 in ('2016-09-05T20:18:56', u'2016-09-05T20:18:56', b'2016-09-05T20:18:56'): for a3 in ( 'Fail for "g\xc3\xb6ran" from 192.0.2.1', - 'Fail for "g\xc3\xb6ran" from 192.0.2.1', + u'Fail for "g\xc3\xb6ran" from 192.0.2.1', b'Fail for "g\xc3\xb6ran" from 192.0.2.1' ): # join should work if all arguments have the same type: @@ -510,7 +510,7 @@ def testWrongTimeOrTZ(self): def testAddAttempt(self): self.filter.setMaxRetry(3) - for i in range(1, 1+3): + for i in xrange(1, 1+3): self.filter.addAttempt('192.0.2.1') self.assertLogged('Attempt 192.0.2.1', '192.0.2.1:%d' % i, all=True, wait=True) self.jail.actions._Actions__checkBan() @@ -547,7 +547,7 @@ def testIgnoreCache(self): # like both test-cases above, just cached (so once per key)... self.filter.ignoreCache = {"key":""} self.filter.ignoreCommand = 'if [ "" = "10.0.0.1" ]; then exit 0; fi; exit 1' - for i in range(5): + for i in xrange(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1")) self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0")) @@ -558,7 +558,7 @@ def testIgnoreCache(self): # by host of IP: self.filter.ignoreCache = {"key":""} self.filter.ignoreCommand = 'if [ "" = "test-host" ]; then exit 0; fi; exit 1' - for i in range(5): + for i in xrange(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList(FailTicket("2001:db8::1"))) self.assertFalse(self.filter.inIgnoreIPList(FailTicket("2001:db8::ffff"))) @@ -570,7 +570,7 @@ def testIgnoreCache(self): self.filter.ignoreCache = {"key":"", "max-count":"10", "max-time":"1h"} self.assertEqual(self.filter.ignoreCache, ["", 10, 60*60]) self.filter.ignoreCommand = 'if [ "" = "tester" ]; then exit 0; fi; exit 1' - for i in range(5): + for i in xrange(5): self.pruneLog() self.assertTrue(self.filter.inIgnoreIPList(FailTicket("tester", data={'user': 'tester'}))) self.assertFalse(self.filter.inIgnoreIPList(FailTicket("root", data={'user': 'root'}))) @@ -673,7 +673,7 @@ def testMissingLogFiles(self): def testDecodeLineWarn(self): # incomplete line (missing byte at end), warning is suppressed: - l = "correct line\n" + l = u"correct line\n" r = l.encode('utf-16le') self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r), l) self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r[0:-1]), l[0:-1]) @@ -733,7 +733,7 @@ def testSeekToTimeSmallFile(self): fc = FileContainer(fname, self.filter.getLogEncoding()) fc.open() # no time - nothing should be found : - for i in range(10): + for i in xrange(10): f.write("[sshd] error: PAM: failure len 1\n") f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) @@ -807,14 +807,14 @@ def testSeekToTimeLargeFile(self): # variable length of file (ca 45K or 450K before and hereafter): # write lines with smaller as search time: t = time - count - 1 - for i in range(count): + for i in xrange(count): f.write("%s [sshd] error: PAM: failure\n" % _tm(t)) t += 1 f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) self.assertEqual(fc.getPos(), 47*count) # write lines with exact search time: - for i in range(10): + for i in xrange(10): f.write("%s [sshd] error: PAM: failure\n" % _tm(time)) f.flush() fc.setPos(0); self.filter.seekToTime(fc, time) @@ -823,8 +823,8 @@ def testSeekToTimeLargeFile(self): self.assertEqual(fc.getPos(), 47*count) # write lines with greater as search time: t = time+1 - for i in range(count//500): - for j in range(500): + for i in xrange(count//500): + for j in xrange(500): f.write("%s [sshd] error: PAM: failure\n" % _tm(t)) t += 1 f.flush() @@ -1634,10 +1634,10 @@ def test_WrongChar(self): # Add direct utf, unicode, blob: for l in ( "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", - "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", + u"error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", b"error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1".decode('utf-8', 'replace'), "error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", - "error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", + u"error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2", b"error: PAM: Authentication failure for \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x9f from 192.0.2.2".decode('utf-8', 'replace') ): fields = self.journal_fields @@ -1666,7 +1666,7 @@ class GetFailures(LogCaptureTestCase): # so that they could be reused by other tests FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, - ['Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3) + [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3) def setUp(self): """Call before every test case.""" @@ -1752,8 +1752,8 @@ def testNLCharAsPartOfUniChar(self): # test on unicode string containing \x0A as part of uni-char, # it must produce exactly 2 lines (both are failures): for l in ( - '%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, - '%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm + u'%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, + u'%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm ): fout.write(l.encode(enc)) fout.close() @@ -1774,8 +1774,8 @@ def testNLCharAsPartOfUniChar(self): def testGetFailures02(self): output = ('141.3.81.106', 4, 1124013539.0, - ['Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' - % m for m in (53, 54, 57, 58)]) + [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' + % m for m in 53, 54, 57, 58]) self.filter.setMaxRetry(4) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) @@ -1886,19 +1886,19 @@ def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) output_yes = ( ('93.184.216.34', 1, 1124013299.0, - ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] ), ('93.184.216.34', 1, 1124013539.0, - ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ), ('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0, - ['Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] ), ) output_no = ( ('93.184.216.34', 1, 1124013539.0, - ['Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ) ) @@ -2004,9 +2004,9 @@ def testCache(self): self.assertTrue(c.get('a') is None) self.assertEqual(c.get('a', 'test'), 'test') # exact 5 elements : - for i in range(5): + for i in xrange(5): c.set(i, i) - for i in range(5): + for i in xrange(5): self.assertEqual(c.get(i), i) # remove unavailable key: c.unset('a'); c.unset('a') @@ -2014,30 +2014,30 @@ def testCache(self): def testCacheMaxSize(self): c = Utils.Cache(maxCount=5, maxTime=60) # exact 5 elements : - for i in range(5): + for i in xrange(5): c.set(i, i) - self.assertEqual([c.get(i) for i in range(5)], [i for i in range(5)]) - self.assertNotIn(-1, (c.get(i, -1) for i in range(5))) + self.assertEqual([c.get(i) for i in xrange(5)], [i for i in xrange(5)]) + self.assertNotIn(-1, (c.get(i, -1) for i in xrange(5))) # add one - too many: c.set(10, i) # one element should be removed : - self.assertIn(-1, (c.get(i, -1) for i in range(5))) + self.assertIn(-1, (c.get(i, -1) for i in xrange(5))) # test max size (not expired): - for i in range(10): + for i in xrange(10): c.set(i, 1) self.assertEqual(len(c), 5) def testCacheMaxTime(self): # test max time (expired, timeout reached) : c = Utils.Cache(maxCount=5, maxTime=0.0005) - for i in range(10): + for i in xrange(10): c.set(i, 1) st = time.time() self.assertTrue(Utils.wait_for(lambda: time.time() >= st + 0.0005, 1)) # we have still 5 elements (or fewer if too slow test mashine): self.assertTrue(len(c) <= 5) # but all that are expiered also: - for i in range(10): + for i in xrange(10): self.assertTrue(c.get(i) is None) # here the whole cache should be empty: self.assertEqual(len(c), 0) @@ -2058,7 +2058,7 @@ def _TestCacheStr2IP(forw=True, result=[], random=False): c = count while c: c -= 1 - s = range(0, 256, 1) if forw else range(255, -1, -1) + s = xrange(0, 256, 1) if forw else xrange(255, -1, -1) if random: shuffle([i for i in s]) for i in s: IPAddr('192.0.2.'+str(i), IPAddr.FAM_IPv4) @@ -2184,16 +2184,16 @@ def testIpToName(self): def testAddr2bin(self): res = IPAddr('10.0.0.0') - self.assertEqual(res.addr, 167772160) + self.assertEqual(res.addr, 167772160L) res = IPAddr('10.0.0.0', cidr=None) - self.assertEqual(res.addr, 167772160) - res = IPAddr('10.0.0.0', cidr=32) - self.assertEqual(res.addr, 167772160) - res = IPAddr('10.0.0.1', cidr=32) - self.assertEqual(res.addr, 167772161) + self.assertEqual(res.addr, 167772160L) + res = IPAddr('10.0.0.0', cidr=32L) + self.assertEqual(res.addr, 167772160L) + res = IPAddr('10.0.0.1', cidr=32L) + self.assertEqual(res.addr, 167772161L) self.assertTrue(res.isSingle) - res = IPAddr('10.0.0.1', cidr=31) - self.assertEqual(res.addr, 167772160) + res = IPAddr('10.0.0.1', cidr=31L) + self.assertEqual(res.addr, 167772160L) self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') @@ -2272,9 +2272,9 @@ def testIPAddr_Compare(self): '93.184.216.34': 'ip4-test', '2606:2800:220:1:248:1893:25c8:1946': 'ip6-test' } - d2 = dict([(IPAddr(k), v) for k, v in d.items()]) - self.assertTrue(isinstance(list(d.keys())[0], str)) - self.assertTrue(isinstance(list(d2.keys())[0], IPAddr)) + d2 = dict([(IPAddr(k), v) for k, v in d.iteritems()]) + self.assertTrue(isinstance(d.keys()[0], basestring)) + self.assertTrue(isinstance(d2.keys()[0], IPAddr)) self.assertEqual(d.get(ip4[2], ''), 'ip4-test') self.assertEqual(d.get(ip6[2], ''), 'ip6-test') self.assertEqual(d2.get(str(ip4[2]), ''), 'ip4-test') diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index d91729fb93..4b02637755 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -29,9 +29,9 @@ import shutil import fnmatch from glob import glob -from io import StringIO +from StringIO import StringIO -from .utils import LogCaptureTestCase, logSys as DefLogSys +from utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, \ getVerbosityFormat, splitwords, uni_decode, uni_string @@ -67,7 +67,7 @@ def testsplitwords(self): self.assertEqual(splitwords(' 1\n 2'), ['1', '2']) self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) # string as unicode: - self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) + self.assertEqual(splitwords(u' 1\n 2, 3'), ['1', '2', '3']) def _sh_call(cmd): @@ -191,12 +191,12 @@ def testmbasename(self): def testUniConverters(self): self.assertRaises(Exception, uni_decode, - (b'test' if sys.version_info >= (3,) else 'test'), 'f2b-test::non-existing-encoding') - uni_decode((b'test\xcf' if sys.version_info >= (3,) else 'test\xcf')) + (b'test' if sys.version_info >= (3,) else u'test'), 'f2b-test::non-existing-encoding') + uni_decode((b'test\xcf' if sys.version_info >= (3,) else u'test\xcf')) uni_string(b'test\xcf') uni_string('test\xcf') if sys.version_info < (3,) and 'PyPy' not in sys.version: - uni_string('test\xcf') + uni_string(u'test\xcf') def testSafeLogging(self): # logging should be exception-safe, to avoid possible errors (concat, str. conversion, representation failures, etc) @@ -208,7 +208,7 @@ def __repr__(self): if self.err: raise Exception('no represenation for test!') else: - return 'conv-error (\xf2\xf0\xe5\xf2\xe8\xe9), unterminated utf \xcf' + return u'conv-error (\xf2\xf0\xe5\xf2\xe8\xe9), unterminated utf \xcf' test = Test() logSys.log(logging.NOTICE, "test 1a: %r", test) self.assertLogged("Traceback", "no represenation for test!") @@ -256,7 +256,7 @@ def deep_function(i): func_raise() try: - print(deep_function(3)) + print deep_function(3) except ValueError: s = tb() @@ -273,7 +273,7 @@ def deep_function(i): self.assertIn(':', s) def _testAssertionErrorRE(self, regexp, fun, *args, **kwargs): - self.assertRaisesRegex(AssertionError, regexp, fun, *args, **kwargs) + self.assertRaisesRegexp(AssertionError, regexp, fun, *args, **kwargs) def testExtendedAssertRaisesRE(self): ## test _testAssertionErrorRE several fail cases: @@ -311,13 +311,13 @@ def testExtendedAssertMethods(self): self._testAssertionErrorRE(r"'a' unexpectedly found in 'cba'", self.assertNotIn, 'a', 'cba') self._testAssertionErrorRE(r"1 unexpectedly found in \[0, 1, 2\]", - self.assertNotIn, 1, range(3)) + self.assertNotIn, 1, xrange(3)) self._testAssertionErrorRE(r"'A' unexpectedly found in \['C', 'A'\]", self.assertNotIn, 'A', (c.upper() for c in 'cba' if c != 'b')) self._testAssertionErrorRE(r"'a' was not found in 'xyz'", self.assertIn, 'a', 'xyz') self._testAssertionErrorRE(r"5 was not found in \[0, 1, 2\]", - self.assertIn, 5, range(3)) + self.assertIn, 5, xrange(3)) self._testAssertionErrorRE(r"'A' was not found in \['C', 'B'\]", self.assertIn, 'A', (c.upper() for c in 'cba' if c != 'a')) ## assertLogged, assertNotLogged positive case: diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index ace1184c68..9b44c6dded 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -68,7 +68,7 @@ def testDefault(self, multipliers = None): a.setBanTimeExtra('multipliers', multipliers) # test algorithm and max time 24 hours : self.assertEqual( - [a.calcBanTime(600, i) for i in range(1, 11)], + [a.calcBanTime(600, i) for i in xrange(1, 11)], [1200, 2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400] ) # with extra large max time (30 days): @@ -80,38 +80,38 @@ def testDefault(self, multipliers = None): if multcnt < 11: arr = arr[0:multcnt-1] + ([arr[multcnt-2]] * (11-multcnt)) self.assertEqual( - [a.calcBanTime(600, i) for i in range(1, 11)], + [a.calcBanTime(600, i) for i in xrange(1, 11)], arr ) a.setBanTimeExtra('maxtime', '1d') # change factor : a.setBanTimeExtra('factor', '2'); self.assertEqual( - [a.calcBanTime(600, i) for i in range(1, 11)], + [a.calcBanTime(600, i) for i in xrange(1, 11)], [2400, 4800, 9600, 19200, 38400, 76800, 86400, 86400, 86400, 86400] ) # factor is float : a.setBanTimeExtra('factor', '1.33'); self.assertEqual( - [int(a.calcBanTime(600, i)) for i in range(1, 11)], + [int(a.calcBanTime(600, i)) for i in xrange(1, 11)], [1596, 3192, 6384, 12768, 25536, 51072, 86400, 86400, 86400, 86400] ) a.setBanTimeExtra('factor', None); # change max time : a.setBanTimeExtra('maxtime', '12h') self.assertEqual( - [a.calcBanTime(600, i) for i in range(1, 11)], + [a.calcBanTime(600, i) for i in xrange(1, 11)], [1200, 2400, 4800, 9600, 19200, 38400, 43200, 43200, 43200, 43200] ) a.setBanTimeExtra('maxtime', '24h') ## test randomization - not possibe all 10 times we have random = 0: a.setBanTimeExtra('rndtime', '5m') self.assertTrue( - False in [1200 in [a.calcBanTime(600, 1) for i in range(10)] for c in range(10)] + False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] ) a.setBanTimeExtra('rndtime', None) self.assertFalse( - False in [1200 in [a.calcBanTime(600, 1) for i in range(10)] for c in range(10)] + False in [1200 in [a.calcBanTime(600, 1) for i in xrange(10)] for c in xrange(10)] ) # restore default: a.setBanTimeExtra('multipliers', None) @@ -123,7 +123,7 @@ def testMultipliers(self): # this multipliers has the same values as default formula, we test stop growing after count 9: self.testDefault('1 2 4 8 16 32 64 128 256') # this multipliers has exactly the same values as default formula, test endless growing (stops by count 31 only): - self.testDefault(' '.join([str(1<= 0: - line1 = next(f) + line1 = f.next() self.assertTrue(line1.endswith("Before file moved\n")) - line2 = next(f) + line2 = f.next() self.assertTrue(line2.endswith("After file moved\n")) try: - n = next(f) + n = f.next() if n.find("Command: ['flushlogs']") >=0: - self.assertRaises(StopIteration, f.__next__) + self.assertRaises(StopIteration, f.next) else: self.fail("Exception StopIteration or Command: ['flushlogs'] expected. Got: %s" % n) except StopIteration: pass # on higher debugging levels this is expected with open(fn,'r') as f: - line1 = next(f) + line1 = f.next() if line1.find('rollover performed on') >= 0: - line1 = next(f) + line1 = f.next() self.assertTrue(line1.endswith("After flushlogs\n")) - self.assertRaises(StopIteration, f.__next__) + self.assertRaises(StopIteration, f.next) f.close() finally: os.remove(fn2) @@ -1185,7 +1185,7 @@ def testStartFailedSockExists(self): os.remove(f) -from .clientreadertestcase import ActionReader, JailsReader, CONFIG_DIR +from clientreadertestcase import ActionReader, JailsReader, CONFIG_DIR class ServerConfigReaderTests(LogCaptureTestCase): diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 15253ce097..e3a07998d6 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -153,7 +153,7 @@ def testSocketConnectBroken(self): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: self.close() - self.assertRaisesRegex(Exception, r"reset by peer|Broken pipe", + self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10))) finally: RequestHandler.found_terminator = org_handler diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 98ec581146..8bcc1431d4 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -35,7 +35,7 @@ import threading import unittest -from io import StringIO +from cStringIO import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode @@ -171,8 +171,8 @@ def initProcess(opts): # Let know the version if opts.verbosity != 0: - print(("Fail2ban %s test suite. Python %s. Please wait..." \ - % (version, str(sys.version).replace('\n', '')))) + print("Fail2ban %s test suite. Python %s. Please wait..." \ + % (version, str(sys.version).replace('\n', ''))) return opts; @@ -303,7 +303,7 @@ def F2B_SkipIfNoNetwork(): c.clear = lambda: logSys.warn('clear CACHE_ipToName is disabled in test suite') # increase max count and max time (too many entries, long time testing): c.setOptions(maxCount=10000, maxTime=5*60) - for i in range(256): + for i in xrange(256): c.set('192.0.2.%s' % i, None) c.set('198.51.100.%s' % i, None) c.set('203.0.113.%s' % i, None) @@ -531,8 +531,8 @@ def addTest(self, suite): import difflib, pprint if not hasattr(unittest.TestCase, 'assertDictEqual'): def assertDictEqual(self, d1, d2, msg=None): - self.assertTrue(isinstance(d1, dict), 'First argument is not a dictionary') - self.assertTrue(isinstance(d2, dict), 'Second argument is not a dictionary') + self.assert_(isinstance(d1, dict), 'First argument is not a dictionary') + self.assert_(isinstance(d2, dict), 'Second argument is not a dictionary') if d1 != d2: standardMsg = '%r != %r' % (d1, d2) diff = ('\n' + '\n'.join(difflib.ndiff( @@ -550,7 +550,7 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None) # used to recognize having element as nested dict, list or tuple: def _is_nested(v): if isinstance(v, dict): - return any(isinstance(v, (dict, list, tuple)) for v in v.values()) + return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) return any(isinstance(v, (dict, list, tuple)) for v in v) if nestedOnly: _nest_sorted = sorted @@ -570,7 +570,7 @@ def _assertSortedEqual(a, b, level, nestedOnly, key): return raise ValueError('%r != %r' % (a, b)) if isinstance(a, dict) and isinstance(b, dict): # compare dict's: - for k, v1 in a.items(): + for k, v1 in a.iteritems(): v2 = b[k] if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key) @@ -605,14 +605,14 @@ def assertRaisesRegexp(self, exccls, regexp, fun, *args, **kwargs): self.fail('\"%s\" does not match \"%s\"' % (regexp, e)) else: self.fail('%s not raised' % getattr(exccls, '__name__')) - unittest.TestCase.assertRaisesRegex = assertRaisesRegexp + unittest.TestCase.assertRaisesRegexp = assertRaisesRegexp # always custom following methods, because we use atm better version of both (support generators) if True: ## if not hasattr(unittest.TestCase, 'assertIn'): def assertIn(self, a, b, msg=None): bb = b wrap = False - if msg is None and hasattr(b, '__iter__') and not isinstance(b, str): + if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring): b, bb = itertools.tee(b) wrap = True if a not in b: @@ -623,7 +623,7 @@ def assertIn(self, a, b, msg=None): def assertNotIn(self, a, b, msg=None): bb = b wrap = False - if msg is None and hasattr(b, '__iter__') and not isinstance(b, str): + if msg is None and hasattr(b, '__iter__') and not isinstance(b, basestring): b, bb = itertools.tee(b) wrap = True if a in b: From 18308e039563faec604b7c080a03015c21a4b147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 11:27:03 +0100 Subject: [PATCH 25/28] fixed typo --- fail2ban/server/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index cc1434a961..ee83b09cf6 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -618,7 +618,7 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): if shutil.which("mmdblookup") and os.path.isfile(geoip2cf): geoipcc = str(subprocess.check_output(["mmdblookup","--file",geoip2cf,"--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") elif shutil.which("geoiplookup"): - if shutil.which("mmdblookup") and !os.path.isfile(geoip2cf): + if shutil.which("mmdblookup") and not os.path.isfile(geoip2cf): self._logWarnOnce("_next_geocfByTimeWarn", ("Found mmdblookup but cannot find mmdb country file at /usr/share/GeoIP/GeoIP2-Country.mmdb, using geoiplookup instead.")) geoipcc = str(subprocess.check_output(["geoiplookup",str(ip)])).split(" ")[3].replace(",","") From 41d7e81786d17dc2894f735508114200d65c6fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 12:00:57 +0100 Subject: [PATCH 26/28] removed diff --- fail2ban/server/filter.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 7f4800541b..8971a12152 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -618,12 +618,8 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): if shutil.which("mmdblookup") and os.path.isfile(geoip2cf): geoipcc = str(subprocess.check_output(["mmdblookup","--file","/usr/share/GeoIP/GeoIP2-Country.mmdb","--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") elif shutil.which("geoiplookup"): -<<<<<<< HEAD if shutil.which("mmdblookup") and not os.path.isfile(geoip2cf): self._logWarnOnce("_next_geocfByTimeWarn", ("Found mmdblookup but cannot find mmdb country file at /usr/share/GeoIP/GeoIP2-Country.mmdb, using geoiplookup instead.")) - -======= ->>>>>>> 56dec6fadd6e9494d1ef35c97a2c4fb399628ff3 geoipcc = str(subprocess.check_output(["geoiplookup",str(ip)])).split(" ")[3].replace(",","") if geoipcc in self.__ignoreGeoSet: self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) From 6e3fa5324ac239331beb219b87b637d4777ef9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 12:26:27 +0100 Subject: [PATCH 27/28] reworked the mmdblookup section to work properly --- fail2ban/server/filter.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 8971a12152..c38a422f06 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -612,20 +612,23 @@ def _inIgnoreIPList(self, ip, ticket, log_ignore=True): return True # check if the IP's geolocation is not on geo ignore list - geoipcc = "" - geoip2cf = "/usr/share/GeoIP/GeoIP2-Country.mmdb" + if self.__ignoreGeoSet: + geoipcc = None + geoip2cf = "/usr/share/GeoIP/GeoIP2-Country.mmdb" + if shutil.which("mmdblookup") and os.path.isfile(geoip2cf): geoipcc = str(subprocess.check_output(["mmdblookup","--file","/usr/share/GeoIP/GeoIP2-Country.mmdb","--ip",str(ip),"country","iso_code"])).split(" ")[2].replace("\"", "") elif shutil.which("geoiplookup"): if shutil.which("mmdblookup") and not os.path.isfile(geoip2cf): - self._logWarnOnce("_next_geocfByTimeWarn", ("Found mmdblookup but cannot find mmdb country file at /usr/share/GeoIP/GeoIP2-Country.mmdb, using geoiplookup instead.")) + logSys.warning("Found mmdblookup but cannot find mmdb country file at /usr/share/GeoIP/GeoIP2-Country.mmdb, using geoiplookup instead.") geoipcc = str(subprocess.check_output(["geoiplookup",str(ip)])).split(" ")[3].replace(",","") - if geoipcc in self.__ignoreGeoSet: - self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) - return True else: - self._logWarnOnce("_next_geoByTimeWarn", ("Cannot find geoiplookup. Geolocation is unavailable.")) + logSys.warning("Cannot find geoiplookup or correctly set mmdblookup. Geolocation is unavailable. If you have mmdblookup installed, please check if /usr/share/GeoIP/GeoIP2-Country.mmdb file is available.") + + if geoipcc in self.__ignoreGeoSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="geo-" + geoipcc) + return True # check if the IP is covered by ignore IP (in set or in subnet/dns): if ip in self.__ignoreIpSet: From 821b754861dd3ae56ccf49f157ac8f17737ddb98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B3bert=20Belanec?= Date: Thu, 23 Feb 2023 12:29:40 +0100 Subject: [PATCH 28/28] added mmdblookup to readme and jai.conf manpage --- README.md | 1 + man/jail.conf.5 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 280b4217d7..e983912035 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Optional: * [python-systemd package](https://www.freedesktop.org/software/systemd/python-systemd/index.html) - [dnspython](http://www.dnspython.org/) - [geoiplookup](https://linux.die.net/man/1/geoiplookup) +- [mmdblookup](https://maxmind.github.io/libmaxminddb/mmdblookup.html) To install: diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 5e319de678..1483c28eb7 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -247,7 +247,7 @@ boolean value (default true) indicates the banning of own IP addresses should be list of IPs not to ban. They can include a DNS resp. CIDR mask too. The option affects additionally to \fBignoreself\fR (if true) and don't need to contain own DNS resp. IPs of the running host. .TP .B ignoregeo -list of GEO location codes of IPs not to ban. For this to work, you should have a `geoiplookup` or other program with the same name installed. This feature was only tested with Maxmind's `geoiplookup`. +list of GEO location codes of IPs not to ban. For this to work, you should have a `geoiplookup` or `mmdblookup` or other program with the same name installed. This feature was only tested with Maxmind's `geoiplookup` or `mmdblookup`. .TP .B ignorecommand command that is executed to determine if the current candidate IP for banning (or failure-ID for raw IDs) should not be banned. The option affects additionally to \fBignoreself\fR and \fBignoreip\fR and will be first executed if both don't hit.