Added a try/except to catch unicode errors.

elgatito · Jan 13, 2022 · 7415707 · 7415707
1 parent 701f323
commit 7415707
Showing 1 changed file with 14 additions and 11 deletions.
diff --git a/burst/normalize.py b/burst/normalize.py
@@ -184,20 +184,23 @@ def normalize_string(string, charset=None, replacing=False):
             string = unicode(string, errors='ignore')
             pass
 
-    string = remove_control_chars(string)
-    string = fix_bad_unicode(string)
-    string = unquote(string)
-    string = string.replace(u'<![CDATA[', u'').replace(u']]', u'')
+    try:
+        string = remove_control_chars(string)
+        string = fix_bad_unicode(string)
+        string = unquote(string)
+        string = string.replace(u'<![CDATA[', u'').replace(u']]', u'')
 
-    if PY3:
-        string = html.unescape(string)
-    else:
-        string = HTMLParser().unescape(string)
+        if PY3:
+            string = html.unescape(string)
+        else:
+            string = HTMLParser().unescape(string)
 
-    if replacing:
-        string = string.replace(u"'", '')
+        if replacing:
+            string = string.replace(u"'", '')
 
-    string = string.lower()
+        string = string.lower()
+    except:
+        pass
 
     return string