Fix the mypy issues

newspaper/extractors/articlebody_extractor.py

@@ -4,6 +4,7 @@
 from statistics import mean
 from typing import Optional
 import lxml
+from newspaper.configuration import Configuration
 import newspaper.extractors.defines as defines
 import newspaper.parsers as parsers
 from newspaper.text import StopWords
@@ -25,7 +26,7 @@
 
 
 class ArticleBodyExtractor:
-    def __init__(self, config):
+    def __init__(self, config: Configuration):
         self.config = config
         self.top_node = None
         self.top_node_complemented = None

newspaper/nlp.py

@@ -31,32 +31,31 @@ def keywords(text: str, stopwords: StopWords, max_keywords: Optional[int] = None
     Returns:
         dict: The top 10 keywords and their frequency scores.
     """
-    text = list(stopwords.tokenizer(text))
+    tokenised_text = list(stopwords.tokenizer(text))
     if not text:
         return dict()
     # of words before removing blacklist words
-    num_words = len(text) or 1
-    text = filter(lambda x: x not in stopwords.stop_words, text)
+    num_words = len(tokenised_text) or 1
+    tokenised_text = list(
+        filter(lambda x: x not in stopwords.stop_words, tokenised_text)
+    )
 
-    freq = Counter(text)
+    freq = Counter(tokenised_text)
 
     keywords_ = freq.most_common(max_keywords)
+    keywords_dict = {k: v * 1.5 / num_words + 1 for k, v in keywords_}
 
-    keywords_ = {k: v * 1.5 / num_words + 1 for k, v in keywords_}
-
-    return keywords_
+    return keywords_dict
 
 
-def summarize(
-    title: str, text: str, stopwords: StopWords, max_sents: Optional[int] = 5
-):
+def summarize(title: str, text: str, stopwords: StopWords, max_sents: int = 5):
     """Summarize an article into the most relevant sentences in the article.
 
     Args:
         title (str): the article title
         text (str): article contents
         stopwords (StopWords): stopwords object for the language of the text
-        max_sents (Optional[int], optional):maximum number of sentences to
+        max_sents (int, optional):maximum number of sentences to
             return in the summary. Sentences are weighted by their relevance
             using the following criteria: sentence position, frequency of
             keywords, title words found in the sentence, and sentence length.

newspaper/parsers.py

@@ -239,7 +239,7 @@ def create_element(tag, text=None, tail=None):
 
 def remove(
     nodes: Union[lxml.html.HtmlElement, List[lxml.html.HtmlElement]],
-    keep_tags: List[str] = None,
+    keep_tags: Optional[List[str]] = None,
 ):
     """Remove the node(s) from the tree
     Arguments:

newspaper/text.py

@@ -15,14 +15,14 @@
 
 from newspaper import settings
 
-punctuation = {
+punctuation_set = {
     c for i in range(sys.maxunicode + 1) if category(c := chr(i)).startswith("P")
 }
-punctuation.update(string.punctuation)
+punctuation_set.update(string.punctuation)
 # remove characters used in contractions
 contraction_separators = set("-'`ʹʻʼʽʾʿˈˊ‘’‛′‵Ꞌꞌ")
-punctuation -= contraction_separators
-punctuation: str = "".join(list(punctuation))
+punctuation_set -= contraction_separators
+punctuation: str = "".join(list(punctuation_set))
 whitespace_tokenizer = WhitespaceTokenizer()
 
 

stubs/indicnlp/tokenize/indic_tokenize.pyi

@@ -0,0 +1 @@
+def trivial_tokenize(text: str, lang: str = ...): ...

stubs/jieba.pyi

@@ -0,0 +1,3 @@
+from typing import Union
+
+def cut(sentence: Union[bytes, str], cut_all: bool = ...): ...

stubs/nltk/__init__.pyi

@@ -2,3 +2,4 @@ from typing import Optional
 from . import data  # noqa: F401
 
 def download(info_or_id: Optional[str] = None): ...
+def word_tokenize(text, language: str = ...): ...

stubs/nltk/tokenize.pyi

@@ -0,0 +1 @@
+class WhitespaceTokenizer: ...

stubs/pythainlp.pyi

@@ -0,0 +1,5 @@
+from typing import List
+
+def word_tokenize(
+    text: str,
+) -> List[str]: ...

stubs/tinysegmenter.pyi

@@ -0,0 +1,2 @@
+class TinySegmenter:
+    def tokenize(self, text: str): ...