refactor: added valid_languages function that returns available langu…

…ages

newspaper/__init__.py

@@ -29,6 +29,8 @@
 import logging
 from logging import NullHandler
 from .exceptions import ArticleBinaryDataException, ArticleException
+from .languages import valid_languages
+
 
 # Set default logging handler to avoid "No handler found" warnings.
 logging.getLogger(__name__).addHandler(NullHandler())
@@ -69,6 +71,7 @@ def article(url: str, language: Optional[str] = "en", **kwargs) -> Article:
     "fulltext",
     "hot",
     "languages",
+    "valid_languages",
     "popular_urls",
     "Config",
     "Article",

newspaper/languages/__init__.py

@@ -1,5 +1,8 @@
+from pathlib import Path
 from typing import Optional
 
+from newspaper import settings
+
 
 languages_tuples = [
     ("aa", "Afar"),
@@ -223,3 +226,14 @@ def get_language_from_iso639_1(iso639_1: str) -> Optional[str]:
         str: Language name (in english)
     """
     return languages_dict.get(iso639_1)
+
+
+def valid_languages():
+    """Returns the List of available Languages"""
+    languages = []
+    for code, language in languages_tuples:
+        stopwords_file = Path(settings.STOPWORDS_DIR) / f"stopwords-{language}.txt"
+        if stopwords_file.exists():
+            languages.append((code, language))
+
+    return languages

newspaper/text.py

@@ -73,13 +73,13 @@ def __init__(self, language="en"):
         self.tokenizer = default_tokenizer
 
         if language not in self._cached_stop_words:
-            stopwordsFile = Path(settings.STOPWORDS_DIR) / f"stopwords-{language}.txt"
-            if not stopwordsFile.exists():
+            stopwords_file = Path(settings.STOPWORDS_DIR) / f"stopwords-{language}.txt"
+            if not stopwords_file.exists():
                 raise FileNotFoundError(
                     f"Stopwords file for language {language} not found! Make sure that "
                     "the language is supported (see `newspaper.languages()`)"
                 )
-            with open(stopwordsFile, "r", encoding="utf-8") as f:
+            with open(stopwords_file, "r", encoding="utf-8") as f:
                 self._cached_stop_words[language] = set(f.read().splitlines())
 
         lang_module = Path(__file__).parent / "languages" / f"{language}.py"

tests/test_languages.py

@@ -1,7 +1,6 @@
 import pytest
 
 import newspaper
-from newspaper import nlp
 from newspaper.article import Article
 from newspaper.text import StopWords
 from tests import conftest
@@ -104,12 +103,12 @@ def test_error_unknown_language(self):
         with pytest.raises(ValueError):
             _ = Article("http://www.cnn.com", language="zz")
 
-    @pytest.mark.skip(reason="valid_languages not implemented")
     def test_stopwords_english(self, valid_language_fixture):
-        for lang in valid_language_fixture:
-            nlp.stopwords = set()
-            nlp.load_stopwords(lang)
-            assert len(nlp.stopwords) > 100
+        for lang, language_name in valid_language_fixture:
+            stopwords = StopWords(lang)
+            assert (
+                len(stopwords.stop_words) > 100
+            ), f"Language {language_name} has too few stopwords"
 
     def test_full_extract(self, language_article_fixture):
         errors = []