Skip to content

Commit

Permalink
Add more logs for category processing
Browse files Browse the repository at this point in the history
  • Loading branch information
benoit74 committed Aug 21, 2023
1 parent d782273 commit 15a471b
Showing 1 changed file with 7 additions and 1 deletion.
8 changes: 7 additions & 1 deletion wikihow2zim/scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ class DomIntegrityError(Exception):

class wikihow2zim(GlobalMixin):
def __init__(self, **kwargs):

Global.conf = Conf(**kwargs)
for option in Global.conf.required:
if getattr(Global.conf, option) is None:
Expand Down Expand Up @@ -326,11 +325,18 @@ def build_expected_categories(self):

while query_list:
item = query_list.pop()
logger.debug(
"Querying categorymembers of "
f"{self.metadata['category_prefix']}:{item}"
)
for query in self.api_site.query(
list="categorymembers",
cmtitle=f"{self.metadata['category_prefix']}:{item}",
cmtype="subcat",
):
logger.debug(
f"categorymembers returned: {query.get('categorymembers')}"
)
for cat_member in query.get("categorymembers"):
# only interested in (sub)categories
if cat_member.get("ns") != 14:
Expand Down

0 comments on commit 15a471b

Please sign in to comment.