Adds wapo/v4 support

ir_datasets/datasets/wapo.py

@@ -1,7 +1,7 @@
 import io
 import json
 import tarfile
-from typing import NamedTuple, Tuple
+from typing import NamedTuple, Tuple, Optional
 import ir_datasets
 from ir_datasets.indices import PickleLz4FullStore
 from ir_datasets.util import Lazy, DownloadConfig, Migrator
@@ -46,7 +46,7 @@ class WapoDoc(NamedTuple):
     url: str
     title: str
     author: str
-    published_date: int
+    published_date: Optional[int]
     kicker: str
     body: str
     body_paras_html: Tuple[str, ...]
@@ -120,7 +120,7 @@ def _docs_iter(self):
                 doc_json['article_url'],
                 doc_json['title'],
                 doc_json['author'],
-                doc_json['published_date'],
+                doc_json.get('published_date'),
                 kicker.rstrip('\n'),
                 body.rstrip('\n'),
                 tuple(body_paras_html),
@@ -195,9 +195,9 @@ def _init():
         TrecQrels(dlc['trec-news-2020/qrels'], BL_QREL_DEFS),
         documentation('v3/trec-news-2020'))
 
-    #subsets['v4'] = Dataset(
-    #    collection_v4,
-    #    documentation('v4'))
+    subsets['v4'] = Dataset(
+       collection_v4,
+       documentation('v4'))
 
     ir_datasets.registry.register(NAME, base)
     for s in sorted(subsets):

ir_datasets/etc/metadata.json

@@ -715,6 +715,7 @@
   "wapo/v2/trec-news-2018": {"docs": {"_ref": "wapo/v2"}, "queries": {"count": 50}, "qrels": {"count": 8508, "fields": {"relevance": {"counts_by_value": {"16": 106, "2": 1189, "0": 6465, "4": 584, "8": 164}}}}},
   "wapo/v2/trec-news-2019": {"docs": {"_ref": "wapo/v2"}, "queries": {"count": 60}, "qrels": {"count": 15655, "fields": {"relevance": {"counts_by_value": {"2": 1677, "0": 12614, "8": 431, "16": 273, "4": 660}}}}},
   "wapo/v3/trec-news-2020": {"queries": {"count": 50}, "qrels": {"count": 17764, "fields": {"relevance": {"counts_by_value": {"0": 15348, "2": 1603, "4": 631, "8": 132, "16": 50}}}}},
+  "wapo/v4": {"docs": {"count": 728626, "fields": {"doc_id": {"max_len": 36, "common_prefix": ""}}}},
   "wikiclir": {},
   "wikiclir/ar": {"docs": {"count": 535118, "fields": {"doc_id": {"max_len": 7, "common_prefix": ""}}}, "queries": {"count": 324489}, "qrels": {"count": 519269, "fields": {"relevance": {"counts_by_value": {"2": 324475, "1": 194794}}}}},
   "wikiclir/ca": {"docs": {"count": 548722, "fields": {"doc_id": {"max_len": 7, "common_prefix": ""}}}, "queries": {"count": 339586}, "qrels": {"count": 965233, "fields": {"relevance": {"counts_by_value": {"2": 339562, "1": 625671}}}}},

test/integration/base.py

@@ -276,9 +276,9 @@ def _replace_regex_namedtuple(self, tup, maxlen=200):
                 count = len(value) - maxlen
                 pattern = b'^' + re.escape(value[:maxlen//2]) + (b'.{%i}' % count) + re.escape(value[-(maxlen//2):]) + b'$'
                 result.append(re.compile(pattern, re.DOTALL))
-            elif isinstance(value, tuple) and isinstance(value[0], tuple):
+            elif isinstance(value, tuple) and len(value) > 0 and isinstance(value[0], tuple):
                 result.append(tuple(self._replace_regex_namedtuple(t) for t in value))
-            elif isinstance(value, list) and isinstance(value[0], tuple):
+            elif isinstance(value, list) and len(value) > 0 and isinstance(value[0], tuple):
                 result.append(list(self._replace_regex_namedtuple(t) for t in value))
             else:
                 result.append(value)