From 8587d309175485154b476a2fbf886adca41a1f26 Mon Sep 17 00:00:00 2001 From: Daniel Simmons-Ritchie <37225902+SimmonsRitchie@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:13:57 -0500 Subject: [PATCH 1/2] Ignore CSVs --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dc864e5..37c3382 100644 --- a/.gitignore +++ b/.gitignore @@ -133,4 +133,5 @@ src/ logs/*.log travis/*.json -# output files: local gitignore added to city_scrapers/local_outputs/ +# output files +*.csv From dc2c98a87a64ea00815f548789dbe8bd6b633cc5 Mon Sep 17 00:00:00 2001 From: Daniel Simmons-Ritchie <37225902+SimmonsRitchie@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:13:34 -0500 Subject: [PATCH 2/2] Build spider: bisnd_mcc --- Pipfile | 1 + Pipfile.lock | 492 +++++---- city_scrapers/spiders/bisnd_mcc.py | 100 ++ tests/files/bisnd_mcc.html | 1658 ++++++++++++++++++++++++++++ tests/test_bisnd_mcc.py | 79 ++ 5 files changed, 2089 insertions(+), 241 deletions(-) create mode 100644 city_scrapers/spiders/bisnd_mcc.py create mode 100644 tests/files/bisnd_mcc.html create mode 100644 tests/test_bisnd_mcc.py diff --git a/Pipfile b/Pipfile index 25ff127..867c2ec 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ scrapy = "*" scrapy-sentry-errors = "1.0.0" city-scrapers-core = {ref = "main", git = "https://github.com/City-Bureau/city-scrapers-core.git", extras = ["azure"]} scrapy-wayback-middleware = "*" +python-dateutil = "*" [dev-packages] freezegun = "*" diff --git a/Pipfile.lock b/Pipfile.lock index bac1595..c802fe5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "aaa944d650a40551e49d1059e9bdba1fac4aa964714caaaef622b6c8b8035f87" + "sha256": "5a008f41025a10bfa7f5526a8e9090d3b02bad38ba77d5494e15eb629bf5f21a" }, "pipfile-spec": 6, "requires": { @@ -33,18 +33,18 @@ }, "azure-core": { "hashes": [ - "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89", - "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35" + "sha256:26273a254131f84269e8ea4464f3560c731f29c0c1f69ac99010845f239c1a8f", + "sha256:7c5ee397e48f281ec4dd773d67a0a47a0962ed6fa833036057f9ea067f688e74" ], "markers": "python_version >= '3.7'", - "version": "==1.30.0" + "version": "==1.30.1" }, "azure-storage-blob": { "hashes": [ - "sha256:26c0a4320a34a3c2a1b74528ba6812ebcb632a04cd67b1c7377232c4b01a5897", - "sha256:7bbc2c9c16678f7a420367fef6b172ba8730a7e66df7f4d7a55d5b3c8216615b" + "sha256:13e16ba42fc54ac2c7e8f976062173a5c82b9ec0594728e134aac372965a11b0", + "sha256:c5530dc51c21c9564e4eb706cd499befca8819b10dd89716d3fc90d747556243" ], - "version": "==12.19.0" + "version": "==12.19.1" }, "certifi": { "hashes": [ @@ -226,41 +226,41 @@ }, "cryptography": { "hashes": [ - "sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380", - "sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589", - "sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea", - "sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65", - "sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a", - "sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3", - "sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008", - "sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1", - "sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2", - "sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635", - "sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2", - "sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90", - "sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee", - "sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a", - "sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242", - "sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12", - "sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2", - "sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d", - "sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be", - "sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee", - "sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6", - "sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529", - "sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929", - "sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1", - "sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6", - "sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a", - "sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446", - "sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9", - "sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888", - "sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4", - "sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33", - "sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f" + "sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee", + "sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576", + "sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d", + "sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30", + "sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413", + "sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb", + "sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da", + "sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4", + "sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd", + "sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc", + "sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8", + "sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1", + "sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc", + "sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e", + "sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8", + "sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940", + "sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400", + "sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7", + "sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16", + "sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278", + "sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74", + "sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec", + "sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1", + "sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2", + "sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c", + "sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922", + "sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a", + "sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6", + "sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1", + "sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e", + "sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac", + "sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7" ], "markers": "python_version >= '3.7'", - "version": "==42.0.2" + "version": "==42.0.5" }, "cssselect": { "hashes": [ @@ -433,11 +433,11 @@ }, "packaging": { "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" ], "markers": "python_version >= '3.7'", - "version": "==23.2" + "version": "==24.0" }, "parsel": { "hashes": [ @@ -488,11 +488,20 @@ }, "pyopenssl": { "hashes": [ - "sha256:6aa33039a93fffa4563e655b61d11364d01264be8ccb49906101e02a334530bf", - "sha256:ba07553fb6fd6a7a2259adb9b84e12302a9a8a75c44046e8bb5d3e5ee887e3c3" + "sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad", + "sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f" ], "markers": "python_version >= '3.7'", - "version": "==24.0.0" + "version": "==24.1.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.9.0.post0" }, "pytz": { "hashes": [ @@ -534,117 +543,117 @@ }, "rpds-py": { "hashes": [ - "sha256:01f58a7306b64e0a4fe042047dd2b7d411ee82e54240284bab63e325762c1147", - "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7", - "sha256:02866e060219514940342a1f84303a1ef7a1dad0ac311792fbbe19b521b489d2", - "sha256:0387ce69ba06e43df54e43968090f3626e231e4bc9150e4c3246947567695f68", - "sha256:060f412230d5f19fc8c8b75f315931b408d8ebf56aec33ef4168d1b9e54200b1", - "sha256:071bc28c589b86bc6351a339114fb7a029f5cddbaca34103aa573eba7b482382", - "sha256:0bfb09bf41fe7c51413f563373e5f537eaa653d7adc4830399d4e9bdc199959d", - "sha256:10162fe3f5f47c37ebf6d8ff5a2368508fe22007e3077bf25b9c7d803454d921", - "sha256:149c5cd24f729e3567b56e1795f74577aa3126c14c11e457bec1b1c90d212e38", - "sha256:1701fc54460ae2e5efc1dd6350eafd7a760f516df8dbe51d4a1c79d69472fbd4", - "sha256:1957a2ab607f9added64478a6982742eb29f109d89d065fa44e01691a20fc20a", - "sha256:1a746a6d49665058a5896000e8d9d2f1a6acba8a03b389c1e4c06e11e0b7f40d", - "sha256:1bfcad3109c1e5ba3cbe2f421614e70439f72897515a96c462ea657261b96518", - "sha256:1d36b2b59e8cc6e576f8f7b671e32f2ff43153f0ad6d0201250a7c07f25d570e", - "sha256:1db228102ab9d1ff4c64148c96320d0be7044fa28bd865a9ce628ce98da5973d", - "sha256:1dc29db3900cb1bb40353772417800f29c3d078dbc8024fd64655a04ee3c4bdf", - "sha256:1e626b365293a2142a62b9a614e1f8e331b28f3ca57b9f05ebbf4cf2a0f0bdc5", - "sha256:1f3c3461ebb4c4f1bbc70b15d20b565759f97a5aaf13af811fcefc892e9197ba", - "sha256:20de7b7179e2031a04042e85dc463a93a82bc177eeba5ddd13ff746325558aa6", - "sha256:24e4900a6643f87058a27320f81336d527ccfe503984528edde4bb660c8c8d59", - "sha256:2528ff96d09f12e638695f3a2e0c609c7b84c6df7c5ae9bfeb9252b6fa686253", - "sha256:25f071737dae674ca8937a73d0f43f5a52e92c2d178330b4c0bb6ab05586ffa6", - "sha256:270987bc22e7e5a962b1094953ae901395e8c1e1e83ad016c5cfcfff75a15a3f", - "sha256:292f7344a3301802e7c25c53792fae7d1593cb0e50964e7bcdcc5cf533d634e3", - "sha256:2953937f83820376b5979318840f3ee47477d94c17b940fe31d9458d79ae7eea", - "sha256:2a792b2e1d3038daa83fa474d559acfd6dc1e3650ee93b2662ddc17dbff20ad1", - "sha256:2a7b2f2f56a16a6d62e55354dd329d929560442bd92e87397b7a9586a32e3e76", - "sha256:2f4eb548daf4836e3b2c662033bfbfc551db58d30fd8fe660314f86bf8510b93", - "sha256:3664d126d3388a887db44c2e293f87d500c4184ec43d5d14d2d2babdb4c64cad", - "sha256:3677fcca7fb728c86a78660c7fb1b07b69b281964673f486ae72860e13f512ad", - "sha256:380e0df2e9d5d5d339803cfc6d183a5442ad7ab3c63c2a0982e8c824566c5ccc", - "sha256:3ac732390d529d8469b831949c78085b034bff67f584559340008d0f6041a049", - "sha256:4128980a14ed805e1b91a7ed551250282a8ddf8201a4e9f8f5b7e6225f54170d", - "sha256:4341bd7579611cf50e7b20bb8c2e23512a3dc79de987a1f411cb458ab670eb90", - "sha256:436474f17733c7dca0fbf096d36ae65277e8645039df12a0fa52445ca494729d", - "sha256:4dc889a9d8a34758d0fcc9ac86adb97bab3fb7f0c4d29794357eb147536483fd", - "sha256:4e21b76075c01d65d0f0f34302b5a7457d95721d5e0667aea65e5bb3ab415c25", - "sha256:516fb8c77805159e97a689e2f1c80655c7658f5af601c34ffdb916605598cda2", - "sha256:5576ee2f3a309d2bb403ec292d5958ce03953b0e57a11d224c1f134feaf8c40f", - "sha256:5a024fa96d541fd7edaa0e9d904601c6445e95a729a2900c5aec6555fe921ed6", - "sha256:5d0e8a6434a3fbf77d11448c9c25b2f25244226cfbec1a5159947cac5b8c5fa4", - "sha256:5e7d63ec01fe7c76c2dbb7e972fece45acbb8836e72682bde138e7e039906e2c", - "sha256:60e820ee1004327609b28db8307acc27f5f2e9a0b185b2064c5f23e815f248f8", - "sha256:637b802f3f069a64436d432117a7e58fab414b4e27a7e81049817ae94de45d8d", - "sha256:65dcf105c1943cba45d19207ef51b8bc46d232a381e94dd38719d52d3980015b", - "sha256:698ea95a60c8b16b58be9d854c9f993c639f5c214cf9ba782eca53a8789d6b19", - "sha256:70fcc6c2906cfa5c6a552ba7ae2ce64b6c32f437d8f3f8eea49925b278a61453", - "sha256:720215373a280f78a1814becb1312d4e4d1077b1202a56d2b0815e95ccb99ce9", - "sha256:7450dbd659fed6dd41d1a7d47ed767e893ba402af8ae664c157c255ec6067fde", - "sha256:7b7d9ca34542099b4e185b3c2a2b2eda2e318a7dbde0b0d83357a6d4421b5296", - "sha256:7fbd70cb8b54fe745301921b0816c08b6d917593429dfc437fd024b5ba713c58", - "sha256:81038ff87a4e04c22e1d81f947c6ac46f122e0c80460b9006e6517c4d842a6ec", - "sha256:810685321f4a304b2b55577c915bece4c4a06dfe38f6e62d9cc1d6ca8ee86b99", - "sha256:82ada4a8ed9e82e443fcef87e22a3eed3654dd3adf6e3b3a0deb70f03e86142a", - "sha256:841320e1841bb53fada91c9725e766bb25009cfd4144e92298db296fb6c894fb", - "sha256:8587fd64c2a91c33cdc39d0cebdaf30e79491cc029a37fcd458ba863f8815383", - "sha256:8ffe53e1d8ef2520ebcf0c9fec15bb721da59e8ef283b6ff3079613b1e30513d", - "sha256:9051e3d2af8f55b42061603e29e744724cb5f65b128a491446cc029b3e2ea896", - "sha256:91e5a8200e65aaac342a791272c564dffcf1281abd635d304d6c4e6b495f29dc", - "sha256:93432e747fb07fa567ad9cc7aaadd6e29710e515aabf939dfbed8046041346c6", - "sha256:938eab7323a736533f015e6069a7d53ef2dcc841e4e533b782c2bfb9fb12d84b", - "sha256:9584f8f52010295a4a417221861df9bea4c72d9632562b6e59b3c7b87a1522b7", - "sha256:9737bdaa0ad33d34c0efc718741abaafce62fadae72c8b251df9b0c823c63b22", - "sha256:99da0a4686ada4ed0f778120a0ea8d066de1a0a92ab0d13ae68492a437db78bf", - "sha256:99f567dae93e10be2daaa896e07513dd4bf9c2ecf0576e0533ac36ba3b1d5394", - "sha256:9bdf1303df671179eaf2cb41e8515a07fc78d9d00f111eadbe3e14262f59c3d0", - "sha256:9f0e4dc0f17dcea4ab9d13ac5c666b6b5337042b4d8f27e01b70fae41dd65c57", - "sha256:a000133a90eea274a6f28adc3084643263b1e7c1a5a66eb0a0a7a36aa757ed74", - "sha256:a3264e3e858de4fc601741498215835ff324ff2482fd4e4af61b46512dd7fc83", - "sha256:a71169d505af63bb4d20d23a8fbd4c6ce272e7bce6cc31f617152aa784436f29", - "sha256:a967dd6afda7715d911c25a6ba1517975acd8d1092b2f326718725461a3d33f9", - "sha256:aa5bfb13f1e89151ade0eb812f7b0d7a4d643406caaad65ce1cbabe0a66d695f", - "sha256:ae35e8e6801c5ab071b992cb2da958eee76340e6926ec693b5ff7d6381441745", - "sha256:b686f25377f9c006acbac63f61614416a6317133ab7fafe5de5f7dc8a06d42eb", - "sha256:b760a56e080a826c2e5af09002c1a037382ed21d03134eb6294812dda268c811", - "sha256:b86b21b348f7e5485fae740d845c65a880f5d1eda1e063bc59bef92d1f7d0c55", - "sha256:b9412abdf0ba70faa6e2ee6c0cc62a8defb772e78860cef419865917d86c7342", - "sha256:bd345a13ce06e94c753dab52f8e71e5252aec1e4f8022d24d56decd31e1b9b23", - "sha256:be22ae34d68544df293152b7e50895ba70d2a833ad9566932d750d3625918b82", - "sha256:bf046179d011e6114daf12a534d874958b039342b347348a78b7cdf0dd9d6041", - "sha256:c3d2010656999b63e628a3c694f23020322b4178c450dc478558a2b6ef3cb9bb", - "sha256:c64602e8be701c6cfe42064b71c84ce62ce66ddc6422c15463fd8127db3d8066", - "sha256:d65e6b4f1443048eb7e833c2accb4fa7ee67cc7d54f31b4f0555b474758bee55", - "sha256:d8bbd8e56f3ba25a7d0cf980fc42b34028848a53a0e36c9918550e0280b9d0b6", - "sha256:da1ead63368c04a9bded7904757dfcae01eba0e0f9bc41d3d7f57ebf1c04015a", - "sha256:dbbb95e6fc91ea3102505d111b327004d1c4ce98d56a4a02e82cd451f9f57140", - "sha256:dbc56680ecf585a384fbd93cd42bc82668b77cb525343170a2d86dafaed2a84b", - "sha256:df3b6f45ba4515632c5064e35ca7f31d51d13d1479673185ba8f9fefbbed58b9", - "sha256:dfe07308b311a8293a0d5ef4e61411c5c20f682db6b5e73de6c7c8824272c256", - "sha256:e796051f2070f47230c745d0a77a91088fbee2cc0502e9b796b9c6471983718c", - "sha256:efa767c220d94aa4ac3a6dd3aeb986e9f229eaf5bce92d8b1b3018d06bed3772", - "sha256:f0b8bf5b8db49d8fd40f54772a1dcf262e8be0ad2ab0206b5a2ec109c176c0a4", - "sha256:f175e95a197f6a4059b50757a3dca33b32b61691bdbd22c29e8a8d21d3914cae", - "sha256:f2f3b28b40fddcb6c1f1f6c88c6f3769cd933fa493ceb79da45968a21dccc920", - "sha256:f6c43b6f97209e370124baf2bf40bb1e8edc25311a158867eb1c3a5d449ebc7a", - "sha256:f7f4cb1f173385e8a39c29510dd11a78bf44e360fb75610594973f5ea141028b", - "sha256:fad059a4bd14c45776600d223ec194e77db6c20255578bb5bcdd7c18fd169361", - "sha256:ff1dcb8e8bc2261a088821b2595ef031c91d499a0c1b031c152d43fe0a6ecec8", - "sha256:ffee088ea9b593cc6160518ba9bd319b5475e5f3e578e4552d63818773c6f56a" + "sha256:01e36a39af54a30f28b73096dd39b6802eddd04c90dbe161c1b8dbe22353189f", + "sha256:044a3e61a7c2dafacae99d1e722cc2d4c05280790ec5a05031b3876809d89a5c", + "sha256:08231ac30a842bd04daabc4d71fddd7e6d26189406d5a69535638e4dcb88fe76", + "sha256:08f9ad53c3f31dfb4baa00da22f1e862900f45908383c062c27628754af2e88e", + "sha256:0ab39c1ba9023914297dd88ec3b3b3c3f33671baeb6acf82ad7ce883f6e8e157", + "sha256:0af039631b6de0397ab2ba16eaf2872e9f8fca391b44d3d8cac317860a700a3f", + "sha256:0b8612cd233543a3781bc659c731b9d607de65890085098986dfd573fc2befe5", + "sha256:11a8c85ef4a07a7638180bf04fe189d12757c696eb41f310d2426895356dcf05", + "sha256:1374f4129f9bcca53a1bba0bb86bf78325a0374577cf7e9e4cd046b1e6f20e24", + "sha256:1d4acf42190d449d5e89654d5c1ed3a4f17925eec71f05e2a41414689cda02d1", + "sha256:1d9a5be316c15ffb2b3c405c4ff14448c36b4435be062a7f578ccd8b01f0c4d8", + "sha256:1df3659d26f539ac74fb3b0c481cdf9d725386e3552c6fa2974f4d33d78e544b", + "sha256:22806714311a69fd0af9b35b7be97c18a0fc2826e6827dbb3a8c94eac6cf7eeb", + "sha256:2644e47de560eb7bd55c20fc59f6daa04682655c58d08185a9b95c1970fa1e07", + "sha256:2e6d75ab12b0bbab7215e5d40f1e5b738aa539598db27ef83b2ec46747df90e1", + "sha256:30f43887bbae0d49113cbaab729a112251a940e9b274536613097ab8b4899cf6", + "sha256:34b18ba135c687f4dac449aa5157d36e2cbb7c03cbea4ddbd88604e076aa836e", + "sha256:36b3ee798c58ace201289024b52788161e1ea133e4ac93fba7d49da5fec0ef9e", + "sha256:39514da80f971362f9267c600b6d459bfbbc549cffc2cef8e47474fddc9b45b1", + "sha256:39f5441553f1c2aed4de4377178ad8ff8f9d733723d6c66d983d75341de265ab", + "sha256:3a96e0c6a41dcdba3a0a581bbf6c44bb863f27c541547fb4b9711fd8cf0ffad4", + "sha256:3f26b5bd1079acdb0c7a5645e350fe54d16b17bfc5e71f371c449383d3342e17", + "sha256:41ef53e7c58aa4ef281da975f62c258950f54b76ec8e45941e93a3d1d8580594", + "sha256:42821446ee7a76f5d9f71f9e33a4fb2ffd724bb3e7f93386150b61a43115788d", + "sha256:43fbac5f22e25bee1d482c97474f930a353542855f05c1161fd804c9dc74a09d", + "sha256:4457a94da0d5c53dc4b3e4de1158bdab077db23c53232f37a3cb7afdb053a4e3", + "sha256:465a3eb5659338cf2a9243e50ad9b2296fa15061736d6e26240e713522b6235c", + "sha256:482103aed1dfe2f3b71a58eff35ba105289b8d862551ea576bd15479aba01f66", + "sha256:4832d7d380477521a8c1644bbab6588dfedea5e30a7d967b5fb75977c45fd77f", + "sha256:4901165d170a5fde6f589acb90a6b33629ad1ec976d4529e769c6f3d885e3e80", + "sha256:5307def11a35f5ae4581a0b658b0af8178c65c530e94893345bebf41cc139d33", + "sha256:5417558f6887e9b6b65b4527232553c139b57ec42c64570569b155262ac0754f", + "sha256:56a737287efecafc16f6d067c2ea0117abadcd078d58721f967952db329a3e5c", + "sha256:586f8204935b9ec884500498ccc91aa869fc652c40c093bd9e1471fbcc25c022", + "sha256:5b4e7d8d6c9b2e8ee2d55c90b59c707ca59bc30058269b3db7b1f8df5763557e", + "sha256:5ddcba87675b6d509139d1b521e0c8250e967e63b5909a7e8f8944d0f90ff36f", + "sha256:618a3d6cae6ef8ec88bb76dd80b83cfe415ad4f1d942ca2a903bf6b6ff97a2da", + "sha256:635dc434ff724b178cb192c70016cc0ad25a275228f749ee0daf0eddbc8183b1", + "sha256:661d25cbffaf8cc42e971dd570d87cb29a665f49f4abe1f9e76be9a5182c4688", + "sha256:66e6a3af5a75363d2c9a48b07cb27c4ea542938b1a2e93b15a503cdfa8490795", + "sha256:67071a6171e92b6da534b8ae326505f7c18022c6f19072a81dcf40db2638767c", + "sha256:685537e07897f173abcf67258bee3c05c374fa6fff89d4c7e42fb391b0605e98", + "sha256:69e64831e22a6b377772e7fb337533c365085b31619005802a79242fee620bc1", + "sha256:6b0817e34942b2ca527b0e9298373e7cc75f429e8da2055607f4931fded23e20", + "sha256:6c81e5f372cd0dc5dc4809553d34f832f60a46034a5f187756d9b90586c2c307", + "sha256:6d7faa6f14017c0b1e69f5e2c357b998731ea75a442ab3841c0dbbbfe902d2c4", + "sha256:6ef0befbb5d79cf32d0266f5cff01545602344eda89480e1dd88aca964260b18", + "sha256:6ef687afab047554a2d366e112dd187b62d261d49eb79b77e386f94644363294", + "sha256:7223a2a5fe0d217e60a60cdae28d6949140dde9c3bcc714063c5b463065e3d66", + "sha256:77f195baa60a54ef9d2de16fbbfd3ff8b04edc0c0140a761b56c267ac11aa467", + "sha256:793968759cd0d96cac1e367afd70c235867831983f876a53389ad869b043c948", + "sha256:7bd339195d84439cbe5771546fe8a4e8a7a045417d8f9de9a368c434e42a721e", + "sha256:7cd863afe7336c62ec78d7d1349a2f34c007a3cc6c2369d667c65aeec412a5b1", + "sha256:7f2facbd386dd60cbbf1a794181e6aa0bd429bd78bfdf775436020172e2a23f0", + "sha256:84ffab12db93b5f6bad84c712c92060a2d321b35c3c9960b43d08d0f639d60d7", + "sha256:8c8370641f1a7f0e0669ddccca22f1da893cef7628396431eb445d46d893e5cd", + "sha256:8db715ebe3bb7d86d77ac1826f7d67ec11a70dbd2376b7cc214199360517b641", + "sha256:8e8916ae4c720529e18afa0b879473049e95949bf97042e938530e072fde061d", + "sha256:8f03bccbd8586e9dd37219bce4d4e0d3ab492e6b3b533e973fa08a112cb2ffc9", + "sha256:8f2fc11e8fe034ee3c34d316d0ad8808f45bc3b9ce5857ff29d513f3ff2923a1", + "sha256:923d39efa3cfb7279a0327e337a7958bff00cc447fd07a25cddb0a1cc9a6d2da", + "sha256:93df1de2f7f7239dc9cc5a4a12408ee1598725036bd2dedadc14d94525192fc3", + "sha256:998e33ad22dc7ec7e030b3df701c43630b5bc0d8fbc2267653577e3fec279afa", + "sha256:99f70b740dc04d09e6b2699b675874367885217a2e9f782bdf5395632ac663b7", + "sha256:9a00312dea9310d4cb7dbd7787e722d2e86a95c2db92fbd7d0155f97127bcb40", + "sha256:9d54553c1136b50fd12cc17e5b11ad07374c316df307e4cfd6441bea5fb68496", + "sha256:9dbbeb27f4e70bfd9eec1be5477517365afe05a9b2c441a0b21929ee61048124", + "sha256:a1ce3ba137ed54f83e56fb983a5859a27d43a40188ba798993812fed73c70836", + "sha256:a34d557a42aa28bd5c48a023c570219ba2593bcbbb8dc1b98d8cf5d529ab1434", + "sha256:a5f446dd5055667aabaee78487f2b5ab72e244f9bc0b2ffebfeec79051679984", + "sha256:ad36cfb355e24f1bd37cac88c112cd7730873f20fb0bdaf8ba59eedf8216079f", + "sha256:aec493917dd45e3c69d00a8874e7cbed844efd935595ef78a0f25f14312e33c6", + "sha256:b316144e85316da2723f9d8dc75bada12fa58489a527091fa1d5a612643d1a0e", + "sha256:b34ae4636dfc4e76a438ab826a0d1eed2589ca7d9a1b2d5bb546978ac6485461", + "sha256:b34b7aa8b261c1dbf7720b5d6f01f38243e9b9daf7e6b8bc1fd4657000062f2c", + "sha256:bc362ee4e314870a70f4ae88772d72d877246537d9f8cb8f7eacf10884862432", + "sha256:bed88b9a458e354014d662d47e7a5baafd7ff81c780fd91584a10d6ec842cb73", + "sha256:c0013fe6b46aa496a6749c77e00a3eb07952832ad6166bd481c74bda0dcb6d58", + "sha256:c0b5dcf9193625afd8ecc92312d6ed78781c46ecbf39af9ad4681fc9f464af88", + "sha256:c4325ff0442a12113a6379af66978c3fe562f846763287ef66bdc1d57925d337", + "sha256:c463ed05f9dfb9baebef68048aed8dcdc94411e4bf3d33a39ba97e271624f8f7", + "sha256:c8362467a0fdeccd47935f22c256bec5e6abe543bf0d66e3d3d57a8fb5731863", + "sha256:cd5bf1af8efe569654bbef5a3e0a56eca45f87cfcffab31dd8dde70da5982475", + "sha256:cf1ea2e34868f6fbf070e1af291c8180480310173de0b0c43fc38a02929fc0e3", + "sha256:d62dec4976954a23d7f91f2f4530852b0c7608116c257833922a896101336c51", + "sha256:d68c93e381010662ab873fea609bf6c0f428b6d0bb00f2c6939782e0818d37bf", + "sha256:d7c36232a90d4755b720fbd76739d8891732b18cf240a9c645d75f00639a9024", + "sha256:dd18772815d5f008fa03d2b9a681ae38d5ae9f0e599f7dda233c439fcaa00d40", + "sha256:ddc2f4dfd396c7bfa18e6ce371cba60e4cf9d2e5cdb71376aa2da264605b60b9", + "sha256:e003b002ec72c8d5a3e3da2989c7d6065b47d9eaa70cd8808b5384fbb970f4ec", + "sha256:e32a92116d4f2a80b629778280103d2a510a5b3f6314ceccd6e38006b5e92dcb", + "sha256:e4461d0f003a0aa9be2bdd1b798a041f177189c1a0f7619fe8c95ad08d9a45d7", + "sha256:e541ec6f2ec456934fd279a3120f856cd0aedd209fc3852eca563f81738f6861", + "sha256:e546e768d08ad55b20b11dbb78a745151acbd938f8f00d0cfbabe8b0199b9880", + "sha256:ea7d4a99f3b38c37eac212dbd6ec42b7a5ec51e2c74b5d3223e43c811609e65f", + "sha256:ed4eb745efbff0a8e9587d22a84be94a5eb7d2d99c02dacf7bd0911713ed14dd", + "sha256:f8a2f084546cc59ea99fda8e070be2fd140c3092dc11524a71aa8f0f3d5a55ca", + "sha256:fcb25daa9219b4cf3a0ab24b0eb9a5cc8949ed4dc72acb8fa16b7e1681aa3c58", + "sha256:fdea4952db2793c4ad0bdccd27c1d8fdd1423a92f04598bc39425bcc2b8ee46e" ], "markers": "python_version >= '3.8'", - "version": "==0.17.1" + "version": "==0.18.0" }, "scrapy": { "hashes": [ - "sha256:3cbdedce0c3f0e0482d61be2d7458683be7cd7cf14b0ee6adfbaddb80f5b36a5", - "sha256:a7f36544d1f5ceb13cff9b7bc904bd7c0fc43a3af0fbe5aa2034fd937cf092d1" + "sha256:733a039c7423e52b69bf2810b5332093d4e42a848460359c07b02ecff8f73ebe", + "sha256:f1edee0cd214512054c01a8d031a8d213dddb53492b02c9e66256e3efe90d175" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==2.11.0" + "version": "==2.11.1" }, "scrapy-sentry-errors": { "hashes": [ @@ -661,15 +670,15 @@ "sha256:fd2b8500d64b909289e3541c201b4d672c0e7b458fc20e77bb37f0d71d93a75a" ], "index": "pypi", - "markers": "python_full_version >= '3.6.1' and python_version < '4.0'", + "markers": "python_version < '4.0' and python_full_version >= '3.6.1'", "version": "==0.3.3" }, "sentry-sdk": { "hashes": [ - "sha256:1bb9cf4ac317906d20787693b5e7f3e42160a90e8bbf1fc544f91c52fa76b68f", - "sha256:69fc5e7512371547207821d801485f45e3c62db629f02f56f58431a10864ac34" + "sha256:4a8364b8f7edbf47f95f7163e48334c96100d9c098f0ae6606e2e18183c223e6", + "sha256:a654ee7e497a3f5f6368b36d4f04baeab1fe92b3105f7f6965d6ef0de35a9ba4" ], - "version": "==1.40.1" + "version": "==1.42.0" }, "service-identity": { "hashes": [ @@ -681,18 +690,18 @@ }, "setuptools": { "hashes": [ - "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05", - "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78" + "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e", + "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c" ], "markers": "python_version >= '3.8'", - "version": "==69.0.3" + "version": "==69.2.0" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "tldextract": { @@ -705,27 +714,27 @@ }, "twisted": { "hashes": [ - "sha256:32acbd40a94f5f46e7b42c109bfae2b302250945561783a8b7a059048f2d4d31", - "sha256:86c55f712cc5ab6f6d64e02503352464f0400f66d4f079096d744080afcccbd0" + "sha256:039f2e6a49ab5108abd94de187fa92377abe5985c7a72d68d0ad266ba19eae63", + "sha256:6b38b6ece7296b5e122c9eb17da2eeab3d98a198f50ca9efd00fb03e5b4fd4ae" ], - "markers": "python_full_version >= '3.7.1'", - "version": "==22.10.0" + "markers": "python_full_version >= '3.8.0'", + "version": "==24.3.0" }, "typing-extensions": { "hashes": [ - "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", - "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" + "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475", + "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb" ], "markers": "python_version >= '3.8'", - "version": "==4.9.0" + "version": "==4.10.0" }, "urllib3": { "hashes": [ - "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20", - "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224" + "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d", + "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19" ], "markers": "python_version >= '3.6'", - "version": "==2.2.0" + "version": "==2.2.1" }, "w3lib": { "hashes": [ @@ -737,76 +746,76 @@ }, "zope.interface": { "hashes": [ - "sha256:0c8cf55261e15590065039696607f6c9c1aeda700ceee40c70478552d323b3ff", - "sha256:13b7d0f2a67eb83c385880489dbb80145e9d344427b4262c49fbf2581677c11c", - "sha256:1f294a15f7723fc0d3b40701ca9b446133ec713eafc1cc6afa7b3d98666ee1ac", - "sha256:239a4a08525c080ff833560171d23b249f7f4d17fcbf9316ef4159f44997616f", - "sha256:2f8d89721834524a813f37fa174bac074ec3d179858e4ad1b7efd4401f8ac45d", - "sha256:2fdc7ccbd6eb6b7df5353012fbed6c3c5d04ceaca0038f75e601060e95345309", - "sha256:34c15ca9248f2e095ef2e93af2d633358c5f048c49fbfddf5fdfc47d5e263736", - "sha256:387545206c56b0315fbadb0431d5129c797f92dc59e276b3ce82db07ac1c6179", - "sha256:43b576c34ef0c1f5a4981163b551a8781896f2a37f71b8655fd20b5af0386abb", - "sha256:57d0a8ce40ce440f96a2c77824ee94bf0d0925e6089df7366c2272ccefcb7941", - "sha256:5a804abc126b33824a44a7aa94f06cd211a18bbf31898ba04bd0924fbe9d282d", - "sha256:67be3ca75012c6e9b109860820a8b6c9a84bfb036fbd1076246b98e56951ca92", - "sha256:6af47f10cfc54c2ba2d825220f180cc1e2d4914d783d6fc0cd93d43d7bc1c78b", - "sha256:6dc998f6de015723196a904045e5a2217f3590b62ea31990672e31fbc5370b41", - "sha256:70d2cef1bf529bff41559be2de9d44d47b002f65e17f43c73ddefc92f32bf00f", - "sha256:7ebc4d34e7620c4f0da7bf162c81978fce0ea820e4fa1e8fc40ee763839805f3", - "sha256:964a7af27379ff4357dad1256d9f215047e70e93009e532d36dcb8909036033d", - "sha256:97806e9ca3651588c1baaebb8d0c5ee3db95430b612db354c199b57378312ee8", - "sha256:9b9bc671626281f6045ad61d93a60f52fd5e8209b1610972cf0ef1bbe6d808e3", - "sha256:9ffdaa5290422ac0f1688cb8adb1b94ca56cee3ad11f29f2ae301df8aecba7d1", - "sha256:a0da79117952a9a41253696ed3e8b560a425197d4e41634a23b1507efe3273f1", - "sha256:a41f87bb93b8048fe866fa9e3d0c51e27fe55149035dcf5f43da4b56732c0a40", - "sha256:aa6fd016e9644406d0a61313e50348c706e911dca29736a3266fc9e28ec4ca6d", - "sha256:ad54ed57bdfa3254d23ae04a4b1ce405954969c1b0550cc2d1d2990e8b439de1", - "sha256:b012d023b4fb59183909b45d7f97fb493ef7a46d2838a5e716e3155081894605", - "sha256:b51b64432eed4c0744241e9ce5c70dcfecac866dff720e746d0a9c82f371dfa7", - "sha256:bbe81def9cf3e46f16ce01d9bfd8bea595e06505e51b7baf45115c77352675fd", - "sha256:c9559138690e1bd4ea6cd0954d22d1e9251e8025ce9ede5d0af0ceae4a401e43", - "sha256:e30506bcb03de8983f78884807e4fd95d8db6e65b69257eea05d13d519b83ac0", - "sha256:e33e86fd65f369f10608b08729c8f1c92ec7e0e485964670b4d2633a4812d36b", - "sha256:e441e8b7d587af0414d25e8d05e27040d78581388eed4c54c30c0c91aad3a379", - "sha256:e8bb9c990ca9027b4214fa543fd4025818dc95f8b7abce79d61dc8a2112b561a", - "sha256:ef43ee91c193f827e49599e824385ec7c7f3cd152d74cb1dfe02cb135f264d83", - "sha256:ef467d86d3cfde8b39ea1b35090208b0447caaabd38405420830f7fd85fbdd56", - "sha256:f89b28772fc2562ed9ad871c865f5320ef761a7fcc188a935e21fe8b31a38ca9", - "sha256:fddbab55a2473f1d3b8833ec6b7ac31e8211b0aa608df5ab09ce07f3727326de" + "sha256:02adbab560683c4eca3789cc0ac487dcc5f5a81cc48695ec247f00803cafe2fe", + "sha256:14e02a6fc1772b458ebb6be1c276528b362041217b9ca37e52ecea2cbdce9fac", + "sha256:25e0af9663eeac6b61b231b43c52293c2cb7f0c232d914bdcbfd3e3bd5c182ad", + "sha256:2606955a06c6852a6cff4abeca38346ed01e83f11e960caa9a821b3626a4467b", + "sha256:396f5c94654301819a7f3a702c5830f0ea7468d7b154d124ceac823e2419d000", + "sha256:3b240883fb43160574f8f738e6d09ddbdbf8fa3e8cea051603d9edfd947d9328", + "sha256:3b6c62813c63c543a06394a636978b22dffa8c5410affc9331ce6cdb5bfa8565", + "sha256:4ae9793f114cee5c464cc0b821ae4d36e1eba961542c6086f391a61aee167b6f", + "sha256:4bce517b85f5debe07b186fc7102b332676760f2e0c92b7185dd49c138734b70", + "sha256:4d45d2ba8195850e3e829f1f0016066a122bfa362cc9dc212527fc3d51369037", + "sha256:4dd374927c00764fcd6fe1046bea243ebdf403fba97a937493ae4be2c8912c2b", + "sha256:506f5410b36e5ba494136d9fa04c548eaf1a0d9c442b0b0e7a0944db7620e0ab", + "sha256:59f7374769b326a217d0b2366f1c176a45a4ff21e8f7cebb3b4a3537077eff85", + "sha256:5ee9789a20b0081dc469f65ff6c5007e67a940d5541419ca03ef20c6213dd099", + "sha256:6fc711acc4a1c702ca931fdbf7bf7c86f2a27d564c85c4964772dadf0e3c52f5", + "sha256:75d2ec3d9b401df759b87bc9e19d1b24db73083147089b43ae748aefa63067ef", + "sha256:76e0531d86523be7a46e15d379b0e975a9db84316617c0efe4af8338dc45b80c", + "sha256:8af82afc5998e1f307d5e72712526dba07403c73a9e287d906a8aa2b1f2e33dd", + "sha256:8f5d2c39f3283e461de3655e03faf10e4742bb87387113f787a7724f32db1e48", + "sha256:97785604824981ec8c81850dd25c8071d5ce04717a34296eeac771231fbdd5cd", + "sha256:a3046e8ab29b590d723821d0785598e0b2e32b636a0272a38409be43e3ae0550", + "sha256:abb0b3f2cb606981c7432f690db23506b1db5899620ad274e29dbbbdd740e797", + "sha256:ac7c2046d907e3b4e2605a130d162b1b783c170292a11216479bb1deb7cadebe", + "sha256:af27b3fe5b6bf9cd01b8e1c5ddea0a0d0a1b8c37dc1c7452f1e90bf817539c6d", + "sha256:b386b8b9d2b6a5e1e4eadd4e62335571244cb9193b7328c2b6e38b64cfda4f0e", + "sha256:b66335bbdbb4c004c25ae01cc4a54fd199afbc1fd164233813c6d3c2293bb7e1", + "sha256:d54f66c511ea01b9ef1d1a57420a93fbb9d48a08ec239f7d9c581092033156d0", + "sha256:de125151a53ecdb39df3cb3deb9951ed834dd6a110a9e795d985b10bb6db4532", + "sha256:de7916380abaef4bb4891740879b1afcba2045aee51799dfd6d6ca9bdc71f35f", + "sha256:e2fefad268ff5c5b314794e27e359e48aeb9c8bb2cbb5748a071757a56f6bb8f", + "sha256:e7b2bed4eea047a949296e618552d3fed00632dc1b795ee430289bdd0e3717f3", + "sha256:e87698e2fea5ca2f0a99dff0a64ce8110ea857b640de536c76d92aaa2a91ff3a", + "sha256:ede888382882f07b9e4cd942255921ffd9f2901684198b88e247c7eabd27a000", + "sha256:f444de0565db46d26c9fa931ca14f497900a295bd5eba480fc3fad25af8c763e", + "sha256:fa994e8937e8ccc7e87395b7b35092818905cf27c651e3ff3e7f29729f5ce3ce", + "sha256:febceb04ee7dd2aef08c2ff3d6f8a07de3052fc90137c507b0ede3ea80c21440" ], "markers": "python_version >= '3.7'", - "version": "==6.1" + "version": "==6.2" } }, "develop": { "black": { "hashes": [ - "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8", - "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6", - "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62", - "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445", - "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c", - "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a", - "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9", - "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2", - "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6", - "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b", - "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4", - "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168", - "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d", - "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5", - "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024", - "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e", - "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b", - "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161", - "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717", - "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8", - "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac", - "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7" + "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8", + "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8", + "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd", + "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9", + "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31", + "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92", + "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f", + "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29", + "sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4", + "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693", + "sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218", + "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a", + "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23", + "sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0", + "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982", + "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894", + "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540", + "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430", + "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b", + "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2", + "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6", + "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==24.1.1" + "version": "==24.2.0" }, "click": { "hashes": [ @@ -877,11 +886,11 @@ }, "packaging": { "hashes": [ - "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", - "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" + "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9" ], "markers": "python_version >= '3.7'", - "version": "==23.2" + "version": "==24.0" }, "pathspec": { "hashes": [ @@ -925,27 +934,28 @@ }, "pytest": { "hashes": [ - "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c", - "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6" + "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7", + "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==8.0.0" + "version": "==8.1.1" }, "python-dateutil": { "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==2.8.2" + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.9.0.post0" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "tomli": { @@ -958,11 +968,11 @@ }, "typing-extensions": { "hashes": [ - "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", - "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" + "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475", + "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb" ], "markers": "python_version >= '3.8'", - "version": "==4.9.0" + "version": "==4.10.0" } } } diff --git a/city_scrapers/spiders/bisnd_mcc.py b/city_scrapers/spiders/bisnd_mcc.py new file mode 100644 index 0000000..60d015e --- /dev/null +++ b/city_scrapers/spiders/bisnd_mcc.py @@ -0,0 +1,100 @@ +from datetime import datetime, time + +from city_scrapers_core.constants import COMMISSION +from city_scrapers_core.items import Meeting +from city_scrapers_core.spiders import CityScrapersSpider +from dateutil.parser import parse + + +class BisndMccSpider(CityScrapersSpider): + name = "bisnd_mcc" + agency = "Morton County Commission" + timezone = "America/Chicago" + start_urls = [ + "https://www.mortonnd.org/?Type=B_BASIC&SEC={8182F8B8-3783-4C56-B690-F78FEE7CAC95}" # noqa + ] + title = "Morton County Commission Meeting" + meeting_time = time(17, 30) + location = { + "address": "Morton County Commission Room", + "name": "Morton County Courthouse, 210 2nd Ave NW, Mandan ND", + } + custom_settings = { + "ROBOTSTXT_OBEY": False, + } + + def parse(self, response): + """ + Parse HTML table and extract meeting information. Due to a lack of info, + we hardcode certain fields, like start time and title + """ + for item in response.css("main[role='main'] table tbody tr:not(:first-child)"): + start, description = self._first_col(item) + if start is None: + self.logger.warning("Invalid date format - skipping") + continue + # ignore meetings that are more than a year ago + if start < datetime.now().replace(year=datetime.now().year - 1): + self.logger.info( + f"Skipping meeting from {start} as it is more than a year ago" + ) + continue + meeting = Meeting( + title=self.title, + description=description, + classification=COMMISSION, + start=start, + end=None, + all_day=False, + time_notes="", + location=self.location, + links=self._parse_links(item), + source=response.url, + ) + + meeting["status"] = self._get_status(meeting) + meeting["id"] = self._get_id(meeting) + + yield meeting + + def _first_col(self, item): + """Extract date of the meeting and any additional text from + the first column of the table row. Date of the meeting is in + "Month Day, Year" format.""" + col_str = item.css("td span::text").extract_first() + description = "" + + # Parse text + if col_str is None: + self.logger.info("No text found in date column") + return None, description + col_words = col_str.split() + if len(col_words) < 3: + # date should always be first three words + self.logger.info(f"Invalid date format: {col_words}") + return None, description + date_str = " ".join(col_words[:3]) + if len(col_words) > 3: + # if there's any other text, use that as the description + description = " ".join(col_words[3:]) + + # parse date + try: + parsed_date = parse(date_str) + # combine date and time + parsed_date = datetime.combine(parsed_date.date(), self.meeting_time) + return parsed_date, description + except ValueError: + self.logger.info(f"Invalid date format: {date_str}") + return None, description + + def _parse_links(self, item): + """Parse or generate links.""" + links = [] + for row in item.css("td:not(:first-child) a"): + link = { + "title": row.css("::text").extract_first(), + "href": row.css("::attr(href)").extract_first(), + } + links.append(link) + return links diff --git a/tests/files/bisnd_mcc.html b/tests/files/bisnd_mcc.html new file mode 100644 index 0000000..77c69a5 --- /dev/null +++ b/tests/files/bisnd_mcc.html @@ -0,0 +1,1658 @@ + + + + + + + + + + + + + + + + + + +County Commission - Morton County, North Dakota +   + + + + + + + + + + + +
+
+
+ +
+
+ + + + + + +
+ + +
+		 +
+ +
+

County Commission Meeting Schedule and Materials

+

Morton County Commission Meetings Schedule and Materials

+

The County Commission meets on the 2nd Thursday and the 4th Tuesday of each month (unless otherwise indicated) at 5:30pm, in the Morton County Commission Room, Morton County Courthouse located at 210 2nd Ave NW, Mandan ND.  All meetings are open to the public.  The following meeting schedule includes agenda, packets, and minutes from meetings, as they become available.  Packets are typically posted on the Tuesday prior to a Thursday meeting and on the Friday prior to a Tuesday meeting. 

County Commission meetings are recorded and played at various dates and times on Government Access, cable channel 2, which is available in the Bismarck Mandan area. All past meetings are available to view online at anytime on www.freetv.org.
 

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Minutes
December 26, 2024Agenda Packet Minutes 
December 12, 2024Agenda Packet Minutes
November 26, 2024Agenda Packet Minutes
November 14, 2024Agenda Packet Minutes
October 22, 2024Agenda Packet Minutes
October 10, 2024Agenda Packet Minutes
September 24, 2024Agenda Packet Minutes 
September 12, 2024Agenda Packet Minutes
August 27, 2024Agenda Packet Minutes 
August 8, 2024Agenda Packet Minutes 
July 23, 2024Agenda Packet Minutes
July 11, 2024Agenda Packet Minutes 
June 25, 2024AgendaPacket Minutes 
June 6, 2024AgendaPacket Minutes
May 28, 2024Agenda Packet Minutes 
May 9, 2024Agenda Packet Minutes 
April 23, 2024Agenda Packet Minutes 
April 11, 2024Agenda Packet Minutes 
March 26, 2024Agenda Packet Minutes 
March 14, 2024Agenda Packet Minutes 
February 27, 2024Agenda Packet Minutes 
February 27 2024 LEC Advisory Bd w/ Burleigh Co  Minute
February 8, 2024Agenda Packet Minutes 
January 23, 2024Agenda Packet Minutes 
January 11, 2024Agenda Packet Minutes 
December 26, 2023AgendaPacketMinutes
December 14, 2023AgendaPacketMinutes
November 28, 2023AgendaPacketMinutes
November 9, 2023AgendaPacketMinutes
October 24, 2023AgendaPacketMinutes
October 12, 2023AgendaPacketMinutes
September 26, 2023AgendaPacketMinutes
September 14, 2023AgendaPacketMinutes
August 22, 2023AgendaPacketMinutes
August 10, 2023AgendaPacketMinutes
July 25, 2023AgendaPacketMinutes
July 13, 2023AgendaPacketMinutes
June  27, 2023AgendaPacketMinutes
June 8, 2023 BOEAgendaPacketMinutes
June 8, 2023AgendaPacketMinutes
May 23, 2023AgendaPacketMinutes
May 11, 2023AgendaPacketMinutes
April 25, 2023AgendaPacketMinutes
April 13, 2023AgendaPacketMinutes
March 28, 2023AgendaPacketMinutes
March 9, 2023AgendaPacketMinutes
+

March 9, 2023

+

Morton/Burleigh Joint Co Commission Meeting

+		Notice  
+

February 28, 2023 Cancelled

+		AgendaPacketMinutes
February 9, 2023AgendaPacketMinutes
January 24, 2023AgendaPacketMinutes
January 12, 2023AgendaPacketMinutes
December 27, 2022AgendaPacketMinutes
December 8, 2022AgendaPacketMinutes
November 22, 2022AgendaPacketMinutes
+

November 10, 2022 Cancelled

+

 

+		AgendaPacketMinutes
October 25, 2022AgendaPacketMinutes
October 19, 2022 Agenda Minutes 
October 13, 2022AgendaPacketMinutes
September 27, 2022AgendaPacketMinutes
September 8, 2022AgendaPacketMinutes
August 23, 2022 CancelledAgendaPacketMinutes
August 11, 2022AgendaPacketMinutes
July 26, 2022AgendaPacketMinutes
July 14, 2022AgendaPacketMinutes
June 28, 2022AgendaPacketMinutes
June 9, 2022  AgendaPacketMinutes
June 9, 2022  BOE  Minutes
May 24, 2022AgendaPacketMinutes
May 12, 2022AgendaPacketMinutes
April 26, 2022AgendaPacketMinutes
April 14, 2022AgendaPacketMinutes
March 22, 2022AgendaPacketMinutes
March 10, 2022AgendaPacketMinutes
February 22, 2022AgendaPacketMinutes
February 10, 2022AgendaPacketMinutes
+

January 28, 2022       Special Commission Mtg

+		AgendaPacketMinutes
+

January 25, 2022

+		AgendaPacketMinutes
+

January 13, 2022

+		AgendaPacketMinutes
+

December 28, 2021 

+		Agenda Packet Minutes 
December 9, 2021 AgendaPacket Minutes 
November 23, 2021 Agenda Packet Minutes 
November 9, 2021 Agenda Packet Minutes 
October 26, 2021 AgendaPacket Minutes 
October 14, 2021 Agenda Packet Minutes 
September 28, 2021 Agenda Packet Minutes 
September 9, 2021 Agenda Packet Minutes 
August 24, 2021 Agenda Packet Minutes 
August 12, 2021 Agenda Packet Minutes 
July 27, 2021 Agenda Packet Minutes 
+

July 21, 2021

+

Special Budget Meeting

+		Agenda Minutes
July 8, 2021 Agenda Packet Minutes 
June 22, 2021 Agenda Packet Minutes 
June 10, 2021 Agenda Packet Minutes 
June 10, 2021 BOE  Minutes
May 25, 2021 Agenda Packet Minutes 
May 13, 2021 Agenda Packet Minutes 
April 27, 2021 Agenda Packet Minutes 
April 8, 2021 Agenda Packet Minutes 
March 23, 2021Agenda Packet Minutes 
March 11, 2021AgendaPacket Minutes 
+

February 23, 2021 

+		Agenda Packet Minutes 
February 11, 2021 Agenda Packet Minutes 
January 26, 2021Agenda Packet Minutes 
January 14, 2021 Agenda Packet Minutes 
    
December 22, 2020AgendaPacket Minutes
December 10, 2020AgendaPacket Minutes
November 24, 2020Agenda  PacketMinutes 
November 12, 2020Agenda PacketMinutes
October 27, 2020 Agenda PacketMinutes 
October 8, 2020AgendaPacket Minutes 
September 22, 2020 Agenda Packet Minutes
September 10, 2020AgendaPacket Minutes 
August 25, 2020 CANCELLEDAgenda PacketMinutes
August 13, 2020 Agenda Packet Minutes
August 4, 2020 Special MtgAgendaPacket Minutes 
July 28, 2002 Agenda Packet Minutes 
July 9, 2020Agenda Packet Minutes 
June 23, 2020 Agenda Packet Minutes
 June 4, 2020 BOE   Minutes 
June 4, 2020 Comm/BOEAgenda Packet Minutes 
May 26, 2020 Agenda Packet Minutes 
May 14, 2020 AgendaPacket Minutes
April 28, 2020 Agenda Packet Minutes 
April 7, 2020 Agenda Packet Minutes 
March 24, 2020 Agenda Packet Minutes 
March 12, 2020 Agenda Packet Minutes 
February 25, 2020 Agenda Packet Minutes 
February 13, 2020 Agenda Packet Minutes 
January 28, 2020 AgendaPacket Minutes 
January 17, 2020 Special Mtg Agenda Packet Minutes 
January 9, 2020 AgendaPacketMinutes 
    
December 26, 2019AgendaPacketMinutes
December 12, 2019AgendaPacketMinutes
November 26, 2019AgendaPacketMinutes
November 14, 2019AgendaPacketMinutes
October 29, 2019AgendaPacketMinutes
October 10, 2019AgendaPacketMinutes
September 24, 2019AgendaPacketMinutes
September 12, 2019AgendaPacketMinutes
August 27, 2019AgendaPacketMinutes
August 8, 2019AgendaPacketMinutes
July 30, 2019AgendaPacket +

Minutes

+
July 23, 2019AgendaPacket +

Minutes

+
July 11, 2019AgendaPacketMinutes
June 25, 2019AgendaPacketMinutes
June 25, 2019 BOEAgendaPacket Minutes
June 10, 2019AgendaPacketMinutes
May 28, 2019AgendaPacketMinutes
May 9, 2019AgendaPacketMinutes
April 23, 2019AgendaPacketMinutes
April 11, 2019AgendaPacketMinutes
March 26, 2019AgendaPacketMinutes
March 14, 2019AgendaPacketMinutes
March 5, 2019 Agenda Minutes
February 26, 2019AgendaPacketMinutes
February 14, 2019AgendaPacketMinutes
January 29, 2019AgendaPacketMinutes
January 10, 2019AgendaPacketMinutes
+

 

+

Morton County Commission Archive





+

 

+
+
+
+ +
+
+
+
+ +
+
+
+ + + + + \ No newline at end of file diff --git a/tests/test_bisnd_mcc.py b/tests/test_bisnd_mcc.py new file mode 100644 index 0000000..8637acd --- /dev/null +++ b/tests/test_bisnd_mcc.py @@ -0,0 +1,79 @@ +from datetime import datetime +from os.path import dirname, join + +import pytest +from city_scrapers_core.constants import COMMISSION, TENTATIVE +from city_scrapers_core.utils import file_response +from freezegun import freeze_time + +from city_scrapers.spiders.bisnd_mcc import BisndMccSpider + +test_response = file_response( + join(dirname(__file__), "files", "bisnd_mcc.html"), + url="https://www.mortonnd.org/?Type=B_BASIC&SEC={8182F8B8-3783-4C56-B690-F78FEE7CAC95}", # noqa +) +spider = BisndMccSpider() + +freezer = freeze_time(datetime(2024, 3, 14, 10, 15)) +freezer.start() + +parsed_items = [item for item in spider.parse(test_response)] +parsed_item = parsed_items[0] +freezer.stop() + + +def test_title(): + assert parsed_item["title"] == "Morton County Commission Meeting" + + +def test_description(): + assert parsed_item["description"] == "" + + +def test_start(): + assert parsed_item["start"] == datetime(2024, 12, 26, 17, 30) + + +def test_end(): + assert parsed_item["end"] is None + + +def test_time_notes(): + assert parsed_item["time_notes"] == "" + + +def test_id(): + assert ( + parsed_item["id"] == "bisnd_mcc/202412261730/x/morton_county_commission_meeting" + ) + + +def test_status(): + assert parsed_item["status"] == TENTATIVE + + +def test_location(): + assert parsed_item["location"] == { + "address": "Morton County Commission Room", + "name": "Morton County Courthouse, 210 2nd Ave NW, Mandan ND", + } + + +def test_source(): + assert ( + parsed_item["source"] + == "https://www.mortonnd.org/?Type=B_BASIC&SEC={8182F8B8-3783-4C56-B690-F78FEE7CAC95}" # noqa + ) + + +def test_links(): + assert parsed_item["links"] == [] + + +def test_classification(): + assert parsed_item["classification"] == COMMISSION + + +@pytest.mark.parametrize("item", parsed_items) +def test_all_day(item): + assert item["all_day"] is False