[FIX] Add cookie in Collector, improve error handling and CI (#29)

* fix: add cookie in `Collector`

* ci: use python 3.11 and 3.13

* fix: improve error handling in collect and downloadImage functions

* fix: standardize header key for cookies to "Cookie"

Author: cwher

.github/workflows/test_on_schedule.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.11", "3.13"]
 
     steps:
       - uses: actions/checkout@v4

README.md

@@ -2,7 +2,7 @@
 
 ![](https://img.shields.io/pypi/v/pixiv-utils)
 [![Daily test](https://github.com/CWHer/PixivCrawler/actions/workflows/test_on_schedule.yml/badge.svg)](https://github.com/CWHer/PixivCrawler/actions/workflows/test_on_schedule.yml)
-![](https://img.shields.io/badge/Python-3.9%20%7C%203.10%20%7C%203.11-green)
+![](https://img.shields.io/badge/Python-3.9%20%7C%203.11%20%7C%203.13-green)
 ![](https://img.shields.io/badge/Platform-Windows%20%7C%20Linux%20%7C%20MacOS-blue)
 
 The Chinese version of README can be found [here](./README_CN.md).

pixiv_utils/pixiv_crawler/collector/collector.py

@@ -78,8 +78,10 @@ def collect(self):
                     f"https://www.pixiv.net/ajax/illust/{illust_id}/pages?lang=zh"
                     for illust_id in self.id_group
                 ]
+                # NOTE: Add COOKIE to collect R18 content
                 additional_headers = [
                     {
+                        "Cookie": user_config.cookie,
                         "Referer": f"https://www.pixiv.net/artworks/{illust_id}",
                         "x-user-id": user_config.user_id,
                     }

pixiv_utils/pixiv_crawler/collector/collector_unit.py

@@ -39,6 +39,8 @@ def collect(
                     printInfo(f"{url} complete")
                 return id_group
 
+            raise RuntimeError(f"Request failed with status code {response.status_code} for {url}")
+
         except Exception as e:
             assertWarn(not debug_config.show_error, e)
             assertWarn(not debug_config.show_error, f"This is {i} attempt to collect {url}")

pixiv_utils/pixiv_crawler/crawlers/bookmark_crawler.py

@@ -44,7 +44,7 @@ def _requestCount(self):
         url = self.user_url + "/bookmark/tags?lang=zh"
         printInfo("===== Requesting bookmark count =====")
 
-        headers = {"COOKIE": user_config.cookie}
+        headers = {"Cookie": user_config.cookie}
         headers.update(network_config.headers)
         for i in range(download_config.retry_times):
             try:
@@ -104,7 +104,7 @@ def collect(self, artworks_per_json: int = 48):
                 )
             )
 
-        additional_headers = {"COOKIE": user_config.cookie}
+        additional_headers = {"Cookie": user_config.cookie}
         collect_bookmark_fn = functools.partial(
             collect, selector=selectBookmark, additional_headers=additional_headers
         )

pixiv_utils/pixiv_crawler/crawlers/keyword_crawler.py

@@ -75,7 +75,7 @@ def collect(self, artworks_per_json: int = 60):
         for i in range(n_page):
             urls.add(url.format(i + 1))
 
-        additional_headers = {"COOKIE": user_config.cookie}
+        additional_headers = {"Cookie": user_config.cookie}
         collect_keyword_fn = functools.partial(
             collect, selector=selectKeyword, additional_headers=additional_headers
         )

pixiv_utils/pixiv_crawler/crawlers/ranking_crawler.py

@@ -76,7 +76,7 @@ def addDate(current: datetime.date, days):
                     {
                         "Referer": re.search("(.*)&p", url).group(1),
                         "x-requested-with": "XMLHttpRequest",
-                        "COOKIE": user_config.cookie,
+                        "Cookie": user_config.cookie,
                     }
                     for url in urls
                 ]

pixiv_utils/pixiv_crawler/crawlers/users_crawler.py

@@ -29,7 +29,7 @@ def collect(self):
         additional_headers = {
             "Referer": f"https://www.pixiv.net/users/{self.artist_id}/illustrations",
             "x-user-id": user_config.user_id,
-            "COOKIE": user_config.cookie,
+            "Cookie": user_config.cookie,
         }
         image_ids = collect(url, selectUser, additional_headers)
         if image_ids is not None:

pixiv_utils/pixiv_crawler/downloader/download_image.py

@@ -53,14 +53,20 @@ def downloadImage(url: str, download_time: float = 10) -> float:
                 if len(response.content) != image_size:
                     time.sleep(download_config.fail_delay)
                     download_time += 2
-                    continue
+                    raise RuntimeError(
+                        f"Image size mismatch: expected {image_size}, got {len(response.content)}"
+                    )
 
                 with open(image_path, "wb") as f:
                     f.write(response.content)
                 if debug_config.verbose:
                     printInfo(f"{image_name} complete")
                 return image_size / 2**20
 
+            raise RuntimeError(
+                f"Request failed with status code {response.status_code} for {image_name}"
+            )
+
         except Exception as e:
             assertWarn(not debug_config.show_error, e)
             assertWarn(not debug_config.show_error, f"This is {i} attempt to download {image_name}")