From fb369db3fdde60b6e5adf7d62f38df46753f3584 Mon Sep 17 00:00:00 2001 From: Seppo Yli-Olli Date: Wed, 5 Oct 2022 17:42:01 +0300 Subject: [PATCH] Stop using child process for downloading URL's This makes things really hard to debug when there are errors and downloading is mostly done with GIL dropped. --- src/buildstream/downloadablefilesource.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/buildstream/downloadablefilesource.py b/src/buildstream/downloadablefilesource.py index e96039a62..83cb1ea9d 100644 --- a/src/buildstream/downloadablefilesource.py +++ b/src/buildstream/downloadablefilesource.py @@ -113,7 +113,8 @@ def _download_file(opener, url, etag, directory): filename = os.path.basename(filename) local_file = os.path.join(directory, filename) with open(local_file, "wb") as dest: - shutil.copyfileobj(response, dest) + # Use megabyte chunk size to avoid hogging GIL + shutil.copyfileobj(response, dest, 1024 * 1024) return local_file, etag @@ -220,9 +221,8 @@ def _ensure_mirror(self, activity_name: str): else: etag = None - local_file, new_etag = self.blocking_activity( - _download_file, (self.__get_urlopener(), self.url, etag, td), activity_name - ) + with self.timed_activity(activity_name): + local_file, new_etag = _download_file(self.__get_urlopener(), self.url, etag, td) if local_file is None: return self.ref