tests.fetchtorrent: verify and delete in postFetch (NixOS#458193)

Author: roberth

pkgs/build-support/fetchtorrent/test-hashes.sha512sum

@@ -0,0 +1,11 @@
+1755c55e08aec3872db177464e0bc3f48c89aa40c1ec32605b472d88f6e92963eb08a10aa37bc47e837ae693db18652f40dfbcd6bf26f07278f0566a19c72cd5  poster.jpg
+3ec33eae8153b6e2bdfff8157b332e6f0cfbc14d0c30e6a34042deecf0cff01ee19ca54314678b5ad970ae669da8cb91425e96253caf91d95ae01a62917827b7  Sintel.de.srt
+ceac1ea59a9224b26946c6683acfaae27e1551c696fb2c632b55151c17f3faa8f930bbd93904cd7a13a896ad3d670b01e2f2bc1a8851060e05fedd85f804ba17  Sintel.en.srt
+6efbc68f872c212f6579c5a5d42d6c2a60d775d32be8a7badcd147dbfc07b24cb2cc97e04a796af4002fcd4c8860a7061aed5d59a1280e4352d1bbd701d612a8  Sintel.es.srt
+5619ad5bc7ca82e7f1f3b5e9197ab6ef6f5edd7293d396a3c036fef95d5ef592cf9e8d403e8532f4ba71a93fc9556614bfba3473eceae392d2e3175116c06f95  Sintel.fr.srt
+fa93c051491b4e11f8e7a643c2138198f795315c942134b94ceace1adda4a5b28dc58b01b507e0da4c26784fc968f47bca1409f985eec2bf80c3e694662ed7c5  Sintel.it.srt
+63426110ed7aa60ed3525078fa324cef4276393b4ada55e0a8cf8292500fdd59c2984e299a4e4f65e4f273b8ae2624da22700de4a55054b00e9d987f606c3c7f  Sintel.mp4
+5fb04c25fda9dada8c808a9f289cec2cca001d60966b69d61ad663240a869c76dd0aff605122823ccbe199e266331f6a4b11a414bba92999a7041c4b534ef432  Sintel.nl.srt
+510f052cbbda0b9fc31679b9a26cc16766df9054fc15b66cff475975725a1e8452e69a28a6f90bb965330a49e49bbf42a90efb13de07eaece7d273ef1ca7978a  Sintel.pl.srt
+510dc0eca608bd0fdd9633ea80365c916f8c4f0fd8e2fdf77534f455f03814401d210df47d940be4fd2fb7ab4c3ff5038a548846c5fa9badbe77b1731b3cae29  Sintel.pt.srt
+57c014bd3d781a94850f85fd6c9364421e17cb2c4e25e22c0a8578f50c874e3f2eaf5a0b3da615db7dc95efbe2e6be7c7238e2e1ba27671c818b9e34b05ee1d0  Sintel.ru.srt

pkgs/build-support/fetchtorrent/test-sintel.torrent

@@ -2,90 +2,130 @@
   lib,
   testers,
   fetchtorrent,
+  emptyDirectory,
   ...
 }:
 
 let
+  # This meta attribute isn't used anywhere, as the actual derivation
+  # realizations are only empty directories.  It's maintained here as a record
+  # of the details of the intermediate product that exists briefly while
+  # building the test derivations.
+  #
+  # Ideally we'd use a smaller download, but neither of the Bittorrent backends
+  # supported by fetchtorrent appear to support torrents that are only reliably
+  # seeded by HTTP sources rather than other people using Bittorrent clients.
+  # Sintel was the smallest torrent I could find that had a free license and
+  # was reliably seeded by other Bittorrent clients.
+  #
+  # For more information, see the discussion at
+  # https://github.com/NixOS/nixpkgs/pull/432091/files/bd13421b2b70f3f125061018c800439ef2d43e8d#r2264073113
   sintel.meta = {
     description = "An open source short film to show off open source technologies.";
     longDescription = ''
       An independently produced short film, initiated by the Blender Foundation
-      as a means to further improve andvalidate the free/open source 3D
+      as a means to further improve and validate the free/open source 3D
       creation suite Blender.
     '';
     license = lib.licenses.cc-by-30;
     homepage = "https://durian.blender.org/";
   };
 
   # Via https://webtorrent.io/free-torrents
-  http.url = "https://webtorrent.io/torrents/sintel.torrent";
+  http.url = "${./test-sintel.torrent}";
   magnet.url = "magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent";
 
-  flattened.hash = "sha256-EzbmBiTEWOlFUNaV5R4eDeD9EBbp6d93rfby88ACg0s=";
-  unflattened.hash = "sha256-lVrlo1AwmFcxwsIsY976VYqb3hAprFH1xWYdmlTuw0U=";
+  # Sintel isn't a massive download, but it's not small.  There's also no real
+  # value in storing copies of it in Nix caches, which is what happens by
+  # default when this test succeeds.  Avoid that by verifying the downloaded
+  # files using `sha512sum` in the post-fetch hook, then deleting the files so
+  # the actual derivation result is an empty directory.
+  #
+  # Chain `&&` in the postFetch phase because the transmission backend does not
+  # run that phase with `errexit` enabled.
+  flattened.postFetch = ''
+    pushd "$out" &&
+    sha512sum --check --strict ${./test-hashes.sha512sum} &&
+    sed 's/.*  //' ${./test-hashes.sha512sum} | xargs rm --verbose &&
+    popd
+  '';
+  unflattened.postFetch = ''
+    pushd "$out" &&
+    pushd Sintel &&
+    sha512sum --check --strict ${./test-hashes.sha512sum} &&
+    sed 's/.*  //' ${./test-hashes.sha512sum} | xargs rm --verbose &&
+    popd &&
+    rm --dir --verbose Sintel &&
+    popd
+  '';
+
+  # Fixed output derivation hash is identical for all derivations: the empty
+  # directory.
+  fetchtorrentWithHash =
+    args:
+    fetchtorrent (
+      {
+        hash = builtins.convertHash {
+          hash = emptyDirectory.outputHash;
+          toHashFormat = "sri";
+          hashAlgo = emptyDirectory.outputHashAlgo;
+        };
+      }
+      // args
+    );
 in
 # Seems almost but not quite worth using lib.mapCartesianProduct...
-builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrent v) {
+builtins.mapAttrs (n: v: testers.invalidateFetcherByDrvHash fetchtorrentWithHash v) {
   http-link = {
     inherit (http) url;
-    inherit (flattened) hash;
-    inherit (sintel) meta;
+    inherit (flattened) postFetch;
   };
   http-link-transmission = {
     inherit (http) url;
     backend = "transmission";
-    inherit (flattened) hash;
-    inherit (sintel) meta;
+    inherit (flattened) postFetch;
   };
   magnet-link = {
     inherit (magnet) url;
-    inherit (flattened) hash;
-    inherit (sintel) meta;
+    inherit (flattened) postFetch;
   };
   magnet-link-transmission = {
     inherit (magnet) url;
     backend = "transmission";
-    inherit (flattened) hash;
-    inherit (sintel) meta;
+    inherit (flattened) postFetch;
   };
   http-link-rqbit = {
     inherit (http) url;
     backend = "rqbit";
-    inherit (flattened) hash;
-    inherit (sintel) meta;
+    inherit (flattened) postFetch;
   };
   magnet-link-rqbit = {
     inherit (magnet) url;
     backend = "rqbit";
-    inherit (flattened) hash;
-    inherit (sintel) meta;
+    inherit (flattened) postFetch;
   };
   http-link-rqbit-flattened = {
     inherit (http) url;
     backend = "rqbit";
     flatten = true;
-    inherit (flattened) hash;
-    inherit (sintel) meta;
+    inherit (flattened) postFetch;
   };
   magnet-link-rqbit-flattened = {
     inherit (magnet) url;
     backend = "rqbit";
     flatten = true;
-    inherit (flattened) hash;
-    inherit (sintel) meta;
+    inherit (flattened) postFetch;
   };
   http-link-rqbit-unflattened = {
     inherit (http) url;
     backend = "rqbit";
     flatten = false;
-    inherit (unflattened) hash;
-    inherit (sintel) meta;
+    inherit (unflattened) postFetch;
   };
   magnet-link-rqbit-unflattened = {
     inherit (magnet) url;
     backend = "rqbit";
     flatten = false;
-    inherit (unflattened) hash;
-    inherit (sintel) meta;
+    inherit (unflattened) postFetch;
   };
 }