crucible.sha256.txt is not updating on every build

[ahl]

The buildomat job is supposed to update crucible.sha256.txt on every build, but many seem to be the same. See oxidecomputer/omicron#1074 for details.

[jmpesp]

I think this may not be a bug. Rust builds seem reproducible. I reproduced locally using the following script:

$ cat repro.sh
#!/bin/bash
git checkout main
for rev in $(cat repro.data);
do
    git diff | patch -p1 -R >/dev/null 2>/dev/null
    git checkout "${rev}" >/dev/null 2>/dev/null
    rm out/crucible.tar.gz >/dev/null 2>/dev/null

    cargo clean --quiet
    cargo build --release --quiet --all-features
    cargo run --release --bin crucible-package --quiet

    echo "$(git rev-parse HEAD) $(sha256sum out/crucible.tar.gz)"
done

from 1074's comments:

$ for commit in $(git log | grep commit | head -n20 | cut -d' ' -f2); do echo "${commit} $(curl -sq https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/${commit}/crucible.tar.gz | sha256sum)"; done;
8ea317a9dfee65d7c018a1fda89fdec63a560ef3 f85bb22797bab58d128c8dd168a2e6ba57feee284bdc9d0adf282197cb45cccb  -
5b6a40978f8f5bf57eb5781f833cf6b2d2b99359 f85bb22797bab58d128c8dd168a2e6ba57feee284bdc9d0adf282197cb45cccb  -
d412b2be7ef8c0658bf5603f7fd746f6972e4666 f85bb22797bab58d128c8dd168a2e6ba57feee284bdc9d0adf282197cb45cccb  -
c708b8024060a06791485fb3712ad839b64351f4 f85bb22797bab58d128c8dd168a2e6ba57feee284bdc9d0adf282197cb45cccb  -
1c89c283ddad0825ebf1b45a2b049c5dc86a9346 3c593037f51df8c2ce57bcec7824eb278f853c44f5ff778757b64274f25b68c2  -
bad407b43de03395be2bbe2f81bd4a895f5a5e97 3c593037f51df8c2ce57bcec7824eb278f853c44f5ff778757b64274f25b68c2  -
6956ef611a2f32051ef9ab620154ae1eb29de880 3c593037f51df8c2ce57bcec7824eb278f853c44f5ff778757b64274f25b68c2  -
212183cf577befc40b20d74c4e7895e0fee3cb5a d205188f8ac5899d9ad7317d1ffd54172b96e68b239a0a6b91775de22b57f94b  -
3cd5ddef060d09f14e1f4a862227dda9c93c820b b3335b4b3252c2158817825f6bde311f8b50d7cc7b87932316aeb29f76d36ae1  -
75ff03de32096fa71d004d306a513feebeba7499 1b1264efb6864c45dd7f8212ee30000533920eb09085f6e5625733fbfaa8baba  -
5a9faeeaffc318c7f8e9354b81aab98728124426 d94368130cc2ea6607c313a99c54c6aa050f46a9c24832227f3009286cf183e9  -
cd9e82123a0261ef3efb148f3a10e77737fc4bb5 360fea6d7d5e482c51a35d676484f48951a83c6cd08f1376c1160e864f36d8f0  -
38236f56a121049de4709e1d7dcde88bba8097c8 f1195a39ac0ef3935bec9ca6e329f3c4a0b934896fa67138dbf5b608ee29440f  -
023f3711748d99f0b478ac08887edd8382b477df 037caa7774af8ea36731af373f16a0b228f3e2b258f5a393b1ddc4764a3fc1c8  -
f1002c3d9057d339bbc5c4fd8eedf6b57f97c318 d336e7b3fc431a6d3ed9f7307c653bbd646bebf9839f6a6c089d74af9673dd87  -
cb6f71b6df8045a074bb1830c14b252b0193d191 d336e7b3fc431a6d3ed9f7307c653bbd646bebf9839f6a6c089d74af9673dd87  -
e4df5eac7b5f5e7110f285ff279c8353473b6d48 d336e7b3fc431a6d3ed9f7307c653bbd646bebf9839f6a6c089d74af9673dd87  -
695762041979ac674fdac747068b25282b615398 d336e7b3fc431a6d3ed9f7307c653bbd646bebf9839f6a6c089d74af9673dd87  -
1be2f4b095160f71611932f7846fd8fea535dc60 610f4dbe6f38c4f3bf2394a07ab05d5680d95d1528a19b796d21a91f748db06a  -
dc6f870f893a3da1b47877143e2bc9e3aef8113a 610f4dbe6f38c4f3bf2394a07ab05d5680d95d1528a19b796d21a91f748db06a  -

and my reproduction:

$ ./repro.sh
Switched to branch 'main'
Your branch is up to date with 'origin/main'.
8ea317a9dfee65d7c018a1fda89fdec63a560ef3 3162c2e433e6e839ee4a35b0b0d4a88f5a3312c9ea9deb3edb5667ffcb614834  out/crucible.tar.gz
5b6a40978f8f5bf57eb5781f833cf6b2d2b99359 3162c2e433e6e839ee4a35b0b0d4a88f5a3312c9ea9deb3edb5667ffcb614834  out/crucible.tar.gz
d412b2be7ef8c0658bf5603f7fd746f6972e4666 3162c2e433e6e839ee4a35b0b0d4a88f5a3312c9ea9deb3edb5667ffcb614834  out/crucible.tar.gz
c708b8024060a06791485fb3712ad839b64351f4 3162c2e433e6e839ee4a35b0b0d4a88f5a3312c9ea9deb3edb5667ffcb614834  out/crucible.tar.gz
1c89c283ddad0825ebf1b45a2b049c5dc86a9346 687f0cbe4f937046bbb3226f8b5ca9be912edf0c2eb5ab2f2ccb7e7b43447609  out/crucible.tar.gz
bad407b43de03395be2bbe2f81bd4a895f5a5e97 687f0cbe4f937046bbb3226f8b5ca9be912edf0c2eb5ab2f2ccb7e7b43447609  out/crucible.tar.gz
6956ef611a2f32051ef9ab620154ae1eb29de880 687f0cbe4f937046bbb3226f8b5ca9be912edf0c2eb5ab2f2ccb7e7b43447609  out/crucible.tar.gz
212183cf577befc40b20d74c4e7895e0fee3cb5a ea35e2cf4cebe22cc7bce678ba9d1290c04f66dd6726597eb64988a614b079c0  out/crucible.tar.gz
3cd5ddef060d09f14e1f4a862227dda9c93c820b 81880bafb12be901b9fe802fdd76965fafac25247a1910d4864098e0c0b44aeb  out/crucible.tar.gz
75ff03de32096fa71d004d306a513feebeba7499 5870be78d933dee3ca991fe37c92af2b19d342095d492c7a7ab0ba6110b76626  out/crucible.tar.gz
5a9faeeaffc318c7f8e9354b81aab98728124426 0c3ceff4b4da4149151cf2958f40284f549a8ac85072c9ad6e4d4b8153035d27  out/crucible.tar.gz
cd9e82123a0261ef3efb148f3a10e77737fc4bb5 44a0249f3c5a4c01061cdcda6fdfcaf5c297fb038a49012017952c6ed6274c34  out/crucible.tar.gz
38236f56a121049de4709e1d7dcde88bba8097c8 1b3378cc75aece50f9b87ca3db8fb12f0fe927b9f296f943c11840ee96d28445  out/crucible.tar.gz
023f3711748d99f0b478ac08887edd8382b477df 9182d80d26b9202b52a73cacbc6e62d5688f4ebe710ae4b23a0023378fe9cd06  out/crucible.tar.gz
f1002c3d9057d339bbc5c4fd8eedf6b57f97c318 abf452419c059c2c4ee6c7d9b12966a85f18342d146c73800052c8d94565d88d  out/crucible.tar.gz
cb6f71b6df8045a074bb1830c14b252b0193d191 abf452419c059c2c4ee6c7d9b12966a85f18342d146c73800052c8d94565d88d  out/crucible.tar.gz
e4df5eac7b5f5e7110f285ff279c8353473b6d48 abf452419c059c2c4ee6c7d9b12966a85f18342d146c73800052c8d94565d88d  out/crucible.tar.gz
695762041979ac674fdac747068b25282b615398 abf452419c059c2c4ee6c7d9b12966a85f18342d146c73800052c8d94565d88d  out/crucible.tar.gz
1be2f4b095160f71611932f7846fd8fea535dc60 eb3643237c8fb7698ce95e13b667e6ce39419091823747b484bbcbb9b514984d  out/crucible.tar.gz
dc6f870f893a3da1b47877143e2bc9e3aef8113a eb3643237c8fb7698ce95e13b667e6ce39419091823747b484bbcbb9b514984d  out/crucible.tar.gz

combining them shows that the repeated sha256 hashes match (grouped below by newlines):

crucible commit                          buildomat sha256                                                 ubuntu workstation repro.sh
---------------------------------------- ---------------------------------------------------------------- ----------------------------------------------------------------
8ea317a9dfee65d7c018a1fda89fdec63a560ef3 f85bb22797bab58d128c8dd168a2e6ba57feee284bdc9d0adf282197cb45cccb 3162c2e433e6e839ee4a35b0b0d4a88f5a3312c9ea9deb3edb5667ffcb614834
5b6a40978f8f5bf57eb5781f833cf6b2d2b99359 f85bb22797bab58d128c8dd168a2e6ba57feee284bdc9d0adf282197cb45cccb 3162c2e433e6e839ee4a35b0b0d4a88f5a3312c9ea9deb3edb5667ffcb614834
d412b2be7ef8c0658bf5603f7fd746f6972e4666 f85bb22797bab58d128c8dd168a2e6ba57feee284bdc9d0adf282197cb45cccb 3162c2e433e6e839ee4a35b0b0d4a88f5a3312c9ea9deb3edb5667ffcb614834
c708b8024060a06791485fb3712ad839b64351f4 f85bb22797bab58d128c8dd168a2e6ba57feee284bdc9d0adf282197cb45cccb 3162c2e433e6e839ee4a35b0b0d4a88f5a3312c9ea9deb3edb5667ffcb614834

1c89c283ddad0825ebf1b45a2b049c5dc86a9346 3c593037f51df8c2ce57bcec7824eb278f853c44f5ff778757b64274f25b68c2 687f0cbe4f937046bbb3226f8b5ca9be912edf0c2eb5ab2f2ccb7e7b43447609
bad407b43de03395be2bbe2f81bd4a895f5a5e97 3c593037f51df8c2ce57bcec7824eb278f853c44f5ff778757b64274f25b68c2 687f0cbe4f937046bbb3226f8b5ca9be912edf0c2eb5ab2f2ccb7e7b43447609
6956ef611a2f32051ef9ab620154ae1eb29de880 3c593037f51df8c2ce57bcec7824eb278f853c44f5ff778757b64274f25b68c2 687f0cbe4f937046bbb3226f8b5ca9be912edf0c2eb5ab2f2ccb7e7b43447609

212183cf577befc40b20d74c4e7895e0fee3cb5a d205188f8ac5899d9ad7317d1ffd54172b96e68b239a0a6b91775de22b57f94b ea35e2cf4cebe22cc7bce678ba9d1290c04f66dd6726597eb64988a614b079c0

3cd5ddef060d09f14e1f4a862227dda9c93c820b b3335b4b3252c2158817825f6bde311f8b50d7cc7b87932316aeb29f76d36ae1 81880bafb12be901b9fe802fdd76965fafac25247a1910d4864098e0c0b44aeb

75ff03de32096fa71d004d306a513feebeba7499 1b1264efb6864c45dd7f8212ee30000533920eb09085f6e5625733fbfaa8baba 5870be78d933dee3ca991fe37c92af2b19d342095d492c7a7ab0ba6110b76626

5a9faeeaffc318c7f8e9354b81aab98728124426 d94368130cc2ea6607c313a99c54c6aa050f46a9c24832227f3009286cf183e9 0c3ceff4b4da4149151cf2958f40284f549a8ac85072c9ad6e4d4b8153035d27

cd9e82123a0261ef3efb148f3a10e77737fc4bb5 360fea6d7d5e482c51a35d676484f48951a83c6cd08f1376c1160e864f36d8f0 44a0249f3c5a4c01061cdcda6fdfcaf5c297fb038a49012017952c6ed6274c34

38236f56a121049de4709e1d7dcde88bba8097c8 f1195a39ac0ef3935bec9ca6e329f3c4a0b934896fa67138dbf5b608ee29440f 1b3378cc75aece50f9b87ca3db8fb12f0fe927b9f296f943c11840ee96d28445

023f3711748d99f0b478ac08887edd8382b477df 037caa7774af8ea36731af373f16a0b228f3e2b258f5a393b1ddc4764a3fc1c8 9182d80d26b9202b52a73cacbc6e62d5688f4ebe710ae4b23a0023378fe9cd06

f1002c3d9057d339bbc5c4fd8eedf6b57f97c318 d336e7b3fc431a6d3ed9f7307c653bbd646bebf9839f6a6c089d74af9673dd87 abf452419c059c2c4ee6c7d9b12966a85f18342d146c73800052c8d94565d88d
cb6f71b6df8045a074bb1830c14b252b0193d191 d336e7b3fc431a6d3ed9f7307c653bbd646bebf9839f6a6c089d74af9673dd87 abf452419c059c2c4ee6c7d9b12966a85f18342d146c73800052c8d94565d88d
e4df5eac7b5f5e7110f285ff279c8353473b6d48 d336e7b3fc431a6d3ed9f7307c653bbd646bebf9839f6a6c089d74af9673dd87 abf452419c059c2c4ee6c7d9b12966a85f18342d146c73800052c8d94565d88d
695762041979ac674fdac747068b25282b615398 d336e7b3fc431a6d3ed9f7307c653bbd646bebf9839f6a6c089d74af9673dd87 abf452419c059c2c4ee6c7d9b12966a85f18342d146c73800052c8d94565d88d

1be2f4b095160f71611932f7846fd8fea535dc60 610f4dbe6f38c4f3bf2394a07ab05d5680d95d1528a19b796d21a91f748db06a eb3643237c8fb7698ce95e13b667e6ce39419091823747b484bbcbb9b514984d
dc6f870f893a3da1b47877143e2bc9e3aef8113a 610f4dbe6f38c4f3bf2394a07ab05d5680d95d1528a19b796d21a91f748db06a eb3643237c8fb7698ce95e13b667e6ce39419091823747b484bbcbb9b514984d