From a36286f7f9d7dd95881e5c55bdee40814383dbe4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 14:42:44 +0100 Subject: [PATCH 01/14] fix fuse tests hanging --- .github/workflows/ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c7a84cd15..4454114fa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,7 @@ jobs: - os: ubuntu-20.04 python-version: '3.10' toxenv: py310-fuse3 - - os: macos-10.15 # macos-latest is macos 11.6.2 and hanging at test_fuse, #6099 + - os: macos-latest python-version: '3.8' toxenv: py38-fuse2 @@ -104,9 +104,9 @@ jobs: if: ${{ runner.os == 'macOS' }} run: | brew install pkg-config || brew upgrade pkg-config - brew install zstd || brew upgrade zstd - brew install lz4 || brew upgrade lz4 - brew install openssl@1.1 || brew upgrade openssl@1.1 + #brew install zstd || brew upgrade zstd + #brew install lz4 || brew upgrade lz4 + #brew install openssl@1.1 || brew upgrade openssl@1.1 brew install --cask macfuse || brew upgrade --cask macfuse # Required for Python llfuse module - name: Install Python requirements @@ -121,8 +121,10 @@ jobs: run: | # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482. #sudo -E bash -c "tox -e py" - tox --skip-missing-interpreters + #tox --skip-missing-interpreters + pytest -k test_fuse - name: Upload coverage to Codecov + if: 0 uses: codecov/codecov-action@v1 env: OS: ${{ runner.os }} From 1df39a242f85b22385e577ebd561dbdae8dd7257 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 14:46:22 +0100 Subject: [PATCH 02/14] temp: remove stuff not needed to speed up debugging this --- .github/workflows/ci.yml | 45 ---------------------------------------- 1 file changed, 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4454114fa2..a94bde0f78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,37 +29,11 @@ on: - '!docs/**' jobs: - lint: - - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Lint with flake8 - run: | - pip install flake8 - flake8 src scripts conftest.py - pytest: - needs: lint strategy: matrix: include: - - os: ubuntu-20.04 - python-version: '3.8' - toxenv: py38-fuse2 - - os: ubuntu-20.04 - python-version: '3.9' - toxenv: py39-fuse3 - - os: ubuntu-20.04 - python-version: '3.10' - toxenv: py310-fuse3 - os: macos-latest python-version: '3.8' toxenv: py38-fuse2 @@ -90,16 +64,6 @@ jobs: ${{ runner.os }}-pip- ${{ runner.os }}- - - name: Install Linux packages - if: ${{ runner.os == 'Linux' }} - run: | - sudo apt-get update - sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config build-essential - sudo apt-get install -y libxxhash-dev || true - sudo apt-get install -y libb2-dev || true - sudo apt-get install -y libfuse-dev fuse || true # Required for Python llfuse module - sudo apt-get install -y libfuse3-dev fuse3 || true # Required for Python pyfuse3 module - - name: Install macOS packages if: ${{ runner.os == 'macOS' }} run: | @@ -123,12 +87,3 @@ jobs: #sudo -E bash -c "tox -e py" #tox --skip-missing-interpreters pytest -k test_fuse - - name: Upload coverage to Codecov - if: 0 - uses: codecov/codecov-action@v1 - env: - OS: ${{ runner.os }} - python: ${{ matrix.python-version }} - with: - token: ${{ secrets.CODECOV_TOKEN }} - env_vars: OS, python From 981468ae2ff6d5ea4ddbd16eb430f99d418e5ce1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 14:47:27 +0100 Subject: [PATCH 03/14] temp: remove stuff not needed to speed up debugging this --- .github/workflows/codeql-analysis.yml | 66 --------------------------- 1 file changed, 66 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 864d71f4e9..0000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,66 +0,0 @@ -# CodeQL semantic code analysis engine - -name: "CodeQL" - -on: - push: - branches: [ master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] - schedule: - - cron: '39 2 * * 5' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'cpp', 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - # just fetching 1 commit is not enough for setuptools-scm, so we fetch all - fetch-depth: 0 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - - name: Cache pip - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - - name: Install requirements, build and install Borg - run: | - sudo apt install libacl1-dev - pip3 install -r requirements.d/development.txt - pip3 install -e . - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 From fe31dfd2fe484a12e5d41c4284171e32ac2718fd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 14:49:47 +0100 Subject: [PATCH 04/14] more verbose --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a94bde0f78..e844104c6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,4 +86,4 @@ jobs: # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482. #sudo -E bash -c "tox -e py" #tox --skip-missing-interpreters - pytest -k test_fuse + pytest -vk test_fuse From a12e25b003a65173cae9f3f265611be0b9fc4ec5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 14:50:55 +0100 Subject: [PATCH 05/14] inst llfuse --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e844104c6f..b1c89c3dfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,4 +86,5 @@ jobs: # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482. #sudo -E bash -c "tox -e py" #tox --skip-missing-interpreters + pip install llfuse pytest -vk test_fuse From 791ec6817c4c14dba321fb1964b6a9228a8055f6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 14:59:13 +0100 Subject: [PATCH 06/14] list allfiles permission entries --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1c89c3dfa..2ca350a4d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,4 +87,6 @@ jobs: #sudo -E bash -c "tox -e py" #tox --skip-missing-interpreters pip install llfuse + echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" + sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles pytest -vk test_fuse From af0e32400998006a0600de7d187c66180ef376e0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 15:07:35 +0100 Subject: [PATCH 07/14] try sudo pytest --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ca350a4d9..3ef56c507b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,4 +89,5 @@ jobs: pip install llfuse echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles - pytest -vk test_fuse + # kTCCServiceSystemPolicyAllFiles|/bin/bash|1|2|0|1||||UNUSED||0|1583997993 + sudo pytest -vk test_fuse From bcc3cd4f6fd31b2166a3b91600c0fe2221ae153e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 15:17:30 +0100 Subject: [PATCH 08/14] more verbose, no sudo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ef56c507b..c66d49b419 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,4 +90,4 @@ jobs: echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles # kTCCServiceSystemPolicyAllFiles|/bin/bash|1|2|0|1||||UNUSED||0|1583997993 - sudo pytest -vk test_fuse + pytest -vvk test_fuse From 7b0a43b91b53eaaa647d0a4050145c8ffb4ceb57 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 15:28:50 +0100 Subject: [PATCH 09/14] tcc --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c66d49b419..a6a6607764 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,4 +90,7 @@ jobs: echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles # kTCCServiceSystemPolicyAllFiles|/bin/bash|1|2|0|1||||UNUSED||0|1583997993 + which borg + sudo sqlite3 "/Library/Application Support/com.apple.TCC/TCC.db" "INSERT OR IGNORE INTO access \ + VALUES('kTCCServiceSystemPolicyAllFiles','/bin/sh',1,2,0,1,NULL,NULL,NULL,'UNUSED',NULL,0,1583997993);" pytest -vvk test_fuse From 0993c9d884e87ad6a4c2077a072306b7dc99511b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 15:46:00 +0100 Subject: [PATCH 10/14] tcc2 --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6a6607764..f1938198c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,7 @@ jobs: #brew install lz4 || brew upgrade lz4 #brew install openssl@1.1 || brew upgrade openssl@1.1 brew install --cask macfuse || brew upgrade --cask macfuse # Required for Python llfuse module + brew install tccutil # needs SIP disabled! - name: Install Python requirements run: | @@ -91,6 +92,8 @@ jobs: sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles # kTCCServiceSystemPolicyAllFiles|/bin/bash|1|2|0|1||||UNUSED||0|1583997993 which borg - sudo sqlite3 "/Library/Application Support/com.apple.TCC/TCC.db" "INSERT OR IGNORE INTO access \ - VALUES('kTCCServiceSystemPolicyAllFiles','/bin/sh',1,2,0,1,NULL,NULL,NULL,'UNUSED',NULL,0,1583997993);" + sudo tccutil --service kTCCServiceSystemPolicyAllFiles --insert /bin/sh + sudo tccutil --enable /bin/sh + echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" + sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles pytest -vvk test_fuse From 8ee7e40ce3ccfa5647c04e714df3e606a5dfebe7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 15:55:47 +0100 Subject: [PATCH 11/14] csrutil --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1938198c7..bb7be6efbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,8 @@ jobs: sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles # kTCCServiceSystemPolicyAllFiles|/bin/bash|1|2|0|1||||UNUSED||0|1583997993 which borg + sudo csrutil disable + sudo csrutil status sudo tccutil --service kTCCServiceSystemPolicyAllFiles --insert /bin/sh sudo tccutil --enable /bin/sh echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" From 755c5e484946d8d57ca53359f80d5a61bdea73bd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 16:00:58 +0100 Subject: [PATCH 12/14] tccutil.py not tccutil --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb7be6efbc..68e1cb69a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,8 +94,8 @@ jobs: which borg sudo csrutil disable sudo csrutil status - sudo tccutil --service kTCCServiceSystemPolicyAllFiles --insert /bin/sh - sudo tccutil --enable /bin/sh + sudo tccutil.py --service kTCCServiceSystemPolicyAllFiles --insert /bin/sh + sudo tccutil.py --enable /bin/sh echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles pytest -vvk test_fuse From 58c39131399b20dc235945df07b4b3a74994b0e4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 16:02:32 +0100 Subject: [PATCH 13/14] remove csrutil --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68e1cb69a7..e89d7446f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,8 +92,9 @@ jobs: sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles # kTCCServiceSystemPolicyAllFiles|/bin/bash|1|2|0|1||||UNUSED||0|1583997993 which borg - sudo csrutil disable - sudo csrutil status + # vvv only available from revovery OS + #sudo csrutil disable + #sudo csrutil status sudo tccutil.py --service kTCCServiceSystemPolicyAllFiles --insert /bin/sh sudo tccutil.py --enable /bin/sh echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" From 5d820a9f9a4143bb088ff762e14ba6251a4509ef Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Jan 2022 17:25:29 +0100 Subject: [PATCH 14/14] fix tccutil name, no .py --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e89d7446f5..43262b0098 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,8 +95,8 @@ jobs: # vvv only available from revovery OS #sudo csrutil disable #sudo csrutil status - sudo tccutil.py --service kTCCServiceSystemPolicyAllFiles --insert /bin/sh - sudo tccutil.py --enable /bin/sh + sudo tccutil --service kTCCServiceSystemPolicyAllFiles --insert /bin/sh + sudo tccutil --enable /bin/sh echo "AllFiles vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" sudo sqlite3 /Library/Application\ Support/com.apple.TCC/TCC.db 'select * from access' | grep AllFiles pytest -vvk test_fuse