From 66aef10ddf4f30f5db06765ab6fa23350c5cadbb Mon Sep 17 00:00:00 2001 From: Luke Eltiste Date: Tue, 8 Oct 2024 16:53:18 -0700 Subject: [PATCH 1/6] update to use pyproject.toml --- Dockerfile | 4 ++-- pyproject.toml | 34 ++++++++++++++++++++++++++++++++++ requirements.txt | 1 - ruff.toml | 4 ---- setup.py | 13 ------------- 5 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 ruff.toml delete mode 100644 setup.py diff --git a/Dockerfile b/Dockerfile index bb03e50..94c7470 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base image -FROM python:3.12-slim +FROM python:3.11-slim # Set work directory WORKDIR /home/docker @@ -11,7 +11,7 @@ RUN apt-get update && \ # Copy the necessary files COPY github-backup/ ./github-backup/ -COPY setup.py . +COPY pyproject.toml . COPY config.json.example ./github-backup/ COPY backup.sh . diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..aae79b6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "github_backup" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "requests~=2.32.3" +] + +[project.optional-dependencies] +dev = [ + "ruff" +] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["github_backup"] + +[tool.ruff] +exclude = [".venv", ".vscode", "venv"] +line-length = 120 +indent-width = 4 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "W", "F", "I", "UP", "B"] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f229360..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index e207a27..0000000 --- a/ruff.toml +++ /dev/null @@ -1,4 +0,0 @@ -line-length = 120 - -[lint] -extend-select = ['F', 'E', 'W', 'I', 'UP', 'B'] diff --git a/setup.py b/setup.py deleted file mode 100644 index 8097a1e..0000000 --- a/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="github_backup", - version="0.1.0", - packages=find_packages(), - install_requires=["requests==2.32.2"], - extras_require={ - "dev": [ - "ruff==0.5.1", - ] - }, -) From 5630393cb57e529b9666f2163ba2523a99b1f331 Mon Sep 17 00:00:00 2001 From: Luke Eltiste Date: Tue, 8 Oct 2024 16:53:53 -0700 Subject: [PATCH 2/6] run ruff --- github-backup/github-backup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/github-backup/github-backup.py b/github-backup/github-backup.py index d8fb39d..eb0b5f3 100644 --- a/github-backup/github-backup.py +++ b/github-backup/github-backup.py @@ -6,13 +6,13 @@ import subprocess import sys import time -from typing import Dict, Iterator, Tuple, Union +from collections.abc import Iterator from urllib.parse import urlparse, urlunparse import requests -def get_json(url: str, session: requests.Session) -> Iterator[Dict]: +def get_json(url: str, session: requests.Session) -> Iterator[dict]: """ Fetch JSON data from a URL using pagination, authentication, and handling rate limits. @@ -148,7 +148,7 @@ def fetch_repo(repo_path: str, repo_url: str) -> None: ) -def mirror(repo_name: str, repo_url: str, to_path: str, username: str, token: str) -> Tuple[str, str]: +def mirror(repo_name: str, repo_url: str, to_path: str, username: str, token: str) -> tuple[str, str]: """ Mirror a GitHub repository to a local directory. @@ -181,7 +181,7 @@ def main(): with open(args.config, "rb") as f: config = json.loads(f.read()) - owners: Union[list[str], None] = config.get("owners") + owners: list[str] | None = config.get("owners") token: str = config["token"] path: str = os.path.expanduser(config["directory"]) if mkdir(path): From c8fde423ecccbdf47d9b9d6bce4b439884312441 Mon Sep 17 00:00:00 2001 From: Luke Eltiste Date: Tue, 8 Oct 2024 16:56:49 -0700 Subject: [PATCH 3/6] Action updates --- .github/workflows/docker-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 15eecb3..658aa08 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -28,14 +28,14 @@ jobs: - name: Install cosign if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@v3.5.0 + uses: sigstore/cosign-installer@v3.7.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.3.0 + uses: docker/setup-buildx-action@v3.7.1 - name: Log into registry ${{ env.REGISTRY }} if: github.event_name != 'pull_request' - uses: docker/login-action@v3.1.0 + uses: docker/login-action@v3.3.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -57,7 +57,7 @@ jobs: - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v5.3.0 + uses: docker/build-push-action@v6.9.0 with: context: . push: ${{ github.event_name != 'pull_request' }} From 346d8c7bdc407abb7f499bfc117051dd69054b3f Mon Sep 17 00:00:00 2001 From: Luke Eltiste Date: Tue, 8 Oct 2024 16:57:01 -0700 Subject: [PATCH 4/6] version 1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aae79b6..beb6784 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "github_backup" -version = "0.1.0" +version = "1.0.0" requires-python = ">=3.11" dependencies = [ "requests~=2.32.3" From 8b3447565db23b7b635e88d93927db4f19bb7514 Mon Sep 17 00:00:00 2001 From: Luke Eltiste Date: Tue, 8 Oct 2024 17:02:27 -0700 Subject: [PATCH 5/6] update ci --- .github/workflows/ci.yaml | 33 +++++++++++++++++++++--------- .github/workflows/docker-image.yml | 1 - 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a9e71fc..6a55cf5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,21 +1,34 @@ name: CI -on: [push, pull_request] + +on: + push: + branches: [main] + pull_request: + branches: [main] jobs: - check: - name: Check + lint: + name: Lint and Format runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + + - name: Run Ruff linter + uses: chartboost/ruff-action@v1 with: args: "check" - format: - name: Format - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 + - name: Check formatting + uses: chartboost/ruff-action@v1 with: args: "format --check" diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 658aa08..3485dfe 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -47,7 +47,6 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | - # set latest tag for default branch type=raw,value=latest,enable={{is_default_branch}} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} From c5d2ce3da05084f2e89195831fe6aab6e265c1ad Mon Sep 17 00:00:00 2001 From: Luke Date: Sat, 19 Oct 2024 14:38:44 -0700 Subject: [PATCH 6/6] retry logic --- github-backup/github-backup.py | 86 ++++++---------------------------- 1 file changed, 14 insertions(+), 72 deletions(-) diff --git a/github-backup/github-backup.py b/github-backup/github-backup.py index eb0b5f3..abc2635 100644 --- a/github-backup/github-backup.py +++ b/github-backup/github-backup.py @@ -10,20 +10,11 @@ from urllib.parse import urlparse, urlunparse import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry def get_json(url: str, session: requests.Session) -> Iterator[dict]: - """ - Fetch JSON data from a URL using pagination, authentication, and handling rate limits. - - Args: - url (str): The base URL to fetch data from. - session (requests.Session): The requests session for making HTTP requests. - token (str): The GitHub personal access token for authentication. - - Yields: - dict: The JSON data from each page of the response. - """ while True: try: response = session.get(url, timeout=10) @@ -60,18 +51,6 @@ def get_json(url: str, session: requests.Session) -> Iterator[dict]: def check_name(name: str) -> str: - """ - Check if a name is valid according to a regular expression pattern. - - Args: - name (str): The name to be checked. - - Raises: - ValueError: If the name is invalid. - - Returns: - str: The validated name. - """ pattern = r"^\w[-\.\w]*$" if not re.match(pattern, name): raise ValueError(f"Invalid name '{name}'") @@ -79,15 +58,6 @@ def check_name(name: str) -> str: def mkdir(path: str) -> bool: - """ - Create a directory if it doesn't exist. - - Args: - path (str): The path to the directory. - - Returns: - bool: True if a new directory was created, False otherwise. - """ try: os.makedirs(path, mode=0o770) return True @@ -98,17 +68,6 @@ def mkdir(path: str) -> bool: def prepare_repo_url(repo_url: str, username: str, token: str) -> str: - """ - Prepare the repository URL for cloning with authentication. - - Args: - repo_url (str): The original repository URL. - username (str): The GitHub username. - token (str): The GitHub personal access token. - - Returns: - str: The prepared repository URL with authentication credentials. - """ parsed = urlparse(repo_url) modified = list(parsed) modified[1] = f"{username}:{token}@{parsed.netloc}" @@ -116,23 +75,10 @@ def prepare_repo_url(repo_url: str, username: str, token: str) -> str: def init_bare_repo(repo_path: str) -> None: - """ - Initialize a bare Git repository at the given path. - - Args: - repo_path (str): The path to the repository. - """ subprocess.run(["git", "init", "--bare", "--quiet"], cwd=repo_path, check=True) def fetch_repo(repo_path: str, repo_url: str) -> None: - """ - Fetch the remote repository into the bare repository. - - Args: - repo_path (str): The path to the bare repository. - repo_url (str): The URL of the remote repository. - """ subprocess.run( [ "git", @@ -149,19 +95,6 @@ def fetch_repo(repo_path: str, repo_url: str) -> None: def mirror(repo_name: str, repo_url: str, to_path: str, username: str, token: str) -> tuple[str, str]: - """ - Mirror a GitHub repository to a local directory. - - Args: - repo_name (str): The name of the repository. - repo_url (str): The URL of the remote repository. - to_path (str): The path to the directory where the repository will be mirrored. - username (str): The GitHub username. - token (str): The GitHub personal access token. - - Returns: - Tuple[str, str]: The owner and name of the mirrored repository. - """ repo_path = os.path.join(to_path, repo_name) os.makedirs(repo_path, exist_ok=True) @@ -173,13 +106,22 @@ def mirror(repo_name: str, repo_url: str, to_path: str, username: str, token: st return repo_name, repo_url.split("/")[-2] +def create_session(retries: int = 3, backoff_factor: float = 0.3) -> requests.Session: + session = requests.Session() + retry = Retry(total=retries, backoff_factor=backoff_factor, status_forcelist=[500, 502, 503, 504]) + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + def main(): parser = argparse.ArgumentParser(description="Backup GitHub repositories") parser.add_argument("config", metavar="CONFIG", help="a configuration file") args = parser.parse_args() - with open(args.config, "rb") as f: - config = json.loads(f.read()) + with open(args.config) as f: + config = json.load(f) owners: list[str] | None = config.get("owners") token: str = config["token"] @@ -187,7 +129,7 @@ def main(): if mkdir(path): print(f"Created directory {path}", file=sys.stderr) - with requests.Session() as session: + with create_session() as session: session.headers.update({"Authorization": f"token {token}"}) user: dict = next(get_json("https://api.github.com/user", session)) for page in get_json("https://api.github.com/user/repos", session):