feat: autoamted benchmakrs + minimalistic static site

.github/workflows/benchmark-pages.yml

@@ -0,0 +1,98 @@
+name: Benchmark and Publish Pages
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  benchmark-and-build:
+    runs-on: ubuntu-latest
+    env:
+      PYTHONUNBUFFERED: "1"
+      DATASET_XZ_URL: "https://huggingface.co/datasets/ladybugdb/livejournal-4m-35m/resolve/main/livejournal-csr.duckdb.xz"
+      DATASET_XZ_PATH: "livejournal-csr.duckdb.xz"
+      DATASET_DB_PATH: "livejournal-csr.duckdb"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup Java (JDK 21)
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: "21"
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install dependencies
+        run: uv sync --group docs
+
+      - name: Download and unpack benchmark data
+        run: |
+          curl -L "$DATASET_XZ_URL" -o "$DATASET_XZ_PATH"
+          xz -d -f "$DATASET_XZ_PATH"
+          test -f "$DATASET_DB_PATH"
+
+      - name: Run icebug scenario
+        run: |
+          uv run python benchmark_pagerank_memory.py compare \
+            --engine icebug \
+            --duckdb-memory-limit 200MB \
+            --output ci-icebug-results.csv
+
+      - name: Run graphframes scenario
+        env:
+          SPARK_TESTING: "1"
+        run: |
+          uv run python benchmark_pagerank_memory.py compare \
+            --engine graphframes \
+            --duckdb-memory-limit 2048MB \
+            --spark-driver-memory 4G \
+            --output ci-graphframes-results.csv
+
+      - name: Generate results markdown and plots
+        run: |
+          uv run python scripts/render_results.py \
+            --icebug-csv ci-icebug-results.csv \
+            --graphframes-csv ci-graphframes-results.csv \
+            --template templates/results.md.j2 \
+            --results-md docs/results.md \
+            --assets-dir docs/assets
+
+      - name: Configure Pages
+        uses: actions/configure-pages@v5
+
+      - name: Build MkDocs site
+        run: uv run mkdocs build --strict
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+  deploy:
+    needs: benchmark-and-build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -0,0 +1,3 @@
+.dir-locals.el
+*.pyc
+*.xz

benchmark_pagerank_memory.py

@@ -290,7 +290,7 @@ def build_spark(args: argparse.Namespace):
         .config("spark.jars.packages", args.spark_package)
     )
     spark = spark_builder.getOrCreate()
-    spark.sparkContext.setLogLevel("WARN")
+    spark.sparkContext.setLogLevel("ERROR")
     return spark
 
 

docs/index.md

@@ -0,0 +1,25 @@
+# Icebug vs GraphFrames
+
+This site presents a focused benchmark comparison between **icebug** and
+**GraphFrames** on the LiveJournal graph dataset.
+
+The benchmark data is stored in **icebug-format** (CSR arrays inside DuckDB).
+icebug consumes this format directly.
+
+GraphFrames does not natively consume icebug-format, so each GraphFrames run
+includes a long **prepare** step that converts data to Parquet before the
+algorithm starts.
+
+The benchmark runs on a standard GitHub-hosted runner in CI with fixed memory
+constraints, captures execution and memory metrics, and publishes reproducible
+results.
+
+| Component | Version | Notes |
+|---|---|---|
+| Runner | GitHub Actions `ubuntu-latest` | Standard GitHub-hosted Linux runner |
+| Python | `3.12` | Installed via `actions/setup-python` |
+| GraphFrames | `0.11.0` | `graphframes-py` package |
+| icebug | `12.6` | Installed from `uv.lock` |
+| Java | JDK `21` | Installed via `actions/setup-java` (Temurin) |
+
+Go to the [Results](results.md) page for the latest generated table and plots.

docs/results.md

@@ -0,0 +1,3 @@
+# Results
+
+Results are generated automatically by CI.

mkdocs.yml

@@ -0,0 +1,7 @@
+site_name: Icebug vs GraphFrames
+site_description: Minimal benchmark site for Icebug and GraphFrames comparison
+theme:
+  name: mkdocs
+nav:
+  - Home: index.md
+  - Results: results.md

pyproject.toml

@@ -15,6 +15,13 @@ dependencies = [
     "typing-extensions>=4.15.0",
 ]
 
+[dependency-groups]
+docs = [
+    "jinja2>=3.1.6",
+    "matplotlib>=3.10.3",
+    "mkdocs>=1.6.1",
+]
+
 [tool.uv]
 package = false
 

scripts/render_results.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import csv
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def _as_float(value: str | None) -> float | None:
+    if value is None or value == "":
+        return None
+    try:
+        return float(value)
+    except ValueError:
+        return None
+
+
+def load_row(path: Path, scenario_name: str) -> dict[str, str]:
+    with path.open("r", encoding="utf-8", newline="") as handle:
+        reader = csv.DictReader(handle)
+        row = next(reader, None)
+        if row is None:
+            raise ValueError(f"No rows in CSV: {path}")
+
+    status = "ok" if row.get("returncode") == "0" else "failed"
+    total_seconds = _as_float(row.get("total_seconds"))
+    prepare_seconds = _as_float(row.get("prepare_seconds"))
+    if prepare_seconds is None:
+        prepare_seconds = _as_float(row.get("load_seconds"))
+    peak_rss_human = row.get("peak_rss_human", "n/a")
+    peak_rss_bytes = _as_float(row.get("peak_rss_bytes")) or 0.0
+
+    return {
+        "scenario": scenario_name,
+        "status": status,
+        "peak_rss_human": peak_rss_human,
+        "peak_rss_bytes": f"{peak_rss_bytes}",
+        "peak_rss_gib": f"{peak_rss_bytes / (1024 ** 3)}",
+        "prepare_seconds": f"{prepare_seconds:.2f}" if prepare_seconds is not None else "n/a",
+        "prepare_seconds_raw": "" if prepare_seconds is None else f"{prepare_seconds}",
+        "total_seconds": f"{total_seconds:.2f}" if total_seconds is not None else "n/a",
+        "total_seconds_raw": "" if total_seconds is None else f"{total_seconds}",
+    }
+
+
+def render_markdown(rows: list[dict[str, str]], template_dir: Path, template_name: str, output_path: Path) -> None:
+    env = Environment(loader=FileSystemLoader(template_dir.as_posix()), autoescape=False)
+    template = env.get_template(template_name)
+    output = template.render(rows=rows)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(output, encoding="utf-8")
+
+
+def plot_metric(rows: list[dict[str, str]], value_key: str, title: str, ylabel: str, output_path: Path) -> None:
+    labels = [row["scenario"] for row in rows]
+    values: list[float] = []
+    for row in rows:
+        raw = row.get(value_key, "")
+        try:
+            values.append(float(raw))
+        except ValueError:
+            values.append(0.0)
+
+    fig, ax = plt.subplots(figsize=(8, 4.5))
+    bars = ax.bar(labels, values, color=["#2f6f5f", "#b45a3c"])
+    ax.set_title(title)
+    ax.set_ylabel(ylabel)
+    ax.grid(axis="y", linestyle="--", alpha=0.3)
+    ax.set_axisbelow(True)
+
+    for bar, value in zip(bars, values):
+        ax.text(bar.get_x() + bar.get_width() / 2.0, bar.get_height(), f"{value:.2f}", ha="center", va="bottom", fontsize=9)
+
+    fig.tight_layout()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output_path, dpi=150)
+    plt.close(fig)
+
+
+def plot_time_grouped(rows: list[dict[str, str]], output_path: Path) -> None:
+    labels = [row["scenario"] for row in rows]
+    prepare_values: list[float] = []
+    total_values: list[float] = []
+
+    for row in rows:
+        prepare_raw = row.get("prepare_seconds_raw", "")
+        total_raw = row.get("total_seconds_raw", "")
+        try:
+            prepare_values.append(float(prepare_raw))
+        except ValueError:
+            prepare_values.append(0.0)
+        try:
+            total_values.append(float(total_raw))
+        except ValueError:
+            total_values.append(0.0)
+
+    x = np.arange(len(labels))
+    width = 0.36
+
+    fig, ax = plt.subplots(figsize=(8, 4.5))
+    prepare_bars = ax.bar(x - width / 2.0, prepare_values, width, label="Prepare time", color="#2f6f5f")
+    total_bars = ax.bar(x + width / 2.0, total_values, width, label="Total time", color="#b45a3c")
+
+    ax.set_title("Runtime Comparison")
+    ax.set_ylabel("Time (seconds)")
+    ax.set_xticks(x)
+    ax.set_xticklabels(labels)
+    ax.grid(axis="y", linestyle="--", alpha=0.3)
+    ax.set_axisbelow(True)
+    ax.legend()
+
+    for bars in (prepare_bars, total_bars):
+        for bar in bars:
+            value = bar.get_height()
+            ax.text(bar.get_x() + bar.get_width() / 2.0, value, f"{value:.2f}", ha="center", va="bottom", fontsize=9)
+
+    fig.tight_layout()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output_path, dpi=150)
+    plt.close(fig)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--icebug-csv", required=True)
+    parser.add_argument("--graphframes-csv", required=True)
+    parser.add_argument("--template", default="templates/results.md.j2")
+    parser.add_argument("--results-md", default="docs/results.md")
+    parser.add_argument("--assets-dir", default="docs/assets")
+    args = parser.parse_args()
+
+    icebug = load_row(Path(args.icebug_csv), "icebug")
+    graphframes = load_row(Path(args.graphframes_csv), "graphframes")
+    rows = [icebug, graphframes]
+
+    template_path = Path(args.template)
+    render_markdown(
+        rows,
+        template_dir=template_path.parent,
+        template_name=template_path.name,
+        output_path=Path(args.results_md),
+    )
+
+    assets_dir = Path(args.assets_dir)
+    plot_metric(
+        rows,
+        value_key="peak_rss_gib",
+        title="Memory Usage Comparison",
+        ylabel="Peak RSS (GiB)",
+        output_path=assets_dir / "memory_usage_comparison.png",
+    )
+    plot_time_grouped(rows, output_path=assets_dir / "runtime_comparison.png")
+
+
+if __name__ == "__main__":
+    main()

templates/results.md.j2

@@ -0,0 +1,17 @@
+# Results
+
+Generated from CI benchmark runs.
+
+| Scenario | Status | Peak RSS | Prepare Time (s) | Total Time (s) |
+|---|---|---:|---:|---:|
+{% for row in rows -%}
+| {{ row.scenario }} | {{ row.status }} | {{ row.peak_rss_human }} | {{ row.prepare_seconds }} | {{ row.total_seconds }} |
+{% endfor %}
+
+## Memory Usage Comparison
+
+![Memory usage comparison](assets/memory_usage_comparison.png)
+
+## Runtime Comparison
+
+![Runtime comparison](assets/runtime_comparison.png)