From d5e0070f599ed68fa47dbf477b1776a867298633 Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 15 Jun 2022 10:01:19 +0200 Subject: [PATCH 1/7] rely on pytest-reportlog to produce the log file --- .github/workflows/parse_logs.py | 123 ++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/.github/workflows/parse_logs.py b/.github/workflows/parse_logs.py index 545beaa4167..c0674aeac0b 100644 --- a/.github/workflows/parse_logs.py +++ b/.github/workflows/parse_logs.py @@ -1,57 +1,102 @@ # type: ignore import argparse -import itertools +import functools +import json import pathlib import textwrap +from dataclasses import dataclass -parser = argparse.ArgumentParser() -parser.add_argument("filepaths", nargs="+", type=pathlib.Path) -args = parser.parse_args() +from pytest import CollectReport, TestReport -filepaths = sorted(p for p in args.filepaths if p.is_file()) +@dataclass +class SessionStart: + pytest_version: str + outcome: str = "status" -def extract_short_test_summary_info(lines): - up_to_start_of_section = itertools.dropwhile( - lambda l: "=== short test summary info ===" not in l, - lines, - ) - up_to_section_content = itertools.islice(up_to_start_of_section, 1, None) - section_content = itertools.takewhile( - lambda l: l.startswith("FAILED") or l.startswith("ERROR"), up_to_section_content - ) - content = "\n".join(section_content) + @classmethod + def _from_json(cls, json): + json_ = json.copy() + json_.pop("$report_type") + return cls(**json_) - return content +@dataclass +class SessionFinish: + exitstatus: str + outcome: str = "status" -def format_log_message(path): - py_version = path.name.split("-")[1] - summary = f"Python {py_version} Test Summary Info" - with open(path) as f: - data = extract_short_test_summary_info(line.rstrip() for line in f) - message = ( - textwrap.dedent( - """\ -
{summary} + @classmethod + def _from_json(cls, json): + json_ = json.copy() + json_.pop("$report_type") + return cls(**json_) - ``` - {data} - ``` -
- """ - ) - .rstrip() - .format(summary=summary, data=data) - ) +def parse_record(record): + report_types = { + "TestReport": TestReport, + "CollectReport": CollectReport, + "SessionStart": SessionStart, + "SessionFinish": SessionFinish, + } + cls = report_types.get(record["$report_type"]) + if cls is None: + raise ValueError(f"unknown report type: {record['$report_type']}") + return cls._from_json(record) + + +@functools.singledispatch +def format_summary(report): + return f"{report.nodeid}: {report}" + + +@format_summary.register +def _(report: TestReport): + message = report.longrepr.chain[0][1].message + return f"{report.nodeid}: {message}" + + +@format_summary.register +def _(report: CollectReport): + message = report.longrepr.split("\n")[-1].removeprefix("E").lstrip() + return f"{report.nodeid}: {message}" + + +def format_report(reports, py_version): + newline = "\n" + summaries = newline.join(format_summary(r) for r in reports) + message = textwrap.dedent( + """\ +
Python {py_version} Test Summary + + ``` + {summaries} + ``` + +
+ """ + ).format(summaries=summaries, py_version=py_version) return message -print("Parsing logs ...") -message = "\n\n".join(format_log_message(path) for path in filepaths) +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("filepath", type=pathlib.Path) + args = parser.parse_args() + + py_version = args.filepath.stem.split("-")[1] + + print("Parsing logs ...") + + lines = args.filepath.read_text().splitlines() + reports = [parse_record(json.loads(line)) for line in lines] + + failed = [report for report in reports if report.outcome == "failed"] + + message = format_report(failed, py_version=py_version) -output_file = pathlib.Path("pytest-logs.txt") -print(f"Writing output file to: {output_file.absolute()}") -output_file.write_text(message) + output_file = pathlib.Path("pytest-logs.txt") + print(f"Writing output file to: {output_file.absolute()}") + output_file.write_text(message) From 5b2566b868654a564e13d75c2e1311c36588c0e0 Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 15 Jun 2022 10:13:07 +0200 Subject: [PATCH 2/7] use the new script to generate the report --- .github/workflows/upstream-dev-ci.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index 8dac95b53c0..dcf0091cdde 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -62,6 +62,7 @@ jobs: environment-name: xarray-tests extra-specs: | python=${{ matrix.python-version }} + pytest-reportlog - name: Install upstream versions run: | bash ci/install-upstream-wheels.sh @@ -80,10 +81,11 @@ jobs: if: success() id: status run: | - set -euo pipefail - python -m pytest --timeout=60 -rf | tee output-${{ matrix.python-version }}-log || ( + python -m pytest --timeout=60 -rf \ + --report-log output-${{ matrix.python-version }}-log.jsonl \ + || ( echo '::set-output name=ARTIFACTS_AVAILABLE::true' && false - ) + ) - name: Upload artifacts if: | failure() @@ -122,7 +124,7 @@ jobs: - name: Parse logs run: | shopt -s globstar - python .github/workflows/parse_logs.py logs/**/*-log + python .github/workflows/parse_logs.py logs/**/*-log* - name: Report failures uses: actions/github-script@v6 with: From 502ff89c4cd889b4e5759d1b8151224a9d149dbf Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 15 Jun 2022 10:16:50 +0200 Subject: [PATCH 3/7] install the dependencies of the parser script --- .github/workflows/upstream-dev-ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index dcf0091cdde..4aeb4d7b475 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -121,6 +121,9 @@ jobs: run: | rsync -a /tmp/workspace/logs/output-*/ ./logs ls -R ./logs + - name: install dependencies + run: | + python -m pip install pytest - name: Parse logs run: | shopt -s globstar From 46c3754ae0aeca401462eb9e445105c443063ba5 Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 15 Jun 2022 10:17:06 +0200 Subject: [PATCH 4/7] temporarily enable report generation [test-upstream] --- .github/workflows/upstream-dev-ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index 4aeb4d7b475..86f476bd50a 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -90,7 +90,6 @@ jobs: if: | failure() && steps.status.outcome == 'failure' - && github.event_name == 'schedule' && github.repository == 'pydata/xarray' uses: actions/upload-artifact@v3 with: @@ -103,7 +102,6 @@ jobs: needs: upstream-dev if: | failure() - && github.event_name == 'schedule' && needs.upstream-dev.outputs.artifacts_availability == 'true' runs-on: ubuntu-latest defaults: @@ -129,6 +127,7 @@ jobs: shopt -s globstar python .github/workflows/parse_logs.py logs/**/*-log* - name: Report failures + if: github.event_name == 'schedule' uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }} From bd843b26214a739a0d0219bd6a0d6018ebd10154 Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 15 Jun 2022 10:54:22 +0200 Subject: [PATCH 5/7] actually upload the log file [skip-ci][test-upstream] --- .github/workflows/upstream-dev-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index 86f476bd50a..3b766ad2135 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -93,8 +93,8 @@ jobs: && github.repository == 'pydata/xarray' uses: actions/upload-artifact@v3 with: - name: output-${{ matrix.python-version }}-log - path: output-${{ matrix.python-version }}-log + name: output-${{ matrix.python-version }}-log.jsonl + path: output-${{ matrix.python-version }}-log.jsonl retention-days: 5 report: From 803db5c7bc0ad760b6a1d6ea159be3bfbd564fc1 Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 15 Jun 2022 11:03:56 +0200 Subject: [PATCH 6/7] print the created report [skip-ci][test-upstream] --- .github/workflows/upstream-dev-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index 3b766ad2135..21b3dfe8a30 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -126,6 +126,7 @@ jobs: run: | shopt -s globstar python .github/workflows/parse_logs.py logs/**/*-log* + cat pytest-logs.txt - name: Report failures if: github.event_name == 'schedule' uses: actions/github-script@v6 From c59ee7b6b1cead6aafb2d0ae15ad734ef6f7c4f3 Mon Sep 17 00:00:00 2001 From: Keewis Date: Wed, 15 Jun 2022 11:18:20 +0200 Subject: [PATCH 7/7] Undo the temporary changes [test-upstream] --- .github/workflows/upstream-dev-ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index 21b3dfe8a30..51ea9720f5b 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -90,6 +90,7 @@ jobs: if: | failure() && steps.status.outcome == 'failure' + && github.event_name == 'schedule' && github.repository == 'pydata/xarray' uses: actions/upload-artifact@v3 with: @@ -102,6 +103,7 @@ jobs: needs: upstream-dev if: | failure() + && github.event_name == 'schedule' && needs.upstream-dev.outputs.artifacts_availability == 'true' runs-on: ubuntu-latest defaults: @@ -128,7 +130,6 @@ jobs: python .github/workflows/parse_logs.py logs/**/*-log* cat pytest-logs.txt - name: Report failures - if: github.event_name == 'schedule' uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }}