From eed5549ff6b41769c20c9288ffbf91be76f94e5b Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 2 Feb 2021 12:53:24 +0100 Subject: [PATCH 1/4] Restore task summary in debug log --- esmvalcore/_task.py | 4 ++-- esmvalcore/preprocessor/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 9312b7497e..6c0a6f8403 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -642,8 +642,8 @@ def __repr__(self): settings_string = pprint.pformat(self.settings) string = (f"{self.__class__.__name__}: {self.name}\n" f"script: {self.script}\n" - f"settings:\n{settings_string}\n") - + f"settings:\n{settings_string}\n" + f"{super().str()}\n") return string diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index a4359e209d..3fd2546ca8 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -492,6 +492,6 @@ def __str__(self): self.__class__.__name__, order, products, - super(PreprocessingTask, self).str(), + super().str(), ) return txt From de4831cdf6661b907ff5345f428b6df3c0f94f2d Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 2 Feb 2021 21:16:55 +0100 Subject: [PATCH 2/4] Clean up code and add tests --- esmvalcore/_task.py | 8 +- esmvalcore/preprocessor/__init__.py | 5 +- tests/unit/task/test_print.py | 156 ++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 tests/unit/task/test_print.py diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 6c0a6f8403..08992a1d2b 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -264,12 +264,12 @@ def get_product_attributes(self) -> dict: for product in self.products } - def str(self): + def print_ancestors(self): """Return a nicely formatted description.""" def _indent(txt): - return '\n'.join('\t' + line for line in txt.split('\n')) + return '\n'.join(' ' + line for line in txt.split('\n')) - txt = 'ancestors:\n{}'.format('\n\n'.join( + txt = 'ancestors:\n{}'.format('\n'.join( _indent(str(task)) for task in self.ancestors) if self.ancestors else 'None') return txt @@ -643,7 +643,7 @@ def __repr__(self): string = (f"{self.__class__.__name__}: {self.name}\n" f"script: {self.script}\n" f"settings:\n{settings_string}\n" - f"{super().str()}\n") + f"{self.print_ancestors()}\n") return string diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 3fd2546ca8..1be87e59af 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -488,10 +488,11 @@ def __str__(self): ] products = '\n\n'.join('\n'.join([str(p), pformat(p.settings)]) for p in self.products) - txt = "{}:\norder: {}\n{}\n{}".format( + txt = "{}: {}\norder: {}\n{}\n{}".format( self.__class__.__name__, + self.name, order, products, - super().str(), + self.print_ancestors(), ) return txt diff --git a/tests/unit/task/test_print.py b/tests/unit/task/test_print.py new file mode 100644 index 0000000000..4a021d5c3d --- /dev/null +++ b/tests/unit/task/test_print.py @@ -0,0 +1,156 @@ +"""Test that a task tree can be printed in a human readable form.""" +import copy +import textwrap + +import pytest + +from esmvalcore._task import DiagnosticTask +from esmvalcore.preprocessor import PreprocessingTask, PreprocessorFile + + +@pytest.fixture +def preproc_file(): + return PreprocessorFile( + attributes={'filename': '/output/preproc/file.nc'}, + settings={ + 'extract_levels': { + 'scheme': 'linear', + 'levels': [95000] + }, + }, + ) + + +@pytest.fixture +def preproc_task(preproc_file): + return PreprocessingTask(products=[preproc_file]) + + +@pytest.fixture +def diagnostic_task(monkeypatch): + script = '/some/where/esmvaltool/diag_scripts/test.py' + settings = { + 'run_dir': str('/output/run'), + } + + monkeypatch.setattr(DiagnosticTask, '_initialize_cmd', lambda self: None) + return DiagnosticTask(script, settings, output_dir='/output/run') + + +def test_repr_preproc_task(preproc_task): + """Test printing a preprocessor task.""" + preproc_task.name = 'diag_1/tas' + result = str(preproc_task) + print(result) + + reference = textwrap.dedent(""" + PreprocessingTask: diag_1/tas + order: ['extract_levels', 'save'] + PreprocessorFile: /output/preproc/file.nc + {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + 'save': {'filename': '/output/preproc/file.nc'}} + ancestors: + None + """) + + assert result.strip() == reference.strip() + + +def test_repr_diagnostic_task(diagnostic_task): + """Test printing a diagnostic task.""" + diagnostic_task.name = 'diag_1/script_1' + result = str(diagnostic_task) + print(result) + + reference = textwrap.dedent(""" + DiagnosticTask: diag_1/script_1 + script: /some/where/esmvaltool/diag_scripts/test.py + settings: + {'run_dir': '/output/run'} + ancestors: + None + """) + + assert result.strip() == reference.strip() + + +def test_repr_simple_tree(preproc_task, diagnostic_task): + """Test the most common task tree.""" + preproc_task.name = 'diag_1/tas' + diagnostic_task.name = 'diag_1/script_1' + diagnostic_task.ancestors = [preproc_task] + result = str(diagnostic_task) + print(result) + + reference = textwrap.dedent(""" + DiagnosticTask: diag_1/script_1 + script: /some/where/esmvaltool/diag_scripts/test.py + settings: + {'run_dir': '/output/run'} + ancestors: + PreprocessingTask: diag_1/tas + order: ['extract_levels', 'save'] + PreprocessorFile: /output/preproc/file.nc + {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + 'save': {'filename': '/output/preproc/file.nc'}} + ancestors: + None + """) + + assert result.strip() == reference.strip() + + +def test_repr_full_tree(preproc_task, diagnostic_task): + """Test a more comlicated task tree.""" + derive_input_task_1 = copy.deepcopy(preproc_task) + derive_input_task_1.name = 'diag_1/tas_derive_input_1' + + derive_input_task_2 = copy.deepcopy(preproc_task) + derive_input_task_2.name = 'diag_1/tas_derive_input_2' + + preproc_task.name = 'diag_1/tas' + preproc_task.ancestors = [derive_input_task_1, derive_input_task_2] + + diagnostic_task_1 = copy.deepcopy(diagnostic_task) + diagnostic_task_1.name = 'diag_1/script_1' + diagnostic_task_1.ancestors = [preproc_task] + + diagnostic_task.name = 'diag_1/script_2' + diagnostic_task.ancestors = [diagnostic_task_1] + result = str(diagnostic_task) + print(result) + + reference = textwrap.dedent(""" + DiagnosticTask: diag_1/script_2 + script: /some/where/esmvaltool/diag_scripts/test.py + settings: + {'run_dir': '/output/run'} + ancestors: + DiagnosticTask: diag_1/script_1 + script: /some/where/esmvaltool/diag_scripts/test.py + settings: + {'run_dir': '/output/run'} + ancestors: + PreprocessingTask: diag_1/tas + order: ['extract_levels', 'save'] + PreprocessorFile: /output/preproc/file.nc + {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + 'save': {'filename': '/output/preproc/file.nc'}} + ancestors: + PreprocessingTask: diag_1/tas_derive_input_1 + order: ['extract_levels', 'save'] + PreprocessorFile: /output/preproc/file.nc + {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + 'save': {'filename': '/output/preproc/file.nc'}} + ancestors: + None + PreprocessingTask: diag_1/tas_derive_input_2 + order: ['extract_levels', 'save'] + PreprocessorFile: /output/preproc/file.nc + {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + 'save': {'filename': '/output/preproc/file.nc'}} + ancestors: + None + """) + + assert result.strip() == reference.strip() From 589cdb4f35ee19f171b012d2a2189998eaf42f3a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 3 Feb 2021 10:31:23 +0100 Subject: [PATCH 3/4] Use textwrap.indent instead of homegrown function --- esmvalcore/_task.py | 8 +++----- tests/unit/task/test_print.py | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 08992a1d2b..7975d983d8 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -8,6 +8,7 @@ import pprint import subprocess import sys +import textwrap import threading import time from copy import deepcopy @@ -266,11 +267,8 @@ def get_product_attributes(self) -> dict: def print_ancestors(self): """Return a nicely formatted description.""" - def _indent(txt): - return '\n'.join(' ' + line for line in txt.split('\n')) - - txt = 'ancestors:\n{}'.format('\n'.join( - _indent(str(task)) + txt = 'ancestors:\n{}'.format('\n\n'.join( + textwrap.indent(str(task), prefix=' ') for task in self.ancestors) if self.ancestors else 'None') return txt diff --git a/tests/unit/task/test_print.py b/tests/unit/task/test_print.py index 4a021d5c3d..807ebfc9ef 100644 --- a/tests/unit/task/test_print.py +++ b/tests/unit/task/test_print.py @@ -144,6 +144,7 @@ def test_repr_full_tree(preproc_task, diagnostic_task): 'save': {'filename': '/output/preproc/file.nc'}} ancestors: None + PreprocessingTask: diag_1/tas_derive_input_2 order: ['extract_levels', 'save'] PreprocessorFile: /output/preproc/file.nc From 461528d6a7ca0a0b72a1ed3d311f504531cc51b1 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 3 Feb 2021 12:18:10 +0100 Subject: [PATCH 4/4] Avoid mocking --- tests/unit/task/test_print.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/unit/task/test_print.py b/tests/unit/task/test_print.py index 807ebfc9ef..0db6b32423 100644 --- a/tests/unit/task/test_print.py +++ b/tests/unit/task/test_print.py @@ -27,14 +27,16 @@ def preproc_task(preproc_file): @pytest.fixture -def diagnostic_task(monkeypatch): - script = '/some/where/esmvaltool/diag_scripts/test.py' +def diagnostic_task(tmp_path): + mock_script = tmp_path / 'script.py' + mock_script.touch() settings = { 'run_dir': str('/output/run'), + 'profile_diagnostic': False, } - - monkeypatch.setattr(DiagnosticTask, '_initialize_cmd', lambda self: None) - return DiagnosticTask(script, settings, output_dir='/output/run') + task = DiagnosticTask(mock_script, settings, output_dir='/output/run') + task.script = '/some/where/esmvaltool/diag_scripts/test.py' + return task def test_repr_preproc_task(preproc_task): @@ -66,7 +68,7 @@ def test_repr_diagnostic_task(diagnostic_task): DiagnosticTask: diag_1/script_1 script: /some/where/esmvaltool/diag_scripts/test.py settings: - {'run_dir': '/output/run'} + {'profile_diagnostic': False, 'run_dir': '/output/run'} ancestors: None """) @@ -86,7 +88,7 @@ def test_repr_simple_tree(preproc_task, diagnostic_task): DiagnosticTask: diag_1/script_1 script: /some/where/esmvaltool/diag_scripts/test.py settings: - {'run_dir': '/output/run'} + {'profile_diagnostic': False, 'run_dir': '/output/run'} ancestors: PreprocessingTask: diag_1/tas order: ['extract_levels', 'save'] @@ -124,12 +126,12 @@ def test_repr_full_tree(preproc_task, diagnostic_task): DiagnosticTask: diag_1/script_2 script: /some/where/esmvaltool/diag_scripts/test.py settings: - {'run_dir': '/output/run'} + {'profile_diagnostic': False, 'run_dir': '/output/run'} ancestors: DiagnosticTask: diag_1/script_1 script: /some/where/esmvaltool/diag_scripts/test.py settings: - {'run_dir': '/output/run'} + {'profile_diagnostic': False, 'run_dir': '/output/run'} ancestors: PreprocessingTask: diag_1/tas order: ['extract_levels', 'save']