From: Yaroslav Halchenko <debian@onerussian.com>
Subject: centralize execution of coverage in the tests and use "debian-compatible" invocation
Origin: Debian
Forwarded: https://github.com/NeurodataWithoutBorders/pynwb/pull/1811
Last-Update: 2023-12-21

--- a/tests/validation/test_validate.py
+++ b/tests/validation/test_validate.py
@@ -1,5 +1,6 @@
 import subprocess
 import re
+import sys
 from unittest.mock import patch
 from io import StringIO
 
@@ -7,26 +8,35 @@ from pynwb.testing import TestCase
 from pynwb import validate, NWBHDF5IO
 
 
+# NOTE we use "coverage run -m pynwb.validate" instead of "python -m pynwb.validate"
+# so that we can both test pynwb.validate and compute code coverage from that test.
+# NOTE we also use "coverage run -p" which will generate a .coverage file with the
+# machine name, process id, and a random number appended to the filename to
+# simplify collecting and merging coverage data from multiple subprocesses. if "-p"
+# is not used, then each "coverage run" will overwrite the .coverage file from a
+# previous "coverage run".
+# NOTE we run "coverage" as "{sys.executable} -m coverage" to 1. make sure to use
+# the same python version, and on Debian systems executable is "python3-coverage", not
+# just "coverage".
+# NOTE the run_coverage.yml GitHub Action runs "python -m coverage combine" to
+# combine the individual coverage reports into one .coverage file.
+def run_coverage(extra_args: list[str]):
+    return subprocess.run(
+        [sys.executable, "-m", "coverage", "run", "-p", "-m", "pynwb.validate"]
+        + extra_args,
+        capture_output=True
+    )
+
+
 class TestValidateCLI(TestCase):
 
     # 1.0.2_nwbfile.nwb has no cached specifications
     # 1.0.3_nwbfile.nwb has cached "core" specification
     # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specifications
 
-    # NOTE we use "coverage run -m pynwb.validate" instead of "python -m pynwb.validate"
-    # so that we can both test pynwb.validate and compute code coverage from that test.
-    # NOTE we also use "coverage run -p" which will generate a .coverage file with the
-    # machine name, process id, and a random number appended to the filename to
-    # simplify collecting and merging coverage data from multiple subprocesses. if "-p"
-    # is not used, then each "coverage run" will overwrite the .coverage file from a
-    # previous "coverage run".
-    # NOTE the run_coverage.yml GitHub Action runs "python -m coverage combine" to
-    # combine the individual coverage reports into one .coverage file.
-
     def test_validate_file_no_cache(self):
         """Test that validating a file with no cached spec against the core namespace succeeds."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate",
-                                 "tests/back_compat/1.0.2_nwbfile.nwb"], capture_output=True)
+        result = run_coverage(["tests/back_compat/1.0.2_nwbfile.nwb"])
 
         stderr_regex = re.compile(
             r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*"
@@ -43,8 +53,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_no_cache_bad_ns(self):
         """Test that validating a file with no cached spec against a specified, unknown namespace fails."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.0.2_nwbfile.nwb",
-                                 "--ns", "notfound"], capture_output=True)
+        result = run_coverage(["tests/back_compat/1.0.2_nwbfile.nwb", "--ns", "notfound"])
 
         stderr_regex = re.compile(
             r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*"
@@ -60,8 +69,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_cached(self):
         """Test that validating a file with cached spec against its cached namespace succeeds."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate",
-                                 "tests/back_compat/1.1.2_nwbfile.nwb"], capture_output=True)
+        result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb"])
 
         self.assertEqual(result.stderr.decode('utf-8'), '')
 
@@ -72,8 +80,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_cached_bad_ns(self):
         """Test that validating a file with cached spec against a specified, unknown namespace fails."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate",
-                                 "tests/back_compat/1.1.2_nwbfile.nwb", "--ns", "notfound"], capture_output=True)
+        result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb", "--ns", "notfound"])
 
         stderr_regex = re.compile(
             r"The namespace 'notfound' could not be found in cached namespace information as only "
@@ -85,8 +92,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_cached_extension(self):
         """Test that validating a file with cached spec against the cached namespaces succeeds."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate",
-                                 "tests/back_compat/2.1.0_nwbfile_with_extension.nwb"], capture_output=True)
+        result = run_coverage(["tests/back_compat/2.1.0_nwbfile_with_extension.nwb"])
 
         self.assertEqual(result.stderr.decode('utf-8'), '')
 
@@ -97,9 +103,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_cached_extension_pass_ns(self):
         """Test that validating a file with cached spec against the extension namespace succeeds."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate",
-                                 "tests/back_compat/2.1.0_nwbfile_with_extension.nwb",
-                                 "--ns", "ndx-testextension"], capture_output=True)
+        result = run_coverage(["tests/back_compat/2.1.0_nwbfile_with_extension.nwb", "--ns", "ndx-testextension"])
 
         self.assertEqual(result.stderr.decode('utf-8'), '')
 
@@ -110,9 +114,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_cached_core(self):
         """Test that validating a file with cached spec against the core namespace succeeds."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate",
-                                 "tests/back_compat/2.1.0_nwbfile_with_extension.nwb",
-                                 "--ns", "core"], capture_output=True)
+        result = run_coverage(["tests/back_compat/2.1.0_nwbfile_with_extension.nwb", "--ns", "core"])
 
         stdout_regex = re.compile(
             r"The namespace 'core' is included by the namespace 'ndx-testextension'. "
@@ -122,8 +124,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_cached_hdmf_common(self):
         """Test that validating a file with cached spec against the hdmf-common namespace fails."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.1.2_nwbfile.nwb",
-                                 "--ns", "hdmf-common"], capture_output=True)
+        result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb", "--ns", "hdmf-common"])
 
         stderr_regex = re.compile(
             r"The namespace 'hdmf-common' is included by the namespace 'core'\. Please validate against that "
@@ -133,8 +134,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_cached_ignore(self):
         """Test that validating a file with cached spec against the core namespace succeeds."""
-        result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.1.2_nwbfile.nwb",
-                                 "--no-cached-namespace"], capture_output=True)
+        result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb", "--no-cached-namespace"]  )
 
         self.assertEqual(result.stderr.decode('utf-8'), '')
 
@@ -145,13 +145,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_invalid(self):
         """Test that validating an invalid file outputs errors."""
-        result = subprocess.run(
-            [
-                "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.0.2_str_experimenter.nwb",
-                "--no-cached-namespace"
-            ],
-            capture_output=True
-        )
+        result = run_coverage(["tests/back_compat/1.0.2_str_experimenter.nwb", "--no-cached-namespace"])
 
         stderr_regex = re.compile(
             r" - found the following errors:\s*"
@@ -167,13 +161,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_list_namespaces_core(self):
         """Test listing namespaces from a file"""
-        result = subprocess.run(
-            [
-                "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.1.2_nwbfile.nwb",
-                "--list-namespaces"
-            ],
-            capture_output=True
-        )
+        result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb", "--list-namespaces"])
 
         self.assertEqual(result.stderr.decode('utf-8'), '')
 
@@ -182,13 +170,7 @@ class TestValidateCLI(TestCase):
 
     def test_validate_file_list_namespaces_extension(self):
         """Test listing namespaces from a file with an extension"""
-        result = subprocess.run(
-            [
-                "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/2.1.0_nwbfile_with_extension.nwb",
-                "--list-namespaces"
-            ],
-            capture_output=True
-        )
+        result = run_coverage(["tests/back_compat/2.1.0_nwbfile_with_extension.nwb", "--list-namespaces"])
 
         self.assertEqual(result.stderr.decode('utf-8'), '')
 
