Add failed test for inherits = ".." in user-defined benchmarks.

pyperformance/tests/__init__.py

@@ -1,8 +1,27 @@
 import contextlib
 import errno
 import os
+import subprocess
+import sys
 import tempfile
 
+DATA_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), 'data'))
+
+
+def run_cmd(cmd):
+    print("Execute: %s" % ' '.join(cmd))
+    proc = subprocess.Popen(cmd)
+    try:
+        proc.wait()
+    except:  # noqa
+        proc.kill()
+        proc.wait()
+        raise
+
+    exitcode = proc.returncode
+    if exitcode:
+        sys.exit(exitcode)
+
 
 @contextlib.contextmanager
 def temporary_file(**kwargs):

pyperformance/tests/data/user_defined_bm/MANIFEST

@@ -0,0 +1,4 @@
+[benchmarks]
+
+name	metafile
+1	<local>

pyperformance/tests/data/user_defined_bm/base.toml

@@ -0,0 +1,3 @@
+[project]
+dynamic = ["name"]
+version = "1.0.0"

pyperformance/tests/data/user_defined_bm/bm_1/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "test_bm_1"
+requires-python = ">=3.8"
+dependencies = ["pyperf"]
+urls = { repository = "https://github.com/python/pyperformance" }
+dynamic = ["version"]
+
+[tool.pyperformance]
+name = "1"
+tags = "test"
+inherits = ".."

pyperformance/tests/test_compare.py

@@ -7,24 +7,7 @@
 import unittest
 
 from pyperformance import tests
-
-
-DATA_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), 'data'))
-
-
-def run_cmd(cmd):
-    print("Execute: %s" % ' '.join(cmd))
-    proc = subprocess.Popen(cmd)
-    try:
-        proc.wait()
-    except:   # noqa
-        proc.kill()
-        proc.wait()
-        raise
-
-    exitcode = proc.returncode
-    if exitcode:
-        sys.exit(exitcode)
+from pyperformance.tests import run_cmd, DATA_DIR
 
 
 class CompareTests(unittest.TestCase):

pyperformance/tests/test_user_defined_bm.py

@@ -0,0 +1,18 @@
+import os.path
+import sys
+import unittest
+
+from pyperformance.tests import DATA_DIR, run_cmd
+
+USER_DEFINED_MANIFEST = os.path.join(DATA_DIR, 'user_defined_bm', 'MANIFEST')
+
+
+class TestBM(unittest.TestCase):
+    def test_user_defined_bm(self):
+        cmd = [sys.executable, '-m', 'pyperformance', 'run', f'--manifest={USER_DEFINED_MANIFEST}']
+
+        run_cmd(cmd)
+
+
+if __name__ == "__main__":
+    unittest.main()

runtests.py

@@ -78,7 +78,8 @@ def run_bench(*cmd):
 def main():
     # Unit tests
     cmd = [sys.executable, '-u',
-           os.path.join('pyperformance', 'tests', 'test_compare.py')]
+           os.path.join('pyperformance', 'tests', 'test_compare.py'),
+           os.path.join('pyperformance', 'tests', 'test_user_defined_bm.py')]
     run_cmd(cmd)
 
     # Functional tests