cleanup (#18)

- remove demo runner

- improve docs

- rename all tests to test_* (previously, also setup_* and cleanup_* existed) to improve stability as it is not guaranteed that pytest.ini is loaded.

- improve logging formatting

- improve full integration test

Reviewed-on: local-it-infrastructure/e2e_tests#18
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

docs/documentation.md

@@ -222,6 +222,7 @@ To understand how a test suite is built, let's have a look at the files
 runner_authentik.py  ->  required, defines the Runner subclass (see below)
 conftest.py  ->  not required. special file for pytest. is automatically discovered and loaded. convenient place to define fixtures and functions to be used in more than one test routine
 setup_authentik.py -> not required. can hold setup routine for authentik, has to be registered in runner_authentik.py
+fixtures_authentik.py -> not required. holds fixtures that are meant to be imported by other test modules that depend on authentik.
 
 # Create a custom Runner
 

pyproject.toml

@@ -59,7 +59,6 @@ line-length = 120
 target-version = "py311"
 
 [tool.pytest.ini_options]
-python_functions = "setup_* test_* cleanup_*"
 norecursedirs = ".* previous-work recipes"
 testpaths = "tests"
 markers = [

pytest_abra/cli.py

@@ -1,5 +1,6 @@
 import argparse
 import os
+import sys
 from pathlib import Path
 
 from loguru import logger
@@ -19,10 +20,11 @@ def run():
     parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;", required=True)
     parser.add_argument("--recipes_dir", type=Path, help="Dir of abra recipes and respective runners", required=True)
     parser.add_argument("--output_dir", type=Path, help="Dir of test outputs", required=True)
-    parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=20_000)
+    parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=30_000)
     parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode")
     parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests")
     parser.add_argument("--session_id", help="Session dir name (inside output_dir). Overwrites --resume")
+    parser.add_argument("--cleanup", help="Force test cleanup. Should not be necessary")
 
     args = parser.parse_args()
     env_paths = [Path(s) for s in args.env_paths.split(";")]
@@ -41,7 +43,9 @@ def run():
     # todo: move to Coordinator
     DIR = DirManager(output_dir=args.output_dir, session_id=session_id)
     log_file = DIR.RESULTS / "coordinator.log"
-    logger.add(log_file)
+    logger.remove()
+    logger.add(log_file, format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}")
+    logger.add(sys.stdout, colorize=True, format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> <level>{message}</level>")
 
     # ---------------------------- initialize and run ---------------------------- #
 

pytest_abra/custom_fixtures.py

@@ -22,7 +22,7 @@ def pytest_addoption(parser: Parser):
     parser.addoption("--runner_index", action="store", type=int)
     parser.addoption("--output_dir", action="store", type=Path)
     parser.addoption("--session_id", action="store", type=str)
-    parser.addoption("--timeout", action="store", type=int, default=20_000)
+    parser.addoption("--timeout", action="store", type=int, default=30_000)
 
 
 @pytest.fixture(autouse=True)

pytest_abra/runner.py

@@ -149,6 +149,7 @@ class Runner:
         # command_arguments.append("--traceconfig")
 
         command_arguments.append("-v")
+
         command_arguments.append(str(full_test_path))
 
         command_arguments.append("--runner_index")

recipes/authentik/tests_authentik/cleanup_authentik.py

@@ -27,7 +27,7 @@ def remove_user(admin_context: BrowserContext, URL: BaseUrl):
     page.get_by_role("dialog").get_by_role("button", name=re.compile(r"Löschen|Delete")).click()
 
 
-def cleanup_delete_user(
+def test_cleanup_delete_user(
     context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists
 ):
     # load admin cookies to context

recipes/authentik/tests_authentik/setup_authentik.py

@@ -12,7 +12,7 @@ TEST_USER = os.environ["TEST_USER"]
 TEST_PASS = os.environ["TEST_PASS"]
 
 
-def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl):
+def test_setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl):
     # go to page
     page = context.new_page()
     page.goto(URL.get())
@@ -50,7 +50,8 @@ def create_invite_link(admin_context: BrowserContext, env_config: dict[str, str]
     page.locator('input[name="name"]').click()
     linkname = "test_link_123"
     page.locator('input[name="name"]').fill(linkname)
-    page.get_by_placeholder("Wählen Sie ein Objekt aus.").click()
+    placeholder_pattern = re.compile(r"Wählen Sie ein|Select an")
+    page.get_by_placeholder(placeholder_pattern).click()
     page.get_by_role("option", name=re.compile(r"invitation-enrollment-flow")).click()
 
     # force, because else we get "intercepts pointer events"
@@ -82,7 +83,7 @@ def create_user(user_context: BrowserContext, invitelink):
     expect(page.locator("ak-library")).to_be_visible()
 
 
-def setup_user_state(
+def test_setup_user_state(
     context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists
 ):
     # load admin cookies to context

recipes/demo/tests_demo/fixtures_demo.py

@@ -1,26 +0,0 @@
-"""
-This file can be used to define fixtures thate are then used by other tests which
-depend on [demo]. For this to work
-
-1. the Runner class of the other test needs to define the depencency as seen
-   by referencing RunnerDemo in the dependencies list:
-
-from pytest_abra.tests_demo.runner_demo import RunnerDemo
-
-class RunnerOther(Runner):
-    dependencies = [RunnerDemo]
-
-
-2. the specific tests that rely on these fixtures need to import the fixtures.
-   To globally import for all tests in 'other', the import should be done in conftest:
-
-in 'conftest.py' in 'test_other' dir:
-from pytest_abra.tests_demo.fixtures_demo import demo_fixture
-"""
-
-import pytest
-
-
-@pytest.fixture
-def demo_fixture():
-    return ""

recipes/demo/tests_demo/runner_demo.py

@@ -1,24 +0,0 @@
-from pytest_abra.runner import Runner, Test
-
-
-class RunnerDemo(Runner):
-    """Every env file has a corresponding runner class"""
-
-    env_type = "demo"  # name of the test, used for logging / output naming
-
-    # this indicates that tests from RunnerDemo depend on the setup from RunnerAuthentik.
-    # RunnerDemo will only execute, when setup_authentik.py has finished successfully.
-    # For example, setup_authentik.py generates session states, that can be used as fixtures
-    # that can be loaded from fixtures_authentik.py
-    dependencies: list[str] = ["authentik"]
-
-    # todo: update these comments
-    # Filename of Demo setup. If defined, it will run 1st by executing pytest
-    # Filename of Demo test. This file contains unconditional tests that will be run in any
-    # case. If defined, it will run 2nd by executing pytest
-    # this list can hold many more tests from RunnerDemo that run conditional. The condition
-    # and the test file can be defined by creating a ConditionalTest instance:
-    # ConditionalTest(condition: Callable, test_file: str)
-    setups: list[Test] = []
-    tests: list[Test] = []
-    cleanups: list[Test] = []

recipes/demo/tests_demo/setup_demo.py

@@ -1,3 +0,0 @@
-# Define functions here that are specifically meant for setup, not for testing. This means
-# all actions that simply are required for other tests from 'demo' to run. Runs before all
-# tests from 'demo'.

recipes/nextcloud/tests_nextcloud/runner_nextcloud.py

@@ -6,7 +6,7 @@ class RunnerNextcloud(Runner):
     dependencies = ["authentik"]
     setups = [Test(test_file="setup_nextcloud.py", prevent_skip=False)]
     tests = [
-        Test(test_file="tests_nextcloud.py", prevent_skip=True),
+        Test(test_file="tests_nextcloud.py"),
         # Test(condition=condition_always_false, test_file="tests_nextcloud_onlyoffice.py"),
     ]
     # cleanups = [Test(test_file="cleanup_nextcloud.py")]

recipes/nextcloud/tests_nextcloud/setup_nextcloud.py

@@ -10,7 +10,7 @@ from pytest_abra import BaseUrl, DirManager
 # https://files.test.dev.local-it.cloud/apps/files/
 
 
-def setup_nextcloud_admin_session(authentik_admin_page: Page, DIR: DirManager, URL: BaseUrl):
+def test_setup_nextcloud_admin_session(authentik_admin_page: Page, DIR: DirManager, URL: BaseUrl):
     """visit nextcloud from authentik with admin_session to create wordpress_admin_session"""
     with authentik_admin_page.expect_popup() as event_context:
         authentik_admin_page.get_by_role("link", name="Nextcloud").click()

recipes/wordpress/tests_wordpress/setup_wordpress.py

@@ -13,7 +13,7 @@ def test_visit_from_domain(authentik_admin_context: BrowserContext, URL: BaseUrl
         expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000)
 
 
-def setup_wordpress_admin_session(authentik_admin_page: Page, DIR: DirManager):
+def test_setup_wordpress_admin_session(authentik_admin_page: Page, DIR: DirManager):
     """visit wordpress from authentik with admin_session to create wordpress_admin_session"""
     with authentik_admin_page.expect_popup() as event_context:
         authentik_admin_page.get_by_role("link", name="Wordpress").click()

recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py

@@ -5,7 +5,7 @@ from playwright.sync_api import Page, expect
 from pytest_abra import BaseUrl
 
 
-def setup_trigger_email(wordpress_admin_page: Page, URL: BaseUrl):
+def test_setup_trigger_email(wordpress_admin_page: Page, URL: BaseUrl):
     """change profile email to EMAIL to trigger email"""
     page = wordpress_admin_page
     page.goto(URL.get("wp-admin/profile.php"))

tests/test_cli_full_integration.py

@@ -1,3 +1,4 @@
+import shutil
 import subprocess
 from pathlib import Path
 
@@ -8,13 +9,25 @@ from pytest_abra.utils import load_json_to_environ
 
 
 @pytest.fixture(scope="session")
-def session_tmp_path_testout(tmp_path_factory: pytest.TempPathFactory) -> Path:
+def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path:
-    return tmp_path_factory.mktemp("test_out")
+    tmp_recipes_target = tmp_path_factory.mktemp("recipes")
+    recipes_dir_source = Path("recipes")
+    shutil.copytree(recipes_dir_source, tmp_recipes_target, dirs_exist_ok=True)
+    return tmp_recipes_target
+
+
+@pytest.fixture(scope="session")
+def tmp_output(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    return tmp_path_factory.mktemp("output")
 
 
 @pytest.mark.slow
-def test_abratest_cli_full_integration(session_tmp_path_testout: Path):
+def test_abratest_cli_full_integration(tmp_output: Path, tmp_recipes: Path):
-    """run abratest against the dev instance"""
+    """Full integration test of abratest against the dev instance. Recipes dir not in path
+
+    this test is hard to debug as the output dir is in tmp. If required, try
+    pytest -s
+    or find the tmp dir to look into test outputs"""
 
     # --------------------- load credentials to env variables -------------------- #
 
@@ -33,9 +46,9 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path):
 
     # ----------------------------------- dirs ----------------------------------- #
 
-    RECIPES_DIR = Path("./recipes").resolve()
+    RECIPES_DIR = tmp_recipes.resolve()
-    # OUTPUT_DIR = Path("./test-output").resolve()
+    # RECIPES_DIR = Path("recipes")
-    OUTPUT_DIR = session_tmp_path_testout.resolve()
+    OUTPUT_DIR = tmp_output.resolve()
 
     # ------------------------------------ run ----------------------------------- #
 
@@ -57,8 +70,8 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path):
 
 
 @pytest.mark.slow
-def test_results_abra(session_tmp_path_testout: Path):
+def test_full_integration_results(tmp_output: Path):
-    OUTPUT_DIR = session_tmp_path_testout.resolve()
+    OUTPUT_DIR = tmp_output.resolve()
 
     DIR = DirManager(output_dir=OUTPUT_DIR, session_id="abc")
     all_files = [f.name for f in DIR.STATUS.rglob("*")]

tests/test_coordinator.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import sys
 from pathlib import Path
 
 import pytest
@@ -36,7 +37,16 @@ def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path:
     return tmp_recipes_target
 
 
-def test_runner_runner_dict_import(tmp_recipes: Path):
+@pytest.fixture
+def clear_sys_path():
+    """clear sys.path before test, restore after"""
+    syspath_copy = sys.path.copy()
+    sys.path.clear()
+    yield
+    sys.path.extend(syspath_copy)
+
+
+def test_runner_runner_dict_import(tmp_recipes: Path, clear_sys_path):
     """import from recipes dict should work, because create_runner_dict has sys.path.append"""
 
     RUNNER_DICT = Coordinator.create_runner_dict(tmp_recipes)

tests/test_dir_manager.py

@@ -1,24 +1,27 @@
-
 import time
-import pytest
-from pytest_abra.dir_manager import DirManager
 from pathlib import Path
 
+import pytest
+
+from pytest_abra.dir_manager import DirManager
+
+
 def test_get_latest_session_id_from_non_existing_dir(tmp_path: Path):
     out = DirManager.get_latest_session_id(tmp_path / "not_exist")
     assert out is None
 
+
 def test_get_latest_session_id_from_empty_dir(tmp_path: Path):
     out = DirManager.get_latest_session_id(tmp_path)
     assert out is None
 
+
 def test_get_latest_session_id_single(tmp_path: Path):
     (tmp_path / "a").mkdir()
     out = DirManager.get_latest_session_id(tmp_path)
     assert out == "a"
 
 
-
 @pytest.mark.slow
 def test_get_latest_session_id(tmp_path: Path):
     (tmp_path / "a").mkdir()
@@ -26,5 +29,3 @@ def test_get_latest_session_id(tmp_path: Path):
     (tmp_path / "b").mkdir()
     out = DirManager.get_latest_session_id(tmp_path)
     assert out == "b"
-
-