various (#16)

* add full integration test of cli / pytest_abra with all tests

* save path of runner_*.py in runner subclass to improve test discovery -> allows for same test name in two different runners

* reorganize output dir names

* use URL fixture everywhere

* rework coordinator interface

* add --session_id to cli args

* add log results table

* plenty of refactoring

* add assert messages

* add plenty of tests

* add /docs dir with plenty of documentation

* fix authentik setup

* add authentik cleanup, remove test user

* add random test user credential generation and integrate into test routine. random creds are saved to STATES

Reviewed-on: local-it-infrastructure/e2e_tests#16
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

tests/test_cli.py

@@ -0,0 +1,54 @@
+import re
+import time
+from pathlib import Path
+
+import pytest
+
+from pytest_abra import DirManager
+from pytest_abra.utils import get_session_id
+
+
+def test_get_session_id_random(tmp_path: Path):
+    args_output_dir = tmp_path
+    args_resume = False
+    args_session_id = None
+    session_id = get_session_id(args_output_dir, args_resume, args_session_id)
+    assert re.search(r"\d+-\d+-\d+", session_id)
+
+
+def test_get_session_id_explicit1(tmp_path: Path):
+    args_output_dir = tmp_path
+    args_resume = False
+    args_session_id = "abc"
+    session_id = get_session_id(args_output_dir, args_resume, args_session_id)
+    assert session_id == "abc"
+
+
+def test_get_session_id_explicit2(tmp_path: Path):
+    args_output_dir = tmp_path
+    args_resume = True
+    args_session_id = "abc"
+    session_id = get_session_id(args_output_dir, args_resume, args_session_id)
+    assert session_id == "abc"
+
+
+@pytest.mark.slow
+def test_get_session_id_integration(tmp_path: Path):
+    assert len(list(tmp_path.iterdir())) == 0
+    session_id_1 = get_session_id(args_output_dir=tmp_path, args_resume=False, args_session_id=None)
+
+    DIR = DirManager(output_dir=tmp_path, session_id=session_id_1)
+    DIR.create_all_dirs()
+    assert len(list(tmp_path.iterdir())) == 1
+
+    time.sleep(1.1)  # get_session_id won't be unique if called without time passed
+    session_id_2 = get_session_id(args_output_dir=tmp_path, args_resume=False, args_session_id=None)
+    DIR = DirManager(output_dir=tmp_path, session_id=session_id_2)
+    DIR.create_all_dirs()
+    assert len(list(tmp_path.iterdir())) == 2
+
+    session_id_3 = get_session_id(args_output_dir=tmp_path, args_resume=True, args_session_id=None)
+    assert session_id_2 == session_id_3
+
+    session_id_4 = get_session_id(args_output_dir=tmp_path, args_resume=True, args_session_id="abc")
+    assert session_id_4 == "abc"

tests/test_cli_full_integration.py

@@ -0,0 +1,68 @@
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from pytest_abra import DirManager
+from pytest_abra.utils import load_json_to_environ
+
+
+@pytest.fixture(scope="session")
+def session_tmp_path_testout(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    return tmp_path_factory.mktemp("test_out")
+
+
+@pytest.mark.slow
+def test_abratest_cli_full_integration(session_tmp_path_testout: Path):
+    """run abratest against the dev instance"""
+
+    # --------------------- load credentials to env variables -------------------- #
+
+    cred_file = Path("credentials.json")
+    load_json_to_environ(cred_file)
+
+    # --------------------------------- env files -------------------------------- #
+
+    ENV_FILES_ROOT = Path("./envfiles").resolve()
+    ENV_FILES = [
+        ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env",  # authentik
+        ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env",  # wordpress
+        ENV_FILES_ROOT / "files.test.dev.local-it.cloud.env",  # nextcloud
+    ]
+    ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES])
+
+    # ----------------------------------- dirs ----------------------------------- #
+
+    RECIPES_DIR = Path("./recipes").resolve()
+    # OUTPUT_DIR = Path("./test-output").resolve()
+    OUTPUT_DIR = session_tmp_path_testout.resolve()
+
+    # ------------------------------------ run ----------------------------------- #
+
+    result = subprocess.run(
+        [
+            "abratest",
+            "--env_paths",
+            ENV_PATHS,
+            "--recipes_dir",
+            RECIPES_DIR,
+            "--output_dir",
+            OUTPUT_DIR,
+            "--session_id",
+            "abc",
+        ]
+    )
+
+    assert result.returncode == 0
+
+
+@pytest.mark.slow
+def test_results_abra(session_tmp_path_testout: Path):
+    OUTPUT_DIR = session_tmp_path_testout.resolve()
+
+    DIR = DirManager(output_dir=OUTPUT_DIR, session_id="abc")
+    all_files = [f.name for f in DIR.STATUS.rglob("*")]
+    passed_files = [f.name for f in DIR.STATUS.rglob("passed-*")]
+    failed_files = set(all_files) - set(passed_files)
+    assert len(all_files) > 0
+    assert not failed_files, failed_files

tests/test_coordinator.py

@@ -0,0 +1,43 @@
+import os
+import shutil
+from pathlib import Path
+
+import pytest
+
+from pytest_abra.coordinator import Coordinator
+from pytest_abra.dir_manager import DirManager
+
+
+def test_load_test_credentials(tmp_path: Path):
+    assert "TEST_USER" not in os.environ
+
+    DIR = DirManager(output_dir=tmp_path, session_id="abc")
+    DIR.create_all_dirs()
+
+    Coordinator.load_test_credentials(DIR)
+    assert (DIR.STATES / "credentials_test.json").is_file()
+
+    assert "TEST_USER" in os.environ
+    test_user_before = os.environ["TEST_USER"]
+
+    # os.environ.clear()  # this breaks pytest!
+    del os.environ["TEST_USER"]
+    assert "TEST_USER" not in os.environ
+
+    Coordinator.load_test_credentials(DIR)
+    assert test_user_before == os.environ["TEST_USER"]
+
+
+@pytest.fixture(scope="session")
+def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    tmp_recipes_target = tmp_path_factory.mktemp("recipes")
+    recipes_dir_source = Path("recipes")
+    shutil.copytree(recipes_dir_source, tmp_recipes_target, dirs_exist_ok=True)
+    return tmp_recipes_target
+
+
+def test_runner_runner_dict_import(tmp_recipes: Path):
+    """import from recipes dict should work, because create_runner_dict has sys.path.append"""
+
+    RUNNER_DICT = Coordinator.create_runner_dict(tmp_recipes)
+    assert len(RUNNER_DICT.keys()) > 0

tests/test_dir_manager.py

@@ -0,0 +1,30 @@
+
+import time
+import pytest
+from pytest_abra.dir_manager import DirManager
+from pathlib import Path
+
+def test_get_latest_session_id_from_non_existing_dir(tmp_path: Path):
+    out = DirManager.get_latest_session_id(tmp_path / "not_exist")
+    assert out is None
+
+def test_get_latest_session_id_from_empty_dir(tmp_path: Path):
+    out = DirManager.get_latest_session_id(tmp_path)
+    assert out is None
+
+def test_get_latest_session_id_single(tmp_path: Path):
+    (tmp_path / "a").mkdir()
+    out = DirManager.get_latest_session_id(tmp_path)
+    assert out == "a"
+
+    
+
+@pytest.mark.slow
+def test_get_latest_session_id(tmp_path: Path):
+    (tmp_path / "a").mkdir()
+    time.sleep(1.1)
+    (tmp_path / "b").mkdir()
+    out = DirManager.get_latest_session_id(tmp_path)
+    assert out == "b"
+
+    

tests/test_env_manager.py

@@ -0,0 +1,137 @@
+import shutil
+from pathlib import Path
+
+import pytest
+
+from pytest_abra.dir_manager import DirManager
+from pytest_abra.env_manager import EnvManager
+from pytest_abra.utils import files_are_same
+
+ENV_PATHS = [
+    Path("envfiles/blog.test.dev.local-it.cloud.env"),  # wordpress
+    Path("envfiles/login.test.dev.local-it.cloud.env"),  # authentik
+    Path("envfiles/login.test.dev.local-it.cloud.env"),  # authentik
+]
+
+
+@pytest.fixture
+def tmp_output(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    return tmp_path_factory.mktemp("output")
+
+
+@pytest.fixture
+def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    return tmp_path_factory.mktemp("recipes")
+
+
+def test_copy_env_files(tmp_output: Path, tmp_recipes: Path):
+    # create dirs in output
+    DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes)
+    DIR.create_all_dirs()
+
+    # confirm dir is empty
+    assert len(list(DIR.ENV_FILES.iterdir())) == 0
+
+    # copy env files
+    env_files = EnvManager._get_env_files(ENV_PATHS)
+    EnvManager.copy_env_files(env_files, DIR)
+
+    # check that each env file is present in DIR.ENV_FILES with correct contents
+    assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files)
+    for index, env_path in enumerate(ENV_PATHS):
+        matching_files = [f for f in DIR.ENV_FILES.iterdir() if index == int(f.name.split("-")[0])]
+        assert len(matching_files) == 1
+        assert files_are_same(env_path, matching_files[0])
+
+
+def test_copy_env_files_twice(tmp_output: Path, tmp_recipes: Path):
+    """Copy the same env files twice"""
+    # create dirs in output
+    DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes)
+    DIR.create_all_dirs()
+
+    # confirm dir is empty
+    assert len(list(DIR.ENV_FILES.iterdir())) == 0
+
+    # copy env files
+    env_files = EnvManager._get_env_files(ENV_PATHS)
+    EnvManager.copy_env_files(env_files, DIR)
+
+    # check that each env file is present in DIR.ENV_FILES with correct contents
+    assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files)
+
+    # copy env files again
+    EnvManager.copy_env_files(env_files, DIR)
+
+    for index, env_path in enumerate(ENV_PATHS):
+        matching_files = [f for f in DIR.ENV_FILES.iterdir() if index == int(f.name.split("-")[0])]
+        assert len(matching_files) == 1
+        assert files_are_same(env_path, matching_files[0])
+
+
+def test_copy_env_files_twice_with_content_change(tmp_output: Path, tmp_recipes: Path, tmp_path: Path):
+    # copy env files to tmp_path
+    assert len(list(tmp_path.iterdir())) == 0
+    for f in ENV_PATHS:
+        shutil.copy(f, tmp_path / f.name)
+    ENV_PATHS_NEW = list(tmp_path.iterdir())
+    assert len(ENV_PATHS_NEW) > 0
+
+    # create dirs in output
+    DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes)
+    DIR.create_all_dirs()
+
+    # confirm dir is empty
+    assert len(list(DIR.ENV_FILES.iterdir())) == 0
+
+    # copy env files from tmp_path to tmp_output
+    env_files = EnvManager._get_env_files(ENV_PATHS_NEW)
+    EnvManager.copy_env_files(env_files, DIR)
+
+    # check that each env file is present in DIR.ENV_FILES with correct contents
+    assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files)
+
+    # change content of one env_file in tmp_path
+    file_path = next(tmp_path.iterdir())
+    with open(file_path, "w") as file:
+        file.write("This is the new content")
+
+    # copy env files again
+    with pytest.raises(AssertionError) as excinfo:
+        EnvManager.copy_env_files(env_files, DIR)
+
+    assert "input env files have changed" in str(excinfo.value)
+
+
+def test_copy_env_files_twice_with_name_change(tmp_output: Path, tmp_recipes: Path, tmp_path: Path):
+    # copy env files to tmp_path
+    assert len(list(tmp_path.iterdir())) == 0
+    for f in ENV_PATHS:
+        shutil.copy(f, tmp_path / f.name)
+    ENV_PATHS_NEW = list(tmp_path.iterdir())
+    assert len(ENV_PATHS_NEW) > 0
+
+    # create dirs in output
+    DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes)
+    DIR.create_all_dirs()
+
+    # confirm dir is empty
+    assert len(list(DIR.ENV_FILES.iterdir())) == 0
+
+    # copy env files from tmp_path to tmp_output
+    env_files = EnvManager._get_env_files(ENV_PATHS_NEW)
+    EnvManager.copy_env_files(env_files, DIR)
+
+    # check that each env file is present in DIR.ENV_FILES with correct contents
+    assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files)
+
+    # change name of one env_file in tmp_path
+    file_path = next(tmp_path.iterdir())
+    file_path.rename(file_path.parent / (file_path.stem + "-other" + file_path.suffix))
+
+    # copy env files from tmp_path to tmp_output again
+    with pytest.raises(AssertionError) as excinfo:
+        env_files = EnvManager._get_env_files(list(tmp_path.iterdir()))
+        EnvManager.copy_env_files(env_files, DIR)
+
+    assert "input env files have changed" in str(excinfo.value)

tests/test_env_resolution.py

@@ -102,3 +102,17 @@ def test_env_manager() -> None:
     assert ENV.env_files[0].env_type == "authentik"
     assert ENV.env_files[1].env_type == "authentik"
     assert ENV.env_files[2].env_type == "wordpress"
+
+
+def test_RUNNER_DICT_missing_key() -> None:
+    """RUNNER_DICT missing wordpress key while .env file with TYPE=wordpress given"""
+    env_paths_list = [
+        Path("envfiles/blog.test.dev.local-it.cloud.env"),  # wordpress
+        Path("envfiles/login.test.dev.local-it.cloud.env"),  # authentik
+        Path("envfiles/login.test.dev.local-it.cloud.env"),  # authentik
+    ]
+    RUNNER_DICT_COPY = RUNNER_DICT.copy()
+    del RUNNER_DICT_COPY["wordpress"]
+    with pytest.raises(AssertionError) as excinfo:
+        EnvManager(env_paths_list, RUNNER_DICT_COPY)
+    assert "no runner for" in str(excinfo.value)

tests/test_html_merge.py

@@ -16,27 +16,31 @@ def session_tmp_path(tmp_path_factory: pytest.TempPathFactory) -> Path:
     return tmp_path_factory.mktemp("html_test")
 
 
-def test_merge_html(session_tmp_path: Path):
+@pytest.fixture(scope="session")
+def html_file(session_tmp_path: Path) -> Path:
     """combines all generated pytest html reports into one"""
 
     in_dir_path = Path(__file__).parent / "assets" / "html_merge"
     in_dir_path = in_dir_path.resolve()
-    ic(in_dir_path)
 
-    out_file_path = session_tmp_path / "test.html"
-    out_assets_dir = session_tmp_path / "assets"
+    html_file = session_tmp_path / "test.html"
 
-    merge_html_reports(in_dir_path.as_posix(), out_file_path.as_posix(), "combined.html")
+    merge_html_reports(in_dir_path.as_posix(), html_file.as_posix(), "combined.html")
+    return html_file
 
-    assert out_file_path.is_file()
-    assert out_assets_dir.is_dir()
-    assert next(out_assets_dir.glob("*"))
+
+def test_merge_html(html_file: Path):
+    assert html_file.is_file()
+    assert html_file.parent.is_dir()
+    assert next(html_file.parent.glob("*"))
 
 
 @pytest.mark.slow
-def test_check_result_with_playwright(session_tmp_path, context: BrowserContext):
-    html_file = session_tmp_path / "test.html"
+def test_check_result_with_playwright(html_file: Path, context: BrowserContext):
+    assert html_file.is_file()
+
     file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get()
+
     page = context.new_page()
     page.goto(file_url)
 

tests/test_runner.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+from pytest_abra import DirManager, Runner
+
+
+def test_runner_create_status_file(tmp_path: Path):
+    """check if _create_status_file prevents duplicates"""
+
+    DIR = DirManager(output_dir=tmp_path, session_id="temp")
+    DIR.create_all_dirs()
+    assert len(list(DIR.STATUS.iterdir())) == 0
+
+    # create first status file
+    Runner._create_status_file(DIR, "passed", "identifier-a")
+    assert len(list(DIR.STATUS.iterdir())) == 1
+
+    # create second status file
+    Runner._create_status_file(DIR, "passed", "identifier-b")
+    assert len(list(DIR.STATUS.iterdir())) == 2
+
+    # check if _get_status_files finds only the correct status file
+    result = Runner._get_status_files(DIR, "identifier-a")
+    assert len(result) == 1
+
+    # overwrite first status file
+    Runner._create_status_file(DIR, "failed", "identifier-a")
+    assert len(list(DIR.STATUS.iterdir())) == 2
+
+    assert Runner._is_test_passed(DIR, "identifier-a") is False