various (#16)

* add full integration test of cli / pytest_abra with all tests

* save path of runner_*.py in runner subclass to improve test discovery -> allows for same test name in two different runners

* reorganize output dir names

* use URL fixture everywhere

* rework coordinator interface

* add --session_id to cli args

* add log results table

* plenty of refactoring

* add assert messages

* add plenty of tests

* add /docs dir with plenty of documentation

* fix authentik setup

* add authentik cleanup, remove test user

* add random test user credential generation and integrate into test routine. random creds are saved to STATES

Reviewed-on: local-it-infrastructure/e2e_tests#16
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

tests/test_cli_full_integration.py

@@ -0,0 +1,68 @@
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from pytest_abra import DirManager
+from pytest_abra.utils import load_json_to_environ
+
+
+@pytest.fixture(scope="session")
+def session_tmp_path_testout(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    return tmp_path_factory.mktemp("test_out")
+
+
+@pytest.mark.slow
+def test_abratest_cli_full_integration(session_tmp_path_testout: Path):
+    """run abratest against the dev instance"""
+
+    # --------------------- load credentials to env variables -------------------- #
+
+    cred_file = Path("credentials.json")
+    load_json_to_environ(cred_file)
+
+    # --------------------------------- env files -------------------------------- #
+
+    ENV_FILES_ROOT = Path("./envfiles").resolve()
+    ENV_FILES = [
+        ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env",  # authentik
+        ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env",  # wordpress
+        ENV_FILES_ROOT / "files.test.dev.local-it.cloud.env",  # nextcloud
+    ]
+    ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES])
+
+    # ----------------------------------- dirs ----------------------------------- #
+
+    RECIPES_DIR = Path("./recipes").resolve()
+    # OUTPUT_DIR = Path("./test-output").resolve()
+    OUTPUT_DIR = session_tmp_path_testout.resolve()
+
+    # ------------------------------------ run ----------------------------------- #
+
+    result = subprocess.run(
+        [
+            "abratest",
+            "--env_paths",
+            ENV_PATHS,
+            "--recipes_dir",
+            RECIPES_DIR,
+            "--output_dir",
+            OUTPUT_DIR,
+            "--session_id",
+            "abc",
+        ]
+    )
+
+    assert result.returncode == 0
+
+
+@pytest.mark.slow
+def test_results_abra(session_tmp_path_testout: Path):
+    OUTPUT_DIR = session_tmp_path_testout.resolve()
+
+    DIR = DirManager(output_dir=OUTPUT_DIR, session_id="abc")
+    all_files = [f.name for f in DIR.STATUS.rglob("*")]
+    passed_files = [f.name for f in DIR.STATUS.rglob("passed-*")]
+    failed_files = set(all_files) - set(passed_files)
+    assert len(all_files) > 0
+    assert not failed_files, failed_files