authentik setup and tracing (#2)

* authentik sessions created successfully during setup without breaking tracing

* setup works on EN and DE localization by using regex patterns

* automated tracing with pytest --trace option, manual hook no longer needed

Reviewed-on: local-it-infrastructure/e2e_tests#2
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

.gitignore

@@ -1,7 +1,8 @@
 __pycache__/
-*.json
+test-output/
-*.zip
 TestResults/
 .vscode/
 *.pyc
+*.json
+*.zip
 credentials*

main.py

@@ -0,0 +1,37 @@
+import json
+import os
+from pathlib import Path
+
+from src.utils import get_session_id
+from src.wrapper import Wrapper
+
+# The env file list is the input to testing framework. each env file triggers
+# the execution of one test Runner and provides configuration to the tests
+# inside the runner. There can be dependencies, for example wordpress requires
+# that authentik ran first to create the admin session and the user session.
+# At the moment, wrong ordering results in unsuccessful test (wrong ordering
+# would be wordpress env file is before authentik env file).
+ENV_FILES = [
+    Path("envfiles/login.test.dev.local-it.cloud.env"),  # authentik
+    Path("envfiles/blog.test.dev.local-it.cloud.env"),  # wordpress
+]
+
+OUTPUT_DIR = Path("./test-output").resolve()
+
+
+# Set environment variables
+
+# os.environ["PWDEBUG"] = "1"
+
+cred_file = Path("credentials.json")
+with open(cred_file, "r") as f:
+    CREDENTIALS = json.load(f)
+
+os.environ["ADMIN_USER"] = CREDENTIALS["admin_user"]
+os.environ["ADMIN_PASS"] = CREDENTIALS["admin_pass"]
+
+
+session_id = get_session_id()
+wrapper = Wrapper(ENV_FILES, output_dir=OUTPUT_DIR, session_id=session_id)
+wrapper.setup_test()
+wrapper.run_test()

pyproject.toml

@@ -17,3 +17,6 @@ package-dir = {"" = "src"}
 [tool.ruff]
 line-length = 120
 target-version = "py311"
+
+[tool.pytest.ini_options]
+python_files = "test_*.py setup*.py"

src/blog.dev.local-it.cloud.env

@@ -1,71 +0,0 @@
-################################################################################
-# DO NOT EDIT THIS FILE, IT IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN #
-################################################################################
-
-TYPE=wordpress
-TIMEOUT=300
-ENABLE_AUTO_UPDATE=true
-COMPOSE_FILE="compose.yml"
-
-# Setup Wordpress settings on each deploy:
-#POST_DEPLOY_CMDS="app core_install"
-
-DOMAIN=blog.dev.local-it.cloud
-## Domain aliases
-#EXTRA_DOMAINS=', `www.blog.dev.local-it.cloud`'
-LETS_ENCRYPT_ENV=production
-
-TITLE="My Example Blog"
-LOCALE=de_DE
-ADMIN_EMAIL=admin@example.com
-
-# Every new user is per default subscriber, uncomment to change it
-DEFAULT_USER_ROLE=administrator
-
-#WORDPRESS_DEBUG=true
-
-## Additional extensions
-#PHP_EXTENSIONS="calendar"
-
-SECRET_DB_ROOT_PASSWORD_VERSION=v1
-SECRET_DB_PASSWORD_VERSION=v1
-
-# Mostly for compatibility with existing database dumps...
-#WORDPRESS_TABLE_PREFIX=wp_
-
-# Multisite
-#WORDPRESS_CONFIG_EXTRA="\
-#define('WP_CACHE', false);\
-#define('WP_ALLOW_MULTISITE', true );"
-
-# Multisite phase 2 (see README)
-#WORDPRESS_CONFIG_EXTRA="define('MULTISITE', true); define('SUBDOMAIN_INSTALL', true); define('DOMAIN_CURRENT_SITE', '${DOMAIN}'); define('PATH_CURRENT_SITE', '/');	define('SITE_ID_CURRENT_SITE', 1); define('BLOG_ID_CURRENT_SITE', 1); define('FORCE_SSL_ADMIN', true ); define('COOKIE_DOMAIN', \$_SERVER['HTTP_HOST']);"
-
-# Local SMTP relay
-#COMPOSE_FILE="$COMPOSE_FILE:compose.mailrelay.yml"
-SMTP_HOST=mail.local-it.org
-MAIL_FROM=noreply@local-it.org
-
-# Remote SMTP relay
-COMPOSE_FILE="$COMPOSE_FILE:compose.smtp.yml"
-SMTP_HOST=mail.local-it.org
-MAIL_FROM=noreply@local-it.org
-SMTP_PORT=587
-SMTP_AUTH=on
-SMTP_TLS=on
-SECRET_SMTP_PASSWORD_VERSION=v1
-
-# Authentik SSO
-COMPOSE_FILE="$COMPOSE_FILE:compose.authentik.yml"
-AUTHENTIK_DOMAIN=dev.local-it.cloud
-SECRET_AUTHENTIK_SECRET_VERSION=v1
-SECRET_AUTHENTIK_ID_VERSION=v1
-LOGIN_TYPE='auto'
-
-# 🚩🚩 dangerous, use only for development sites!
-
-# Allow remote connections to db
-#COMPOSE_FILE="$COMPOSE_FILE:compose.public-db.yml
-
-# Wide-open CORS
-#CORS_ALLOW_ALL=1

src/conftest.py

@@ -5,16 +5,14 @@
 # sys.path. It is thus good practise for projects to either put conftest.py under
 # a package scope or to never import anything from a conftest.py file.
 
-
 from pathlib import Path
 
 import pytest
-from dirmanager import DirManager
 from dotenv import dotenv_values
 
-pytest_plugins = [
+from src.dirmanager import DirManager
-    "setup.setup_authentik",
+
-]
+TIMEOUT = 5000
 
 
 def pytest_addoption(parser):
@@ -23,7 +21,7 @@ def pytest_addoption(parser):
         action="store",
     )
     parser.addoption(
-        "--tests_dir",
+        "--output_dir",
         action="store",
     )
     parser.addoption(
@@ -33,32 +31,49 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture(scope="session", autouse=True)
-def dirmanager(request) -> DirManager:
+def DIR(request) -> DirManager:
-    tests_dir = request.config.getoption("--tests_dir")
+    """Fixture holding test directories
-    tests_dir = Path(tests_dir)
+
+    DIR.OUTPUT
+    DIR.SESSION
+    DIR.RECORDS
+    DIR.STATES
+    DIR.RESULTS
+    DIR.PROGRESS"""
+
+    output_dir = request.config.getoption("--output_dir")
+    assert output_dir is not None, "required pytest command line argument not given"
+    output_dir = Path(output_dir)
     session_id = request.config.getoption("--session_id")
-    return DirManager(tests_dir=tests_dir, session_id=session_id)
+    assert session_id is not None, "required pytest command line argument not given"
+    dirmanager = DirManager(output_dir=output_dir, session_id=session_id)
+    dirmanager.create_all_dirs()
+    return dirmanager
 
 
 @pytest.fixture(scope="session", autouse=True)
 def dotenv_config(request) -> dict[str, str]:
     dotenv_path = request.config.getoption("--env_file")
+    assert dotenv_path is not None, "required pytest command line argument not given"
     dotenv_path = Path(dotenv_path)
     assert dotenv_path.is_file()
-    return dotenv_values(dotenv_path)
+    return dotenv_values(dotenv_path)  # type: ignore
 
 
-@pytest.fixture(scope="session", autouse=True)
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
-def RECORDS(dirmanager) -> Path:
+def pytest_runtest_makereport(item, call):
-    assert isinstance(dirmanager, DirManager)
+    """saves traceback when test fails"""
-    return dirmanager.dirs["records"]
 
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
 
-@pytest.fixture(scope="session", autouse=True)
+    # we only look at actual failing test calls, not setup/teardown
-def STATES(dirmanager) -> Path:
+    if rep.when == "call" and rep.failed:
-    return dirmanager.dirs["states"]
+        # saves traceback as .txt for failed test
-
+        filename = f"failed-{item.nodeid}.txt"
-
+        filename = filename.replace("/", "-")
-@pytest.fixture(scope="session", autouse=True)
+        filename = filename.replace("::", "-")
-def RESULTS(dirmanager) -> Path:
+        filepath = item.funcargs["DIR"].RESULTS / filename
-    return dirmanager.dirs["results"]
+        with open(filepath, "a") as f:
+            f.write(rep.longreprtext + "\n")

src/dirmanager.py

@@ -8,37 +8,49 @@ class DirManager:
     The structures is as follows:
     tests dir/
         session_dir-1/
+            progress
             records
-            states
             results
+            states
         session_dir-2/
             records
             ...
     """
 
-    def __init__(self, tests_dir: Path, session_id: str):
+    def __init__(self, output_dir: Path | str, session_id: str):
         # root test dir
-        self.tests_dir = tests_dir
+        if isinstance(output_dir, str):
+            output_dir = Path(output_dir)
+        self._output_dir = output_dir.resolve()
         self.session_id = session_id
 
-        self.dirs = self._get_all_dirs()
-
     def create_all_dirs(self):
-        self.create_dirs(self.tests_dir, exist_ok=True)
+        self.create_dirs(self._output_dir, exist_ok=True)
-        self.create_dirs(self.dirs)
+        self.create_dirs([self.SESSION, self.RECORDS, self.STATES, self.RESULTS, self.PROGRESS], exist_ok=True)
 
-    def _get_all_dirs(self):
+    @property
-        dirs = {}
+    def OUTPUT(self):
-        dirs["session"] = self.tests_dir / f"test-{self.session_id}"
+        return self._output_dir
-        dirs.update(self._get_subdirs(session_dir=dirs["session"]))
-        return dirs
 
-    def _get_subdirs(self, session_dir: Path):
+    @property
-        return {
+    def SESSION(self):
-            "records": session_dir / Path("records"),
+        return self._output_dir / f"test-{self.session_id}"
-            "states": session_dir / Path("states"),
+
-            "results": session_dir / Path("results"),
+    @property
-        }
+    def RECORDS(self):
+        return self.SESSION / Path("records")
+
+    @property
+    def STATES(self):
+        return self.SESSION / Path("states")
+
+    @property
+    def RESULTS(self):
+        return self.SESSION / Path("results")
+
+    @property
+    def PROGRESS(self):
+        return self.SESSION / Path("progress")
 
     @staticmethod
     def create_dirs(dirs: Path | list[Path] | dict[str, Path], exist_ok=False):

src/main.py

@@ -1,76 +0,0 @@
-from datetime import datetime
-from pathlib import Path
-from typing import Protocol
-
-from dirmanager import DirManager
-from dotenv import dotenv_values
-from icecream import ic
-from tests_authentik.runner_authentik import RunnerAuthentik
-from tests_wordpress.runner_wordpress import RunnerWordpress
-
-TESTS_DIR = Path("../tests")
-
-ENV_FILES = [
-    Path("../envfiles/login.test.dev.local-it.cloud.env"),  # authentik
-    # Path("../envfiles/blog.test.dev.local-it.cloud.env"),  # wordpress
-]
-
-
-class TestRunner(Protocol):
-    def __init__(self, dotenv_path: Path, tests_dir: Path, session_id: str):
-        ...
-
-    def run_tests(self):
-        ...
-
-
-# Register all runners here. A .env file with TYPE=authentik will be ran with RunnerAuthentik
-RUNNER_DICT: dict[str, type[TestRunner]] = {
-    "authentik": RunnerAuthentik,
-    "wordpress": RunnerWordpress,
-}
-
-
-class Wrapper:
-    def __init__(self, env_files: list[Path], session_id: str):
-        self.env_files = env_files
-        self.check_env_files(self.env_files)
-        self.session_id = session_id
-
-    def setup_test(self):
-        self.dir_manager = DirManager(tests_dir=TESTS_DIR, session_id=self.session_id)
-        self.dir_manager.create_all_dirs()
-
-    def run_test(self):
-        self.runners: list[TestRunner] = self.load_runners(self.env_files)
-        self.run_tests(self.runners)
-
-    def load_runners(self, env_files: list[Path]) -> list[TestRunner]:
-        runners = []
-        for env_file in env_files:
-            config: dict[str, str] = dotenv_values(env_file)
-            RunnerClass = RUNNER_DICT[config["TYPE"]]
-            runners.append(RunnerClass(dotenv_path=env_file, tests_dir=TESTS_DIR, session_id=self.session_id))
-        return runners
-
-    def run_tests(self, runners: list[TestRunner]):
-        for runner in runners:
-            runner.run_tests()
-
-    @staticmethod
-    def check_env_files(env_files: list[Path]):
-        """checks if file exist for every file in list"""
-        for env_file in env_files:
-            assert env_file.is_file()
-
-    @staticmethod
-    def get_session_id() -> str:
-        current_datetime = datetime.now()
-        return current_datetime.strftime("%Y-%m-%d-%H-%M-%S")
-
-
-if __name__ == "__main__":
-    session_id = Wrapper.get_session_id()
-    wrapper = Wrapper(ENV_FILES, session_id=session_id)
-    wrapper.setup_test()
-    wrapper.run_test()

src/runner.py

@@ -2,8 +2,11 @@ from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
 import pytest
+from dotenv import dotenv_values
 from icecream import ic
 
+from src.dirmanager import DirManager
+
 
 class SubTest(TypedDict):
     condition: Callable[[Path], bool]
@@ -11,23 +14,31 @@ class SubTest(TypedDict):
 
 
 class Runner:
-    test_dir_name: Optional[Path] = None
+    name: Optional[str] = None
+    test_dir_name: Optional[str] = None
+    main_setup_name: Optional[str] = None
     main_test_name: Optional[str] = None
     sub_tests: list[SubTest] = []
+    dependencies: list[str] = []
 
-    def __init__(self, dotenv_path: Path, tests_dir: Path, session_id: str):
+    def __init__(self, dotenv_path: Path, output_dir: Path, session_id: str):
         self.dotenv_path = dotenv_path
-        self.tests_dir = tests_dir
+        self.config: dict[str, str] = dotenv_values(dotenv_path)  # type: ignore
+        self.output_dir = output_dir
         self.session_id = session_id
+        self.DIRS = DirManager(output_dir, session_id)
 
         ic(f"creating instance of {self.__class__.__name__}")
         assert self.test_dir_name is not None
         self.root_dir = Path(__file__).parent
 
     def _run_main_test(self):
+        if isinstance(self.main_setup_name, str):
+            full_path = self.root_dir / self.test_dir_name / self.main_setup_name
+            self._run_pytest(full_path)
         if isinstance(self.main_test_name, str):
-            full_test_path = self.root_dir / self.test_dir_name / self.main_test_name
+            full_path = self.root_dir / self.test_dir_name / self.main_test_name
-            self._run_pytest(full_test_path)
+            self._run_pytest(full_path)
 
     def _run_pytest(self, full_test_path: Path):
         """runs pytest programmatically
@@ -35,24 +46,45 @@ class Runner:
         will run all tests in the file at full_test_path with some command line arguments"""
 
         ic(f"running test: {full_test_path}")
-        pytest.main(
-            [
-                "-v",
-                "-rp",
-                str(full_test_path),
-                "--env_file",
-                str(self.dotenv_path),
-                "--tests_dir",
-                str(self.tests_dir),
-                "--session_id",
-                self.session_id,
-            ]
-        )
 
-    def show_files(self):
+        command_arguments = []
-        ic(list(self.root_dir.glob("*")))
+
+        # command_arguments.append("-v")
+        # command_arguments.append("-rx")
+        command_arguments.append(str(full_test_path))
+
+        command_arguments.append("--env_file")
+        command_arguments.append(str(self.dotenv_path))
+
+        command_arguments.append("--output_dir")
+        command_arguments.append(str(self.DIRS.OUTPUT))
+
+        command_arguments.append("--session_id")
+        command_arguments.append(self.session_id)
+
+        # artifacts dir
+        # warning: https://github.com/microsoft/playwright-pytest/issues/111
+        # --output only works with the given context and page fixture
+        # folder needs to be unique! traces will not appear, if every pytest run has same output dir
+        output = self.DIRS.RESULTS / full_test_path.stem
+        command_arguments.append("--output")
+        command_arguments.append(str(output))
+
+        # tracing
+        command_arguments.append("--tracing")
+        command_arguments.append("retain-on-failure")
+        # command_arguments.append("on")
+
+        # Disable capturing. With -s set, prints will go to console as if pytest is not there.
+        # command_arguments.append("-s")
+
+        # headed
+        # command_arguments.append("--headed")
+
+        pytest.main(command_arguments)
 
     def run_tests(self):
+        self._check_dependencies_finished()
         self._run_main_test()
         for sub_test in self.sub_tests:
             condition_function = sub_test["condition"]
@@ -60,3 +92,16 @@ class Runner:
                 test_name = sub_test["test_file"]
                 full_test_path = self.root_dir / self.test_dir_name / test_name
                 self._run_pytest(full_test_path)
+        self._create_progress_file()
+
+    def _create_progress_file(self):
+        """create progress file to indicated finished test"""
+        file_path = self.DIRS.PROGRESS / self.name
+        with open(file_path, "w") as _:
+            pass  # create empty file
+
+    def _check_dependencies_finished(self):
+        """look for progress file of dependencies to confirm they have ran"""
+        finished_tests = [result.name for result in self.DIRS.PROGRESS.glob("*")]
+        for dependencie in self.dependencies:
+            assert dependencie in finished_tests

src/setup-deprecated/deprecated-setup_authentik-deprecated.py

@@ -0,0 +1,100 @@
+import json
+from pathlib import Path
+
+import pytest
+from icecream import ic
+from playwright.sync_api import Browser, Locator, expect
+
+cred_file = Path("credentials.json")
+with open(cred_file, "r") as f:
+    CREDENTIALS = json.load(f)
+
+print(CREDENTIALS)
+
+TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"}
+TIMEOUT = 5000
+
+
+def check_for(locator: Locator):
+    expect(locator).to_be_visible(timeout=TIMEOUT)
+
+
+# todo: why is this a fixture? to get dotenv_config
+@pytest.fixture(scope="session", autouse=True)
+def create_admin_login(browser: Browser, dotenv_config, STATES):
+    # ic(dotenv_config)
+
+    # go to page
+    context = browser.new_context()
+    context.set_default_timeout(TIMEOUT)
+    page = context.new_page()
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
+
+    # check welcome message
+    welcome_message = dotenv_config.get("welcome_message")
+    if welcome_message:
+        check_for(page.get_by_text(welcome_message))
+
+    # login
+    page.locator('input[name="uidField"]').fill(CREDENTIALS["admin"])
+    page.locator('ak-stage-identification input[name="password"]').fill(CREDENTIALS["admin_pw"])
+    page.get_by_role("button", name="Log In").click()
+    check_for(page.locator("ak-library"))
+
+    # save state
+    context.storage_state(path=f"{STATES}/admin_state.json")
+    page.close()
+    context.close()
+
+
+def create_invite_link(page):
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
+    page.get_by_role("link", name="Admin Interface").click()
+    page.get_by_text("Verzeichnis").click()
+    page.get_by_text("Benutzer").nth(2).click()
+    page.get_by_text("Einladungen").click()
+    page.get_by_role("button", name="Erstellen").first.click()
+    page.locator('input[name="name"]').click()
+    linkname = "testlink9433"
+    page.locator('input[name="name"]').fill(linkname)
+    page.get_by_placeholder("Wählen Sie ein Objekt aus.").click()
+    page.get_by_role("option", name="invitation-enrollment-flow invitation-enrollment-flow").click()
+    page.get_by_text("Erstellen", exact=True).first.click()
+    linklocator = page.get_by_role("rowgroup").filter(has=page.get_by_text(linkname))
+    linklocator.locator(".fa-angle-down").click()
+    invitelink = linklocator.get_by_role("textbox").get_attribute(name="value")
+    return invitelink
+
+
+def create_user(context, invitelink):
+    page = context.new_page()
+    page.goto(invitelink)
+    page.get_by_placeholder("Benutzername").click()
+    page.get_by_placeholder("Benutzername").fill(testuser["username"])
+    page.locator('input[name="name"]').click()
+    page.locator('input[name="name"]').fill(testuser["name"])
+    page.locator('input[name="email"]').click()
+    page.locator('input[name="email"]').fill(testuser["email"])
+    page.get_by_placeholder("Passwort", exact=True).click()
+    page.get_by_placeholder("Passwort", exact=True).fill(testuser["password"])
+    page.get_by_placeholder("Passwort (wiederholen)").click()
+    page.get_by_placeholder("Passwort (wiederholen)").fill(testuser["password"])
+    page.get_by_role("button", name="Weiter").click()
+    check_for(page.locator("ak-library"))
+    context.storage_state(path=f"{STATES}/user_state.json")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def create_user_session(browser: Browser, admin_login):
+    admin_context = browser.new_context(storage_state=f"{STATES}/admin_state.json")
+    # admin_context = setup_context(browser, f"{STATES}/admin_state.json")
+    admin_page = admin_context.new_page()
+    invitelink = create_invite_link(admin_page)
+    admin_context.tracing.stop(path=f"{RECORDS}/create_invite_link.zip")
+    admin_context.close()
+    user_context = setup_context(browser)
+    create_user(user_context, invitelink)
+    user_context.tracing.stop(path=f"{RECORDS}/create_user.zip")
+    user_context.close()

src/setup/setup_authentik.py

@@ -1,47 +0,0 @@
-import json
-from pathlib import Path
-
-import pytest
-from icecream import ic
-from playwright.sync_api import Browser, Locator, expect
-
-cred_file = Path("../credentials.json")
-with open(cred_file, "r") as f:
-    CREDENTIALS = json.load(f)
-
-print(CREDENTIALS)
-
-TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"}
-TIMEOUT = 5000
-
-
-def check_for(locator: Locator):
-    expect(locator).to_be_visible(timeout=TIMEOUT)
-
-
-@pytest.fixture(scope="session", autouse=True)
-def admin_login(browser: Browser, dotenv_config, STATES):
-    # ic(dotenv_config)
-
-    # go to page
-    context = browser.new_context()
-    context.set_default_timeout(TIMEOUT)
-    page = context.new_page()
-    url = "https://" + dotenv_config["DOMAIN"]
-    page.goto(url)
-
-    # check welcome message
-    welcome_message = dotenv_config.get("welcome_message")
-    if welcome_message:
-        check_for(page.get_by_text(welcome_message))
-
-    # login
-    page.locator('input[name="uidField"]').fill(CREDENTIALS["admin"])
-    page.locator('ak-stage-identification input[name="password"]').fill(CREDENTIALS["admin_pw"])
-    page.get_by_role("button", name="Log In").click()
-    check_for(page.locator("ak-library"))
-
-    # save state
-    context.storage_state(path=f"{STATES}/admin_state.json")
-    page.close()
-    context.close()

src/tests_authentik/fixtures_authentik.py

@@ -0,0 +1,22 @@
+import json
+
+import pytest
+from playwright.sync_api import BrowserContext
+
+from src.dirmanager import DirManager
+
+
+@pytest.fixture
+def admin_session(context: BrowserContext, DIR: DirManager) -> BrowserContext:
+    state_file = DIR.STATES / "admin_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+    return context
+
+
+@pytest.fixture
+def user_session(context: BrowserContext, DIR: DirManager) -> BrowserContext:
+    state_file = DIR.STATES / "user_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+    return context

src/tests_authentik/plugin_authentik.py

@@ -0,0 +1,3 @@
+# will be loaded in conftest.py
+# will provide context for other tests (wordpress etc.)
+# that depend on authentik (which is all of them)

src/tests_authentik/runner_authentik.py

@@ -1,6 +1,8 @@
 from pathlib import Path
 
-from runner import Runner, SubTest
+from src.runner import Runner, SubTest
+
+# from src.tests_authentik.setup_authentik import setup_authentik
 
 
 def condition_always_true(dotenv_path: Path) -> bool:
@@ -12,5 +14,7 @@ def condition_always_false(dotenv_path: Path) -> bool:
 
 
 class RunnerAuthentik(Runner):
+    name = "authentik"
     test_dir_name = "tests_authentik"
-    main_test_name = "test_authentik_dummy.py"
+    main_setup_name = "setup_authentik.py"
+    # main_test_name = "test_authentik_dummy.py"

src/tests_authentik/setup_authentik.py

@@ -0,0 +1,122 @@
+import json
+import os
+import re
+
+from icecream import ic
+from playwright.sync_api import BrowserContext, expect
+
+from src.dirmanager import DirManager
+
+ADMIN_USER = os.environ["ADMIN_USER"]
+ADMIN_PASS = os.environ["ADMIN_PASS"]
+
+
+TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"}
+TIMEOUT = 10000
+
+
+def test_create_admin_login(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager):
+    # go to page
+    page = context.new_page()
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
+
+    # check welcome message
+    welcome_message = dotenv_config.get("welcome_message")
+    if welcome_message:
+        expect(page.get_by_text(welcome_message)).to_be_visible()
+
+    # login
+    page.locator('input[name="uidField"]').fill(ADMIN_USER)
+    page.locator('ak-stage-identification input[name="password"]').fill(ADMIN_PASS)
+    page.get_by_role("button", name="Log In").click()
+    expect(page.locator("ak-library")).to_be_visible()
+
+    # save state
+    context.storage_state(path=f"{DIR.STATES}/admin_state.json")
+
+
+def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, str]):
+    # go to admin page
+    page = admin_context.new_page()
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
+    page.get_by_role("link", name="Admin Interface").click()
+    nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis"))
+    nav.click()
+    nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click()
+    result = page.get_by_text(TESTUSER["username"]).is_visible(timeout=TIMEOUT)
+    return result
+
+
+def create_invite_link(admin_context: BrowserContext, dotenv_config: dict[str, str]):
+    # go to admin page
+    page = admin_context.new_page()
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
+    page.get_by_role("link", name="Admin Interface").click()
+
+    nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis"))
+    nav.click()
+    nav.get_by_role("link", name=re.compile(r"Invitations|Einladungen")).click()
+
+    # todo: only works if no links have been created yet (empty list)
+    page.get_by_role("cell", name=re.compile(r"Keine Objekte|objects")).get_by_role(
+        "button"
+    ).click()  # todo: confirm "objects" for en lang
+
+    page.locator('input[name="name"]').click()
+    linkname = "test_link_123"
+    page.locator('input[name="name"]').fill(linkname)
+    page.get_by_placeholder("Wählen Sie ein Objekt aus.").click()
+    page.get_by_role("option", name=re.compile(r"invitation-enrollment-flow")).click()
+
+    # force, because else we get "intercepts pointer events"
+    page.locator("footer").locator("ak-spinner-button").first.click(force=True)
+
+    linklocator = page.get_by_role("rowgroup").filter(has=page.get_by_text(linkname))
+    linklocator.locator(".fa-angle-down").click()
+    # page.get_by_text(linkname).click()
+    invitelink = linklocator.get_by_role("textbox").get_attribute(name="value")
+    return invitelink
+
+
+def create_user(user_context: BrowserContext, invitelink):
+    # warning: only works on german site
+    page = user_context.new_page()
+    page.goto(invitelink)
+    page.get_by_placeholder("Benutzername").click()
+    page.get_by_placeholder("Benutzername").fill(TESTUSER["username"])
+    page.locator('input[name="name"]').click()
+    page.locator('input[name="name"]').fill(TESTUSER["name"])
+    page.locator('input[name="email"]').click()
+    page.locator('input[name="email"]').fill(TESTUSER["email"])
+    page.get_by_placeholder("Passwort", exact=True).click()
+    page.get_by_placeholder("Passwort", exact=True).fill(TESTUSER["password"])
+    page.get_by_placeholder("Passwort (wiederholen)").click()
+    page.get_by_placeholder("Passwort (wiederholen)").fill(TESTUSER["password"])
+    page.get_by_role("button", name="Weiter").click()
+    expect(page.locator("ak-library")).to_be_visible()
+
+
+def test_create_user_session(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager):
+    context.set_default_timeout(TIMEOUT)
+
+    # load admin cookies
+    state_file = DIR.STATES / "admin_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+
+    if check_if_user_exists(context, dotenv_config):
+        # just login with user
+        pass
+        context.clear_cookies()
+    else:
+        ## create user
+        # create invite_link
+        invite_link = create_invite_link(context, dotenv_config)
+        # create user
+        context.clear_cookies()
+        create_user(context, invite_link)
+
+    context.storage_state(path=f"{DIR.STATES}/user_state.json")

src/tests_authentik/test_authentik_dummy.py

@@ -1,2 +1,6 @@
 def test_true():
     assert 1 + 1 == 2
+
+
+def test_not_true():
+    assert 1 + 1 == 3

src/tests_wordpress/conftest.py

@@ -1,28 +1 @@
-# this conftest cannot be executed directly if there is a second conftest
+from src.tests_authentik.fixtures_authentik import admin_session, user_session
-# on a higher level. might work if other tests are executed though
-# -> at least bad for debugging
-
-import json
-from pathlib import Path
-
-import pytest
-from playwright.sync_api import Browser, Locator, expect, sync_playwright
-
-# playwright = sync_playwright().start()
-# browser = playwright.chromium.launch(headless=False)
-
-
-cred_file = Path("../credentials.json")
-with open(cred_file, "r") as f:
-    CREDENTIALS = json.load(f)
-
-print(CREDENTIALS)
-
-RECORDS = Path("records")
-RECORDS.mkdir(exist_ok=True)
-STATES = Path("states")
-STATES.mkdir(exist_ok=True)
-
-
-def test_dummy():
-    assert 1 + 1 == 2

src/tests_wordpress/runner_wordpress.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from runner import Runner, SubTest
+from src.runner import Runner, SubTest
 
 
 def condition_always_true(dotenv_path: Path) -> bool:
@@ -12,9 +12,10 @@ def condition_always_false(dotenv_path: Path) -> bool:
 
 
 class RunnerWordpress(Runner):
+    name = "wordpress"
     test_dir_name = "tests_wordpress"
     # main_test_name = "test_wordpress.py"
     sub_tests = [
-        SubTest(condition=condition_always_false, test_file="test_wordpress_feature1.py"),
+        SubTest(condition=condition_always_true, test_file="test_wordpress_feature1.py"),
-        SubTest(condition=condition_always_true, test_file="conftest.py"),
     ]
+    dependencies: list[str] = ["authentik"]

src/tests_wordpress/test_wordpress_feature1.py

@@ -1,26 +1,31 @@
 import re
 
 from icecream import ic
-from playwright.sync_api import Page, expect
+from playwright.sync_api import BrowserContext, Page, expect
 
 
-def test_one(config):
+def test_demo(admin_session: BrowserContext):
-    ic(config)
+    admin_session.new_page()
     assert 1 + 1 == 2
 
 
-def test_has_title(page: Page):
+# def test_one(config):
-    page.goto("https://playwright.dev/")
+#     ic(config)
-
+#     assert 1 + 1 == 2
-    # Expect a title "to contain" a substring.
-    expect(page).to_have_title(re.compile("Playwright"))
 
 
-def test_get_started_link(page: Page):
+# def test_has_title(page: Page):
-    page.goto("https://playwright.dev/")
+#     page.goto("https://playwright.dev/")
 
-    # Click the get started link.
+#     # Expect a title "to contain" a substring.
-    page.get_by_role("link", name="Get started").click()
+#     expect(page).to_have_title(re.compile("Playwright"))
 
-    # Expects page to have a heading with the name of Installation.
+
-    expect(page.get_by_role("heading", name="Installation")).to_be_visible()
+# def test_get_started_link(page: Page):
+#     page.goto("https://playwright.dev/")
+
+#     # Click the get started link.
+#     page.get_by_role("link", name="Get started").click()
+
+#     # Expects page to have a heading with the name of Installation.
+#     expect(page.get_by_role("heading", name="Installation")).to_be_visible()

src/utils.py

@@ -0,0 +1,7 @@
+from datetime import datetime
+
+
+@staticmethod
+def get_session_id() -> str:
+    current_datetime = datetime.now()
+    return current_datetime.strftime("%Y-%m-%d-%H-%M-%S")

src/wrapper.py

@@ -0,0 +1,55 @@
+import os
+from pathlib import Path
+from typing import Protocol
+
+from dotenv import dotenv_values
+
+from src.dirmanager import DirManager
+from src.tests_authentik.runner_authentik import RunnerAuthentik
+from src.tests_wordpress.runner_wordpress import RunnerWordpress
+
+
+class TestRunner(Protocol):
+    def __init__(self, dotenv_path: Path, output_dir: Path, session_id: str):
+        ...
+
+    def run_tests(self):
+        ...
+
+
+# Register all runners here. A .env file with TYPE=authentik will be ran with RunnerAuthentik
+RUNNER_DICT: dict[str, type[TestRunner]] = {
+    "authentik": RunnerAuthentik,
+    "wordpress": RunnerWordpress,
+}
+
+
+class Wrapper:
+    def __init__(self, env_files: list[Path], output_dir: Path, session_id: str):
+        self.env_files = env_files
+        self.check_env_files(self.env_files)
+        self.output_dir = output_dir
+        self.session_id = session_id
+
+    def setup_test(self):
+        self.dir_manager = DirManager(output_dir=self.output_dir, session_id=self.session_id)
+        self.dir_manager.create_all_dirs()
+
+    def run_test(self):
+        self.runners: list[TestRunner] = self._load_runners(self.env_files)
+        for runner in self.runners:
+            runner.run_tests()
+
+    def _load_runners(self, env_files: list[Path]) -> list[TestRunner]:
+        runners = []
+        for env_file in env_files:
+            config: dict[str, str] = dotenv_values(env_file)  # type: ignore
+            RunnerClass = RUNNER_DICT[config["TYPE"]]
+            runners.append(RunnerClass(dotenv_path=env_file, output_dir=self.output_dir, session_id=self.session_id))
+        return runners
+
+    @staticmethod
+    def check_env_files(env_files: list[Path]):
+        """checks if file exist for every file in list"""
+        for env_file in env_files:
+            assert env_file.is_file()