various (#16)

* add full integration test of cli / pytest_abra with all tests

* save path of runner_*.py in runner subclass to improve test discovery -> allows for same test name in two different runners

* reorganize output dir names

* use URL fixture everywhere

* rework coordinator interface

* add --session_id to cli args

* add log results table

* plenty of refactoring

* add assert messages

* add plenty of tests

* add /docs dir with plenty of documentation

* fix authentik setup

* add authentik cleanup, remove test user

* add random test user credential generation and integrate into test routine. random creds are saved to STATES

Reviewed-on: local-it-infrastructure/e2e_tests#16
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

recipes/authentik/tests_authentik/cleanup_authentik.py

@@ -0,0 +1,40 @@
+import json
+import os
+import re
+
+from playwright.sync_api import BrowserContext
+
+from pytest_abra import BaseUrl, DirManager
+
+ADMIN_USER = os.environ["ADMIN_USER"]
+ADMIN_PASS = os.environ["ADMIN_PASS"]
+TEST_USER = os.environ["TEST_USER"]
+TEST_PASS = os.environ["TEST_PASS"]
+
+
+def remove_user(admin_context: BrowserContext, URL: BaseUrl):
+    """removes TEST_USER account from authentik"""
+    page = admin_context.new_page()
+    page.goto(URL.get())
+    page.get_by_role("link", name="Admin Interface").click()
+    nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis"))
+    nav.click()
+    nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click()
+
+    name_pattern = re.compile(TEST_USER)
+    page.get_by_role("row", name=name_pattern).get_by_label("").check()
+    page.get_by_role("button", name=re.compile(r"Löschen|Delete")).click()
+    page.get_by_role("dialog").get_by_role("button", name=re.compile(r"Löschen|Delete")).click()
+
+
+def cleanup_delete_user(
+    context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists
+):
+    # load admin cookies to context
+    state_file = DIR.STATES / "authentik_admin_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+
+    if check_if_user_exists(context, env_config, URL):
+        remove_user(context, URL)
+        assert not check_if_user_exists(context, env_config, URL)

recipes/authentik/tests_authentik/conftest.py

@@ -0,0 +1,46 @@
+import os
+import re
+from typing import Callable, Generator
+
+import pytest
+from playwright.sync_api import APIRequestContext, BrowserContext, Playwright, TimeoutError
+
+from pytest_abra import BaseUrl, DirManager
+
+
+@pytest.fixture(scope="session")
+def api_request_context(
+    playwright: Playwright,
+    DIR: DirManager,
+) -> Generator[APIRequestContext, None, None]:
+    state_file = DIR.STATES / "authentik_admin_state.json"
+    request_context = playwright.request.new_context(storage_state=state_file)
+    yield request_context
+    request_context.dispose()
+
+
+@pytest.fixture
+def check_if_user_exists() -> Callable[[BrowserContext, dict[str, str], BaseUrl], bool]:
+    """This is actually a normal function supplied by a fixture. We do this, because imports from
+    tests_authentik are difficult as it is not part of the python environment. We expect
+    from X import function
+    to fail here. However, pytest handles the loading of fixtures from conftest.py automatically,
+    hence we use that to load functions too."""
+
+    def inner_check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl) -> bool:
+        # go to admin page
+        page = admin_context.new_page()
+        page.goto(URL.get())
+        page.get_by_role("link", name="Admin Interface").click()
+        nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis"))
+        nav.click()
+        nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click()
+
+        user = page.get_by_text(os.environ["TEST_USER"])
+        try:
+            user.wait_for(state="visible", timeout=5_000)
+            return True
+        except TimeoutError:
+            return False
+
+    return inner_check_if_user_exists

recipes/authentik/tests_authentik/fixtures_authentik.py

@@ -3,13 +3,13 @@ import json
 import pytest
 from playwright.sync_api import BrowserContext, Page
 
-from pytest_abra.dir_manager import DirManager
-from pytest_abra.utils import BaseUrl
+from pytest_abra import BaseUrl, DirManager
 
 
 @pytest.fixture
 def authentik_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext:
     state_file = DIR.STATES / "authentik_admin_state.json"
+    assert state_file.is_file(), "authentik setup did not finish successfully"
     storage_state = json.loads(state_file.read_bytes())
     context.add_cookies(storage_state["cookies"])
     return context
@@ -27,6 +27,7 @@ def authentik_admin_page(authentik_admin_context: BrowserContext, DIR: DirManage
 @pytest.fixture
 def authentik_user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext:
     state_file = DIR.STATES / "authentik_user_state.json"
+    assert state_file.is_file(), "authentik setup did not finish successfully"
     storage_state = json.loads(state_file.read_bytes())
     context.add_cookies(storage_state["cookies"])
     return context

recipes/authentik/tests_authentik/runner_authentik.py

@@ -5,3 +5,4 @@ class RunnerAuthentik(Runner):
     env_type = "authentik"
     setups = [Test(test_file="setup_authentik.py")]
     tests = [Test(test_file="test_authentik_blueprint_api.py")]
+    cleanups = [Test(test_file="cleanup_authentik.py")]

recipes/authentik/tests_authentik/setup_authentik.py

@@ -4,21 +4,18 @@ import re
 
 from playwright.sync_api import BrowserContext, expect
 
-from pytest_abra.dir_manager import DirManager
-from pytest_abra.utils import BaseUrl
+from pytest_abra import BaseUrl, DirManager
 
 ADMIN_USER = os.environ["ADMIN_USER"]
 ADMIN_PASS = os.environ["ADMIN_PASS"]
+TEST_USER = os.environ["TEST_USER"]
+TEST_PASS = os.environ["TEST_PASS"]
 
 
-TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"}
-
-
-def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager):
+def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl):
     # go to page
     page = context.new_page()
-    url = "https://" + env_config["DOMAIN"]
-    page.goto(url)
+    page.goto(URL.get())
 
     # check welcome message
     welcome_message = env_config.get("welcome_message")
@@ -35,20 +32,6 @@ def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR:
     context.storage_state(path=DIR.STATES / "authentik_admin_state.json")
 
 
-def check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl):
-    # go to admin page
-    page = admin_context.new_page()
-    page.goto(URL.get())
-    page.get_by_role("link", name="Admin Interface").click()
-    nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis"))
-    nav.click()
-    nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click()
-
-    user = page.get_by_text(TESTUSER["username"])
-    user.wait_for(state="visible")
-    return user.is_visible()
-
-
 def create_invite_link(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl):
     # go to admin page
     page = admin_context.new_page()
@@ -85,20 +68,23 @@ def create_user(user_context: BrowserContext, invitelink):
     page = user_context.new_page()
     page.goto(invitelink)
     page.get_by_placeholder("Benutzername").click()
-    page.get_by_placeholder("Benutzername").fill(TESTUSER["username"])
+    page.get_by_placeholder("Benutzername").fill(TEST_USER)
     page.locator('input[name="name"]').click()
-    page.locator('input[name="name"]').fill(TESTUSER["name"])
+    page.locator('input[name="name"]').fill("name")
     page.locator('input[name="email"]').click()
-    page.locator('input[name="email"]').fill(TESTUSER["email"])
+    email = os.environ["IMAP_EMAIL"] if "IMAP_EMAIL" in os.environ else "test@domain.com"
+    page.locator('input[name="email"]').fill(email)
     page.get_by_placeholder("Passwort", exact=True).click()
-    page.get_by_placeholder("Passwort", exact=True).fill(TESTUSER["password"])
+    page.get_by_placeholder("Passwort", exact=True).fill(TEST_PASS)
     page.get_by_placeholder("Passwort (wiederholen)").click()
-    page.get_by_placeholder("Passwort (wiederholen)").fill(TESTUSER["password"])
+    page.get_by_placeholder("Passwort (wiederholen)").fill(TEST_PASS)
     page.get_by_role("button", name="Weiter").click()
     expect(page.locator("ak-library")).to_be_visible()
 
 
-def setup_user_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl):
+def setup_user_state(
+    context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists
+):
     # load admin cookies to context
     state_file = DIR.STATES / "authentik_admin_state.json"
     storage_state = json.loads(state_file.read_bytes())

recipes/authentik/tests_authentik/test_authentik_blueprint_api.py

@@ -17,7 +17,7 @@ def test_authentik_blueprint_status(
     blueprints = api_request_context.get(URL.get("api/v3/managed/blueprints"))
     assert blueprints.ok
     blueprints_data = blueprints.json()
-    ic(blueprints_data)
+    # ic(blueprints_data)
 
     # fake failed blueprint
     # blueprints_data["results"][10]["status"] = "failed"

recipes/nextcloud/tests_nextcloud/conftest.py

@@ -4,8 +4,7 @@ import os
 import pytest
 from playwright.sync_api import BrowserContext, Page
 
-from pytest_abra.dir_manager import DirManager
-from pytest_abra.utils import BaseUrl
+from pytest_abra import BaseUrl, DirManager
 
 pytest_plugins = "authentik.tests_authentik.fixtures_authentik"
 

recipes/nextcloud/tests_nextcloud/setup_nextcloud.py

@@ -2,8 +2,7 @@ import re
 
 from playwright.sync_api import Page, expect
 
-from pytest_abra.dir_manager import DirManager
-from pytest_abra.utils import BaseUrl
+from pytest_abra import BaseUrl, DirManager
 
 # url dashboard
 # https://files.test.dev.local-it.cloud/apps/dashboard/

recipes/wordpress/tests_wordpress/runner_wordpress.py

@@ -1,11 +1,12 @@
 from pytest_abra import ConditionArgs, Runner, Test
 
 
-def condition_has_locale(args: ConditionArgs) -> bool:
+def env_config_has_locale(args: ConditionArgs) -> bool:
     env_config = args.env_config
-    if "de" in env_config.get("LOCALE", ""):
+    if "LOCALE" in env_config:
         return True
-    return False
+    else:
+        return False
 
 
 class RunnerWordpress(Runner):
@@ -16,6 +17,6 @@ class RunnerWordpress(Runner):
         Test(test_file="setup_wordpress_trigger_email.py"),
     ]
     tests = [
-        Test(test_file="test_wordpress_receive_email.py", prevent_skip=True),
-        # Test(condition=condition_has_locale, test_file="test_wordpress_localization.py"),
+        # Test(test_file="test_wordpress_receive_email.py", prevent_skip=True),
+        Test(condition=env_config_has_locale, test_file="test_wordpress_localization.py"),
     ]

recipes/wordpress/tests_wordpress/setup_wordpress.py

@@ -1,14 +1,13 @@
 import pytest
 from playwright.sync_api import BrowserContext, Page, expect
 
-from pytest_abra.dir_manager import DirManager
+from pytest_abra import BaseUrl, DirManager
 
 
-def test_visit_from_domain(authentik_admin_context: BrowserContext, env_config: dict[str, str]):
+def test_visit_from_domain(authentik_admin_context: BrowserContext, URL: BaseUrl):
     """visit wordpress directly with admin_session, expect not to be logged in"""
     page = authentik_admin_context.new_page()
-    url = "https://" + env_config["DOMAIN"]
-    page.goto(url)
+    page.goto(URL.get())
     with pytest.raises(AssertionError):
         # look for admin bar
         expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000)

recipes/wordpress/tests_wordpress/test_wordpress_localization.py

@@ -2,14 +2,13 @@
 
 from playwright.sync_api import BrowserContext, expect
 
-from pytest_abra.dir_manager import DirManager
+from pytest_abra import BaseUrl
 
 
-def test_welcome_message(context: BrowserContext, env_config: dict[str, str], DIR: DirManager):
+def test_de_welcome_message(context: BrowserContext, env_config: dict[str, str], URL: BaseUrl):
     page = context.new_page()
-    url = "https://" + env_config["DOMAIN"]
-    page.goto(url)
+    page.goto(URL.get())
 
     expect(page.locator(".wp-block-heading")).to_be_visible()
-    if "locale" in env_config and "de" in env_config["locale"]:
+    if "de" in env_config.get("locale", ""):
         expect(page.get_by_role("heading")).to_have_text("Willkommen bei WordPress!")

recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py

@@ -1,8 +1,10 @@
+import pytest
 from icecream import ic
 
 from pytest_abra.custom_fixtures import Message
 
 
+@pytest.mark.skip
 def test_demo(imap_recent_messages: list[Message]):
     for message in imap_recent_messages:
         print(dir(message))