refactor for independent test dirs (#7)

* make it so that the actual tests can be moved anywhere, for example in abra recipe repos
-> major refactoring with pytest test discovery magic

* create RUNNER_DICT dynamically with importlib
-> none of the tests are hardcoded, more tests can be added by placing a folder

* autoload fixtures with pytest plugins

* add URL fixture to navigate on web pages. Includes url parser based on python urllib to generate correct links

* fix nextcloud setups and tests

*  add email groundwork with imbox

Reviewed-on: local-it-infrastructure/e2e_tests#7
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

recipes/authentik/tests_authentik/fixtures_authentik.py

@@ -0,0 +1,43 @@
+import json
+
+import pytest
+from dotenv import dotenv_values
+from playwright.sync_api import BrowserContext, Page
+
+from abratest.dir_manager import DirManager
+
+
+@pytest.fixture
+def authentik_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext:
+    state_file = DIR.STATES / "authentik_admin_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+    return context
+
+
+@pytest.fixture
+def authentik_admin_page(authentik_admin_context: BrowserContext, DIR: DirManager) -> Page:
+    page = authentik_admin_context.new_page()
+    env_file = DIR.ENV_FILES / "authentik"
+    config: dict[str, str] = dotenv_values(env_file)  # type: ignore
+    url = "https://" + config["DOMAIN"]
+    page.goto(url)
+    return page
+
+
+@pytest.fixture
+def authentik_user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext:
+    state_file = DIR.STATES / "authentik_user_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+    return context
+
+
+@pytest.fixture
+def authentik_user_page(authentik_user_context: BrowserContext, DIR: DirManager) -> Page:
+    page = authentik_user_context.new_page()
+    env_file = DIR.ENV_FILES / "authentik"
+    config: dict[str, str] = dotenv_values(env_file)  # type: ignore
+    url = "https://" + config["DOMAIN"]
+    page.goto(url)
+    return page

recipes/authentik/tests_authentik/runner_authentik.py

@@ -0,0 +1,16 @@
+from abratest.runner import Runner, Test
+
+
+def condition_always_true(dotenv_config: dict[str, str]) -> bool:
+    return True
+
+
+def condition_always_false(dotenv_config: dict[str, str]) -> bool:
+    return False
+
+
+class RunnerAuthentik(Runner):
+    name = "authentik"
+    test_dir_name = "tests_authentik"
+    setups = [Test(test_file="setup_authentik.py")]
+    # tests = [Test(test_file="test_authentik_dummy.py")]

recipes/authentik/tests_authentik/setup_authentik.py

@@ -0,0 +1,118 @@
+import json
+import os
+import re
+
+from playwright.sync_api import BrowserContext, expect
+
+from abratest.dir_manager import DirManager
+from abratest.utils import BaseUrl
+
+ADMIN_USER = os.environ["ADMIN_USER"]
+ADMIN_PASS = os.environ["ADMIN_PASS"]
+
+
+TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"}
+
+
+def setup_admin_state(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager):
+    # go to page
+    page = context.new_page()
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
+
+    # check welcome message
+    welcome_message = dotenv_config.get("welcome_message")
+    if welcome_message:
+        expect(page.get_by_text(welcome_message)).to_be_visible()
+
+    # login
+    page.locator('input[name="uidField"]').fill(ADMIN_USER)
+    page.locator('ak-stage-identification input[name="password"]').fill(ADMIN_PASS)
+    page.get_by_role("button", name="Log In").click()
+    expect(page.locator("ak-library")).to_be_visible()
+
+    # save state
+    context.storage_state(path=DIR.STATES / "authentik_admin_state.json")
+
+
+def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, str], URL: BaseUrl):
+    # go to admin page
+    page = admin_context.new_page()
+    page.goto(URL.get())
+    page.get_by_role("link", name="Admin Interface").click()
+    nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis"))
+    nav.click()
+    nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click()
+
+    user = page.get_by_text(TESTUSER["username"])
+    user.wait_for(state="visible")
+    return user.is_visible()
+
+
+def create_invite_link(admin_context: BrowserContext, dotenv_config: dict[str, str], URL: BaseUrl):
+    # go to admin page
+    page = admin_context.new_page()
+    page.goto(URL.get())
+    page.get_by_role("link", name="Admin Interface").click()
+
+    nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis"))
+    nav.click()
+    nav.get_by_role("link", name=re.compile(r"Invitations|Einladungen")).click()
+
+    # todo: only works if no links have been created yet (empty list)
+    page.get_by_role("cell", name=re.compile(r"Keine Objekte|objects")).get_by_role(
+        "button"
+    ).click()  # todo: confirm "objects" for en lang
+
+    page.locator('input[name="name"]').click()
+    linkname = "test_link_123"
+    page.locator('input[name="name"]').fill(linkname)
+    page.get_by_placeholder("Wählen Sie ein Objekt aus.").click()
+    page.get_by_role("option", name=re.compile(r"invitation-enrollment-flow")).click()
+
+    # force, because else we get "intercepts pointer events"
+    page.locator("footer").locator("ak-spinner-button").first.click(force=True)
+
+    linklocator = page.get_by_role("rowgroup").filter(has=page.get_by_text(linkname))
+    linklocator.locator(".fa-angle-down").click()
+    # page.get_by_text(linkname).click()
+    invitelink = linklocator.get_by_role("textbox").get_attribute(name="value")
+    return invitelink
+
+
+def create_user(user_context: BrowserContext, invitelink):
+    # warning: only works on german site
+    page = user_context.new_page()
+    page.goto(invitelink)
+    page.get_by_placeholder("Benutzername").click()
+    page.get_by_placeholder("Benutzername").fill(TESTUSER["username"])
+    page.locator('input[name="name"]').click()
+    page.locator('input[name="name"]').fill(TESTUSER["name"])
+    page.locator('input[name="email"]').click()
+    page.locator('input[name="email"]').fill(TESTUSER["email"])
+    page.get_by_placeholder("Passwort", exact=True).click()
+    page.get_by_placeholder("Passwort", exact=True).fill(TESTUSER["password"])
+    page.get_by_placeholder("Passwort (wiederholen)").click()
+    page.get_by_placeholder("Passwort (wiederholen)").fill(TESTUSER["password"])
+    page.get_by_role("button", name="Weiter").click()
+    expect(page.locator("ak-library")).to_be_visible()
+
+
+def setup_user_state(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager, URL: BaseUrl):
+    # load admin cookies to context
+    state_file = DIR.STATES / "authentik_admin_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+
+    if check_if_user_exists(context, dotenv_config, URL):
+        # just login with user
+        pass
+        context.clear_cookies()
+    else:
+        # get invite_link
+        invite_link = create_invite_link(context, dotenv_config, URL)
+        # create user
+        context.clear_cookies()
+        create_user(context, invite_link)
+
+    context.storage_state(path=DIR.STATES / "authentik_user_state.json")

recipes/authentik/tests_authentik/test_authentik_dummy.py

@@ -0,0 +1,6 @@
+def test_true():
+    assert 1 + 1 == 2
+
+
+def test_not_true():
+    assert 1 + 1 == 3

recipes/demo/tests_demo/fixtures_demo.py

@@ -0,0 +1,26 @@
+"""
+This file can be used to define fixtures thate are then used by other tests which
+depend on [demo]. For this to work
+
+1. the Runner class of the other test needs to define the depencency as seen
+   by referencing RunnerDemo in the dependencies list:
+
+from abratest.tests_demo.runner_demo import RunnerDemo
+
+class RunnerOther(Runner):
+    dependencies = [RunnerDemo]
+
+
+2. the specific tests that rely on these fixtures need to import the fixtures.
+   To globally import for all tests in 'other', the import should be done in conftest:
+
+in 'conftest.py' in 'test_other' dir:
+from abratest.tests_demo.fixtures_demo import demo_fixture
+"""
+
+import pytest
+
+
+@pytest.fixture
+def demo_fixture():
+    return ""

recipes/demo/tests_demo/runner_demo.py

@@ -0,0 +1,25 @@
+from abratest.runner import Runner, Test
+
+
+class RunnerDemo(Runner):
+    """Every env file has a corresponding runner class"""
+
+    name: str = "demo"  # name of the test, used for logging / output naming
+    test_dir_name: str = "tests_demo"  # dir name holding all tests related to RunnerDemo
+
+    # this indicates that tests from RunnerDemo depend on the setup from RunnerAuthentik.
+    # RunnerDemo will only execute, when setup_authentik.py has finished successfully.
+    # For example, setup_authentik.py generates session states, that can be used as fixtures
+    # that can be loaded from fixtures_authentik.py
+    dependencies: list[str] = ["authentik"]
+
+    # todo: update these comments
+    # Filename of Demo setup. If defined, it will run 1st by executing pytest
+    # Filename of Demo test. This file contains unconditional tests that will be run in any
+    # case. If defined, it will run 2nd by executing pytest
+    # this list can hold many more tests from RunnerDemo that run conditional. The condition
+    # and the test file can be defined by creating a ConditionalTest instance:
+    # ConditionalTest(condition: Callable, test_file: str)
+    setups: list[Test] = []
+    tests: list[Test] = []
+    cleanups: list[Test] = []

recipes/demo/tests_demo/setup_demo.py

@@ -0,0 +1,3 @@
+# Define functions here that are specifically meant for setup, not for testing. This means
+# all actions that simply are required for other tests from 'demo' to run. Runs before all
+# tests from 'demo'.

recipes/nextcloud/tests_nextcloud/cleanup_nextcloud.py

@@ -0,0 +1,17 @@
+import os
+
+from playwright.sync_api import Page
+
+
+def delete_nextcloud_user(authentik_admin_page: Page):
+    """Delete Nextcloud User"""
+    with authentik_admin_page.expect_popup() as nextcloud_info:
+        authentik_admin_page.get_by_role("link", name="Nextcloud").click()
+    nextcloud = nextcloud_info.value
+    nextcloud.get_by_role("link", name="Open settings menu").click()
+    nextcloud.get_by_role("link", name="Users").click()
+    nextcloud.locator("#app-content div").filter(has_text=os.environ["NEXTCLOUD_USER"]).get_by_role(
+        "button", name="Toggle user actions menu"
+    ).click()
+    nextcloud.get_by_role("button", name="Delete user").click()
+    nextcloud.get_by_role("button", name=f"Delete authentik-{os.environ["NEXTCLOUD_USER"]}'s account").click()

recipes/nextcloud/tests_nextcloud/conftest.py

@@ -0,0 +1,33 @@
+import json
+import os
+
+import pytest
+from playwright.sync_api import BrowserContext, Page
+
+from abratest.dir_manager import DirManager
+from abratest.utils import BaseUrl
+
+pytest_plugins = "tests_authentik.fixtures_authentik"
+
+NEXTCLOUD_DEMO_USER = {
+    "NEXTCLOUD_USER": "next_demo_user",
+    "NEXTCLOUD_PASS": "P@ss.123",
+}
+
+for key, value in NEXTCLOUD_DEMO_USER.items():
+    os.environ[key] = value
+
+
+@pytest.fixture
+def nextcloud_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext:
+    state_file = DIR.STATES / "nextcloud_admin_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+    return context
+
+
+@pytest.fixture
+def nextcloud_admin_page(nextcloud_admin_context: BrowserContext, DIR: DirManager, URL: BaseUrl) -> Page:
+    page = nextcloud_admin_context.new_page()
+    page.goto(URL.get("/apps/files"))
+    return page

recipes/nextcloud/tests_nextcloud/runner_nextcloud.py

@@ -0,0 +1,17 @@
+from abratest.runner import Runner, Test
+
+
+def condition_always_false(dotenv_config: dict[str, str]) -> bool:
+    return False
+
+
+class RunnerNextcloud(Runner):
+    name: str = "nextcloud"
+    test_dir_name: str = "tests_nextcloud"
+    dependencies = ["authentik"]
+    setups = [Test(test_file="setup_nextcloud.py", prevent_skip=False)]
+    tests = [
+        Test(test_file="tests_nextcloud.py", prevent_skip=True),
+        # Test(condition=condition_always_false, test_file="tests_nextcloud_onlyoffice.py"),
+    ]
+    # cleanups = [Test(test_file="cleanup_nextcloud.py")]

recipes/nextcloud/tests_nextcloud/setup_nextcloud.py

@@ -0,0 +1,21 @@
+from playwright.sync_api import Page, expect
+
+from abratest.dir_manager import DirManager
+from abratest.utils import BaseUrl
+
+# url dashboard
+# https://files.test.dev.local-it.cloud/apps/dashboard/
+# url files
+# https://files.test.dev.local-it.cloud/apps/files/
+
+
+def setup_nextcloud_admin_session(authentik_admin_page: Page, DIR: DirManager, URL: BaseUrl):
+    """visit nextcloud from authentik with admin_session to create wordpress_admin_session"""
+    with authentik_admin_page.expect_popup() as event_context:
+        authentik_admin_page.get_by_role("link", name="Nextcloud").click()
+    page_nextcloud = event_context.value
+    context = page_nextcloud.context
+
+    page_nextcloud.goto(URL.get("/apps/files"))
+    expect(page_nextcloud.get_by_role("link", name="Name")).to_be_visible()
+    context.storage_state(path=DIR.STATES / "nextcloud_admin_state.json")

recipes/nextcloud/tests_nextcloud/tests_nextcloud.py

@@ -0,0 +1,32 @@
+import re
+
+import pytest
+from playwright.sync_api import Page, expect
+
+
+def test_nextcloud_quota(nextcloud_admin_page: Page, dotenv_config: dict[str, str]):
+    """Test Nextcloud"""
+    if dotenv_config.get("DEFAULT_QUOTA"):
+        # get quota from website
+        quota_string = nextcloud_admin_page.get_by_text(
+            re.compile(r"\d*,\d .* \d*,\d")
+        ).inner_text()  # "37,7 MB von 104,9 MB verwendet"
+        out = re.search(r"\d*,\d .* (\d*,\d).", quota_string)
+        out_number = out[1]  # 104,9
+        out_number = out_number.replace(",", ".")
+        quota_website = float(out_number)
+
+        # get quota from env
+        quota_config_string = dotenv_config["DEFAULT_QUOTA"]  # "100 MB"
+        assert "MB" in quota_config_string
+        quota_config = float(quota_config_string.strip("MB"))
+
+        assert quota_website == pytest.approx(quota_config, rel=0.1)  # within 10%
+    else:
+        pytest.skip("DEFAULT_QUOTA not defined in env file")
+
+
+@pytest.mark.skip
+def test_nextcloud_apps(nextcloud_admin_page: Page, dotenv_config: dict[str, str]):
+    for app in dotenv_config["nc_apps"]:
+        expect(nextcloud_admin_page.get_by_role("link", name=app)).to_be_visible()

recipes/nextcloud/tests_nextcloud/tests_nextcloud_onlyoffice.py

@@ -0,0 +1,19 @@
+def test_onlyoffice(nc_session):
+    """Test Onlyoffice in Nextcloud"""
+    context, page = nc_session
+    # if page.query_selector('.close-icon'):
+    #    page.get_by_role("button", name="Close modal").click()
+    page.get_by_role("link", name="New file/folder menu").click()
+    page.get_by_role("link", name="New document").click()
+    page.locator("#view9-input-file").fill("test.docx")
+    page.get_by_role("button", name="Submit").click()
+    outer_frame = page.frame_locator("#onlyofficeFrame")
+    check_for(outer_frame.locator("body"))
+    inner_frame = outer_frame.frame_locator("#app > iframe")
+    check_for(inner_frame.locator("body"))
+    onlyoffice = page.frame("frameEditor")
+    check_for(onlyoffice.locator('//*[@id="area_id"]'))
+    onlyoffice.locator("#btn-goback").click()
+    page.get_by_role("link", name="Not favorited test .docx Share Actions").get_by_role("link", name="Actions").click()
+    page.get_by_role("link", name="Delete file").click()
+    context.tracing.stop(path=f"{RECORDS}/onlyoffice.zip")

recipes/wordpress/tests_wordpress/conftest.py

@@ -0,0 +1,27 @@
+import json
+
+import pytest
+from dotenv import dotenv_values
+from playwright.sync_api import BrowserContext, Page
+
+from abratest.dir_manager import DirManager
+
+pytest_plugins = "tests_authentik.fixtures_authentik"
+
+
+@pytest.fixture
+def wordpress_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext:
+    state_file = DIR.STATES / "wordpress_admin_state.json"
+    storage_state = json.loads(state_file.read_bytes())
+    context.add_cookies(storage_state["cookies"])
+    return context
+
+
+@pytest.fixture
+def wordpress_admin_page(wordpress_admin_context: BrowserContext, DIR: DirManager) -> Page:
+    page = wordpress_admin_context.new_page()
+    env_file = DIR.ENV_FILES / "wordpress"
+    config: dict[str, str] = dotenv_values(env_file)  # type: ignore
+    url = "https://" + config["DOMAIN"]
+    page.goto(url)
+    return page

recipes/wordpress/tests_wordpress/runner_wordpress.py

@@ -0,0 +1,27 @@
+from abratest.runner import Runner, Test
+
+
+def condition_always_true(dotenv_config: dict[str, str]) -> bool:
+    return True
+
+
+def condition_always_false(dotenv_config: dict[str, str]) -> bool:
+    return False
+
+
+def condition_has_locale(dotenv_config: dict[str, str]) -> bool:
+    if "LOCALE" in dotenv_config:
+        if "de" in dotenv_config["LOCALE"]:
+            return True
+    return False
+
+
+class RunnerWordpress(Runner):
+    name = "wordpress"
+    test_dir_name = "tests_wordpress"
+    dependencies = ["authentik"]
+    setups = [Test(test_file="setup_wordpress.py")]
+    tests = [
+        Test(test_file="test_wordpress.py"),
+        Test(condition=condition_has_locale, test_file="test_wordpress_localization.py"),
+    ]

recipes/wordpress/tests_wordpress/setup_wordpress.py

@@ -0,0 +1,28 @@
+import pytest
+from playwright.sync_api import BrowserContext, Page, expect
+
+from abratest.dir_manager import DirManager
+
+
+def test_visit_from_domain(authentik_admin_context: BrowserContext, dotenv_config: dict[str, str]):
+    """visit wordpress directly with admin_session, expect not to be logged in"""
+    page = authentik_admin_context.new_page()
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
+    with pytest.raises(AssertionError):
+        # look for admin bar
+        expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000)
+
+
+def setup_wordpress_admin_session(authentik_admin_page: Page, DIR: DirManager):
+    """visit wordpress from authentik with admin_session to create wordpress_admin_session"""
+    with authentik_admin_page.expect_popup() as event_context:
+        authentik_admin_page.get_by_role("link", name="Wordpress").click()
+    page_wordpress = event_context.value
+    # look for content wrapper
+    expect(page_wordpress.locator("#wpcontent")).to_be_visible()
+    # look for admin bar
+    expect(page_wordpress.locator("#wpadminbar")).to_be_visible()
+    # save session
+    context = page_wordpress.context
+    context.storage_state(path=DIR.STATES / "wordpress_admin_state.json")

recipes/wordpress/tests_wordpress/test_wordpress_localization.py

@@ -0,0 +1,15 @@
+# WIP localization
+
+from playwright.sync_api import BrowserContext, expect
+
+from abratest.dir_manager import DirManager
+
+
+def test_welcome_message(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager):
+    page = context.new_page()
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
+
+    expect(page.locator(".wp-block-heading")).to_be_visible()
+    if "locale" in dotenv_config and "de" in dotenv_config["locale"]:
+        expect(page.get_by_role("heading")).to_have_text("Willkommen bei WordPress!")