rework-output-and-test-logic (#3)

* fix flakey tests in authentik / wordpress

* make it possible to rerun tests partially -> passed will be skipped, failed will be repeated

* improve organization of all outputs (moving, renaming, keeping multiple versions etc.)

* add html reports, replace .txt tracebacks

* combine all html reports into one

* add demo runner with comments for documentation purposes

Reviewed-on: local-it-infrastructure/e2e_tests#3
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

src/tests_wordpress/conftest.py

@@ -1 +1 @@
-from src.tests_authentik.fixtures_authentik import admin_session, user_session
+from src.tests_authentik.fixtures_authentik import admin_context, user_context

src/tests_wordpress/runner_wordpress.py

@@ -1,21 +1,28 @@
-from pathlib import Path
-
 from src.runner import Runner, SubTest
+from src.tests_authentik.runner_authentik import RunnerAuthentik
 
 
-def condition_always_true(dotenv_path: Path) -> bool:
+def condition_always_true(dotenv_config: dict[str, str]) -> bool:
     return True
 
 
-def condition_always_false(dotenv_path: Path) -> bool:
+def condition_always_false(dotenv_config: dict[str, str]) -> bool:
+    return False
+
+
+def condition_has_locale(dotenv_config: dict[str, str]) -> bool:
+    if "LOCALE" in dotenv_config:
+        if "de" in dotenv_config["LOCALE"]:
+            return True
     return False
 
 
 class RunnerWordpress(Runner):
     name = "wordpress"
     test_dir_name = "tests_wordpress"
-    # main_test_name = "test_wordpress.py"
+    main_test_name = "test_wordpress.py"
+    dependencies: list[type[Runner]] = [RunnerAuthentik]
     sub_tests = [
-        SubTest(condition=condition_always_true, test_file="test_wordpress_feature1.py"),
+        SubTest(condition=condition_has_locale, test_file="test_wordpress_localization.py"),
     ]
-    dependencies: list[str] = ["authentik"]
+    prevent_skip = True

src/tests_wordpress/test_wordpress.py

@@ -1,29 +1,14 @@
 import re
 
-import pytest
-from playwright.sync_api import Page, expect
+from playwright.sync_api import BrowserContext, expect
+
+from src.dirmanager import DirManager
 
 
-def test_one():
-    assert 1 + 1 == 2
+def test_visit_from_authentik(admin_context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager):
+    page_authentik = admin_context.new_page()
+    with page_authentik.expect_popup() as event_context:
+        page_authentik.get_by_role("link", name="Wordpress").click()
+    page_wordpress = event_context.value
 
-
-def test_two():
-    assert 2 + 1 == 3
-
-
-def test_has_title(page: Page):
-    page.goto("https://playwright.dev/")
-
-    # Expect a title "to contain" a substring.
-    expect(page).to_have_title(re.compile("Playwright"))
-
-
-def test_get_started_link(page: Page):
-    page.goto("https://playwright.dev/")
-
-    # Click the get started link.
-    page.get_by_role("link", name="Get started").click()
-
-    # Expects page to have a heading with the name of Installation.
-    expect(page.get_by_role("heading", name="Installation")).to_be_visible()
+    expect(page_wordpress.locator("#wpcontent")).to_be_visible()

src/tests_wordpress/test_wordpress_feature1.py

@@ -1,31 +0,0 @@
-import re
-
-from icecream import ic
-from playwright.sync_api import BrowserContext, Page, expect
-
-
-def test_demo(admin_session: BrowserContext):
-    admin_session.new_page()
-    assert 1 + 1 == 2
-
-
-# def test_one(config):
-#     ic(config)
-#     assert 1 + 1 == 2
-
-
-# def test_has_title(page: Page):
-#     page.goto("https://playwright.dev/")
-
-#     # Expect a title "to contain" a substring.
-#     expect(page).to_have_title(re.compile("Playwright"))
-
-
-# def test_get_started_link(page: Page):
-#     page.goto("https://playwright.dev/")
-
-#     # Click the get started link.
-#     page.get_by_role("link", name="Get started").click()
-
-#     # Expects page to have a heading with the name of Installation.
-#     expect(page.get_by_role("heading", name="Installation")).to_be_visible()

src/tests_wordpress/test_wordpress_localization.py

@@ -1,22 +1,15 @@
 # WIP localization
 
-from playwright.sync_api import Page, expect
+from playwright.sync_api import BrowserContext, expect
+
+from src.dirmanager import DirManager
 
 
-def test_has_title(page: Page):
-    page.goto("https://playwright.dev/")
+def test_welcome_message(user_context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager):
+    page = user_context.new_page()
+    url = "https://" + dotenv_config["DOMAIN"]
+    page.goto(url)
 
-    # Expect a title "to contain" a substring.
-    expect(page).to_have_title(re.compile("Playwright"))
-
-
-def test_wordpress(admin_session):
-    context, page = admin_session
-    with page.expect_popup() as info:
-        page.get_by_role("link", name="Wordpress").click()
-
-    wordpress = info.value
-    check_for(wordpress.locator("#wpcontent"))
-    if CONFIG["locale"] == "de":
-        check_for(wordpress.get_by_role("heading", name="Willkommen bei WordPress!"))
-    context.tracing.stop(path=f"{RECORDS}/wordpress.zip")
+    expect(page.locator("#wpcontent")).to_be_visible()
+    if "locale" in dotenv_config and "de" in dotenv_config["locale"]:
+        expect(page.get_by_role("heading")).to_have_text("Willkommen bei WordPress!")