rework-output-and-test-logic (#3)

* fix flakey tests in authentik / wordpress

* make it possible to rerun tests partially -> passed will be skipped, failed will be repeated

* improve organization of all outputs (moving, renaming, keeping multiple versions etc.)

* add html reports, replace .txt tracebacks

* combine all html reports into one

* add demo runner with comments for documentation purposes

Reviewed-on: local-it-infrastructure/e2e_tests#3
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

src/tests_demo/fixtures_demo.py

@@ -0,0 +1,26 @@
+"""
+This file can be used to define fixtures thate are then used by other tests which
+depend on [demo]. For this to work
+
+1. the Runner class of the other test needs to define the depencency as seen
+   by referencing RunnerDemo in the dependencies list:
+
+from src.tests_demo.runner_demo import RunnerDemo
+
+class RunnerOther(Runner):
+    dependencies = [RunnerDemo]
+
+
+2. the specific tests that rely on these fixtures need to import the fixtures.
+   To globally import for all tests in 'other', the import should be done in conftest:
+
+in 'conftest.py' in 'test_other' dir:
+from src.tests_demo.fixtures_demo import demo_fixture
+"""
+
+import pytest
+
+
+@pytest.fixture
+def demo_fixture():
+    return ""

src/tests_demo/runner_demo.py

@@ -0,0 +1,29 @@
+from typing import Optional
+
+from src.runner import Runner, SubTest
+from src.tests_authentik.runner_authentik import RunnerAuthentik
+
+
+class RunnerDemo(Runner):
+    """Every env file has a corresponding runner class"""
+
+    name: str = "demo"  # name of the test, used for logging / output naming
+    test_dir_name: str = "tests_demo"  # dir name holding all tests related to RunnerDemo
+
+    # Filename of Demo setup. If defined, it will run 1st by executing pytest
+    main_setup_name: Optional[str] = "setup_demo.py"
+
+    # Filename of Demo test. This file contains unconditional tests that will be run in any
+    # case. If defined, it will run 2nd by executing pytest
+    main_test_name: Optional[str] = None
+
+    # this indicates that tests from RunnerDemo depend on the setup from RunnerAuthentik.
+    # RunnerDemo will only execute, when setup_authentik.py has finished successfully.
+    # For example, setup_authentik.py generates session states, that can be used as fixtures
+    # that can be loaded from fixtures_authentik.py
+    dependencies: list[type["Runner"]] = [RunnerAuthentik]
+
+    # this list can hold many more tests from RunnerDemo that run conditional. The condition
+    # and the test file can be defined by creating a SubTest instance:
+    # SubTest(condition: Callable, test_file: str)
+    sub_tests: list[SubTest] = []

src/tests_demo/setup_demo.py

@@ -0,0 +1,3 @@
+# Define functions here that are specifically meant for setup, not for testing. This means
+# all actions that simply are required for other tests from 'demo' to run. Runs before all
+# tests from 'demo'.