README.md

@@ -35,14 +35,15 @@ Run the script with
 ```bash
 python main.py  # run abratest
 pytest  # test abratest
+pytest  --collect-only  # debug test abratest
 ```
 
 # 2.2 Run with Docker
 
 ```bash
 docker compose build  # build the image
-docker compose run --rm app python ./main.py  # run AbraTest
-docker compose run --rm app pytest  # test AbraTest
+docker compose run --rm app ./run_abratest.sh  # run AbraTest
+docker compose run --rm app ./test_abratest.sh  # test AbraTest
 ```
 
 Force rebuild with cache

abratest/coordinator.py

@@ -42,14 +42,9 @@ class Coordinator:
     def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]:
         """Creates an instance of the correct Runner class for each given env file"""
         runners: list[Runner] = []
-        for env_file in env_files:
+        for index, env_file in enumerate(env_files):
             RunnerClass = self.RUNNER_DICT[env_file.config["TYPE"]]
-            dependency_classes: list[type[Runner]] = []
-            for dependency in RunnerClass.dependencies:
-                dependency_classes.append(self.RUNNER_DICT[dependency])
-            runner_instance = RunnerClass(dotenv_path=env_file.env_path, DIR=self.DIR)
-            runner_instance._dependency_runners = dependency_classes
-            runners.append(runner_instance)
+            runners.append(RunnerClass(coordinator=self, runner_index=index))
         return runners
 
     def combine_html(self) -> None:
@@ -104,5 +99,5 @@ class Coordinator:
             assert len(runner_class_names) == 1
             runner_class_name = runner_class_names[0]
             RunnerClass: type[Runner] = getattr(module, runner_class_name)
-            RUNNER_DICT[RunnerClass.name] = RunnerClass
+            RUNNER_DICT[RunnerClass.env_type] = RunnerClass
         return RUNNER_DICT

abratest/dir_manager.py

@@ -1,5 +1,7 @@
 from pathlib import Path
 
+from dotenv import dotenv_values
+
 
 class DirManager:
     """Manages directories for the tests and should be used to create and find
@@ -69,3 +71,8 @@ class DirManager:
     @property
     def RECIPES(self):
         return self.recipes_dir
+
+    def get_config(self, search_string: str) -> dict[str, str]:
+        env_file = next(self.ENV_FILES.glob(f"*{search_string}*"))
+        config: dict[str, str] = dotenv_values(env_file)  # type: ignore
+        return config

abratest/env_manager.py

@@ -46,7 +46,7 @@ class EnvManager:
         for env_file in env_files:
             child_runner_class = RUNNER_DICT[env_file.env_type]
             for dependency in child_runner_class.dependencies:
-                dependency_rule = DependencyRule(child=child_runner_class.name, dependency=dependency)
+                dependency_rule = DependencyRule(child=child_runner_class.env_type, dependency=dependency)
                 dependency_rules.append(dependency_rule)
         return dependency_rules
 
@@ -93,8 +93,10 @@ class EnvManager:
         )
 
     def copy_env_files(self, DIR: DirManager) -> None:
-        """Copies all env files to STATES/env_files. Files will be renamed to their own TYPE value."""
-        env_files_dir = DIR.STATES / "env_files"
-        env_files_dir.mkdir(exist_ok=True)
-        for env_file in self.env_files:
-            shutil.copy(env_file.env_path, env_files_dir / env_file.env_type)
+        """Copies all env files to STATES/env_files. Files will be renamed to
+        <index>-<env_type>-<original_name>
+        00-authentik-login.test.dev.local-it.cloud.env"""
+
+        for index, env_file in enumerate(self.env_files):
+            file_name = "-".join([str(index).zfill(2), env_file.env_type, env_file.env_path.name])
+            shutil.copy(env_file.env_path, DIR.ENV_FILES / file_name)

abratest/runner.py

@@ -1,12 +1,13 @@
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Callable
+from typing import TYPE_CHECKING, Callable
 
 import pytest
-from dotenv import dotenv_values
 from loguru import logger
 
-from abratest.dir_manager import DirManager
+if TYPE_CHECKING:
+    from abratest.coordinator import Coordinator
+    from abratest.env_manager import EnvFile
 
 
 @dataclass
@@ -17,22 +18,25 @@ class Test:
 
 
 class Runner:
-    name: str = ""
-    test_dir_name: str = ""
+    env_type: str = ""
     setups: list[Test] = []
     tests: list[Test] = []
     cleanups: list[Test] = []
     dependencies: list[str] = []
-    _dependency_runners: list[type["Runner"]] = []
 
-    def __init__(self, dotenv_path: Path, DIR: DirManager):
-        self.dotenv_path = dotenv_path
-        self.config: dict[str, str] = dotenv_values(dotenv_path)  # type: ignore
-        self.DIR = DIR
+    def __init__(self, coordinator: "Coordinator", runner_index: int):
+        self.coordinator = coordinator  # needed?
+        self.runner_index = runner_index  # needed?
+
+        self.DIR = coordinator.DIR
+        self.ENV = coordinator.ENV
+        self.RUNNER_DICT = coordinator.RUNNER_DICT
+
+        self.env_file: EnvFile = self.ENV.env_files[self.runner_index]
+        self.dotenv_path = self.env_file.env_path
+        self.config = self.env_file.config
 
         logger.info(f"creating instance of {self.__class__.__name__}")
-        assert self.test_dir_name
-        self.root_dir = Path(__file__).parent
 
     def run_setups(self):
         """runs the setup scripts if available"""
@@ -50,7 +54,7 @@ class Runner:
         """runs the main test script and if available and sub test scripts if their running condition is met"""
         # check if required dependencies have passed
         if not self._dependencies_passed():
-            logger.warning(f"skipping run_tests() of {self.name}, because some dependencies have not passed")
+            logger.warning(f"skipping run_tests() of {self.env_type}, because some dependencies have not passed")
             return
 
         for test in test_list:
@@ -63,8 +67,11 @@ class Runner:
         # condition_available: true / pass
         # condition_met: true / false
 
-        identifier_string = self.combine_names(self.name, test.test_file)
-        full_test_path = self.DIR.RECIPES / self.name / self.test_dir_name / test.test_file
+        identifier_string = self.combine_names(self.env_type, test.test_file)
+
+        results = list(self.DIR.RECIPES.rglob(test.test_file))
+        assert len(results) == 1, f"{test.test_file} should exist exactly 1 time, but found {len(results)} times"
+        full_test_path = results[0]
 
         # check if test aleady passed
         if self._is_test_passed(identifier_string, remove_existing=True):
@@ -168,9 +175,10 @@ class Runner:
 
         passed_tests = [r.name for r in self.DIR.RESULTS.glob("*") if "passed" in r.name]
         results = []
-        for dependency_runner in self._dependency_runners:
+        for dependency in self.dependencies:
+            dependency_runner = self.coordinator.RUNNER_DICT[dependency]
             for setup_name in dependency_runner.setups:
-                dependencie_identifier = self.combine_names(dependency_runner.name, setup_name.test_file)
+                dependencie_identifier = self.combine_names(dependency_runner.env_type, setup_name.test_file)
                 results.append(any(dependencie_identifier in f for f in passed_tests))
         return all(results)
 

pyproject.toml

@@ -20,4 +20,5 @@ target-version = "py311"
 
 [tool.pytest.ini_options]
 python_functions = "test_* setup_*"
-norecursedirs = "previous-work recipes"
+norecursedirs = ".* previous-work recipes"
+testpaths = "tests"

recipes/authentik/tests_authentik/fixtures_authentik.py

@@ -1,10 +1,10 @@
 import json
 
 import pytest
-from dotenv import dotenv_values
 from playwright.sync_api import BrowserContext, Page
 
 from abratest.dir_manager import DirManager
+from abratest.utils import BaseUrl
 
 
 @pytest.fixture
@@ -18,10 +18,9 @@ def authentik_admin_context(context: BrowserContext, DIR: DirManager) -> Browser
 @pytest.fixture
 def authentik_admin_page(authentik_admin_context: BrowserContext, DIR: DirManager) -> Page:
     page = authentik_admin_context.new_page()
-    env_file = DIR.ENV_FILES / "authentik"
-    config: dict[str, str] = dotenv_values(env_file)  # type: ignore
-    url = "https://" + config["DOMAIN"]
-    page.goto(url)
+    config = DIR.get_config("authentik")
+    base_url = BaseUrl(config["DOMAIN"])
+    page.goto(base_url.get())
     return page
 
 
@@ -36,8 +35,7 @@ def authentik_user_context(context: BrowserContext, DIR: DirManager) -> BrowserC
 @pytest.fixture
 def authentik_user_page(authentik_user_context: BrowserContext, DIR: DirManager) -> Page:
     page = authentik_user_context.new_page()
-    env_file = DIR.ENV_FILES / "authentik"
-    config: dict[str, str] = dotenv_values(env_file)  # type: ignore
-    url = "https://" + config["DOMAIN"]
-    page.goto(url)
+    config = DIR.get_config("authentik")
+    base_url = BaseUrl(config["DOMAIN"])
+    page.goto(base_url.get())
     return page

recipes/authentik/tests_authentik/runner_authentik.py

@@ -10,7 +10,6 @@ def condition_always_false(dotenv_config: dict[str, str]) -> bool:
 
 
 class RunnerAuthentik(Runner):
-    name = "authentik"
-    test_dir_name = "tests_authentik"
+    env_type = "authentik"
     setups = [Test(test_file="setup_authentik.py")]
     # tests = [Test(test_file="test_authentik_dummy.py")]

recipes/demo/tests_demo/runner_demo.py

@@ -4,8 +4,7 @@ from abratest.runner import Runner, Test
 class RunnerDemo(Runner):
     """Every env file has a corresponding runner class"""
 
-    name: str = "demo"  # name of the test, used for logging / output naming
-    test_dir_name: str = "tests_demo"  # dir name holding all tests related to RunnerDemo
+    env_type = "demo"  # name of the test, used for logging / output naming
 
     # this indicates that tests from RunnerDemo depend on the setup from RunnerAuthentik.
     # RunnerDemo will only execute, when setup_authentik.py has finished successfully.

recipes/nextcloud/tests_nextcloud/conftest.py

@@ -7,7 +7,7 @@ from playwright.sync_api import BrowserContext, Page
 from abratest.dir_manager import DirManager
 from abratest.utils import BaseUrl
 
-pytest_plugins = "tests_authentik.fixtures_authentik"
+pytest_plugins = "authentik.tests_authentik.fixtures_authentik"
 
 NEXTCLOUD_DEMO_USER = {
     "NEXTCLOUD_USER": "next_demo_user",

recipes/nextcloud/tests_nextcloud/runner_nextcloud.py

@@ -6,8 +6,7 @@ def condition_always_false(dotenv_config: dict[str, str]) -> bool:
 
 
 class RunnerNextcloud(Runner):
-    name: str = "nextcloud"
-    test_dir_name: str = "tests_nextcloud"
+    env_type = "nextcloud"
     dependencies = ["authentik"]
     setups = [Test(test_file="setup_nextcloud.py", prevent_skip=False)]
     tests = [

recipes/wordpress/tests_wordpress/conftest.py

@@ -6,7 +6,7 @@ from playwright.sync_api import BrowserContext, Page
 
 from abratest.dir_manager import DirManager
 
-pytest_plugins = "tests_authentik.fixtures_authentik"
+pytest_plugins = "authentik.tests_authentik.fixtures_authentik"
 
 
 @pytest.fixture

recipes/wordpress/tests_wordpress/runner_wordpress.py

@@ -17,8 +17,7 @@ def condition_has_locale(dotenv_config: dict[str, str]) -> bool:
 
 
 class RunnerWordpress(Runner):
-    name = "wordpress"
-    test_dir_name = "tests_wordpress"
+    env_type = "wordpress"
     dependencies = ["authentik"]
     setups = [Test(test_file="setup_wordpress.py")]
     tests = [

run_abratest.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+RECIPES_PATH=$PWD/recipes 
+export PYTHONPATH=${PYTHONPATH}:$RECIPES_PATH
+
+python main.py

test_abratest.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+PWD_PATH=$PWD
+RECIPES_PATH=$PWD/recipes 
+export PYTHONPATH=$PWD_PATH:$RECIPES_PATH
+
+pytest