From 3ffa2f8ecdfd12c8cb5c40ad10505527c7bdc421 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 10 Dec 2023 18:00:36 +0100 Subject: [PATCH 01/91] turn create_result_file into classmethod --- pytest_abra/runner.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 3d75562..fb6d270 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, Callable, NamedTuple import pytest from loguru import logger +from pytest_abra import DirManager + if TYPE_CHECKING: from pytest_abra.coordinator import Coordinator from pytest_abra.env_manager import EnvFile @@ -89,12 +91,13 @@ class Runner: if not condition_result: # test condition is defined but not met logger.info(f"skipping {identifier_string} (test condition is not met)") + self.create_result_file(self.DIR, result="skipped", identifier_string=identifier_string) return # test condition is undefined or not met logger.info(f"running {identifier_string}") result = self._call_pytest(full_test_path) - self._create_result_file(result=result, identifier_string=identifier_string) + self.create_result_file(self.DIR, result=result, identifier_string=identifier_string) def _run_condition(self, condition_function: Callable[[ConditionArgs], bool]): """run the test condition function with multiple arguments""" @@ -174,15 +177,19 @@ class Runner: return pytest.main(command_arguments) - def _create_result_file( - self, - result: int, + @classmethod + def create_result_file( + cls, + DIR: DirManager, + result: int | str, identifier_string: str, ): """create result file to indicated passed/failed or skipped test""" - full_name = self.combine_names(self.result_int_to_str(result), identifier_string) - file_path = self.DIR.RESULTS / full_name + if isinstance(result, int): + result = cls.result_int_to_str(result) + full_name = cls.combine_names(result, identifier_string) + file_path = DIR.RESULTS / full_name with open(file_path, "w") as _: pass # create empty file @@ -204,8 +211,6 @@ class Runner: def result_int_to_str(result_int: int) -> str: """converts the pytest exit code (int) into a meaningful string""" match result_int: - case -1: - return "skipped" case 0: return "passed" case _: -- 2.47.2 From ebac7f49fdbdd5d4890d3adf3ca0047b51d3f056 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 10 Dec 2023 18:00:44 +0100 Subject: [PATCH 02/91] add todos --- pytest_abra/coordinator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 52f5e77..c302c76 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -32,11 +32,16 @@ class Coordinator: self.ENV = EnvManager(env_paths=env_paths, RUNNER_DICT=self.RUNNER_DICT) self.TIMEOUT = timeout + # todo: prepare tests def setup_test(self) -> None: logger.info("calling setup_test()") self.DIR.create_all_dirs() self.ENV.copy_env_files(self.DIR) + # todo: check that tests are unique + # todo: run setups + # todo: run tests + # todo: run cleanups def run_test(self) -> None: logger.info("calling run_test()") self.runners: list[Runner] = self._load_runners(self.ENV.env_files) -- 2.47.2 From a8479a56e3fb979ed2242d156c60ca6459ca9418 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 10 Dec 2023 18:00:52 +0100 Subject: [PATCH 03/91] add generate_random_string --- pytest_abra/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index 828e6b9..b5bd3cf 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -1,3 +1,5 @@ +import random +import string from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -35,3 +37,12 @@ def rmtree(root_dir: Path): child.unlink() root_dir.rmdir() + + +def generate_random_string(length: int, punctuation=False) -> str: + """returns a random string of the given length""" + characters = string.ascii_letters + string.digits + if punctuation: + characters += string.punctuation + random_string = "".join(random.choice(characters) for _ in range(length)) + return random_string -- 2.47.2 From fec4e0a6ea23a204de1a0ec083f35c81965785a0 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 10 Dec 2023 18:01:42 +0100 Subject: [PATCH 04/91] add todo --- pytest_abra/coordinator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index c302c76..c044276 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -38,6 +38,7 @@ class Coordinator: self.DIR.create_all_dirs() self.ENV.copy_env_files(self.DIR) # todo: check that tests are unique + # todo: create random testuser creds and load them # todo: run setups # todo: run tests -- 2.47.2 From 4f8bceb587d43fa6323fe21fbc4b928af36c7e38 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 10 Dec 2023 23:20:06 +0100 Subject: [PATCH 05/91] rename functions --- pytest_abra/cli.py | 4 ++-- pytest_abra/coordinator.py | 14 +++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 1ebf852..53d534b 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -49,7 +49,7 @@ def run(): recipes_dir=args.recipes_dir, timeout=args.timeout, ) - coordinator.setup_test() - coordinator.run_test() + coordinator.prepare_tests() + coordinator.run_tests() coordinator.combine_html() coordinator.collect_traces() diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index c044276..67cd776 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -32,19 +32,15 @@ class Coordinator: self.ENV = EnvManager(env_paths=env_paths, RUNNER_DICT=self.RUNNER_DICT) self.TIMEOUT = timeout - # todo: prepare tests - def setup_test(self) -> None: - logger.info("calling setup_test()") + def prepare_tests(self) -> None: + logger.info("calling prepare_tests()") self.DIR.create_all_dirs() self.ENV.copy_env_files(self.DIR) # todo: check that tests are unique # todo: create random testuser creds and load them - # todo: run setups - # todo: run tests - # todo: run cleanups - def run_test(self) -> None: - logger.info("calling run_test()") + def run_tests(self) -> None: + logger.info("calling run_tests()") self.runners: list[Runner] = self._load_runners(self.ENV.env_files) for runner in self.runners: runner.run_setups() @@ -52,7 +48,7 @@ class Coordinator: runner.run_tests() for runner in self.runners: runner.run_cleanups() - logger.info("run_test() finished") + logger.info("run_tests() finished") def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]: """Creates an instance of the correct Runner class for each given env file""" -- 2.47.2 From 7ec75cd6a0a79630456313ff5264bf423b9e89b6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 10 Dec 2023 23:53:49 +0100 Subject: [PATCH 06/91] Use RunnerMeta to save path along with Runner subclass --- pytest_abra/coordinator.py | 24 ++++++++++++------------ pytest_abra/env_manager.py | 7 ++++--- pytest_abra/runner.py | 10 +++++----- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 67cd776..f1e557a 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -2,6 +2,7 @@ import importlib import re import sys from pathlib import Path +from typing import NamedTuple from loguru import logger @@ -12,6 +13,11 @@ from pytest_abra.runner import Runner from pytest_abra.utils import rmtree +class RunnerMeta(NamedTuple): + cls: type[Runner] + path: Path + + class Coordinator: def __init__( self, @@ -54,8 +60,9 @@ class Coordinator: """Creates an instance of the correct Runner class for each given env file""" runners: list[Runner] = [] for index, env_file in enumerate(env_files): - RunnerClass = self.RUNNER_DICT[env_file.env_config["TYPE"]] - runners.append(RunnerClass(coordinator=self, runner_index=index)) + meta = self.RUNNER_DICT[env_file.env_config["TYPE"]] + RunnerClass = meta.cls + runners.append(RunnerClass(coordinator=self, runner_index=index, runner_dir=meta.path)) return runners def combine_html(self) -> None: @@ -89,21 +96,14 @@ class Coordinator: rmtree(trace_root_dir) @staticmethod - def create_runner_dict(recipes_dir: Path) -> dict[str, type["Runner"]]: + def create_runner_dict(recipes_dir: Path) -> dict[str, RunnerMeta]: """Creates a dictionary holding all the RunnerClasses that can be discovered in recipes_dir - example: - RUNNER_DICT: dict[str, type["Runner"]] = { - "authentik": RunnerAuthentik, - "wordpress": RunnerWordpress, - "nextcloud": RunnerNextcloud, - } - The Runner classes are automatically imported with importlib. The imports are successful because recipes_dir is added to sys.path. """ - RUNNER_DICT: dict[str, type["Runner"]] = dict() + RUNNER_DICT: dict[str, RunnerMeta] = dict() runner_discovery_pattern = re.compile("Runner.+") # make it possible to import modules from recipes_dir @@ -116,5 +116,5 @@ class Coordinator: assert len(runner_class_names) == 1 runner_class_name = runner_class_names[0] RunnerClass: type[Runner] = getattr(module, runner_class_name) - RUNNER_DICT[RunnerClass.env_type] = RunnerClass + RUNNER_DICT[RunnerClass.env_type] = RunnerMeta(cls=RunnerClass, path=module_path) return RUNNER_DICT diff --git a/pytest_abra/env_manager.py b/pytest_abra/env_manager.py index f9d79cd..2490462 100644 --- a/pytest_abra/env_manager.py +++ b/pytest_abra/env_manager.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, NamedTuple from dotenv import dotenv_values if TYPE_CHECKING: + from pytest_abra.coordinator import RunnerMeta from pytest_abra.dir_manager import DirManager from pytest_abra.runner import Runner @@ -24,7 +25,7 @@ class DependencyRule(NamedTuple): class EnvManager: - def __init__(self, env_paths: list[Path], RUNNER_DICT: dict[str, type["Runner"]]): + def __init__(self, env_paths: list[Path], RUNNER_DICT: dict[str, "RunnerMeta"]): self.env_files: list[EnvFile] = self._get_env_files(env_paths) self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files, RUNNER_DICT) self.env_files = self.sort_env_files_by_rule(self.env_files, self.dependency_rules) @@ -42,10 +43,10 @@ class EnvManager: return env_files @staticmethod - def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, type["Runner"]]) -> list[DependencyRule]: + def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, "RunnerMeta"]) -> list[DependencyRule]: dependency_rules: list[DependencyRule] = [] for env_file in env_files: - child_runner_class = RUNNER_DICT[env_file.env_type] + child_runner_class = RUNNER_DICT[env_file.env_type].cls for dependency in child_runner_class.dependencies: dependency_rule = DependencyRule(child=child_runner_class.env_type, dependency=dependency) dependency_rules.append(dependency_rule) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index fb6d270..d2424ea 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -6,9 +6,8 @@ from typing import TYPE_CHECKING, Callable, NamedTuple import pytest from loguru import logger -from pytest_abra import DirManager - if TYPE_CHECKING: + from pytest_abra import DirManager from pytest_abra.coordinator import Coordinator from pytest_abra.env_manager import EnvFile @@ -33,9 +32,10 @@ class Runner: cleanups: list[Test] = [] dependencies: list[str] = [] - def __init__(self, coordinator: "Coordinator", runner_index: int): + def __init__(self, coordinator: "Coordinator", runner_index: int, runner_dir: Path): self.coordinator = coordinator self.runner_index = runner_index + self.runner_dir = runner_dir self.DIR = coordinator.DIR self.ENV = coordinator.ENV @@ -180,7 +180,7 @@ class Runner: @classmethod def create_result_file( cls, - DIR: DirManager, + DIR: "DirManager", result: int | str, identifier_string: str, ): @@ -201,7 +201,7 @@ class Runner: passed_tests = [r.name for r in self.DIR.RESULTS.glob("*") if "passed" in r.name] results = [] for dependency in self.dependencies: - dependency_runner = self.coordinator.RUNNER_DICT[dependency] + dependency_runner = self.coordinator.RUNNER_DICT[dependency].cls for setup_name in dependency_runner.setups: dependencie_identifier = self.combine_names(dependency_runner.env_type, setup_name.test_file) results.append(any(dependencie_identifier in f for f in passed_tests)) -- 2.47.2 From edc8c9a2f5d45bced65772fd3b9305f8588423ec Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:00:16 +0100 Subject: [PATCH 07/91] remove RunnerMeta, save path to class var instead --- pytest_abra/coordinator.py | 18 ++++++------------ pytest_abra/env_manager.py | 7 +++---- pytest_abra/runner.py | 6 +++--- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index f1e557a..47a4a57 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -2,7 +2,6 @@ import importlib import re import sys from pathlib import Path -from typing import NamedTuple from loguru import logger @@ -13,11 +12,6 @@ from pytest_abra.runner import Runner from pytest_abra.utils import rmtree -class RunnerMeta(NamedTuple): - cls: type[Runner] - path: Path - - class Coordinator: def __init__( self, @@ -60,9 +54,8 @@ class Coordinator: """Creates an instance of the correct Runner class for each given env file""" runners: list[Runner] = [] for index, env_file in enumerate(env_files): - meta = self.RUNNER_DICT[env_file.env_config["TYPE"]] - RunnerClass = meta.cls - runners.append(RunnerClass(coordinator=self, runner_index=index, runner_dir=meta.path)) + RunnerClass = self.RUNNER_DICT[env_file.env_config["TYPE"]] + runners.append(RunnerClass(coordinator=self, runner_index=index)) return runners def combine_html(self) -> None: @@ -96,14 +89,14 @@ class Coordinator: rmtree(trace_root_dir) @staticmethod - def create_runner_dict(recipes_dir: Path) -> dict[str, RunnerMeta]: + def create_runner_dict(recipes_dir: Path) -> dict[str, type[Runner]]: """Creates a dictionary holding all the RunnerClasses that can be discovered in recipes_dir The Runner classes are automatically imported with importlib. The imports are successful because recipes_dir is added to sys.path. """ - RUNNER_DICT: dict[str, RunnerMeta] = dict() + RUNNER_DICT: dict[str, type[Runner]] = dict() runner_discovery_pattern = re.compile("Runner.+") # make it possible to import modules from recipes_dir @@ -116,5 +109,6 @@ class Coordinator: assert len(runner_class_names) == 1 runner_class_name = runner_class_names[0] RunnerClass: type[Runner] = getattr(module, runner_class_name) - RUNNER_DICT[RunnerClass.env_type] = RunnerMeta(cls=RunnerClass, path=module_path) + RunnerClass._tests_path = module_path + RUNNER_DICT[RunnerClass.env_type] = RunnerClass return RUNNER_DICT diff --git a/pytest_abra/env_manager.py b/pytest_abra/env_manager.py index 2490462..7de06e4 100644 --- a/pytest_abra/env_manager.py +++ b/pytest_abra/env_manager.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, NamedTuple from dotenv import dotenv_values if TYPE_CHECKING: - from pytest_abra.coordinator import RunnerMeta from pytest_abra.dir_manager import DirManager from pytest_abra.runner import Runner @@ -25,7 +24,7 @@ class DependencyRule(NamedTuple): class EnvManager: - def __init__(self, env_paths: list[Path], RUNNER_DICT: dict[str, "RunnerMeta"]): + def __init__(self, env_paths: list[Path], RUNNER_DICT: dict[str, type[Runner]]): self.env_files: list[EnvFile] = self._get_env_files(env_paths) self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files, RUNNER_DICT) self.env_files = self.sort_env_files_by_rule(self.env_files, self.dependency_rules) @@ -43,10 +42,10 @@ class EnvManager: return env_files @staticmethod - def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, "RunnerMeta"]) -> list[DependencyRule]: + def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, type[Runner]]) -> list[DependencyRule]: dependency_rules: list[DependencyRule] = [] for env_file in env_files: - child_runner_class = RUNNER_DICT[env_file.env_type].cls + child_runner_class = RUNNER_DICT[env_file.env_type] for dependency in child_runner_class.dependencies: dependency_rule = DependencyRule(child=child_runner_class.env_type, dependency=dependency) dependency_rules.append(dependency_rule) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index d2424ea..b193795 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -1,7 +1,7 @@ import os from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Callable, NamedTuple +from typing import TYPE_CHECKING, Callable, NamedTuple, Optional import pytest from loguru import logger @@ -31,11 +31,11 @@ class Runner: tests: list[Test] = [] cleanups: list[Test] = [] dependencies: list[str] = [] + _tests_path: Optional[Path] = None - def __init__(self, coordinator: "Coordinator", runner_index: int, runner_dir: Path): + def __init__(self, coordinator: "Coordinator", runner_index: int): self.coordinator = coordinator self.runner_index = runner_index - self.runner_dir = runner_dir self.DIR = coordinator.DIR self.ENV = coordinator.ENV -- 2.47.2 From de6a71b9c8275f9cafe4f0cd35d0a72945764b95 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:07:53 +0100 Subject: [PATCH 08/91] fixup --- pytest_abra/coordinator.py | 2 +- pytest_abra/env_manager.py | 4 ++-- pytest_abra/runner.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 47a4a57..99da549 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -109,6 +109,6 @@ class Coordinator: assert len(runner_class_names) == 1 runner_class_name = runner_class_names[0] RunnerClass: type[Runner] = getattr(module, runner_class_name) - RunnerClass._tests_path = module_path + RunnerClass._tests_path = module_path.parent RUNNER_DICT[RunnerClass.env_type] = RunnerClass return RUNNER_DICT diff --git a/pytest_abra/env_manager.py b/pytest_abra/env_manager.py index 7de06e4..f9d79cd 100644 --- a/pytest_abra/env_manager.py +++ b/pytest_abra/env_manager.py @@ -24,7 +24,7 @@ class DependencyRule(NamedTuple): class EnvManager: - def __init__(self, env_paths: list[Path], RUNNER_DICT: dict[str, type[Runner]]): + def __init__(self, env_paths: list[Path], RUNNER_DICT: dict[str, type["Runner"]]): self.env_files: list[EnvFile] = self._get_env_files(env_paths) self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files, RUNNER_DICT) self.env_files = self.sort_env_files_by_rule(self.env_files, self.dependency_rules) @@ -42,7 +42,7 @@ class EnvManager: return env_files @staticmethod - def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, type[Runner]]) -> list[DependencyRule]: + def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, type["Runner"]]) -> list[DependencyRule]: dependency_rules: list[DependencyRule] = [] for env_file in env_files: child_runner_class = RUNNER_DICT[env_file.env_type] diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index b193795..c8c3812 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -1,7 +1,7 @@ import os from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Callable, NamedTuple, Optional +from typing import TYPE_CHECKING, Callable, NamedTuple import pytest from loguru import logger @@ -31,7 +31,7 @@ class Runner: tests: list[Test] = [] cleanups: list[Test] = [] dependencies: list[str] = [] - _tests_path: Optional[Path] = None + _tests_path: Path = Path("undefined") def __init__(self, coordinator: "Coordinator", runner_index: int): self.coordinator = coordinator @@ -74,7 +74,7 @@ class Runner: identifier_string = self.combine_names(self.env_type, test.test_file) - results = list(self.DIR.RECIPES.rglob(test.test_file)) + results = list(self._tests_path.rglob(test.test_file)) assert len(results) == 1, f"{test.test_file} should exist exactly 1 time, but found {len(results)} times" full_test_path = results[0] @@ -201,7 +201,7 @@ class Runner: passed_tests = [r.name for r in self.DIR.RESULTS.glob("*") if "passed" in r.name] results = [] for dependency in self.dependencies: - dependency_runner = self.coordinator.RUNNER_DICT[dependency].cls + dependency_runner = self.coordinator.RUNNER_DICT[dependency] for setup_name in dependency_runner.setups: dependencie_identifier = self.combine_names(dependency_runner.env_type, setup_name.test_file) results.append(any(dependencie_identifier in f for f in passed_tests)) -- 2.47.2 From 5ce6be974576627e04e228310073f402c016c582 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:07:59 +0100 Subject: [PATCH 09/91] disable test_wordpress_receive_email --- recipes/wordpress/tests_wordpress/runner_wordpress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/wordpress/tests_wordpress/runner_wordpress.py b/recipes/wordpress/tests_wordpress/runner_wordpress.py index d58bd74..21cdeab 100644 --- a/recipes/wordpress/tests_wordpress/runner_wordpress.py +++ b/recipes/wordpress/tests_wordpress/runner_wordpress.py @@ -16,6 +16,6 @@ class RunnerWordpress(Runner): Test(test_file="setup_wordpress_trigger_email.py"), ] tests = [ - Test(test_file="test_wordpress_receive_email.py", prevent_skip=True), + # Test(test_file="test_wordpress_receive_email.py", prevent_skip=True), # Test(condition=condition_has_locale, test_file="test_wordpress_localization.py"), ] -- 2.47.2 From 744a017eedf2c6792c3297856b1964499181dca8 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:08:25 +0100 Subject: [PATCH 10/91] unique check no longer necessary --- pytest_abra/coordinator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 99da549..84d08a5 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -36,7 +36,6 @@ class Coordinator: logger.info("calling prepare_tests()") self.DIR.create_all_dirs() self.ENV.copy_env_files(self.DIR) - # todo: check that tests are unique # todo: create random testuser creds and load them def run_tests(self) -> None: -- 2.47.2 From fed7d27136f02ec74b0996445c69fa969ea6beb7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:09:28 +0100 Subject: [PATCH 11/91] add example --- pytest_abra/coordinator.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 84d08a5..e0f2162 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -91,6 +91,13 @@ class Coordinator: def create_runner_dict(recipes_dir: Path) -> dict[str, type[Runner]]: """Creates a dictionary holding all the RunnerClasses that can be discovered in recipes_dir + example: + RUNNER_DICT: dict[str, type["Runner"]] = { + "authentik": RunnerAuthentik, + "wordpress": RunnerWordpress, + "nextcloud": RunnerNextcloud, + } + The Runner classes are automatically imported with importlib. The imports are successful because recipes_dir is added to sys.path. """ -- 2.47.2 From 2f6d0c47e59f312534ab66a32eead59b2c7027c1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:12:51 +0100 Subject: [PATCH 12/91] add load_json_to_environ --- main.py | 10 +++------- pytest_abra/utils.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index aa809bc..9a37f57 100644 --- a/main.py +++ b/main.py @@ -1,16 +1,12 @@ -import json -import os import subprocess from pathlib import Path +from pytest_abra.utils import load_json_to_environ + # --------------------- load credentials to env variables -------------------- # cred_file = Path("credentials.json") -with open(cred_file, "r") as f: - CREDENTIALS = json.load(f) - -for key, value in CREDENTIALS.items(): - os.environ[key] = value +load_json_to_environ(cred_file) # --------------------------------- env files -------------------------------- # diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index b5bd3cf..6de521c 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -1,3 +1,5 @@ +import json +import os import random import string from dataclasses import dataclass @@ -46,3 +48,11 @@ def generate_random_string(length: int, punctuation=False) -> str: characters += string.punctuation random_string = "".join(random.choice(characters) for _ in range(length)) return random_string + + +def load_json_to_environ(cred_file: Path): + with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + + for key, value in CREDENTIALS.items(): + os.environ[key] = value -- 2.47.2 From f49029aeed8fd2bbeece906035f986cfcaaa4969 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:14:13 +0100 Subject: [PATCH 13/91] add docstring --- pytest_abra/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index 6de521c..407df11 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -51,6 +51,7 @@ def generate_random_string(length: int, punctuation=False) -> str: def load_json_to_environ(cred_file: Path): + """Load the contents of a json file directly into os.environ. Variable names are inherited""" with open(cred_file, "r") as f: CREDENTIALS = json.load(f) -- 2.47.2 From 27a0ff8a2e976b2a40f74599338b1fe4ce00cb54 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:20:49 +0100 Subject: [PATCH 14/91] add load_test_credentials --- pytest_abra/coordinator.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index e0f2162..b599458 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -1,4 +1,5 @@ import importlib +import json import re import sys from pathlib import Path @@ -9,7 +10,7 @@ from pytest_abra.dir_manager import DirManager from pytest_abra.env_manager import EnvFile, EnvManager from pytest_abra.html_helper import merge_html_reports from pytest_abra.runner import Runner -from pytest_abra.utils import rmtree +from pytest_abra.utils import generate_random_string, load_json_to_environ, rmtree class Coordinator: @@ -36,7 +37,7 @@ class Coordinator: logger.info("calling prepare_tests()") self.DIR.create_all_dirs() self.ENV.copy_env_files(self.DIR) - # todo: create random testuser creds and load them + self.load_test_credentials() def run_tests(self) -> None: logger.info("calling run_tests()") @@ -87,6 +88,25 @@ class Coordinator: f.parent.rename(new_path) rmtree(trace_root_dir) + def load_test_credentials(self): + """Load test user credentials. If not available, create them randomly. + + Test users are created during testing but should be deleted after the test. In case test + users are not deleted after tests by accident, the user credentials are not known to an + attacker.""" + + test_credentials_path = self.DIR.STATES / "credentials_test.json" + if not test_credentials_path.is_file(): + test_credentials = { + "TEST_USER": "test-" + generate_random_string(6), + "TEST_PASS": generate_random_string(12, punctuation=True), + } + + with open(test_credentials_path, "w") as json_file: + json.dump(test_credentials, json_file) + + load_json_to_environ(test_credentials_path) + @staticmethod def create_runner_dict(recipes_dir: Path) -> dict[str, type[Runner]]: """Creates a dictionary holding all the RunnerClasses that can be discovered in recipes_dir -- 2.47.2 From 093818bc816e2de4a948f71c4a5a597e810e5a8a Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:41:07 +0100 Subject: [PATCH 15/91] results to status --- pytest_abra/dir_manager.py | 5 +++++ pytest_abra/runner.py | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pytest_abra/dir_manager.py b/pytest_abra/dir_manager.py index 1ff4c11..c54954b 100644 --- a/pytest_abra/dir_manager.py +++ b/pytest_abra/dir_manager.py @@ -37,6 +37,7 @@ class DirManager: self.STATES, self.ENV_FILES, self.RESULTS, + self.STATUS, ] for d in dirs: d.mkdir(exist_ok=True) @@ -69,6 +70,10 @@ class DirManager: def RESULTS(self): return self.SESSION / "results" + @property + def STATUS(self): + return self.SESSION / "status" + @property def RECIPES(self): return self.recipes_dir diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index c8c3812..055c7c1 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -119,13 +119,13 @@ class Runner: other than 'passed' will be deleted""" already_passed = False - for result in self.DIR.RESULTS.glob("*"): - if identifier_string in result.name: + for status in self.DIR.STATUS.glob("*"): + if identifier_string in status.name: # process any result file (passed / failed / skipped) if it exists - if "passed" in result.name: + if "passed" in status.name: already_passed = True elif remove_existing: - result.unlink() + status.unlink() return already_passed def _call_pytest(self, full_test_path: Path) -> int: @@ -189,7 +189,7 @@ class Runner: if isinstance(result, int): result = cls.result_int_to_str(result) full_name = cls.combine_names(result, identifier_string) - file_path = DIR.RESULTS / full_name + file_path = DIR.STATUS / full_name with open(file_path, "w") as _: pass # create empty file @@ -198,7 +198,7 @@ class Runner: # todo: what about conditional setups? - passed_tests = [r.name for r in self.DIR.RESULTS.glob("*") if "passed" in r.name] + passed_tests = [r.name for r in self.DIR.STATUS.glob("*") if "passed" in r.name] results = [] for dependency in self.dependencies: dependency_runner = self.coordinator.RUNNER_DICT[dependency] -- 2.47.2 From 7c1f1ff5d4b51cb4502747b1695003c1e8d66cbd Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 00:43:12 +0100 Subject: [PATCH 16/91] records -> results --- pytest_abra/cli.py | 2 +- pytest_abra/coordinator.py | 10 +++++----- pytest_abra/custom_fixtures.py | 4 ++-- pytest_abra/dir_manager.py | 11 +++-------- pytest_abra/runner.py | 4 ++-- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 53d534b..2a9c3ae 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -37,7 +37,7 @@ def run(): # todo: move to Coordinator DIR = DirManager(output_dir=args.output_dir, session_id=session_id) - log_file = DIR.RECORDS / "coordinator.log" + log_file = DIR.RESULTS / "coordinator.log" logger.add(log_file) # ---------------------------- initialize and run ---------------------------- # diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index b599458..8819388 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -60,13 +60,13 @@ class Coordinator: def combine_html(self) -> None: """combines all generated pytest html reports into one""" - in_dir_path = str(self.DIR.RECORDS / "html") - out_file_path = str(self.DIR.RECORDS / "full-report.html") + in_dir_path = str(self.DIR.RESULTS / "html") + out_file_path = str(self.DIR.RESULTS / "full-report.html") title = "combined.html" merge_html_reports(in_dir_path, out_file_path, title) def collect_traces(self): - """moves all traces into SESSION/RECORDS dir + """moves all traces into SESSION/RESULTS dir if tests are rerun and generate another trace, the new trace will get a unique name such as tracename-0 @@ -82,9 +82,9 @@ class Coordinator: index += 1 return get_new_path(root_dir, base_name, index=index) - trace_root_dir = self.DIR.RECORDS / "traces" + trace_root_dir = self.DIR.RESULTS / "traces" for f in trace_root_dir.rglob("*/trace.zip"): - new_path = get_new_path(self.DIR.RECORDS, f.parent.name) + new_path = get_new_path(self.DIR.RESULTS, f.parent.name) f.parent.rename(new_path) rmtree(trace_root_dir) diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index 82bc0bd..d627812 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -49,9 +49,9 @@ def DIR(request) -> DirManager: DIR.OUTPUT DIR.SESSION - DIR.RECORDS DIR.STATES - DIR.RESULTS""" + DIR.RESULTS + DIR.STATUS""" output_dir = request.config.getoption("--output_dir") assert output_dir, "pytest argument --output_dir not set" diff --git a/pytest_abra/dir_manager.py b/pytest_abra/dir_manager.py index c54954b..b290ab5 100644 --- a/pytest_abra/dir_manager.py +++ b/pytest_abra/dir_manager.py @@ -11,11 +11,11 @@ class DirManager: The structures is as follows: tests dir/ session_id-1/ - records results states + status session_id-2/ - records + results ... """ @@ -32,7 +32,6 @@ class DirManager: dirs: list[Path] = [ self.OUTPUT_DIR, self.SESSION, - self.RECORDS, self.HTML, self.STATES, self.ENV_FILES, @@ -50,13 +49,9 @@ class DirManager: def SESSION(self): return self.OUTPUT_DIR / self.session_id - @property - def RECORDS(self): - return self.SESSION / "records" - @property def HTML(self): - return self.RECORDS / "html" + return self.RESULTS / "html" @property def STATES(self): diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 055c7c1..251068a 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -158,7 +158,7 @@ class Runner: # --output only works with the given context and page fixture # folder needs to be unique! traces will not appear, if every pytest run has same output dir command_arguments.append("--output") - command_arguments.append(str(self.DIR.RECORDS / "traces" / full_test_path.stem)) + command_arguments.append(str(self.DIR.RESULTS / "traces" / full_test_path.stem)) # tracing command_arguments.append("--tracing") # "on", "off", "retain-on-failure" @@ -173,7 +173,7 @@ class Runner: # command_arguments.append("--headed") # html report. Will be combined into one file later. - command_arguments.append(f"--html={self.DIR.RECORDS / 'html' / full_test_path.with_suffix('.html').name}") + command_arguments.append(f"--html={self.DIR.RESULTS / 'html' / full_test_path.with_suffix('.html').name}") return pytest.main(command_arguments) -- 2.47.2 From 5368b667e67723b16c973e79dc21342d21487d6f Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 01:14:18 +0100 Subject: [PATCH 17/91] add dev dependencies --- pyproject.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d436714..2fd610a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,9 +21,16 @@ dependencies = [ "loguru == 0.7.2", "beautifulsoup4 == 4.12.2", "imbox == 0.9.8", + "tabulate", "hatchling == 1.18.0", "icecream", - "tabulate", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "mypy", + "ruff", ] [project.entry-points.pytest11] -- 2.47.2 From 4d8033ca9d8dd40f1883f044d7a69c6662b4d45f Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 01:14:44 +0100 Subject: [PATCH 18/91] fix type --- pytest_abra/custom_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index d627812..c5e8b6c 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -93,7 +93,7 @@ def URL(env_config: dict[str, str]) -> BaseUrl: @pytest.fixture(scope="session") -def imap_client() -> None: +def imap_client() -> Generator[Imbox]: """imap email client using credentials from environment variables""" assert os.environ["IMAP_HOST"] -- 2.47.2 From b5bd3615695c49607c0c2984c72bc48b086666f6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 01:14:58 +0100 Subject: [PATCH 19/91] wip pre-commit config --- .pre-commit-config.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4eca63b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +repos: + - repo: local + hooks: + # - id: whitespace + # name: strip whitespace + # entry: ./strip_whitespace.sh + # language: system + # always_run: true + # pass_filenames: false + - id: ruff + name: ruff + entry: python -m ruff . --preview + language: system + always_run: true + pass_filenames: false + - id: mypy + name: mypy + entry: mypy pytest_abra/ + language: system + always_run: true + pass_filenames: false + - id: tests + name: run all tests that are not marked slow + entry: pytest -m "not slow" + language: system + always_run: true + pass_filenames: false -- 2.47.2 From 5a45255bd6c6bc1640545f671c623c3f2d3ca057 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 01:25:47 +0100 Subject: [PATCH 20/91] test --- .pre-commit-config.yaml | 47 ++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4eca63b..2b2ef57 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,27 +1,36 @@ repos: - - repo: local - hooks: + # - repo: local + # hooks: # - id: whitespace # name: strip whitespace # entry: ./strip_whitespace.sh # language: system # always_run: true # pass_filenames: false + + # - id: mypy + # name: mypy + # entry: mypy pytest_abra/ + # language: system + # always_run: true + # pass_filenames: false + # - id: tests + # name: run all tests that are not marked slow + # entry: pytest -m "not slow" + # language: system + # always_run: true + # pass_filenames: false + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: '' # Use the sha / tag you want to point at + hooks: + - id: mypy + + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.1.7 + hooks: + # Run the linter. - id: ruff - name: ruff - entry: python -m ruff . --preview - language: system - always_run: true - pass_filenames: false - - id: mypy - name: mypy - entry: mypy pytest_abra/ - language: system - always_run: true - pass_filenames: false - - id: tests - name: run all tests that are not marked slow - entry: pytest -m "not slow" - language: system - always_run: true - pass_filenames: false + # Run the formatter. + - id: ruff-format \ No newline at end of file -- 2.47.2 From 8ba0121415a783a23c20982704994637e1f602a7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 01:27:00 +0100 Subject: [PATCH 21/91] add working config --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b2ef57..b7c3d03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,6 @@ repos: - id: mypy - repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. rev: v0.1.7 hooks: # Run the linter. -- 2.47.2 From ba33d97c536adb014cb55a10276bc0a242c4314e Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 01:44:02 +0100 Subject: [PATCH 22/91] fix type and config for tests --- .pre-commit-config.yaml | 29 +++++++++-------------------- pytest_abra/custom_fixtures.py | 10 +++++----- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7c3d03..fabed9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,30 +1,19 @@ repos: # - repo: local - # hooks: - # - id: whitespace - # name: strip whitespace - # entry: ./strip_whitespace.sh - # language: system - # always_run: true - # pass_filenames: false - - # - id: mypy - # name: mypy - # entry: mypy pytest_abra/ - # language: system - # always_run: true - # pass_filenames: false - # - id: tests - # name: run all tests that are not marked slow - # entry: pytest -m "not slow" - # language: system - # always_run: true - # pass_filenames: false + # hooks: + # - id: tests + # name: run all tests that are not marked slow + # entry: python -m pytest -m "not slow" + # language: system + # language_version: default + # always_run: true + # pass_filenames: false - repo: https://github.com/pre-commit/mirrors-mypy rev: '' # Use the sha / tag you want to point at hooks: - id: mypy + args: [--ignore-missing-imports] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.7 diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index c5e8b6c..972b07e 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -3,7 +3,8 @@ import os import re -from datetime import datetime, timedelta + +# from datetime import datetime, timedelta from pathlib import Path from typing import Generator, Protocol, TypedDict @@ -93,7 +94,7 @@ def URL(env_config: dict[str, str]) -> BaseUrl: @pytest.fixture(scope="session") -def imap_client() -> Generator[Imbox]: +def imap_client() -> Generator[Imbox, None, None]: """imap email client using credentials from environment variables""" assert os.environ["IMAP_HOST"] @@ -138,9 +139,8 @@ def imap_recent_messages(imap_client: Imbox) -> list[Message]: for uid, message in messages: print(uid, message.subject, message.date)""" - N_MINUTES = 30 - - n_minutes_ago = datetime.now() - timedelta(minutes=N_MINUTES) + # N_MINUTES = 30 + # n_minutes_ago = datetime.now() - timedelta(minutes=N_MINUTES) uids: list[bytes] = [] messages: list[Message] = [] # for uid, message in imap_client.messages(date__gt=n_minutes_ago): -- 2.47.2 From 705287c84d9e9a863a72ac13e9639f09cc643ed2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 01:59:11 +0100 Subject: [PATCH 23/91] use URL --- recipes/authentik/tests_authentik/setup_authentik.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/recipes/authentik/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py index 9aa6e3b..96e72e8 100644 --- a/recipes/authentik/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -14,11 +14,10 @@ ADMIN_PASS = os.environ["ADMIN_PASS"] TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} -def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager): +def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl): # go to page page = context.new_page() - url = "https://" + env_config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) # check welcome message welcome_message = env_config.get("welcome_message") -- 2.47.2 From eacfd6582b6ca0233315c901b98625fa25deeb42 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 02:00:02 +0100 Subject: [PATCH 24/91] use URL --- recipes/wordpress/tests_wordpress/setup_wordpress.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/recipes/wordpress/tests_wordpress/setup_wordpress.py b/recipes/wordpress/tests_wordpress/setup_wordpress.py index 4c44799..a65d8fe 100644 --- a/recipes/wordpress/tests_wordpress/setup_wordpress.py +++ b/recipes/wordpress/tests_wordpress/setup_wordpress.py @@ -1,14 +1,13 @@ import pytest from playwright.sync_api import BrowserContext, Page, expect -from pytest_abra.dir_manager import DirManager +from pytest_abra import BaseUrl, DirManager -def test_visit_from_domain(authentik_admin_context: BrowserContext, env_config: dict[str, str]): +def test_visit_from_domain(authentik_admin_context: BrowserContext, URL: BaseUrl): """visit wordpress directly with admin_session, expect not to be logged in""" page = authentik_admin_context.new_page() - url = "https://" + env_config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) with pytest.raises(AssertionError): # look for admin bar expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000) -- 2.47.2 From f1c862c903762cf78a13b5df609c23cf714fb5b4 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 02:00:48 +0100 Subject: [PATCH 25/91] use URL --- .../tests_wordpress/test_wordpress_localization.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py index 51601cb..4731a0c 100644 --- a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py +++ b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py @@ -2,13 +2,12 @@ from playwright.sync_api import BrowserContext, expect -from pytest_abra.dir_manager import DirManager +from pytest_abra import BaseUrl -def test_welcome_message(context: BrowserContext, env_config: dict[str, str], DIR: DirManager): +def test_welcome_message(context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): page = context.new_page() - url = "https://" + env_config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) expect(page.locator(".wp-block-heading")).to_be_visible() if "locale" in env_config and "de" in env_config["locale"]: -- 2.47.2 From d22af87ca196eb85aa30c12674da31fda7b09079 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 02:01:42 +0100 Subject: [PATCH 26/91] improve imports --- recipes/authentik/tests_authentik/fixtures_authentik.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipes/authentik/tests_authentik/fixtures_authentik.py b/recipes/authentik/tests_authentik/fixtures_authentik.py index fa50e3f..b19b06a 100644 --- a/recipes/authentik/tests_authentik/fixtures_authentik.py +++ b/recipes/authentik/tests_authentik/fixtures_authentik.py @@ -3,8 +3,7 @@ import json import pytest from playwright.sync_api import BrowserContext, Page -from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager @pytest.fixture -- 2.47.2 From cf4cfdc4c9446e32d26d0cc517b30eee81d8f5fd Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 02:05:29 +0100 Subject: [PATCH 27/91] improve imports --- pytest_abra/__init__.py | 3 ++- pytest_abra/coordinator.py | 4 +--- pytest_abra/env_manager.py | 3 +-- pytest_abra/runner.py | 4 +--- recipes/authentik/tests_authentik/setup_authentik.py | 3 +-- recipes/nextcloud/tests_nextcloud/conftest.py | 3 +-- recipes/nextcloud/tests_nextcloud/setup_nextcloud.py | 3 +-- 7 files changed, 8 insertions(+), 15 deletions(-) diff --git a/pytest_abra/__init__.py b/pytest_abra/__init__.py index 0b134ec..5ec5233 100644 --- a/pytest_abra/__init__.py +++ b/pytest_abra/__init__.py @@ -1,6 +1,6 @@ from pytest_abra.coordinator import Coordinator from pytest_abra.dir_manager import DirManager -from pytest_abra.env_manager import EnvFile +from pytest_abra.env_manager import EnvFile, EnvManager from pytest_abra.runner import ConditionArgs, Runner, Test from pytest_abra.utils import BaseUrl @@ -12,4 +12,5 @@ __all__ = [ "DirManager", "BaseUrl", "EnvFile", + "EnvManager", ] diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 8819388..738f084 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -6,10 +6,8 @@ from pathlib import Path from loguru import logger -from pytest_abra.dir_manager import DirManager -from pytest_abra.env_manager import EnvFile, EnvManager +from pytest_abra import DirManager, EnvFile, EnvManager, Runner from pytest_abra.html_helper import merge_html_reports -from pytest_abra.runner import Runner from pytest_abra.utils import generate_random_string, load_json_to_environ, rmtree diff --git a/pytest_abra/env_manager.py b/pytest_abra/env_manager.py index f9d79cd..ab3754a 100644 --- a/pytest_abra/env_manager.py +++ b/pytest_abra/env_manager.py @@ -5,8 +5,7 @@ from typing import TYPE_CHECKING, NamedTuple from dotenv import dotenv_values if TYPE_CHECKING: - from pytest_abra.dir_manager import DirManager - from pytest_abra.runner import Runner + from pytest_abra import DirManager, Runner class EnvFile(NamedTuple): diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 251068a..72561ac 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -7,9 +7,7 @@ import pytest from loguru import logger if TYPE_CHECKING: - from pytest_abra import DirManager - from pytest_abra.coordinator import Coordinator - from pytest_abra.env_manager import EnvFile + from pytest_abra import Coordinator, DirManager, EnvFile class ConditionArgs(NamedTuple): diff --git a/recipes/authentik/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py index 96e72e8..5fbde7a 100644 --- a/recipes/authentik/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -4,8 +4,7 @@ import re from playwright.sync_api import BrowserContext, expect -from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager ADMIN_USER = os.environ["ADMIN_USER"] ADMIN_PASS = os.environ["ADMIN_PASS"] diff --git a/recipes/nextcloud/tests_nextcloud/conftest.py b/recipes/nextcloud/tests_nextcloud/conftest.py index ce56379..4151c7a 100644 --- a/recipes/nextcloud/tests_nextcloud/conftest.py +++ b/recipes/nextcloud/tests_nextcloud/conftest.py @@ -4,8 +4,7 @@ import os import pytest from playwright.sync_api import BrowserContext, Page -from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager pytest_plugins = "authentik.tests_authentik.fixtures_authentik" diff --git a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py index 64c8e94..3f18b2c 100644 --- a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py @@ -2,8 +2,7 @@ import re from playwright.sync_api import Page, expect -from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager # url dashboard # https://files.test.dev.local-it.cloud/apps/dashboard/ -- 2.47.2 From 019d2c028c5b0c8cd39aa7723ea3a60ed0c512e6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 02:06:43 +0100 Subject: [PATCH 28/91] fixup --- pytest_abra/coordinator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 738f084..8819388 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -6,8 +6,10 @@ from pathlib import Path from loguru import logger -from pytest_abra import DirManager, EnvFile, EnvManager, Runner +from pytest_abra.dir_manager import DirManager +from pytest_abra.env_manager import EnvFile, EnvManager from pytest_abra.html_helper import merge_html_reports +from pytest_abra.runner import Runner from pytest_abra.utils import generate_random_string, load_json_to_environ, rmtree -- 2.47.2 From 94f03f946b459aff4ae37ce7ea26846799ad4e96 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 02:17:08 +0100 Subject: [PATCH 29/91] move docs to docs dir --- {prototyping => docs}/documentation.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {prototyping => docs}/documentation.md (100%) diff --git a/prototyping/documentation.md b/docs/documentation.md similarity index 100% rename from prototyping/documentation.md rename to docs/documentation.md -- 2.47.2 From cd00de6d017b4f35f6c2bf413b44aae272bf0a9b Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 02:24:34 +0100 Subject: [PATCH 30/91] change discvoery mask --- pytest_abra/coordinator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 8819388..2ec3182 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -128,7 +128,7 @@ class Coordinator: # make it possible to import modules from recipes_dir sys.path.append(recipes_dir.as_posix()) - for module_path in recipes_dir.rglob("*/runner*.py"): + for module_path in recipes_dir.rglob("*/runner_*.py"): rel_path = module_path.relative_to(recipes_dir).as_posix().replace("/", ".").replace(".py", "") module = importlib.import_module(rel_path) runner_class_names = [name for name in dir(module) if runner_discovery_pattern.match(name)] -- 2.47.2 From 050d8cde38f50b52f48b4c71d287265730ed84f6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 02:24:39 +0100 Subject: [PATCH 31/91] add discovery segment --- docs/documentation.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/documentation.md b/docs/documentation.md index e27c359..013e57e 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -96,6 +96,21 @@ def condition_function(args: ConditionArgs) -> bool: ... ``` +## Discovery of `Runners` and `Tests` + +- Runners will be discovered, if they are defined in a moduled of name `runner_*.py` including a class of name `Runner*`. + +- Tests will be discovered by filename as long as they are placed in the parent dir of `runner_*.py` or in any subdirectory. + +``` +DIR parent_dir +├── FILE runner_*.py +├── FILE test1.py +└── DIR subdir + ├── DIR subsubdir + │ └── test2.py + └── test3.py +``` # Create custom Tests -- 2.47.2 From db70a49badd44537e7de1e80481e15cb235a57e9 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 11:32:30 +0100 Subject: [PATCH 32/91] add file exists check to load_json_to_environ --- pytest_abra/utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index 407df11..b9fcb2d 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -7,6 +7,8 @@ from datetime import datetime from pathlib import Path from urllib.parse import urlunparse +from loguru import logger + @dataclass class BaseUrl: @@ -28,7 +30,7 @@ def get_datetime_string() -> str: return current_datetime.strftime("%Y-%m-%d-%H-%M-%S") -def rmtree(root_dir: Path): +def rmtree(root_dir: Path) -> None: """removes a folder with content recursively""" if not root_dir.is_dir(): return @@ -50,8 +52,13 @@ def generate_random_string(length: int, punctuation=False) -> str: return random_string -def load_json_to_environ(cred_file: Path): +def load_json_to_environ(cred_file: Path) -> None: """Load the contents of a json file directly into os.environ. Variable names are inherited""" + + if not cred_file.is_file(): + logger.warning(f"{cred_file} could not be found, no credentials loaded") + return + with open(cred_file, "r") as f: CREDENTIALS = json.load(f) -- 2.47.2 From 2c406c3a343df0e27a461d0d6c6688d9a2909972 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 11:32:35 +0100 Subject: [PATCH 33/91] add example credentials --- .gitignore | 3 ++- credentials-example.json | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 credentials-example.json diff --git a/.gitignore b/.gitignore index 78cd629..f541c4b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ TestResults/ *.json *.zip *.egg-info -credentials* \ No newline at end of file +credentials* +!credentials-example.json \ No newline at end of file diff --git a/credentials-example.json b/credentials-example.json new file mode 100644 index 0000000..fa0f58b --- /dev/null +++ b/credentials-example.json @@ -0,0 +1,9 @@ +{ + "ADMIN_USER": "admin", + "ADMIN_PASS": "password", + "IMAP_EMAIL": "test@domain.com", + "IMAP_HOST": "mail.domain.com", + "IMAP_PORT": "993", + "IMAP_USER": "imap_user", + "IMAP_PASS": "password" +} \ No newline at end of file -- 2.47.2 From dd5fe859e86ed0048a5cc6867214508658769f38 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 11:34:05 +0100 Subject: [PATCH 34/91] rename _execute_tests_list --- pytest_abra/runner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 72561ac..99c494f 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -43,18 +43,18 @@ class Runner: def run_setups(self): """runs the setup scripts if available""" - self._execute_test_list(self.setups) + self._execute_tests_list(self.setups) def run_tests(self): """runs the test scripts if available""" - self._execute_test_list(self.tests) + self._execute_tests_list(self.tests) def run_cleanups(self): """runs the cleanup scripts if available""" - self._execute_test_list(self.cleanups) + self._execute_tests_list(self.cleanups) - def _execute_test_list(self, test_list: list[Test]): - """runs the main test script and if available and sub test scripts if their running condition is met""" + def _execute_tests_list(self, test_list: list[Test]): + """Runs all tests given in the list. If condition is defined, it is also checked.""" # check if required dependencies have passed if not self._dependencies_passed(): logger.warning(f"skipping run_tests() of {self.env_type} (one or more dependencies have not passed)") -- 2.47.2 From 0c8999b070b21304cd145c0a931aacd5271e1aa1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 11:51:32 +0100 Subject: [PATCH 35/91] huge refactor of runner functions --- pytest_abra/runner.py | 86 ++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 99c494f..5667cd9 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -64,20 +64,14 @@ class Runner: self._run_test_with_checks(test) def _run_test_with_checks(self, test: Test): - # dependency passed: true / false - # already_passed: true / false - # prevent_skip: true / false - # condition_available: true / pass - # condition_met: true / false - identifier_string = self.combine_names(self.env_type, test.test_file) - results = list(self._tests_path.rglob(test.test_file)) - assert len(results) == 1, f"{test.test_file} should exist exactly 1 time, but found {len(results)} times" - full_test_path = results[0] + test_files = list(self._tests_path.rglob(test.test_file)) + assert len(test_files) == 1, f"{test.test_file} should exist exactly once, but found {len(test_files)} times" + full_test_path = test_files[0] # check if test aleady passed - if self._is_test_passed(identifier_string, remove_existing=True): + if self._is_test_passed(self.DIR, identifier_string): if test.prevent_skip: logger.info(f"continuing {identifier_string} (passed before but prevent_skip=True)") else: @@ -85,19 +79,19 @@ class Runner: return if test.condition: - condition_result = self._run_condition(test.condition) + condition_result = self._call_condition_function(test.condition) if not condition_result: # test condition is defined but not met logger.info(f"skipping {identifier_string} (test condition is not met)") - self.create_result_file(self.DIR, result="skipped", identifier_string=identifier_string) + self._create_status_file(self.DIR, status="skipped", identifier_string=identifier_string) return # test condition is undefined or not met logger.info(f"running {identifier_string}") - result = self._call_pytest(full_test_path) - self.create_result_file(self.DIR, result=result, identifier_string=identifier_string) + exit_code = self._call_pytest(full_test_path) + self._create_status_file(self.DIR, status=exit_code, identifier_string=identifier_string) - def _run_condition(self, condition_function: Callable[[ConditionArgs], bool]): + def _call_condition_function(self, condition_function: Callable[[ConditionArgs], bool]): """run the test condition function with multiple arguments""" # more arguments can be added later without changing the function signature conditon_args = ConditionArgs( @@ -107,24 +101,38 @@ class Runner: ) return condition_function(conditon_args) - def _is_test_passed(self, identifier_string: str, remove_existing: bool = False) -> bool: - """returns True if the selected test matching identifier_string already passed + @classmethod + def _create_status_file( + cls, + DIR: "DirManager", + status: int | str, + identifier_string: str, + ): + """create result file to indicated passed/failed/skipped test""" - This is determined by the presence of a specific output file in the RESULTS folder that - matches identifier_string + if isinstance(status, int): + status = cls.exit_code_to_str(status) - remove_existing: If True, result files matching identifier_string with a status - other than 'passed' will be deleted""" + # remove matching files + [f.unlink() for f in DIR.STATUS.glob("*") if identifier_string in f.name] - already_passed = False - for status in self.DIR.STATUS.glob("*"): - if identifier_string in status.name: - # process any result file (passed / failed / skipped) if it exists - if "passed" in status.name: - already_passed = True - elif remove_existing: - status.unlink() - return already_passed + full_name = cls.combine_names(status, identifier_string) + file_path = DIR.STATUS / full_name + with open(file_path, "w") as _: + pass # create empty file + + @staticmethod + def _is_test_passed(DIR: "DirManager", identifier_string: str) -> bool: + """returns True if the selected test matching identifier_string already passed""" + + matching_files = [f for f in DIR.STATUS.glob("*") if identifier_string in f.name] + if len(matching_files) == 1: + status_file = matching_files[0] + if "passed" in status_file.name: + return True + elif len(matching_files) > 1: + logger.warning("more than one matching status file found") + return False def _call_pytest(self, full_test_path: Path) -> int: """runs pytest programmatically with a specific file @@ -175,22 +183,6 @@ class Runner: return pytest.main(command_arguments) - @classmethod - def create_result_file( - cls, - DIR: "DirManager", - result: int | str, - identifier_string: str, - ): - """create result file to indicated passed/failed or skipped test""" - - if isinstance(result, int): - result = cls.result_int_to_str(result) - full_name = cls.combine_names(result, identifier_string) - file_path = DIR.STATUS / full_name - with open(file_path, "w") as _: - pass # create empty file - def _dependencies_passed(self): """returns true if all setups of each dependency have passed""" @@ -206,7 +198,7 @@ class Runner: return all(results) @staticmethod - def result_int_to_str(result_int: int) -> str: + def exit_code_to_str(result_int: int) -> str: """converts the pytest exit code (int) into a meaningful string""" match result_int: case 0: -- 2.47.2 From a4d99a2e7d2facc3af3898208e3bafad0dd2385d Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 12:02:39 +0100 Subject: [PATCH 36/91] change dir order --- pytest_abra/dir_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pytest_abra/dir_manager.py b/pytest_abra/dir_manager.py index b290ab5..c20f03c 100644 --- a/pytest_abra/dir_manager.py +++ b/pytest_abra/dir_manager.py @@ -32,10 +32,10 @@ class DirManager: dirs: list[Path] = [ self.OUTPUT_DIR, self.SESSION, - self.HTML, self.STATES, self.ENV_FILES, self.RESULTS, + self.HTML, self.STATUS, ] for d in dirs: @@ -49,10 +49,6 @@ class DirManager: def SESSION(self): return self.OUTPUT_DIR / self.session_id - @property - def HTML(self): - return self.RESULTS / "html" - @property def STATES(self): return self.SESSION / "states" @@ -65,6 +61,10 @@ class DirManager: def RESULTS(self): return self.SESSION / "results" + @property + def HTML(self): + return self.RESULTS / "html" + @property def STATUS(self): return self.SESSION / "status" -- 2.47.2 From 131477557da2489b79298be541a1fc1d016309a2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 12:03:31 +0100 Subject: [PATCH 37/91] more refactoring --- pytest_abra/runner.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 5667cd9..1951cce 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -114,7 +114,8 @@ class Runner: status = cls.exit_code_to_str(status) # remove matching files - [f.unlink() for f in DIR.STATUS.glob("*") if identifier_string in f.name] + for status_file in cls._get_status_files(DIR, identifier_string): + status_file.unlink() full_name = cls.combine_names(status, identifier_string) file_path = DIR.STATUS / full_name @@ -122,10 +123,14 @@ class Runner: pass # create empty file @staticmethod - def _is_test_passed(DIR: "DirManager", identifier_string: str) -> bool: + def _get_status_files(DIR: "DirManager", identifier_string: str) -> list[Path]: + return [f for f in DIR.STATUS.glob("*") if identifier_string in f.name] + + @classmethod + def _is_test_passed(cls, DIR: "DirManager", identifier_string: str) -> bool: """returns True if the selected test matching identifier_string already passed""" - matching_files = [f for f in DIR.STATUS.glob("*") if identifier_string in f.name] + matching_files = cls._get_status_files(DIR, identifier_string) if len(matching_files) == 1: status_file = matching_files[0] if "passed" in status_file.name: -- 2.47.2 From cf93cc80468891bdab89753dfad1d6d0efd92cce Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 12:10:04 +0100 Subject: [PATCH 38/91] add test for runner functions --- tests/test_runner.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/test_runner.py diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..24f9bf6 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,29 @@ +from pathlib import Path + +from pytest_abra import DirManager, Runner + + +def test_runner_create_status_file(tmp_path: Path): + """check if _create_status_file prevents duplicates""" + + DIR = DirManager(output_dir=tmp_path, session_id="temp") + DIR.create_all_dirs() + assert len(list(DIR.STATUS.iterdir())) == 0 + + # create first status file + Runner._create_status_file(DIR, "passed", "identifier-a") + assert len(list(DIR.STATUS.iterdir())) == 1 + + # create second status file + Runner._create_status_file(DIR, "passed", "identifier-b") + assert len(list(DIR.STATUS.iterdir())) == 2 + + # check if _get_status_files finds only the correct status file + result = Runner._get_status_files(DIR, "identifier-a") + assert len(result) == 1 + + # overwrite first status file + Runner._create_status_file(DIR, "failed", "identifier-a") + assert len(list(DIR.STATUS.iterdir())) == 2 + + assert Runner._is_test_passed(DIR, "identifier-a") is False -- 2.47.2 From 016d0e6b183497ef14cc6d91d1555816f98cfcba Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 12:43:29 +0100 Subject: [PATCH 39/91] add --session_id and get_session_id --- pytest_abra/cli.py | 10 ++++------ pytest_abra/utils.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 2a9c3ae..c26f79f 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -6,7 +6,7 @@ from loguru import logger from pytest_abra import Coordinator from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import get_datetime_string +from pytest_abra.utils import get_session_id def run(): @@ -17,6 +17,8 @@ def run(): parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=20_000) parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode") parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests") + parser.add_argument("--session_id", help="Session dir name (inside output_dir). Overwrites --resume") + args = parser.parse_args() env_paths = [Path(s) for s in args.env_paths.split(";")] @@ -27,11 +29,7 @@ def run(): # ----------------------------- define session_id ---------------------------- # - session_id = "test-" + get_datetime_string() - if args.resume: - latest_session_id = DirManager.get_latest_session_id(args.output_dir) - if latest_session_id: - session_id = DirManager.get_latest_session_id(args.output_dir) + session_id = get_session_id(args.session_id, args.resume, args.output_dir) # ------------------------------- setup logging ------------------------------ # diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index b9fcb2d..8171a97 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -9,6 +9,8 @@ from urllib.parse import urlunparse from loguru import logger +from pytest_abra.dir_manager import DirManager + @dataclass class BaseUrl: @@ -64,3 +66,15 @@ def load_json_to_environ(cred_file: Path) -> None: for key, value in CREDENTIALS.items(): os.environ[key] = value + + +def get_session_id(args_session_id: str, args_resume: bool, args_output_dir: Path) -> str: + """converts the cli arguments to the correct session_id""" + session_id = args_session_id + if not session_id: + session_id = "test-" + get_datetime_string() + if args_resume: + latest_session_id = DirManager.get_latest_session_id(args_output_dir) + if latest_session_id: + session_id = latest_session_id + return session_id -- 2.47.2 From d7b3373145296e85c26bff5b797f433355a2f77b Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 12:55:05 +0100 Subject: [PATCH 40/91] adjust arguments --- pytest_abra/cli.py | 2 +- pytest_abra/utils.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index c26f79f..0f21f30 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -29,7 +29,7 @@ def run(): # ----------------------------- define session_id ---------------------------- # - session_id = get_session_id(args.session_id, args.resume, args.output_dir) + session_id = get_session_id(args.output_dir, args.resume, args.session_id) # ------------------------------- setup logging ------------------------------ # diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index 8171a97..95880a6 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -5,6 +5,7 @@ import string from dataclasses import dataclass from datetime import datetime from pathlib import Path +from typing import Optional from urllib.parse import urlunparse from loguru import logger @@ -68,7 +69,7 @@ def load_json_to_environ(cred_file: Path) -> None: os.environ[key] = value -def get_session_id(args_session_id: str, args_resume: bool, args_output_dir: Path) -> str: +def get_session_id(args_output_dir: Path, args_resume: bool, args_session_id: Optional[str]) -> str: """converts the cli arguments to the correct session_id""" session_id = args_session_id if not session_id: -- 2.47.2 From 106e40920c5fd466782a1a0da8bcb022bf24856a Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 13:56:26 +0100 Subject: [PATCH 41/91] add cli test --- tests/test_cli.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/test_cli.py diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..633bc4a --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,54 @@ +import re +import time +from pathlib import Path + +import pytest + +from pytest_abra import DirManager +from pytest_abra.utils import get_session_id + + +def test_get_session_id_random(tmp_path: Path): + args_output_dir = tmp_path + args_resume = False + args_session_id = None + session_id = get_session_id(args_output_dir, args_resume, args_session_id) + assert re.search("\d+-\d+-\d+", session_id) + + +def test_get_session_id_explicit1(tmp_path: Path): + args_output_dir = tmp_path + args_resume = False + args_session_id = "abc" + session_id = get_session_id(args_output_dir, args_resume, args_session_id) + assert session_id == "abc" + + +def test_get_session_id_explicit2(tmp_path: Path): + args_output_dir = tmp_path + args_resume = True + args_session_id = "abc" + session_id = get_session_id(args_output_dir, args_resume, args_session_id) + assert session_id == "abc" + + +@pytest.mark.slow +def test_get_session_id_integration(tmp_path: Path): + assert len(list(tmp_path.iterdir())) == 0 + session_id_1 = get_session_id(args_output_dir=tmp_path, args_resume=False, args_session_id=None) + + DIR = DirManager(output_dir=tmp_path, session_id=session_id_1) + DIR.create_all_dirs() + assert len(list(tmp_path.iterdir())) == 1 + + time.sleep(1.1) # get_session_id won't be unique if called without time passed + session_id_2 = get_session_id(args_output_dir=tmp_path, args_resume=False, args_session_id=None) + DIR = DirManager(output_dir=tmp_path, session_id=session_id_2) + DIR.create_all_dirs() + assert len(list(tmp_path.iterdir())) == 2 + + session_id_3 = get_session_id(args_output_dir=tmp_path, args_resume=True, args_session_id=None) + assert session_id_2 == session_id_3 + + session_id_4 = get_session_id(args_output_dir=tmp_path, args_resume=True, args_session_id="abc") + assert session_id_4 == "abc" -- 2.47.2 From 2b8ba3f9c495f6d3bad63aa26946c6120966b5b7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 13:59:53 +0100 Subject: [PATCH 42/91] make load_test_credentials a staticmethod --- pytest_abra/coordinator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 2ec3182..a8dc94f 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -37,7 +37,7 @@ class Coordinator: logger.info("calling prepare_tests()") self.DIR.create_all_dirs() self.ENV.copy_env_files(self.DIR) - self.load_test_credentials() + self.load_test_credentials(self.DIR) def run_tests(self) -> None: logger.info("calling run_tests()") @@ -88,14 +88,15 @@ class Coordinator: f.parent.rename(new_path) rmtree(trace_root_dir) - def load_test_credentials(self): + @staticmethod + def load_test_credentials(DIR: DirManager): """Load test user credentials. If not available, create them randomly. Test users are created during testing but should be deleted after the test. In case test users are not deleted after tests by accident, the user credentials are not known to an attacker.""" - test_credentials_path = self.DIR.STATES / "credentials_test.json" + test_credentials_path = DIR.STATES / "credentials_test.json" if not test_credentials_path.is_file(): test_credentials = { "TEST_USER": "test-" + generate_random_string(6), -- 2.47.2 From 0af72d13a72975a318c2e29650790d6270adec7b Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 14:03:40 +0100 Subject: [PATCH 43/91] add test_load_test_credentials --- tests/test_coordinator.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/test_coordinator.py diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py new file mode 100644 index 0000000..1902966 --- /dev/null +++ b/tests/test_coordinator.py @@ -0,0 +1,18 @@ +import os +from pathlib import Path + +from pytest_abra.coordinator import Coordinator +from pytest_abra.dir_manager import DirManager + + +def test_load_test_credentials(tmp_path: Path): + assert "TEST_USER" not in os.environ + + DIR = DirManager(output_dir=tmp_path, session_id="abc") + DIR.create_all_dirs() + + Coordinator.load_test_credentials(DIR) + + assert "TEST_USER" in os.environ + + assert (DIR.STATES / "credentials_test.json").is_file() -- 2.47.2 From 616fe8a491263f68d0feb076ef1e8c925e5fef12 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 14:06:07 +0100 Subject: [PATCH 44/91] improve test --- tests/test_coordinator.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py index 1902966..c52e6db 100644 --- a/tests/test_coordinator.py +++ b/tests/test_coordinator.py @@ -12,7 +12,13 @@ def test_load_test_credentials(tmp_path: Path): DIR.create_all_dirs() Coordinator.load_test_credentials(DIR) + assert (DIR.STATES / "credentials_test.json").is_file() assert "TEST_USER" in os.environ + test_user_before = os.environ["TEST_USER"] - assert (DIR.STATES / "credentials_test.json").is_file() + os.environ.clear() + assert "TEST_USER" not in os.environ + + Coordinator.load_test_credentials(DIR) + assert test_user_before == os.environ["TEST_USER"] -- 2.47.2 From 1e6676697363731826660902b3e784c44d7b61d7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 14:29:18 +0100 Subject: [PATCH 45/91] add TestResult, all test executions return TestResult --- pytest_abra/coordinator.py | 8 +++++--- pytest_abra/runner.py | 40 ++++++++++++++++++------------------- pytest_abra/shared_types.py | 16 +++++++++++++++ 3 files changed, 41 insertions(+), 23 deletions(-) create mode 100644 pytest_abra/shared_types.py diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index a8dc94f..5149787 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -10,6 +10,7 @@ from pytest_abra.dir_manager import DirManager from pytest_abra.env_manager import EnvFile, EnvManager from pytest_abra.html_helper import merge_html_reports from pytest_abra.runner import Runner +from pytest_abra.shared_types import TestResult from pytest_abra.utils import generate_random_string, load_json_to_environ, rmtree @@ -42,12 +43,13 @@ class Coordinator: def run_tests(self) -> None: logger.info("calling run_tests()") self.runners: list[Runner] = self._load_runners(self.ENV.env_files) + status_list: list[TestResult] = [] for runner in self.runners: - runner.run_setups() + status_list.extend(runner.run_setups()) for runner in self.runners: - runner.run_tests() + status_list.extend(runner.run_tests()) for runner in self.runners: - runner.run_cleanups() + status_list.extend(runner.run_cleanups()) logger.info("run_tests() finished") def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]: diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 1951cce..894283c 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -6,6 +6,8 @@ from typing import TYPE_CHECKING, Callable, NamedTuple import pytest from loguru import logger +from pytest_abra.shared_types import STATUS, TestResult + if TYPE_CHECKING: from pytest_abra import Coordinator, DirManager, EnvFile @@ -41,29 +43,28 @@ class Runner: logger.info(f"creating instance of {self.__class__.__name__}") - def run_setups(self): + def run_setups(self) -> list[TestResult]: """runs the setup scripts if available""" - self._execute_tests_list(self.setups) + return self._execute_tests_list(self.setups) - def run_tests(self): + def run_tests(self) -> list[TestResult]: """runs the test scripts if available""" - self._execute_tests_list(self.tests) + return self._execute_tests_list(self.tests) - def run_cleanups(self): + def run_cleanups(self) -> list[TestResult]: """runs the cleanup scripts if available""" - self._execute_tests_list(self.cleanups) + return self._execute_tests_list(self.cleanups) - def _execute_tests_list(self, test_list: list[Test]): + def _execute_tests_list(self, test_list: list[Test]) -> list[TestResult]: """Runs all tests given in the list. If condition is defined, it is also checked.""" # check if required dependencies have passed if not self._dependencies_passed(): logger.warning(f"skipping run_tests() of {self.env_type} (one or more dependencies have not passed)") - return + return [TestResult("skipped_dep", test.test_file) for test in test_list] - for test in test_list: - self._run_test_with_checks(test) + return [self._run_test_with_checks(test) for test in test_list] - def _run_test_with_checks(self, test: Test): + def _run_test_with_checks(self, test: Test) -> TestResult: identifier_string = self.combine_names(self.env_type, test.test_file) test_files = list(self._tests_path.rglob(test.test_file)) @@ -76,20 +77,22 @@ class Runner: logger.info(f"continuing {identifier_string} (passed before but prevent_skip=True)") else: logger.info(f"skipping {identifier_string} (test has passed)") - return + return TestResult("skipped_pas", test.test_file) if test.condition: condition_result = self._call_condition_function(test.condition) if not condition_result: # test condition is defined but not met logger.info(f"skipping {identifier_string} (test condition is not met)") - self._create_status_file(self.DIR, status="skipped", identifier_string=identifier_string) - return + self._create_status_file(self.DIR, status="skipped_con", identifier_string=identifier_string) + return TestResult("skipped_con", test.test_file) # test condition is undefined or not met logger.info(f"running {identifier_string}") exit_code = self._call_pytest(full_test_path) - self._create_status_file(self.DIR, status=exit_code, identifier_string=identifier_string) + status = self.exit_code_to_str(exit_code) + self._create_status_file(self.DIR, status=status, identifier_string=identifier_string) + return TestResult(status, test.test_file) def _call_condition_function(self, condition_function: Callable[[ConditionArgs], bool]): """run the test condition function with multiple arguments""" @@ -105,14 +108,11 @@ class Runner: def _create_status_file( cls, DIR: "DirManager", - status: int | str, + status: STATUS, identifier_string: str, ): """create result file to indicated passed/failed/skipped test""" - if isinstance(status, int): - status = cls.exit_code_to_str(status) - # remove matching files for status_file in cls._get_status_files(DIR, identifier_string): status_file.unlink() @@ -203,7 +203,7 @@ class Runner: return all(results) @staticmethod - def exit_code_to_str(result_int: int) -> str: + def exit_code_to_str(result_int: int) -> STATUS: """converts the pytest exit code (int) into a meaningful string""" match result_int: case 0: diff --git a/pytest_abra/shared_types.py b/pytest_abra/shared_types.py new file mode 100644 index 0000000..ff0e147 --- /dev/null +++ b/pytest_abra/shared_types.py @@ -0,0 +1,16 @@ +from typing import Literal, NamedTuple + +""" +passed: test passed +failed: test failed +skipped_con: test skipped because condition was not met +skipped_dep: test skipped because dependencies did not finish +skipped_pas: test skipped because it passed before +""" + +STATUS = Literal["passed", "failed", "skipped_con", "skipped_dep", "skipped_pas"] + + +class TestResult(NamedTuple): + status: STATUS + test_name: str -- 2.47.2 From c24f09a9ab8fdc43b778331ddf4502721f9561ca Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 14:43:29 +0100 Subject: [PATCH 46/91] remove test dependency --- tests/test_html_merge.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_html_merge.py b/tests/test_html_merge.py index d97ab83..acebfef 100644 --- a/tests/test_html_merge.py +++ b/tests/test_html_merge.py @@ -16,27 +16,32 @@ def session_tmp_path(tmp_path_factory: pytest.TempPathFactory) -> Path: return tmp_path_factory.mktemp("html_test") -def test_merge_html(session_tmp_path: Path): +@pytest.fixture(scope="session") +def html_file(session_tmp_path: Path) -> Path: """combines all generated pytest html reports into one""" in_dir_path = Path(__file__).parent / "assets" / "html_merge" in_dir_path = in_dir_path.resolve() ic(in_dir_path) - out_file_path = session_tmp_path / "test.html" - out_assets_dir = session_tmp_path / "assets" + html_file = session_tmp_path / "test.html" - merge_html_reports(in_dir_path.as_posix(), out_file_path.as_posix(), "combined.html") + merge_html_reports(in_dir_path.as_posix(), html_file.as_posix(), "combined.html") + return html_file - assert out_file_path.is_file() - assert out_assets_dir.is_dir() - assert next(out_assets_dir.glob("*")) + +def test_merge_html(html_file: Path): + assert html_file.is_file() + assert html_file.parent.is_dir() + assert next(html_file.parent.glob("*")) @pytest.mark.slow -def test_check_result_with_playwright(session_tmp_path, context: BrowserContext): - html_file = session_tmp_path / "test.html" +def test_check_result_with_playwright(html_file: Path, context: BrowserContext): + assert html_file.is_file() + file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() + page = context.new_page() page.goto(file_url) -- 2.47.2 From a9dbd16901be95fd1f3d66b28158899a988ea523 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:05:37 +0100 Subject: [PATCH 47/91] log results table --- pytest_abra/coordinator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 5149787..4e2b371 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -5,6 +5,7 @@ import sys from pathlib import Path from loguru import logger +from tabulate import tabulate # type: ignore from pytest_abra.dir_manager import DirManager from pytest_abra.env_manager import EnvFile, EnvManager @@ -50,7 +51,8 @@ class Coordinator: status_list.extend(runner.run_tests()) for runner in self.runners: status_list.extend(runner.run_cleanups()) - logger.info("run_tests() finished") + result_table = tabulate([[t.test_name, t.status] for t in status_list], headers=["name", "status"]) + logger.info(f"run_tests() finished\n{result_table}") def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]: """Creates an instance of the correct Runner class for each given env file""" -- 2.47.2 From e0d2b7cd21d088d0fcb87b935fde4c8a1ae0eec9 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:26:17 +0100 Subject: [PATCH 48/91] simplify error --- tests/test_html_merge.py | 44 ++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/tests/test_html_merge.py b/tests/test_html_merge.py index acebfef..1fab89e 100644 --- a/tests/test_html_merge.py +++ b/tests/test_html_merge.py @@ -36,30 +36,34 @@ def test_merge_html(html_file: Path): assert next(html_file.parent.glob("*")) -@pytest.mark.slow -def test_check_result_with_playwright(html_file: Path, context: BrowserContext): - assert html_file.is_file() +# @pytest.mark.slow +# def test_check_result_with_playwright(html_file: Path, context: BrowserContext): +# assert html_file.is_file() - file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() +# file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() - page = context.new_page() - page.goto(file_url) +# page = context.new_page() +# page.goto(file_url) - # check if combined is correct - expect(page.get_by_text("2 Passed,")).to_be_visible() - expect(page.get_by_text("2 Failed,")).to_be_visible() - expect(page.get_by_text("tests ran in 12.946 seconds")).to_be_visible() +# # check if combined is correct +# expect(page.get_by_text("2 Passed,")).to_be_visible() +# expect(page.get_by_text("2 Failed,")).to_be_visible() +# expect(page.get_by_text("tests ran in 12.946 seconds")).to_be_visible() - # check if heading is correct - expect(page.get_by_role("heading", name="combined.html")).to_be_visible() +# # check if heading is correct +# expect(page.get_by_role("heading", name="combined.html")).to_be_visible() - # check if traceback is included - expect(page.get_by_text("E AssertionError: One or more")).to_be_visible() +# # check if traceback is included +# expect(page.get_by_text("E AssertionError: One or more")).to_be_visible() - # check if asset works - with page.expect_popup() as page1_info: - page.get_by_role("link", name="Authentik Blueprint Status").click() - page1 = page1_info.value +# # check if asset works +# with page.expect_popup() as page1_info: +# page.get_by_role("link", name="Authentik Blueprint Status").click() +# page1 = page1_info.value - # see if content of txt file is correct - expect(page1.get_by_text("failed")).to_be_visible() +# # see if content of txt file is correct +# expect(page1.get_by_text("failed")).to_be_visible() + + +def test_demo(context: BrowserContext): + assert True -- 2.47.2 From 89a8b8d3e379a378ba3f482545d89cd40fe041bb Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:37:58 +0100 Subject: [PATCH 49/91] remove os.environ.clear() -> breaks context fixture --- tests/test_coordinator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py index c52e6db..7758cae 100644 --- a/tests/test_coordinator.py +++ b/tests/test_coordinator.py @@ -17,7 +17,8 @@ def test_load_test_credentials(tmp_path: Path): assert "TEST_USER" in os.environ test_user_before = os.environ["TEST_USER"] - os.environ.clear() + # os.environ.clear() # this breaks pytest! + del os.environ["TEST_USER"] assert "TEST_USER" not in os.environ Coordinator.load_test_credentials(DIR) -- 2.47.2 From 63309b67c0cfaeecd2e8ed862780fad3ec91f1bc Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:38:26 +0100 Subject: [PATCH 50/91] test_check_result_with_playwright fully working --- tests/test_html_merge.py | 44 ++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/test_html_merge.py b/tests/test_html_merge.py index 1fab89e..acebfef 100644 --- a/tests/test_html_merge.py +++ b/tests/test_html_merge.py @@ -36,34 +36,30 @@ def test_merge_html(html_file: Path): assert next(html_file.parent.glob("*")) -# @pytest.mark.slow -# def test_check_result_with_playwright(html_file: Path, context: BrowserContext): -# assert html_file.is_file() +@pytest.mark.slow +def test_check_result_with_playwright(html_file: Path, context: BrowserContext): + assert html_file.is_file() -# file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() + file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() -# page = context.new_page() -# page.goto(file_url) + page = context.new_page() + page.goto(file_url) -# # check if combined is correct -# expect(page.get_by_text("2 Passed,")).to_be_visible() -# expect(page.get_by_text("2 Failed,")).to_be_visible() -# expect(page.get_by_text("tests ran in 12.946 seconds")).to_be_visible() + # check if combined is correct + expect(page.get_by_text("2 Passed,")).to_be_visible() + expect(page.get_by_text("2 Failed,")).to_be_visible() + expect(page.get_by_text("tests ran in 12.946 seconds")).to_be_visible() -# # check if heading is correct -# expect(page.get_by_role("heading", name="combined.html")).to_be_visible() + # check if heading is correct + expect(page.get_by_role("heading", name="combined.html")).to_be_visible() -# # check if traceback is included -# expect(page.get_by_text("E AssertionError: One or more")).to_be_visible() + # check if traceback is included + expect(page.get_by_text("E AssertionError: One or more")).to_be_visible() -# # check if asset works -# with page.expect_popup() as page1_info: -# page.get_by_role("link", name="Authentik Blueprint Status").click() -# page1 = page1_info.value + # check if asset works + with page.expect_popup() as page1_info: + page.get_by_role("link", name="Authentik Blueprint Status").click() + page1 = page1_info.value -# # see if content of txt file is correct -# expect(page1.get_by_text("failed")).to_be_visible() - - -def test_demo(context: BrowserContext): - assert True + # see if content of txt file is correct + expect(page1.get_by_text("failed")).to_be_visible() -- 2.47.2 From b62c2bee2529e50966150c48b7ea609a3f0d1b09 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:42:07 +0100 Subject: [PATCH 51/91] add docs for optinal arguments --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 7fb9702..9a5e4eb 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,12 @@ To run successfully, very specific arguments are required. The easiest way to us - `--recipes_dir`: directory of all available abra recipes - `--output_dir`: target directory for all test results +Furtheremore, there are these optional arguments: + +- `--resume`: `abratest` will take the directory in `output_dir` with the most recent creation date and resume the tests there. +- `--debug`: enables playwright debug mode, see docs [here](https://playwright.dev/python/docs/running-tests#debugging-tests) +- `--timeout`: will overwrite the default playwright timeouts in [ms], see docs [here](https://playwright.dev/python/docs/api/class-browsercontext#browser-context-set-default-timeout) and [here](https://playwright.dev/python/docs/test-assertions#global-timeout). In our current setup, some tests can fail at 10s but will pass with 20s. + ### env_paths [string] The variable env_paths consists of one or more paths pointing at .env files. The paths are separated with ";". These .env files are actually configuration files for `abra` recipes, but `pytest-abra` uses the same files for test configuration. -- 2.47.2 From ed403954b6261dd312a492b9eae4c48c116236ec Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:42:27 +0100 Subject: [PATCH 52/91] wip test_pytest_abra --- tests/test_pytest_abra.py | 69 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/test_pytest_abra.py diff --git a/tests/test_pytest_abra.py b/tests/test_pytest_abra.py new file mode 100644 index 0000000..4418064 --- /dev/null +++ b/tests/test_pytest_abra.py @@ -0,0 +1,69 @@ +import json +import os +import subprocess +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session") +def session_tmp_path_testout(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("test_out") + + +@pytest.mark.slow +def test_pytest_abra(session_tmp_path_testout: Path): + """run abratest against the dev instance""" + + # --------------------- load credentials to env variables -------------------- # + + cred_file = Path("credentials.json") + with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + + for key, value in CREDENTIALS.items(): + os.environ[key] = value + + # --------------------------------- env files -------------------------------- # + + ENV_FILES_ROOT = Path("../envfiles").resolve() + ENV_FILES = [ + ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env", # authentik + ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env", # wordpress + ENV_FILES_ROOT / "files.test.dev.local-it.cloud.env", # nextcloud + ] + ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES]) + + # ----------------------------------- dirs ----------------------------------- # + + RECIPES_DIR = Path("../recipes").resolve() + OUTPUT_DIR = Path("./test-output").resolve() + # OUTPUT_DIR = session_tmp_path_testout.resolve() + + # ------------------------------------ run ----------------------------------- # + + result = subprocess.run( + [ + "abratest", + "--env_paths", + ENV_PATHS, + "--recipes_dir", + RECIPES_DIR, + "--output_dir", + OUTPUT_DIR, + ] + ) + + assert result.returncode == 0 + + assert "failed" not in str(result.stdout) + + with open(OUTPUT_DIR / "tempfile.txt", "w") as f: + f.write(str(result.stdout)) + + +# @pytest.mark.slow +# def test_results_abra(session_tmp_path_testout: Path): +# OUTPUT_DIR = Path("./test-output").resolve() +# print(list(OUTPUT_DIR.rglob("*"))) +# assert False -- 2.47.2 From 24b1ca3bf70c444678fbb913b76ffd69f51edd9c Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:42:27 +0100 Subject: [PATCH 53/91] skip test --- .../wordpress/tests_wordpress/test_wordpress_receive_email.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py b/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py index 736f18d..f806d13 100644 --- a/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py +++ b/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py @@ -1,8 +1,10 @@ +import pytest from icecream import ic from pytest_abra.custom_fixtures import Message +@pytest.mark.skip def test_demo(imap_recent_messages: list[Message]): for message in imap_recent_messages: print(dir(message)) -- 2.47.2 From 4882df4d78875c77166f7da4feac7ee8367dbea7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:42:27 +0100 Subject: [PATCH 54/91] assert state_file.is_file() --- recipes/authentik/tests_authentik/fixtures_authentik.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes/authentik/tests_authentik/fixtures_authentik.py b/recipes/authentik/tests_authentik/fixtures_authentik.py index b19b06a..f1c5823 100644 --- a/recipes/authentik/tests_authentik/fixtures_authentik.py +++ b/recipes/authentik/tests_authentik/fixtures_authentik.py @@ -9,6 +9,7 @@ from pytest_abra import BaseUrl, DirManager @pytest.fixture def authentik_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "authentik_admin_state.json" + assert state_file.is_file() storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) return context @@ -26,6 +27,7 @@ def authentik_admin_page(authentik_admin_context: BrowserContext, DIR: DirManage @pytest.fixture def authentik_user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "authentik_user_state.json" + assert state_file.is_file() storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) return context -- 2.47.2 From 44203f3050c5c513b4d9a3cb032e38e6e352a00f Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:42:27 +0100 Subject: [PATCH 55/91] remove print --- .../authentik/tests_authentik/test_authentik_blueprint_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py index 87eb452..cf5914b 100644 --- a/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py +++ b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py @@ -17,7 +17,7 @@ def test_authentik_blueprint_status( blueprints = api_request_context.get(URL.get("api/v3/managed/blueprints")) assert blueprints.ok blueprints_data = blueprints.json() - ic(blueprints_data) + # ic(blueprints_data) # fake failed blueprint # blueprints_data["results"][10]["status"] = "failed" -- 2.47.2 From 8feebb92704f05c032911dc5cd2a5a9a7b2e8211 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:42:27 +0100 Subject: [PATCH 56/91] rename test --- tests/test_pytest_abra.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_pytest_abra.py b/tests/test_pytest_abra.py index 4418064..b87e788 100644 --- a/tests/test_pytest_abra.py +++ b/tests/test_pytest_abra.py @@ -12,7 +12,7 @@ def session_tmp_path_testout(tmp_path_factory: pytest.TempPathFactory) -> Path: @pytest.mark.slow -def test_pytest_abra(session_tmp_path_testout: Path): +def test_abratest_cli_full_integration(session_tmp_path_testout: Path): """run abratest against the dev instance""" # --------------------- load credentials to env variables -------------------- # @@ -51,15 +51,23 @@ def test_pytest_abra(session_tmp_path_testout: Path): RECIPES_DIR, "--output_dir", OUTPUT_DIR, - ] + ], + capture_output=True, ) + with open(OUTPUT_DIR / "tempfile_stdout.txt", "w") as f: + f.write(str(result.stdout)) + + with open(OUTPUT_DIR / "tempfile_stderr.txt", "w") as f: + f.write(str(result.stderr)) + + # print(result) + assert result.returncode == 0 assert "failed" not in str(result.stdout) - with open(OUTPUT_DIR / "tempfile.txt", "w") as f: - f.write(str(result.stdout)) + # assert False # @pytest.mark.slow -- 2.47.2 From aba66158a1eebf99927cba86285c9dab6a388024 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:43:34 +0100 Subject: [PATCH 57/91] use load_json_to_environ --- tests/test_pytest_abra.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test_pytest_abra.py b/tests/test_pytest_abra.py index b87e788..0bc03d8 100644 --- a/tests/test_pytest_abra.py +++ b/tests/test_pytest_abra.py @@ -1,10 +1,10 @@ -import json -import os import subprocess from pathlib import Path import pytest +from pytest_abra.utils import load_json_to_environ + @pytest.fixture(scope="session") def session_tmp_path_testout(tmp_path_factory: pytest.TempPathFactory) -> Path: @@ -18,11 +18,7 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path): # --------------------- load credentials to env variables -------------------- # cred_file = Path("credentials.json") - with open(cred_file, "r") as f: - CREDENTIALS = json.load(f) - - for key, value in CREDENTIALS.items(): - os.environ[key] = value + load_json_to_environ(cred_file) # --------------------------------- env files -------------------------------- # -- 2.47.2 From 766e7909d0cf01a6d40a80821fd99cf396322913 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 15:53:22 +0100 Subject: [PATCH 58/91] test_abratest_cli_full_integration working --- tests/test_pytest_abra.py | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/tests/test_pytest_abra.py b/tests/test_pytest_abra.py index 0bc03d8..baed132 100644 --- a/tests/test_pytest_abra.py +++ b/tests/test_pytest_abra.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest +from pytest_abra import DirManager from pytest_abra.utils import load_json_to_environ @@ -33,8 +34,8 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path): # ----------------------------------- dirs ----------------------------------- # RECIPES_DIR = Path("../recipes").resolve() - OUTPUT_DIR = Path("./test-output").resolve() - # OUTPUT_DIR = session_tmp_path_testout.resolve() + # OUTPUT_DIR = Path("./test-output").resolve() + OUTPUT_DIR = session_tmp_path_testout.resolve() # ------------------------------------ run ----------------------------------- # @@ -47,27 +48,17 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path): RECIPES_DIR, "--output_dir", OUTPUT_DIR, - ], - capture_output=True, + "--session_id", + "abc", + ] ) - with open(OUTPUT_DIR / "tempfile_stdout.txt", "w") as f: - f.write(str(result.stdout)) - with open(OUTPUT_DIR / "tempfile_stderr.txt", "w") as f: - f.write(str(result.stderr)) +@pytest.mark.slow +def test_results_abra(session_tmp_path_testout: Path): + OUTPUT_DIR = session_tmp_path_testout.resolve() - # print(result) - - assert result.returncode == 0 - - assert "failed" not in str(result.stdout) - - # assert False - - -# @pytest.mark.slow -# def test_results_abra(session_tmp_path_testout: Path): -# OUTPUT_DIR = Path("./test-output").resolve() -# print(list(OUTPUT_DIR.rglob("*"))) -# assert False + DIR = DirManager(output_dir=OUTPUT_DIR, session_id="abc") + all_files = list(DIR.STATUS.rglob("*")) + passed_files = list(DIR.STATUS.rglob("passed-*")) + assert len(all_files) == len(passed_files) -- 2.47.2 From d3e25c5052bd717f2767113e30a6833a96dd733e Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 22:44:55 +0100 Subject: [PATCH 59/91] simpolify default path --- pytest_abra/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 894283c..c9eb650 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -31,7 +31,7 @@ class Runner: tests: list[Test] = [] cleanups: list[Test] = [] dependencies: list[str] = [] - _tests_path: Path = Path("undefined") + _tests_path: Path = Path() def __init__(self, coordinator: "Coordinator", runner_index: int): self.coordinator = coordinator -- 2.47.2 From e7910a88a54a14c7ef8dfcfca238b0eccc28744e Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 22:56:05 +0100 Subject: [PATCH 60/91] add assert message --- recipes/authentik/tests_authentik/fixtures_authentik.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/authentik/tests_authentik/fixtures_authentik.py b/recipes/authentik/tests_authentik/fixtures_authentik.py index f1c5823..fdba975 100644 --- a/recipes/authentik/tests_authentik/fixtures_authentik.py +++ b/recipes/authentik/tests_authentik/fixtures_authentik.py @@ -9,7 +9,7 @@ from pytest_abra import BaseUrl, DirManager @pytest.fixture def authentik_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "authentik_admin_state.json" - assert state_file.is_file() + assert state_file.is_file(), "authentik setup did not finish successfully" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) return context @@ -27,7 +27,7 @@ def authentik_admin_page(authentik_admin_context: BrowserContext, DIR: DirManage @pytest.fixture def authentik_user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "authentik_user_state.json" - assert state_file.is_file() + assert state_file.is_file(), "authentik setup did not finish successfully" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) return context -- 2.47.2 From 1160f769fc8a2967350bb321fa4ccf59cec86ba2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 11 Dec 2023 23:11:27 +0100 Subject: [PATCH 61/91] add assert message --- pytest_abra/custom_fixtures.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index 972b07e..5aa86d6 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -97,10 +97,10 @@ def URL(env_config: dict[str, str]) -> BaseUrl: def imap_client() -> Generator[Imbox, None, None]: """imap email client using credentials from environment variables""" - assert os.environ["IMAP_HOST"] - assert os.environ["IMAP_PORT"] - assert os.environ["IMAP_USER"] - assert os.environ["IMAP_PASS"] + assert os.environ["IMAP_HOST"], "required environment variable is undefined" + assert os.environ["IMAP_PORT"], "required environment variable is undefined" + assert os.environ["IMAP_USER"], "required environment variable is undefined" + assert os.environ["IMAP_PASS"], "required environment variable is undefined" imbox = Imbox( hostname=os.environ["IMAP_HOST"], -- 2.47.2 From b39802094bbbd3bd56d2a59be58980a6d5a4c9e6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 12 Dec 2023 11:09:29 +0100 Subject: [PATCH 62/91] remove pre-commit --- .pre-commit-config.yaml | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index fabed9e..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -repos: - # - repo: local - # hooks: - # - id: tests - # name: run all tests that are not marked slow - # entry: python -m pytest -m "not slow" - # language: system - # language_version: default - # always_run: true - # pass_filenames: false - - - repo: https://github.com/pre-commit/mirrors-mypy - rev: '' # Use the sha / tag you want to point at - hooks: - - id: mypy - args: [--ignore-missing-imports] - - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.7 - hooks: - # Run the linter. - - id: ruff - # Run the formatter. - - id: ruff-format \ No newline at end of file -- 2.47.2 From 7620f1d4b75045307c84f537e4c3ad449726f2f4 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 12 Dec 2023 11:09:39 +0100 Subject: [PATCH 63/91] rename table --- pytest_abra/coordinator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 4e2b371..fbbe9cf 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -51,8 +51,8 @@ class Coordinator: status_list.extend(runner.run_tests()) for runner in self.runners: status_list.extend(runner.run_cleanups()) - result_table = tabulate([[t.test_name, t.status] for t in status_list], headers=["name", "status"]) - logger.info(f"run_tests() finished\n{result_table}") + status_table = tabulate([[t.test_name, t.status] for t in status_list], headers=["name", "status"]) + logger.info(f"run_tests() finished\n{status_table}") def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]: """Creates an instance of the correct Runner class for each given env file""" -- 2.47.2 From 231f9d24e6ae2fb19f8b65d425c7a4112466b4d3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 12 Dec 2023 11:12:02 +0100 Subject: [PATCH 64/91] pin all versions --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2fd610a..8625d38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,14 +21,13 @@ dependencies = [ "loguru == 0.7.2", "beautifulsoup4 == 4.12.2", "imbox == 0.9.8", - "tabulate", + "tabulate == 0.9.0", "hatchling == 1.18.0", - "icecream", + "icecream == 2.1.3", ] [project.optional-dependencies] dev = [ - "pre-commit", "mypy", "ruff", ] -- 2.47.2 From 5d6696cf1952d02ee4a0a1a172b06ef9db0dd34a Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 12 Dec 2023 11:54:03 +0100 Subject: [PATCH 65/91] add --version --- pytest_abra/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 0f21f30..dfcf90a 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -2,6 +2,7 @@ import argparse import os from pathlib import Path +import pkg_resources # type: ignore from loguru import logger from pytest_abra import Coordinator @@ -9,8 +10,13 @@ from pytest_abra.dir_manager import DirManager from pytest_abra.utils import get_session_id +def get_version(): + return pkg_resources.get_distribution("pytest_abra").version + + def run(): parser = argparse.ArgumentParser() + parser.add_argument("--version", "-V", action="version", version=get_version(), help="output the version number") parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;", required=True) parser.add_argument("--recipes_dir", type=Path, help="List of loaded env files separated with ;", required=True) parser.add_argument("--output_dir", type=Path, help="List of loaded env files separated with ;", required=True) -- 2.47.2 From 622b2a1f8e3678836555a240185a423d7fda03f1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 12 Dec 2023 15:38:28 +0100 Subject: [PATCH 66/91] improve docs --- README.md | 160 +++++++++------------------------------ docs/documentation.md | 170 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 201 insertions(+), 129 deletions(-) diff --git a/README.md b/README.md index 9a5e4eb..2a1b8f0 100644 --- a/README.md +++ b/README.md @@ -1,96 +1,18 @@ # pytest-abra -Pytest-Abra is an installable python package design to test instances created with [abra](https://docs.coopcloud.tech/abra/). After installation, you will have two things: +Pytest-Abra is an installable python package baed on pytest, designed to test instances created with [abra](https://docs.coopcloud.tech/abra/). After installation, you will have two things: - `abratest` CLI command. *Used to initialize the testing.* -- `pytest-abra` Pytest plugin. *Automatically loads custom fixtures in any pytest (see `pytest_abra/custom_fixtures.py`)* - -## CLI (`abratest`) - -`abratest` can be called via terminal: - -```bash -abratest [arguments] -``` - -To run successfully, very specific arguments are required. The easiest way to use abratest is with the helper script in `main.py`. Of yourse you can implement a similar helper script in the language of your liking. The cli command `abratest` has 3 **required arguments**: - -- `--env_paths`: list of the .env files used in the test -- `--recipes_dir`: directory of all available abra recipes -- `--output_dir`: target directory for all test results - -Furtheremore, there are these optional arguments: - -- `--resume`: `abratest` will take the directory in `output_dir` with the most recent creation date and resume the tests there. -- `--debug`: enables playwright debug mode, see docs [here](https://playwright.dev/python/docs/running-tests#debugging-tests) -- `--timeout`: will overwrite the default playwright timeouts in [ms], see docs [here](https://playwright.dev/python/docs/api/class-browsercontext#browser-context-set-default-timeout) and [here](https://playwright.dev/python/docs/test-assertions#global-timeout). In our current setup, some tests can fail at 10s but will pass with 20s. - -### env_paths [string] - -The variable env_paths consists of one or more paths pointing at .env files. The paths are separated with ";". These .env files are actually configuration files for `abra` recipes, but `pytest-abra` uses the same files for test configuration. - -To run `abratest` with these `.env` configuration files - -``` -/path/to/config_1.env -/path/to/config_2.env -/path/to/config_3.env -``` - -we simply call - -``` -abratest --env_paths /path/to/config_1.env;/path/to/config_2.env;/path/to/config_3.env -``` - -Under the hood, each `.env` file in `--env_paths` will create one instance of a `Runner` subclass. Let's say we have `wordpress_configuration.env` containing `TYPE=wordpress`. This will create an instance of `RunnerWordpress`. This class has to be imported from `recipes_dir`. - -### recipes_dir [string] - -The required argument `--recipes_dir` has to point to the directory, where all the abra recipes are stored. We can call `abratest` with - -``` -abratest --recipes_dir /path/to/abra/recipes -``` - -The expected dir structure inside of `recipes_dir` is as follows: - -``` -DIR recipes_dir [contains abra recipes] -│ -├── DIR authentik [authentik recipe] -│ ├── [files from authentik recipe] -│ └── DIR tests_authentik [pytest tests for authentik] -│ ├── FILE runner_authentik.py # containing RunnerAuthentik class -│ └── [pytest_files] -│ -└── DIR wordpress [wordpress recipe] - ├── [files from wordpress recipe] - └── DIR tests_wordpress [pytest tests for wordpress] - ├── FILE runner_wordpress.py # containing RunnerWordpress class - └── [pytest_files] -``` - -The class `RunnerWordpress` will be automatically imported using `importlib` library, which is equivalent to the code below. Note that `recipes_dir` will be added to sys.path automatically for the import to work and that every `Runner` class matching `recipes_dir.rglob("*/runner*.py")` will be imported. - -```python -from wordpress.tests_wordpress.runner_wordpress import RunnerWordpress -``` - -### output_dir [string] - -Path to the directory where all test outputs are stored (test report, tracebacks, playwright traces etc.) - -``` -abratest --output_dir /path/to/output -``` +- `pytest-abra` Pytest plugin. *Automatically loads custom fixtures in any pytest run (see `pytest_abra/custom_fixtures.py`)* # Usage -To use pytest-abra, follow these steps: +Pytest-abra can easily be installed on any system but also offers a Docker image. To use pytest-abra, follow these steps: -## 1. GIT clone [with & without Docker] +## Usage [without Docker] + +### Installation [without Docker] To clone with submodules, use these git commands: @@ -101,14 +23,6 @@ git submodule update --init // add submodule after normal cloning git submodule update --remote // update submodules ``` -## Run - -You can run pytest-abra with and without Docker. Choose now and follow the steps accordingly: - -## 2.1 Run without Docker - -### Installation - Create a python environment and install all dependencies via ```bash @@ -116,46 +30,40 @@ pip install -e . playwright install ``` -Run the script with +### Run [without Docker] + +Run the helper script or directly use the cli command (see docs) ```bash -python main.py +python main.py # run pytest-abra +abratest [options] ``` -## 2.2 Run with Docker +## Usage [with docker] + +### Installation [with docker] + +To clone with submodules, use these git commands: + +```bash +git clone --recurse-submodules +// optional: +git submodule update --init // add submodule after normal cloning +git submodule update --remote // update submodules +``` + +Build the image ```bash docker compose build # build the image +docker compose build --no-cache # Force rebuild without cache +``` + +### Run [with docker] + +Run the script + +```bash docker compose run --rm app python main.py # run pytest-abra -docker compose run --rm -it app /bin/bash # debug the container -``` - -Force rebuild with cache - -```bash -docker-compose up --build -``` - -Force rebuild without cache - -```bash -docker-compose build --no-cache -``` - -## Playwright Debug & Codegen - -Use playwright debug mode or codegen to create testing code easily by recording browser actions https://playwright.dev/python/docs/codegen - -```bash -abratest --debug # launch your tests in debug mode -playwright codegen demo.playwright.dev/todomvc # visit given url in codegen mode -``` - -## Development - -```bash -pytest # test pytest-abra -pytest -m "not slow" # test pytest-abra without slow tests -pytest --collect-only # debug test pytest-abra -docker compose run --rm app pytest # run pytest-abra +docker compose run --rm -it app /bin/bash # use the container interactively ``` diff --git a/docs/documentation.md b/docs/documentation.md index 013e57e..8b3d30b 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -1,15 +1,161 @@ -# RUN +# pytest-abra +Pytest-Abra is an installable python package baed on pytest, designed to test instances created with [abra](https://docs.coopcloud.tech/abra/). After installation, you will have two things: +- `abratest` CLI command. *Used to initialize the testing.* +- `pytest-abra` Pytest plugin. *Automatically loads custom fixtures in any pytest run (see `pytest_abra/custom_fixtures.py`)* +# Getting Started +Pytest-abra can easily be installed on any system but also offers a Docker image. To use pytest-abra, follow these steps: +## Usage [without Docker] +### Installation [without Docker] + +To clone with submodules, use these git commands: + +```bash +git clone --recurse-submodules +// optional: +git submodule update --init // add submodule after normal cloning +git submodule update --remote // update submodules +``` + +Create a python environment and install all dependencies via + +```bash +pip install -e . +playwright install +``` + +### Run [without Docker] + +Run the helper script or directly use the cli command (see docs) + +```bash +python main.py # run pytest-abra +abratest [options] +``` + +## Usage [with docker] + +### Installation [with docker] + +To clone with submodules, use these git commands: + +```bash +git clone --recurse-submodules +// optional: +git submodule update --init // add submodule after normal cloning +git submodule update --remote // update submodules +``` + +Build the image + +```bash +docker compose build # build the image +docker compose build --no-cache # Force rebuild without cache +``` + +### Run [with docker] + +Run the script + +```bash +docker compose run --rm app python main.py # run pytest-abra +docker compose run --rm -it app /bin/bash # use the container interactively +``` + +# Documentation + +After Installation, `abratest` can be called via terminal: + +```bash +abratest [arguments] +``` + +To run successfully, very specific arguments are required. The easiest way to use `abratest` is with the helper script `main.py`. Of yourse you can implement a similar helper script in the language of your liking. + +## CLI Interface + +The cli command `abratest` has 3 **required arguments**: + +- `--env_paths ENV_PATHS`: list of the .env files used in the test +- `--recipes_dir RECIPES_DIR`: directory of all available abra recipes +- `--output_dir OUTPUT_DIR`: target directory for all test results + +Furtheremore, there are these optional arguments: + +- `--resume`: `abratest` will take the directory in `output_dir` with the most recent creation date and resume the tests there. +- `--session_id SESSION_ID`: Instead of generating a new session_id, the given session_id is used to run or resume the test. Overwrites --resume to False. +- `--debug`: enables playwright debug mode, see docs [here](https://playwright.dev/python/docs/running-tests#debugging-tests) +- `--timeout`: will overwrite the default playwright timeouts in [ms], see docs [here](https://playwright.dev/python/docs/api/class-browsercontext#browser-context-set-default-timeout) and [here](https://playwright.dev/python/docs/test-assertions#global-timeout). In our current setup, some tests can fail at 10s but will pass with 20s. + +### env_paths [required | string] + +The .env files provied through the `--env_paths` argument are the most important input to abratest, as they serve as configuration for the tests. One or more paths pointing at .env files can be provided, multiple paths are separated with ";". These .env files are actually the same files that are used to configure the `abra` recipes for instance creation. + +To run `abratest` with these `.env` configuration files + +- `/path/config_1.env` [of TYPE authentik] +- `/path/config_2.env` [of TYPE wordpress] +- `/path/config_3.env` [of TYPE wordpress] + +we simply call + +``` +abratest --env_paths /path/config_1.env;/path/config_2.env;/path/config_3.env [...other args] +``` + +Under the hood, each `.env` file in `--env_paths` will create one instance of a `Runner` subclass. Let's say we have `config_2.env` containing `TYPE=wordpress`. This will create an instance of `RunnerWordpress`. This class has to be imported from `recipes_dir`. + +### recipes_dir [required | string] + +The required argument `--recipes_dir` has to point to the directory, where all the abra recipes are stored. We can call `abratest` with + +``` +abratest --recipes_dir /path/to/abra/recipes +``` + +The expected dir structure inside of `recipes_dir` is as follows: + +``` +DIR recipes_dir [contains abra recipes] +│ +├── DIR authentik [authentik recipe] +│ ├── [files from authentik recipe] +│ └── DIR tests_authentik [pytest tests for authentik] +│ ├── FILE runner_authentik.py # containing RunnerAuthentik class +│ └── [pytest_files] +│ +└── DIR wordpress [wordpress recipe] + ├── [files from wordpress recipe] + └── DIR tests_wordpress [pytest tests for wordpress] + ├── FILE runner_wordpress.py # containing RunnerWordpress class + └── [pytest_files] +``` + +The class `RunnerWordpress` will be automatically imported using `importlib` library, which is equivalent to the code below. Note that `recipes_dir` will be added to sys.path automatically for the import to work and that every `Runner` class matching `recipes_dir.rglob("*/runner*.py")` will be imported. + +```python +from wordpress.tests_wordpress.runner_wordpress import RunnerWordpress +``` + +### output_dir [required | string] + +Path to the directory where all test outputs are stored (test report, tracebacks, playwright traces etc.) + +``` +abratest --output_dir /path/to/output +``` + +# Functionality Abratest has 3 required inputs, but most importantly the test configuration is done through the .env files given with the --env_paths argument. So let's say we want to run abratest with these 3 .env files: -- config1.env [of TYPE authentik] +- `config1.env` [of TYPE authentik] - config2.env [of TYPE wordpress] @@ -116,4 +262,22 @@ DIR parent_dir The test files are written in the same way as any other pytest test file. The only difference is that pytest-abra provides custom fixtures that make it easy to get the configuration by the provided .env files and to deal with URLS etc. -# todo: add example \ No newline at end of file +# todo: add example + +# Playwright Debug & Codegen + +Use playwright debug mode or codegen to create testing code easily by recording browser actions https://playwright.dev/python/docs/codegen + +```bash +abratest --debug # launch your tests in debug mode +playwright codegen demo.playwright.dev/todomvc # visit given url in codegen mode +``` + +## Development + +```bash +pytest # test pytest-abra +pytest -m "not slow" # test pytest-abra without slow tests +pytest --collect-only # debug test pytest-abra +docker compose run --rm app pytest # run pytest-abra +``` -- 2.47.2 From 4fdca7924784fc1780764f269b78a6cd5afc461e Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 13 Dec 2023 00:16:50 +0100 Subject: [PATCH 67/91] add docs for # Create custom Tests --- docs/documentation.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/documentation.md b/docs/documentation.md index 8b3d30b..560ce6e 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -262,6 +262,42 @@ DIR parent_dir The test files are written in the same way as any other pytest test file. The only difference is that pytest-abra provides custom fixtures that make it easy to get the configuration by the provided .env files and to deal with URLS etc. + +### Step 1) Add new Test + +Create a new testfile `new_test.py` in the same directory or a subdirectory of `runner_wordpress.py`. +Register `new_test.py` as a test in the `RunnerWordpress` class. +Set prevent_skip=True, so that you can run your new test over and over again for debugging, without it being skipped + +```python +# runner_wordpress.py +from pytest_abra import Runner, Test + +class RunnerWordpress(Runner): + env_type = "wordpress" + tests = [ + Test(test_file="working_test.py"), + Test(test_file="new_test.py", prevent_skip=True), + ] +``` + +```python +# new_test.py + +def test_new(): + ... +``` + +### Step 2) Call abratest + +Call abratest with `--debug` to enable playwright debug mode and either `--session_id` or `--resume`. + +```bash +abratest [required-options] --debug --session_id debug_session +``` + +This could be done by modifying `main.py`. The first time you run abratest, all tests will be executed as usual. The second time, all tests will be skipped as they have passed already. Only your new test will be run again and again, as the prevent_skip option is enabled. So you can run all tests once and then skip all tests besides your new test you want to debug. + # todo: add example # Playwright Debug & Codegen -- 2.47.2 From 3d8c72e3731b63afa358630cfb323f8c12e50962 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 13 Dec 2023 00:17:29 +0100 Subject: [PATCH 68/91] add test for Coordinator.create_runner_dict --- tests/test_coordinator.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py index 7758cae..ac44900 100644 --- a/tests/test_coordinator.py +++ b/tests/test_coordinator.py @@ -1,6 +1,11 @@ import os +import shutil +import sys +from collections import UserList from pathlib import Path +import pytest + from pytest_abra.coordinator import Coordinator from pytest_abra.dir_manager import DirManager @@ -23,3 +28,35 @@ def test_load_test_credentials(tmp_path: Path): Coordinator.load_test_credentials(DIR) assert test_user_before == os.environ["TEST_USER"] + + +@pytest.fixture(scope="session") +def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path: + tmp_recipes_target = tmp_path_factory.mktemp("recipes") + recipes_dir_source = Path("recipes") + shutil.copytree(recipes_dir_source, tmp_recipes_target, dirs_exist_ok=True) + return tmp_recipes_target + + +def test_runner_runner_dict_import(tmp_recipes: Path): + """import from recipes dict should work, because create_runner_dict has sys.path.append""" + + RUNNER_DICT = Coordinator.create_runner_dict(tmp_recipes) + assert len(RUNNER_DICT.keys()) > 0 + + +# def test_runner_runner_dict_import_patched(tmp_recipes: Path, monkeypatch): +# """import from recipes dict should fail without sys.path.append""" + +# class MockPath(UserList): +# def append(self, item): +# pass + +# monkeypatch.setattr(sys, "path", MockPath()) + +# with pytest.raises(ModuleNotFoundError): +# Coordinator.create_runner_dict(tmp_recipes) + + +# def test_something(): +# import authentik -- 2.47.2 From 25c1f6fd50a02146333546fbfac7ab2badfc8d91 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 13 Dec 2023 00:36:09 +0100 Subject: [PATCH 69/91] remove ic --- tests/test_html_merge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_html_merge.py b/tests/test_html_merge.py index acebfef..a216dfa 100644 --- a/tests/test_html_merge.py +++ b/tests/test_html_merge.py @@ -22,7 +22,6 @@ def html_file(session_tmp_path: Path) -> Path: in_dir_path = Path(__file__).parent / "assets" / "html_merge" in_dir_path = in_dir_path.resolve() - ic(in_dir_path) html_file = session_tmp_path / "test.html" -- 2.47.2 From d9b65c6a6f960d05b33065950305b5341d5ac6c3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 13 Dec 2023 00:43:11 +0100 Subject: [PATCH 70/91] cleanup --- tests/test_coordinator.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py index ac44900..3c00ed6 100644 --- a/tests/test_coordinator.py +++ b/tests/test_coordinator.py @@ -1,7 +1,5 @@ import os import shutil -import sys -from collections import UserList from pathlib import Path import pytest @@ -43,20 +41,3 @@ def test_runner_runner_dict_import(tmp_recipes: Path): RUNNER_DICT = Coordinator.create_runner_dict(tmp_recipes) assert len(RUNNER_DICT.keys()) > 0 - - -# def test_runner_runner_dict_import_patched(tmp_recipes: Path, monkeypatch): -# """import from recipes dict should fail without sys.path.append""" - -# class MockPath(UserList): -# def append(self, item): -# pass - -# monkeypatch.setattr(sys, "path", MockPath()) - -# with pytest.raises(ModuleNotFoundError): -# Coordinator.create_runner_dict(tmp_recipes) - - -# def test_something(): -# import authentik -- 2.47.2 From 78fafc1e5ccadb071cd99b47009400c76c2845b1 Mon Sep 17 00:00:00 2001 From: Daniel Brummerloh Date: Wed, 13 Dec 2023 17:00:34 +0100 Subject: [PATCH 71/91] fix paths --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 9a37f57..f03903c 100644 --- a/main.py +++ b/main.py @@ -14,7 +14,7 @@ load_json_to_environ(cred_file) # triggers the execution of one test Runner and provides configuration to the # tests inside the runner. -ENV_FILES_ROOT = Path("../envfiles").resolve() +ENV_FILES_ROOT = Path("./envfiles").resolve() ENV_FILES = [ ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env", # authentik ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env", # wordpress @@ -24,7 +24,7 @@ ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES]) # ----------------------------------- dirs ----------------------------------- # -RECIPES_DIR = Path("../recipes").resolve() +RECIPES_DIR = Path("./recipes").resolve() OUTPUT_DIR = Path("./test-output").resolve() # ------------------------------------ run ----------------------------------- # -- 2.47.2 From 0d6bd2d0f87290aa8da20d388994b82354a68383 Mon Sep 17 00:00:00 2001 From: Daniel Brummerloh Date: Wed, 13 Dec 2023 17:00:56 +0100 Subject: [PATCH 72/91] move fixtures --- pytest_abra/custom_fixtures.py | 13 +------------ recipes/authentik/tests_authentik/conftest.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 12 deletions(-) create mode 100644 recipes/authentik/tests_authentik/conftest.py diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index 5aa86d6..0f0a5f0 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -12,7 +12,7 @@ import pytest from dotenv import dotenv_values from icecream import ic # type: ignore from imbox import Imbox # type: ignore -from playwright.sync_api import APIRequestContext, BrowserContext, Playwright, expect +from playwright.sync_api import BrowserContext, expect from pytest import Parser from pytest_abra import BaseUrl, DirManager, EnvFile @@ -150,14 +150,3 @@ def imap_recent_messages(imap_client: Imbox) -> list[Message]: messages.append(message) return messages - - -@pytest.fixture(scope="session") -def api_request_context( - playwright: Playwright, - DIR: DirManager, -) -> Generator[APIRequestContext, None, None]: - state_file = DIR.STATES / "authentik_admin_state.json" - request_context = playwright.request.new_context(storage_state=state_file) - yield request_context - request_context.dispose() diff --git a/recipes/authentik/tests_authentik/conftest.py b/recipes/authentik/tests_authentik/conftest.py new file mode 100644 index 0000000..6542c21 --- /dev/null +++ b/recipes/authentik/tests_authentik/conftest.py @@ -0,0 +1,17 @@ +from typing import Generator + +import pytest +from playwright.sync_api import APIRequestContext, Playwright + +from pytest_abra import DirManager + + +@pytest.fixture(scope="session") +def api_request_context( + playwright: Playwright, + DIR: DirManager, +) -> Generator[APIRequestContext, None, None]: + state_file = DIR.STATES / "authentik_admin_state.json" + request_context = playwright.request.new_context(storage_state=state_file) + yield request_context + request_context.dispose() -- 2.47.2 From 18d9782a8a4e7ab134090a03f2667c8945379c15 Mon Sep 17 00:00:00 2001 From: Daniel Brummerloh Date: Wed, 13 Dec 2023 17:01:25 +0100 Subject: [PATCH 73/91] fix paths and assert return code --- tests/test_pytest_abra.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_pytest_abra.py b/tests/test_pytest_abra.py index baed132..805ce4c 100644 --- a/tests/test_pytest_abra.py +++ b/tests/test_pytest_abra.py @@ -23,7 +23,7 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path): # --------------------------------- env files -------------------------------- # - ENV_FILES_ROOT = Path("../envfiles").resolve() + ENV_FILES_ROOT = Path("./envfiles").resolve() ENV_FILES = [ ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env", # authentik ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env", # wordpress @@ -33,7 +33,7 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path): # ----------------------------------- dirs ----------------------------------- # - RECIPES_DIR = Path("../recipes").resolve() + RECIPES_DIR = Path("./recipes").resolve() # OUTPUT_DIR = Path("./test-output").resolve() OUTPUT_DIR = session_tmp_path_testout.resolve() @@ -53,6 +53,8 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path): ] ) + assert result.returncode == 0 + @pytest.mark.slow def test_results_abra(session_tmp_path_testout: Path): @@ -61,4 +63,5 @@ def test_results_abra(session_tmp_path_testout: Path): DIR = DirManager(output_dir=OUTPUT_DIR, session_id="abc") all_files = list(DIR.STATUS.rglob("*")) passed_files = list(DIR.STATUS.rglob("passed-*")) + assert len(all_files) > 0 assert len(all_files) == len(passed_files) -- 2.47.2 From 1c2a957f993c4f85b0d9ed93b03e0c6480c124c2 Mon Sep 17 00:00:00 2001 From: Daniel Brummerloh Date: Wed, 13 Dec 2023 17:01:50 +0100 Subject: [PATCH 74/91] fix bug in get_latest_session_id when dir didnt exist --- pytest_abra/dir_manager.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pytest_abra/dir_manager.py b/pytest_abra/dir_manager.py index c20f03c..b5ca29e 100644 --- a/pytest_abra/dir_manager.py +++ b/pytest_abra/dir_manager.py @@ -80,7 +80,13 @@ class DirManager: @staticmethod def get_latest_session_id(output_dir: Path) -> Optional[str]: - """returns the name of the newest dir inside of output_dir""" + """returns the name of the newest dir inside of output_dir + + if output_dir does not exists or is empty, None is returned""" + + if not output_dir.is_dir(): + return None + all_dirs = [d for d in output_dir.iterdir() if d.is_dir()] if all_dirs: newest_dir: Path = max(all_dirs, key=lambda x: x.stat().st_ctime) -- 2.47.2 From aa541de52fc26ea1cd57b0972f29591ba3b89992 Mon Sep 17 00:00:00 2001 From: Daniel Brummerloh Date: Wed, 13 Dec 2023 17:01:56 +0100 Subject: [PATCH 75/91] add tests for dirmanager --- tests/test_dir_manager.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/test_dir_manager.py diff --git a/tests/test_dir_manager.py b/tests/test_dir_manager.py new file mode 100644 index 0000000..8dd78b1 --- /dev/null +++ b/tests/test_dir_manager.py @@ -0,0 +1,30 @@ + +import time +import pytest +from pytest_abra.dir_manager import DirManager +from pathlib import Path + +def test_get_latest_session_id_from_non_existing_dir(tmp_path: Path): + out = DirManager.get_latest_session_id(tmp_path / "not_exist") + assert out is None + +def test_get_latest_session_id_from_empty_dir(tmp_path: Path): + out = DirManager.get_latest_session_id(tmp_path) + assert out is None + +def test_get_latest_session_id_single(tmp_path: Path): + (tmp_path / "a").mkdir() + out = DirManager.get_latest_session_id(tmp_path) + assert out == "a" + + + +@pytest.mark.slow +def test_get_latest_session_id(tmp_path: Path): + (tmp_path / "a").mkdir() + time.sleep(1.1) + (tmp_path / "b").mkdir() + out = DirManager.get_latest_session_id(tmp_path) + assert out == "b" + + \ No newline at end of file -- 2.47.2 From 88d466c7451601dac07fb006da0e3e749aae8aa2 Mon Sep 17 00:00:00 2001 From: Daniel Brummerloh Date: Wed, 13 Dec 2023 17:40:49 +0100 Subject: [PATCH 76/91] fix escape chars in regex pattern --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 633bc4a..8a94ee6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,7 +13,7 @@ def test_get_session_id_random(tmp_path: Path): args_resume = False args_session_id = None session_id = get_session_id(args_output_dir, args_resume, args_session_id) - assert re.search("\d+-\d+-\d+", session_id) + assert re.search(r"\d+-\d+-\d+", session_id) def test_get_session_id_explicit1(tmp_path: Path): -- 2.47.2 From 290b3f879a0e453392c66e5fe9e931a1973322c3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 10:44:08 +0100 Subject: [PATCH 77/91] add missing key assertion in _get_dependency_rules and add test case --- pytest_abra/env_manager.py | 1 + tests/test_env_resolution.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/pytest_abra/env_manager.py b/pytest_abra/env_manager.py index ab3754a..8664624 100644 --- a/pytest_abra/env_manager.py +++ b/pytest_abra/env_manager.py @@ -44,6 +44,7 @@ class EnvManager: def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, type["Runner"]]) -> list[DependencyRule]: dependency_rules: list[DependencyRule] = [] for env_file in env_files: + assert env_file.env_type in RUNNER_DICT, f"no runner for env_type={env_file.env_type} found in RUNNER_DICT" child_runner_class = RUNNER_DICT[env_file.env_type] for dependency in child_runner_class.dependencies: dependency_rule = DependencyRule(child=child_runner_class.env_type, dependency=dependency) diff --git a/tests/test_env_resolution.py b/tests/test_env_resolution.py index ddb5fc4..bbdf8f0 100644 --- a/tests/test_env_resolution.py +++ b/tests/test_env_resolution.py @@ -102,3 +102,17 @@ def test_env_manager() -> None: assert ENV.env_files[0].env_type == "authentik" assert ENV.env_files[1].env_type == "authentik" assert ENV.env_files[2].env_type == "wordpress" + + +def test_RUNNER_DICT_missing_key() -> None: + """RUNNER_DICT missing wordpress key while .env file with TYPE=wordpress given""" + env_paths_list = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + ] + RUNNER_DICT_COPY = RUNNER_DICT.copy() + del RUNNER_DICT_COPY["wordpress"] + with pytest.raises(AssertionError) as excinfo: + EnvManager(env_paths_list, RUNNER_DICT_COPY) + assert "no runner for" in str(excinfo.value) -- 2.47.2 From 5102cf2d9113ea42784638f3a5b2265400010ce1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 11:09:49 +0100 Subject: [PATCH 78/91] add files_are_same --- pytest_abra/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index 95880a6..0dfdb5d 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -79,3 +79,8 @@ def get_session_id(args_output_dir: Path, args_resume: bool, args_session_id: Op if latest_session_id: session_id = latest_session_id return session_id + + +def files_are_same(file1: Path, file2: Path) -> bool: + with open(file1, "r") as f1, open(file2, "r") as f2: + return f1.read() == f2.read() -- 2.47.2 From 8e926d4e64972fb63d3e8a0ec51fc10fa58865f6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 11:38:55 +0100 Subject: [PATCH 79/91] add file change check to copy_env_files, add test cases for copy_env_files --- pytest_abra/coordinator.py | 2 +- pytest_abra/env_manager.py | 26 +++++-- tests/test_env_manager.py | 137 +++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 tests/test_env_manager.py diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index fbbe9cf..6704360 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -38,7 +38,7 @@ class Coordinator: def prepare_tests(self) -> None: logger.info("calling prepare_tests()") self.DIR.create_all_dirs() - self.ENV.copy_env_files(self.DIR) + self.ENV.copy_env_files(self.ENV.env_files, self.DIR) self.load_test_credentials(self.DIR) def run_tests(self) -> None: diff --git a/pytest_abra/env_manager.py b/pytest_abra/env_manager.py index 8664624..7076268 100644 --- a/pytest_abra/env_manager.py +++ b/pytest_abra/env_manager.py @@ -4,6 +4,8 @@ from typing import TYPE_CHECKING, NamedTuple from dotenv import dotenv_values +from pytest_abra.utils import files_are_same + if TYPE_CHECKING: from pytest_abra import DirManager, Runner @@ -93,11 +95,25 @@ class EnvManager: "Could not resolve test order. This is possibly due to a circular dependency (a on b, b on c, c on a)" ) - def copy_env_files(self, DIR: "DirManager") -> None: - """Copies all env files to STATES/env_files. Files will be renamed to - -- - 00-authentik-login.test.dev.local-it.cloud.env""" + @staticmethod + def copy_env_files(env_files: list[EnvFile], DIR: "DirManager") -> None: + """Copies all env files to STATES/env_files. - for index, env_file in enumerate(self.env_files): + Files will be renamed to --. Example: + 00-authentik-login.test.dev.local-it.cloud.env + + Does nothing when called twice with same env_files. Throws an AssertionError if either + contents or filenames of env_files have changed (probably test rerun with different input)""" + + dir_was_not_empty = len(list(DIR.ENV_FILES.iterdir())) > 0 + + for index, env_file in enumerate(env_files): file_name = "-".join([str(index).zfill(2), env_file.env_type, env_file.env_path.name]) + if dir_was_not_empty: + # check that the copied env files have not changed + present_files = [f.name for f in DIR.ENV_FILES.iterdir()] + assert ( + file_name in present_files and files_are_same(env_file.env_path, DIR.ENV_FILES / file_name) + ), "It appears that you are resuming a test while the input env files have changed. Start a new test instead" + shutil.copy(env_file.env_path, DIR.ENV_FILES / file_name) diff --git a/tests/test_env_manager.py b/tests/test_env_manager.py new file mode 100644 index 0000000..05255f4 --- /dev/null +++ b/tests/test_env_manager.py @@ -0,0 +1,137 @@ +import shutil +from pathlib import Path + +import pytest + +from pytest_abra.dir_manager import DirManager +from pytest_abra.env_manager import EnvManager +from pytest_abra.utils import files_are_same + +ENV_PATHS = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik +] + + +@pytest.fixture +def tmp_output(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("output") + + +@pytest.fixture +def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("recipes") + + +def test_copy_env_files(tmp_output: Path, tmp_recipes: Path): + # create dirs in output + DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes) + DIR.create_all_dirs() + + # confirm dir is empty + assert len(list(DIR.ENV_FILES.iterdir())) == 0 + + # copy env files + env_files = EnvManager._get_env_files(ENV_PATHS) + EnvManager.copy_env_files(env_files, DIR) + + # check that each env file is present in DIR.ENV_FILES with correct contents + assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files) + for index, env_path in enumerate(ENV_PATHS): + matching_files = [f for f in DIR.ENV_FILES.iterdir() if index == int(f.name.split("-")[0])] + assert len(matching_files) == 1 + assert files_are_same(env_path, matching_files[0]) + + +def test_copy_env_files_twice(tmp_output: Path, tmp_recipes: Path): + """Copy the same env files twice""" + # create dirs in output + DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes) + DIR.create_all_dirs() + + # confirm dir is empty + assert len(list(DIR.ENV_FILES.iterdir())) == 0 + + # copy env files + env_files = EnvManager._get_env_files(ENV_PATHS) + EnvManager.copy_env_files(env_files, DIR) + + # check that each env file is present in DIR.ENV_FILES with correct contents + assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files) + + # copy env files again + EnvManager.copy_env_files(env_files, DIR) + + for index, env_path in enumerate(ENV_PATHS): + matching_files = [f for f in DIR.ENV_FILES.iterdir() if index == int(f.name.split("-")[0])] + assert len(matching_files) == 1 + assert files_are_same(env_path, matching_files[0]) + + +def test_copy_env_files_twice_with_content_change(tmp_output: Path, tmp_recipes: Path, tmp_path: Path): + # copy env files to tmp_path + assert len(list(tmp_path.iterdir())) == 0 + for f in ENV_PATHS: + shutil.copy(f, tmp_path / f.name) + ENV_PATHS_NEW = list(tmp_path.iterdir()) + assert len(ENV_PATHS_NEW) > 0 + + # create dirs in output + DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes) + DIR.create_all_dirs() + + # confirm dir is empty + assert len(list(DIR.ENV_FILES.iterdir())) == 0 + + # copy env files from tmp_path to tmp_output + env_files = EnvManager._get_env_files(ENV_PATHS_NEW) + EnvManager.copy_env_files(env_files, DIR) + + # check that each env file is present in DIR.ENV_FILES with correct contents + assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files) + + # change content of one env_file in tmp_path + file_path = next(tmp_path.iterdir()) + with open(file_path, "w") as file: + file.write("This is the new content") + + # copy env files again + with pytest.raises(AssertionError) as excinfo: + EnvManager.copy_env_files(env_files, DIR) + + assert "input env files have changed" in str(excinfo.value) + + +def test_copy_env_files_twice_with_name_change(tmp_output: Path, tmp_recipes: Path, tmp_path: Path): + # copy env files to tmp_path + assert len(list(tmp_path.iterdir())) == 0 + for f in ENV_PATHS: + shutil.copy(f, tmp_path / f.name) + ENV_PATHS_NEW = list(tmp_path.iterdir()) + assert len(ENV_PATHS_NEW) > 0 + + # create dirs in output + DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes) + DIR.create_all_dirs() + + # confirm dir is empty + assert len(list(DIR.ENV_FILES.iterdir())) == 0 + + # copy env files from tmp_path to tmp_output + env_files = EnvManager._get_env_files(ENV_PATHS_NEW) + EnvManager.copy_env_files(env_files, DIR) + + # check that each env file is present in DIR.ENV_FILES with correct contents + assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files) + + # change name of one env_file in tmp_path + file_path = next(tmp_path.iterdir()) + file_path.rename(file_path.parent / (file_path.stem + "-other" + file_path.suffix)) + + # copy env files from tmp_path to tmp_output again + with pytest.raises(AssertionError) as excinfo: + env_files = EnvManager._get_env_files(list(tmp_path.iterdir())) + EnvManager.copy_env_files(env_files, DIR) + + assert "input env files have changed" in str(excinfo.value) -- 2.47.2 From 6be9fdc53515b000770ed57559c8fb9d8bd6ac23 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 12:13:00 +0100 Subject: [PATCH 80/91] use os.environ["TEST_USER"], fix check_if_user_exists --- .../tests_authentik/setup_authentik.py | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/recipes/authentik/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py index 5fbde7a..0881307 100644 --- a/recipes/authentik/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -2,15 +2,14 @@ import json import os import re -from playwright.sync_api import BrowserContext, expect +from playwright.sync_api import BrowserContext, TimeoutError, expect from pytest_abra import BaseUrl, DirManager ADMIN_USER = os.environ["ADMIN_USER"] ADMIN_PASS = os.environ["ADMIN_PASS"] - - -TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} +TEST_USER = os.environ["TEST_USER"] +TEST_PASS = os.environ["TEST_PASS"] def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl): @@ -42,9 +41,12 @@ def check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, st nav.click() nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() - user = page.get_by_text(TESTUSER["username"]) - user.wait_for(state="visible") - return user.is_visible() + user = page.get_by_text(TEST_USER) + try: + user.wait_for(state="visible", timeout=5_000) + return True + except TimeoutError: + return False def create_invite_link(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): @@ -83,15 +85,16 @@ def create_user(user_context: BrowserContext, invitelink): page = user_context.new_page() page.goto(invitelink) page.get_by_placeholder("Benutzername").click() - page.get_by_placeholder("Benutzername").fill(TESTUSER["username"]) + page.get_by_placeholder("Benutzername").fill(TEST_USER) page.locator('input[name="name"]').click() - page.locator('input[name="name"]').fill(TESTUSER["name"]) + page.locator('input[name="name"]').fill("name") page.locator('input[name="email"]').click() - page.locator('input[name="email"]').fill(TESTUSER["email"]) + email = os.environ["IMAP_EMAIL"] if "IMAP_EMAIL" in os.environ else "test@domain.com" + page.locator('input[name="email"]').fill(email) page.get_by_placeholder("Passwort", exact=True).click() - page.get_by_placeholder("Passwort", exact=True).fill(TESTUSER["password"]) + page.get_by_placeholder("Passwort", exact=True).fill(TEST_PASS) page.get_by_placeholder("Passwort (wiederholen)").click() - page.get_by_placeholder("Passwort (wiederholen)").fill(TESTUSER["password"]) + page.get_by_placeholder("Passwort (wiederholen)").fill(TEST_PASS) page.get_by_role("button", name="Weiter").click() expect(page.locator("ak-library")).to_be_visible() -- 2.47.2 From 359a18eae5bb6977c120e9431e23c2ffd296014d Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 12:29:06 +0100 Subject: [PATCH 81/91] turn check_if_user_exists into a fixture --- recipes/authentik/tests_authentik/conftest.py | 34 +++++++++++++++++-- .../tests_authentik/setup_authentik.py | 23 +++---------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/recipes/authentik/tests_authentik/conftest.py b/recipes/authentik/tests_authentik/conftest.py index 6542c21..5922893 100644 --- a/recipes/authentik/tests_authentik/conftest.py +++ b/recipes/authentik/tests_authentik/conftest.py @@ -1,9 +1,11 @@ -from typing import Generator +import os +import re +from typing import Callable, Generator import pytest -from playwright.sync_api import APIRequestContext, Playwright +from playwright.sync_api import APIRequestContext, BrowserContext, Playwright, TimeoutError -from pytest_abra import DirManager +from pytest_abra import BaseUrl, DirManager @pytest.fixture(scope="session") @@ -15,3 +17,29 @@ def api_request_context( request_context = playwright.request.new_context(storage_state=state_file) yield request_context request_context.dispose() + + +@pytest.fixture +def check_if_user_exists() -> Callable[[BrowserContext, dict[str, str], BaseUrl], bool]: + """This is actually a normal function supplied as a fixture. This is because normal imports from tests_authentik is difficult. + tests_authentik is not part of the python environment, so + from ... import function + will most likely fail. However, pytest handles the loading of fixtures from conftest.py automatically, hence we use that to load functions too.""" + + def inner_check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl) -> bool: + # go to admin page + page = admin_context.new_page() + page.goto(URL.get()) + page.get_by_role("link", name="Admin Interface").click() + nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) + nav.click() + nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() + + user = page.get_by_text(os.environ["TEST_USER"]) + try: + user.wait_for(state="visible", timeout=5_000) + return True + except TimeoutError: + return False + + return inner_check_if_user_exists diff --git a/recipes/authentik/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py index 0881307..b66b41d 100644 --- a/recipes/authentik/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -2,7 +2,7 @@ import json import os import re -from playwright.sync_api import BrowserContext, TimeoutError, expect +from playwright.sync_api import BrowserContext, expect from pytest_abra import BaseUrl, DirManager @@ -32,23 +32,6 @@ def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: context.storage_state(path=DIR.STATES / "authentik_admin_state.json") -def check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): - # go to admin page - page = admin_context.new_page() - page.goto(URL.get()) - page.get_by_role("link", name="Admin Interface").click() - nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) - nav.click() - nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() - - user = page.get_by_text(TEST_USER) - try: - user.wait_for(state="visible", timeout=5_000) - return True - except TimeoutError: - return False - - def create_invite_link(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): # go to admin page page = admin_context.new_page() @@ -99,7 +82,9 @@ def create_user(user_context: BrowserContext, invitelink): expect(page.locator("ak-library")).to_be_visible() -def setup_user_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl): +def setup_user_state( + context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists +): # load admin cookies to context state_file = DIR.STATES / "authentik_admin_state.json" storage_state = json.loads(state_file.read_bytes()) -- 2.47.2 From b13303d404679e18fe07df1da76b2e4529aac7ce Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:02:31 +0100 Subject: [PATCH 82/91] wip add cleanup routine to delete user --- .../tests_authentik/cleanup_authentik.py | 30 +++++++++++++++++++ .../tests_authentik/runner_authentik.py | 1 + 2 files changed, 31 insertions(+) create mode 100644 recipes/authentik/tests_authentik/cleanup_authentik.py diff --git a/recipes/authentik/tests_authentik/cleanup_authentik.py b/recipes/authentik/tests_authentik/cleanup_authentik.py new file mode 100644 index 0000000..3d65308 --- /dev/null +++ b/recipes/authentik/tests_authentik/cleanup_authentik.py @@ -0,0 +1,30 @@ +import json +import os + +from playwright.sync_api import BrowserContext + +from pytest_abra import BaseUrl, DirManager + +ADMIN_USER = os.environ["ADMIN_USER"] +ADMIN_PASS = os.environ["ADMIN_PASS"] +TEST_USER = os.environ["TEST_USER"] +TEST_PASS = os.environ["TEST_PASS"] + + +def remove_user(): + pass + + +def cleanup_delete_user( + context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists +): + # load admin cookies to context + state_file = DIR.STATES / "authentik_admin_state.json" + storage_state = json.loads(state_file.read_bytes()) + context.add_cookies(storage_state["cookies"]) + + if check_if_user_exists(context, env_config, URL): + # just login with user + assert False, "yes" + remove_user() + assert False, "no" diff --git a/recipes/authentik/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py index 2543dc6..1079409 100644 --- a/recipes/authentik/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -5,3 +5,4 @@ class RunnerAuthentik(Runner): env_type = "authentik" setups = [Test(test_file="setup_authentik.py")] tests = [Test(test_file="test_authentik_blueprint_api.py")] + cleanups = [Test(test_file="cleanup_authentik.py", prevent_skip=True)] -- 2.47.2 From baf6f6a6557c022af54eca2679da5494e74807b7 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:15:17 +0100 Subject: [PATCH 83/91] implement remove_user in authentik --- .../tests_authentik/cleanup_authentik.py | 22 ++++++++++++++----- .../tests_authentik/runner_authentik.py | 2 +- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/recipes/authentik/tests_authentik/cleanup_authentik.py b/recipes/authentik/tests_authentik/cleanup_authentik.py index 3d65308..b556ea4 100644 --- a/recipes/authentik/tests_authentik/cleanup_authentik.py +++ b/recipes/authentik/tests_authentik/cleanup_authentik.py @@ -1,5 +1,6 @@ import json import os +import re from playwright.sync_api import BrowserContext @@ -11,8 +12,19 @@ TEST_USER = os.environ["TEST_USER"] TEST_PASS = os.environ["TEST_PASS"] -def remove_user(): - pass +def remove_user(admin_context: BrowserContext, URL: BaseUrl): + """removes TEST_USER account from authentik""" + page = admin_context.new_page() + page.goto(URL.get()) + page.get_by_role("link", name="Admin Interface").click() + nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) + nav.click() + nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() + + name_pattern = re.compile(TEST_USER) + page.get_by_role("row", name=name_pattern).get_by_label("").check() + page.get_by_role("button", name=re.compile(r"Löschen|Delete")).click() + page.get_by_role("dialog").get_by_role("button", name=re.compile(r"Löschen|Delete")).click() def cleanup_delete_user( @@ -24,7 +36,5 @@ def cleanup_delete_user( context.add_cookies(storage_state["cookies"]) if check_if_user_exists(context, env_config, URL): - # just login with user - assert False, "yes" - remove_user() - assert False, "no" + remove_user(context, URL) + assert not check_if_user_exists(context, env_config, URL) diff --git a/recipes/authentik/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py index 1079409..4e9b81a 100644 --- a/recipes/authentik/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -5,4 +5,4 @@ class RunnerAuthentik(Runner): env_type = "authentik" setups = [Test(test_file="setup_authentik.py")] tests = [Test(test_file="test_authentik_blueprint_api.py")] - cleanups = [Test(test_file="cleanup_authentik.py", prevent_skip=True)] + cleanups = [Test(test_file="cleanup_authentik.py")] -- 2.47.2 From 5e3df614bb6ea19b3c5a448a47a14391416cfd58 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:15:54 +0100 Subject: [PATCH 84/91] add optional session_id cli arg --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index f03903c..9d3ac37 100644 --- a/main.py +++ b/main.py @@ -40,5 +40,7 @@ subprocess.run( OUTPUT_DIR, "--resume", # "--debug", + # "--session_id", + # "abc", ] ) -- 2.47.2 From 9ff1d1a2a0efc6ecc4982d26e09182973f41ccc1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:28:57 +0100 Subject: [PATCH 85/91] wip more docs --- docs/documentation.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/documentation.md b/docs/documentation.md index 560ce6e..b84c7da 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -213,6 +213,16 @@ Furthermore, some `Runner` classes can depend on others. For example, `RunnerWor | 9. | Wordpress-2 | cleanups | +# Create a test suite for a recipe + +todo + +To understand how a test suite is built, let's have a look at the files + +runner_authentik.py -> required, defines the Runner subclass (see below) +conftest.py -> not required. special file for pytest. is automatically discovered and loaded. convenient place to define fixtures and functions to be used in more than one test routine +setup_authentik.py -> not required. can hold setup routine for authentik, has to be registered in runner_authentik.py + # Create a custom Runner To comprehend the process of creating a new subclass of `Runner`, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. -- 2.47.2 From 2b0a79f8f63fc44871016e38fc01642f5678f076 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:33:12 +0100 Subject: [PATCH 86/91] improve docstring --- recipes/authentik/tests_authentik/conftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/recipes/authentik/tests_authentik/conftest.py b/recipes/authentik/tests_authentik/conftest.py index 5922893..df919d9 100644 --- a/recipes/authentik/tests_authentik/conftest.py +++ b/recipes/authentik/tests_authentik/conftest.py @@ -21,10 +21,11 @@ def api_request_context( @pytest.fixture def check_if_user_exists() -> Callable[[BrowserContext, dict[str, str], BaseUrl], bool]: - """This is actually a normal function supplied as a fixture. This is because normal imports from tests_authentik is difficult. - tests_authentik is not part of the python environment, so - from ... import function - will most likely fail. However, pytest handles the loading of fixtures from conftest.py automatically, hence we use that to load functions too.""" + """This is actually a normal function supplied by a fixture. We do this, because imports from + tests_authentik are difficult as it is not part of the python environment. We expect + from X import function + to fail here. However, pytest handles the loading of fixtures from conftest.py automatically, + hence we use that to load functions too.""" def inner_check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl) -> bool: # go to admin page -- 2.47.2 From 0602899fca392ac764505ff3b9969c7c6a60f99e Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:47:24 +0100 Subject: [PATCH 87/91] enable test_wordpress_localization --- recipes/wordpress/tests_wordpress/runner_wordpress.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/recipes/wordpress/tests_wordpress/runner_wordpress.py b/recipes/wordpress/tests_wordpress/runner_wordpress.py index 21cdeab..aef1273 100644 --- a/recipes/wordpress/tests_wordpress/runner_wordpress.py +++ b/recipes/wordpress/tests_wordpress/runner_wordpress.py @@ -1,11 +1,12 @@ from pytest_abra import ConditionArgs, Runner, Test -def condition_has_locale(args: ConditionArgs) -> bool: +def env_config_has_locale(args: ConditionArgs) -> bool: env_config = args.env_config - if "de" in env_config.get("LOCALE", ""): + if "LOCALE" in env_config: return True - return False + else: + return False class RunnerWordpress(Runner): @@ -17,5 +18,5 @@ class RunnerWordpress(Runner): ] tests = [ # Test(test_file="test_wordpress_receive_email.py", prevent_skip=True), - # Test(condition=condition_has_locale, test_file="test_wordpress_localization.py"), + Test(condition=env_config_has_locale, test_file="test_wordpress_localization.py"), ] -- 2.47.2 From 1481a3a49a584b99aff74f5eb2f1ed267f09fa96 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:52:18 +0100 Subject: [PATCH 88/91] improve output in case of failed tests --- tests/test_pytest_abra.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_pytest_abra.py b/tests/test_pytest_abra.py index 805ce4c..e2d8520 100644 --- a/tests/test_pytest_abra.py +++ b/tests/test_pytest_abra.py @@ -61,7 +61,8 @@ def test_results_abra(session_tmp_path_testout: Path): OUTPUT_DIR = session_tmp_path_testout.resolve() DIR = DirManager(output_dir=OUTPUT_DIR, session_id="abc") - all_files = list(DIR.STATUS.rglob("*")) - passed_files = list(DIR.STATUS.rglob("passed-*")) + all_files = [f.name for f in DIR.STATUS.rglob("*")] + passed_files = [f.name for f in DIR.STATUS.rglob("passed-*")] + failed_files = set(all_files) - set(passed_files) assert len(all_files) > 0 - assert len(all_files) == len(passed_files) + assert not failed_files, failed_files -- 2.47.2 From 756dcc96f678cf891cc7f30f2e5b24ab7430d41a Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:53:15 +0100 Subject: [PATCH 89/91] rename --- tests/{test_pytest_abra.py => test_cli_full_integration.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_pytest_abra.py => test_cli_full_integration.py} (100%) diff --git a/tests/test_pytest_abra.py b/tests/test_cli_full_integration.py similarity index 100% rename from tests/test_pytest_abra.py rename to tests/test_cli_full_integration.py -- 2.47.2 From 336ed269870470a4a7aa934e0c26790de45b1032 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 13:53:32 +0100 Subject: [PATCH 90/91] cleanup --- .../wordpress/tests_wordpress/test_wordpress_localization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py index 4731a0c..2e01d79 100644 --- a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py +++ b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py @@ -5,10 +5,10 @@ from playwright.sync_api import BrowserContext, expect from pytest_abra import BaseUrl -def test_welcome_message(context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): +def test_de_welcome_message(context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): page = context.new_page() page.goto(URL.get()) expect(page.locator(".wp-block-heading")).to_be_visible() - if "locale" in env_config and "de" in env_config["locale"]: + if "de" in env_config.get("locale", ""): expect(page.get_by_role("heading")).to_have_text("Willkommen bei WordPress!") -- 2.47.2 From 1fa97e402fa821d33551a1b9b83a9d7aca296285 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 14:01:59 +0100 Subject: [PATCH 91/91] update cli help --- pytest_abra/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index dfcf90a..ca39cf2 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -18,8 +18,8 @@ def run(): parser = argparse.ArgumentParser() parser.add_argument("--version", "-V", action="version", version=get_version(), help="output the version number") parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;", required=True) - parser.add_argument("--recipes_dir", type=Path, help="List of loaded env files separated with ;", required=True) - parser.add_argument("--output_dir", type=Path, help="List of loaded env files separated with ;", required=True) + parser.add_argument("--recipes_dir", type=Path, help="Dir of abra recipes and respective runners", required=True) + parser.add_argument("--output_dir", type=Path, help="Dir of test outputs", required=True) parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=20_000) parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode") parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests") -- 2.47.2