local-it/e2e_tests
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

various (#16)

Browse source 
* add full integration test of cli / pytest_abra with all tests

* save path of runner_*.py in runner subclass to improve test discovery -> allows for same test name in two different runners

* reorganize output dir names

* use URL fixture everywhere

* rework coordinator interface

* add --session_id to cli args

* add log results table

* plenty of refactoring

* add assert messages

* add plenty of tests

* add /docs dir with plenty of documentation

* fix authentik setup

* add authentik cleanup, remove test user

* add random test user credential generation and integrate into test routine. random creds are saved to STATES

Reviewed-on: local-it-infrastructure/e2e_tests#16
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>
This commit is contained in:
Daniel 2023-12-14 14:03:58 +01:00 • committed by dan
parent 016b88a68d
commit 2dd765a974
36 changed files with 1145 additions and 432 deletions
Download patch file Download diff file Expand all files Collapse all files

63
pytest_abra/coordinator.py
View file

@ -1,15 +1,18 @@
import importlib
import json
import re
import sys
from pathlib import Path
from loguru import logger
from tabulate import tabulate # type: ignore
from pytest_abra.dir_manager import DirManager
from pytest_abra.env_manager import EnvFile, EnvManager
from pytest_abra.html_helper import merge_html_reports
from pytest_abra.runner import Runner
from pytest_abra.utils import rmtree
from pytest_abra.shared_types import TestResult
from pytest_abra.utils import generate_random_string, load_json_to_environ, rmtree
class Coordinator:
@ -32,21 +35,24 @@ class Coordinator:
self.ENV = EnvManager(env_paths=env_paths, RUNNER_DICT=self.RUNNER_DICT)
self.TIMEOUT = timeout
def setup_test(self) -> None:
logger.info("calling setup_test()")
def prepare_tests(self) -> None:
logger.info("calling prepare_tests()")
self.DIR.create_all_dirs()
self.ENV.copy_env_files(self.DIR)
self.ENV.copy_env_files(self.ENV.env_files, self.DIR)
self.load_test_credentials(self.DIR)
def run_test(self) -> None:
logger.info("calling run_test()")
def run_tests(self) -> None:
logger.info("calling run_tests()")
self.runners: list[Runner] = self._load_runners(self.ENV.env_files)
status_list: list[TestResult] = []
for runner in self.runners:
runner.run_setups()
status_list.extend(runner.run_setups())
for runner in self.runners:
runner.run_tests()
status_list.extend(runner.run_tests())
for runner in self.runners:
runner.run_cleanups()
logger.info("run_test() finished")
status_list.extend(runner.run_cleanups())
status_table = tabulate([[t.test_name, t.status] for t in status_list], headers=["name", "status"])
logger.info(f"run_tests() finished\n{status_table}")
def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]:
"""Creates an instance of the correct Runner class for each given env file"""
@ -58,13 +64,13 @@ class Coordinator:
def combine_html(self) -> None:
"""combines all generated pytest html reports into one"""
in_dir_path = str(self.DIR.RECORDS / "html")
out_file_path = str(self.DIR.RECORDS / "full-report.html")
in_dir_path = str(self.DIR.RESULTS / "html")
out_file_path = str(self.DIR.RESULTS / "full-report.html")
title = "combined.html"
merge_html_reports(in_dir_path, out_file_path, title)
def collect_traces(self):
"""moves all traces into SESSION/RECORDS dir
"""moves all traces into SESSION/RESULTS dir
if tests are rerun and generate another trace, the new trace will get a unique name such as
tracename-0
@ -80,14 +86,34 @@ class Coordinator:
index += 1
return get_new_path(root_dir, base_name, index=index)
trace_root_dir = self.DIR.RECORDS / "traces"
trace_root_dir = self.DIR.RESULTS / "traces"
for f in trace_root_dir.rglob("*/trace.zip"):
new_path = get_new_path(self.DIR.RECORDS, f.parent.name)
new_path = get_new_path(self.DIR.RESULTS, f.parent.name)
f.parent.rename(new_path)
rmtree(trace_root_dir)
@staticmethod
def create_runner_dict(recipes_dir: Path) -> dict[str, type["Runner"]]:
def load_test_credentials(DIR: DirManager):
"""Load test user credentials. If not available, create them randomly.
Test users are created during testing but should be deleted after the test. In case test
users are not deleted after tests by accident, the user credentials are not known to an
attacker."""
test_credentials_path = DIR.STATES / "credentials_test.json"
if not test_credentials_path.is_file():
test_credentials = {
"TEST_USER": "test-" + generate_random_string(6),
"TEST_PASS": generate_random_string(12, punctuation=True),
}
with open(test_credentials_path, "w") as json_file:
json.dump(test_credentials, json_file)
load_json_to_environ(test_credentials_path)
@staticmethod
def create_runner_dict(recipes_dir: Path) -> dict[str, type[Runner]]:
"""Creates a dictionary holding all the RunnerClasses that can be discovered in recipes_dir
example:
@ -101,18 +127,19 @@ class Coordinator:
because recipes_dir is added to sys.path.
"""
RUNNER_DICT: dict[str, type["Runner"]] = dict()
RUNNER_DICT: dict[str, type[Runner]] = dict()
runner_discovery_pattern = re.compile("Runner.+")
# make it possible to import modules from recipes_dir
sys.path.append(recipes_dir.as_posix())
for module_path in recipes_dir.rglob("*/runner*.py"):
for module_path in recipes_dir.rglob("*/runner_*.py"):
rel_path = module_path.relative_to(recipes_dir).as_posix().replace("/", ".").replace(".py", "")
module = importlib.import_module(rel_path)
runner_class_names = [name for name in dir(module) if runner_discovery_pattern.match(name)]
assert len(runner_class_names) == 1
runner_class_name = runner_class_names[0]
RunnerClass: type[Runner] = getattr(module, runner_class_name)
RunnerClass._tests_path = module_path.parent
RUNNER_DICT[RunnerClass.env_type] = RunnerClass
return RUNNER_DICT