local-it/e2e_tests
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

rework-output-and-test-logic (#3)

Browse source 
* fix flakey tests in authentik / wordpress

* make it possible to rerun tests partially -> passed will be skipped, failed will be repeated

* improve organization of all outputs (moving, renaming, keeping multiple versions etc.)

* add html reports, replace .txt tracebacks

* combine all html reports into one

* add demo runner with comments for documentation purposes

Reviewed-on: local-it-infrastructure/e2e_tests#3
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>
This commit is contained in:
Daniel 2023-11-29 14:14:46 +01:00 • committed by dan
parent d2cd6ba47f
commit 8172f685de
24 changed files with 588 additions and 418 deletions
Download patch file Download diff file Expand all files Collapse all files

31
src/conftest.py
View file

@ -9,24 +9,28 @@ from pathlib import Path
import pytest
from dotenv import dotenv_values
from pytest import Parser
from src.dirmanager import DirManager
TIMEOUT = 5000
def pytest_addoption(parser):
def pytest_addoption(parser: Parser):
parser.addoption(
"--env_file",
action="store",
required=True,
)
parser.addoption(
"--output_dir",
action="store",
required=True,
)
parser.addoption(
"--session_id",
action="store",
required=True,
)
@ -38,14 +42,11 @@ def DIR(request) -> DirManager:
DIR.SESSION
DIR.RECORDS
DIR.STATES
DIR.RESULTS
DIR.PROGRESS"""
DIR.RESULTS"""
output_dir = request.config.getoption("--output_dir")
assert output_dir is not None, "required pytest command line argument not given"
output_dir = Path(output_dir)
session_id = request.config.getoption("--session_id")
assert session_id is not None, "required pytest command line argument not given"
dirmanager = DirManager(output_dir=output_dir, session_id=session_id)
dirmanager.create_all_dirs()
return dirmanager
@ -54,26 +55,6 @@ def DIR(request) -> DirManager:
@pytest.fixture(scope="session", autouse=True)
def dotenv_config(request) -> dict[str, str]:
dotenv_path = request.config.getoption("--env_file")
assert dotenv_path is not None, "required pytest command line argument not given"
dotenv_path = Path(dotenv_path)
assert dotenv_path.is_file()
return dotenv_values(dotenv_path) # type: ignore
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
"""saves traceback when test fails"""
# execute all other hooks to obtain the report object
outcome = yield
rep = outcome.get_result()
# we only look at actual failing test calls, not setup/teardown
if rep.when == "call" and rep.failed:
# saves traceback as .txt for failed test
filename = f"failed-{item.nodeid}.txt"
filename = filename.replace("/", "-")
filename = filename.replace("::", "-")
filepath = item.funcargs["DIR"].RESULTS / filename
with open(filepath, "a") as f:
f.write(rep.longreprtext + "\n")