From 97ed87c79fe903205d829f24ae695d6fb155e245 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 22 Nov 2023 21:40:13 +0100 Subject: [PATCH 01/18] [WIP] Add new automated test framework (#1) Co-authored-by: Daniel Co-committed-by: Daniel --- .gitignore | 3 + .gitmodules | 3 + Dockerfile | 12 ++ README.md | 21 +- docker-compose.yml | 6 + envfiles | 1 + previous-work/README.md | 11 ++ .../authentik_test.py | 0 config.yaml => previous-work/config.yaml | 0 conftest.py => previous-work/conftest.py | 0 .../nextcloud_test.py | 0 pytest.ini => previous-work/pytest.ini | 0 .../vikunja_test.py | 0 wekan_test.py => previous-work/wekan_test.py | 0 .../wordpress_test.py | 0 pyproject.toml | 19 ++ requirements.txt | 4 + src/blog.dev.local-it.cloud.env | 71 +++++++ src/conftest.py | 64 +++++++ src/dirmanager.py | 53 ++++++ src/main.py | 76 ++++++++ src/runner.py | 62 ++++++ src/setup/setup_authentik.py | 47 +++++ src/tests_authentik/runner_authentik.py | 16 ++ src/tests_authentik/test_authentik_dummy.py | 2 + src/tests_authentik/test_authentik_setup.py | 179 ++++++++++++++++++ src/tests_wordpress/conftest.py | 28 +++ src/tests_wordpress/runner_wordpress.py | 20 ++ src/tests_wordpress/test_wordpress.py | 29 +++ .../test_wordpress_feature1.py | 26 +++ .../test_wordpress_localization.py | 22 +++ 31 files changed, 769 insertions(+), 6 deletions(-) create mode 100644 .gitmodules create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 160000 envfiles create mode 100644 previous-work/README.md rename authentik_test.py => previous-work/authentik_test.py (100%) rename config.yaml => previous-work/config.yaml (100%) rename conftest.py => previous-work/conftest.py (100%) rename nextcloud_test.py => previous-work/nextcloud_test.py (100%) rename pytest.ini => previous-work/pytest.ini (100%) rename vikunja_test.py => previous-work/vikunja_test.py (100%) rename wekan_test.py => previous-work/wekan_test.py (100%) rename wordpress_test.py => previous-work/wordpress_test.py (100%) create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 src/blog.dev.local-it.cloud.env create mode 100644 src/conftest.py create mode 100644 src/dirmanager.py create mode 100644 src/main.py create mode 100644 src/runner.py create mode 100644 src/setup/setup_authentik.py create mode 100644 src/tests_authentik/runner_authentik.py create mode 100644 src/tests_authentik/test_authentik_dummy.py create mode 100644 src/tests_authentik/test_authentik_setup.py create mode 100644 src/tests_wordpress/conftest.py create mode 100644 src/tests_wordpress/runner_wordpress.py create mode 100644 src/tests_wordpress/test_wordpress.py create mode 100644 src/tests_wordpress/test_wordpress_feature1.py create mode 100644 src/tests_wordpress/test_wordpress_localization.py diff --git a/.gitignore b/.gitignore index 47caf4d..4395fbc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ __pycache__/ *.json *.zip TestResults/ +.vscode/ +*.pyc +credentials* \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..2cb2d96 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "envfiles"] + path = envfiles + url = ssh://git@git.local-it.org:2222/local-it-infrastructure/dev.local-it.cloud.git diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..662da0f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12-slim + +RUN pip install --no-cache-dir pytest-playwright + +RUN playwright install + +RUN playwright install-deps + +COPY ./requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +WORKDIR /code/src \ No newline at end of file diff --git a/README.md b/README.md index 4d956f1..acf5183 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,20 @@ +# Readme -# Installation +```bash +docker compose build +docker compose run --rm app python ./main.py +docker compose run --rm app pytest +# docker-compose up +``` -pip install pytest-playwright -playwright install +Force rebuild with cache -# Run Tests: +```bash +docker-compose up --build +``` -pytest -k nextcloud +Force rebuild wtihtout cache -playwright show-trace trace.zip +```bash +docker-compose build --no-cache +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5da046e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,6 @@ +services: + app: + build: . + container_name: python-env + volumes: + - .:/code diff --git a/envfiles b/envfiles new file mode 160000 index 0000000..a8375f6 --- /dev/null +++ b/envfiles @@ -0,0 +1 @@ +Subproject commit a8375f6fc7a285a1000b5553be47eaf19b0be0a6 diff --git a/previous-work/README.md b/previous-work/README.md new file mode 100644 index 0000000..4d956f1 --- /dev/null +++ b/previous-work/README.md @@ -0,0 +1,11 @@ + +# Installation + +pip install pytest-playwright +playwright install + +# Run Tests: + +pytest -k nextcloud + +playwright show-trace trace.zip diff --git a/authentik_test.py b/previous-work/authentik_test.py similarity index 100% rename from authentik_test.py rename to previous-work/authentik_test.py diff --git a/config.yaml b/previous-work/config.yaml similarity index 100% rename from config.yaml rename to previous-work/config.yaml diff --git a/conftest.py b/previous-work/conftest.py similarity index 100% rename from conftest.py rename to previous-work/conftest.py diff --git a/nextcloud_test.py b/previous-work/nextcloud_test.py similarity index 100% rename from nextcloud_test.py rename to previous-work/nextcloud_test.py diff --git a/pytest.ini b/previous-work/pytest.ini similarity index 100% rename from pytest.ini rename to previous-work/pytest.ini diff --git a/vikunja_test.py b/previous-work/vikunja_test.py similarity index 100% rename from vikunja_test.py rename to previous-work/vikunja_test.py diff --git a/wekan_test.py b/previous-work/wekan_test.py similarity index 100% rename from wekan_test.py rename to previous-work/wekan_test.py diff --git a/wordpress_test.py b/previous-work/wordpress_test.py similarity index 100% rename from wordpress_test.py rename to previous-work/wordpress_test.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4a80e86 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "locit-testing" +version = "0.1.0" +requires-python = "~=3.11" +dependencies = [ + "pytest == 7.4.3", +] + +[project.optional-dependencies] +dev = [ + "ruff >= 0.1.3", +] + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.ruff] +line-length = 120 +target-version = "py311" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..66c69c3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pytest +pytest-playwright +python-dotenv +icecream \ No newline at end of file diff --git a/src/blog.dev.local-it.cloud.env b/src/blog.dev.local-it.cloud.env new file mode 100644 index 0000000..600aaf6 --- /dev/null +++ b/src/blog.dev.local-it.cloud.env @@ -0,0 +1,71 @@ +################################################################################ +# DO NOT EDIT THIS FILE, IT IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN # +################################################################################ + +TYPE=wordpress +TIMEOUT=300 +ENABLE_AUTO_UPDATE=true +COMPOSE_FILE="compose.yml" + +# Setup Wordpress settings on each deploy: +#POST_DEPLOY_CMDS="app core_install" + +DOMAIN=blog.dev.local-it.cloud +## Domain aliases +#EXTRA_DOMAINS=', `www.blog.dev.local-it.cloud`' +LETS_ENCRYPT_ENV=production + +TITLE="My Example Blog" +LOCALE=de_DE +ADMIN_EMAIL=admin@example.com + +# Every new user is per default subscriber, uncomment to change it +DEFAULT_USER_ROLE=administrator + +#WORDPRESS_DEBUG=true + +## Additional extensions +#PHP_EXTENSIONS="calendar" + +SECRET_DB_ROOT_PASSWORD_VERSION=v1 +SECRET_DB_PASSWORD_VERSION=v1 + +# Mostly for compatibility with existing database dumps... +#WORDPRESS_TABLE_PREFIX=wp_ + +# Multisite +#WORDPRESS_CONFIG_EXTRA="\ +#define('WP_CACHE', false);\ +#define('WP_ALLOW_MULTISITE', true );" + +# Multisite phase 2 (see README) +#WORDPRESS_CONFIG_EXTRA="define('MULTISITE', true); define('SUBDOMAIN_INSTALL', true); define('DOMAIN_CURRENT_SITE', '${DOMAIN}'); define('PATH_CURRENT_SITE', '/'); define('SITE_ID_CURRENT_SITE', 1); define('BLOG_ID_CURRENT_SITE', 1); define('FORCE_SSL_ADMIN', true ); define('COOKIE_DOMAIN', \$_SERVER['HTTP_HOST']);" + +# Local SMTP relay +#COMPOSE_FILE="$COMPOSE_FILE:compose.mailrelay.yml" +SMTP_HOST=mail.local-it.org +MAIL_FROM=noreply@local-it.org + +# Remote SMTP relay +COMPOSE_FILE="$COMPOSE_FILE:compose.smtp.yml" +SMTP_HOST=mail.local-it.org +MAIL_FROM=noreply@local-it.org +SMTP_PORT=587 +SMTP_AUTH=on +SMTP_TLS=on +SECRET_SMTP_PASSWORD_VERSION=v1 + +# Authentik SSO +COMPOSE_FILE="$COMPOSE_FILE:compose.authentik.yml" +AUTHENTIK_DOMAIN=dev.local-it.cloud +SECRET_AUTHENTIK_SECRET_VERSION=v1 +SECRET_AUTHENTIK_ID_VERSION=v1 +LOGIN_TYPE='auto' + +# 🚩🚩 dangerous, use only for development sites! + +# Allow remote connections to db +#COMPOSE_FILE="$COMPOSE_FILE:compose.public-db.yml + +# Wide-open CORS +#CORS_ALLOW_ALL=1 diff --git a/src/conftest.py b/src/conftest.py new file mode 100644 index 0000000..5c0bcee --- /dev/null +++ b/src/conftest.py @@ -0,0 +1,64 @@ +# regarding conftest: +# If you have conftest.py files which do not reside in a python package directory +# (i.e. one containing an __init__.py) then “import conftest” can be ambiguous +# because there might be other conftest.py files as well on your PYTHONPATH or +# sys.path. It is thus good practise for projects to either put conftest.py under +# a package scope or to never import anything from a conftest.py file. + + +from pathlib import Path + +import pytest +from dirmanager import DirManager +from dotenv import dotenv_values + +pytest_plugins = [ + "setup.setup_authentik", +] + + +def pytest_addoption(parser): + parser.addoption( + "--env_file", + action="store", + ) + parser.addoption( + "--tests_dir", + action="store", + ) + parser.addoption( + "--session_id", + action="store", + ) + + +@pytest.fixture(scope="session", autouse=True) +def dirmanager(request) -> DirManager: + tests_dir = request.config.getoption("--tests_dir") + tests_dir = Path(tests_dir) + session_id = request.config.getoption("--session_id") + return DirManager(tests_dir=tests_dir, session_id=session_id) + + +@pytest.fixture(scope="session", autouse=True) +def dotenv_config(request) -> dict[str, str]: + dotenv_path = request.config.getoption("--env_file") + dotenv_path = Path(dotenv_path) + assert dotenv_path.is_file() + return dotenv_values(dotenv_path) + + +@pytest.fixture(scope="session", autouse=True) +def RECORDS(dirmanager) -> Path: + assert isinstance(dirmanager, DirManager) + return dirmanager.dirs["records"] + + +@pytest.fixture(scope="session", autouse=True) +def STATES(dirmanager) -> Path: + return dirmanager.dirs["states"] + + +@pytest.fixture(scope="session", autouse=True) +def RESULTS(dirmanager) -> Path: + return dirmanager.dirs["results"] diff --git a/src/dirmanager.py b/src/dirmanager.py new file mode 100644 index 0000000..dfc149d --- /dev/null +++ b/src/dirmanager.py @@ -0,0 +1,53 @@ +from pathlib import Path + + +class DirManager: + """Manages directories for the tests and should be used to create and find + and use the correct directories. + + The structures is as follows: + tests dir/ + session_dir-1/ + records + states + results + session_dir-2/ + records + ... + """ + + def __init__(self, tests_dir: Path, session_id: str): + # root test dir + self.tests_dir = tests_dir + self.session_id = session_id + + self.dirs = self._get_all_dirs() + + def create_all_dirs(self): + self.create_dirs(self.tests_dir, exist_ok=True) + self.create_dirs(self.dirs) + + def _get_all_dirs(self): + dirs = {} + dirs["session"] = self.tests_dir / f"test-{self.session_id}" + dirs.update(self._get_subdirs(session_dir=dirs["session"])) + return dirs + + def _get_subdirs(self, session_dir: Path): + return { + "records": session_dir / Path("records"), + "states": session_dir / Path("states"), + "results": session_dir / Path("results"), + } + + @staticmethod + def create_dirs(dirs: Path | list[Path] | dict[str, Path], exist_ok=False): + match dirs: + case Path(): + dirs.mkdir(exist_ok=exist_ok) + case list(): + for d in dirs: + d.mkdir(exist_ok=exist_ok) + case dict(): + for d in dirs.values(): + d.mkdir(exist_ok=exist_ok) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..3100621 --- /dev/null +++ b/src/main.py @@ -0,0 +1,76 @@ +from datetime import datetime +from pathlib import Path +from typing import Protocol + +from dirmanager import DirManager +from dotenv import dotenv_values +from icecream import ic +from tests_authentik.runner_authentik import RunnerAuthentik +from tests_wordpress.runner_wordpress import RunnerWordpress + +TESTS_DIR = Path("../tests") + +ENV_FILES = [ + Path("../envfiles/login.test.dev.local-it.cloud.env"), # authentik + # Path("../envfiles/blog.test.dev.local-it.cloud.env"), # wordpress +] + + +class TestRunner(Protocol): + def __init__(self, dotenv_path: Path, tests_dir: Path, session_id: str): + ... + + def run_tests(self): + ... + + +# Register all runners here. A .env file with TYPE=authentik will be ran with RunnerAuthentik +RUNNER_DICT: dict[str, type[TestRunner]] = { + "authentik": RunnerAuthentik, + "wordpress": RunnerWordpress, +} + + +class Wrapper: + def __init__(self, env_files: list[Path], session_id: str): + self.env_files = env_files + self.check_env_files(self.env_files) + self.session_id = session_id + + def setup_test(self): + self.dir_manager = DirManager(tests_dir=TESTS_DIR, session_id=self.session_id) + self.dir_manager.create_all_dirs() + + def run_test(self): + self.runners: list[TestRunner] = self.load_runners(self.env_files) + self.run_tests(self.runners) + + def load_runners(self, env_files: list[Path]) -> list[TestRunner]: + runners = [] + for env_file in env_files: + config: dict[str, str] = dotenv_values(env_file) + RunnerClass = RUNNER_DICT[config["TYPE"]] + runners.append(RunnerClass(dotenv_path=env_file, tests_dir=TESTS_DIR, session_id=self.session_id)) + return runners + + def run_tests(self, runners: list[TestRunner]): + for runner in runners: + runner.run_tests() + + @staticmethod + def check_env_files(env_files: list[Path]): + """checks if file exist for every file in list""" + for env_file in env_files: + assert env_file.is_file() + + @staticmethod + def get_session_id() -> str: + current_datetime = datetime.now() + return current_datetime.strftime("%Y-%m-%d-%H-%M-%S") + + +if __name__ == "__main__": + session_id = Wrapper.get_session_id() + wrapper = Wrapper(ENV_FILES, session_id=session_id) + wrapper.setup_test() + wrapper.run_test() diff --git a/src/runner.py b/src/runner.py new file mode 100644 index 0000000..9e0a09b --- /dev/null +++ b/src/runner.py @@ -0,0 +1,62 @@ +from pathlib import Path +from typing import Callable, Optional, TypedDict + +import pytest +from icecream import ic + + +class SubTest(TypedDict): + condition: Callable[[Path], bool] + test_file: str + + +class Runner: + test_dir_name: Optional[Path] = None + main_test_name: Optional[str] = None + sub_tests: list[SubTest] = [] + + def __init__(self, dotenv_path: Path, tests_dir: Path, session_id: str): + self.dotenv_path = dotenv_path + self.tests_dir = tests_dir + self.session_id = session_id + + ic(f"creating instance of {self.__class__.__name__}") + assert self.test_dir_name is not None + self.root_dir = Path(__file__).parent + + def _run_main_test(self): + if isinstance(self.main_test_name, str): + full_test_path = self.root_dir / self.test_dir_name / self.main_test_name + self._run_pytest(full_test_path) + + def _run_pytest(self, full_test_path: Path): + """runs pytest programmatically + + will run all tests in the file at full_test_path with some command line arguments""" + + ic(f"running test: {full_test_path}") + pytest.main( + [ + "-v", + "-rp", + str(full_test_path), + "--env_file", + str(self.dotenv_path), + "--tests_dir", + str(self.tests_dir), + "--session_id", + self.session_id, + ] + ) + + def show_files(self): + ic(list(self.root_dir.glob("*"))) + + def run_tests(self): + self._run_main_test() + for sub_test in self.sub_tests: + condition_function = sub_test["condition"] + if condition_function(self.dotenv_path): + test_name = sub_test["test_file"] + full_test_path = self.root_dir / self.test_dir_name / test_name + self._run_pytest(full_test_path) diff --git a/src/setup/setup_authentik.py b/src/setup/setup_authentik.py new file mode 100644 index 0000000..0c7307b --- /dev/null +++ b/src/setup/setup_authentik.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +import pytest +from icecream import ic +from playwright.sync_api import Browser, Locator, expect + +cred_file = Path("../credentials.json") +with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + +print(CREDENTIALS) + +TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} +TIMEOUT = 5000 + + +def check_for(locator: Locator): + expect(locator).to_be_visible(timeout=TIMEOUT) + + +@pytest.fixture(scope="session", autouse=True) +def admin_login(browser: Browser, dotenv_config, STATES): + # ic(dotenv_config) + + # go to page + context = browser.new_context() + context.set_default_timeout(TIMEOUT) + page = context.new_page() + url = "https://" + dotenv_config["DOMAIN"] + page.goto(url) + + # check welcome message + welcome_message = dotenv_config.get("welcome_message") + if welcome_message: + check_for(page.get_by_text(welcome_message)) + + # login + page.locator('input[name="uidField"]').fill(CREDENTIALS["admin"]) + page.locator('ak-stage-identification input[name="password"]').fill(CREDENTIALS["admin_pw"]) + page.get_by_role("button", name="Log In").click() + check_for(page.locator("ak-library")) + + # save state + context.storage_state(path=f"{STATES}/admin_state.json") + page.close() + context.close() diff --git a/src/tests_authentik/runner_authentik.py b/src/tests_authentik/runner_authentik.py new file mode 100644 index 0000000..1ef97b8 --- /dev/null +++ b/src/tests_authentik/runner_authentik.py @@ -0,0 +1,16 @@ +from pathlib import Path + +from runner import Runner, SubTest + + +def condition_always_true(dotenv_path: Path) -> bool: + return True + + +def condition_always_false(dotenv_path: Path) -> bool: + return False + + +class RunnerAuthentik(Runner): + test_dir_name = "tests_authentik" + main_test_name = "test_authentik_dummy.py" diff --git a/src/tests_authentik/test_authentik_dummy.py b/src/tests_authentik/test_authentik_dummy.py new file mode 100644 index 0000000..d20e1cf --- /dev/null +++ b/src/tests_authentik/test_authentik_dummy.py @@ -0,0 +1,2 @@ +def test_true(): + assert 1 + 1 == 2 diff --git a/src/tests_authentik/test_authentik_setup.py b/src/tests_authentik/test_authentik_setup.py new file mode 100644 index 0000000..36a22f3 --- /dev/null +++ b/src/tests_authentik/test_authentik_setup.py @@ -0,0 +1,179 @@ +## this file will not be used later +## split into +# -> setup.setup_authentic.py +# and +# -> tests + +import pytest +from icecream import ic +from playwright.sync_api import Browser, Locator, expect + +# playwright = sync_playwright().start() +# browser = playwright.chromium.launch(headless=False) + + +testuser = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} + + +TIMEOUT = 5000 + + +def check_for(locator: Locator): + expect(locator).to_be_visible(timeout=TIMEOUT) + + +def setup_context(browser, state_file=None): + if state_file: + context = browser.new_context(storage_state=state_file) + else: + context = browser.new_context() + context.set_default_timeout(TIMEOUT) + return context + + +""" Test Authentik Login and DE Locale """ + + +@pytest.fixture(scope="session", autouse=True) +def admin_login(browser: Browser, dotenv_config, STATES): + # ic(dotenv_config) + CONFIG = dotenv_config + context = setup_context(browser) + page = context.new_page() + url = "https://" + CONFIG["DOMAIN"] + ic(url) + page.goto(url) + welcome_message = CONFIG.get("welcome_message") + if welcome_message: + check_for(page.get_by_text(welcome_message)) + if CONFIG["locale"] == "de": + check_for(page.get_by_text("Benutzername oder Passwort vergessen?")) + check_for(page.get_by_text("E-Mail or Anmeldename")) + check_for(page.get_by_text("Passwort", exact=True)) + page.locator('input[name="uidField"]').fill(CONFIG["admin"]) + page.locator('ak-stage-identification input[name="password"]').fill(CONFIG["admin_pw"]) + page.get_by_role("button", name="Log In").click() + check_for(page.locator("ak-library")) + if CONFIG["locale"] == "de": + check_for(page.get_by_text("Meine Anwendungen")) + context.storage_state(path=f"{STATES}/admin_state.json") + page.close() + context.close() + + +""" Create User """ + + +@pytest.fixture(scope="session", autouse=True) +def init_create_user(browser: Browser, dotenv_config, STATES): + admin_context = setup_context(browser, f"{STATES}/admin_state.json") + admin_page = admin_context.new_page() + invitelink = create_invite_link(admin_page, dotenv_config) + admin_context.close() + user_context = setup_context(browser) + create_user(user_context, invitelink) + user_context.close() + + +""" Delete User """ + + +@pytest.fixture(scope="session", autouse=True) +def post_delete_user(browser: Browser, dotenv_config, RECORDS, STATES): + yield + context = browser.new_context(storage_state=f"{STATES}/admin_state.json") + context.tracing.start(screenshots=True, snapshots=True, sources=True) + context.set_default_timeout(TIMEOUT) + page = context.new_page() + # delete_nextcloud_user(page) + delete_authentik_user(page, dotenv_config) + context.tracing.stop(path=f"{RECORDS}/delete_user.zip") + + +""" Create Invite Link """ + + +def create_invite_link(page, dotenv_config): + CONFIG = dotenv_config + page.goto(CONFIG["domain"]) + page.get_by_role("link", name="Admin Interface").click() + page.get_by_text("Verzeichnis").click() + page.get_by_text("Benutzer").nth(2).click() + page.get_by_text("Einladungen").click() + page.get_by_role("button", name="Erstellen").first.click() + page.locator('input[name="name"]').click() + linkname = "testlink9433" + page.locator('input[name="name"]').fill(linkname) + page.get_by_placeholder("Wählen Sie ein Objekt aus.").click() + page.get_by_role("option", name="invitation-enrollment-flow invitation-enrollment-flow").click() + page.get_by_text("Erstellen", exact=True).first.click() + linklocator = page.get_by_role("rowgroup").filter(has=page.get_by_text(linkname)) + linklocator.locator(".fa-angle-down").click() + invitelink = linklocator.get_by_role("textbox").get_attribute(name="value") + return invitelink + + +""" Create User from invitelink """ + + +def create_user(context, invitelink, STATES): + page = context.new_page() + page.goto(invitelink) + page.get_by_placeholder("Benutzername").click() + page.get_by_placeholder("Benutzername").fill(testuser["username"]) + page.locator('input[name="name"]').click() + page.locator('input[name="name"]').fill(testuser["name"]) + page.locator('input[name="email"]').click() + page.locator('input[name="email"]').fill(testuser["email"]) + page.get_by_placeholder("Passwort", exact=True).click() + page.get_by_placeholder("Passwort", exact=True).fill(testuser["password"]) + page.get_by_placeholder("Passwort (wiederholen)").click() + page.get_by_placeholder("Passwort (wiederholen)").fill(testuser["password"]) + page.get_by_role("button", name="Weiter").click() + check_for(page.locator("ak-library")) + context.storage_state(path=f"{STATES}/user_state.json") + + +""" Delete Authentik Account """ + + +def delete_authentik_user(page, dotenv_config): + CONFIG = dotenv_config + page.goto(CONFIG["domain"]) + page.get_by_role("link", name="Admin Interface").click() + page.get_by_text("Verzeichnis").click() + page.get_by_text("Benutzer").nth(2).click() + page.get_by_role("row").filter(has=page.get_by_text(testuser["username"])).get_by_role("checkbox").click() + page.get_by_role("button", name="Löschen").click() + page.get_by_role("dialog").get_by_role("button", name="Löschen").click() + check_for(page.get_by_text("1 Benutzer erfolgreich gelöscht")) + + +""" Reuse Authentik Admin Session """ + + +@pytest.fixture +def admin_session(browser: Browser, dotenv_config, STATES): + CONFIG = dotenv_config + context = setup_context(browser, f"{STATES}/admin_state.json") + page = context.new_page() + page.goto(CONFIG["domain"]) + yield context, page + context.close() + + +""" Reuse Authentik User Session """ + + +@pytest.fixture +def user_session(browser: Browser, dotenv_config, STATES): + CONFIG = dotenv_config + context = setup_context(browser, f"{STATES}/user_state.json") + page = context.new_page() + page.goto(CONFIG["domain"]) + yield context, page + context.close() + + +def test_true(): + assert 1 + 1 == 2 diff --git a/src/tests_wordpress/conftest.py b/src/tests_wordpress/conftest.py new file mode 100644 index 0000000..d78bb85 --- /dev/null +++ b/src/tests_wordpress/conftest.py @@ -0,0 +1,28 @@ +# this conftest cannot be executed directly if there is a second conftest +# on a higher level. might work if other tests are executed though +# -> at least bad for debugging + +import json +from pathlib import Path + +import pytest +from playwright.sync_api import Browser, Locator, expect, sync_playwright + +# playwright = sync_playwright().start() +# browser = playwright.chromium.launch(headless=False) + + +cred_file = Path("../credentials.json") +with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + +print(CREDENTIALS) + +RECORDS = Path("records") +RECORDS.mkdir(exist_ok=True) +STATES = Path("states") +STATES.mkdir(exist_ok=True) + + +def test_dummy(): + assert 1 + 1 == 2 diff --git a/src/tests_wordpress/runner_wordpress.py b/src/tests_wordpress/runner_wordpress.py new file mode 100644 index 0000000..3d60aa2 --- /dev/null +++ b/src/tests_wordpress/runner_wordpress.py @@ -0,0 +1,20 @@ +from pathlib import Path + +from runner import Runner, SubTest + + +def condition_always_true(dotenv_path: Path) -> bool: + return True + + +def condition_always_false(dotenv_path: Path) -> bool: + return False + + +class RunnerWordpress(Runner): + test_dir_name = "tests_wordpress" + # main_test_name = "test_wordpress.py" + sub_tests = [ + SubTest(condition=condition_always_false, test_file="test_wordpress_feature1.py"), + SubTest(condition=condition_always_true, test_file="conftest.py"), + ] diff --git a/src/tests_wordpress/test_wordpress.py b/src/tests_wordpress/test_wordpress.py new file mode 100644 index 0000000..5d5b407 --- /dev/null +++ b/src/tests_wordpress/test_wordpress.py @@ -0,0 +1,29 @@ +import re + +import pytest +from playwright.sync_api import Page, expect + + +def test_one(): + assert 1 + 1 == 2 + + +def test_two(): + assert 2 + 1 == 3 + + +def test_has_title(page: Page): + page.goto("https://playwright.dev/") + + # Expect a title "to contain" a substring. + expect(page).to_have_title(re.compile("Playwright")) + + +def test_get_started_link(page: Page): + page.goto("https://playwright.dev/") + + # Click the get started link. + page.get_by_role("link", name="Get started").click() + + # Expects page to have a heading with the name of Installation. + expect(page.get_by_role("heading", name="Installation")).to_be_visible() diff --git a/src/tests_wordpress/test_wordpress_feature1.py b/src/tests_wordpress/test_wordpress_feature1.py new file mode 100644 index 0000000..865fd93 --- /dev/null +++ b/src/tests_wordpress/test_wordpress_feature1.py @@ -0,0 +1,26 @@ +import re + +from icecream import ic +from playwright.sync_api import Page, expect + + +def test_one(config): + ic(config) + assert 1 + 1 == 2 + + +def test_has_title(page: Page): + page.goto("https://playwright.dev/") + + # Expect a title "to contain" a substring. + expect(page).to_have_title(re.compile("Playwright")) + + +def test_get_started_link(page: Page): + page.goto("https://playwright.dev/") + + # Click the get started link. + page.get_by_role("link", name="Get started").click() + + # Expects page to have a heading with the name of Installation. + expect(page.get_by_role("heading", name="Installation")).to_be_visible() diff --git a/src/tests_wordpress/test_wordpress_localization.py b/src/tests_wordpress/test_wordpress_localization.py new file mode 100644 index 0000000..ca41ffe --- /dev/null +++ b/src/tests_wordpress/test_wordpress_localization.py @@ -0,0 +1,22 @@ +# WIP localization + +from playwright.sync_api import Page, expect + + +def test_has_title(page: Page): + page.goto("https://playwright.dev/") + + # Expect a title "to contain" a substring. + expect(page).to_have_title(re.compile("Playwright")) + + +def test_wordpress(admin_session): + context, page = admin_session + with page.expect_popup() as info: + page.get_by_role("link", name="Wordpress").click() + + wordpress = info.value + check_for(wordpress.locator("#wpcontent")) + if CONFIG["locale"] == "de": + check_for(wordpress.get_by_role("heading", name="Willkommen bei WordPress!")) + context.tracing.stop(path=f"{RECORDS}/wordpress.zip") From d2cd6ba47f21d0aedc5e73e0aa9566bde5754d4e Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 27 Nov 2023 17:01:45 +0100 Subject: [PATCH 02/18] authentik setup and tracing (#2) * authentik sessions created successfully during setup without breaking tracing * setup works on EN and DE localization by using regex patterns * automated tracing with pytest --trace option, manual hook no longer needed Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/2 Co-authored-by: Daniel Co-committed-by: Daniel --- .gitignore | 5 +- main.py | 37 ++++++ pyproject.toml | 3 + src/__init__.py | 0 src/blog.dev.local-it.cloud.env | 71 ---------- src/conftest.py | 61 +++++---- src/dirmanager.py | 48 ++++--- src/main.py | 76 ----------- src/runner.py | 85 +++++++++--- .../deprecated-setup_authentik-deprecated.py | 100 ++++++++++++++ src/setup/setup_authentik.py | 47 ------- src/tests_authentik/fixtures_authentik.py | 22 ++++ src/tests_authentik/plugin_authentik.py | 3 + src/tests_authentik/runner_authentik.py | 8 +- src/tests_authentik/setup_authentik.py | 122 ++++++++++++++++++ src/tests_authentik/test_authentik_dummy.py | 4 + ...thentik_setup.py => test_authentik_old.py} | 0 src/tests_wordpress/conftest.py | 29 +---- src/tests_wordpress/runner_wordpress.py | 7 +- .../test_wordpress_feature1.py | 33 +++-- src/utils.py | 7 + src/wrapper.py | 55 ++++++++ 22 files changed, 519 insertions(+), 304 deletions(-) create mode 100644 main.py create mode 100644 src/__init__.py delete mode 100644 src/blog.dev.local-it.cloud.env delete mode 100644 src/main.py create mode 100644 src/setup-deprecated/deprecated-setup_authentik-deprecated.py delete mode 100644 src/setup/setup_authentik.py create mode 100644 src/tests_authentik/fixtures_authentik.py create mode 100644 src/tests_authentik/plugin_authentik.py create mode 100644 src/tests_authentik/setup_authentik.py rename src/tests_authentik/{test_authentik_setup.py => test_authentik_old.py} (100%) create mode 100644 src/utils.py create mode 100644 src/wrapper.py diff --git a/.gitignore b/.gitignore index 4395fbc..46d7f87 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ __pycache__/ -*.json -*.zip +test-output/ TestResults/ .vscode/ *.pyc +*.json +*.zip credentials* \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..1461740 --- /dev/null +++ b/main.py @@ -0,0 +1,37 @@ +import json +import os +from pathlib import Path + +from src.utils import get_session_id +from src.wrapper import Wrapper + +# The env file list is the input to testing framework. each env file triggers +# the execution of one test Runner and provides configuration to the tests +# inside the runner. There can be dependencies, for example wordpress requires +# that authentik ran first to create the admin session and the user session. +# At the moment, wrong ordering results in unsuccessful test (wrong ordering +# would be wordpress env file is before authentik env file). +ENV_FILES = [ + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress +] + +OUTPUT_DIR = Path("./test-output").resolve() + + +# Set environment variables + +# os.environ["PWDEBUG"] = "1" + +cred_file = Path("credentials.json") +with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + +os.environ["ADMIN_USER"] = CREDENTIALS["admin_user"] +os.environ["ADMIN_PASS"] = CREDENTIALS["admin_pass"] + + +session_id = get_session_id() +wrapper = Wrapper(ENV_FILES, output_dir=OUTPUT_DIR, session_id=session_id) +wrapper.setup_test() +wrapper.run_test() diff --git a/pyproject.toml b/pyproject.toml index 4a80e86..d6f9e14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,6 @@ package-dir = {"" = "src"} [tool.ruff] line-length = 120 target-version = "py311" + +[tool.pytest.ini_options] +python_files = "test_*.py setup*.py" \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/blog.dev.local-it.cloud.env b/src/blog.dev.local-it.cloud.env deleted file mode 100644 index 600aaf6..0000000 --- a/src/blog.dev.local-it.cloud.env +++ /dev/null @@ -1,71 +0,0 @@ -################################################################################ -# DO NOT EDIT THIS FILE, IT IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN # -################################################################################ - -TYPE=wordpress -TIMEOUT=300 -ENABLE_AUTO_UPDATE=true -COMPOSE_FILE="compose.yml" - -# Setup Wordpress settings on each deploy: -#POST_DEPLOY_CMDS="app core_install" - -DOMAIN=blog.dev.local-it.cloud -## Domain aliases -#EXTRA_DOMAINS=', `www.blog.dev.local-it.cloud`' -LETS_ENCRYPT_ENV=production - -TITLE="My Example Blog" -LOCALE=de_DE -ADMIN_EMAIL=admin@example.com - -# Every new user is per default subscriber, uncomment to change it -DEFAULT_USER_ROLE=administrator - -#WORDPRESS_DEBUG=true - -## Additional extensions -#PHP_EXTENSIONS="calendar" - -SECRET_DB_ROOT_PASSWORD_VERSION=v1 -SECRET_DB_PASSWORD_VERSION=v1 - -# Mostly for compatibility with existing database dumps... -#WORDPRESS_TABLE_PREFIX=wp_ - -# Multisite -#WORDPRESS_CONFIG_EXTRA="\ -#define('WP_CACHE', false);\ -#define('WP_ALLOW_MULTISITE', true );" - -# Multisite phase 2 (see README) -#WORDPRESS_CONFIG_EXTRA="define('MULTISITE', true); define('SUBDOMAIN_INSTALL', true); define('DOMAIN_CURRENT_SITE', '${DOMAIN}'); define('PATH_CURRENT_SITE', '/'); define('SITE_ID_CURRENT_SITE', 1); define('BLOG_ID_CURRENT_SITE', 1); define('FORCE_SSL_ADMIN', true ); define('COOKIE_DOMAIN', \$_SERVER['HTTP_HOST']);" - -# Local SMTP relay -#COMPOSE_FILE="$COMPOSE_FILE:compose.mailrelay.yml" -SMTP_HOST=mail.local-it.org -MAIL_FROM=noreply@local-it.org - -# Remote SMTP relay -COMPOSE_FILE="$COMPOSE_FILE:compose.smtp.yml" -SMTP_HOST=mail.local-it.org -MAIL_FROM=noreply@local-it.org -SMTP_PORT=587 -SMTP_AUTH=on -SMTP_TLS=on -SECRET_SMTP_PASSWORD_VERSION=v1 - -# Authentik SSO -COMPOSE_FILE="$COMPOSE_FILE:compose.authentik.yml" -AUTHENTIK_DOMAIN=dev.local-it.cloud -SECRET_AUTHENTIK_SECRET_VERSION=v1 -SECRET_AUTHENTIK_ID_VERSION=v1 -LOGIN_TYPE='auto' - -# 🚩🚩 dangerous, use only for development sites! - -# Allow remote connections to db -#COMPOSE_FILE="$COMPOSE_FILE:compose.public-db.yml - -# Wide-open CORS -#CORS_ALLOW_ALL=1 diff --git a/src/conftest.py b/src/conftest.py index 5c0bcee..143dd86 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -5,16 +5,14 @@ # sys.path. It is thus good practise for projects to either put conftest.py under # a package scope or to never import anything from a conftest.py file. - from pathlib import Path import pytest -from dirmanager import DirManager from dotenv import dotenv_values -pytest_plugins = [ - "setup.setup_authentik", -] +from src.dirmanager import DirManager + +TIMEOUT = 5000 def pytest_addoption(parser): @@ -23,7 +21,7 @@ def pytest_addoption(parser): action="store", ) parser.addoption( - "--tests_dir", + "--output_dir", action="store", ) parser.addoption( @@ -33,32 +31,49 @@ def pytest_addoption(parser): @pytest.fixture(scope="session", autouse=True) -def dirmanager(request) -> DirManager: - tests_dir = request.config.getoption("--tests_dir") - tests_dir = Path(tests_dir) +def DIR(request) -> DirManager: + """Fixture holding test directories + + DIR.OUTPUT + DIR.SESSION + DIR.RECORDS + DIR.STATES + DIR.RESULTS + DIR.PROGRESS""" + + output_dir = request.config.getoption("--output_dir") + assert output_dir is not None, "required pytest command line argument not given" + output_dir = Path(output_dir) session_id = request.config.getoption("--session_id") - return DirManager(tests_dir=tests_dir, session_id=session_id) + assert session_id is not None, "required pytest command line argument not given" + dirmanager = DirManager(output_dir=output_dir, session_id=session_id) + dirmanager.create_all_dirs() + return dirmanager @pytest.fixture(scope="session", autouse=True) def dotenv_config(request) -> dict[str, str]: dotenv_path = request.config.getoption("--env_file") + assert dotenv_path is not None, "required pytest command line argument not given" dotenv_path = Path(dotenv_path) assert dotenv_path.is_file() - return dotenv_values(dotenv_path) + return dotenv_values(dotenv_path) # type: ignore -@pytest.fixture(scope="session", autouse=True) -def RECORDS(dirmanager) -> Path: - assert isinstance(dirmanager, DirManager) - return dirmanager.dirs["records"] +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + """saves traceback when test fails""" + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() -@pytest.fixture(scope="session", autouse=True) -def STATES(dirmanager) -> Path: - return dirmanager.dirs["states"] - - -@pytest.fixture(scope="session", autouse=True) -def RESULTS(dirmanager) -> Path: - return dirmanager.dirs["results"] + # we only look at actual failing test calls, not setup/teardown + if rep.when == "call" and rep.failed: + # saves traceback as .txt for failed test + filename = f"failed-{item.nodeid}.txt" + filename = filename.replace("/", "-") + filename = filename.replace("::", "-") + filepath = item.funcargs["DIR"].RESULTS / filename + with open(filepath, "a") as f: + f.write(rep.longreprtext + "\n") diff --git a/src/dirmanager.py b/src/dirmanager.py index dfc149d..e5066fa 100644 --- a/src/dirmanager.py +++ b/src/dirmanager.py @@ -8,37 +8,49 @@ class DirManager: The structures is as follows: tests dir/ session_dir-1/ + progress records - states results + states session_dir-2/ records ... """ - def __init__(self, tests_dir: Path, session_id: str): + def __init__(self, output_dir: Path | str, session_id: str): # root test dir - self.tests_dir = tests_dir + if isinstance(output_dir, str): + output_dir = Path(output_dir) + self._output_dir = output_dir.resolve() self.session_id = session_id - self.dirs = self._get_all_dirs() - def create_all_dirs(self): - self.create_dirs(self.tests_dir, exist_ok=True) - self.create_dirs(self.dirs) + self.create_dirs(self._output_dir, exist_ok=True) + self.create_dirs([self.SESSION, self.RECORDS, self.STATES, self.RESULTS, self.PROGRESS], exist_ok=True) - def _get_all_dirs(self): - dirs = {} - dirs["session"] = self.tests_dir / f"test-{self.session_id}" - dirs.update(self._get_subdirs(session_dir=dirs["session"])) - return dirs + @property + def OUTPUT(self): + return self._output_dir - def _get_subdirs(self, session_dir: Path): - return { - "records": session_dir / Path("records"), - "states": session_dir / Path("states"), - "results": session_dir / Path("results"), - } + @property + def SESSION(self): + return self._output_dir / f"test-{self.session_id}" + + @property + def RECORDS(self): + return self.SESSION / Path("records") + + @property + def STATES(self): + return self.SESSION / Path("states") + + @property + def RESULTS(self): + return self.SESSION / Path("results") + + @property + def PROGRESS(self): + return self.SESSION / Path("progress") @staticmethod def create_dirs(dirs: Path | list[Path] | dict[str, Path], exist_ok=False): diff --git a/src/main.py b/src/main.py deleted file mode 100644 index 3100621..0000000 --- a/src/main.py +++ /dev/null @@ -1,76 +0,0 @@ -from datetime import datetime -from pathlib import Path -from typing import Protocol - -from dirmanager import DirManager -from dotenv import dotenv_values -from icecream import ic -from tests_authentik.runner_authentik import RunnerAuthentik -from tests_wordpress.runner_wordpress import RunnerWordpress - -TESTS_DIR = Path("../tests") - -ENV_FILES = [ - Path("../envfiles/login.test.dev.local-it.cloud.env"), # authentik - # Path("../envfiles/blog.test.dev.local-it.cloud.env"), # wordpress -] - - -class TestRunner(Protocol): - def __init__(self, dotenv_path: Path, tests_dir: Path, session_id: str): - ... - - def run_tests(self): - ... - - -# Register all runners here. A .env file with TYPE=authentik will be ran with RunnerAuthentik -RUNNER_DICT: dict[str, type[TestRunner]] = { - "authentik": RunnerAuthentik, - "wordpress": RunnerWordpress, -} - - -class Wrapper: - def __init__(self, env_files: list[Path], session_id: str): - self.env_files = env_files - self.check_env_files(self.env_files) - self.session_id = session_id - - def setup_test(self): - self.dir_manager = DirManager(tests_dir=TESTS_DIR, session_id=self.session_id) - self.dir_manager.create_all_dirs() - - def run_test(self): - self.runners: list[TestRunner] = self.load_runners(self.env_files) - self.run_tests(self.runners) - - def load_runners(self, env_files: list[Path]) -> list[TestRunner]: - runners = [] - for env_file in env_files: - config: dict[str, str] = dotenv_values(env_file) - RunnerClass = RUNNER_DICT[config["TYPE"]] - runners.append(RunnerClass(dotenv_path=env_file, tests_dir=TESTS_DIR, session_id=self.session_id)) - return runners - - def run_tests(self, runners: list[TestRunner]): - for runner in runners: - runner.run_tests() - - @staticmethod - def check_env_files(env_files: list[Path]): - """checks if file exist for every file in list""" - for env_file in env_files: - assert env_file.is_file() - - @staticmethod - def get_session_id() -> str: - current_datetime = datetime.now() - return current_datetime.strftime("%Y-%m-%d-%H-%M-%S") - - -if __name__ == "__main__": - session_id = Wrapper.get_session_id() - wrapper = Wrapper(ENV_FILES, session_id=session_id) - wrapper.setup_test() - wrapper.run_test() diff --git a/src/runner.py b/src/runner.py index 9e0a09b..2aa814b 100644 --- a/src/runner.py +++ b/src/runner.py @@ -2,8 +2,11 @@ from pathlib import Path from typing import Callable, Optional, TypedDict import pytest +from dotenv import dotenv_values from icecream import ic +from src.dirmanager import DirManager + class SubTest(TypedDict): condition: Callable[[Path], bool] @@ -11,23 +14,31 @@ class SubTest(TypedDict): class Runner: - test_dir_name: Optional[Path] = None + name: Optional[str] = None + test_dir_name: Optional[str] = None + main_setup_name: Optional[str] = None main_test_name: Optional[str] = None sub_tests: list[SubTest] = [] + dependencies: list[str] = [] - def __init__(self, dotenv_path: Path, tests_dir: Path, session_id: str): + def __init__(self, dotenv_path: Path, output_dir: Path, session_id: str): self.dotenv_path = dotenv_path - self.tests_dir = tests_dir + self.config: dict[str, str] = dotenv_values(dotenv_path) # type: ignore + self.output_dir = output_dir self.session_id = session_id + self.DIRS = DirManager(output_dir, session_id) ic(f"creating instance of {self.__class__.__name__}") assert self.test_dir_name is not None self.root_dir = Path(__file__).parent def _run_main_test(self): + if isinstance(self.main_setup_name, str): + full_path = self.root_dir / self.test_dir_name / self.main_setup_name + self._run_pytest(full_path) if isinstance(self.main_test_name, str): - full_test_path = self.root_dir / self.test_dir_name / self.main_test_name - self._run_pytest(full_test_path) + full_path = self.root_dir / self.test_dir_name / self.main_test_name + self._run_pytest(full_path) def _run_pytest(self, full_test_path: Path): """runs pytest programmatically @@ -35,24 +46,45 @@ class Runner: will run all tests in the file at full_test_path with some command line arguments""" ic(f"running test: {full_test_path}") - pytest.main( - [ - "-v", - "-rp", - str(full_test_path), - "--env_file", - str(self.dotenv_path), - "--tests_dir", - str(self.tests_dir), - "--session_id", - self.session_id, - ] - ) - def show_files(self): - ic(list(self.root_dir.glob("*"))) + command_arguments = [] + + # command_arguments.append("-v") + # command_arguments.append("-rx") + command_arguments.append(str(full_test_path)) + + command_arguments.append("--env_file") + command_arguments.append(str(self.dotenv_path)) + + command_arguments.append("--output_dir") + command_arguments.append(str(self.DIRS.OUTPUT)) + + command_arguments.append("--session_id") + command_arguments.append(self.session_id) + + # artifacts dir + # warning: https://github.com/microsoft/playwright-pytest/issues/111 + # --output only works with the given context and page fixture + # folder needs to be unique! traces will not appear, if every pytest run has same output dir + output = self.DIRS.RESULTS / full_test_path.stem + command_arguments.append("--output") + command_arguments.append(str(output)) + + # tracing + command_arguments.append("--tracing") + command_arguments.append("retain-on-failure") + # command_arguments.append("on") + + # Disable capturing. With -s set, prints will go to console as if pytest is not there. + # command_arguments.append("-s") + + # headed + # command_arguments.append("--headed") + + pytest.main(command_arguments) def run_tests(self): + self._check_dependencies_finished() self._run_main_test() for sub_test in self.sub_tests: condition_function = sub_test["condition"] @@ -60,3 +92,16 @@ class Runner: test_name = sub_test["test_file"] full_test_path = self.root_dir / self.test_dir_name / test_name self._run_pytest(full_test_path) + self._create_progress_file() + + def _create_progress_file(self): + """create progress file to indicated finished test""" + file_path = self.DIRS.PROGRESS / self.name + with open(file_path, "w") as _: + pass # create empty file + + def _check_dependencies_finished(self): + """look for progress file of dependencies to confirm they have ran""" + finished_tests = [result.name for result in self.DIRS.PROGRESS.glob("*")] + for dependencie in self.dependencies: + assert dependencie in finished_tests diff --git a/src/setup-deprecated/deprecated-setup_authentik-deprecated.py b/src/setup-deprecated/deprecated-setup_authentik-deprecated.py new file mode 100644 index 0000000..4ac4fd5 --- /dev/null +++ b/src/setup-deprecated/deprecated-setup_authentik-deprecated.py @@ -0,0 +1,100 @@ +import json +from pathlib import Path + +import pytest +from icecream import ic +from playwright.sync_api import Browser, Locator, expect + +cred_file = Path("credentials.json") +with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + +print(CREDENTIALS) + +TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} +TIMEOUT = 5000 + + +def check_for(locator: Locator): + expect(locator).to_be_visible(timeout=TIMEOUT) + + +# todo: why is this a fixture? to get dotenv_config +@pytest.fixture(scope="session", autouse=True) +def create_admin_login(browser: Browser, dotenv_config, STATES): + # ic(dotenv_config) + + # go to page + context = browser.new_context() + context.set_default_timeout(TIMEOUT) + page = context.new_page() + url = "https://" + dotenv_config["DOMAIN"] + page.goto(url) + + # check welcome message + welcome_message = dotenv_config.get("welcome_message") + if welcome_message: + check_for(page.get_by_text(welcome_message)) + + # login + page.locator('input[name="uidField"]').fill(CREDENTIALS["admin"]) + page.locator('ak-stage-identification input[name="password"]').fill(CREDENTIALS["admin_pw"]) + page.get_by_role("button", name="Log In").click() + check_for(page.locator("ak-library")) + + # save state + context.storage_state(path=f"{STATES}/admin_state.json") + page.close() + context.close() + + +def create_invite_link(page): + url = "https://" + dotenv_config["DOMAIN"] + page.goto(url) + page.get_by_role("link", name="Admin Interface").click() + page.get_by_text("Verzeichnis").click() + page.get_by_text("Benutzer").nth(2).click() + page.get_by_text("Einladungen").click() + page.get_by_role("button", name="Erstellen").first.click() + page.locator('input[name="name"]').click() + linkname = "testlink9433" + page.locator('input[name="name"]').fill(linkname) + page.get_by_placeholder("Wählen Sie ein Objekt aus.").click() + page.get_by_role("option", name="invitation-enrollment-flow invitation-enrollment-flow").click() + page.get_by_text("Erstellen", exact=True).first.click() + linklocator = page.get_by_role("rowgroup").filter(has=page.get_by_text(linkname)) + linklocator.locator(".fa-angle-down").click() + invitelink = linklocator.get_by_role("textbox").get_attribute(name="value") + return invitelink + + +def create_user(context, invitelink): + page = context.new_page() + page.goto(invitelink) + page.get_by_placeholder("Benutzername").click() + page.get_by_placeholder("Benutzername").fill(testuser["username"]) + page.locator('input[name="name"]').click() + page.locator('input[name="name"]').fill(testuser["name"]) + page.locator('input[name="email"]').click() + page.locator('input[name="email"]').fill(testuser["email"]) + page.get_by_placeholder("Passwort", exact=True).click() + page.get_by_placeholder("Passwort", exact=True).fill(testuser["password"]) + page.get_by_placeholder("Passwort (wiederholen)").click() + page.get_by_placeholder("Passwort (wiederholen)").fill(testuser["password"]) + page.get_by_role("button", name="Weiter").click() + check_for(page.locator("ak-library")) + context.storage_state(path=f"{STATES}/user_state.json") + + +@pytest.fixture(scope="session", autouse=True) +def create_user_session(browser: Browser, admin_login): + admin_context = browser.new_context(storage_state=f"{STATES}/admin_state.json") + # admin_context = setup_context(browser, f"{STATES}/admin_state.json") + admin_page = admin_context.new_page() + invitelink = create_invite_link(admin_page) + admin_context.tracing.stop(path=f"{RECORDS}/create_invite_link.zip") + admin_context.close() + user_context = setup_context(browser) + create_user(user_context, invitelink) + user_context.tracing.stop(path=f"{RECORDS}/create_user.zip") + user_context.close() diff --git a/src/setup/setup_authentik.py b/src/setup/setup_authentik.py deleted file mode 100644 index 0c7307b..0000000 --- a/src/setup/setup_authentik.py +++ /dev/null @@ -1,47 +0,0 @@ -import json -from pathlib import Path - -import pytest -from icecream import ic -from playwright.sync_api import Browser, Locator, expect - -cred_file = Path("../credentials.json") -with open(cred_file, "r") as f: - CREDENTIALS = json.load(f) - -print(CREDENTIALS) - -TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} -TIMEOUT = 5000 - - -def check_for(locator: Locator): - expect(locator).to_be_visible(timeout=TIMEOUT) - - -@pytest.fixture(scope="session", autouse=True) -def admin_login(browser: Browser, dotenv_config, STATES): - # ic(dotenv_config) - - # go to page - context = browser.new_context() - context.set_default_timeout(TIMEOUT) - page = context.new_page() - url = "https://" + dotenv_config["DOMAIN"] - page.goto(url) - - # check welcome message - welcome_message = dotenv_config.get("welcome_message") - if welcome_message: - check_for(page.get_by_text(welcome_message)) - - # login - page.locator('input[name="uidField"]').fill(CREDENTIALS["admin"]) - page.locator('ak-stage-identification input[name="password"]').fill(CREDENTIALS["admin_pw"]) - page.get_by_role("button", name="Log In").click() - check_for(page.locator("ak-library")) - - # save state - context.storage_state(path=f"{STATES}/admin_state.json") - page.close() - context.close() diff --git a/src/tests_authentik/fixtures_authentik.py b/src/tests_authentik/fixtures_authentik.py new file mode 100644 index 0000000..bcd106d --- /dev/null +++ b/src/tests_authentik/fixtures_authentik.py @@ -0,0 +1,22 @@ +import json + +import pytest +from playwright.sync_api import BrowserContext + +from src.dirmanager import DirManager + + +@pytest.fixture +def admin_session(context: BrowserContext, DIR: DirManager) -> BrowserContext: + state_file = DIR.STATES / "admin_state.json" + storage_state = json.loads(state_file.read_bytes()) + context.add_cookies(storage_state["cookies"]) + return context + + +@pytest.fixture +def user_session(context: BrowserContext, DIR: DirManager) -> BrowserContext: + state_file = DIR.STATES / "user_state.json" + storage_state = json.loads(state_file.read_bytes()) + context.add_cookies(storage_state["cookies"]) + return context diff --git a/src/tests_authentik/plugin_authentik.py b/src/tests_authentik/plugin_authentik.py new file mode 100644 index 0000000..14f31c9 --- /dev/null +++ b/src/tests_authentik/plugin_authentik.py @@ -0,0 +1,3 @@ +# will be loaded in conftest.py +# will provide context for other tests (wordpress etc.) +# that depend on authentik (which is all of them) diff --git a/src/tests_authentik/runner_authentik.py b/src/tests_authentik/runner_authentik.py index 1ef97b8..fbdf44c 100644 --- a/src/tests_authentik/runner_authentik.py +++ b/src/tests_authentik/runner_authentik.py @@ -1,6 +1,8 @@ from pathlib import Path -from runner import Runner, SubTest +from src.runner import Runner, SubTest + +# from src.tests_authentik.setup_authentik import setup_authentik def condition_always_true(dotenv_path: Path) -> bool: @@ -12,5 +14,7 @@ def condition_always_false(dotenv_path: Path) -> bool: class RunnerAuthentik(Runner): + name = "authentik" test_dir_name = "tests_authentik" - main_test_name = "test_authentik_dummy.py" + main_setup_name = "setup_authentik.py" + # main_test_name = "test_authentik_dummy.py" diff --git a/src/tests_authentik/setup_authentik.py b/src/tests_authentik/setup_authentik.py new file mode 100644 index 0000000..51706a3 --- /dev/null +++ b/src/tests_authentik/setup_authentik.py @@ -0,0 +1,122 @@ +import json +import os +import re + +from icecream import ic +from playwright.sync_api import BrowserContext, expect + +from src.dirmanager import DirManager + +ADMIN_USER = os.environ["ADMIN_USER"] +ADMIN_PASS = os.environ["ADMIN_PASS"] + + +TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} +TIMEOUT = 10000 + + +def test_create_admin_login(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): + # go to page + page = context.new_page() + url = "https://" + dotenv_config["DOMAIN"] + page.goto(url) + + # check welcome message + welcome_message = dotenv_config.get("welcome_message") + if welcome_message: + expect(page.get_by_text(welcome_message)).to_be_visible() + + # login + page.locator('input[name="uidField"]').fill(ADMIN_USER) + page.locator('ak-stage-identification input[name="password"]').fill(ADMIN_PASS) + page.get_by_role("button", name="Log In").click() + expect(page.locator("ak-library")).to_be_visible() + + # save state + context.storage_state(path=f"{DIR.STATES}/admin_state.json") + + +def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, str]): + # go to admin page + page = admin_context.new_page() + url = "https://" + dotenv_config["DOMAIN"] + page.goto(url) + page.get_by_role("link", name="Admin Interface").click() + nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) + nav.click() + nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() + result = page.get_by_text(TESTUSER["username"]).is_visible(timeout=TIMEOUT) + return result + + +def create_invite_link(admin_context: BrowserContext, dotenv_config: dict[str, str]): + # go to admin page + page = admin_context.new_page() + url = "https://" + dotenv_config["DOMAIN"] + page.goto(url) + page.get_by_role("link", name="Admin Interface").click() + + nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) + nav.click() + nav.get_by_role("link", name=re.compile(r"Invitations|Einladungen")).click() + + # todo: only works if no links have been created yet (empty list) + page.get_by_role("cell", name=re.compile(r"Keine Objekte|objects")).get_by_role( + "button" + ).click() # todo: confirm "objects" for en lang + + page.locator('input[name="name"]').click() + linkname = "test_link_123" + page.locator('input[name="name"]').fill(linkname) + page.get_by_placeholder("Wählen Sie ein Objekt aus.").click() + page.get_by_role("option", name=re.compile(r"invitation-enrollment-flow")).click() + + # force, because else we get "intercepts pointer events" + page.locator("footer").locator("ak-spinner-button").first.click(force=True) + + linklocator = page.get_by_role("rowgroup").filter(has=page.get_by_text(linkname)) + linklocator.locator(".fa-angle-down").click() + # page.get_by_text(linkname).click() + invitelink = linklocator.get_by_role("textbox").get_attribute(name="value") + return invitelink + + +def create_user(user_context: BrowserContext, invitelink): + # warning: only works on german site + page = user_context.new_page() + page.goto(invitelink) + page.get_by_placeholder("Benutzername").click() + page.get_by_placeholder("Benutzername").fill(TESTUSER["username"]) + page.locator('input[name="name"]').click() + page.locator('input[name="name"]').fill(TESTUSER["name"]) + page.locator('input[name="email"]').click() + page.locator('input[name="email"]').fill(TESTUSER["email"]) + page.get_by_placeholder("Passwort", exact=True).click() + page.get_by_placeholder("Passwort", exact=True).fill(TESTUSER["password"]) + page.get_by_placeholder("Passwort (wiederholen)").click() + page.get_by_placeholder("Passwort (wiederholen)").fill(TESTUSER["password"]) + page.get_by_role("button", name="Weiter").click() + expect(page.locator("ak-library")).to_be_visible() + + +def test_create_user_session(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): + context.set_default_timeout(TIMEOUT) + + # load admin cookies + state_file = DIR.STATES / "admin_state.json" + storage_state = json.loads(state_file.read_bytes()) + context.add_cookies(storage_state["cookies"]) + + if check_if_user_exists(context, dotenv_config): + # just login with user + pass + context.clear_cookies() + else: + ## create user + # create invite_link + invite_link = create_invite_link(context, dotenv_config) + # create user + context.clear_cookies() + create_user(context, invite_link) + + context.storage_state(path=f"{DIR.STATES}/user_state.json") diff --git a/src/tests_authentik/test_authentik_dummy.py b/src/tests_authentik/test_authentik_dummy.py index d20e1cf..bddb561 100644 --- a/src/tests_authentik/test_authentik_dummy.py +++ b/src/tests_authentik/test_authentik_dummy.py @@ -1,2 +1,6 @@ def test_true(): assert 1 + 1 == 2 + + +def test_not_true(): + assert 1 + 1 == 3 diff --git a/src/tests_authentik/test_authentik_setup.py b/src/tests_authentik/test_authentik_old.py similarity index 100% rename from src/tests_authentik/test_authentik_setup.py rename to src/tests_authentik/test_authentik_old.py diff --git a/src/tests_wordpress/conftest.py b/src/tests_wordpress/conftest.py index d78bb85..dcd6fde 100644 --- a/src/tests_wordpress/conftest.py +++ b/src/tests_wordpress/conftest.py @@ -1,28 +1 @@ -# this conftest cannot be executed directly if there is a second conftest -# on a higher level. might work if other tests are executed though -# -> at least bad for debugging - -import json -from pathlib import Path - -import pytest -from playwright.sync_api import Browser, Locator, expect, sync_playwright - -# playwright = sync_playwright().start() -# browser = playwright.chromium.launch(headless=False) - - -cred_file = Path("../credentials.json") -with open(cred_file, "r") as f: - CREDENTIALS = json.load(f) - -print(CREDENTIALS) - -RECORDS = Path("records") -RECORDS.mkdir(exist_ok=True) -STATES = Path("states") -STATES.mkdir(exist_ok=True) - - -def test_dummy(): - assert 1 + 1 == 2 +from src.tests_authentik.fixtures_authentik import admin_session, user_session diff --git a/src/tests_wordpress/runner_wordpress.py b/src/tests_wordpress/runner_wordpress.py index 3d60aa2..de9e501 100644 --- a/src/tests_wordpress/runner_wordpress.py +++ b/src/tests_wordpress/runner_wordpress.py @@ -1,6 +1,6 @@ from pathlib import Path -from runner import Runner, SubTest +from src.runner import Runner, SubTest def condition_always_true(dotenv_path: Path) -> bool: @@ -12,9 +12,10 @@ def condition_always_false(dotenv_path: Path) -> bool: class RunnerWordpress(Runner): + name = "wordpress" test_dir_name = "tests_wordpress" # main_test_name = "test_wordpress.py" sub_tests = [ - SubTest(condition=condition_always_false, test_file="test_wordpress_feature1.py"), - SubTest(condition=condition_always_true, test_file="conftest.py"), + SubTest(condition=condition_always_true, test_file="test_wordpress_feature1.py"), ] + dependencies: list[str] = ["authentik"] diff --git a/src/tests_wordpress/test_wordpress_feature1.py b/src/tests_wordpress/test_wordpress_feature1.py index 865fd93..7f81d8c 100644 --- a/src/tests_wordpress/test_wordpress_feature1.py +++ b/src/tests_wordpress/test_wordpress_feature1.py @@ -1,26 +1,31 @@ import re from icecream import ic -from playwright.sync_api import Page, expect +from playwright.sync_api import BrowserContext, Page, expect -def test_one(config): - ic(config) +def test_demo(admin_session: BrowserContext): + admin_session.new_page() assert 1 + 1 == 2 -def test_has_title(page: Page): - page.goto("https://playwright.dev/") - - # Expect a title "to contain" a substring. - expect(page).to_have_title(re.compile("Playwright")) +# def test_one(config): +# ic(config) +# assert 1 + 1 == 2 -def test_get_started_link(page: Page): - page.goto("https://playwright.dev/") +# def test_has_title(page: Page): +# page.goto("https://playwright.dev/") - # Click the get started link. - page.get_by_role("link", name="Get started").click() +# # Expect a title "to contain" a substring. +# expect(page).to_have_title(re.compile("Playwright")) - # Expects page to have a heading with the name of Installation. - expect(page.get_by_role("heading", name="Installation")).to_be_visible() + +# def test_get_started_link(page: Page): +# page.goto("https://playwright.dev/") + +# # Click the get started link. +# page.get_by_role("link", name="Get started").click() + +# # Expects page to have a heading with the name of Installation. +# expect(page.get_by_role("heading", name="Installation")).to_be_visible() diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..3f96269 --- /dev/null +++ b/src/utils.py @@ -0,0 +1,7 @@ +from datetime import datetime + + +@staticmethod +def get_session_id() -> str: + current_datetime = datetime.now() + return current_datetime.strftime("%Y-%m-%d-%H-%M-%S") diff --git a/src/wrapper.py b/src/wrapper.py new file mode 100644 index 0000000..161900d --- /dev/null +++ b/src/wrapper.py @@ -0,0 +1,55 @@ +import os +from pathlib import Path +from typing import Protocol + +from dotenv import dotenv_values + +from src.dirmanager import DirManager +from src.tests_authentik.runner_authentik import RunnerAuthentik +from src.tests_wordpress.runner_wordpress import RunnerWordpress + + +class TestRunner(Protocol): + def __init__(self, dotenv_path: Path, output_dir: Path, session_id: str): + ... + + def run_tests(self): + ... + + +# Register all runners here. A .env file with TYPE=authentik will be ran with RunnerAuthentik +RUNNER_DICT: dict[str, type[TestRunner]] = { + "authentik": RunnerAuthentik, + "wordpress": RunnerWordpress, +} + + +class Wrapper: + def __init__(self, env_files: list[Path], output_dir: Path, session_id: str): + self.env_files = env_files + self.check_env_files(self.env_files) + self.output_dir = output_dir + self.session_id = session_id + + def setup_test(self): + self.dir_manager = DirManager(output_dir=self.output_dir, session_id=self.session_id) + self.dir_manager.create_all_dirs() + + def run_test(self): + self.runners: list[TestRunner] = self._load_runners(self.env_files) + for runner in self.runners: + runner.run_tests() + + def _load_runners(self, env_files: list[Path]) -> list[TestRunner]: + runners = [] + for env_file in env_files: + config: dict[str, str] = dotenv_values(env_file) # type: ignore + RunnerClass = RUNNER_DICT[config["TYPE"]] + runners.append(RunnerClass(dotenv_path=env_file, output_dir=self.output_dir, session_id=self.session_id)) + return runners + + @staticmethod + def check_env_files(env_files: list[Path]): + """checks if file exist for every file in list""" + for env_file in env_files: + assert env_file.is_file() From 8172f685de0adba01b147e935e16fba50e7edc59 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 29 Nov 2023 14:14:46 +0100 Subject: [PATCH 03/18] rework-output-and-test-logic (#3) * fix flakey tests in authentik / wordpress * make it possible to rerun tests partially -> passed will be skipped, failed will be repeated * improve organization of all outputs (moving, renaming, keeping multiple versions etc.) * add html reports, replace .txt tracebacks * combine all html reports into one * add demo runner with comments for documentation purposes Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/3 Co-authored-by: Daniel Co-committed-by: Daniel --- Dockerfile | 2 +- README.md | 12 +- main.py | 57 +++-- previous-work/wordpress_test.py | 22 +- requirements.txt | 4 +- src/conftest.py | 31 +-- src/coordinator.py | 99 +++++++++ src/dirmanager.py | 7 +- src/html_helper.py | 196 ++++++++++++++++++ src/runner.py | 125 ++++++++--- src/tests_authentik/fixtures_authentik.py | 8 +- src/tests_authentik/runner_authentik.py | 8 +- src/tests_authentik/setup_authentik.py | 14 +- src/tests_authentik/test_authentik_old.py | 179 ---------------- src/tests_demo/fixtures_demo.py | 26 +++ src/tests_demo/runner_demo.py | 29 +++ src/tests_demo/setup_demo.py | 3 + src/tests_wordpress/conftest.py | 2 +- src/tests_wordpress/runner_wordpress.py | 21 +- src/tests_wordpress/test_wordpress.py | 33 +-- .../test_wordpress_feature1.py | 31 --- .../test_wordpress_localization.py | 27 +-- src/utils.py | 15 +- src/wrapper.py | 55 ----- 24 files changed, 588 insertions(+), 418 deletions(-) create mode 100644 src/coordinator.py create mode 100644 src/html_helper.py delete mode 100644 src/tests_authentik/test_authentik_old.py create mode 100644 src/tests_demo/fixtures_demo.py create mode 100644 src/tests_demo/runner_demo.py create mode 100644 src/tests_demo/setup_demo.py delete mode 100644 src/tests_wordpress/test_wordpress_feature1.py delete mode 100644 src/wrapper.py diff --git a/Dockerfile b/Dockerfile index 662da0f..8a7efdd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,4 +9,4 @@ RUN playwright install-deps COPY ./requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt -WORKDIR /code/src \ No newline at end of file +WORKDIR /code \ No newline at end of file diff --git a/README.md b/README.md index acf5183..a4ffa00 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,14 @@ -# Readme +# Clone + +To clone with submodules, use these git commands: + +```bash +git clone --recurse-submodules +git submodule update --init // add submodule after normal cloning +git submodule update --remote // update submodules +``` + +# Run ```bash docker compose build diff --git a/main.py b/main.py index 1461740..f71cc25 100644 --- a/main.py +++ b/main.py @@ -2,27 +2,42 @@ import json import os from pathlib import Path -from src.utils import get_session_id -from src.wrapper import Wrapper +from loguru import logger + +from src.coordinator import Coordinator +from src.dirmanager import DirManager +from src.utils import get_session_id + +# ----------------------------- lookup env files ----------------------------- # + + +# This list of env files is the input to testing framework. each env file +# triggers the execution of one test Runner and provides configuration to the +# tests inside the runner. There can be dependencies, for example wordpress +# requires that authentik ran first to create the admin session and the user +# session. At the moment, wrong ordering results in unsuccessful test +# (wrong ordering would be wordpress env file is before authentik env file). -# The env file list is the input to testing framework. each env file triggers -# the execution of one test Runner and provides configuration to the tests -# inside the runner. There can be dependencies, for example wordpress requires -# that authentik ran first to create the admin session and the user session. -# At the moment, wrong ordering results in unsuccessful test (wrong ordering -# would be wordpress env file is before authentik env file). ENV_FILES = [ Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress ] + +# ----------------------------- define ouptut dir ---------------------------- # + + OUTPUT_DIR = Path("./test-output").resolve() -# Set environment variables +# -------------------------- enable playwright debug ------------------------- # + # os.environ["PWDEBUG"] = "1" + +# --------------------- load credentials to env variables -------------------- # + cred_file = Path("credentials.json") with open(cred_file, "r") as f: CREDENTIALS = json.load(f) @@ -31,7 +46,25 @@ os.environ["ADMIN_USER"] = CREDENTIALS["admin_user"] os.environ["ADMIN_PASS"] = CREDENTIALS["admin_pass"] +# ----------------------------- define session_id ---------------------------- # + + session_id = get_session_id() -wrapper = Wrapper(ENV_FILES, output_dir=OUTPUT_DIR, session_id=session_id) -wrapper.setup_test() -wrapper.run_test() +# session_id = "abc" + + +# ------------------------------- setup logging ------------------------------ # + +DIR = DirManager(output_dir=OUTPUT_DIR, session_id=session_id) +log_file = DIR.RESULTS / "full.log" +logger.add(log_file) + + +# ---------------------------- initialize and run ---------------------------- # + + +coordinator = Coordinator(ENV_FILES, output_dir=OUTPUT_DIR, session_id=session_id) +coordinator.setup_test() +coordinator.run_test() +coordinator.combine_html() +coordinator.collect_traces() diff --git a/previous-work/wordpress_test.py b/previous-work/wordpress_test.py index c58e4af..1d65311 100644 --- a/previous-work/wordpress_test.py +++ b/previous-work/wordpress_test.py @@ -1,14 +1,14 @@ -from conftest import CONFIG, check_for, RECORDS +from playwright.sync_api import BrowserContext, expect -""" Test Wordpress """ -def test_wordpress(admin_session): - context, page = admin_session - with page.expect_popup() as info: - page.get_by_role("link", name="Wordpress").click() +from src.dirmanager import DirManager - wordpress = info.value - check_for(wordpress.locator("#wpcontent")) - if CONFIG['locale'] == 'de': - check_for(wordpress.get_by_role("heading", name="Willkommen bei WordPress!")) - context.tracing.stop(path=f"{RECORDS}/wordpress.zip") +def test_wordpress(admin_session: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): + page_authentik = admin_session.new_page() + with page_authentik.expect_popup() as event_context: + page_authentik.get_by_role("link", name="Wordpress").click() + page_wordpress = event_context.value + + expect(page_wordpress.locator("#wpcontent")).to_be_visible() + if "locale" in dotenv_config and "de" in dotenv_config["locale"]: + expect(page_wordpress.get_by_role("heading")).to_have_text("Willkommen bei WordPress!") diff --git a/requirements.txt b/requirements.txt index 66c69c3..f1577e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ pytest pytest-playwright python-dotenv -icecream \ No newline at end of file +icecream +loguru +beautifulsoup4 \ No newline at end of file diff --git a/src/conftest.py b/src/conftest.py index 143dd86..3eb168c 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -9,24 +9,28 @@ from pathlib import Path import pytest from dotenv import dotenv_values +from pytest import Parser from src.dirmanager import DirManager TIMEOUT = 5000 -def pytest_addoption(parser): +def pytest_addoption(parser: Parser): parser.addoption( "--env_file", action="store", + required=True, ) parser.addoption( "--output_dir", action="store", + required=True, ) parser.addoption( "--session_id", action="store", + required=True, ) @@ -38,14 +42,11 @@ def DIR(request) -> DirManager: DIR.SESSION DIR.RECORDS DIR.STATES - DIR.RESULTS - DIR.PROGRESS""" + DIR.RESULTS""" output_dir = request.config.getoption("--output_dir") - assert output_dir is not None, "required pytest command line argument not given" output_dir = Path(output_dir) session_id = request.config.getoption("--session_id") - assert session_id is not None, "required pytest command line argument not given" dirmanager = DirManager(output_dir=output_dir, session_id=session_id) dirmanager.create_all_dirs() return dirmanager @@ -54,26 +55,6 @@ def DIR(request) -> DirManager: @pytest.fixture(scope="session", autouse=True) def dotenv_config(request) -> dict[str, str]: dotenv_path = request.config.getoption("--env_file") - assert dotenv_path is not None, "required pytest command line argument not given" dotenv_path = Path(dotenv_path) assert dotenv_path.is_file() return dotenv_values(dotenv_path) # type: ignore - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - """saves traceback when test fails""" - - # execute all other hooks to obtain the report object - outcome = yield - rep = outcome.get_result() - - # we only look at actual failing test calls, not setup/teardown - if rep.when == "call" and rep.failed: - # saves traceback as .txt for failed test - filename = f"failed-{item.nodeid}.txt" - filename = filename.replace("/", "-") - filename = filename.replace("::", "-") - filepath = item.funcargs["DIR"].RESULTS / filename - with open(filepath, "a") as f: - f.write(rep.longreprtext + "\n") diff --git a/src/coordinator.py b/src/coordinator.py new file mode 100644 index 0000000..b466e1c --- /dev/null +++ b/src/coordinator.py @@ -0,0 +1,99 @@ +import shutil +from pathlib import Path + +from dotenv import dotenv_values +from loguru import logger + +from src.dirmanager import DirManager +from src.html_helper import merge_html_files +from src.runner import Runner +from src.tests_authentik.runner_authentik import RunnerAuthentik +from src.tests_wordpress.runner_wordpress import RunnerWordpress +from src.utils import rmtree + +# Register all runners here. Each .env file with TYPE=authentik will be ran with RunnerAuthentik +RUNNER_DICT: dict[str, type[Runner]] = { + "authentik": RunnerAuthentik, + "wordpress": RunnerWordpress, +} + + +class Coordinator: + def __init__(self, env_paths_list: list[Path], output_dir: Path, session_id: str): + out_string = "".join([e.name + "\n" for e in env_paths_list]) + out_string += f"output_dir = {output_dir}\n" + out_string += f"session_id = {session_id}" + logger.info(f"initialize Coordinator instance with\nenv_paths_list =\n{out_string}") + + self.DIR = DirManager(output_dir=output_dir, session_id=session_id) + self.output_dir = output_dir + self.session_id = session_id + + self.env_paths: dict[str, Path] = dict() + self.env_configs: dict[str, dict[str, str]] = dict() # todo: needed? + self._parse_env_files(env_paths_list) + + def _parse_env_files(self, env_paths: list[Path]): + for env_path in env_paths: + assert env_path.is_file(), f"the env file {env_path} does not exist" + config: dict[str, str] = dotenv_values(env_path) # type: ignore + assert "TYPE" in config, f"the env file {env_path} does not specify the required TYPE key." + env_type = config["TYPE"] + self.env_paths[env_type] = env_path + self.env_configs[env_type] = config # todo: needed? + + def setup_test(self): + logger.info("calling setup_test()") + self.DIR.create_all_dirs() + self._copy_env_files() + + def _copy_env_files(self): + """Copies all env filesto STATES/env_files. Files will be renamed to their own TYPE value.""" + env_files_dir = self.DIR.STATES / "env_files" + env_files_dir.mkdir(exist_ok=True) + for type_key, env_path in self.env_paths.items(): + shutil.copy(env_path, env_files_dir / type_key) + + def run_test(self): + logger.info("calling run_test()") + self.runners: list[Runner] = self._load_runners(self.env_paths.values()) + for runner in self.runners: + runner.run_tests() + logger.info("run_test() finished") + + def _load_runners(self, env_files: list[Path]) -> list[Runner]: + runners = [] + for env_file in env_files: + config: dict[str, str] = dotenv_values(env_file) # type: ignore + RunnerClass = RUNNER_DICT[config["TYPE"]] + runners.append(RunnerClass(dotenv_path=env_file, output_dir=self.output_dir, session_id=self.session_id)) + return runners + + def combine_html(self): + in_path = str(self.DIR.RECORDS / "html") + out_path = str(self.DIR.RECORDS / "full-report.html") + title = "combined.html" + merge_html_files(in_path, out_path, title) + + def collect_traces(self): + """moves all traces into SESSION/RECORDS dir + + if tests are rerun and generate another trace, the new trace will get a unique name such as + tracename-0 + tracename-1 + ... + """ + + def get_new_path(root_dir: Path, base_name: str, index=0) -> Path: + new_name_alt = base_name + f"-{index}" + if not (root_dir / new_name_alt).is_dir(): + return root_dir / new_name_alt + else: + index += 1 + return get_new_path(root_dir, base_name, index=index) + + trace_root_dir = self.DIR.RECORDS / "traces" + for f in trace_root_dir.rglob("*/trace.zip"): + new_path = get_new_path(self.DIR.RECORDS, f.parent.name) + f.parent.rename(new_path) + rmtree(trace_root_dir) diff --git a/src/dirmanager.py b/src/dirmanager.py index e5066fa..41dcfaa 100644 --- a/src/dirmanager.py +++ b/src/dirmanager.py @@ -8,7 +8,6 @@ class DirManager: The structures is as follows: tests dir/ session_dir-1/ - progress records results states @@ -26,7 +25,7 @@ class DirManager: def create_all_dirs(self): self.create_dirs(self._output_dir, exist_ok=True) - self.create_dirs([self.SESSION, self.RECORDS, self.STATES, self.RESULTS, self.PROGRESS], exist_ok=True) + self.create_dirs([self.SESSION, self.RECORDS, self.RECORDS / "html", self.STATES, self.RESULTS], exist_ok=True) @property def OUTPUT(self): @@ -48,10 +47,6 @@ class DirManager: def RESULTS(self): return self.SESSION / Path("results") - @property - def PROGRESS(self): - return self.SESSION / Path("progress") - @staticmethod def create_dirs(dirs: Path | list[Path] | dict[str, Path], exist_ok=False): match dirs: diff --git a/src/html_helper.py b/src/html_helper.py new file mode 100644 index 0000000..8f94b67 --- /dev/null +++ b/src/html_helper.py @@ -0,0 +1,196 @@ +# code from +# https://github.com/akavbathen/pytest_html_merger/tree/main + +import json +import os +import pathlib +import re + +from bs4 import BeautifulSoup +from packaging import version + +CHECKBOX_REGEX = r"^(?P0|[1-9]\d*) (?P.*)" + + +def merge_html_files(in_path: str, out_path: str, title: str): + paths = get_html_files(in_path, out_path) + if not paths: + raise RuntimeError(f"Unable to find html files in {in_path}") + + assets_dir_path = get_assets_path(in_path) + + first_file = BeautifulSoup("".join(open(paths[0])), features="html.parser") + paths.pop(0) + + try: + first_file.find("link").decompose() + except: + pass + + if assets_dir_path is None: + print( + f"Will assume css is embedded in the reports. If this is not the case, " + f"Please make sure that you have 'assets' directory inside {in_path} " + f"which contains css files generated by pytest-html." + ) + else: + with open(os.path.join(assets_dir_path, "style.css"), "r") as f: + content = f.read() + + head = first_file.head + head.append(first_file.new_tag("style", type="text/css")) + head.style.append(content) + + h = first_file.find("h1") + h.string = title or os.path.basename(out_path) + + ps = first_file.find_all("p") + pytest_version = ps[0].text.split(" ")[-1] + ps.pop(0) + + cb_types = { + "passed": [0, ""], + "skipped": [0, ""], + "failed": [0, ""], + "error": [0, ""], + "xfailed": [0, ""], + "xpassed": [0, ""], + } + + html_ver = version.parse(pytest_version) + if html_ver >= version.parse("4.0.0rc"): + cb_types["rerun"] = [0, ""] + + for cb_type in cb_types: + cb_val = get_checkbox_value(first_file, cb_type) + cb_types[cb_type][0] = cb_val[0] + cb_types[cb_type][1] = cb_val[1] + + dur, test_count, fp = get_test_count_and_duration(ps, html_ver) + + if html_ver < version.parse("4.0.0rc"): + t = first_file.find("table", {"id": "results-table"}) + else: + f_json_blob = first_file.find("div", {"id": "data-container"}).get("data-jsonblob") + # Convert the JSON string into a dictionary + f_data_dict = json.loads(f_json_blob) + + for path in paths: + cur_file = BeautifulSoup("".join(open(path)), features="html.parser") + + if html_ver < version.parse("4.0.0rc"): + tbody_res = cur_file.find_all("tbody", {"class": "results-table-row"}) + for elm in tbody_res: + t.append(elm) + else: + f_json_blob = cur_file.find("div", {"id": "data-container"}).get("data-jsonblob") + # Convert the JSON string into a dictionary + c_data_dict = json.loads(f_json_blob) + + f_data_dict["tests"].update(c_data_dict["tests"]) + + p_res = cur_file.find_all("p") + _dur, _test_count, _ = get_test_count_and_duration(p_res, html_ver) + dur += _dur + test_count += _test_count + + for cb_type in cb_types: + tmp = get_checkbox_value(cur_file, cb_type) + cb_types[cb_type][0] += tmp[0] + + fp.string = f"{test_count} tests ran in {dur} seconds" + + if html_ver >= version.parse("4.0.0rc"): + first_file.find("div", {"id": "data-container"})["data-jsonblob"] = json.dumps(f_data_dict) + + for cb_type in cb_types: + set_checkbox_value(first_file, cb_type, cb_types[cb_type]) + + with open(out_path, "w") as f: + f.write(str(first_file)) + + +def get_test_count_and_duration(ps, html_ver): + test_count = 0 + dur = 0 + fp = None + + for p in ps: + if html_ver >= version.parse("4.0.0"): + match = re.search(r"test.* took ", p.text) + if match: + tmp = p.text.split(" ") + test_count = int(tmp[0]) + + if "ms." in tmp: + dur = int(tmp[3]) / 1000 + else: + hours, minutes, seconds = map(int, tmp[3][:-1].split(":")) + dur = hours * 3600 + minutes * 60 + seconds + + fp = p + + break + + if html_ver < version.parse("4.0.0"): + if " tests ran" in p.text: + tmp = p.text.split(" ") + test_count = int(tmp[0]) + dur = float(tmp[4]) + fp = p + + break + + return dur, test_count, fp + + +def set_checkbox_value(root_soap, cb_type, val): + elem = root_soap.find("span", {"class": cb_type}) + match = re.search(CHECKBOX_REGEX, elem.text) + if match is None: + raise RuntimeError(f"{cb_type} not found") + + elem.string = f"{val[0]} {val[1]}" + + elem = root_soap.find("input", {"data-test-result": cb_type}) + if val[0] != 0: + del elem["disabled"] + del elem["hidden"] + + +def get_checkbox_value(root_soap, cb_type): + elem = root_soap.find("span", {"class": cb_type}) + match = re.search(CHECKBOX_REGEX, elem.text) + if match is None: + raise RuntimeError(f"{cb_type} not found") + + gdict = match.groupdict() + + return int(gdict["num"]), gdict["txt1"] + + +def get_html_files(path, output_file_path): + onlyfiles = [] + output_file_path = os.path.abspath(output_file_path) + + for p in pathlib.Path(path).rglob("*.html"): + print(p) + res = str(p.absolute()) + if output_file_path in res: + print("damn") + continue + + tmp = BeautifulSoup("".join(open(res)), features="html.parser") + p = tmp.find("p") + if p and "Report generated on " in p.text: + onlyfiles.append(res) + + return sorted(onlyfiles, reverse=True) + + +def get_assets_path(path): + res = None + for p in pathlib.Path(path).rglob("assets"): + return str(p.absolute()) + + return res diff --git a/src/runner.py b/src/runner.py index 2aa814b..d232334 100644 --- a/src/runner.py +++ b/src/runner.py @@ -3,23 +3,24 @@ from typing import Callable, Optional, TypedDict import pytest from dotenv import dotenv_values -from icecream import ic +from loguru import logger from src.dirmanager import DirManager class SubTest(TypedDict): - condition: Callable[[Path], bool] + condition: Callable[[dict[str, str]], bool] test_file: str class Runner: - name: Optional[str] = None - test_dir_name: Optional[str] = None + name: str = "" + test_dir_name: str = "" main_setup_name: Optional[str] = None main_test_name: Optional[str] = None + dependencies: list[type["Runner"]] = [] sub_tests: list[SubTest] = [] - dependencies: list[str] = [] + prevent_skip = False def __init__(self, dotenv_path: Path, output_dir: Path, session_id: str): self.dotenv_path = dotenv_path @@ -28,24 +29,54 @@ class Runner: self.session_id = session_id self.DIRS = DirManager(output_dir, session_id) - ic(f"creating instance of {self.__class__.__name__}") - assert self.test_dir_name is not None + logger.info(f"creating instance of {self.__class__.__name__}") + assert self.test_dir_name self.root_dir = Path(__file__).parent - def _run_main_test(self): + def _run_main_setup_and_test(self): if isinstance(self.main_setup_name, str): - full_path = self.root_dir / self.test_dir_name / self.main_setup_name - self._run_pytest(full_path) + self._run_test_if_required( + identifier_string=self.combine_names(self.name, self.main_setup_name), + test_path=self.root_dir / self.test_dir_name / self.main_setup_name, + ) + if isinstance(self.main_test_name, str): - full_path = self.root_dir / self.test_dir_name / self.main_test_name - self._run_pytest(full_path) + self._run_test_if_required( + identifier_string=self.combine_names(self.name, self.main_test_name), + test_path=self.root_dir / self.test_dir_name / self.main_test_name, + ) - def _run_pytest(self, full_test_path: Path): - """runs pytest programmatically + def _run_test_if_required(self, identifier_string: str, test_path: Path): + if not self.prevent_skip and self._test_already_passed(identifier_string, remove_existing=True): + logger.info(f"skipping {identifier_string}") + else: + logger.info(f"running {identifier_string}") + result = self._call_pytest(test_path) + self._create_result_file(result=result, identifier_string=identifier_string) - will run all tests in the file at full_test_path with some command line arguments""" + def _test_already_passed(self, identifier_string: str, remove_existing: bool = False) -> bool: + """returns True if the selected test (matching test_name + sub_test_name) already passed - ic(f"running test: {full_test_path}") + This is determined by the presence of a specific output file in the RESULTS folder that + matches identifier_string + + remove_existing: If True, result files matching test_name + sub_test_name with a status + other than 'passed' will be deleted""" + + already_passed = False + for result in self.DIRS.RESULTS.glob("*"): + if identifier_string in result.name: + # process any result file (passed / failed / skipped) if it exists + if "passed" in result.name: + already_passed = True + elif remove_existing: + result.unlink() + return already_passed + + def _call_pytest(self, full_test_path: Path) -> int: + """runs pytest programmatically on a specific file + + all tests in the file [full_test_path] will be run along with command line arguments""" command_arguments = [] @@ -66,9 +97,8 @@ class Runner: # warning: https://github.com/microsoft/playwright-pytest/issues/111 # --output only works with the given context and page fixture # folder needs to be unique! traces will not appear, if every pytest run has same output dir - output = self.DIRS.RESULTS / full_test_path.stem command_arguments.append("--output") - command_arguments.append(str(output)) + command_arguments.append(str(self.DIRS.RECORDS / "traces" / full_test_path.stem)) # tracing command_arguments.append("--tracing") @@ -81,27 +111,56 @@ class Runner: # headed # command_arguments.append("--headed") - pytest.main(command_arguments) + # html report. Will be combined into one file later. + command_arguments.append(f"--html={self.DIRS.RECORDS / 'html' / full_test_path.with_suffix('.html').name}") + + return pytest.main(command_arguments) def run_tests(self): - self._check_dependencies_finished() - self._run_main_test() + self._assert_dependencies_passed() + self._run_main_setup_and_test() for sub_test in self.sub_tests: condition_function = sub_test["condition"] - if condition_function(self.dotenv_path): - test_name = sub_test["test_file"] - full_test_path = self.root_dir / self.test_dir_name / test_name - self._run_pytest(full_test_path) - self._create_progress_file() + sub_test_name = sub_test["test_file"] + identifier_string = self.combine_names(self.name, sub_test_name) + if condition_function(self.config): + test_path = self.root_dir / self.test_dir_name / sub_test_name + self._run_test_if_required(identifier_string=identifier_string, test_path=test_path) + else: + self._create_result_file(result=-1, identifier_string=identifier_string) - def _create_progress_file(self): - """create progress file to indicated finished test""" - file_path = self.DIRS.PROGRESS / self.name + def _create_result_file( + self, + result: int, + identifier_string: str, + ): + """create result file to indicated passed/failed or skipped test""" + + full_name = self.combine_names(self.result_int_to_str(result), identifier_string) + file_path = self.DIRS.RESULTS / full_name with open(file_path, "w") as _: pass # create empty file - def _check_dependencies_finished(self): - """look for progress file of dependencies to confirm they have ran""" - finished_tests = [result.name for result in self.DIRS.PROGRESS.glob("*")] + @staticmethod + def result_int_to_str(result_int: int) -> str: + match result_int: + case -1: + return "skipped" + case 0: + return "passed" + case _: + return "failed" + + @staticmethod + def combine_names(*names: str) -> str: + return "-".join(names) + + def _assert_dependencies_passed(self): + """assert that all dependencie setups passed before""" + + passed_tests = [r.name for r in self.DIRS.RESULTS.glob("*") if "passed" in r.name] for dependencie in self.dependencies: - assert dependencie in finished_tests + dependencie_identifier = self.combine_names(dependencie.name, dependencie.main_setup_name) + assert any( + dependencie_identifier in f for f in passed_tests + ), f"could not run {self.name} because {dependencie} did not run before" diff --git a/src/tests_authentik/fixtures_authentik.py b/src/tests_authentik/fixtures_authentik.py index bcd106d..dab4e42 100644 --- a/src/tests_authentik/fixtures_authentik.py +++ b/src/tests_authentik/fixtures_authentik.py @@ -5,18 +5,22 @@ from playwright.sync_api import BrowserContext from src.dirmanager import DirManager +TIMEOUT = 5000 + @pytest.fixture -def admin_session(context: BrowserContext, DIR: DirManager) -> BrowserContext: +def admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "admin_state.json" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) + context.set_default_timeout(TIMEOUT) return context @pytest.fixture -def user_session(context: BrowserContext, DIR: DirManager) -> BrowserContext: +def user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "user_state.json" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) + context.set_default_timeout(TIMEOUT) return context diff --git a/src/tests_authentik/runner_authentik.py b/src/tests_authentik/runner_authentik.py index fbdf44c..59c486e 100644 --- a/src/tests_authentik/runner_authentik.py +++ b/src/tests_authentik/runner_authentik.py @@ -1,15 +1,11 @@ -from pathlib import Path - from src.runner import Runner, SubTest -# from src.tests_authentik.setup_authentik import setup_authentik - -def condition_always_true(dotenv_path: Path) -> bool: +def condition_always_true(dotenv_config: dict[str, str]) -> bool: return True -def condition_always_false(dotenv_path: Path) -> bool: +def condition_always_false(dotenv_config: dict[str, str]) -> bool: return False diff --git a/src/tests_authentik/setup_authentik.py b/src/tests_authentik/setup_authentik.py index 51706a3..831a137 100644 --- a/src/tests_authentik/setup_authentik.py +++ b/src/tests_authentik/setup_authentik.py @@ -2,7 +2,6 @@ import json import os import re -from icecream import ic from playwright.sync_api import BrowserContext, expect from src.dirmanager import DirManager @@ -10,13 +9,15 @@ from src.dirmanager import DirManager ADMIN_USER = os.environ["ADMIN_USER"] ADMIN_PASS = os.environ["ADMIN_PASS"] - +LOCALE = {"Accept-Language": "de_DE"} TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} -TIMEOUT = 10000 +TIMEOUT = 6000 def test_create_admin_login(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): # go to page + context.set_extra_http_headers(LOCALE) + context.set_default_timeout(TIMEOUT) page = context.new_page() url = "https://" + dotenv_config["DOMAIN"] page.goto(url) @@ -45,8 +46,10 @@ def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) nav.click() nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() - result = page.get_by_text(TESTUSER["username"]).is_visible(timeout=TIMEOUT) - return result + + user = page.get_by_text(TESTUSER["username"]) + user.wait_for(state="visible") + return user.is_visible() def create_invite_link(admin_context: BrowserContext, dotenv_config: dict[str, str]): @@ -100,6 +103,7 @@ def create_user(user_context: BrowserContext, invitelink): def test_create_user_session(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): + context.set_extra_http_headers(LOCALE) context.set_default_timeout(TIMEOUT) # load admin cookies diff --git a/src/tests_authentik/test_authentik_old.py b/src/tests_authentik/test_authentik_old.py deleted file mode 100644 index 36a22f3..0000000 --- a/src/tests_authentik/test_authentik_old.py +++ /dev/null @@ -1,179 +0,0 @@ -## this file will not be used later -## split into -# -> setup.setup_authentic.py -# and -# -> tests - -import pytest -from icecream import ic -from playwright.sync_api import Browser, Locator, expect - -# playwright = sync_playwright().start() -# browser = playwright.chromium.launch(headless=False) - - -testuser = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} - - -TIMEOUT = 5000 - - -def check_for(locator: Locator): - expect(locator).to_be_visible(timeout=TIMEOUT) - - -def setup_context(browser, state_file=None): - if state_file: - context = browser.new_context(storage_state=state_file) - else: - context = browser.new_context() - context.set_default_timeout(TIMEOUT) - return context - - -""" Test Authentik Login and DE Locale """ - - -@pytest.fixture(scope="session", autouse=True) -def admin_login(browser: Browser, dotenv_config, STATES): - # ic(dotenv_config) - CONFIG = dotenv_config - context = setup_context(browser) - page = context.new_page() - url = "https://" + CONFIG["DOMAIN"] - ic(url) - page.goto(url) - welcome_message = CONFIG.get("welcome_message") - if welcome_message: - check_for(page.get_by_text(welcome_message)) - if CONFIG["locale"] == "de": - check_for(page.get_by_text("Benutzername oder Passwort vergessen?")) - check_for(page.get_by_text("E-Mail or Anmeldename")) - check_for(page.get_by_text("Passwort", exact=True)) - page.locator('input[name="uidField"]').fill(CONFIG["admin"]) - page.locator('ak-stage-identification input[name="password"]').fill(CONFIG["admin_pw"]) - page.get_by_role("button", name="Log In").click() - check_for(page.locator("ak-library")) - if CONFIG["locale"] == "de": - check_for(page.get_by_text("Meine Anwendungen")) - context.storage_state(path=f"{STATES}/admin_state.json") - page.close() - context.close() - - -""" Create User """ - - -@pytest.fixture(scope="session", autouse=True) -def init_create_user(browser: Browser, dotenv_config, STATES): - admin_context = setup_context(browser, f"{STATES}/admin_state.json") - admin_page = admin_context.new_page() - invitelink = create_invite_link(admin_page, dotenv_config) - admin_context.close() - user_context = setup_context(browser) - create_user(user_context, invitelink) - user_context.close() - - -""" Delete User """ - - -@pytest.fixture(scope="session", autouse=True) -def post_delete_user(browser: Browser, dotenv_config, RECORDS, STATES): - yield - context = browser.new_context(storage_state=f"{STATES}/admin_state.json") - context.tracing.start(screenshots=True, snapshots=True, sources=True) - context.set_default_timeout(TIMEOUT) - page = context.new_page() - # delete_nextcloud_user(page) - delete_authentik_user(page, dotenv_config) - context.tracing.stop(path=f"{RECORDS}/delete_user.zip") - - -""" Create Invite Link """ - - -def create_invite_link(page, dotenv_config): - CONFIG = dotenv_config - page.goto(CONFIG["domain"]) - page.get_by_role("link", name="Admin Interface").click() - page.get_by_text("Verzeichnis").click() - page.get_by_text("Benutzer").nth(2).click() - page.get_by_text("Einladungen").click() - page.get_by_role("button", name="Erstellen").first.click() - page.locator('input[name="name"]').click() - linkname = "testlink9433" - page.locator('input[name="name"]').fill(linkname) - page.get_by_placeholder("Wählen Sie ein Objekt aus.").click() - page.get_by_role("option", name="invitation-enrollment-flow invitation-enrollment-flow").click() - page.get_by_text("Erstellen", exact=True).first.click() - linklocator = page.get_by_role("rowgroup").filter(has=page.get_by_text(linkname)) - linklocator.locator(".fa-angle-down").click() - invitelink = linklocator.get_by_role("textbox").get_attribute(name="value") - return invitelink - - -""" Create User from invitelink """ - - -def create_user(context, invitelink, STATES): - page = context.new_page() - page.goto(invitelink) - page.get_by_placeholder("Benutzername").click() - page.get_by_placeholder("Benutzername").fill(testuser["username"]) - page.locator('input[name="name"]').click() - page.locator('input[name="name"]').fill(testuser["name"]) - page.locator('input[name="email"]').click() - page.locator('input[name="email"]').fill(testuser["email"]) - page.get_by_placeholder("Passwort", exact=True).click() - page.get_by_placeholder("Passwort", exact=True).fill(testuser["password"]) - page.get_by_placeholder("Passwort (wiederholen)").click() - page.get_by_placeholder("Passwort (wiederholen)").fill(testuser["password"]) - page.get_by_role("button", name="Weiter").click() - check_for(page.locator("ak-library")) - context.storage_state(path=f"{STATES}/user_state.json") - - -""" Delete Authentik Account """ - - -def delete_authentik_user(page, dotenv_config): - CONFIG = dotenv_config - page.goto(CONFIG["domain"]) - page.get_by_role("link", name="Admin Interface").click() - page.get_by_text("Verzeichnis").click() - page.get_by_text("Benutzer").nth(2).click() - page.get_by_role("row").filter(has=page.get_by_text(testuser["username"])).get_by_role("checkbox").click() - page.get_by_role("button", name="Löschen").click() - page.get_by_role("dialog").get_by_role("button", name="Löschen").click() - check_for(page.get_by_text("1 Benutzer erfolgreich gelöscht")) - - -""" Reuse Authentik Admin Session """ - - -@pytest.fixture -def admin_session(browser: Browser, dotenv_config, STATES): - CONFIG = dotenv_config - context = setup_context(browser, f"{STATES}/admin_state.json") - page = context.new_page() - page.goto(CONFIG["domain"]) - yield context, page - context.close() - - -""" Reuse Authentik User Session """ - - -@pytest.fixture -def user_session(browser: Browser, dotenv_config, STATES): - CONFIG = dotenv_config - context = setup_context(browser, f"{STATES}/user_state.json") - page = context.new_page() - page.goto(CONFIG["domain"]) - yield context, page - context.close() - - -def test_true(): - assert 1 + 1 == 2 diff --git a/src/tests_demo/fixtures_demo.py b/src/tests_demo/fixtures_demo.py new file mode 100644 index 0000000..f4a3919 --- /dev/null +++ b/src/tests_demo/fixtures_demo.py @@ -0,0 +1,26 @@ +""" +This file can be used to define fixtures thate are then used by other tests which +depend on [demo]. For this to work + +1. the Runner class of the other test needs to define the depencency as seen + by referencing RunnerDemo in the dependencies list: + +from src.tests_demo.runner_demo import RunnerDemo + +class RunnerOther(Runner): + dependencies = [RunnerDemo] + + +2. the specific tests that rely on these fixtures need to import the fixtures. + To globally import for all tests in 'other', the import should be done in conftest: + +in 'conftest.py' in 'test_other' dir: +from src.tests_demo.fixtures_demo import demo_fixture +""" + +import pytest + + +@pytest.fixture +def demo_fixture(): + return "" diff --git a/src/tests_demo/runner_demo.py b/src/tests_demo/runner_demo.py new file mode 100644 index 0000000..8f36e9c --- /dev/null +++ b/src/tests_demo/runner_demo.py @@ -0,0 +1,29 @@ +from typing import Optional + +from src.runner import Runner, SubTest +from src.tests_authentik.runner_authentik import RunnerAuthentik + + +class RunnerDemo(Runner): + """Every env file has a corresponding runner class""" + + name: str = "demo" # name of the test, used for logging / output naming + test_dir_name: str = "tests_demo" # dir name holding all tests related to RunnerDemo + + # Filename of Demo setup. If defined, it will run 1st by executing pytest + main_setup_name: Optional[str] = "setup_demo.py" + + # Filename of Demo test. This file contains unconditional tests that will be run in any + # case. If defined, it will run 2nd by executing pytest + main_test_name: Optional[str] = None + + # this indicates that tests from RunnerDemo depend on the setup from RunnerAuthentik. + # RunnerDemo will only execute, when setup_authentik.py has finished successfully. + # For example, setup_authentik.py generates session states, that can be used as fixtures + # that can be loaded from fixtures_authentik.py + dependencies: list[type["Runner"]] = [RunnerAuthentik] + + # this list can hold many more tests from RunnerDemo that run conditional. The condition + # and the test file can be defined by creating a SubTest instance: + # SubTest(condition: Callable, test_file: str) + sub_tests: list[SubTest] = [] diff --git a/src/tests_demo/setup_demo.py b/src/tests_demo/setup_demo.py new file mode 100644 index 0000000..d46a9d6 --- /dev/null +++ b/src/tests_demo/setup_demo.py @@ -0,0 +1,3 @@ +# Define functions here that are specifically meant for setup, not for testing. This means +# all actions that simply are required for other tests from 'demo' to run. Runs before all +# tests from 'demo'. diff --git a/src/tests_wordpress/conftest.py b/src/tests_wordpress/conftest.py index dcd6fde..c7a791d 100644 --- a/src/tests_wordpress/conftest.py +++ b/src/tests_wordpress/conftest.py @@ -1 +1 @@ -from src.tests_authentik.fixtures_authentik import admin_session, user_session +from src.tests_authentik.fixtures_authentik import admin_context, user_context diff --git a/src/tests_wordpress/runner_wordpress.py b/src/tests_wordpress/runner_wordpress.py index de9e501..4355f22 100644 --- a/src/tests_wordpress/runner_wordpress.py +++ b/src/tests_wordpress/runner_wordpress.py @@ -1,21 +1,28 @@ -from pathlib import Path - from src.runner import Runner, SubTest +from src.tests_authentik.runner_authentik import RunnerAuthentik -def condition_always_true(dotenv_path: Path) -> bool: +def condition_always_true(dotenv_config: dict[str, str]) -> bool: return True -def condition_always_false(dotenv_path: Path) -> bool: +def condition_always_false(dotenv_config: dict[str, str]) -> bool: + return False + + +def condition_has_locale(dotenv_config: dict[str, str]) -> bool: + if "LOCALE" in dotenv_config: + if "de" in dotenv_config["LOCALE"]: + return True return False class RunnerWordpress(Runner): name = "wordpress" test_dir_name = "tests_wordpress" - # main_test_name = "test_wordpress.py" + main_test_name = "test_wordpress.py" + dependencies: list[type[Runner]] = [RunnerAuthentik] sub_tests = [ - SubTest(condition=condition_always_true, test_file="test_wordpress_feature1.py"), + SubTest(condition=condition_has_locale, test_file="test_wordpress_localization.py"), ] - dependencies: list[str] = ["authentik"] + prevent_skip = True diff --git a/src/tests_wordpress/test_wordpress.py b/src/tests_wordpress/test_wordpress.py index 5d5b407..6c85114 100644 --- a/src/tests_wordpress/test_wordpress.py +++ b/src/tests_wordpress/test_wordpress.py @@ -1,29 +1,14 @@ import re -import pytest -from playwright.sync_api import Page, expect +from playwright.sync_api import BrowserContext, expect + +from src.dirmanager import DirManager -def test_one(): - assert 1 + 1 == 2 +def test_visit_from_authentik(admin_context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): + page_authentik = admin_context.new_page() + with page_authentik.expect_popup() as event_context: + page_authentik.get_by_role("link", name="Wordpress").click() + page_wordpress = event_context.value - -def test_two(): - assert 2 + 1 == 3 - - -def test_has_title(page: Page): - page.goto("https://playwright.dev/") - - # Expect a title "to contain" a substring. - expect(page).to_have_title(re.compile("Playwright")) - - -def test_get_started_link(page: Page): - page.goto("https://playwright.dev/") - - # Click the get started link. - page.get_by_role("link", name="Get started").click() - - # Expects page to have a heading with the name of Installation. - expect(page.get_by_role("heading", name="Installation")).to_be_visible() + expect(page_wordpress.locator("#wpcontent")).to_be_visible() diff --git a/src/tests_wordpress/test_wordpress_feature1.py b/src/tests_wordpress/test_wordpress_feature1.py deleted file mode 100644 index 7f81d8c..0000000 --- a/src/tests_wordpress/test_wordpress_feature1.py +++ /dev/null @@ -1,31 +0,0 @@ -import re - -from icecream import ic -from playwright.sync_api import BrowserContext, Page, expect - - -def test_demo(admin_session: BrowserContext): - admin_session.new_page() - assert 1 + 1 == 2 - - -# def test_one(config): -# ic(config) -# assert 1 + 1 == 2 - - -# def test_has_title(page: Page): -# page.goto("https://playwright.dev/") - -# # Expect a title "to contain" a substring. -# expect(page).to_have_title(re.compile("Playwright")) - - -# def test_get_started_link(page: Page): -# page.goto("https://playwright.dev/") - -# # Click the get started link. -# page.get_by_role("link", name="Get started").click() - -# # Expects page to have a heading with the name of Installation. -# expect(page.get_by_role("heading", name="Installation")).to_be_visible() diff --git a/src/tests_wordpress/test_wordpress_localization.py b/src/tests_wordpress/test_wordpress_localization.py index ca41ffe..a3a8a91 100644 --- a/src/tests_wordpress/test_wordpress_localization.py +++ b/src/tests_wordpress/test_wordpress_localization.py @@ -1,22 +1,15 @@ # WIP localization -from playwright.sync_api import Page, expect +from playwright.sync_api import BrowserContext, expect + +from src.dirmanager import DirManager -def test_has_title(page: Page): - page.goto("https://playwright.dev/") +def test_welcome_message(user_context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): + page = user_context.new_page() + url = "https://" + dotenv_config["DOMAIN"] + page.goto(url) - # Expect a title "to contain" a substring. - expect(page).to_have_title(re.compile("Playwright")) - - -def test_wordpress(admin_session): - context, page = admin_session - with page.expect_popup() as info: - page.get_by_role("link", name="Wordpress").click() - - wordpress = info.value - check_for(wordpress.locator("#wpcontent")) - if CONFIG["locale"] == "de": - check_for(wordpress.get_by_role("heading", name="Willkommen bei WordPress!")) - context.tracing.stop(path=f"{RECORDS}/wordpress.zip") + expect(page.locator("#wpcontent")).to_be_visible() + if "locale" in dotenv_config and "de" in dotenv_config["locale"]: + expect(page.get_by_role("heading")).to_have_text("Willkommen bei WordPress!") diff --git a/src/utils.py b/src/utils.py index 3f96269..92f7b26 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,7 +1,20 @@ from datetime import datetime +from pathlib import Path -@staticmethod def get_session_id() -> str: current_datetime = datetime.now() return current_datetime.strftime("%Y-%m-%d-%H-%M-%S") + + +def rmtree(root_dir: Path): + """removes a folder with content recursively""" + if not root_dir.is_dir(): + return + for child in root_dir.iterdir(): + if child.is_dir(): + rmtree(child) + else: + child.unlink() + + root_dir.rmdir() diff --git a/src/wrapper.py b/src/wrapper.py deleted file mode 100644 index 161900d..0000000 --- a/src/wrapper.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -from pathlib import Path -from typing import Protocol - -from dotenv import dotenv_values - -from src.dirmanager import DirManager -from src.tests_authentik.runner_authentik import RunnerAuthentik -from src.tests_wordpress.runner_wordpress import RunnerWordpress - - -class TestRunner(Protocol): - def __init__(self, dotenv_path: Path, output_dir: Path, session_id: str): - ... - - def run_tests(self): - ... - - -# Register all runners here. A .env file with TYPE=authentik will be ran with RunnerAuthentik -RUNNER_DICT: dict[str, type[TestRunner]] = { - "authentik": RunnerAuthentik, - "wordpress": RunnerWordpress, -} - - -class Wrapper: - def __init__(self, env_files: list[Path], output_dir: Path, session_id: str): - self.env_files = env_files - self.check_env_files(self.env_files) - self.output_dir = output_dir - self.session_id = session_id - - def setup_test(self): - self.dir_manager = DirManager(output_dir=self.output_dir, session_id=self.session_id) - self.dir_manager.create_all_dirs() - - def run_test(self): - self.runners: list[TestRunner] = self._load_runners(self.env_files) - for runner in self.runners: - runner.run_tests() - - def _load_runners(self, env_files: list[Path]) -> list[TestRunner]: - runners = [] - for env_file in env_files: - config: dict[str, str] = dotenv_values(env_file) # type: ignore - RunnerClass = RUNNER_DICT[config["TYPE"]] - runners.append(RunnerClass(dotenv_path=env_file, output_dir=self.output_dir, session_id=self.session_id)) - return runners - - @staticmethod - def check_env_files(env_files: list[Path]): - """checks if file exist for every file in list""" - for env_file in env_files: - assert env_file.is_file() From 2e33f8f014e99e45ed4f366caa3f521f2610193c Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 30 Nov 2023 10:53:20 +0100 Subject: [PATCH 04/18] make-all-env-files-available (#4) Before, a test had only access to it's own env file / configuration (wordpress could see wordpress env file). Now, all env files are available. Wordpress test can also read authentik env file, for example to get the authentik domain. Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/4 Co-authored-by: Daniel Co-committed-by: Daniel --- src/dirmanager.py | 18 +++++-- src/runner.py | 61 ++++++++++++----------- src/tests_authentik/fixtures_authentik.py | 14 +++++- src/tests_wordpress/conftest.py | 2 +- src/tests_wordpress/test_wordpress.py | 17 +++---- 5 files changed, 68 insertions(+), 44 deletions(-) diff --git a/src/dirmanager.py b/src/dirmanager.py index 41dcfaa..c210d25 100644 --- a/src/dirmanager.py +++ b/src/dirmanager.py @@ -25,7 +25,9 @@ class DirManager: def create_all_dirs(self): self.create_dirs(self._output_dir, exist_ok=True) - self.create_dirs([self.SESSION, self.RECORDS, self.RECORDS / "html", self.STATES, self.RESULTS], exist_ok=True) + self.create_dirs( + [self.SESSION, self.RECORDS, self.HTML, self.STATES, self.ENV_FILES, self.RESULTS], exist_ok=True + ) @property def OUTPUT(self): @@ -37,15 +39,23 @@ class DirManager: @property def RECORDS(self): - return self.SESSION / Path("records") + return self.SESSION / "records" + + @property + def HTML(self): + return self.RECORDS / "html" @property def STATES(self): - return self.SESSION / Path("states") + return self.SESSION / "states" + + @property + def ENV_FILES(self): + return self.STATES / "env_files" @property def RESULTS(self): - return self.SESSION / Path("results") + return self.SESSION / "results" @staticmethod def create_dirs(dirs: Path | list[Path] | dict[str, Path], exist_ok=False): diff --git a/src/runner.py b/src/runner.py index d232334..7d0011a 100644 --- a/src/runner.py +++ b/src/runner.py @@ -33,28 +33,44 @@ class Runner: assert self.test_dir_name self.root_dir = Path(__file__).parent - def _run_main_setup_and_test(self): + def run_tests(self): + # check if required dependencies have passed + self._assert_dependencies_passed() + + # run main setup if available if isinstance(self.main_setup_name, str): - self._run_test_if_required( + self._run_or_skip_test( identifier_string=self.combine_names(self.name, self.main_setup_name), test_path=self.root_dir / self.test_dir_name / self.main_setup_name, ) + # run main test if available if isinstance(self.main_test_name, str): - self._run_test_if_required( + self._run_or_skip_test( identifier_string=self.combine_names(self.name, self.main_test_name), test_path=self.root_dir / self.test_dir_name / self.main_test_name, ) - def _run_test_if_required(self, identifier_string: str, test_path: Path): - if not self.prevent_skip and self._test_already_passed(identifier_string, remove_existing=True): + # run sub tests if conditions are met + for sub_test in self.sub_tests: + condition_function = sub_test["condition"] + sub_test_name = sub_test["test_file"] + identifier_string = self.combine_names(self.name, sub_test_name) + if condition_function(self.config): + test_path = self.root_dir / self.test_dir_name / sub_test_name + self._run_or_skip_test(identifier_string=identifier_string, test_path=test_path) + else: + self._create_result_file(result=-1, identifier_string=identifier_string) + + def _run_or_skip_test(self, identifier_string: str, test_path: Path): + if not self.prevent_skip and self._is_test_passed(identifier_string, remove_existing=True): logger.info(f"skipping {identifier_string}") else: logger.info(f"running {identifier_string}") result = self._call_pytest(test_path) self._create_result_file(result=result, identifier_string=identifier_string) - def _test_already_passed(self, identifier_string: str, remove_existing: bool = False) -> bool: + def _is_test_passed(self, identifier_string: str, remove_existing: bool = False) -> bool: """returns True if the selected test (matching test_name + sub_test_name) already passed This is determined by the presence of a specific output file in the RESULTS folder that @@ -116,19 +132,6 @@ class Runner: return pytest.main(command_arguments) - def run_tests(self): - self._assert_dependencies_passed() - self._run_main_setup_and_test() - for sub_test in self.sub_tests: - condition_function = sub_test["condition"] - sub_test_name = sub_test["test_file"] - identifier_string = self.combine_names(self.name, sub_test_name) - if condition_function(self.config): - test_path = self.root_dir / self.test_dir_name / sub_test_name - self._run_test_if_required(identifier_string=identifier_string, test_path=test_path) - else: - self._create_result_file(result=-1, identifier_string=identifier_string) - def _create_result_file( self, result: int, @@ -141,6 +144,16 @@ class Runner: with open(file_path, "w") as _: pass # create empty file + def _assert_dependencies_passed(self): + """assert that all dependencie setups passed before""" + + passed_tests = [r.name for r in self.DIRS.RESULTS.glob("*") if "passed" in r.name] + for dependencie in self.dependencies: + dependencie_identifier = self.combine_names(dependencie.name, dependencie.main_setup_name) + assert any( + dependencie_identifier in f for f in passed_tests + ), f"could not run {self.name} because {dependencie} did not run before" + @staticmethod def result_int_to_str(result_int: int) -> str: match result_int: @@ -154,13 +167,3 @@ class Runner: @staticmethod def combine_names(*names: str) -> str: return "-".join(names) - - def _assert_dependencies_passed(self): - """assert that all dependencie setups passed before""" - - passed_tests = [r.name for r in self.DIRS.RESULTS.glob("*") if "passed" in r.name] - for dependencie in self.dependencies: - dependencie_identifier = self.combine_names(dependencie.name, dependencie.main_setup_name) - assert any( - dependencie_identifier in f for f in passed_tests - ), f"could not run {self.name} because {dependencie} did not run before" diff --git a/src/tests_authentik/fixtures_authentik.py b/src/tests_authentik/fixtures_authentik.py index dab4e42..21bddbe 100644 --- a/src/tests_authentik/fixtures_authentik.py +++ b/src/tests_authentik/fixtures_authentik.py @@ -1,7 +1,8 @@ import json import pytest -from playwright.sync_api import BrowserContext +from dotenv import dotenv_values +from playwright.sync_api import BrowserContext, Page from src.dirmanager import DirManager @@ -17,6 +18,17 @@ def admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: return context +@pytest.fixture +def authentik_admin_page(admin_context: BrowserContext, DIR: DirManager) -> Page: + page = admin_context.new_page() + page.pause() + authentik_env_file = DIR.ENV_FILES / "authentik" + authentik_config: dict[str, str] = dotenv_values(authentik_env_file) # type: ignore + url = "https://" + authentik_config["DOMAIN"] + page.goto(url) + return page + + @pytest.fixture def user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "user_state.json" diff --git a/src/tests_wordpress/conftest.py b/src/tests_wordpress/conftest.py index c7a791d..df1eac3 100644 --- a/src/tests_wordpress/conftest.py +++ b/src/tests_wordpress/conftest.py @@ -1 +1 @@ -from src.tests_authentik.fixtures_authentik import admin_context, user_context +from src.tests_authentik.fixtures_authentik import admin_context, authentik_admin_page, user_context diff --git a/src/tests_wordpress/test_wordpress.py b/src/tests_wordpress/test_wordpress.py index 6c85114..bb9275d 100644 --- a/src/tests_wordpress/test_wordpress.py +++ b/src/tests_wordpress/test_wordpress.py @@ -1,14 +1,13 @@ -import re - -from playwright.sync_api import BrowserContext, expect - -from src.dirmanager import DirManager +from playwright.sync_api import Page, expect -def test_visit_from_authentik(admin_context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): - page_authentik = admin_context.new_page() - with page_authentik.expect_popup() as event_context: - page_authentik.get_by_role("link", name="Wordpress").click() +def test_visit_from_authentik(authentik_admin_page: Page): + with authentik_admin_page.expect_popup() as event_context: + authentik_admin_page.get_by_role("link", name="Wordpress").click() page_wordpress = event_context.value + # look for content wrapper expect(page_wordpress.locator("#wpcontent")).to_be_visible() + + # look for admin bar + expect(page_wordpress.locator("#wpadminbar")).to_be_visible() From d3dc0f942a78ed685ecacc5e89b46be2d41a7fc5 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 4 Dec 2023 12:46:30 +0100 Subject: [PATCH 05/18] new-features (#5) * refactoring and rework: runner now has setups / tests / cleanups as lists * add nextcloud runner * add email testing prototype with imap fixture * add dependency resolution (sort env files in input so that test order is correct) Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/5 Co-authored-by: Daniel Co-committed-by: Daniel --- README.md | 29 ++++- main.py | 9 +- prototyping/__init__.py | 0 prototyping/email_stuff.py | 47 +++++++ prototyping/test_urllib.py | 10 ++ pyproject.toml | 3 +- requirements.txt | 1 + src/conftest.py | 30 ++++- src/coordinator.py | 63 +++++++--- src/dirmanager.py | 4 +- src/env_file_helper.py | 52 ++++++++ src/html_helper.py | 2 - src/runner.py | 115 +++++++++++------- .../deprecated-setup_authentik-deprecated.py | 100 --------------- src/tests_authentik/fixtures_authentik.py | 33 ++--- src/tests_authentik/plugin_authentik.py | 3 - src/tests_authentik/runner_authentik.py | 6 +- src/tests_authentik/setup_authentik.py | 21 ++-- src/tests_demo/runner_demo.py | 23 ++-- src/tests_nextcloud/cleanup_nextcloud.py | 25 ++++ src/tests_nextcloud/conftest.py | 1 + src/tests_nextcloud/runner_nextcloud.py | 18 +++ src/tests_nextcloud/setup_nextcloud.py | 35 ++++++ src/tests_nextcloud/tests_nextcloud.py | 13 ++ .../tests_nextcloud_onlyoffice.py | 19 +++ src/tests_wordpress/conftest.py | 35 +++++- src/tests_wordpress/runner_wordpress.py | 10 +- src/tests_wordpress/setup_wordpress.py | 30 +++++ src/tests_wordpress/test_wordpress.py | 13 -- .../test_wordpress_localization.py | 6 +- src/utils.py | 5 + tests/test_env_resolution.py | 59 +++++++++ 32 files changed, 573 insertions(+), 247 deletions(-) create mode 100644 prototyping/__init__.py create mode 100644 prototyping/email_stuff.py create mode 100644 prototyping/test_urllib.py create mode 100644 src/env_file_helper.py delete mode 100644 src/setup-deprecated/deprecated-setup_authentik-deprecated.py delete mode 100644 src/tests_authentik/plugin_authentik.py create mode 100644 src/tests_nextcloud/cleanup_nextcloud.py create mode 100644 src/tests_nextcloud/conftest.py create mode 100644 src/tests_nextcloud/runner_nextcloud.py create mode 100644 src/tests_nextcloud/setup_nextcloud.py create mode 100644 src/tests_nextcloud/tests_nextcloud.py create mode 100644 src/tests_nextcloud/tests_nextcloud_onlyoffice.py create mode 100644 src/tests_wordpress/setup_wordpress.py create mode 100644 tests/test_env_resolution.py diff --git a/README.md b/README.md index a4ffa00..a046244 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Clone +# GIT Clone To clone with submodules, use these git commands: @@ -8,7 +8,24 @@ git submodule update --init // add submodule after normal cloning git submodule update --remote // update submodules ``` -# Run +# Run without Docker + +### Installation + +Create a python environment and install all dependencies via + +```bash +pip install -r requirements.txt +playwright install +``` + +Run the script with + +```bash +python main.py +``` + +# Run with Docker ```bash docker compose build @@ -28,3 +45,11 @@ Force rebuild wtihtout cache ```bash docker-compose build --no-cache ``` + +# Codegen + +Use playwright codegen to create code for new testes easily https://playwright.dev/python/docs/codegen + +```bash +playwright codegen demo.playwright.dev/todomvc +``` \ No newline at end of file diff --git a/main.py b/main.py index f71cc25..9005385 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,8 @@ from src.utils import get_session_id # requires that authentik ran first to create the admin session and the user # session. At the moment, wrong ordering results in unsuccessful test # (wrong ordering would be wordpress env file is before authentik env file). +# At the moment, functionailty is only guaranteed if each env file use +# a unique TYPE var. ENV_FILES = [ Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik @@ -38,12 +40,13 @@ OUTPUT_DIR = Path("./test-output").resolve() # --------------------- load credentials to env variables -------------------- # + cred_file = Path("credentials.json") with open(cred_file, "r") as f: CREDENTIALS = json.load(f) -os.environ["ADMIN_USER"] = CREDENTIALS["admin_user"] -os.environ["ADMIN_PASS"] = CREDENTIALS["admin_pass"] +for key, value in CREDENTIALS.items(): + os.environ[key] = value # ----------------------------- define session_id ---------------------------- # @@ -56,7 +59,7 @@ session_id = get_session_id() # ------------------------------- setup logging ------------------------------ # DIR = DirManager(output_dir=OUTPUT_DIR, session_id=session_id) -log_file = DIR.RESULTS / "full.log" +log_file = DIR.RECORDS / "coordinator.log" logger.add(log_file) diff --git a/prototyping/__init__.py b/prototyping/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prototyping/email_stuff.py b/prototyping/email_stuff.py new file mode 100644 index 0000000..a8896f8 --- /dev/null +++ b/prototyping/email_stuff.py @@ -0,0 +1,47 @@ +# %% +import email +import json +from email.header import decode_header +from imaplib import IMAP4, IMAP4_SSL +from pathlib import Path + +# -------------------------------- credentials ------------------------------- # + +cred_file = Path("../credentials.json") +with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + +username = CREDENTIALS["imap_user"] +password = CREDENTIALS["imap_pass"] + + +# ----------------------------------- imap ----------------------------------- # + +host = "mail.local-it.org" +imap_port = 143 +imap_ssl_port = 993 + + +with IMAP4_SSL(host=host) as imap_server: + imap_server.login(username, password) + imap_server.select("INBOX") + + # Search for all emails in the folder + status, email_ids = imap_server.search(None, "ALL") + email_ids = email_ids[0].split() + + # Fetch email details using the retrieved IDs + for email_id in email_ids: + result, data = imap_server.fetch(email_id, "(RFC822)") + raw_email = data[0][1] # Raw content of the email + email_message = email.message_from_bytes(raw_email) + + # Extract the subject + subject_encoded = email_message.get("Subject") + decoded_subject = decode_header(subject_encoded)[0][0] + + if isinstance(decoded_subject, bytes): + decoded_subject = decoded_subject.decode() + + # Print or use the subject as needed + print("Subject:", decoded_subject) diff --git a/prototyping/test_urllib.py b/prototyping/test_urllib.py new file mode 100644 index 0000000..074b0a5 --- /dev/null +++ b/prototyping/test_urllib.py @@ -0,0 +1,10 @@ +# %% +from urllib.parse import parse_qs, urlencode, urlparse, urlunparse + +string = "blog.dev.local-it.cloud" + + +parsed_url = urlparse(string, scheme="https") +print(parsed_url) + +print(urlunparse(parsed_url)) diff --git a/pyproject.toml b/pyproject.toml index d6f9e14..6912f0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,4 +19,5 @@ line-length = 120 target-version = "py311" [tool.pytest.ini_options] -python_files = "test_*.py setup*.py" \ No newline at end of file +python_functions = "test_* setup_*" +norecursedirs = "previous-work src" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f1577e2..a17d5a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pytest +pytest-html pytest-playwright python-dotenv icecream diff --git a/src/conftest.py b/src/conftest.py index 3eb168c..690aaf6 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -5,15 +5,28 @@ # sys.path. It is thus good practise for projects to either put conftest.py under # a package scope or to never import anything from a conftest.py file. +import os +from imaplib import IMAP4_SSL from pathlib import Path import pytest from dotenv import dotenv_values +from playwright.sync_api import BrowserContext, expect from pytest import Parser from src.dirmanager import DirManager -TIMEOUT = 5000 +# global timeout and LOCALE +LOCALE = {"Accept-Language": "de_DE"} +TIMEOUT = 7_000 +expect.set_options(timeout=TIMEOUT) + + +@pytest.fixture +def context(context: BrowserContext) -> BrowserContext: + context.set_default_timeout(TIMEOUT) + context.set_extra_http_headers(LOCALE) + return context def pytest_addoption(parser: Parser): @@ -58,3 +71,18 @@ def dotenv_config(request) -> dict[str, str]: dotenv_path = Path(dotenv_path) assert dotenv_path.is_file() return dotenv_values(dotenv_path) # type: ignore + + +@pytest.fixture(scope="session") +def imap_ssl_email_client() -> None: + assert os.environ["IMAP_HOST"] + assert os.environ["IMAP_PORT"] + assert os.environ["IMAP_USER"] + assert os.environ["IMAP_PASS"] + port = int(os.environ["IMAP_PORT"]) + imap_client = IMAP4_SSL(host=os.environ["IMAP_HOST"], port=port) + imap_client.login(os.environ["IMAP_USER"], os.environ["IMAP_PASS"]) + imap_client.select("INBOX") + yield imap_client + imap_client.close() + imap_client.logout() diff --git a/src/coordinator.py b/src/coordinator.py index b466e1c..0baf934 100644 --- a/src/coordinator.py +++ b/src/coordinator.py @@ -5,21 +5,24 @@ from dotenv import dotenv_values from loguru import logger from src.dirmanager import DirManager +from src.env_file_helper import DependencyRule, EnvFile, sort_env_files_by_rule from src.html_helper import merge_html_files from src.runner import Runner from src.tests_authentik.runner_authentik import RunnerAuthentik +from src.tests_nextcloud.runner_nextcloud import RunnerNextcloud from src.tests_wordpress.runner_wordpress import RunnerWordpress from src.utils import rmtree -# Register all runners here. Each .env file with TYPE=authentik will be ran with RunnerAuthentik +# Register all runners here. Each .env file with TYPE=authentik will be run with RunnerAuthentik RUNNER_DICT: dict[str, type[Runner]] = { "authentik": RunnerAuthentik, "wordpress": RunnerWordpress, + "nextcloud": RunnerNextcloud, } class Coordinator: - def __init__(self, env_paths_list: list[Path], output_dir: Path, session_id: str): + def __init__(self, env_paths_list: list[Path], output_dir: Path, session_id: str) -> None: out_string = "".join([e.name + "\n" for e in env_paths_list]) out_string += f"output_dir = {output_dir}\n" out_string += f"session_id = {session_id}" @@ -29,47 +32,67 @@ class Coordinator: self.output_dir = output_dir self.session_id = session_id - self.env_paths: dict[str, Path] = dict() - self.env_configs: dict[str, dict[str, str]] = dict() # todo: needed? - self._parse_env_files(env_paths_list) + # parse env files + self.env_files: list[EnvFile] = self._getn_env_files_list(env_paths_list) + self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files) - def _parse_env_files(self, env_paths: list[Path]): + @staticmethod + def _getn_env_files_list(env_paths: list[Path]) -> list[EnvFile]: + """Returns a list of EnvFile objects created from the given env files""" + env_files: list[EnvFile] = [] for env_path in env_paths: assert env_path.is_file(), f"the env file {env_path} does not exist" config: dict[str, str] = dotenv_values(env_path) # type: ignore assert "TYPE" in config, f"the env file {env_path} does not specify the required TYPE key." env_type = config["TYPE"] - self.env_paths[env_type] = env_path - self.env_configs[env_type] = config # todo: needed? + env_files.append(EnvFile(env_path=env_path, config=config, env_type=env_type)) + return env_files - def setup_test(self): + @staticmethod + def _get_dependency_rules(env_files: list[EnvFile]) -> list[DependencyRule]: + dependency_rules: list[DependencyRule] = [] + for env_file in env_files: + child_runner_class = RUNNER_DICT[env_file.env_type] + for dependency in child_runner_class.dependencies: + dependency_rule = DependencyRule(child=child_runner_class.name, dependency=dependency.name) + dependency_rules.append(dependency_rule) + return dependency_rules + + def setup_test(self) -> None: logger.info("calling setup_test()") self.DIR.create_all_dirs() self._copy_env_files() - def _copy_env_files(self): - """Copies all env filesto STATES/env_files. Files will be renamed to their own TYPE value.""" + def _copy_env_files(self) -> None: + """Copies all env files to STATES/env_files. Files will be renamed to their own TYPE value.""" env_files_dir = self.DIR.STATES / "env_files" env_files_dir.mkdir(exist_ok=True) - for type_key, env_path in self.env_paths.items(): - shutil.copy(env_path, env_files_dir / type_key) + for env_file in self.env_files: + shutil.copy(env_file.env_path, env_files_dir / env_file.env_type) - def run_test(self): + def run_test(self) -> None: logger.info("calling run_test()") - self.runners: list[Runner] = self._load_runners(self.env_paths.values()) + self.runners: list[Runner] = self._load_runners(self.env_files) + for runner in self.runners: + runner.run_setups() for runner in self.runners: runner.run_tests() + for runner in self.runners: + runner.run_cleanups() logger.info("run_test() finished") - def _load_runners(self, env_files: list[Path]) -> list[Runner]: + def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]: + """Creates an instance of the correct Runner class for each given env file""" runners = [] for env_file in env_files: - config: dict[str, str] = dotenv_values(env_file) # type: ignore - RunnerClass = RUNNER_DICT[config["TYPE"]] - runners.append(RunnerClass(dotenv_path=env_file, output_dir=self.output_dir, session_id=self.session_id)) + RunnerClass = RUNNER_DICT[env_file.config["TYPE"]] + runners.append( + RunnerClass(dotenv_path=env_file.env_path, output_dir=self.output_dir, session_id=self.session_id) + ) return runners - def combine_html(self): + def combine_html(self) -> None: + """combines all generated pytest html reports into one""" in_path = str(self.DIR.RECORDS / "html") out_path = str(self.DIR.RECORDS / "full-report.html") title = "combined.html" diff --git a/src/dirmanager.py b/src/dirmanager.py index c210d25..b5228bf 100644 --- a/src/dirmanager.py +++ b/src/dirmanager.py @@ -30,12 +30,12 @@ class DirManager: ) @property - def OUTPUT(self): + def OUTPUT_DIR(self): return self._output_dir @property def SESSION(self): - return self._output_dir / f"test-{self.session_id}" + return self.OUTPUT_DIR / f"test-{self.session_id}" @property def RECORDS(self): diff --git a/src/env_file_helper.py b/src/env_file_helper.py new file mode 100644 index 0000000..29fb914 --- /dev/null +++ b/src/env_file_helper.py @@ -0,0 +1,52 @@ +from pathlib import Path +from typing import NamedTuple + +from loguru import logger + + +class EnvFile(NamedTuple): + env_path: Path + config: dict[str, str] + env_type: str + + def __repr__(self) -> str: + return f"EnvFile(type={self.env_type})" + + +class DependencyRule(NamedTuple): + child: str + dependency: str + + +def _is_rule_satisfied(in_list: list, rule: DependencyRule) -> tuple[bool, int]: + child_indices = [index for index, element in enumerate(in_list) if element.env_type == rule.child] + child_index = min(child_indices) + # child_index = in_list.index(rule.child) + parent_indices = [index for index, element in enumerate(in_list) if element.env_type == rule.dependency] + parent_index = max(parent_indices) + # parent_index = in_list.index(rule.dependency) + return parent_index < child_index, parent_index + + +def sort_env_files_by_rule(env_list: list[EnvFile], rules: list[DependencyRule]) -> list: + in_list = env_list.copy() + + def swap_item_with_previous(in_list: list[EnvFile], index: int): + """swaps item at index N with item at index N-1""" + assert index > 0, "cannot swap with negative index" + in_list[index], in_list[index - 1] = in_list[index - 1], in_list[index] + + for _ in range(10_000): + rule_satisfied: list[bool] = [] + for rule in rules: + is_rule_satisfied, parent_index = _is_rule_satisfied(in_list, rule) + if is_rule_satisfied: + rule_satisfied.append(True) + else: + rule_satisfied.append(False) + # parent_index = in_list.index(rule.dependency) + swap_item_with_previous(in_list, parent_index) + if all(rule_satisfied): + return in_list + logger.error("could not find order that satisfys all rules") + raise ValueError diff --git a/src/html_helper.py b/src/html_helper.py index 8f94b67..6e81c7e 100644 --- a/src/html_helper.py +++ b/src/html_helper.py @@ -174,10 +174,8 @@ def get_html_files(path, output_file_path): output_file_path = os.path.abspath(output_file_path) for p in pathlib.Path(path).rglob("*.html"): - print(p) res = str(p.absolute()) if output_file_path in res: - print("damn") continue tmp = BeautifulSoup("".join(open(res)), features="html.parser") diff --git a/src/runner.py b/src/runner.py index 7d0011a..71ca7ed 100644 --- a/src/runner.py +++ b/src/runner.py @@ -1,5 +1,6 @@ +from dataclasses import dataclass from pathlib import Path -from typing import Callable, Optional, TypedDict +from typing import Callable import pytest from dotenv import dotenv_values @@ -8,18 +9,20 @@ from loguru import logger from src.dirmanager import DirManager -class SubTest(TypedDict): - condition: Callable[[dict[str, str]], bool] +@dataclass +class Test: test_file: str + condition: Callable[[dict[str, str]], bool] | None = None + prevent_skip: bool = False class Runner: name: str = "" test_dir_name: str = "" - main_setup_name: Optional[str] = None - main_test_name: Optional[str] = None + setups: list[Test] = [] + tests: list[Test] = [] + cleanups: list[Test] = [] dependencies: list[type["Runner"]] = [] - sub_tests: list[SubTest] = [] prevent_skip = False def __init__(self, dotenv_path: Path, output_dir: Path, session_id: str): @@ -33,50 +36,63 @@ class Runner: assert self.test_dir_name self.root_dir = Path(__file__).parent + def run_setups(self): + """runs the setup scripts if available""" + self._execute_test_list(self.setups) + def run_tests(self): + """runs the test scripts if available""" + self._execute_test_list(self.tests) + + def run_cleanups(self): + """runs the cleanup scripts if available""" + self._execute_test_list(self.cleanups) + + def _execute_test_list(self, test_list: list[Test]): + """runs the main test script and if available and sub test scripts if their running condition is met""" # check if required dependencies have passed - self._assert_dependencies_passed() + if not self._dependencies_passed(): + logger.warning(f"skipping run_tests() of {self.name}, because some dependencies have not passed") + return - # run main setup if available - if isinstance(self.main_setup_name, str): - self._run_or_skip_test( - identifier_string=self.combine_names(self.name, self.main_setup_name), - test_path=self.root_dir / self.test_dir_name / self.main_setup_name, - ) + for test in test_list: + self._run_test_with_checks(test) - # run main test if available - if isinstance(self.main_test_name, str): - self._run_or_skip_test( - identifier_string=self.combine_names(self.name, self.main_test_name), - test_path=self.root_dir / self.test_dir_name / self.main_test_name, - ) + def _run_test_with_checks(self, test: Test): + # dependency passed: true / false + # already_passed: true / false + # prevent_skip: true / false + # condition_available: true / pass + # condition_met: true / false - # run sub tests if conditions are met - for sub_test in self.sub_tests: - condition_function = sub_test["condition"] - sub_test_name = sub_test["test_file"] - identifier_string = self.combine_names(self.name, sub_test_name) - if condition_function(self.config): - test_path = self.root_dir / self.test_dir_name / sub_test_name - self._run_or_skip_test(identifier_string=identifier_string, test_path=test_path) + identifier_string = self.combine_names(self.name, test.test_file) + test_path = self.root_dir / self.test_dir_name / test.test_file + + # check if test aleady passed + if self._is_test_passed(identifier_string, remove_existing=True): + if test.prevent_skip: + logger.info(f"continuing , test {identifier_string} has passed but prevent_skip=True") else: - self._create_result_file(result=-1, identifier_string=identifier_string) + logger.info(f"skipping {identifier_string}, test has passed") + return - def _run_or_skip_test(self, identifier_string: str, test_path: Path): - if not self.prevent_skip and self._is_test_passed(identifier_string, remove_existing=True): - logger.info(f"skipping {identifier_string}") - else: - logger.info(f"running {identifier_string}") - result = self._call_pytest(test_path) - self._create_result_file(result=result, identifier_string=identifier_string) + if test.condition and not test.condition(self.config): + # test condition is defined but not met + logger.info(f"skipping {identifier_string}, test condition is not met") + return + + # test condition is undefined or not met + logger.info(f"running {identifier_string}") + result = self._call_pytest(test_path) + self._create_result_file(result=result, identifier_string=identifier_string) def _is_test_passed(self, identifier_string: str, remove_existing: bool = False) -> bool: - """returns True if the selected test (matching test_name + sub_test_name) already passed + """returns True if the selected test matching identifier_string already passed This is determined by the presence of a specific output file in the RESULTS folder that matches identifier_string - remove_existing: If True, result files matching test_name + sub_test_name with a status + remove_existing: If True, result files matching identifier_string with a status other than 'passed' will be deleted""" already_passed = False @@ -96,20 +112,21 @@ class Runner: command_arguments = [] - # command_arguments.append("-v") + command_arguments.append("-v") # command_arguments.append("-rx") command_arguments.append(str(full_test_path)) command_arguments.append("--env_file") command_arguments.append(str(self.dotenv_path)) + # set root dir for tests output (used in DirManager). this is our custom argument command_arguments.append("--output_dir") - command_arguments.append(str(self.DIRS.OUTPUT)) + command_arguments.append(str(self.DIRS.OUTPUT_DIR)) command_arguments.append("--session_id") command_arguments.append(self.session_id) - # artifacts dir + # artifacts dir from pytest # warning: https://github.com/microsoft/playwright-pytest/issues/111 # --output only works with the given context and page fixture # folder needs to be unique! traces will not appear, if every pytest run has same output dir @@ -144,18 +161,22 @@ class Runner: with open(file_path, "w") as _: pass # create empty file - def _assert_dependencies_passed(self): - """assert that all dependencie setups passed before""" + def _dependencies_passed(self): + """returns true if all setups of each dependency have passed""" + + # todo: what about conditional setups? passed_tests = [r.name for r in self.DIRS.RESULTS.glob("*") if "passed" in r.name] - for dependencie in self.dependencies: - dependencie_identifier = self.combine_names(dependencie.name, dependencie.main_setup_name) - assert any( - dependencie_identifier in f for f in passed_tests - ), f"could not run {self.name} because {dependencie} did not run before" + results = [] + for dependencie_runner in self.dependencies: + for setup_name in dependencie_runner.setups: + dependencie_identifier = self.combine_names(dependencie_runner.name, setup_name.test_file) + results.append(any(dependencie_identifier in f for f in passed_tests)) + return all(results) @staticmethod def result_int_to_str(result_int: int) -> str: + """converts the pytest exit code (int) into a meaningful string""" match result_int: case -1: return "skipped" diff --git a/src/setup-deprecated/deprecated-setup_authentik-deprecated.py b/src/setup-deprecated/deprecated-setup_authentik-deprecated.py deleted file mode 100644 index 4ac4fd5..0000000 --- a/src/setup-deprecated/deprecated-setup_authentik-deprecated.py +++ /dev/null @@ -1,100 +0,0 @@ -import json -from pathlib import Path - -import pytest -from icecream import ic -from playwright.sync_api import Browser, Locator, expect - -cred_file = Path("credentials.json") -with open(cred_file, "r") as f: - CREDENTIALS = json.load(f) - -print(CREDENTIALS) - -TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} -TIMEOUT = 5000 - - -def check_for(locator: Locator): - expect(locator).to_be_visible(timeout=TIMEOUT) - - -# todo: why is this a fixture? to get dotenv_config -@pytest.fixture(scope="session", autouse=True) -def create_admin_login(browser: Browser, dotenv_config, STATES): - # ic(dotenv_config) - - # go to page - context = browser.new_context() - context.set_default_timeout(TIMEOUT) - page = context.new_page() - url = "https://" + dotenv_config["DOMAIN"] - page.goto(url) - - # check welcome message - welcome_message = dotenv_config.get("welcome_message") - if welcome_message: - check_for(page.get_by_text(welcome_message)) - - # login - page.locator('input[name="uidField"]').fill(CREDENTIALS["admin"]) - page.locator('ak-stage-identification input[name="password"]').fill(CREDENTIALS["admin_pw"]) - page.get_by_role("button", name="Log In").click() - check_for(page.locator("ak-library")) - - # save state - context.storage_state(path=f"{STATES}/admin_state.json") - page.close() - context.close() - - -def create_invite_link(page): - url = "https://" + dotenv_config["DOMAIN"] - page.goto(url) - page.get_by_role("link", name="Admin Interface").click() - page.get_by_text("Verzeichnis").click() - page.get_by_text("Benutzer").nth(2).click() - page.get_by_text("Einladungen").click() - page.get_by_role("button", name="Erstellen").first.click() - page.locator('input[name="name"]').click() - linkname = "testlink9433" - page.locator('input[name="name"]').fill(linkname) - page.get_by_placeholder("Wählen Sie ein Objekt aus.").click() - page.get_by_role("option", name="invitation-enrollment-flow invitation-enrollment-flow").click() - page.get_by_text("Erstellen", exact=True).first.click() - linklocator = page.get_by_role("rowgroup").filter(has=page.get_by_text(linkname)) - linklocator.locator(".fa-angle-down").click() - invitelink = linklocator.get_by_role("textbox").get_attribute(name="value") - return invitelink - - -def create_user(context, invitelink): - page = context.new_page() - page.goto(invitelink) - page.get_by_placeholder("Benutzername").click() - page.get_by_placeholder("Benutzername").fill(testuser["username"]) - page.locator('input[name="name"]').click() - page.locator('input[name="name"]').fill(testuser["name"]) - page.locator('input[name="email"]').click() - page.locator('input[name="email"]').fill(testuser["email"]) - page.get_by_placeholder("Passwort", exact=True).click() - page.get_by_placeholder("Passwort", exact=True).fill(testuser["password"]) - page.get_by_placeholder("Passwort (wiederholen)").click() - page.get_by_placeholder("Passwort (wiederholen)").fill(testuser["password"]) - page.get_by_role("button", name="Weiter").click() - check_for(page.locator("ak-library")) - context.storage_state(path=f"{STATES}/user_state.json") - - -@pytest.fixture(scope="session", autouse=True) -def create_user_session(browser: Browser, admin_login): - admin_context = browser.new_context(storage_state=f"{STATES}/admin_state.json") - # admin_context = setup_context(browser, f"{STATES}/admin_state.json") - admin_page = admin_context.new_page() - invitelink = create_invite_link(admin_page) - admin_context.tracing.stop(path=f"{RECORDS}/create_invite_link.zip") - admin_context.close() - user_context = setup_context(browser) - create_user(user_context, invitelink) - user_context.tracing.stop(path=f"{RECORDS}/create_user.zip") - user_context.close() diff --git a/src/tests_authentik/fixtures_authentik.py b/src/tests_authentik/fixtures_authentik.py index 21bddbe..d4c21c8 100644 --- a/src/tests_authentik/fixtures_authentik.py +++ b/src/tests_authentik/fixtures_authentik.py @@ -6,33 +6,38 @@ from playwright.sync_api import BrowserContext, Page from src.dirmanager import DirManager -TIMEOUT = 5000 - @pytest.fixture -def admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: - state_file = DIR.STATES / "admin_state.json" +def authentik_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: + state_file = DIR.STATES / "authentik_admin_state.json" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) - context.set_default_timeout(TIMEOUT) return context @pytest.fixture -def authentik_admin_page(admin_context: BrowserContext, DIR: DirManager) -> Page: - page = admin_context.new_page() - page.pause() - authentik_env_file = DIR.ENV_FILES / "authentik" - authentik_config: dict[str, str] = dotenv_values(authentik_env_file) # type: ignore - url = "https://" + authentik_config["DOMAIN"] +def authentik_admin_page(authentik_admin_context: BrowserContext, DIR: DirManager) -> Page: + page = authentik_admin_context.new_page() + env_file = DIR.ENV_FILES / "authentik" + config: dict[str, str] = dotenv_values(env_file) # type: ignore + url = "https://" + config["DOMAIN"] page.goto(url) return page @pytest.fixture -def user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: - state_file = DIR.STATES / "user_state.json" +def authentik_user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: + state_file = DIR.STATES / "authentik_user_state.json" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) - context.set_default_timeout(TIMEOUT) return context + + +@pytest.fixture +def authentik_user_page(authentik_user_context: BrowserContext, DIR: DirManager) -> Page: + page = authentik_user_context.new_page() + env_file = DIR.ENV_FILES / "authentik" + config: dict[str, str] = dotenv_values(env_file) # type: ignore + url = "https://" + config["DOMAIN"] + page.goto(url) + return page diff --git a/src/tests_authentik/plugin_authentik.py b/src/tests_authentik/plugin_authentik.py deleted file mode 100644 index 14f31c9..0000000 --- a/src/tests_authentik/plugin_authentik.py +++ /dev/null @@ -1,3 +0,0 @@ -# will be loaded in conftest.py -# will provide context for other tests (wordpress etc.) -# that depend on authentik (which is all of them) diff --git a/src/tests_authentik/runner_authentik.py b/src/tests_authentik/runner_authentik.py index 59c486e..5b19788 100644 --- a/src/tests_authentik/runner_authentik.py +++ b/src/tests_authentik/runner_authentik.py @@ -1,4 +1,4 @@ -from src.runner import Runner, SubTest +from src.runner import Runner, Test def condition_always_true(dotenv_config: dict[str, str]) -> bool: @@ -12,5 +12,5 @@ def condition_always_false(dotenv_config: dict[str, str]) -> bool: class RunnerAuthentik(Runner): name = "authentik" test_dir_name = "tests_authentik" - main_setup_name = "setup_authentik.py" - # main_test_name = "test_authentik_dummy.py" + setups = [Test(test_file="setup_authentik.py")] + # tests = [Test(test_file="test_authentik_dummy.py")] diff --git a/src/tests_authentik/setup_authentik.py b/src/tests_authentik/setup_authentik.py index 831a137..7927895 100644 --- a/src/tests_authentik/setup_authentik.py +++ b/src/tests_authentik/setup_authentik.py @@ -9,15 +9,12 @@ from src.dirmanager import DirManager ADMIN_USER = os.environ["ADMIN_USER"] ADMIN_PASS = os.environ["ADMIN_PASS"] -LOCALE = {"Accept-Language": "de_DE"} + TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} -TIMEOUT = 6000 -def test_create_admin_login(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): +def setup_admin_state(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): # go to page - context.set_extra_http_headers(LOCALE) - context.set_default_timeout(TIMEOUT) page = context.new_page() url = "https://" + dotenv_config["DOMAIN"] page.goto(url) @@ -34,7 +31,7 @@ def test_create_admin_login(context: BrowserContext, dotenv_config: dict[str, st expect(page.locator("ak-library")).to_be_visible() # save state - context.storage_state(path=f"{DIR.STATES}/admin_state.json") + context.storage_state(path=f"{DIR.STATES}/authentik_admin_state.json") def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, str]): @@ -102,12 +99,9 @@ def create_user(user_context: BrowserContext, invitelink): expect(page.locator("ak-library")).to_be_visible() -def test_create_user_session(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): - context.set_extra_http_headers(LOCALE) - context.set_default_timeout(TIMEOUT) - +def setup_user_state(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): # load admin cookies - state_file = DIR.STATES / "admin_state.json" + state_file = DIR.STATES / "authentik_admin_state.json" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) @@ -116,11 +110,10 @@ def test_create_user_session(context: BrowserContext, dotenv_config: dict[str, s pass context.clear_cookies() else: - ## create user - # create invite_link + # get invite_link invite_link = create_invite_link(context, dotenv_config) # create user context.clear_cookies() create_user(context, invite_link) - context.storage_state(path=f"{DIR.STATES}/user_state.json") + context.storage_state(path=f"{DIR.STATES}/authentik_user_state.json") diff --git a/src/tests_demo/runner_demo.py b/src/tests_demo/runner_demo.py index 8f36e9c..2957ad5 100644 --- a/src/tests_demo/runner_demo.py +++ b/src/tests_demo/runner_demo.py @@ -1,6 +1,4 @@ -from typing import Optional - -from src.runner import Runner, SubTest +from src.runner import Runner, Test from src.tests_authentik.runner_authentik import RunnerAuthentik @@ -10,20 +8,19 @@ class RunnerDemo(Runner): name: str = "demo" # name of the test, used for logging / output naming test_dir_name: str = "tests_demo" # dir name holding all tests related to RunnerDemo - # Filename of Demo setup. If defined, it will run 1st by executing pytest - main_setup_name: Optional[str] = "setup_demo.py" - - # Filename of Demo test. This file contains unconditional tests that will be run in any - # case. If defined, it will run 2nd by executing pytest - main_test_name: Optional[str] = None - # this indicates that tests from RunnerDemo depend on the setup from RunnerAuthentik. # RunnerDemo will only execute, when setup_authentik.py has finished successfully. # For example, setup_authentik.py generates session states, that can be used as fixtures # that can be loaded from fixtures_authentik.py dependencies: list[type["Runner"]] = [RunnerAuthentik] + # todo: update these comments + # Filename of Demo setup. If defined, it will run 1st by executing pytest + # Filename of Demo test. This file contains unconditional tests that will be run in any + # case. If defined, it will run 2nd by executing pytest # this list can hold many more tests from RunnerDemo that run conditional. The condition - # and the test file can be defined by creating a SubTest instance: - # SubTest(condition: Callable, test_file: str) - sub_tests: list[SubTest] = [] + # and the test file can be defined by creating a ConditionalTest instance: + # ConditionalTest(condition: Callable, test_file: str) + setups: list[Test] = [] + tests: list[Test] = [] + cleanups: list[Test] = [] diff --git a/src/tests_nextcloud/cleanup_nextcloud.py b/src/tests_nextcloud/cleanup_nextcloud.py new file mode 100644 index 0000000..0bd456f --- /dev/null +++ b/src/tests_nextcloud/cleanup_nextcloud.py @@ -0,0 +1,25 @@ +import pytest +from playwright.sync_api import BrowserContext, expect + +# todo: what is this test for, why is it a fixture? -> ignore for now + + +@pytest.fixture(scope="session", autouse=True) +def delete_nextcloud_user(admin_context: BrowserContext): + """Delete Nextcloud User""" + yield + context = setup_context(browser, f"{STATES}/admin_state.json") + page = context.new_page() + page.goto(CONFIG["domain"]) + with page.expect_popup() as nextcloud_info: + page.get_by_role("link", name="Nextcloud").click() + nextcloud = nextcloud_info.value + nextcloud.get_by_role("link", name="Open settings menu").click() + nextcloud.get_by_role("link", name="Users").click() + nextcloud.locator("#app-content div").filter(has_text=testuser["username"]).get_by_role( + "button", name="Toggle user actions menu" + ).click() + nextcloud.get_by_role("button", name="Delete user").click() + nextcloud.get_by_role("button", name=f"Delete authentik-{testuser['username']}'s account").click() + context.tracing.stop(path=f"{RECORDS}/nextcloud_delete_user.zip") + context.close() diff --git a/src/tests_nextcloud/conftest.py b/src/tests_nextcloud/conftest.py new file mode 100644 index 0000000..df1eac3 --- /dev/null +++ b/src/tests_nextcloud/conftest.py @@ -0,0 +1 @@ +from src.tests_authentik.fixtures_authentik import admin_context, authentik_admin_page, user_context diff --git a/src/tests_nextcloud/runner_nextcloud.py b/src/tests_nextcloud/runner_nextcloud.py new file mode 100644 index 0000000..8468815 --- /dev/null +++ b/src/tests_nextcloud/runner_nextcloud.py @@ -0,0 +1,18 @@ +from src.runner import Runner, Test +from src.tests_authentik.runner_authentik import RunnerAuthentik + + +def condition_always_false(dotenv_config: dict[str, str]) -> bool: + return False + + +class RunnerNextcloud(Runner): + name: str = "nextcloud" + test_dir_name: str = "tests_nextcloud" + dependencies = [RunnerAuthentik] + setups = [Test(test_file="setup_nextcloud.py")] + tests = [ + Test(test_file="tests_nextcloud.py"), + Test(condition=condition_always_false, test_file="tests_nextcloud_onlyoffice.py"), + ] + # cleanups = [Test(test_file="cleanup_nextcloud.py")] diff --git a/src/tests_nextcloud/setup_nextcloud.py b/src/tests_nextcloud/setup_nextcloud.py new file mode 100644 index 0000000..eb53182 --- /dev/null +++ b/src/tests_nextcloud/setup_nextcloud.py @@ -0,0 +1,35 @@ +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def nc_login(browser: Browser): + """Nextcloud Login""" + context = setup_context(browser, f"{STATES}/user_state.json") + page = context.new_page() + page.goto(CONFIG["domain"]) + with page.expect_popup() as nextcloud_info: + link = page.get_by_role("link", name="Nextcloud") + CONFIG["nc_domain"] = link.get_attribute("href") + link.click() + nextcloud = nextcloud_info.value + check_for(nextcloud.get_by_role("link", name="Name")) + if nextcloud.query_selector(".close-icon"): + close_button = nextcloud.get_by_role("button", name="Close modal") + close_button.click() + expect(close_button).to_be_hidden() + nextcloud.wait_for_timeout(2000) + context.storage_state(path=f"{STATES}/nc_user_state.json") + context.tracing.stop(path=f"{RECORDS}/nextcloud_login_user.zip") + context.close() + + +@pytest.fixture +def nc_session(browser: Browser): + """Reuse Nextcloud User Session""" + context = setup_context(browser, f"{STATES}/nc_user_state.json") + page = context.new_page() + page.goto(CONFIG["nc_domain"]) + if page.query_selector(".close-icon"): + page.get_by_role("button", name="Close modal").click() + yield context, page + context.close() diff --git a/src/tests_nextcloud/tests_nextcloud.py b/src/tests_nextcloud/tests_nextcloud.py new file mode 100644 index 0000000..c9bc5be --- /dev/null +++ b/src/tests_nextcloud/tests_nextcloud.py @@ -0,0 +1,13 @@ +def test_nextcloud(nc_session): + """Test Nextcloud""" + context, page = nc_session + # if page.query_selector('.close-icon'): + # page.get_by_role("button", name="Close modal").click() + if CONFIG.get("default_quota"): + quota = int( + page.get_by_role("listitem", name="Storage informations").get_by_role("link").inner_text().split()[3] + ) + assert quota == CONFIG["default_quota"] + for app in CONFIG["nc_apps"]: + check_for(page.get_by_role("link", name=app)) + context.tracing.stop(path=f"{RECORDS}/nextcloud.zip") diff --git a/src/tests_nextcloud/tests_nextcloud_onlyoffice.py b/src/tests_nextcloud/tests_nextcloud_onlyoffice.py new file mode 100644 index 0000000..0b78e5a --- /dev/null +++ b/src/tests_nextcloud/tests_nextcloud_onlyoffice.py @@ -0,0 +1,19 @@ +def test_onlyoffice(nc_session): + """Test Onlyoffice in Nextcloud""" + context, page = nc_session + # if page.query_selector('.close-icon'): + # page.get_by_role("button", name="Close modal").click() + page.get_by_role("link", name="New file/folder menu").click() + page.get_by_role("link", name="New document").click() + page.locator("#view9-input-file").fill("test.docx") + page.get_by_role("button", name="Submit").click() + outer_frame = page.frame_locator("#onlyofficeFrame") + check_for(outer_frame.locator("body")) + inner_frame = outer_frame.frame_locator("#app > iframe") + check_for(inner_frame.locator("body")) + onlyoffice = page.frame("frameEditor") + check_for(onlyoffice.locator('//*[@id="area_id"]')) + onlyoffice.locator("#btn-goback").click() + page.get_by_role("link", name="Not favorited test .docx Share Actions").get_by_role("link", name="Actions").click() + page.get_by_role("link", name="Delete file").click() + context.tracing.stop(path=f"{RECORDS}/onlyoffice.zip") diff --git a/src/tests_wordpress/conftest.py b/src/tests_wordpress/conftest.py index df1eac3..f2d4e2c 100644 --- a/src/tests_wordpress/conftest.py +++ b/src/tests_wordpress/conftest.py @@ -1 +1,34 @@ -from src.tests_authentik.fixtures_authentik import admin_context, authentik_admin_page, user_context +import json + +import pytest +from dotenv import dotenv_values +from playwright.sync_api import BrowserContext, Page + +from src.dirmanager import DirManager + +# from src.tests_authentik.fixtures_authentik import ( +# authentik_admin_context, +# authentik_admin_page, +# authentik_user_context, +# authentik_user_page, +# ) + +pytest_plugins = "src.tests_authentik.fixtures_authentik" + + +@pytest.fixture +def wordpress_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: + state_file = DIR.STATES / "wordpress_admin_state.json" + storage_state = json.loads(state_file.read_bytes()) + context.add_cookies(storage_state["cookies"]) + return context + + +@pytest.fixture +def wordpress_admin_page(wordpress_admin_context: BrowserContext, DIR: DirManager) -> Page: + page = wordpress_admin_context.new_page() + env_file = DIR.ENV_FILES / "wordpress" + config: dict[str, str] = dotenv_values(env_file) # type: ignore + url = "https://" + config["DOMAIN"] + page.goto(url) + return page diff --git a/src/tests_wordpress/runner_wordpress.py b/src/tests_wordpress/runner_wordpress.py index 4355f22..097a66a 100644 --- a/src/tests_wordpress/runner_wordpress.py +++ b/src/tests_wordpress/runner_wordpress.py @@ -1,4 +1,4 @@ -from src.runner import Runner, SubTest +from src.runner import Runner, Test from src.tests_authentik.runner_authentik import RunnerAuthentik @@ -20,9 +20,9 @@ def condition_has_locale(dotenv_config: dict[str, str]) -> bool: class RunnerWordpress(Runner): name = "wordpress" test_dir_name = "tests_wordpress" - main_test_name = "test_wordpress.py" dependencies: list[type[Runner]] = [RunnerAuthentik] - sub_tests = [ - SubTest(condition=condition_has_locale, test_file="test_wordpress_localization.py"), + setups = [Test(test_file="setup_wordpress.py")] + tests = [ + Test(test_file="test_wordpress.py"), + Test(condition=condition_has_locale, test_file="test_wordpress_localization.py"), ] - prevent_skip = True diff --git a/src/tests_wordpress/setup_wordpress.py b/src/tests_wordpress/setup_wordpress.py new file mode 100644 index 0000000..6543c9d --- /dev/null +++ b/src/tests_wordpress/setup_wordpress.py @@ -0,0 +1,30 @@ +import pytest +from playwright.sync_api import BrowserContext, Page, expect + +from src.dirmanager import DirManager + + +@pytest.mark.xfail(reason="wordpress sso login has not been generated") +def test_visit_from_domain(authentik_admin_context: BrowserContext, dotenv_config: dict[str, str]): + """visit wordpress directly with admin_session, expect not to be logged in""" + page = authentik_admin_context.new_page() + url = "https://" + dotenv_config["DOMAIN"] + page.goto(url) + # look for content wrapper + expect(page.locator("#wpcontent")).to_be_visible(timeout=3_000) + # look for admin bar + expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000) + + +def setup_wordpress_admin_session(authentik_admin_page: Page, DIR: DirManager): + """visit wordpress from authentik with admin_session to create wordpress_admin_session""" + with authentik_admin_page.expect_popup() as event_context: + authentik_admin_page.get_by_role("link", name="Wordpress").click() + page_wordpress = event_context.value + # look for content wrapper + expect(page_wordpress.locator("#wpcontent")).to_be_visible() + # look for admin bar + expect(page_wordpress.locator("#wpadminbar")).to_be_visible() + # save session + context = page_wordpress.context + context.storage_state(path=f"{DIR.STATES}/wordpress_admin_state.json") diff --git a/src/tests_wordpress/test_wordpress.py b/src/tests_wordpress/test_wordpress.py index bb9275d..e69de29 100644 --- a/src/tests_wordpress/test_wordpress.py +++ b/src/tests_wordpress/test_wordpress.py @@ -1,13 +0,0 @@ -from playwright.sync_api import Page, expect - - -def test_visit_from_authentik(authentik_admin_page: Page): - with authentik_admin_page.expect_popup() as event_context: - authentik_admin_page.get_by_role("link", name="Wordpress").click() - page_wordpress = event_context.value - - # look for content wrapper - expect(page_wordpress.locator("#wpcontent")).to_be_visible() - - # look for admin bar - expect(page_wordpress.locator("#wpadminbar")).to_be_visible() diff --git a/src/tests_wordpress/test_wordpress_localization.py b/src/tests_wordpress/test_wordpress_localization.py index a3a8a91..77bce95 100644 --- a/src/tests_wordpress/test_wordpress_localization.py +++ b/src/tests_wordpress/test_wordpress_localization.py @@ -5,11 +5,11 @@ from playwright.sync_api import BrowserContext, expect from src.dirmanager import DirManager -def test_welcome_message(user_context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): - page = user_context.new_page() +def test_welcome_message(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): + page = context.new_page() url = "https://" + dotenv_config["DOMAIN"] page.goto(url) - expect(page.locator("#wpcontent")).to_be_visible() + expect(page.locator(".wp-block-heading")).to_be_visible() if "locale" in dotenv_config and "de" in dotenv_config["locale"]: expect(page.get_by_role("heading")).to_have_text("Willkommen bei WordPress!") diff --git a/src/utils.py b/src/utils.py index 92f7b26..dcf66f5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -18,3 +18,8 @@ def rmtree(root_dir: Path): child.unlink() root_dir.rmdir() + + +def make_url(domain: str) -> str: + """adds 'http://' at the beginning of a string""" + return "https://" + domain diff --git a/tests/test_env_resolution.py b/tests/test_env_resolution.py new file mode 100644 index 0000000..9d14908 --- /dev/null +++ b/tests/test_env_resolution.py @@ -0,0 +1,59 @@ +import sys +from pathlib import Path + +from icecream import ic + +sys.path.append(Path(__file__).parent.parent.resolve().__str__()) + +# import pytest + +# from prototyping.sorting_algo import Rule, is_rule_satisfied, sort_by_rules +from src.coordinator import Coordinator +from src.env_file_helper import DependencyRule, EnvFile, sort_env_files_by_rule + +# @pytest.fixture +# def in_list(): +# return ["a", "b", "c", "d", "e", "f", "g"] + + +# @pytest.fixture +# def rules() -> list[Rule]: +# return [ # X depends on Y +# Rule("a", "e"), +# Rule("b", "e"), +# Rule("b", "f"), +# Rule("c", "e"), +# Rule("d", "e"), +# Rule("f", "e"), +# ] + + +# def has_rules_satisfied(in_list, rules): +# rule_satisfied: list[bool] = [] +# for rule in rules: +# if is_rule_satisfied(in_list, rule): +# rule_satisfied.append(True) +# else: +# rule_satisfied.append(False) +# return all(rule_satisfied) + + +# def test_stuff(in_list, rules): +# sort_by_rules(in_list, rules) +# assert has_unique_elements(in_list) +# assert has_rules_satisfied(in_list, rules) + + +ENV_FILES = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik +] + + +env_files: list[EnvFile] = Coordinator._getn_env_files_list(ENV_FILES) +dependency_rules: list[DependencyRule] = Coordinator._get_dependency_rules(env_files) + +ic(env_files) +sorted_env_files = sort_env_files_by_rule(env_files, dependency_rules) +ic(env_files) +ic(sorted_env_files) From 3fa10aaa69582c6beec113d4ed7da58052dc4cff Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 4 Dec 2023 17:09:01 +0100 Subject: [PATCH 06/18] implement env manager (#6) * add EnvManager class * holds all functions that are env file related * integrates runner dependency resolution * add integration and unit tests for EnvManager Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/6 Co-authored-by: Daniel Co-committed-by: Daniel --- conftest.py | 2 + main.py | 2 +- previous-work/wordpress_test.py | 2 +- src/conftest.py | 2 +- src/coordinator.py | 62 ++------ src/{dirmanager.py => dir_manager.py} | 33 ++--- src/env_file_helper.py | 52 ------- src/env_manager.py | 100 +++++++++++++ src/runner.py | 2 +- src/runner_dict.py | 16 +++ src/tests_authentik/fixtures_authentik.py | 2 +- src/tests_authentik/setup_authentik.py | 2 +- src/tests_nextcloud/cleanup_nextcloud.py | 22 +-- src/tests_nextcloud/conftest.py | 17 ++- src/tests_wordpress/conftest.py | 2 +- src/tests_wordpress/setup_wordpress.py | 10 +- .../test_wordpress_localization.py | 2 +- tests/test_env_resolution.py | 132 ++++++++++++------ 18 files changed, 264 insertions(+), 198 deletions(-) create mode 100644 conftest.py rename src/{dirmanager.py => dir_manager.py} (60%) delete mode 100644 src/env_file_helper.py create mode 100644 src/env_manager.py create mode 100644 src/runner_dict.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..8704865 --- /dev/null +++ b/conftest.py @@ -0,0 +1,2 @@ +# this file exists so that tests inside /tests always find /src imports, +# because this will cause the root (/) to be added to sys.path diff --git a/main.py b/main.py index 9005385..5821bdf 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ from pathlib import Path from loguru import logger from src.coordinator import Coordinator -from src.dirmanager import DirManager +from src.dir_manager import DirManager from src.utils import get_session_id # ----------------------------- lookup env files ----------------------------- # diff --git a/previous-work/wordpress_test.py b/previous-work/wordpress_test.py index 1d65311..0e84a46 100644 --- a/previous-work/wordpress_test.py +++ b/previous-work/wordpress_test.py @@ -1,6 +1,6 @@ from playwright.sync_api import BrowserContext, expect -from src.dirmanager import DirManager +from src.dir_manager import DirManager def test_wordpress(admin_session: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): diff --git a/src/conftest.py b/src/conftest.py index 690aaf6..6098f4e 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -14,7 +14,7 @@ from dotenv import dotenv_values from playwright.sync_api import BrowserContext, expect from pytest import Parser -from src.dirmanager import DirManager +from src.dir_manager import DirManager # global timeout and LOCALE LOCALE = {"Accept-Language": "de_DE"} diff --git a/src/coordinator.py b/src/coordinator.py index 0baf934..509aa3e 100644 --- a/src/coordinator.py +++ b/src/coordinator.py @@ -1,78 +1,34 @@ -import shutil from pathlib import Path -from dotenv import dotenv_values from loguru import logger -from src.dirmanager import DirManager -from src.env_file_helper import DependencyRule, EnvFile, sort_env_files_by_rule +from src.dir_manager import DirManager +from src.env_manager import EnvFile, EnvManager from src.html_helper import merge_html_files from src.runner import Runner -from src.tests_authentik.runner_authentik import RunnerAuthentik -from src.tests_nextcloud.runner_nextcloud import RunnerNextcloud -from src.tests_wordpress.runner_wordpress import RunnerWordpress +from src.runner_dict import RUNNER_DICT from src.utils import rmtree -# Register all runners here. Each .env file with TYPE=authentik will be run with RunnerAuthentik -RUNNER_DICT: dict[str, type[Runner]] = { - "authentik": RunnerAuthentik, - "wordpress": RunnerWordpress, - "nextcloud": RunnerNextcloud, -} - class Coordinator: def __init__(self, env_paths_list: list[Path], output_dir: Path, session_id: str) -> None: + # logging out_string = "".join([e.name + "\n" for e in env_paths_list]) out_string += f"output_dir = {output_dir}\n" out_string += f"session_id = {session_id}" logger.info(f"initialize Coordinator instance with\nenv_paths_list =\n{out_string}") self.DIR = DirManager(output_dir=output_dir, session_id=session_id) - self.output_dir = output_dir - self.session_id = session_id - - # parse env files - self.env_files: list[EnvFile] = self._getn_env_files_list(env_paths_list) - self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files) - - @staticmethod - def _getn_env_files_list(env_paths: list[Path]) -> list[EnvFile]: - """Returns a list of EnvFile objects created from the given env files""" - env_files: list[EnvFile] = [] - for env_path in env_paths: - assert env_path.is_file(), f"the env file {env_path} does not exist" - config: dict[str, str] = dotenv_values(env_path) # type: ignore - assert "TYPE" in config, f"the env file {env_path} does not specify the required TYPE key." - env_type = config["TYPE"] - env_files.append(EnvFile(env_path=env_path, config=config, env_type=env_type)) - return env_files - - @staticmethod - def _get_dependency_rules(env_files: list[EnvFile]) -> list[DependencyRule]: - dependency_rules: list[DependencyRule] = [] - for env_file in env_files: - child_runner_class = RUNNER_DICT[env_file.env_type] - for dependency in child_runner_class.dependencies: - dependency_rule = DependencyRule(child=child_runner_class.name, dependency=dependency.name) - dependency_rules.append(dependency_rule) - return dependency_rules + self.ENV = EnvManager(env_paths_list) def setup_test(self) -> None: logger.info("calling setup_test()") self.DIR.create_all_dirs() - self._copy_env_files() - - def _copy_env_files(self) -> None: - """Copies all env files to STATES/env_files. Files will be renamed to their own TYPE value.""" - env_files_dir = self.DIR.STATES / "env_files" - env_files_dir.mkdir(exist_ok=True) - for env_file in self.env_files: - shutil.copy(env_file.env_path, env_files_dir / env_file.env_type) + self.ENV.copy_env_files(self.DIR) def run_test(self) -> None: logger.info("calling run_test()") - self.runners: list[Runner] = self._load_runners(self.env_files) + self.runners: list[Runner] = self._load_runners(self.ENV.env_files) for runner in self.runners: runner.run_setups() for runner in self.runners: @@ -87,7 +43,9 @@ class Coordinator: for env_file in env_files: RunnerClass = RUNNER_DICT[env_file.config["TYPE"]] runners.append( - RunnerClass(dotenv_path=env_file.env_path, output_dir=self.output_dir, session_id=self.session_id) + RunnerClass( + dotenv_path=env_file.env_path, output_dir=self.DIR.output_dir, session_id=self.DIR.session_id + ) ) return runners diff --git a/src/dirmanager.py b/src/dir_manager.py similarity index 60% rename from src/dirmanager.py rename to src/dir_manager.py index b5228bf..10cb867 100644 --- a/src/dirmanager.py +++ b/src/dir_manager.py @@ -20,18 +20,25 @@ class DirManager: # root test dir if isinstance(output_dir, str): output_dir = Path(output_dir) - self._output_dir = output_dir.resolve() + self.output_dir = output_dir.resolve() self.session_id = session_id - def create_all_dirs(self): - self.create_dirs(self._output_dir, exist_ok=True) - self.create_dirs( - [self.SESSION, self.RECORDS, self.HTML, self.STATES, self.ENV_FILES, self.RESULTS], exist_ok=True - ) + def create_all_dirs(self) -> None: + dirs: list[Path] = [ + self.OUTPUT_DIR, + self.SESSION, + self.RECORDS, + self.HTML, + self.STATES, + self.ENV_FILES, + self.RESULTS, + ] + for d in dirs: + d.mkdir(exist_ok=True) @property def OUTPUT_DIR(self): - return self._output_dir + return self.output_dir @property def SESSION(self): @@ -56,15 +63,3 @@ class DirManager: @property def RESULTS(self): return self.SESSION / "results" - - @staticmethod - def create_dirs(dirs: Path | list[Path] | dict[str, Path], exist_ok=False): - match dirs: - case Path(): - dirs.mkdir(exist_ok=exist_ok) - case list(): - for d in dirs: - d.mkdir(exist_ok=exist_ok) - case dict(): - for d in dirs.values(): - d.mkdir(exist_ok=exist_ok) diff --git a/src/env_file_helper.py b/src/env_file_helper.py deleted file mode 100644 index 29fb914..0000000 --- a/src/env_file_helper.py +++ /dev/null @@ -1,52 +0,0 @@ -from pathlib import Path -from typing import NamedTuple - -from loguru import logger - - -class EnvFile(NamedTuple): - env_path: Path - config: dict[str, str] - env_type: str - - def __repr__(self) -> str: - return f"EnvFile(type={self.env_type})" - - -class DependencyRule(NamedTuple): - child: str - dependency: str - - -def _is_rule_satisfied(in_list: list, rule: DependencyRule) -> tuple[bool, int]: - child_indices = [index for index, element in enumerate(in_list) if element.env_type == rule.child] - child_index = min(child_indices) - # child_index = in_list.index(rule.child) - parent_indices = [index for index, element in enumerate(in_list) if element.env_type == rule.dependency] - parent_index = max(parent_indices) - # parent_index = in_list.index(rule.dependency) - return parent_index < child_index, parent_index - - -def sort_env_files_by_rule(env_list: list[EnvFile], rules: list[DependencyRule]) -> list: - in_list = env_list.copy() - - def swap_item_with_previous(in_list: list[EnvFile], index: int): - """swaps item at index N with item at index N-1""" - assert index > 0, "cannot swap with negative index" - in_list[index], in_list[index - 1] = in_list[index - 1], in_list[index] - - for _ in range(10_000): - rule_satisfied: list[bool] = [] - for rule in rules: - is_rule_satisfied, parent_index = _is_rule_satisfied(in_list, rule) - if is_rule_satisfied: - rule_satisfied.append(True) - else: - rule_satisfied.append(False) - # parent_index = in_list.index(rule.dependency) - swap_item_with_previous(in_list, parent_index) - if all(rule_satisfied): - return in_list - logger.error("could not find order that satisfys all rules") - raise ValueError diff --git a/src/env_manager.py b/src/env_manager.py new file mode 100644 index 0000000..cfc81fb --- /dev/null +++ b/src/env_manager.py @@ -0,0 +1,100 @@ +import shutil +from pathlib import Path +from typing import NamedTuple + +from dotenv import dotenv_values + +from src.dir_manager import DirManager +from src.runner_dict import RUNNER_DICT + + +class EnvFile(NamedTuple): + env_path: Path + config: dict[str, str] + env_type: str + + def __repr__(self) -> str: + return f"EnvFile(type={self.env_type})" + + +class DependencyRule(NamedTuple): + child: str + dependency: str + + +class EnvManager: + def __init__(self, env_paths_list: list[Path]): + self.env_files: list[EnvFile] = self._get_env_files(env_paths_list) + self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files) + self.env_files = self.sort_env_files_by_rule(self.env_files, self.dependency_rules) + + @staticmethod + def _get_env_files(env_paths: list[Path]) -> list[EnvFile]: + """Returns a list of EnvFile objects created from the given env files""" + env_files: list[EnvFile] = [] + for env_path in env_paths: + assert env_path.is_file(), f"the env file {env_path} does not exist" + config: dict[str, str] = dotenv_values(env_path) # type: ignore + assert "TYPE" in config, f"the env file {env_path} does not specify the required TYPE key." + env_type = config["TYPE"] + env_files.append(EnvFile(env_path=env_path, config=config, env_type=env_type)) + return env_files + + @staticmethod + def _get_dependency_rules(env_files: list[EnvFile]) -> list[DependencyRule]: + dependency_rules: list[DependencyRule] = [] + for env_file in env_files: + child_runner_class = RUNNER_DICT[env_file.env_type] + for dependency in child_runner_class.dependencies: + dependency_rule = DependencyRule(child=child_runner_class.name, dependency=dependency.name) + dependency_rules.append(dependency_rule) + return dependency_rules + + @staticmethod + def _get_indices_by_string(in_list: list[EnvFile], string: str) -> list[int]: + """returns all indices of items in in_list, where item.env_type matches string""" + return [index for index, element in enumerate(in_list) if element.env_type == string] + + @staticmethod + def _swap_item_with_previous(in_list: list[EnvFile], index: int): + """swaps item at index N with item at index N-1""" + assert index > 0, "cannot swap with negative index" + in_list[index], in_list[index - 1] = in_list[index - 1], in_list[index] + + @classmethod + def is_rule_satisfied(cls, env_list: list[EnvFile], rule: DependencyRule, swap=False) -> bool: + """returns if the ordering in in_list is compliant with the given rule + + if swap=True, some reordering will happen in case of a violated rule""" + + child_indices = cls._get_indices_by_string(env_list, rule.child) + parent_indices = cls._get_indices_by_string(env_list, rule.dependency) + for child_index in child_indices: + for parent_index in parent_indices: + if not parent_index < child_index: + if swap: + cls._swap_item_with_previous(env_list, parent_index) + return False + return True + + @classmethod + def sort_env_files_by_rule(cls, env_list: list[EnvFile], rules: list[DependencyRule]) -> list[EnvFile]: + out_list = env_list.copy() + + for _ in range(10_000): + rule_satisfied: list[bool] = [] + for rule in rules: + rule_satisfied.append(cls.is_rule_satisfied(out_list, rule, swap=True)) + + if all(rule_satisfied): + return out_list + raise ValueError( + "Could not resolve test order. This is possibly due to a circular dependency (a on b, b on c, c on a)" + ) + + def copy_env_files(self, DIR: DirManager) -> None: + """Copies all env files to STATES/env_files. Files will be renamed to their own TYPE value.""" + env_files_dir = DIR.STATES / "env_files" + env_files_dir.mkdir(exist_ok=True) + for env_file in self.env_files: + shutil.copy(env_file.env_path, env_files_dir / env_file.env_type) diff --git a/src/runner.py b/src/runner.py index 71ca7ed..8ef444a 100644 --- a/src/runner.py +++ b/src/runner.py @@ -6,7 +6,7 @@ import pytest from dotenv import dotenv_values from loguru import logger -from src.dirmanager import DirManager +from src.dir_manager import DirManager @dataclass diff --git a/src/runner_dict.py b/src/runner_dict.py new file mode 100644 index 0000000..5b000db --- /dev/null +++ b/src/runner_dict.py @@ -0,0 +1,16 @@ +from typing import TYPE_CHECKING + +from src.tests_authentik.runner_authentik import RunnerAuthentik +from src.tests_nextcloud.runner_nextcloud import RunnerNextcloud +from src.tests_wordpress.runner_wordpress import RunnerWordpress + +if TYPE_CHECKING: + from src.runner import Runner + +# Register all runners here. Each .env file with TYPE=authentik will be run with RunnerAuthentik + +RUNNER_DICT: dict[str, type["Runner"]] = { + "authentik": RunnerAuthentik, + "wordpress": RunnerWordpress, + "nextcloud": RunnerNextcloud, +} diff --git a/src/tests_authentik/fixtures_authentik.py b/src/tests_authentik/fixtures_authentik.py index d4c21c8..6b24538 100644 --- a/src/tests_authentik/fixtures_authentik.py +++ b/src/tests_authentik/fixtures_authentik.py @@ -4,7 +4,7 @@ import pytest from dotenv import dotenv_values from playwright.sync_api import BrowserContext, Page -from src.dirmanager import DirManager +from src.dir_manager import DirManager @pytest.fixture diff --git a/src/tests_authentik/setup_authentik.py b/src/tests_authentik/setup_authentik.py index 7927895..18a4e3f 100644 --- a/src/tests_authentik/setup_authentik.py +++ b/src/tests_authentik/setup_authentik.py @@ -4,7 +4,7 @@ import re from playwright.sync_api import BrowserContext, expect -from src.dirmanager import DirManager +from src.dir_manager import DirManager ADMIN_USER = os.environ["ADMIN_USER"] ADMIN_PASS = os.environ["ADMIN_PASS"] diff --git a/src/tests_nextcloud/cleanup_nextcloud.py b/src/tests_nextcloud/cleanup_nextcloud.py index 0bd456f..22e4f4f 100644 --- a/src/tests_nextcloud/cleanup_nextcloud.py +++ b/src/tests_nextcloud/cleanup_nextcloud.py @@ -1,25 +1,17 @@ -import pytest -from playwright.sync_api import BrowserContext, expect +import os -# todo: what is this test for, why is it a fixture? -> ignore for now +from playwright.sync_api import Page -@pytest.fixture(scope="session", autouse=True) -def delete_nextcloud_user(admin_context: BrowserContext): +def delete_nextcloud_user(authentik_admin_page: Page): """Delete Nextcloud User""" - yield - context = setup_context(browser, f"{STATES}/admin_state.json") - page = context.new_page() - page.goto(CONFIG["domain"]) - with page.expect_popup() as nextcloud_info: - page.get_by_role("link", name="Nextcloud").click() + with authentik_admin_page.expect_popup() as nextcloud_info: + authentik_admin_page.get_by_role("link", name="Nextcloud").click() nextcloud = nextcloud_info.value nextcloud.get_by_role("link", name="Open settings menu").click() nextcloud.get_by_role("link", name="Users").click() - nextcloud.locator("#app-content div").filter(has_text=testuser["username"]).get_by_role( + nextcloud.locator("#app-content div").filter(has_text=os.environ["NEXTCLOUD_USER"]).get_by_role( "button", name="Toggle user actions menu" ).click() nextcloud.get_by_role("button", name="Delete user").click() - nextcloud.get_by_role("button", name=f"Delete authentik-{testuser['username']}'s account").click() - context.tracing.stop(path=f"{RECORDS}/nextcloud_delete_user.zip") - context.close() + nextcloud.get_by_role("button", name=f"Delete authentik-{os.environ["NEXTCLOUD_USER"]}'s account").click() diff --git a/src/tests_nextcloud/conftest.py b/src/tests_nextcloud/conftest.py index df1eac3..0805ce5 100644 --- a/src/tests_nextcloud/conftest.py +++ b/src/tests_nextcloud/conftest.py @@ -1 +1,16 @@ -from src.tests_authentik.fixtures_authentik import admin_context, authentik_admin_page, user_context +import os + +from src.tests_authentik.fixtures_authentik import ( + authentik_admin_context, + authentik_admin_page, + authentik_user_context, + authentik_user_page, +) + +NEXTCLOUD_DEMO_USER = { + "NEXTCLOUD_USER": "next_demo_user", + "NEXTCLOUD_PASS": "P@ss.123", +} + +for key, value in NEXTCLOUD_DEMO_USER.items(): + os.environ[key] = value diff --git a/src/tests_wordpress/conftest.py b/src/tests_wordpress/conftest.py index f2d4e2c..ac095dc 100644 --- a/src/tests_wordpress/conftest.py +++ b/src/tests_wordpress/conftest.py @@ -4,7 +4,7 @@ import pytest from dotenv import dotenv_values from playwright.sync_api import BrowserContext, Page -from src.dirmanager import DirManager +from src.dir_manager import DirManager # from src.tests_authentik.fixtures_authentik import ( # authentik_admin_context, diff --git a/src/tests_wordpress/setup_wordpress.py b/src/tests_wordpress/setup_wordpress.py index 6543c9d..4feca93 100644 --- a/src/tests_wordpress/setup_wordpress.py +++ b/src/tests_wordpress/setup_wordpress.py @@ -1,19 +1,17 @@ import pytest from playwright.sync_api import BrowserContext, Page, expect -from src.dirmanager import DirManager +from src.dir_manager import DirManager -@pytest.mark.xfail(reason="wordpress sso login has not been generated") def test_visit_from_domain(authentik_admin_context: BrowserContext, dotenv_config: dict[str, str]): """visit wordpress directly with admin_session, expect not to be logged in""" page = authentik_admin_context.new_page() url = "https://" + dotenv_config["DOMAIN"] page.goto(url) - # look for content wrapper - expect(page.locator("#wpcontent")).to_be_visible(timeout=3_000) - # look for admin bar - expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000) + with pytest.raises(AssertionError): + # look for admin bar + expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000) def setup_wordpress_admin_session(authentik_admin_page: Page, DIR: DirManager): diff --git a/src/tests_wordpress/test_wordpress_localization.py b/src/tests_wordpress/test_wordpress_localization.py index 77bce95..8c9ffa4 100644 --- a/src/tests_wordpress/test_wordpress_localization.py +++ b/src/tests_wordpress/test_wordpress_localization.py @@ -2,7 +2,7 @@ from playwright.sync_api import BrowserContext, expect -from src.dirmanager import DirManager +from src.dir_manager import DirManager def test_welcome_message(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): diff --git a/tests/test_env_resolution.py b/tests/test_env_resolution.py index 9d14908..2c1bc97 100644 --- a/tests/test_env_resolution.py +++ b/tests/test_env_resolution.py @@ -1,59 +1,101 @@ -import sys from pathlib import Path -from icecream import ic +import pytest -sys.path.append(Path(__file__).parent.parent.resolve().__str__()) - -# import pytest - -# from prototyping.sorting_algo import Rule, is_rule_satisfied, sort_by_rules -from src.coordinator import Coordinator -from src.env_file_helper import DependencyRule, EnvFile, sort_env_files_by_rule - -# @pytest.fixture -# def in_list(): -# return ["a", "b", "c", "d", "e", "f", "g"] +# from src.env_file_helper import DependencyRule, EnvFile, sort_env_files_by_rule +from src.env_manager import DependencyRule, EnvFile, EnvManager -# @pytest.fixture -# def rules() -> list[Rule]: -# return [ # X depends on Y -# Rule("a", "e"), -# Rule("b", "e"), -# Rule("b", "f"), -# Rule("c", "e"), -# Rule("d", "e"), -# Rule("f", "e"), -# ] +def test_complex_sorting() -> None: + demo_rules = [ # X depends on Y + DependencyRule("a", "e"), + DependencyRule("b", "e"), + DependencyRule("b", "f"), + DependencyRule("c", "e"), + DependencyRule("d", "e"), + DependencyRule("f", "e"), + ] + + demo_types = ["a", "b", "c", "d", "e", "f", "g"] + env_files = [EnvFile(env_type=t, env_path=Path(), config=dict()) for t in demo_types] + EnvManager.sort_env_files_by_rule + sorted_env_files = EnvManager.sort_env_files_by_rule(env_files, demo_rules) + + assert sorted_env_files[0].env_type == "e" -# def has_rules_satisfied(in_list, rules): -# rule_satisfied: list[bool] = [] -# for rule in rules: -# if is_rule_satisfied(in_list, rule): -# rule_satisfied.append(True) -# else: -# rule_satisfied.append(False) -# return all(rule_satisfied) +def test_circular_import() -> None: + """This test will raise ValueError because the example input cannot be correctly ordered""" + demo_rules = [ + DependencyRule("a", "b"), + DependencyRule("b", "c"), + DependencyRule("c", "a"), + ] + + demo_types = ["a", "b", "c"] + env_files = [EnvFile(env_type=t, env_path=Path(), config=dict()) for t in demo_types] + with pytest.raises(ValueError): + EnvManager.sort_env_files_by_rule(env_files, demo_rules) -# def test_stuff(in_list, rules): -# sort_by_rules(in_list, rules) -# assert has_unique_elements(in_list) -# assert has_rules_satisfied(in_list, rules) +def test_real_env_files() -> None: + """authentik should be first""" + + ENV_FILES = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + ] + env_files: list[EnvFile] = EnvManager._get_env_files(ENV_FILES) + dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files) + sorted_env_files = EnvManager.sort_env_files_by_rule(env_files, dependency_rules) + assert sorted_env_files[0].env_type == "authentik" -ENV_FILES = [ - Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress - Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik -] +def test_real_env_files_duplicate() -> None: + """authentik should be first""" + + ENV_FILES = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + ] + env_files: list[EnvFile] = EnvManager._get_env_files(ENV_FILES) + dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files) + sorted_env_files = EnvManager.sort_env_files_by_rule(env_files, dependency_rules) + assert sorted_env_files[0].env_type == "authentik" + assert sorted_env_files[1].env_type == "authentik" + assert sorted_env_files[2].env_type == "wordpress" -env_files: list[EnvFile] = Coordinator._getn_env_files_list(ENV_FILES) -dependency_rules: list[DependencyRule] = Coordinator._get_dependency_rules(env_files) +def test_real_env_files_duplicate_six() -> None: + """authentik should be first""" -ic(env_files) -sorted_env_files = sort_env_files_by_rule(env_files, dependency_rules) -ic(env_files) -ic(sorted_env_files) + ENV_FILES = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + ] + env_files: list[EnvFile] = EnvManager._get_env_files(ENV_FILES) + dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files) + sorted_env_files = EnvManager.sort_env_files_by_rule(env_files, dependency_rules) + assert sorted_env_files[0].env_type == "authentik" + assert sorted_env_files[1].env_type == "authentik" + assert sorted_env_files[2].env_type == "authentik" + assert sorted_env_files[3].env_type == "wordpress" + assert sorted_env_files[4].env_type == "wordpress" + assert sorted_env_files[5].env_type == "wordpress" + + +def test_env_manager() -> None: + env_paths_list = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + ] + ENV = EnvManager(env_paths_list) + assert ENV.env_files[0].env_type == "authentik" + assert ENV.env_files[1].env_type == "authentik" + assert ENV.env_files[2].env_type == "wordpress" From f9c21c6e6b147046f4c0c35f7a37ec59351923ae Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 5 Dec 2023 21:41:43 +0100 Subject: [PATCH 07/18] refactor for independent test dirs (#7) * make it so that the actual tests can be moved anywhere, for example in abra recipe repos -> major refactoring with pytest test discovery magic * create RUNNER_DICT dynamically with importlib -> none of the tests are hardcoded, more tests can be added by placing a folder * autoload fixtures with pytest plugins * add URL fixture to navigate on web pages. Includes url parser based on python urllib to generate correct links * fix nextcloud setups and tests * add email groundwork with imbox Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/7 Co-authored-by: Daniel Co-committed-by: Daniel --- README.md | 29 ++++++--- {src => abratest}/__init__.py | 0 {src => abratest}/coordinator.py | 60 ++++++++++++++----- {src => abratest}/dir_manager.py | 10 +++- {src => abratest}/env_manager.py | 12 ++-- {src => abratest}/html_helper.py | 0 src/conftest.py => abratest/plugin-abra.py | 10 +++- {src => abratest}/runner.py | 38 ++++++------ {src => abratest}/utils.py | 15 +++++ main.py | 27 +++++---- previous-work/wordpress_test.py | 2 +- prototyping/email_stuff.py | 18 +++--- prototyping/email_stuff_2.py | 53 ++++++++++++++++ prototyping/test_urllib.py | 10 ---- pyproject.toml | 10 ++-- .../authentik/tests_authentik/__init__.py | 0 .../tests_authentik/fixtures_authentik.py | 2 +- .../tests_authentik/runner_authentik.py | 2 +- .../tests_authentik/setup_authentik.py | 25 ++++---- .../tests_authentik/test_authentik_dummy.py | 0 recipes/demo/tests_demo/__init__.py | 0 .../demo}/tests_demo/fixtures_demo.py | 4 +- .../demo}/tests_demo/runner_demo.py | 5 +- .../demo}/tests_demo/setup_demo.py | 0 recipes/nextcloud/tests_nextcloud/__init__.py | 0 .../tests_nextcloud/cleanup_nextcloud.py | 0 recipes/nextcloud/tests_nextcloud/conftest.py | 33 ++++++++++ .../tests_nextcloud/runner_nextcloud.py | 17 ++++++ .../tests_nextcloud/setup_nextcloud.py | 21 +++++++ .../tests_nextcloud/tests_nextcloud.py | 32 ++++++++++ .../tests_nextcloud_onlyoffice.py | 0 recipes/wordpress/tests_wordpress/__init__.py | 0 .../wordpress}/tests_wordpress/conftest.py | 11 +--- .../tests_wordpress/runner_wordpress.py | 5 +- .../tests_wordpress/setup_wordpress.py | 4 +- .../tests_wordpress/test_wordpress.py | 0 .../test_wordpress_localization.py | 2 +- requirements.txt | 3 +- src/runner_dict.py | 16 ----- src/tests_nextcloud/conftest.py | 16 ----- src/tests_nextcloud/runner_nextcloud.py | 18 ------ src/tests_nextcloud/setup_nextcloud.py | 35 ----------- src/tests_nextcloud/tests_nextcloud.py | 13 ---- tests/test_env_resolution.py | 15 +++-- tests/test_url.py | 28 +++++++++ 45 files changed, 373 insertions(+), 228 deletions(-) rename {src => abratest}/__init__.py (100%) rename {src => abratest}/coordinator.py (53%) rename {src => abratest}/dir_manager.py (82%) rename {src => abratest}/env_manager.py (91%) rename {src => abratest}/html_helper.py (100%) rename src/conftest.py => abratest/plugin-abra.py (90%) rename {src => abratest}/runner.py (84%) rename {src => abratest}/utils.py (61%) create mode 100644 prototyping/email_stuff_2.py delete mode 100644 prototyping/test_urllib.py rename src/tests_wordpress/test_wordpress.py => recipes/authentik/tests_authentik/__init__.py (100%) rename {src => recipes/authentik}/tests_authentik/fixtures_authentik.py (96%) rename {src => recipes/authentik}/tests_authentik/runner_authentik.py (90%) rename {src => recipes/authentik}/tests_authentik/setup_authentik.py (88%) rename {src => recipes/authentik}/tests_authentik/test_authentik_dummy.py (100%) create mode 100644 recipes/demo/tests_demo/__init__.py rename {src => recipes/demo}/tests_demo/fixtures_demo.py (83%) rename {src => recipes/demo}/tests_demo/runner_demo.py (87%) rename {src => recipes/demo}/tests_demo/setup_demo.py (100%) create mode 100644 recipes/nextcloud/tests_nextcloud/__init__.py rename {src => recipes/nextcloud}/tests_nextcloud/cleanup_nextcloud.py (100%) create mode 100644 recipes/nextcloud/tests_nextcloud/conftest.py create mode 100644 recipes/nextcloud/tests_nextcloud/runner_nextcloud.py create mode 100644 recipes/nextcloud/tests_nextcloud/setup_nextcloud.py create mode 100644 recipes/nextcloud/tests_nextcloud/tests_nextcloud.py rename {src => recipes/nextcloud}/tests_nextcloud/tests_nextcloud_onlyoffice.py (100%) create mode 100644 recipes/wordpress/tests_wordpress/__init__.py rename {src => recipes/wordpress}/tests_wordpress/conftest.py (73%) rename {src => recipes/wordpress}/tests_wordpress/runner_wordpress.py (80%) rename {src => recipes/wordpress}/tests_wordpress/setup_wordpress.py (90%) create mode 100644 recipes/wordpress/tests_wordpress/test_wordpress.py rename {src => recipes/wordpress}/tests_wordpress/test_wordpress_localization.py (91%) delete mode 100644 src/runner_dict.py delete mode 100644 src/tests_nextcloud/conftest.py delete mode 100644 src/tests_nextcloud/runner_nextcloud.py delete mode 100644 src/tests_nextcloud/setup_nextcloud.py delete mode 100644 src/tests_nextcloud/tests_nextcloud.py create mode 100644 tests/test_url.py diff --git a/README.md b/README.md index a046244..54547be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ -# GIT Clone +# AbraTest +...description... + +# Usage + +To use AbraTest, follow these steps: + +## 1. GIT Clone To clone with submodules, use these git commands: @@ -8,7 +15,11 @@ git submodule update --init // add submodule after normal cloning git submodule update --remote // update submodules ``` -# Run without Docker +## Run + +You can run AbraTest with and without Docker. Choose now and follow the steps accordingly: + +## 2.1 Run without Docker ### Installation @@ -22,16 +33,16 @@ playwright install Run the script with ```bash -python main.py +python main.py # run abratest +pytest # test abratest ``` -# Run with Docker +# 2.2 Run with Docker ```bash -docker compose build -docker compose run --rm app python ./main.py -docker compose run --rm app pytest -# docker-compose up +docker compose build # build the image +docker compose run --rm app python ./main.py # run AbraTest +docker compose run --rm app pytest # test AbraTest ``` Force rebuild with cache @@ -46,7 +57,7 @@ Force rebuild wtihtout cache docker-compose build --no-cache ``` -# Codegen +## Codegen Use playwright codegen to create code for new testes easily https://playwright.dev/python/docs/codegen diff --git a/src/__init__.py b/abratest/__init__.py similarity index 100% rename from src/__init__.py rename to abratest/__init__.py diff --git a/src/coordinator.py b/abratest/coordinator.py similarity index 53% rename from src/coordinator.py rename to abratest/coordinator.py index 509aa3e..11e4b7f 100644 --- a/src/coordinator.py +++ b/abratest/coordinator.py @@ -1,25 +1,27 @@ +import importlib +import re from pathlib import Path from loguru import logger -from src.dir_manager import DirManager -from src.env_manager import EnvFile, EnvManager -from src.html_helper import merge_html_files -from src.runner import Runner -from src.runner_dict import RUNNER_DICT -from src.utils import rmtree +from abratest.dir_manager import DirManager +from abratest.env_manager import EnvFile, EnvManager +from abratest.html_helper import merge_html_files +from abratest.runner import Runner +from abratest.utils import rmtree class Coordinator: - def __init__(self, env_paths_list: list[Path], output_dir: Path, session_id: str) -> None: + def __init__(self, env_paths_list: list[Path], output_dir: Path, session_id: str, recipes_dir: Path) -> None: # logging out_string = "".join([e.name + "\n" for e in env_paths_list]) out_string += f"output_dir = {output_dir}\n" out_string += f"session_id = {session_id}" logger.info(f"initialize Coordinator instance with\nenv_paths_list =\n{out_string}") - self.DIR = DirManager(output_dir=output_dir, session_id=session_id) - self.ENV = EnvManager(env_paths_list) + self.RUNNER_DICT = self.create_runner_dict(recipes_dir) + self.DIR = DirManager(output_dir=output_dir, session_id=session_id, recipes_dir=recipes_dir) + self.ENV = EnvManager(env_paths_list, self.RUNNER_DICT) def setup_test(self) -> None: logger.info("calling setup_test()") @@ -39,14 +41,15 @@ class Coordinator: def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]: """Creates an instance of the correct Runner class for each given env file""" - runners = [] + runners: list[Runner] = [] for env_file in env_files: - RunnerClass = RUNNER_DICT[env_file.config["TYPE"]] - runners.append( - RunnerClass( - dotenv_path=env_file.env_path, output_dir=self.DIR.output_dir, session_id=self.DIR.session_id - ) - ) + RunnerClass = self.RUNNER_DICT[env_file.config["TYPE"]] + dependency_classes: list[type[Runner]] = [] + for dependency in RunnerClass.dependencies: + dependency_classes.append(self.RUNNER_DICT[dependency]) + runner_instance = RunnerClass(dotenv_path=env_file.env_path, DIR=self.DIR) + runner_instance._dependency_runners = dependency_classes + runners.append(runner_instance) return runners def combine_html(self) -> None: @@ -78,3 +81,28 @@ class Coordinator: new_path = get_new_path(self.DIR.RECORDS, f.parent.name) f.parent.rename(new_path) rmtree(trace_root_dir) + + @staticmethod + def create_runner_dict(recipes_dir: Path) -> dict[str, type["Runner"]]: + """Creates a dictionary holding all the RunnerClasses that can be discovered in recipes_dir + + example: + RUNNER_DICT: dict[str, type["Runner"]] = { + "authentik": RunnerAuthentik, + "wordpress": RunnerWordpress, + "nextcloud": RunnerNextcloud, + } + """ + + RUNNER_DICT: dict[str, type["Runner"]] = dict() + runner_discovery_pattern = re.compile("Runner.+") + + for module_path in recipes_dir.rglob("*/runner*.py"): + rel_path = module_path.relative_to(recipes_dir).as_posix().replace("/", ".").replace(".py", "") + module = importlib.import_module(rel_path) + runner_class_names = [name for name in dir(module) if runner_discovery_pattern.match(name)] + assert len(runner_class_names) == 1 + runner_class_name = runner_class_names[0] + RunnerClass: type[Runner] = getattr(module, runner_class_name) + RUNNER_DICT[RunnerClass.name] = RunnerClass + return RUNNER_DICT diff --git a/src/dir_manager.py b/abratest/dir_manager.py similarity index 82% rename from src/dir_manager.py rename to abratest/dir_manager.py index 10cb867..e172ef5 100644 --- a/src/dir_manager.py +++ b/abratest/dir_manager.py @@ -16,12 +16,14 @@ class DirManager: ... """ - def __init__(self, output_dir: Path | str, session_id: str): - # root test dir + def __init__(self, output_dir: Path | str, session_id: str, recipes_dir: Path | str = ""): if isinstance(output_dir, str): output_dir = Path(output_dir) self.output_dir = output_dir.resolve() self.session_id = session_id + if isinstance(recipes_dir, str): + recipes_dir = Path(recipes_dir) + self.recipes_dir = recipes_dir def create_all_dirs(self) -> None: dirs: list[Path] = [ @@ -63,3 +65,7 @@ class DirManager: @property def RESULTS(self): return self.SESSION / "results" + + @property + def RECIPES(self): + return self.recipes_dir diff --git a/src/env_manager.py b/abratest/env_manager.py similarity index 91% rename from src/env_manager.py rename to abratest/env_manager.py index cfc81fb..18086e2 100644 --- a/src/env_manager.py +++ b/abratest/env_manager.py @@ -4,8 +4,8 @@ from typing import NamedTuple from dotenv import dotenv_values -from src.dir_manager import DirManager -from src.runner_dict import RUNNER_DICT +from abratest.dir_manager import DirManager +from abratest.runner import Runner class EnvFile(NamedTuple): @@ -23,9 +23,9 @@ class DependencyRule(NamedTuple): class EnvManager: - def __init__(self, env_paths_list: list[Path]): + def __init__(self, env_paths_list: list[Path], RUNNER_DICT: dict[str, type["Runner"]]): self.env_files: list[EnvFile] = self._get_env_files(env_paths_list) - self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files) + self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files, RUNNER_DICT) self.env_files = self.sort_env_files_by_rule(self.env_files, self.dependency_rules) @staticmethod @@ -41,12 +41,12 @@ class EnvManager: return env_files @staticmethod - def _get_dependency_rules(env_files: list[EnvFile]) -> list[DependencyRule]: + def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, type["Runner"]]) -> list[DependencyRule]: dependency_rules: list[DependencyRule] = [] for env_file in env_files: child_runner_class = RUNNER_DICT[env_file.env_type] for dependency in child_runner_class.dependencies: - dependency_rule = DependencyRule(child=child_runner_class.name, dependency=dependency.name) + dependency_rule = DependencyRule(child=child_runner_class.name, dependency=dependency) dependency_rules.append(dependency_rule) return dependency_rules diff --git a/src/html_helper.py b/abratest/html_helper.py similarity index 100% rename from src/html_helper.py rename to abratest/html_helper.py diff --git a/src/conftest.py b/abratest/plugin-abra.py similarity index 90% rename from src/conftest.py rename to abratest/plugin-abra.py index 6098f4e..c72b21b 100644 --- a/src/conftest.py +++ b/abratest/plugin-abra.py @@ -14,11 +14,12 @@ from dotenv import dotenv_values from playwright.sync_api import BrowserContext, expect from pytest import Parser -from src.dir_manager import DirManager +from abratest.dir_manager import DirManager +from abratest.utils import BaseUrl # global timeout and LOCALE LOCALE = {"Accept-Language": "de_DE"} -TIMEOUT = 7_000 +TIMEOUT = 20_000 expect.set_options(timeout=TIMEOUT) @@ -73,6 +74,11 @@ def dotenv_config(request) -> dict[str, str]: return dotenv_values(dotenv_path) # type: ignore +@pytest.fixture(scope="session", autouse=True) +def URL(dotenv_config: dict[str, str]) -> BaseUrl: + return BaseUrl(netloc=dotenv_config["DOMAIN"]) + + @pytest.fixture(scope="session") def imap_ssl_email_client() -> None: assert os.environ["IMAP_HOST"] diff --git a/src/runner.py b/abratest/runner.py similarity index 84% rename from src/runner.py rename to abratest/runner.py index 8ef444a..8ea95a4 100644 --- a/src/runner.py +++ b/abratest/runner.py @@ -6,7 +6,7 @@ import pytest from dotenv import dotenv_values from loguru import logger -from src.dir_manager import DirManager +from abratest.dir_manager import DirManager @dataclass @@ -22,15 +22,13 @@ class Runner: setups: list[Test] = [] tests: list[Test] = [] cleanups: list[Test] = [] - dependencies: list[type["Runner"]] = [] - prevent_skip = False + dependencies: list[str] = [] + _dependency_runners: list[type["Runner"]] = [] - def __init__(self, dotenv_path: Path, output_dir: Path, session_id: str): + def __init__(self, dotenv_path: Path, DIR: DirManager): self.dotenv_path = dotenv_path self.config: dict[str, str] = dotenv_values(dotenv_path) # type: ignore - self.output_dir = output_dir - self.session_id = session_id - self.DIRS = DirManager(output_dir, session_id) + self.DIR = DIR logger.info(f"creating instance of {self.__class__.__name__}") assert self.test_dir_name @@ -66,7 +64,7 @@ class Runner: # condition_met: true / false identifier_string = self.combine_names(self.name, test.test_file) - test_path = self.root_dir / self.test_dir_name / test.test_file + full_test_path = self.DIR.RECIPES / self.name / self.test_dir_name / test.test_file # check if test aleady passed if self._is_test_passed(identifier_string, remove_existing=True): @@ -83,7 +81,7 @@ class Runner: # test condition is undefined or not met logger.info(f"running {identifier_string}") - result = self._call_pytest(test_path) + result = self._call_pytest(full_test_path) self._create_result_file(result=result, identifier_string=identifier_string) def _is_test_passed(self, identifier_string: str, remove_existing: bool = False) -> bool: @@ -96,7 +94,7 @@ class Runner: other than 'passed' will be deleted""" already_passed = False - for result in self.DIRS.RESULTS.glob("*"): + for result in self.DIR.RESULTS.glob("*"): if identifier_string in result.name: # process any result file (passed / failed / skipped) if it exists if "passed" in result.name: @@ -112,6 +110,8 @@ class Runner: command_arguments = [] + # command_arguments.append("--traceconfig") + command_arguments.append("-v") # command_arguments.append("-rx") command_arguments.append(str(full_test_path)) @@ -121,17 +121,17 @@ class Runner: # set root dir for tests output (used in DirManager). this is our custom argument command_arguments.append("--output_dir") - command_arguments.append(str(self.DIRS.OUTPUT_DIR)) + command_arguments.append(str(self.DIR.OUTPUT_DIR)) command_arguments.append("--session_id") - command_arguments.append(self.session_id) + command_arguments.append(self.DIR.session_id) # artifacts dir from pytest # warning: https://github.com/microsoft/playwright-pytest/issues/111 # --output only works with the given context and page fixture # folder needs to be unique! traces will not appear, if every pytest run has same output dir command_arguments.append("--output") - command_arguments.append(str(self.DIRS.RECORDS / "traces" / full_test_path.stem)) + command_arguments.append(str(self.DIR.RECORDS / "traces" / full_test_path.stem)) # tracing command_arguments.append("--tracing") @@ -145,7 +145,7 @@ class Runner: # command_arguments.append("--headed") # html report. Will be combined into one file later. - command_arguments.append(f"--html={self.DIRS.RECORDS / 'html' / full_test_path.with_suffix('.html').name}") + command_arguments.append(f"--html={self.DIR.RECORDS / 'html' / full_test_path.with_suffix('.html').name}") return pytest.main(command_arguments) @@ -157,7 +157,7 @@ class Runner: """create result file to indicated passed/failed or skipped test""" full_name = self.combine_names(self.result_int_to_str(result), identifier_string) - file_path = self.DIRS.RESULTS / full_name + file_path = self.DIR.RESULTS / full_name with open(file_path, "w") as _: pass # create empty file @@ -166,11 +166,11 @@ class Runner: # todo: what about conditional setups? - passed_tests = [r.name for r in self.DIRS.RESULTS.glob("*") if "passed" in r.name] + passed_tests = [r.name for r in self.DIR.RESULTS.glob("*") if "passed" in r.name] results = [] - for dependencie_runner in self.dependencies: - for setup_name in dependencie_runner.setups: - dependencie_identifier = self.combine_names(dependencie_runner.name, setup_name.test_file) + for dependency_runner in self._dependency_runners: + for setup_name in dependency_runner.setups: + dependencie_identifier = self.combine_names(dependency_runner.name, setup_name.test_file) results.append(any(dependencie_identifier in f for f in passed_tests)) return all(results) diff --git a/src/utils.py b/abratest/utils.py similarity index 61% rename from src/utils.py rename to abratest/utils.py index dcf66f5..e98fa9b 100644 --- a/src/utils.py +++ b/abratest/utils.py @@ -1,5 +1,20 @@ +from dataclasses import dataclass from datetime import datetime from pathlib import Path +from urllib.parse import urlunparse + + +@dataclass +class BaseUrl: + netloc: str + scheme: str = "https" + path: str = "" + params: str = "" + query: str = "" + fragment: str = "" + + def get(self, path: str = ""): + return urlunparse((self.scheme, self.netloc, path, self.params, self.query, self.fragment)) def get_session_id() -> str: diff --git a/main.py b/main.py index 5821bdf..8c2d739 100644 --- a/main.py +++ b/main.py @@ -4,9 +4,9 @@ from pathlib import Path from loguru import logger -from src.coordinator import Coordinator -from src.dir_manager import DirManager -from src.utils import get_session_id +from abratest.coordinator import Coordinator +from abratest.dir_manager import DirManager +from abratest.utils import get_session_id # ----------------------------- lookup env files ----------------------------- # @@ -23,6 +23,7 @@ from src.utils import get_session_id ENV_FILES = [ Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + # Path("envfiles/files.test.dev.local-it.cloud.env"), # nextcloud ] @@ -30,12 +31,7 @@ ENV_FILES = [ OUTPUT_DIR = Path("./test-output").resolve() - - -# -------------------------- enable playwright debug ------------------------- # - - -# os.environ["PWDEBUG"] = "1" +RECIPES_DIR = Path("./recipes").resolve() # --------------------- load credentials to env variables -------------------- # @@ -49,6 +45,14 @@ for key, value in CREDENTIALS.items(): os.environ[key] = value +# -------------------------- enable playwright debug ------------------------- # + + +# add abra-testing dir +os.environ["PYTEST_PLUGINS"] = "abratest.plugin-abra" # "abratest.plugin,abratest.other" +# os.environ["PWDEBUG"] = "1" + + # ----------------------------- define session_id ---------------------------- # @@ -58,6 +62,7 @@ session_id = get_session_id() # ------------------------------- setup logging ------------------------------ # + DIR = DirManager(output_dir=OUTPUT_DIR, session_id=session_id) log_file = DIR.RECORDS / "coordinator.log" logger.add(log_file) @@ -66,7 +71,9 @@ logger.add(log_file) # ---------------------------- initialize and run ---------------------------- # -coordinator = Coordinator(ENV_FILES, output_dir=OUTPUT_DIR, session_id=session_id) +coordinator = Coordinator( + env_paths_list=ENV_FILES, output_dir=OUTPUT_DIR, session_id=session_id, recipes_dir=RECIPES_DIR +) coordinator.setup_test() coordinator.run_test() coordinator.combine_html() diff --git a/previous-work/wordpress_test.py b/previous-work/wordpress_test.py index 0e84a46..bf61351 100644 --- a/previous-work/wordpress_test.py +++ b/previous-work/wordpress_test.py @@ -1,6 +1,6 @@ from playwright.sync_api import BrowserContext, expect -from src.dir_manager import DirManager +from abratest.dir_manager import DirManager def test_wordpress(admin_session: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): diff --git a/prototyping/email_stuff.py b/prototyping/email_stuff.py index a8896f8..ce0d850 100644 --- a/prototyping/email_stuff.py +++ b/prototyping/email_stuff.py @@ -1,6 +1,7 @@ # %% import email import json +import os from email.header import decode_header from imaplib import IMAP4, IMAP4_SSL from pathlib import Path @@ -11,19 +12,20 @@ cred_file = Path("../credentials.json") with open(cred_file, "r") as f: CREDENTIALS = json.load(f) -username = CREDENTIALS["imap_user"] -password = CREDENTIALS["imap_pass"] +for key, value in CREDENTIALS.items(): + os.environ[key] = value + +IMAP_HOST = os.environ["IMAP_HOST"] +IMAP_PORT = os.environ["IMAP_PORT"] +IMAP_USER = os.environ["IMAP_USER"] +IMAP_PASS = os.environ["IMAP_PASS"] # ----------------------------------- imap ----------------------------------- # -host = "mail.local-it.org" -imap_port = 143 -imap_ssl_port = 993 - -with IMAP4_SSL(host=host) as imap_server: - imap_server.login(username, password) +with IMAP4_SSL(host=IMAP_HOST) as imap_server: + imap_server.login(IMAP_USER, IMAP_PASS) imap_server.select("INBOX") # Search for all emails in the folder diff --git a/prototyping/email_stuff_2.py b/prototyping/email_stuff_2.py new file mode 100644 index 0000000..15cf562 --- /dev/null +++ b/prototyping/email_stuff_2.py @@ -0,0 +1,53 @@ +# %% +import datetime +import json +import os +from pathlib import Path + +from imbox import Imbox + +cred_file = Path("../credentials.json") +with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + +for key, value in CREDENTIALS.items(): + os.environ[key] = value + +IMAP_HOST = os.environ["IMAP_HOST"] +IMAP_PORT = os.environ["IMAP_PORT"] +IMAP_USER = os.environ["IMAP_USER"] +IMAP_PASS = os.environ["IMAP_PASS"] + + +with Imbox( + hostname=os.environ["IMAP_HOST"], + port=os.environ["IMAP_PORT"], + username=os.environ["IMAP_USER"], + password=os.environ["IMAP_PASS"], + ssl=True, + ssl_context=None, + starttls=False, +) as imbox: + # Get all folders + status, folders_with_additional_info = imbox.folders() + + # Gets all messages from the inbox + all_inbox_messages = imbox.messages() + + # Messages received after specific date + inbox_messages_received_after = imbox.messages(date__gt=datetime.date(2018, 7, 30)) + + # Messages whose subjects contain a string + inbox_messages_subject_christmas = imbox.messages(subject="Christmas") + + for uid, message in all_inbox_messages: + print(uid, message.subject, message.date) + # # Every message is an object with the following keys + + # message.sent_from + # message.sent_to + # message.subject + # message.headers + # message.message_id + # message.date + # message.body.plain diff --git a/prototyping/test_urllib.py b/prototyping/test_urllib.py deleted file mode 100644 index 074b0a5..0000000 --- a/prototyping/test_urllib.py +++ /dev/null @@ -1,10 +0,0 @@ -# %% -from urllib.parse import parse_qs, urlencode, urlparse, urlunparse - -string = "blog.dev.local-it.cloud" - - -parsed_url = urlparse(string, scheme="https") -print(parsed_url) - -print(urlunparse(parsed_url)) diff --git a/pyproject.toml b/pyproject.toml index 6912f0b..24f1aa4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] -name = "locit-testing" -version = "0.1.0" +name = "abratest" +version = "0.2.0" requires-python = "~=3.11" dependencies = [ "pytest == 7.4.3", @@ -8,11 +8,11 @@ dependencies = [ [project.optional-dependencies] dev = [ - "ruff >= 0.1.3", + "ruff >= 0.1.7", ] [tool.setuptools] -package-dir = {"" = "src"} +package-dir = {"" = "abratest"} [tool.ruff] line-length = 120 @@ -20,4 +20,4 @@ target-version = "py311" [tool.pytest.ini_options] python_functions = "test_* setup_*" -norecursedirs = "previous-work src" \ No newline at end of file +norecursedirs = "previous-work recipes" \ No newline at end of file diff --git a/src/tests_wordpress/test_wordpress.py b/recipes/authentik/tests_authentik/__init__.py similarity index 100% rename from src/tests_wordpress/test_wordpress.py rename to recipes/authentik/tests_authentik/__init__.py diff --git a/src/tests_authentik/fixtures_authentik.py b/recipes/authentik/tests_authentik/fixtures_authentik.py similarity index 96% rename from src/tests_authentik/fixtures_authentik.py rename to recipes/authentik/tests_authentik/fixtures_authentik.py index 6b24538..3f27b43 100644 --- a/src/tests_authentik/fixtures_authentik.py +++ b/recipes/authentik/tests_authentik/fixtures_authentik.py @@ -4,7 +4,7 @@ import pytest from dotenv import dotenv_values from playwright.sync_api import BrowserContext, Page -from src.dir_manager import DirManager +from abratest.dir_manager import DirManager @pytest.fixture diff --git a/src/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py similarity index 90% rename from src/tests_authentik/runner_authentik.py rename to recipes/authentik/tests_authentik/runner_authentik.py index 5b19788..a3bba32 100644 --- a/src/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -1,4 +1,4 @@ -from src.runner import Runner, Test +from abratest.runner import Runner, Test def condition_always_true(dotenv_config: dict[str, str]) -> bool: diff --git a/src/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py similarity index 88% rename from src/tests_authentik/setup_authentik.py rename to recipes/authentik/tests_authentik/setup_authentik.py index 18a4e3f..b7f7f37 100644 --- a/src/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -4,7 +4,8 @@ import re from playwright.sync_api import BrowserContext, expect -from src.dir_manager import DirManager +from abratest.dir_manager import DirManager +from abratest.utils import BaseUrl ADMIN_USER = os.environ["ADMIN_USER"] ADMIN_PASS = os.environ["ADMIN_PASS"] @@ -31,14 +32,13 @@ def setup_admin_state(context: BrowserContext, dotenv_config: dict[str, str], DI expect(page.locator("ak-library")).to_be_visible() # save state - context.storage_state(path=f"{DIR.STATES}/authentik_admin_state.json") + context.storage_state(path=DIR.STATES / "authentik_admin_state.json") -def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, str]): +def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, str], URL: BaseUrl): # go to admin page page = admin_context.new_page() - url = "https://" + dotenv_config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) page.get_by_role("link", name="Admin Interface").click() nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) nav.click() @@ -49,11 +49,10 @@ def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, return user.is_visible() -def create_invite_link(admin_context: BrowserContext, dotenv_config: dict[str, str]): +def create_invite_link(admin_context: BrowserContext, dotenv_config: dict[str, str], URL: BaseUrl): # go to admin page page = admin_context.new_page() - url = "https://" + dotenv_config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) page.get_by_role("link", name="Admin Interface").click() nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) @@ -99,21 +98,21 @@ def create_user(user_context: BrowserContext, invitelink): expect(page.locator("ak-library")).to_be_visible() -def setup_user_state(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): - # load admin cookies +def setup_user_state(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager, URL: BaseUrl): + # load admin cookies to context state_file = DIR.STATES / "authentik_admin_state.json" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) - if check_if_user_exists(context, dotenv_config): + if check_if_user_exists(context, dotenv_config, URL): # just login with user pass context.clear_cookies() else: # get invite_link - invite_link = create_invite_link(context, dotenv_config) + invite_link = create_invite_link(context, dotenv_config, URL) # create user context.clear_cookies() create_user(context, invite_link) - context.storage_state(path=f"{DIR.STATES}/authentik_user_state.json") + context.storage_state(path=DIR.STATES / "authentik_user_state.json") diff --git a/src/tests_authentik/test_authentik_dummy.py b/recipes/authentik/tests_authentik/test_authentik_dummy.py similarity index 100% rename from src/tests_authentik/test_authentik_dummy.py rename to recipes/authentik/tests_authentik/test_authentik_dummy.py diff --git a/recipes/demo/tests_demo/__init__.py b/recipes/demo/tests_demo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests_demo/fixtures_demo.py b/recipes/demo/tests_demo/fixtures_demo.py similarity index 83% rename from src/tests_demo/fixtures_demo.py rename to recipes/demo/tests_demo/fixtures_demo.py index f4a3919..b271e60 100644 --- a/src/tests_demo/fixtures_demo.py +++ b/recipes/demo/tests_demo/fixtures_demo.py @@ -5,7 +5,7 @@ depend on [demo]. For this to work 1. the Runner class of the other test needs to define the depencency as seen by referencing RunnerDemo in the dependencies list: -from src.tests_demo.runner_demo import RunnerDemo +from abratest.tests_demo.runner_demo import RunnerDemo class RunnerOther(Runner): dependencies = [RunnerDemo] @@ -15,7 +15,7 @@ class RunnerOther(Runner): To globally import for all tests in 'other', the import should be done in conftest: in 'conftest.py' in 'test_other' dir: -from src.tests_demo.fixtures_demo import demo_fixture +from abratest.tests_demo.fixtures_demo import demo_fixture """ import pytest diff --git a/src/tests_demo/runner_demo.py b/recipes/demo/tests_demo/runner_demo.py similarity index 87% rename from src/tests_demo/runner_demo.py rename to recipes/demo/tests_demo/runner_demo.py index 2957ad5..ab2618c 100644 --- a/src/tests_demo/runner_demo.py +++ b/recipes/demo/tests_demo/runner_demo.py @@ -1,5 +1,4 @@ -from src.runner import Runner, Test -from src.tests_authentik.runner_authentik import RunnerAuthentik +from abratest.runner import Runner, Test class RunnerDemo(Runner): @@ -12,7 +11,7 @@ class RunnerDemo(Runner): # RunnerDemo will only execute, when setup_authentik.py has finished successfully. # For example, setup_authentik.py generates session states, that can be used as fixtures # that can be loaded from fixtures_authentik.py - dependencies: list[type["Runner"]] = [RunnerAuthentik] + dependencies: list[str] = ["authentik"] # todo: update these comments # Filename of Demo setup. If defined, it will run 1st by executing pytest diff --git a/src/tests_demo/setup_demo.py b/recipes/demo/tests_demo/setup_demo.py similarity index 100% rename from src/tests_demo/setup_demo.py rename to recipes/demo/tests_demo/setup_demo.py diff --git a/recipes/nextcloud/tests_nextcloud/__init__.py b/recipes/nextcloud/tests_nextcloud/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests_nextcloud/cleanup_nextcloud.py b/recipes/nextcloud/tests_nextcloud/cleanup_nextcloud.py similarity index 100% rename from src/tests_nextcloud/cleanup_nextcloud.py rename to recipes/nextcloud/tests_nextcloud/cleanup_nextcloud.py diff --git a/recipes/nextcloud/tests_nextcloud/conftest.py b/recipes/nextcloud/tests_nextcloud/conftest.py new file mode 100644 index 0000000..88f5b09 --- /dev/null +++ b/recipes/nextcloud/tests_nextcloud/conftest.py @@ -0,0 +1,33 @@ +import json +import os + +import pytest +from playwright.sync_api import BrowserContext, Page + +from abratest.dir_manager import DirManager +from abratest.utils import BaseUrl + +pytest_plugins = "tests_authentik.fixtures_authentik" + +NEXTCLOUD_DEMO_USER = { + "NEXTCLOUD_USER": "next_demo_user", + "NEXTCLOUD_PASS": "P@ss.123", +} + +for key, value in NEXTCLOUD_DEMO_USER.items(): + os.environ[key] = value + + +@pytest.fixture +def nextcloud_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: + state_file = DIR.STATES / "nextcloud_admin_state.json" + storage_state = json.loads(state_file.read_bytes()) + context.add_cookies(storage_state["cookies"]) + return context + + +@pytest.fixture +def nextcloud_admin_page(nextcloud_admin_context: BrowserContext, DIR: DirManager, URL: BaseUrl) -> Page: + page = nextcloud_admin_context.new_page() + page.goto(URL.get("/apps/files")) + return page diff --git a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py new file mode 100644 index 0000000..8ea7063 --- /dev/null +++ b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py @@ -0,0 +1,17 @@ +from abratest.runner import Runner, Test + + +def condition_always_false(dotenv_config: dict[str, str]) -> bool: + return False + + +class RunnerNextcloud(Runner): + name: str = "nextcloud" + test_dir_name: str = "tests_nextcloud" + dependencies = ["authentik"] + setups = [Test(test_file="setup_nextcloud.py", prevent_skip=False)] + tests = [ + Test(test_file="tests_nextcloud.py", prevent_skip=True), + # Test(condition=condition_always_false, test_file="tests_nextcloud_onlyoffice.py"), + ] + # cleanups = [Test(test_file="cleanup_nextcloud.py")] diff --git a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py new file mode 100644 index 0000000..020d56d --- /dev/null +++ b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py @@ -0,0 +1,21 @@ +from playwright.sync_api import Page, expect + +from abratest.dir_manager import DirManager +from abratest.utils import BaseUrl + +# url dashboard +# https://files.test.dev.local-it.cloud/apps/dashboard/ +# url files +# https://files.test.dev.local-it.cloud/apps/files/ + + +def setup_nextcloud_admin_session(authentik_admin_page: Page, DIR: DirManager, URL: BaseUrl): + """visit nextcloud from authentik with admin_session to create wordpress_admin_session""" + with authentik_admin_page.expect_popup() as event_context: + authentik_admin_page.get_by_role("link", name="Nextcloud").click() + page_nextcloud = event_context.value + context = page_nextcloud.context + + page_nextcloud.goto(URL.get("/apps/files")) + expect(page_nextcloud.get_by_role("link", name="Name")).to_be_visible() + context.storage_state(path=DIR.STATES / "nextcloud_admin_state.json") diff --git a/recipes/nextcloud/tests_nextcloud/tests_nextcloud.py b/recipes/nextcloud/tests_nextcloud/tests_nextcloud.py new file mode 100644 index 0000000..b79150f --- /dev/null +++ b/recipes/nextcloud/tests_nextcloud/tests_nextcloud.py @@ -0,0 +1,32 @@ +import re + +import pytest +from playwright.sync_api import Page, expect + + +def test_nextcloud_quota(nextcloud_admin_page: Page, dotenv_config: dict[str, str]): + """Test Nextcloud""" + if dotenv_config.get("DEFAULT_QUOTA"): + # get quota from website + quota_string = nextcloud_admin_page.get_by_text( + re.compile(r"\d*,\d .* \d*,\d") + ).inner_text() # "37,7 MB von 104,9 MB verwendet" + out = re.search(r"\d*,\d .* (\d*,\d).", quota_string) + out_number = out[1] # 104,9 + out_number = out_number.replace(",", ".") + quota_website = float(out_number) + + # get quota from env + quota_config_string = dotenv_config["DEFAULT_QUOTA"] # "100 MB" + assert "MB" in quota_config_string + quota_config = float(quota_config_string.strip("MB")) + + assert quota_website == pytest.approx(quota_config, rel=0.1) # within 10% + else: + pytest.skip("DEFAULT_QUOTA not defined in env file") + + +@pytest.mark.skip +def test_nextcloud_apps(nextcloud_admin_page: Page, dotenv_config: dict[str, str]): + for app in dotenv_config["nc_apps"]: + expect(nextcloud_admin_page.get_by_role("link", name=app)).to_be_visible() diff --git a/src/tests_nextcloud/tests_nextcloud_onlyoffice.py b/recipes/nextcloud/tests_nextcloud/tests_nextcloud_onlyoffice.py similarity index 100% rename from src/tests_nextcloud/tests_nextcloud_onlyoffice.py rename to recipes/nextcloud/tests_nextcloud/tests_nextcloud_onlyoffice.py diff --git a/recipes/wordpress/tests_wordpress/__init__.py b/recipes/wordpress/tests_wordpress/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests_wordpress/conftest.py b/recipes/wordpress/tests_wordpress/conftest.py similarity index 73% rename from src/tests_wordpress/conftest.py rename to recipes/wordpress/tests_wordpress/conftest.py index ac095dc..db4e529 100644 --- a/src/tests_wordpress/conftest.py +++ b/recipes/wordpress/tests_wordpress/conftest.py @@ -4,16 +4,9 @@ import pytest from dotenv import dotenv_values from playwright.sync_api import BrowserContext, Page -from src.dir_manager import DirManager +from abratest.dir_manager import DirManager -# from src.tests_authentik.fixtures_authentik import ( -# authentik_admin_context, -# authentik_admin_page, -# authentik_user_context, -# authentik_user_page, -# ) - -pytest_plugins = "src.tests_authentik.fixtures_authentik" +pytest_plugins = "tests_authentik.fixtures_authentik" @pytest.fixture diff --git a/src/tests_wordpress/runner_wordpress.py b/recipes/wordpress/tests_wordpress/runner_wordpress.py similarity index 80% rename from src/tests_wordpress/runner_wordpress.py rename to recipes/wordpress/tests_wordpress/runner_wordpress.py index 097a66a..7167ce8 100644 --- a/src/tests_wordpress/runner_wordpress.py +++ b/recipes/wordpress/tests_wordpress/runner_wordpress.py @@ -1,5 +1,4 @@ -from src.runner import Runner, Test -from src.tests_authentik.runner_authentik import RunnerAuthentik +from abratest.runner import Runner, Test def condition_always_true(dotenv_config: dict[str, str]) -> bool: @@ -20,7 +19,7 @@ def condition_has_locale(dotenv_config: dict[str, str]) -> bool: class RunnerWordpress(Runner): name = "wordpress" test_dir_name = "tests_wordpress" - dependencies: list[type[Runner]] = [RunnerAuthentik] + dependencies = ["authentik"] setups = [Test(test_file="setup_wordpress.py")] tests = [ Test(test_file="test_wordpress.py"), diff --git a/src/tests_wordpress/setup_wordpress.py b/recipes/wordpress/tests_wordpress/setup_wordpress.py similarity index 90% rename from src/tests_wordpress/setup_wordpress.py rename to recipes/wordpress/tests_wordpress/setup_wordpress.py index 4feca93..0d8243a 100644 --- a/src/tests_wordpress/setup_wordpress.py +++ b/recipes/wordpress/tests_wordpress/setup_wordpress.py @@ -1,7 +1,7 @@ import pytest from playwright.sync_api import BrowserContext, Page, expect -from src.dir_manager import DirManager +from abratest.dir_manager import DirManager def test_visit_from_domain(authentik_admin_context: BrowserContext, dotenv_config: dict[str, str]): @@ -25,4 +25,4 @@ def setup_wordpress_admin_session(authentik_admin_page: Page, DIR: DirManager): expect(page_wordpress.locator("#wpadminbar")).to_be_visible() # save session context = page_wordpress.context - context.storage_state(path=f"{DIR.STATES}/wordpress_admin_state.json") + context.storage_state(path=DIR.STATES / "wordpress_admin_state.json") diff --git a/recipes/wordpress/tests_wordpress/test_wordpress.py b/recipes/wordpress/tests_wordpress/test_wordpress.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests_wordpress/test_wordpress_localization.py b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py similarity index 91% rename from src/tests_wordpress/test_wordpress_localization.py rename to recipes/wordpress/tests_wordpress/test_wordpress_localization.py index 8c9ffa4..1583a9f 100644 --- a/src/tests_wordpress/test_wordpress_localization.py +++ b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py @@ -2,7 +2,7 @@ from playwright.sync_api import BrowserContext, expect -from src.dir_manager import DirManager +from abratest.dir_manager import DirManager def test_welcome_message(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): diff --git a/requirements.txt b/requirements.txt index a17d5a2..88b903b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ pytest-playwright python-dotenv icecream loguru -beautifulsoup4 \ No newline at end of file +beautifulsoup4 +imbox diff --git a/src/runner_dict.py b/src/runner_dict.py deleted file mode 100644 index 5b000db..0000000 --- a/src/runner_dict.py +++ /dev/null @@ -1,16 +0,0 @@ -from typing import TYPE_CHECKING - -from src.tests_authentik.runner_authentik import RunnerAuthentik -from src.tests_nextcloud.runner_nextcloud import RunnerNextcloud -from src.tests_wordpress.runner_wordpress import RunnerWordpress - -if TYPE_CHECKING: - from src.runner import Runner - -# Register all runners here. Each .env file with TYPE=authentik will be run with RunnerAuthentik - -RUNNER_DICT: dict[str, type["Runner"]] = { - "authentik": RunnerAuthentik, - "wordpress": RunnerWordpress, - "nextcloud": RunnerNextcloud, -} diff --git a/src/tests_nextcloud/conftest.py b/src/tests_nextcloud/conftest.py deleted file mode 100644 index 0805ce5..0000000 --- a/src/tests_nextcloud/conftest.py +++ /dev/null @@ -1,16 +0,0 @@ -import os - -from src.tests_authentik.fixtures_authentik import ( - authentik_admin_context, - authentik_admin_page, - authentik_user_context, - authentik_user_page, -) - -NEXTCLOUD_DEMO_USER = { - "NEXTCLOUD_USER": "next_demo_user", - "NEXTCLOUD_PASS": "P@ss.123", -} - -for key, value in NEXTCLOUD_DEMO_USER.items(): - os.environ[key] = value diff --git a/src/tests_nextcloud/runner_nextcloud.py b/src/tests_nextcloud/runner_nextcloud.py deleted file mode 100644 index 8468815..0000000 --- a/src/tests_nextcloud/runner_nextcloud.py +++ /dev/null @@ -1,18 +0,0 @@ -from src.runner import Runner, Test -from src.tests_authentik.runner_authentik import RunnerAuthentik - - -def condition_always_false(dotenv_config: dict[str, str]) -> bool: - return False - - -class RunnerNextcloud(Runner): - name: str = "nextcloud" - test_dir_name: str = "tests_nextcloud" - dependencies = [RunnerAuthentik] - setups = [Test(test_file="setup_nextcloud.py")] - tests = [ - Test(test_file="tests_nextcloud.py"), - Test(condition=condition_always_false, test_file="tests_nextcloud_onlyoffice.py"), - ] - # cleanups = [Test(test_file="cleanup_nextcloud.py")] diff --git a/src/tests_nextcloud/setup_nextcloud.py b/src/tests_nextcloud/setup_nextcloud.py deleted file mode 100644 index eb53182..0000000 --- a/src/tests_nextcloud/setup_nextcloud.py +++ /dev/null @@ -1,35 +0,0 @@ -import pytest - - -@pytest.fixture(scope="session", autouse=True) -def nc_login(browser: Browser): - """Nextcloud Login""" - context = setup_context(browser, f"{STATES}/user_state.json") - page = context.new_page() - page.goto(CONFIG["domain"]) - with page.expect_popup() as nextcloud_info: - link = page.get_by_role("link", name="Nextcloud") - CONFIG["nc_domain"] = link.get_attribute("href") - link.click() - nextcloud = nextcloud_info.value - check_for(nextcloud.get_by_role("link", name="Name")) - if nextcloud.query_selector(".close-icon"): - close_button = nextcloud.get_by_role("button", name="Close modal") - close_button.click() - expect(close_button).to_be_hidden() - nextcloud.wait_for_timeout(2000) - context.storage_state(path=f"{STATES}/nc_user_state.json") - context.tracing.stop(path=f"{RECORDS}/nextcloud_login_user.zip") - context.close() - - -@pytest.fixture -def nc_session(browser: Browser): - """Reuse Nextcloud User Session""" - context = setup_context(browser, f"{STATES}/nc_user_state.json") - page = context.new_page() - page.goto(CONFIG["nc_domain"]) - if page.query_selector(".close-icon"): - page.get_by_role("button", name="Close modal").click() - yield context, page - context.close() diff --git a/src/tests_nextcloud/tests_nextcloud.py b/src/tests_nextcloud/tests_nextcloud.py deleted file mode 100644 index c9bc5be..0000000 --- a/src/tests_nextcloud/tests_nextcloud.py +++ /dev/null @@ -1,13 +0,0 @@ -def test_nextcloud(nc_session): - """Test Nextcloud""" - context, page = nc_session - # if page.query_selector('.close-icon'): - # page.get_by_role("button", name="Close modal").click() - if CONFIG.get("default_quota"): - quota = int( - page.get_by_role("listitem", name="Storage informations").get_by_role("link").inner_text().split()[3] - ) - assert quota == CONFIG["default_quota"] - for app in CONFIG["nc_apps"]: - check_for(page.get_by_role("link", name=app)) - context.tracing.stop(path=f"{RECORDS}/nextcloud.zip") diff --git a/tests/test_env_resolution.py b/tests/test_env_resolution.py index 2c1bc97..87f6f7c 100644 --- a/tests/test_env_resolution.py +++ b/tests/test_env_resolution.py @@ -2,8 +2,11 @@ from pathlib import Path import pytest -# from src.env_file_helper import DependencyRule, EnvFile, sort_env_files_by_rule -from src.env_manager import DependencyRule, EnvFile, EnvManager +from abratest.coordinator import Coordinator +from abratest.env_manager import DependencyRule, EnvFile, EnvManager + +RECIPES_DIR = Path("./recipes").resolve() +RUNNER_DICT = Coordinator.create_runner_dict(RECIPES_DIR) def test_complex_sorting() -> None: @@ -46,7 +49,7 @@ def test_real_env_files() -> None: Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik ] env_files: list[EnvFile] = EnvManager._get_env_files(ENV_FILES) - dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files) + dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files, RUNNER_DICT) sorted_env_files = EnvManager.sort_env_files_by_rule(env_files, dependency_rules) assert sorted_env_files[0].env_type == "authentik" @@ -60,7 +63,7 @@ def test_real_env_files_duplicate() -> None: Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik ] env_files: list[EnvFile] = EnvManager._get_env_files(ENV_FILES) - dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files) + dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files, RUNNER_DICT) sorted_env_files = EnvManager.sort_env_files_by_rule(env_files, dependency_rules) assert sorted_env_files[0].env_type == "authentik" assert sorted_env_files[1].env_type == "authentik" @@ -79,7 +82,7 @@ def test_real_env_files_duplicate_six() -> None: Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress ] env_files: list[EnvFile] = EnvManager._get_env_files(ENV_FILES) - dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files) + dependency_rules: list[DependencyRule] = EnvManager._get_dependency_rules(env_files, RUNNER_DICT) sorted_env_files = EnvManager.sort_env_files_by_rule(env_files, dependency_rules) assert sorted_env_files[0].env_type == "authentik" assert sorted_env_files[1].env_type == "authentik" @@ -95,7 +98,7 @@ def test_env_manager() -> None: Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik ] - ENV = EnvManager(env_paths_list) + ENV = EnvManager(env_paths_list, RUNNER_DICT) assert ENV.env_files[0].env_type == "authentik" assert ENV.env_files[1].env_type == "authentik" assert ENV.env_files[2].env_type == "wordpress" diff --git a/tests/test_url.py b/tests/test_url.py new file mode 100644 index 0000000..680023f --- /dev/null +++ b/tests/test_url.py @@ -0,0 +1,28 @@ +from abratest.utils import BaseUrl + +url_input = { + "netloc": "blog.dev.local-it.cloud", + "scheme": "https", +} + +url_obj = BaseUrl(**url_input) + + +def test_urllib_domain_only(): + assert url_obj.get() == "https://blog.dev.local-it.cloud" + + +def test_urllib_path_single(): + assert url_obj.get(path="something") == "https://blog.dev.local-it.cloud/something" + + +def test_urllib_path_double(): + assert url_obj.get(path="something/else") == "https://blog.dev.local-it.cloud/something/else" + + +def test_urllib_path_signle_suc_slash(): + assert url_obj.get(path="something/else/") == "https://blog.dev.local-it.cloud/something/else/" + + +def test_urllib_path_signle_pre_slash(): + assert url_obj.get(path="/something/else") == "https://blog.dev.local-it.cloud/something/else" From 4c5a470a700a0fa6bfaf8a679e9452c3bc107711 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 6 Dec 2023 12:05:13 +0100 Subject: [PATCH 08/18] refactor so that coordinator instance is available in runner instance (#8) -> all program states available Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/8 Co-authored-by: Daniel Co-committed-by: Daniel --- README.md | 5 ++- abratest/coordinator.py | 11 ++--- abratest/dir_manager.py | 7 ++++ abratest/env_manager.py | 14 ++++--- abratest/runner.py | 42 +++++++++++-------- pyproject.toml | 3 +- .../tests_authentik/fixtures_authentik.py | 16 ++++--- .../tests_authentik/runner_authentik.py | 3 +- recipes/demo/tests_demo/runner_demo.py | 3 +- recipes/nextcloud/tests_nextcloud/conftest.py | 2 +- .../tests_nextcloud/runner_nextcloud.py | 3 +- recipes/wordpress/tests_wordpress/conftest.py | 2 +- .../tests_wordpress/runner_wordpress.py | 3 +- run_abratest.sh | 6 +++ test_abratest.sh | 7 ++++ 15 files changed, 74 insertions(+), 53 deletions(-) create mode 100644 run_abratest.sh create mode 100644 test_abratest.sh diff --git a/README.md b/README.md index 54547be..2ceb297 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,15 @@ Run the script with ```bash python main.py # run abratest pytest # test abratest +pytest --collect-only # debug test abratest ``` # 2.2 Run with Docker ```bash docker compose build # build the image -docker compose run --rm app python ./main.py # run AbraTest -docker compose run --rm app pytest # test AbraTest +docker compose run --rm app ./run_abratest.sh # run AbraTest +docker compose run --rm app ./test_abratest.sh # test AbraTest ``` Force rebuild with cache diff --git a/abratest/coordinator.py b/abratest/coordinator.py index 11e4b7f..28a5102 100644 --- a/abratest/coordinator.py +++ b/abratest/coordinator.py @@ -42,14 +42,9 @@ class Coordinator: def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]: """Creates an instance of the correct Runner class for each given env file""" runners: list[Runner] = [] - for env_file in env_files: + for index, env_file in enumerate(env_files): RunnerClass = self.RUNNER_DICT[env_file.config["TYPE"]] - dependency_classes: list[type[Runner]] = [] - for dependency in RunnerClass.dependencies: - dependency_classes.append(self.RUNNER_DICT[dependency]) - runner_instance = RunnerClass(dotenv_path=env_file.env_path, DIR=self.DIR) - runner_instance._dependency_runners = dependency_classes - runners.append(runner_instance) + runners.append(RunnerClass(coordinator=self, runner_index=index)) return runners def combine_html(self) -> None: @@ -104,5 +99,5 @@ class Coordinator: assert len(runner_class_names) == 1 runner_class_name = runner_class_names[0] RunnerClass: type[Runner] = getattr(module, runner_class_name) - RUNNER_DICT[RunnerClass.name] = RunnerClass + RUNNER_DICT[RunnerClass.env_type] = RunnerClass return RUNNER_DICT diff --git a/abratest/dir_manager.py b/abratest/dir_manager.py index e172ef5..2104a68 100644 --- a/abratest/dir_manager.py +++ b/abratest/dir_manager.py @@ -1,5 +1,7 @@ from pathlib import Path +from dotenv import dotenv_values + class DirManager: """Manages directories for the tests and should be used to create and find @@ -69,3 +71,8 @@ class DirManager: @property def RECIPES(self): return self.recipes_dir + + def get_config(self, search_string: str) -> dict[str, str]: + env_file = next(self.ENV_FILES.glob(f"*{search_string}*")) + config: dict[str, str] = dotenv_values(env_file) # type: ignore + return config diff --git a/abratest/env_manager.py b/abratest/env_manager.py index 18086e2..780436f 100644 --- a/abratest/env_manager.py +++ b/abratest/env_manager.py @@ -46,7 +46,7 @@ class EnvManager: for env_file in env_files: child_runner_class = RUNNER_DICT[env_file.env_type] for dependency in child_runner_class.dependencies: - dependency_rule = DependencyRule(child=child_runner_class.name, dependency=dependency) + dependency_rule = DependencyRule(child=child_runner_class.env_type, dependency=dependency) dependency_rules.append(dependency_rule) return dependency_rules @@ -93,8 +93,10 @@ class EnvManager: ) def copy_env_files(self, DIR: DirManager) -> None: - """Copies all env files to STATES/env_files. Files will be renamed to their own TYPE value.""" - env_files_dir = DIR.STATES / "env_files" - env_files_dir.mkdir(exist_ok=True) - for env_file in self.env_files: - shutil.copy(env_file.env_path, env_files_dir / env_file.env_type) + """Copies all env files to STATES/env_files. Files will be renamed to + -- + 00-authentik-login.test.dev.local-it.cloud.env""" + + for index, env_file in enumerate(self.env_files): + file_name = "-".join([str(index).zfill(2), env_file.env_type, env_file.env_path.name]) + shutil.copy(env_file.env_path, DIR.ENV_FILES / file_name) diff --git a/abratest/runner.py b/abratest/runner.py index 8ea95a4..c9cbc9e 100644 --- a/abratest/runner.py +++ b/abratest/runner.py @@ -1,12 +1,13 @@ from dataclasses import dataclass from pathlib import Path -from typing import Callable +from typing import TYPE_CHECKING, Callable import pytest -from dotenv import dotenv_values from loguru import logger -from abratest.dir_manager import DirManager +if TYPE_CHECKING: + from abratest.coordinator import Coordinator + from abratest.env_manager import EnvFile @dataclass @@ -17,22 +18,25 @@ class Test: class Runner: - name: str = "" - test_dir_name: str = "" + env_type: str = "" setups: list[Test] = [] tests: list[Test] = [] cleanups: list[Test] = [] dependencies: list[str] = [] - _dependency_runners: list[type["Runner"]] = [] - def __init__(self, dotenv_path: Path, DIR: DirManager): - self.dotenv_path = dotenv_path - self.config: dict[str, str] = dotenv_values(dotenv_path) # type: ignore - self.DIR = DIR + def __init__(self, coordinator: "Coordinator", runner_index: int): + self.coordinator = coordinator # needed? + self.runner_index = runner_index # needed? + + self.DIR = coordinator.DIR + self.ENV = coordinator.ENV + self.RUNNER_DICT = coordinator.RUNNER_DICT + + self.env_file: EnvFile = self.ENV.env_files[self.runner_index] + self.dotenv_path = self.env_file.env_path + self.config = self.env_file.config logger.info(f"creating instance of {self.__class__.__name__}") - assert self.test_dir_name - self.root_dir = Path(__file__).parent def run_setups(self): """runs the setup scripts if available""" @@ -50,7 +54,7 @@ class Runner: """runs the main test script and if available and sub test scripts if their running condition is met""" # check if required dependencies have passed if not self._dependencies_passed(): - logger.warning(f"skipping run_tests() of {self.name}, because some dependencies have not passed") + logger.warning(f"skipping run_tests() of {self.env_type}, because some dependencies have not passed") return for test in test_list: @@ -63,8 +67,11 @@ class Runner: # condition_available: true / pass # condition_met: true / false - identifier_string = self.combine_names(self.name, test.test_file) - full_test_path = self.DIR.RECIPES / self.name / self.test_dir_name / test.test_file + identifier_string = self.combine_names(self.env_type, test.test_file) + + results = list(self.DIR.RECIPES.rglob(test.test_file)) + assert len(results) == 1, f"{test.test_file} should exist exactly 1 time, but found {len(results)} times" + full_test_path = results[0] # check if test aleady passed if self._is_test_passed(identifier_string, remove_existing=True): @@ -168,9 +175,10 @@ class Runner: passed_tests = [r.name for r in self.DIR.RESULTS.glob("*") if "passed" in r.name] results = [] - for dependency_runner in self._dependency_runners: + for dependency in self.dependencies: + dependency_runner = self.coordinator.RUNNER_DICT[dependency] for setup_name in dependency_runner.setups: - dependencie_identifier = self.combine_names(dependency_runner.name, setup_name.test_file) + dependencie_identifier = self.combine_names(dependency_runner.env_type, setup_name.test_file) results.append(any(dependencie_identifier in f for f in passed_tests)) return all(results) diff --git a/pyproject.toml b/pyproject.toml index 24f1aa4..ec0809f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,4 +20,5 @@ target-version = "py311" [tool.pytest.ini_options] python_functions = "test_* setup_*" -norecursedirs = "previous-work recipes" \ No newline at end of file +norecursedirs = ".* previous-work recipes" +testpaths = "tests" \ No newline at end of file diff --git a/recipes/authentik/tests_authentik/fixtures_authentik.py b/recipes/authentik/tests_authentik/fixtures_authentik.py index 3f27b43..34ed3d7 100644 --- a/recipes/authentik/tests_authentik/fixtures_authentik.py +++ b/recipes/authentik/tests_authentik/fixtures_authentik.py @@ -1,10 +1,10 @@ import json import pytest -from dotenv import dotenv_values from playwright.sync_api import BrowserContext, Page from abratest.dir_manager import DirManager +from abratest.utils import BaseUrl @pytest.fixture @@ -18,10 +18,9 @@ def authentik_admin_context(context: BrowserContext, DIR: DirManager) -> Browser @pytest.fixture def authentik_admin_page(authentik_admin_context: BrowserContext, DIR: DirManager) -> Page: page = authentik_admin_context.new_page() - env_file = DIR.ENV_FILES / "authentik" - config: dict[str, str] = dotenv_values(env_file) # type: ignore - url = "https://" + config["DOMAIN"] - page.goto(url) + config = DIR.get_config("authentik") + base_url = BaseUrl(config["DOMAIN"]) + page.goto(base_url.get()) return page @@ -36,8 +35,7 @@ def authentik_user_context(context: BrowserContext, DIR: DirManager) -> BrowserC @pytest.fixture def authentik_user_page(authentik_user_context: BrowserContext, DIR: DirManager) -> Page: page = authentik_user_context.new_page() - env_file = DIR.ENV_FILES / "authentik" - config: dict[str, str] = dotenv_values(env_file) # type: ignore - url = "https://" + config["DOMAIN"] - page.goto(url) + config = DIR.get_config("authentik") + base_url = BaseUrl(config["DOMAIN"]) + page.goto(base_url.get()) return page diff --git a/recipes/authentik/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py index a3bba32..b83da2f 100644 --- a/recipes/authentik/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -10,7 +10,6 @@ def condition_always_false(dotenv_config: dict[str, str]) -> bool: class RunnerAuthentik(Runner): - name = "authentik" - test_dir_name = "tests_authentik" + env_type = "authentik" setups = [Test(test_file="setup_authentik.py")] # tests = [Test(test_file="test_authentik_dummy.py")] diff --git a/recipes/demo/tests_demo/runner_demo.py b/recipes/demo/tests_demo/runner_demo.py index ab2618c..2b9aa78 100644 --- a/recipes/demo/tests_demo/runner_demo.py +++ b/recipes/demo/tests_demo/runner_demo.py @@ -4,8 +4,7 @@ from abratest.runner import Runner, Test class RunnerDemo(Runner): """Every env file has a corresponding runner class""" - name: str = "demo" # name of the test, used for logging / output naming - test_dir_name: str = "tests_demo" # dir name holding all tests related to RunnerDemo + env_type = "demo" # name of the test, used for logging / output naming # this indicates that tests from RunnerDemo depend on the setup from RunnerAuthentik. # RunnerDemo will only execute, when setup_authentik.py has finished successfully. diff --git a/recipes/nextcloud/tests_nextcloud/conftest.py b/recipes/nextcloud/tests_nextcloud/conftest.py index 88f5b09..abdf9f9 100644 --- a/recipes/nextcloud/tests_nextcloud/conftest.py +++ b/recipes/nextcloud/tests_nextcloud/conftest.py @@ -7,7 +7,7 @@ from playwright.sync_api import BrowserContext, Page from abratest.dir_manager import DirManager from abratest.utils import BaseUrl -pytest_plugins = "tests_authentik.fixtures_authentik" +pytest_plugins = "authentik.tests_authentik.fixtures_authentik" NEXTCLOUD_DEMO_USER = { "NEXTCLOUD_USER": "next_demo_user", diff --git a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py index 8ea7063..5f7fe27 100644 --- a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py @@ -6,8 +6,7 @@ def condition_always_false(dotenv_config: dict[str, str]) -> bool: class RunnerNextcloud(Runner): - name: str = "nextcloud" - test_dir_name: str = "tests_nextcloud" + env_type = "nextcloud" dependencies = ["authentik"] setups = [Test(test_file="setup_nextcloud.py", prevent_skip=False)] tests = [ diff --git a/recipes/wordpress/tests_wordpress/conftest.py b/recipes/wordpress/tests_wordpress/conftest.py index db4e529..c91a525 100644 --- a/recipes/wordpress/tests_wordpress/conftest.py +++ b/recipes/wordpress/tests_wordpress/conftest.py @@ -6,7 +6,7 @@ from playwright.sync_api import BrowserContext, Page from abratest.dir_manager import DirManager -pytest_plugins = "tests_authentik.fixtures_authentik" +pytest_plugins = "authentik.tests_authentik.fixtures_authentik" @pytest.fixture diff --git a/recipes/wordpress/tests_wordpress/runner_wordpress.py b/recipes/wordpress/tests_wordpress/runner_wordpress.py index 7167ce8..846f069 100644 --- a/recipes/wordpress/tests_wordpress/runner_wordpress.py +++ b/recipes/wordpress/tests_wordpress/runner_wordpress.py @@ -17,8 +17,7 @@ def condition_has_locale(dotenv_config: dict[str, str]) -> bool: class RunnerWordpress(Runner): - name = "wordpress" - test_dir_name = "tests_wordpress" + env_type = "wordpress" dependencies = ["authentik"] setups = [Test(test_file="setup_wordpress.py")] tests = [ diff --git a/run_abratest.sh b/run_abratest.sh new file mode 100644 index 0000000..ddffb80 --- /dev/null +++ b/run_abratest.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +RECIPES_PATH=$PWD/recipes +export PYTHONPATH=${PYTHONPATH}:$RECIPES_PATH + +python main.py \ No newline at end of file diff --git a/test_abratest.sh b/test_abratest.sh new file mode 100644 index 0000000..d7abaaa --- /dev/null +++ b/test_abratest.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +PWD_PATH=$PWD +RECIPES_PATH=$PWD/recipes +export PYTHONPATH=$PWD_PATH:$RECIPES_PATH + +pytest \ No newline at end of file From 868568869886232d4ead4a91858c0e0e3f134478 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 7 Dec 2023 11:32:01 +0100 Subject: [PATCH 09/18] installable package (#9) * turn repo into installable package (pip install -e .) * add hatchling build packend * call it pytest-abra * add pytest entrypoint, so that it gets loaded automatically if installed (and pytest is run) * make fixtures optional, so that pytest can still be used in other context * add cli script -> you can now directly run "pytest-abra" in console Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/9 Co-authored-by: Daniel Co-committed-by: Daniel --- .gitignore | 1 + README.md | 23 +++-- abratest/plugin-abra.py | 94 ------------------ conftest.py | 2 - main.py | 92 ++++++------------ previous-work/wordpress_test.py | 2 +- prototyping/env_var_subprocess.py | 17 ++++ pyproject.toml | 46 +++++++-- {abratest => pytest_abra}/__init__.py | 0 pytest_abra/cli.py | 56 +++++++++++ {abratest => pytest_abra}/coordinator.py | 15 +-- pytest_abra/demo.py | 1 + {abratest => pytest_abra}/dir_manager.py | 2 +- {abratest => pytest_abra}/env_manager.py | 4 +- {abratest => pytest_abra}/html_helper.py | 0 pytest_abra/pytest_abra.py | 95 +++++++++++++++++++ {abratest => pytest_abra}/runner.py | 11 ++- {abratest => pytest_abra}/utils.py | 2 +- .../tests_authentik/fixtures_authentik.py | 4 +- .../tests_authentik/runner_authentik.py | 2 +- .../tests_authentik/setup_authentik.py | 4 +- recipes/demo/tests_demo/fixtures_demo.py | 4 +- recipes/demo/tests_demo/runner_demo.py | 2 +- recipes/nextcloud/tests_nextcloud/conftest.py | 4 +- .../tests_nextcloud/runner_nextcloud.py | 2 +- .../tests_nextcloud/setup_nextcloud.py | 4 +- recipes/wordpress/tests_wordpress/conftest.py | 2 +- .../tests_wordpress/runner_wordpress.py | 2 +- .../tests_wordpress/setup_wordpress.py | 2 +- .../test_wordpress_localization.py | 2 +- requirements.txt | 1 + tests/test_env_resolution.py | 4 +- tests/test_url.py | 2 +- 33 files changed, 294 insertions(+), 210 deletions(-) delete mode 100644 abratest/plugin-abra.py delete mode 100644 conftest.py create mode 100644 prototyping/env_var_subprocess.py rename {abratest => pytest_abra}/__init__.py (100%) create mode 100644 pytest_abra/cli.py rename {abratest => pytest_abra}/coordinator.py (90%) create mode 100644 pytest_abra/demo.py rename {abratest => pytest_abra}/dir_manager.py (96%) rename {abratest => pytest_abra}/env_manager.py (98%) rename {abratest => pytest_abra}/html_helper.py (100%) create mode 100644 pytest_abra/pytest_abra.py rename {abratest => pytest_abra}/runner.py (95%) rename {abratest => pytest_abra}/utils.py (96%) diff --git a/.gitignore b/.gitignore index 46d7f87..78cd629 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ TestResults/ *.pyc *.json *.zip +*.egg-info credentials* \ No newline at end of file diff --git a/README.md b/README.md index 2ceb297..d3d3604 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# AbraTest +# pytest-abra ...description... # Usage -To use AbraTest, follow these steps: +To use pytest-abra, follow these steps: ## 1. GIT Clone @@ -17,7 +17,7 @@ git submodule update --remote // update submodules ## Run -You can run AbraTest with and without Docker. Choose now and follow the steps accordingly: +You can run pytest-abra with and without Docker. Choose now and follow the steps accordingly: ## 2.1 Run without Docker @@ -33,17 +33,14 @@ playwright install Run the script with ```bash -python main.py # run abratest -pytest # test abratest -pytest --collect-only # debug test abratest +python main.py ``` # 2.2 Run with Docker ```bash docker compose build # build the image -docker compose run --rm app ./run_abratest.sh # run AbraTest -docker compose run --rm app ./test_abratest.sh # test AbraTest +docker compose run --rm app ./run_pytest-abra.sh # run pytest-abra ``` Force rebuild with cache @@ -64,4 +61,12 @@ Use playwright codegen to create code for new testes easily https://playwright.d ```bash playwright codegen demo.playwright.dev/todomvc -``` \ No newline at end of file +``` + +## Development + +```bash +pytest # test pytest-abra +pytest --collect-only # debug test pytest-abra +docker compose run --rm app ./test_pytest-abra.sh # test pytest-abra +``` diff --git a/abratest/plugin-abra.py b/abratest/plugin-abra.py deleted file mode 100644 index c72b21b..0000000 --- a/abratest/plugin-abra.py +++ /dev/null @@ -1,94 +0,0 @@ -# regarding conftest: -# If you have conftest.py files which do not reside in a python package directory -# (i.e. one containing an __init__.py) then “import conftest” can be ambiguous -# because there might be other conftest.py files as well on your PYTHONPATH or -# sys.path. It is thus good practise for projects to either put conftest.py under -# a package scope or to never import anything from a conftest.py file. - -import os -from imaplib import IMAP4_SSL -from pathlib import Path - -import pytest -from dotenv import dotenv_values -from playwright.sync_api import BrowserContext, expect -from pytest import Parser - -from abratest.dir_manager import DirManager -from abratest.utils import BaseUrl - -# global timeout and LOCALE -LOCALE = {"Accept-Language": "de_DE"} -TIMEOUT = 20_000 -expect.set_options(timeout=TIMEOUT) - - -@pytest.fixture -def context(context: BrowserContext) -> BrowserContext: - context.set_default_timeout(TIMEOUT) - context.set_extra_http_headers(LOCALE) - return context - - -def pytest_addoption(parser: Parser): - parser.addoption( - "--env_file", - action="store", - required=True, - ) - parser.addoption( - "--output_dir", - action="store", - required=True, - ) - parser.addoption( - "--session_id", - action="store", - required=True, - ) - - -@pytest.fixture(scope="session", autouse=True) -def DIR(request) -> DirManager: - """Fixture holding test directories - - DIR.OUTPUT - DIR.SESSION - DIR.RECORDS - DIR.STATES - DIR.RESULTS""" - - output_dir = request.config.getoption("--output_dir") - output_dir = Path(output_dir) - session_id = request.config.getoption("--session_id") - dirmanager = DirManager(output_dir=output_dir, session_id=session_id) - dirmanager.create_all_dirs() - return dirmanager - - -@pytest.fixture(scope="session", autouse=True) -def dotenv_config(request) -> dict[str, str]: - dotenv_path = request.config.getoption("--env_file") - dotenv_path = Path(dotenv_path) - assert dotenv_path.is_file() - return dotenv_values(dotenv_path) # type: ignore - - -@pytest.fixture(scope="session", autouse=True) -def URL(dotenv_config: dict[str, str]) -> BaseUrl: - return BaseUrl(netloc=dotenv_config["DOMAIN"]) - - -@pytest.fixture(scope="session") -def imap_ssl_email_client() -> None: - assert os.environ["IMAP_HOST"] - assert os.environ["IMAP_PORT"] - assert os.environ["IMAP_USER"] - assert os.environ["IMAP_PASS"] - port = int(os.environ["IMAP_PORT"]) - imap_client = IMAP4_SSL(host=os.environ["IMAP_HOST"], port=port) - imap_client.login(os.environ["IMAP_USER"], os.environ["IMAP_PASS"]) - imap_client.select("INBOX") - yield imap_client - imap_client.close() - imap_client.logout() diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 8704865..0000000 --- a/conftest.py +++ /dev/null @@ -1,2 +0,0 @@ -# this file exists so that tests inside /tests always find /src imports, -# because this will cause the root (/) to be added to sys.path diff --git a/main.py b/main.py index 8c2d739..e7b02a2 100644 --- a/main.py +++ b/main.py @@ -1,42 +1,10 @@ import json import os +import subprocess from pathlib import Path -from loguru import logger - -from abratest.coordinator import Coordinator -from abratest.dir_manager import DirManager -from abratest.utils import get_session_id - -# ----------------------------- lookup env files ----------------------------- # - - -# This list of env files is the input to testing framework. each env file -# triggers the execution of one test Runner and provides configuration to the -# tests inside the runner. There can be dependencies, for example wordpress -# requires that authentik ran first to create the admin session and the user -# session. At the moment, wrong ordering results in unsuccessful test -# (wrong ordering would be wordpress env file is before authentik env file). -# At the moment, functionailty is only guaranteed if each env file use -# a unique TYPE var. - -ENV_FILES = [ - Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik - Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress - # Path("envfiles/files.test.dev.local-it.cloud.env"), # nextcloud -] - - -# ----------------------------- define ouptut dir ---------------------------- # - - -OUTPUT_DIR = Path("./test-output").resolve() -RECIPES_DIR = Path("./recipes").resolve() - - # --------------------- load credentials to env variables -------------------- # - cred_file = Path("credentials.json") with open(cred_file, "r") as f: CREDENTIALS = json.load(f) @@ -44,37 +12,35 @@ with open(cred_file, "r") as f: for key, value in CREDENTIALS.items(): os.environ[key] = value +# --------------------------------- env files -------------------------------- # -# -------------------------- enable playwright debug ------------------------- # +# This list of env files is the input to testing framework. each env file +# triggers the execution of one test Runner and provides configuration to the +# tests inside the runner. + +ENV_FILES_ROOT = Path("../envfiles").resolve() +ENV_FILES = [ + ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env", # authentik + ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env", # wordpress + ENV_FILES_ROOT / "files.test.dev.local-it.cloud.env", # nextcloud +] +ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES]) + +# ----------------------------------- dirs ----------------------------------- # + +RECIPES_DIR = Path("../recipes").resolve() +OUTPUT_DIR = Path("./test-output").resolve() -# add abra-testing dir -os.environ["PYTEST_PLUGINS"] = "abratest.plugin-abra" # "abratest.plugin,abratest.other" -# os.environ["PWDEBUG"] = "1" - - -# ----------------------------- define session_id ---------------------------- # - - -session_id = get_session_id() -# session_id = "abc" - - -# ------------------------------- setup logging ------------------------------ # - - -DIR = DirManager(output_dir=OUTPUT_DIR, session_id=session_id) -log_file = DIR.RECORDS / "coordinator.log" -logger.add(log_file) - - -# ---------------------------- initialize and run ---------------------------- # - - -coordinator = Coordinator( - env_paths_list=ENV_FILES, output_dir=OUTPUT_DIR, session_id=session_id, recipes_dir=RECIPES_DIR +subprocess.run( + [ + "abratest", + "--env_paths", + ENV_PATHS, + "--recipes_dir", + RECIPES_DIR, + "--output_dir", + OUTPUT_DIR, + # "--debug", + ] ) -coordinator.setup_test() -coordinator.run_test() -coordinator.combine_html() -coordinator.collect_traces() diff --git a/previous-work/wordpress_test.py b/previous-work/wordpress_test.py index bf61351..9f8173f 100644 --- a/previous-work/wordpress_test.py +++ b/previous-work/wordpress_test.py @@ -1,6 +1,6 @@ from playwright.sync_api import BrowserContext, expect -from abratest.dir_manager import DirManager +from pytest_abra.dir_manager import DirManager def test_wordpress(admin_session: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): diff --git a/prototyping/env_var_subprocess.py b/prototyping/env_var_subprocess.py new file mode 100644 index 0000000..aa0a027 --- /dev/null +++ b/prototyping/env_var_subprocess.py @@ -0,0 +1,17 @@ +import os +import subprocess + +# Set an environment variable in the parent process +os.environ["PARENT_VARIABLE"] = "12345s" + +# Spawn a subprocess and modify the environment variable +subprocess.run( + [ + "python", + "-c", + "import os; print('b', os.environ['PARENT_VARIABLE']); os.environ['PARENT_VARIABLE'] = 'modified_value'; print('c', os.environ['PARENT_VARIABLE'])", + ] +) + +# Check if the modification in the subprocess affected the parent process +print("a", os.environ["PARENT_VARIABLE"]) # This will print 'parent_value', not 'modified_value' diff --git a/pyproject.toml b/pyproject.toml index ec0809f..c951457 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,50 @@ [project] -name = "abratest" +name = "pytest-abra" +description = "A pytest plugin to test instances of abra recipes" +authors = [{name = "Local-IT e.V."}] +readme = "README.md" version = "0.2.0" -requires-python = "~=3.11" +requires-python = ">=3.10" +classifiers = [ +"Programming Language :: Python :: 3", +"Programming Language :: Python :: 3.10", +"Programming Language :: Python :: 3.11", +"Programming Language :: Python :: 3.12", +"Framework :: Pytest", +] dependencies = [ "pytest == 7.4.3", + "playwright == 1.40", + "pytest-html == 4.1.1", + "pytest-playwright == 0.4.3", + "python-dotenv == 1.0.0", + "loguru == 0.7.2", + "beautifulsoup4 == 4.12.2", + "imbox == 0.9.8", + "hatchling == 1.18.0", + "icecream", ] -[project.optional-dependencies] -dev = [ - "ruff >= 0.1.7", +[project.entry_points] +pytest11 = [ + "pytest_abra = pytest_abra.pytest_abra", +] + +[project.scripts] +abratest = "pytest_abra.cli:run" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +include = [ + "pytest_abra/*.py", +] +exclude = [ + "*.json", ] -[tool.setuptools] -package-dir = {"" = "abratest"} [tool.ruff] line-length = 120 diff --git a/abratest/__init__.py b/pytest_abra/__init__.py similarity index 100% rename from abratest/__init__.py rename to pytest_abra/__init__.py diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py new file mode 100644 index 0000000..2b1dfa7 --- /dev/null +++ b/pytest_abra/cli.py @@ -0,0 +1,56 @@ +import argparse +import os +from pathlib import Path + +from loguru import logger + +from pytest_abra.coordinator import Coordinator +from pytest_abra.dir_manager import DirManager +from pytest_abra.utils import get_datetime_string + + +def run(): + parser = argparse.ArgumentParser() + parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;") + parser.add_argument("--recipes_dir", type=Path, help="List of loaded env files separated with ;") + parser.add_argument("--output_dir", type=Path, help="List of loaded env files separated with ;") + parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=20_000) + parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode") + parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests") + args = parser.parse_args() + + ENV_FILES = [Path(s) for s in args.env_paths.split(";")] + + # -------------------------- enable playwright debug ------------------------- # + + if args.debug: + os.environ["PWDEBUG"] = "1" + + # ----------------------------- define session_id ---------------------------- # + + session_id = "test-" + get_datetime_string() + if args.resume: + # look for previous session_id + pass + # session_id = "abc" + + # ------------------------------- setup logging ------------------------------ # + + # todo: move to Coordinator + DIR = DirManager(output_dir=args.output_dir, session_id=session_id) + log_file = DIR.RECORDS / "coordinator.log" + logger.add(log_file) + + # ---------------------------- initialize and run ---------------------------- # + + coordinator = Coordinator( + env_paths_list=ENV_FILES, + output_dir=args.output_dir, + session_id=session_id, + recipes_dir=args.recipes_dir, + timeout=args.timeout, + ) + coordinator.setup_test() + coordinator.run_test() + coordinator.combine_html() + coordinator.collect_traces() diff --git a/abratest/coordinator.py b/pytest_abra/coordinator.py similarity index 90% rename from abratest/coordinator.py rename to pytest_abra/coordinator.py index 28a5102..fb0f422 100644 --- a/abratest/coordinator.py +++ b/pytest_abra/coordinator.py @@ -4,15 +4,17 @@ from pathlib import Path from loguru import logger -from abratest.dir_manager import DirManager -from abratest.env_manager import EnvFile, EnvManager -from abratest.html_helper import merge_html_files -from abratest.runner import Runner -from abratest.utils import rmtree +from pytest_abra.dir_manager import DirManager +from pytest_abra.env_manager import EnvFile, EnvManager +from pytest_abra.html_helper import merge_html_files +from pytest_abra.runner import Runner +from pytest_abra.utils import rmtree class Coordinator: - def __init__(self, env_paths_list: list[Path], output_dir: Path, session_id: str, recipes_dir: Path) -> None: + def __init__( + self, env_paths_list: list[Path], output_dir: Path, session_id: str, recipes_dir: Path, timeout: int + ) -> None: # logging out_string = "".join([e.name + "\n" for e in env_paths_list]) out_string += f"output_dir = {output_dir}\n" @@ -22,6 +24,7 @@ class Coordinator: self.RUNNER_DICT = self.create_runner_dict(recipes_dir) self.DIR = DirManager(output_dir=output_dir, session_id=session_id, recipes_dir=recipes_dir) self.ENV = EnvManager(env_paths_list, self.RUNNER_DICT) + self.TIMEOUT = timeout def setup_test(self) -> None: logger.info("calling setup_test()") diff --git a/pytest_abra/demo.py b/pytest_abra/demo.py new file mode 100644 index 0000000..0ece3c2 --- /dev/null +++ b/pytest_abra/demo.py @@ -0,0 +1 @@ +print("wooooorking") diff --git a/abratest/dir_manager.py b/pytest_abra/dir_manager.py similarity index 96% rename from abratest/dir_manager.py rename to pytest_abra/dir_manager.py index 2104a68..0125522 100644 --- a/abratest/dir_manager.py +++ b/pytest_abra/dir_manager.py @@ -46,7 +46,7 @@ class DirManager: @property def SESSION(self): - return self.OUTPUT_DIR / f"test-{self.session_id}" + return self.OUTPUT_DIR / self.session_id @property def RECORDS(self): diff --git a/abratest/env_manager.py b/pytest_abra/env_manager.py similarity index 98% rename from abratest/env_manager.py rename to pytest_abra/env_manager.py index 780436f..f6f16b5 100644 --- a/abratest/env_manager.py +++ b/pytest_abra/env_manager.py @@ -4,8 +4,8 @@ from typing import NamedTuple from dotenv import dotenv_values -from abratest.dir_manager import DirManager -from abratest.runner import Runner +from pytest_abra.dir_manager import DirManager +from pytest_abra.runner import Runner class EnvFile(NamedTuple): diff --git a/abratest/html_helper.py b/pytest_abra/html_helper.py similarity index 100% rename from abratest/html_helper.py rename to pytest_abra/html_helper.py diff --git a/pytest_abra/pytest_abra.py b/pytest_abra/pytest_abra.py new file mode 100644 index 0000000..21350af --- /dev/null +++ b/pytest_abra/pytest_abra.py @@ -0,0 +1,95 @@ +# This file is registered as a pytest plugin, meaning it will automatically loaded. +# All fixtures in this file will be available without manual loading. + +import os +import re +from imaplib import IMAP4_SSL +from pathlib import Path + +import pytest +from dotenv import dotenv_values +from playwright.sync_api import BrowserContext, expect +from pytest import Parser + +from pytest_abra.dir_manager import DirManager +from pytest_abra.env_manager import EnvFile +from pytest_abra.utils import BaseUrl + + +def pytest_addoption(parser: Parser): + parser.addoption("--runner_index", action="store", type=int) + parser.addoption("--output_dir", action="store", type=Path) + parser.addoption("--session_id", action="store", type=str) + parser.addoption("--timeout", action="store", type=int, default=20_000) + + +@pytest.fixture +def context(context: BrowserContext, request) -> BrowserContext: + # note: because this has the existing context fixture as an argument, it is ensured + # that the original fixture is called first and then overwritten by this custom one. + + TIMEOUT = request.config.getoption("--timeout") + LOCALE = {"Accept-Language": "de_DE"} + + context.set_default_timeout(TIMEOUT) + context.set_extra_http_headers(LOCALE) + expect.set_options(timeout=TIMEOUT) + return context + + +@pytest.fixture(scope="session") +def DIR(request) -> DirManager: + """Fixture holding test directories + + DIR.OUTPUT + DIR.SESSION + DIR.RECORDS + DIR.STATES + DIR.RESULTS""" + + output_dir = request.config.getoption("--output_dir") + assert output_dir, "pytest argument --output_dir not set" + session_id = request.config.getoption("--session_id") + assert session_id, "pytest argument --session_id not set" + dirmanager = DirManager(output_dir=output_dir, session_id=session_id) + dirmanager.create_all_dirs() + return dirmanager + + +@pytest.fixture(scope="session") +def ENV_FILES(DIR: DirManager) -> dict[int, EnvFile]: + out: dict[int, EnvFile] = dict() + for env_path in DIR.ENV_FILES.glob("*.env"): + config: dict[str, str] = dotenv_values(env_path) # type: ignore + env_type = config["TYPE"] + result = re.search(r"(\d+)-*", env_path.name) + assert result + runner_index = int(result[1]) + out[runner_index] = EnvFile(env_path=env_path, config=config, env_type=env_type) + return out + + +@pytest.fixture(scope="session") +def dotenv_config(request, ENV_FILES: dict[int, EnvFile]) -> dict[str, str]: + runner_index = request.config.getoption("--runner_index") + return ENV_FILES[runner_index].config + + +@pytest.fixture(scope="session") +def URL(dotenv_config: dict[str, str]) -> BaseUrl: + return BaseUrl(netloc=dotenv_config["DOMAIN"]) + + +@pytest.fixture(scope="session") +def imap_ssl_email_client() -> None: + assert os.environ["IMAP_HOST"] + assert os.environ["IMAP_PORT"] + assert os.environ["IMAP_USER"] + assert os.environ["IMAP_PASS"] + port = int(os.environ["IMAP_PORT"]) + imap_client = IMAP4_SSL(host=os.environ["IMAP_HOST"], port=port) + imap_client.login(os.environ["IMAP_USER"], os.environ["IMAP_PASS"]) + imap_client.select("INBOX") + yield imap_client + imap_client.close() + imap_client.logout() diff --git a/abratest/runner.py b/pytest_abra/runner.py similarity index 95% rename from abratest/runner.py rename to pytest_abra/runner.py index c9cbc9e..d17b756 100644 --- a/abratest/runner.py +++ b/pytest_abra/runner.py @@ -6,8 +6,8 @@ import pytest from loguru import logger if TYPE_CHECKING: - from abratest.coordinator import Coordinator - from abratest.env_manager import EnvFile + from pytest_abra.coordinator import Coordinator + from pytest_abra.env_manager import EnvFile @dataclass @@ -123,8 +123,8 @@ class Runner: # command_arguments.append("-rx") command_arguments.append(str(full_test_path)) - command_arguments.append("--env_file") - command_arguments.append(str(self.dotenv_path)) + command_arguments.append("--runner_index") + command_arguments.append(str(self.runner_index)) # set root dir for tests output (used in DirManager). this is our custom argument command_arguments.append("--output_dir") @@ -133,6 +133,9 @@ class Runner: command_arguments.append("--session_id") command_arguments.append(self.DIR.session_id) + command_arguments.append("--timeout") + command_arguments.append(str(self.coordinator.TIMEOUT)) + # artifacts dir from pytest # warning: https://github.com/microsoft/playwright-pytest/issues/111 # --output only works with the given context and page fixture diff --git a/abratest/utils.py b/pytest_abra/utils.py similarity index 96% rename from abratest/utils.py rename to pytest_abra/utils.py index e98fa9b..b820416 100644 --- a/abratest/utils.py +++ b/pytest_abra/utils.py @@ -17,7 +17,7 @@ class BaseUrl: return urlunparse((self.scheme, self.netloc, path, self.params, self.query, self.fragment)) -def get_session_id() -> str: +def get_datetime_string() -> str: current_datetime = datetime.now() return current_datetime.strftime("%Y-%m-%d-%H-%M-%S") diff --git a/recipes/authentik/tests_authentik/fixtures_authentik.py b/recipes/authentik/tests_authentik/fixtures_authentik.py index 34ed3d7..fa50e3f 100644 --- a/recipes/authentik/tests_authentik/fixtures_authentik.py +++ b/recipes/authentik/tests_authentik/fixtures_authentik.py @@ -3,8 +3,8 @@ import json import pytest from playwright.sync_api import BrowserContext, Page -from abratest.dir_manager import DirManager -from abratest.utils import BaseUrl +from pytest_abra.dir_manager import DirManager +from pytest_abra.utils import BaseUrl @pytest.fixture diff --git a/recipes/authentik/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py index b83da2f..fb8c3b2 100644 --- a/recipes/authentik/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -1,4 +1,4 @@ -from abratest.runner import Runner, Test +from pytest_abra.runner import Runner, Test def condition_always_true(dotenv_config: dict[str, str]) -> bool: diff --git a/recipes/authentik/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py index b7f7f37..680a85d 100644 --- a/recipes/authentik/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -4,8 +4,8 @@ import re from playwright.sync_api import BrowserContext, expect -from abratest.dir_manager import DirManager -from abratest.utils import BaseUrl +from pytest_abra.dir_manager import DirManager +from pytest_abra.utils import BaseUrl ADMIN_USER = os.environ["ADMIN_USER"] ADMIN_PASS = os.environ["ADMIN_PASS"] diff --git a/recipes/demo/tests_demo/fixtures_demo.py b/recipes/demo/tests_demo/fixtures_demo.py index b271e60..c650202 100644 --- a/recipes/demo/tests_demo/fixtures_demo.py +++ b/recipes/demo/tests_demo/fixtures_demo.py @@ -5,7 +5,7 @@ depend on [demo]. For this to work 1. the Runner class of the other test needs to define the depencency as seen by referencing RunnerDemo in the dependencies list: -from abratest.tests_demo.runner_demo import RunnerDemo +from pytest_abra.tests_demo.runner_demo import RunnerDemo class RunnerOther(Runner): dependencies = [RunnerDemo] @@ -15,7 +15,7 @@ class RunnerOther(Runner): To globally import for all tests in 'other', the import should be done in conftest: in 'conftest.py' in 'test_other' dir: -from abratest.tests_demo.fixtures_demo import demo_fixture +from pytest_abra.tests_demo.fixtures_demo import demo_fixture """ import pytest diff --git a/recipes/demo/tests_demo/runner_demo.py b/recipes/demo/tests_demo/runner_demo.py index 2b9aa78..e735e5f 100644 --- a/recipes/demo/tests_demo/runner_demo.py +++ b/recipes/demo/tests_demo/runner_demo.py @@ -1,4 +1,4 @@ -from abratest.runner import Runner, Test +from pytest_abra.runner import Runner, Test class RunnerDemo(Runner): diff --git a/recipes/nextcloud/tests_nextcloud/conftest.py b/recipes/nextcloud/tests_nextcloud/conftest.py index abdf9f9..ce56379 100644 --- a/recipes/nextcloud/tests_nextcloud/conftest.py +++ b/recipes/nextcloud/tests_nextcloud/conftest.py @@ -4,8 +4,8 @@ import os import pytest from playwright.sync_api import BrowserContext, Page -from abratest.dir_manager import DirManager -from abratest.utils import BaseUrl +from pytest_abra.dir_manager import DirManager +from pytest_abra.utils import BaseUrl pytest_plugins = "authentik.tests_authentik.fixtures_authentik" diff --git a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py index 5f7fe27..79b5050 100644 --- a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py @@ -1,4 +1,4 @@ -from abratest.runner import Runner, Test +from pytest_abra.runner import Runner, Test def condition_always_false(dotenv_config: dict[str, str]) -> bool: diff --git a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py index 020d56d..f400ce3 100644 --- a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py @@ -1,7 +1,7 @@ from playwright.sync_api import Page, expect -from abratest.dir_manager import DirManager -from abratest.utils import BaseUrl +from pytest_abra.dir_manager import DirManager +from pytest_abra.utils import BaseUrl # url dashboard # https://files.test.dev.local-it.cloud/apps/dashboard/ diff --git a/recipes/wordpress/tests_wordpress/conftest.py b/recipes/wordpress/tests_wordpress/conftest.py index c91a525..77977a9 100644 --- a/recipes/wordpress/tests_wordpress/conftest.py +++ b/recipes/wordpress/tests_wordpress/conftest.py @@ -4,7 +4,7 @@ import pytest from dotenv import dotenv_values from playwright.sync_api import BrowserContext, Page -from abratest.dir_manager import DirManager +from pytest_abra.dir_manager import DirManager pytest_plugins = "authentik.tests_authentik.fixtures_authentik" diff --git a/recipes/wordpress/tests_wordpress/runner_wordpress.py b/recipes/wordpress/tests_wordpress/runner_wordpress.py index 846f069..eae53ee 100644 --- a/recipes/wordpress/tests_wordpress/runner_wordpress.py +++ b/recipes/wordpress/tests_wordpress/runner_wordpress.py @@ -1,4 +1,4 @@ -from abratest.runner import Runner, Test +from pytest_abra.runner import Runner, Test def condition_always_true(dotenv_config: dict[str, str]) -> bool: diff --git a/recipes/wordpress/tests_wordpress/setup_wordpress.py b/recipes/wordpress/tests_wordpress/setup_wordpress.py index 0d8243a..b9fd773 100644 --- a/recipes/wordpress/tests_wordpress/setup_wordpress.py +++ b/recipes/wordpress/tests_wordpress/setup_wordpress.py @@ -1,7 +1,7 @@ import pytest from playwright.sync_api import BrowserContext, Page, expect -from abratest.dir_manager import DirManager +from pytest_abra.dir_manager import DirManager def test_visit_from_domain(authentik_admin_context: BrowserContext, dotenv_config: dict[str, str]): diff --git a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py index 1583a9f..a0c76ba 100644 --- a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py +++ b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py @@ -2,7 +2,7 @@ from playwright.sync_api import BrowserContext, expect -from abratest.dir_manager import DirManager +from pytest_abra.dir_manager import DirManager def test_welcome_message(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): diff --git a/requirements.txt b/requirements.txt index 88b903b..be7218a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ icecream loguru beautifulsoup4 imbox +hatchling \ No newline at end of file diff --git a/tests/test_env_resolution.py b/tests/test_env_resolution.py index 87f6f7c..4b11941 100644 --- a/tests/test_env_resolution.py +++ b/tests/test_env_resolution.py @@ -2,8 +2,8 @@ from pathlib import Path import pytest -from abratest.coordinator import Coordinator -from abratest.env_manager import DependencyRule, EnvFile, EnvManager +from pytest_abra.coordinator import Coordinator +from pytest_abra.env_manager import DependencyRule, EnvFile, EnvManager RECIPES_DIR = Path("./recipes").resolve() RUNNER_DICT = Coordinator.create_runner_dict(RECIPES_DIR) diff --git a/tests/test_url.py b/tests/test_url.py index 680023f..82b9f44 100644 --- a/tests/test_url.py +++ b/tests/test_url.py @@ -1,4 +1,4 @@ -from abratest.utils import BaseUrl +from pytest_abra.utils import BaseUrl url_input = { "netloc": "blog.dev.local-it.cloud", From d2cfc089c36f59f773f54648ff128c74bb72c3f2 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 7 Dec 2023 13:02:39 +0100 Subject: [PATCH 10/18] fix-docker (#10) * fix docker -> use "pip install -e ." in installation -> add symlinks in docker image -> docker / non docker execution can run same main.py + cli * remove sh scripts * remove requirements.txt Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/10 Co-authored-by: Daniel Co-committed-by: Daniel --- Dockerfile | 12 +++++++++--- README.md | 9 +++++---- main.py | 7 +++++++ pyproject.toml | 6 ++---- pytest_abra/cli.py | 7 +++---- requirements.txt | 9 --------- run_abratest.sh | 6 ------ test_abratest.sh | 7 ------- 8 files changed, 26 insertions(+), 37 deletions(-) delete mode 100644 requirements.txt delete mode 100644 run_abratest.sh delete mode 100644 test_abratest.sh diff --git a/Dockerfile b/Dockerfile index 8a7efdd..609f0a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,13 @@ RUN playwright install RUN playwright install-deps -COPY ./requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt +COPY . /code -WORKDIR /code \ No newline at end of file +WORKDIR /code + +RUN pip install --no-cache-dir -e . + +RUN rm -rf /code + +RUN ln -s /code/recipes /recipes +RUN ln -s /code/envfiles /envfiles diff --git a/README.md b/README.md index d3d3604..3418450 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ You can run pytest-abra with and without Docker. Choose now and follow the steps Create a python environment and install all dependencies via ```bash -pip install -r requirements.txt +pip install -e . playwright install ``` @@ -40,7 +40,8 @@ python main.py ```bash docker compose build # build the image -docker compose run --rm app ./run_pytest-abra.sh # run pytest-abra +docker compose run --rm app python main.py # run pytest-abra +docker compose run --rm -it app /bin/bash # debug the container ``` Force rebuild with cache @@ -49,7 +50,7 @@ Force rebuild with cache docker-compose up --build ``` -Force rebuild wtihtout cache +Force rebuild without cache ```bash docker-compose build --no-cache @@ -68,5 +69,5 @@ playwright codegen demo.playwright.dev/todomvc ```bash pytest # test pytest-abra pytest --collect-only # debug test pytest-abra -docker compose run --rm app ./test_pytest-abra.sh # test pytest-abra +docker compose run --rm app pytest # run pytest-abra ``` diff --git a/main.py b/main.py index e7b02a2..9de2d4f 100644 --- a/main.py +++ b/main.py @@ -31,6 +31,13 @@ ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES]) RECIPES_DIR = Path("../recipes").resolve() OUTPUT_DIR = Path("./test-output").resolve() +# -------------------------------- pythonpath -------------------------------- # + +# add recipes dir to pythonpath, so that python imports from there are possible +# the custom classes of Runner will be imported from there +os.environ["PYTHONPATH"] = RECIPES_DIR.as_posix() + +# ------------------------------------ run ----------------------------------- # subprocess.run( [ diff --git a/pyproject.toml b/pyproject.toml index c951457..20d9b07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,10 +25,8 @@ dependencies = [ "icecream", ] -[project.entry_points] -pytest11 = [ - "pytest_abra = pytest_abra.pytest_abra", -] +[project.entry-points.pytest11] +pytest_abra = "pytest_abra.pytest_abra" [project.scripts] abratest = "pytest_abra.cli:run" diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 2b1dfa7..0d82951 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -11,14 +11,13 @@ from pytest_abra.utils import get_datetime_string def run(): parser = argparse.ArgumentParser() - parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;") - parser.add_argument("--recipes_dir", type=Path, help="List of loaded env files separated with ;") - parser.add_argument("--output_dir", type=Path, help="List of loaded env files separated with ;") + parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;", required=True) + parser.add_argument("--recipes_dir", type=Path, help="List of loaded env files separated with ;", required=True) + parser.add_argument("--output_dir", type=Path, help="List of loaded env files separated with ;", required=True) parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=20_000) parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode") parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests") args = parser.parse_args() - ENV_FILES = [Path(s) for s in args.env_paths.split(";")] # -------------------------- enable playwright debug ------------------------- # diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index be7218a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -pytest -pytest-html -pytest-playwright -python-dotenv -icecream -loguru -beautifulsoup4 -imbox -hatchling \ No newline at end of file diff --git a/run_abratest.sh b/run_abratest.sh deleted file mode 100644 index ddffb80..0000000 --- a/run_abratest.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -RECIPES_PATH=$PWD/recipes -export PYTHONPATH=${PYTHONPATH}:$RECIPES_PATH - -python main.py \ No newline at end of file diff --git a/test_abratest.sh b/test_abratest.sh deleted file mode 100644 index d7abaaa..0000000 --- a/test_abratest.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -PWD_PATH=$PWD -RECIPES_PATH=$PWD/recipes -export PYTHONPATH=$PWD_PATH:$RECIPES_PATH - -pytest \ No newline at end of file From 0b4e0a0c16a900326517876706b2e74db4a23003 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 7 Dec 2023 17:21:19 +0100 Subject: [PATCH 11/18] remove-pythonpath-requirement (#11) Before, recipes_dir had to be present in the importable paths of the python interpreter. This was solved by adding it to the PYTHONPATH env var. Now, abratest handles this by itself. Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/11 Co-authored-by: Daniel Co-committed-by: Daniel --- README.md | 76 +++++++++++++++++++++++++++++++++++++- main.py | 6 --- pytest_abra/coordinator.py | 7 ++++ 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3418450..54fd681 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,88 @@ # pytest-abra -...description... + +Pytest-Abra is an installable python package design to test instances created with [abra](https://docs.coopcloud.tech/abra/). After installation, you will have two things: + +- `abratest` CLI command + +- `pytest-abra` Pytest plugin + +## CLI (abratest) + +The easiest way to call abratest is via the helper script in `main.py`. You can also directly call abratest via terminal, but you will have to make sure that the requirements below are met. To do that, you can call `abratest` with: + +```bash +abratest [arguments] +``` + +The cli command abratest has 3 **required arguments**: + +- `--env_paths`: list of the .env files used in the test +- `--recipes_dir`: directory of all available abra recipes +- `--output_dir`: target directory for all test results + +### env_paths [string] + +The variable env_paths consists of one or more paths pointing at .env files. The paths are separated with ";". These .env files are actually configuration files for `abra` recipes, but `pytest-abra` uses the same files for test configuration. + +To run `abratest` with these `.env` configuration files + +``` +/path/to/config_1.env +/path/to/config_2.env +/path/to/config_3.env +``` + +we simply call + +``` +abratest --env_paths /path/to/config_1.env;/path/to/config_2.env;/path/to/config_3.env +``` + +Under the hood, each `.env` file in `--env_paths` will create one instance of a `Runner` subclass. Let's say we have `wordpress_configuration.env` containing `TYPE=wordpress`. This will create an instance of `RunnerWordpress`. This class has to be imported from `recipes_dir`. + +### recipes_dir [string] + +The required argument `--recipes_dir` has to point to the directory, where all the abra recipes are stored. We can call `abratest` with + +``` +abratest --recipes_dir /path/to/abra/recipes +``` + +The expected dir structure inside of `recipes_dir` is as follows: + +``` +DIR recipes_dir [contains abra recipes] +│ +├── DIR authentik [authentik recipe] +│ ├── [files from authentik recipe] +│ └── DIR tests_authentik [pytest tests for authentik] +│ ├── FILE runner_authentik.py # containing RunnerAuthentik class +│ └── [pytest_files] +│ +└── DIR wordpress [wordpress recipe] + ├── [files from wordpress recipe] + └── DIR tests_wordpress [pytest tests for wordpress] + ├── FILE runner_wordpress.py # containing RunnerWordpress class + └── [pytest_files] +``` + +The class `RunnerWordpress` will be automatically imported by `importlib`, which is equivalent to + +```python +from wordpress.tests_wordpress.runner_wordpress import RunnerWordpress +``` # Usage To use pytest-abra, follow these steps: -## 1. GIT Clone +## 1. GIT clone [with & without Docker] To clone with submodules, use these git commands: ```bash git clone --recurse-submodules +// optional: git submodule update --init // add submodule after normal cloning git submodule update --remote // update submodules ``` diff --git a/main.py b/main.py index 9de2d4f..00a451c 100644 --- a/main.py +++ b/main.py @@ -31,12 +31,6 @@ ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES]) RECIPES_DIR = Path("../recipes").resolve() OUTPUT_DIR = Path("./test-output").resolve() -# -------------------------------- pythonpath -------------------------------- # - -# add recipes dir to pythonpath, so that python imports from there are possible -# the custom classes of Runner will be imported from there -os.environ["PYTHONPATH"] = RECIPES_DIR.as_posix() - # ------------------------------------ run ----------------------------------- # subprocess.run( diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index fb0f422..74c4ea2 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -1,5 +1,6 @@ import importlib import re +import sys from pathlib import Path from loguru import logger @@ -90,11 +91,17 @@ class Coordinator: "wordpress": RunnerWordpress, "nextcloud": RunnerNextcloud, } + + The Runner classes are automatically imported with importlib. The imports are successful + because recipes_dir is added to sys.path. """ RUNNER_DICT: dict[str, type["Runner"]] = dict() runner_discovery_pattern = re.compile("Runner.+") + # make it possible to import modules from recipes_dir + sys.path.append(recipes_dir.as_posix()) + for module_path in recipes_dir.rglob("*/runner*.py"): rel_path = module_path.relative_to(recipes_dir).as_posix().replace("/", ".").replace(".py", "") module = importlib.import_module(rel_path) From 41a042f07d3665d53dd99f617efd16b76fe99c46 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 7 Dec 2023 19:38:17 +0100 Subject: [PATCH 12/18] add-resume (#12) * add functionality to --resume flag. latest test will resume by running failed tests again * fix nextcloud setup -> all tests passing * fix expect timeout by moving it to its own fixture Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/12 Co-authored-by: Daniel Co-committed-by: Daniel --- main.py | 1 + pytest_abra/cli.py | 6 +++--- pytest_abra/coordinator.py | 7 ++++++- pytest_abra/dir_manager.py | 11 +++++++++++ pytest_abra/pytest_abra.py | 7 ++++++- recipes/nextcloud/tests_nextcloud/setup_nextcloud.py | 7 ++++++- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 00a451c..aa809bc 100644 --- a/main.py +++ b/main.py @@ -42,6 +42,7 @@ subprocess.run( RECIPES_DIR, "--output_dir", OUTPUT_DIR, + "--resume", # "--debug", ] ) diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 0d82951..8658dbb 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -29,9 +29,9 @@ def run(): session_id = "test-" + get_datetime_string() if args.resume: - # look for previous session_id - pass - # session_id = "abc" + latest_session_id = DirManager.get_latest_session_id(args.output_dir) + if latest_session_id: + session_id = DirManager.get_latest_session_id(args.output_dir) # ------------------------------- setup logging ------------------------------ # diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 74c4ea2..66e953a 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -14,7 +14,12 @@ from pytest_abra.utils import rmtree class Coordinator: def __init__( - self, env_paths_list: list[Path], output_dir: Path, session_id: str, recipes_dir: Path, timeout: int + self, + env_paths_list: list[Path], + output_dir: Path, + session_id: str, + recipes_dir: Path, + timeout: int, ) -> None: # logging out_string = "".join([e.name + "\n" for e in env_paths_list]) diff --git a/pytest_abra/dir_manager.py b/pytest_abra/dir_manager.py index 0125522..fb29e4d 100644 --- a/pytest_abra/dir_manager.py +++ b/pytest_abra/dir_manager.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Optional from dotenv import dotenv_values @@ -76,3 +77,13 @@ class DirManager: env_file = next(self.ENV_FILES.glob(f"*{search_string}*")) config: dict[str, str] = dotenv_values(env_file) # type: ignore return config + + @staticmethod + def get_latest_session_id(output_dir: Path) -> Optional[str]: + """returns the name of the newest dir inside of output_dir""" + all_dirs = [d for d in output_dir.iterdir() if d.is_dir()] + if all_dirs: + newest_dir: Path = max(all_dirs, key=lambda x: x.stat().st_ctime) + return newest_dir.name + else: + return None diff --git a/pytest_abra/pytest_abra.py b/pytest_abra/pytest_abra.py index 21350af..33a74ae 100644 --- a/pytest_abra/pytest_abra.py +++ b/pytest_abra/pytest_abra.py @@ -23,6 +23,12 @@ def pytest_addoption(parser: Parser): parser.addoption("--timeout", action="store", type=int, default=20_000) +@pytest.fixture(autouse=True) +def set_expect_timeout(request): + TIMEOUT = request.config.getoption("--timeout") + expect.set_options(timeout=TIMEOUT) + + @pytest.fixture def context(context: BrowserContext, request) -> BrowserContext: # note: because this has the existing context fixture as an argument, it is ensured @@ -33,7 +39,6 @@ def context(context: BrowserContext, request) -> BrowserContext: context.set_default_timeout(TIMEOUT) context.set_extra_http_headers(LOCALE) - expect.set_options(timeout=TIMEOUT) return context diff --git a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py index f400ce3..64c8e94 100644 --- a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py @@ -1,3 +1,5 @@ +import re + from playwright.sync_api import Page, expect from pytest_abra.dir_manager import DirManager @@ -16,6 +18,9 @@ def setup_nextcloud_admin_session(authentik_admin_page: Page, DIR: DirManager, U page_nextcloud = event_context.value context = page_nextcloud.context + # expect quota stats on files page to confirm successful login page_nextcloud.goto(URL.get("/apps/files")) - expect(page_nextcloud.get_by_role("link", name="Name")).to_be_visible() + quota_pattern = re.compile(r"\d*,\d .* (\d*,\d).") + expect(page_nextcloud.get_by_text(quota_pattern)).to_be_visible() + context.storage_state(path=DIR.STATES / "nextcloud_admin_state.json") From d1ff1183a5714fb9699cb35d4b6a8c2ed005f571 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 Dec 2023 18:17:31 +0100 Subject: [PATCH 13/18] refactoring (#13) * general project refactoring * various small improvements * improve imap fixture with helper functions and typing * add wordpress send email setup * add wordpress receive email test * add various documentation Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/13 Co-authored-by: Daniel Co-committed-by: Daniel --- README.md | 29 +++--- previous-work/wordpress_test.py | 4 +- prototyping/__init__.py | 0 prototyping/dependency_injection.py | 19 ++++ .../{email_stuff_2.py => email_imbox.py} | 0 prototyping/email_stuff.py | 49 ----------- prototyping/structure.md | 80 +++++++++++++++++ pyproject.toml | 2 +- pytest_abra/__init__.py | 13 +++ pytest_abra/cli.py | 6 +- pytest_abra/coordinator.py | 8 +- .../{pytest_abra.py => custom_fixtures.py} | 88 +++++++++++++++---- pytest_abra/demo.py | 1 - pytest_abra/dir_manager.py | 4 +- pytest_abra/env_manager.py | 17 ++-- pytest_abra/runner.py | 38 +++++--- pytest_abra/utils.py | 7 +- .../tests_authentik/runner_authentik.py | 10 +-- .../tests_authentik/setup_authentik.py | 20 ++--- .../tests_nextcloud/runner_nextcloud.py | 6 +- .../tests_nextcloud/tests_nextcloud.py | 12 +-- recipes/wordpress/tests_wordpress/conftest.py | 10 +-- .../tests_wordpress/runner_wordpress.py | 29 +++--- .../tests_wordpress/setup_wordpress.py | 4 +- .../setup_wordpress_trigger_email.py | 19 ++++ .../tests_wordpress/test_wordpress.py | 0 .../test_wordpress_localization.py | 6 +- .../test_wordpress_receive_email.py | 13 +++ tests/test_env_resolution.py | 4 +- 29 files changed, 323 insertions(+), 175 deletions(-) delete mode 100644 prototyping/__init__.py create mode 100644 prototyping/dependency_injection.py rename prototyping/{email_stuff_2.py => email_imbox.py} (100%) delete mode 100644 prototyping/email_stuff.py create mode 100644 prototyping/structure.md rename pytest_abra/{pytest_abra.py => custom_fixtures.py} (56%) delete mode 100644 pytest_abra/demo.py create mode 100644 recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py delete mode 100644 recipes/wordpress/tests_wordpress/test_wordpress.py create mode 100644 recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py diff --git a/README.md b/README.md index 54fd681..159e16e 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,19 @@ Pytest-Abra is an installable python package design to test instances created with [abra](https://docs.coopcloud.tech/abra/). After installation, you will have two things: -- `abratest` CLI command +- `abratest` CLI command. *Used to initialize the testing.* -- `pytest-abra` Pytest plugin +- `pytest-abra` Pytest plugin. *Automatically loads custom fixtures in any pytest (see `pytest_abra/custom_fixtures.py`)* -## CLI (abratest) +## CLI (`abratest`) -The easiest way to call abratest is via the helper script in `main.py`. You can also directly call abratest via terminal, but you will have to make sure that the requirements below are met. To do that, you can call `abratest` with: +`abratest` can be called via terminal: ```bash abratest [arguments] ``` -The cli command abratest has 3 **required arguments**: +To run successfully, very specific arguments are required. The easiest way to use abratest is with the helper script in `main.py`. Of yourse you can implement a similar helper script in the language of your liking. The cli command `abratest` has 3 **required arguments**: - `--env_paths`: list of the .env files used in the test - `--recipes_dir`: directory of all available abra recipes @@ -66,12 +66,20 @@ DIR recipes_dir [contains abra recipes] └── [pytest_files] ``` -The class `RunnerWordpress` will be automatically imported by `importlib`, which is equivalent to +The class `RunnerWordpress` will be automatically imported using `importlib` library, which is equivalent to the code below. Note that `recipes_dir` will be added to sys.path automatically for the import to work and that every `Runner` class matching `recipes_dir.rglob("*/runner*.py")` will be imported. ```python from wordpress.tests_wordpress.runner_wordpress import RunnerWordpress ``` +### output_dir [string] + +Path to the directory where all test outputs are stored (test report, tracebacks, playwright traces etc.) + +``` +abratest --output_dir /path/to/output +``` + # Usage To use pytest-abra, follow these steps: @@ -108,7 +116,7 @@ Run the script with python main.py ``` -# 2.2 Run with Docker +## 2.2 Run with Docker ```bash docker compose build # build the image @@ -128,12 +136,13 @@ Force rebuild without cache docker-compose build --no-cache ``` -## Codegen +## Playwright Debug & Codegen -Use playwright codegen to create code for new testes easily https://playwright.dev/python/docs/codegen +Use playwright debug mode or codegen to create testing code easily by recording browser actions https://playwright.dev/python/docs/codegen ```bash -playwright codegen demo.playwright.dev/todomvc +abratest --debug # launch your tests in debug mode +playwright codegen demo.playwright.dev/todomvc # visit given url in codegen mode ``` ## Development diff --git a/previous-work/wordpress_test.py b/previous-work/wordpress_test.py index 9f8173f..960fa72 100644 --- a/previous-work/wordpress_test.py +++ b/previous-work/wordpress_test.py @@ -3,12 +3,12 @@ from playwright.sync_api import BrowserContext, expect from pytest_abra.dir_manager import DirManager -def test_wordpress(admin_session: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): +def test_wordpress(admin_session: BrowserContext, env_config: dict[str, str], DIR: DirManager): page_authentik = admin_session.new_page() with page_authentik.expect_popup() as event_context: page_authentik.get_by_role("link", name="Wordpress").click() page_wordpress = event_context.value expect(page_wordpress.locator("#wpcontent")).to_be_visible() - if "locale" in dotenv_config and "de" in dotenv_config["locale"]: + if "locale" in env_config and "de" in env_config["locale"]: expect(page_wordpress.get_by_role("heading")).to_have_text("Willkommen bei WordPress!") diff --git a/prototyping/__init__.py b/prototyping/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/prototyping/dependency_injection.py b/prototyping/dependency_injection.py new file mode 100644 index 0000000..dbd3760 --- /dev/null +++ b/prototyping/dependency_injection.py @@ -0,0 +1,19 @@ +import inspect + +a = 2 +b = 3 +c = 4 + + +def func(a: int, c: int) -> int: + return a + c + + +arg_names = inspect.getfullargspec(func).args +print(arg_names) # ['a', 'c'] + +arguments = {arg: globals()[arg] for arg in arg_names if arg in globals()} +print(arguments) # {'a': 2, 'c': 4} + +result = func(**arguments) +print(result) # 6 diff --git a/prototyping/email_stuff_2.py b/prototyping/email_imbox.py similarity index 100% rename from prototyping/email_stuff_2.py rename to prototyping/email_imbox.py diff --git a/prototyping/email_stuff.py b/prototyping/email_stuff.py deleted file mode 100644 index ce0d850..0000000 --- a/prototyping/email_stuff.py +++ /dev/null @@ -1,49 +0,0 @@ -# %% -import email -import json -import os -from email.header import decode_header -from imaplib import IMAP4, IMAP4_SSL -from pathlib import Path - -# -------------------------------- credentials ------------------------------- # - -cred_file = Path("../credentials.json") -with open(cred_file, "r") as f: - CREDENTIALS = json.load(f) - -for key, value in CREDENTIALS.items(): - os.environ[key] = value - -IMAP_HOST = os.environ["IMAP_HOST"] -IMAP_PORT = os.environ["IMAP_PORT"] -IMAP_USER = os.environ["IMAP_USER"] -IMAP_PASS = os.environ["IMAP_PASS"] - - -# ----------------------------------- imap ----------------------------------- # - - -with IMAP4_SSL(host=IMAP_HOST) as imap_server: - imap_server.login(IMAP_USER, IMAP_PASS) - imap_server.select("INBOX") - - # Search for all emails in the folder - status, email_ids = imap_server.search(None, "ALL") - email_ids = email_ids[0].split() - - # Fetch email details using the retrieved IDs - for email_id in email_ids: - result, data = imap_server.fetch(email_id, "(RFC822)") - raw_email = data[0][1] # Raw content of the email - email_message = email.message_from_bytes(raw_email) - - # Extract the subject - subject_encoded = email_message.get("Subject") - decoded_subject = decode_header(subject_encoded)[0][0] - - if isinstance(decoded_subject, bytes): - decoded_subject = decoded_subject.decode() - - # Print or use the subject as needed - print("Subject:", decoded_subject) diff --git a/prototyping/structure.md b/prototyping/structure.md new file mode 100644 index 0000000..15c820f --- /dev/null +++ b/prototyping/structure.md @@ -0,0 +1,80 @@ + + +Abratest has 3 required inputs, but most importantly the test configuration is done through the .env files given with the --env_paths argument. So let's say we want to run abratest with these 3 .env files: + +- config1.env [of TYPE authentik] + +- config2.env [of TYPE wordpress] + +- config3.env [of TYPE wordpress] + +Now we run + +```bash +abratest --env_paths path/config1.env;path/config2.env;path/config3.env [...other args] +``` + + +``` +abratest -> create Coordinator() instance +└── Coordinator() -> create Runner() subclass instances + ├── RunnerAuthentik() [based on config1.env, loaded + │ │ from abra/recipes/authentik] + │ │ # RunnerAuthentik with 3 test files: + │ ├── RUN pytest path/setup_authentik.py + │ ├── RUN pytest path/test_authentik_1.py + │ └── RUN pytest path/test_authentik_2.py + ├── RunnerWordpress() [based on config2.env, loaded + │ │ from abra/recipes/wordpress] + │ │ # RunnerWordpress with 1 test file + │ ├── RUN pytest path/setup_authentik.py + │ ├── RUN pytest path/test_authentik_1.py + │ └── RUN pytest path/test_authentik_2.py + └── RunnerWordpress() [based on config3.env, loaded + │ from abra/recipes/wordpress] + │ # RunnerWordpress with 1 test file + ├── RUN pytest path/setup_authentik.py + ├── RUN pytest path/test_authentik_1.py + └── RUN pytest path/test_authentik_2.py + + +``` + +Coordinator will take care of the correct order of the tests. In general, tests are placed in one of 3 categories: `setups`, `tests` and `cleanups`. To associate a test with one of these categories, place the Test in the corresponding list of the Runner class, i.e. Runner.setups = [test] or Runner.tests = [test]. The execution order will be. + +> [setups] ➔ [tests] ➔ [cleanups] + + +Furthermore, some `Runner` classes can depend on others. For example, `RunnerWordpress` depends on `RunnerAuthentik`. Therefore, `Coordinator` will make sure that `RunnerAuthentik` runs before `RunnerWordpress`. We will end up with with this order: + +| # | Runner | Type | +| --- | -------------- | -------- | +| 1. | Authentik | setups | +| 2. | Wordpress-1 | setups | +| 3. | Wordpress-2 | setups | +| 4. | Authentik | tests | +| 5. | Wordpress-1 | tests | +| 6. | Wordpress-2 | tests | +| 7. | Authentik | cleanups | +| 8. | Wordpress-1 | cleanups | +| 9. | Wordpress-2 | cleanups | + + + +To comprehend this process, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. + + +```python +class RunnerWordpress(Runner): + env_type = "wordpress" + dependencies = ["authentik"] + setups = [ + Test(test_file="setup_wordpress_1.py"), + Test(test_file="setup_wordpress_2.py"), + ] + tests = [ + Test(test_file="test_wordpress.py"), + Test(condition=condition_function, test_file="test_wordpress_conditional.py"), + ] + cleanups = [] +``` diff --git a/pyproject.toml b/pyproject.toml index 20d9b07..9bf9361 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ ] [project.entry-points.pytest11] -pytest_abra = "pytest_abra.pytest_abra" +pytest_abra = "pytest_abra.custom_fixtures" [project.scripts] abratest = "pytest_abra.cli:run" diff --git a/pytest_abra/__init__.py b/pytest_abra/__init__.py index e69de29..18988b8 100644 --- a/pytest_abra/__init__.py +++ b/pytest_abra/__init__.py @@ -0,0 +1,13 @@ +from pytest_abra.coordinator import Coordinator +from pytest_abra.dir_manager import DirManager +from pytest_abra.runner import ConditionArgs, Runner, Test +from pytest_abra.utils import BaseUrl + +__all__ = [ + "Coordinator", + "ConditionArgs", + "Runner", + "Test", + "DirManager", + "BaseUrl", +] diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 8658dbb..1ebf852 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -4,7 +4,7 @@ from pathlib import Path from loguru import logger -from pytest_abra.coordinator import Coordinator +from pytest_abra import Coordinator from pytest_abra.dir_manager import DirManager from pytest_abra.utils import get_datetime_string @@ -18,7 +18,7 @@ def run(): parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode") parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests") args = parser.parse_args() - ENV_FILES = [Path(s) for s in args.env_paths.split(";")] + env_paths = [Path(s) for s in args.env_paths.split(";")] # -------------------------- enable playwright debug ------------------------- # @@ -43,7 +43,7 @@ def run(): # ---------------------------- initialize and run ---------------------------- # coordinator = Coordinator( - env_paths_list=ENV_FILES, + env_paths=env_paths, output_dir=args.output_dir, session_id=session_id, recipes_dir=args.recipes_dir, diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 66e953a..86cecdf 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -15,21 +15,21 @@ from pytest_abra.utils import rmtree class Coordinator: def __init__( self, - env_paths_list: list[Path], + env_paths: list[Path], output_dir: Path, session_id: str, recipes_dir: Path, timeout: int, ) -> None: # logging - out_string = "".join([e.name + "\n" for e in env_paths_list]) + out_string = "".join([e.name + "\n" for e in env_paths]) out_string += f"output_dir = {output_dir}\n" out_string += f"session_id = {session_id}" logger.info(f"initialize Coordinator instance with\nenv_paths_list =\n{out_string}") self.RUNNER_DICT = self.create_runner_dict(recipes_dir) self.DIR = DirManager(output_dir=output_dir, session_id=session_id, recipes_dir=recipes_dir) - self.ENV = EnvManager(env_paths_list, self.RUNNER_DICT) + self.ENV = EnvManager(env_paths=env_paths, RUNNER_DICT=self.RUNNER_DICT) self.TIMEOUT = timeout def setup_test(self) -> None: @@ -52,7 +52,7 @@ class Coordinator: """Creates an instance of the correct Runner class for each given env file""" runners: list[Runner] = [] for index, env_file in enumerate(env_files): - RunnerClass = self.RUNNER_DICT[env_file.config["TYPE"]] + RunnerClass = self.RUNNER_DICT[env_file.env_config["TYPE"]] runners.append(RunnerClass(coordinator=self, runner_index=index)) return runners diff --git a/pytest_abra/pytest_abra.py b/pytest_abra/custom_fixtures.py similarity index 56% rename from pytest_abra/pytest_abra.py rename to pytest_abra/custom_fixtures.py index 33a74ae..2c343a9 100644 --- a/pytest_abra/pytest_abra.py +++ b/pytest_abra/custom_fixtures.py @@ -3,11 +3,14 @@ import os import re -from imaplib import IMAP4_SSL +from datetime import datetime, timedelta from pathlib import Path +from typing import Protocol, TypedDict import pytest from dotenv import dotenv_values +from icecream import ic +from imbox import Imbox # type: ignore from playwright.sync_api import BrowserContext, expect from pytest import Parser @@ -62,39 +65,90 @@ def DIR(request) -> DirManager: @pytest.fixture(scope="session") -def ENV_FILES(DIR: DirManager) -> dict[int, EnvFile]: - out: dict[int, EnvFile] = dict() +def env_files(DIR: DirManager) -> list[EnvFile]: + """list of EnvFile objects created from the given env files""" + + env_files_dict: dict[int, EnvFile] = dict() for env_path in DIR.ENV_FILES.glob("*.env"): config: dict[str, str] = dotenv_values(env_path) # type: ignore env_type = config["TYPE"] result = re.search(r"(\d+)-*", env_path.name) assert result runner_index = int(result[1]) - out[runner_index] = EnvFile(env_path=env_path, config=config, env_type=env_type) - return out + env_files_dict[runner_index] = EnvFile(env_path=env_path, env_config=config, env_type=env_type) + keys = list(env_files_dict.keys()) + keys.sort() + return [env_files_dict[key] for key in keys] @pytest.fixture(scope="session") -def dotenv_config(request, ENV_FILES: dict[int, EnvFile]) -> dict[str, str]: +def env_config(request, env_files: list[EnvFile]) -> dict[str, str]: + """Current env_config""" runner_index = request.config.getoption("--runner_index") - return ENV_FILES[runner_index].config + return env_files[runner_index].env_config @pytest.fixture(scope="session") -def URL(dotenv_config: dict[str, str]) -> BaseUrl: - return BaseUrl(netloc=dotenv_config["DOMAIN"]) +def URL(env_config: dict[str, str]) -> BaseUrl: + """BaseUrl object based on current DOMAIN""" + return BaseUrl(netloc=env_config["DOMAIN"]) @pytest.fixture(scope="session") -def imap_ssl_email_client() -> None: +def imap_client() -> None: + """imap email client using credentials from environment variables""" + assert os.environ["IMAP_HOST"] assert os.environ["IMAP_PORT"] assert os.environ["IMAP_USER"] assert os.environ["IMAP_PASS"] - port = int(os.environ["IMAP_PORT"]) - imap_client = IMAP4_SSL(host=os.environ["IMAP_HOST"], port=port) - imap_client.login(os.environ["IMAP_USER"], os.environ["IMAP_PASS"]) - imap_client.select("INBOX") - yield imap_client - imap_client.close() - imap_client.logout() + + imbox = Imbox( + hostname=os.environ["IMAP_HOST"], + port=os.environ["IMAP_PORT"], + username=os.environ["IMAP_USER"], + password=os.environ["IMAP_PASS"], + ssl=True, + ssl_context=None, + starttls=False, + ) + + yield imbox + + imbox.logout() + + +class Body(TypedDict): + plain: list + html: list + + +class Message(Protocol): + sent_from: list + sent_to: list + subject: str + headers: list + date: str + body: Body + + +@pytest.fixture +def imap_recent_messages(imap_client: Imbox) -> list[Message]: + """Get all messages from [n_minutes] ago till now. + + # iterate with + for uid, message in messages: + print(uid, message.subject, message.date)""" + + N_MINUTES = 30 + + n_minutes_ago = datetime.now() - timedelta(minutes=N_MINUTES) + uids: list[bytes] = [] + messages: list[Message] = [] + # for uid, message in imap_client.messages(date__gt=n_minutes_ago): + for uid, message in imap_client.messages(): + ic("one time") + uids.append(uid) + messages.append(message) + + return messages diff --git a/pytest_abra/demo.py b/pytest_abra/demo.py deleted file mode 100644 index 0ece3c2..0000000 --- a/pytest_abra/demo.py +++ /dev/null @@ -1 +0,0 @@ -print("wooooorking") diff --git a/pytest_abra/dir_manager.py b/pytest_abra/dir_manager.py index fb29e4d..1ff4c11 100644 --- a/pytest_abra/dir_manager.py +++ b/pytest_abra/dir_manager.py @@ -10,11 +10,11 @@ class DirManager: The structures is as follows: tests dir/ - session_dir-1/ + session_id-1/ records results states - session_dir-2/ + session_id-2/ records ... """ diff --git a/pytest_abra/env_manager.py b/pytest_abra/env_manager.py index f6f16b5..f9d79cd 100644 --- a/pytest_abra/env_manager.py +++ b/pytest_abra/env_manager.py @@ -1,16 +1,17 @@ import shutil from pathlib import Path -from typing import NamedTuple +from typing import TYPE_CHECKING, NamedTuple from dotenv import dotenv_values -from pytest_abra.dir_manager import DirManager -from pytest_abra.runner import Runner +if TYPE_CHECKING: + from pytest_abra.dir_manager import DirManager + from pytest_abra.runner import Runner class EnvFile(NamedTuple): env_path: Path - config: dict[str, str] + env_config: dict[str, str] env_type: str def __repr__(self) -> str: @@ -23,8 +24,8 @@ class DependencyRule(NamedTuple): class EnvManager: - def __init__(self, env_paths_list: list[Path], RUNNER_DICT: dict[str, type["Runner"]]): - self.env_files: list[EnvFile] = self._get_env_files(env_paths_list) + def __init__(self, env_paths: list[Path], RUNNER_DICT: dict[str, type["Runner"]]): + self.env_files: list[EnvFile] = self._get_env_files(env_paths) self.dependency_rules: list[DependencyRule] = self._get_dependency_rules(self.env_files, RUNNER_DICT) self.env_files = self.sort_env_files_by_rule(self.env_files, self.dependency_rules) @@ -37,7 +38,7 @@ class EnvManager: config: dict[str, str] = dotenv_values(env_path) # type: ignore assert "TYPE" in config, f"the env file {env_path} does not specify the required TYPE key." env_type = config["TYPE"] - env_files.append(EnvFile(env_path=env_path, config=config, env_type=env_type)) + env_files.append(EnvFile(env_path=env_path, env_config=config, env_type=env_type)) return env_files @staticmethod @@ -92,7 +93,7 @@ class EnvManager: "Could not resolve test order. This is possibly due to a circular dependency (a on b, b on c, c on a)" ) - def copy_env_files(self, DIR: DirManager) -> None: + def copy_env_files(self, DIR: "DirManager") -> None: """Copies all env files to STATES/env_files. Files will be renamed to -- 00-authentik-login.test.dev.local-it.cloud.env""" diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index d17b756..1a75d27 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING, Callable, NamedTuple import pytest from loguru import logger @@ -10,10 +10,16 @@ if TYPE_CHECKING: from pytest_abra.env_manager import EnvFile +class ConditionArgs(NamedTuple): + env_config: dict[str, str] + runner_index: int + env_files: list["EnvFile"] + + @dataclass class Test: test_file: str - condition: Callable[[dict[str, str]], bool] | None = None + condition: Callable[[ConditionArgs], bool] | None = None prevent_skip: bool = False @@ -25,17 +31,13 @@ class Runner: dependencies: list[str] = [] def __init__(self, coordinator: "Coordinator", runner_index: int): - self.coordinator = coordinator # needed? - self.runner_index = runner_index # needed? + self.coordinator = coordinator + self.runner_index = runner_index self.DIR = coordinator.DIR self.ENV = coordinator.ENV self.RUNNER_DICT = coordinator.RUNNER_DICT - self.env_file: EnvFile = self.ENV.env_files[self.runner_index] - self.dotenv_path = self.env_file.env_path - self.config = self.env_file.config - logger.info(f"creating instance of {self.__class__.__name__}") def run_setups(self): @@ -81,16 +83,28 @@ class Runner: logger.info(f"skipping {identifier_string}, test has passed") return - if test.condition and not test.condition(self.config): - # test condition is defined but not met - logger.info(f"skipping {identifier_string}, test condition is not met") - return + if test.condition: + condition_result = self._run_condition(test.condition) + if not condition_result: + # test condition is defined but not met + logger.info(f"skipping {identifier_string}, test condition is not met") + return # test condition is undefined or not met logger.info(f"running {identifier_string}") result = self._call_pytest(full_test_path) self._create_result_file(result=result, identifier_string=identifier_string) + def _run_condition(self, condition_function: Callable[[ConditionArgs], bool]): + """run the test condition function with multiple arguments""" + # more arguments can be added later without changing the function signature + conditon_args = ConditionArgs( + env_files=self.ENV.env_files, + runner_index=self.runner_index, + env_config=self.ENV.env_files[self.runner_index].env_config, + ) + return condition_function(conditon_args) + def _is_test_passed(self, identifier_string: str, remove_existing: bool = False) -> bool: """returns True if the selected test matching identifier_string already passed diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index b820416..828e6b9 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -6,6 +6,8 @@ from urllib.parse import urlunparse @dataclass class BaseUrl: + """utility class to create a url string with urllib""" + netloc: str scheme: str = "https" path: str = "" @@ -33,8 +35,3 @@ def rmtree(root_dir: Path): child.unlink() root_dir.rmdir() - - -def make_url(domain: str) -> str: - """adds 'http://' at the beginning of a string""" - return "https://" + domain diff --git a/recipes/authentik/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py index fb8c3b2..0570db7 100644 --- a/recipes/authentik/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -1,12 +1,4 @@ -from pytest_abra.runner import Runner, Test - - -def condition_always_true(dotenv_config: dict[str, str]) -> bool: - return True - - -def condition_always_false(dotenv_config: dict[str, str]) -> bool: - return False +from pytest_abra import Runner, Test class RunnerAuthentik(Runner): diff --git a/recipes/authentik/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py index 680a85d..9aa6e3b 100644 --- a/recipes/authentik/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -14,20 +14,20 @@ ADMIN_PASS = os.environ["ADMIN_PASS"] TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} -def setup_admin_state(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): +def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager): # go to page page = context.new_page() - url = "https://" + dotenv_config["DOMAIN"] + url = "https://" + env_config["DOMAIN"] page.goto(url) # check welcome message - welcome_message = dotenv_config.get("welcome_message") + welcome_message = env_config.get("welcome_message") if welcome_message: expect(page.get_by_text(welcome_message)).to_be_visible() # login - page.locator('input[name="uidField"]').fill(ADMIN_USER) - page.locator('ak-stage-identification input[name="password"]').fill(ADMIN_PASS) + page.locator("input[name='uidField']").fill(ADMIN_USER) + page.locator("ak-stage-identification input[name='password']").fill(ADMIN_PASS) page.get_by_role("button", name="Log In").click() expect(page.locator("ak-library")).to_be_visible() @@ -35,7 +35,7 @@ def setup_admin_state(context: BrowserContext, dotenv_config: dict[str, str], DI context.storage_state(path=DIR.STATES / "authentik_admin_state.json") -def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, str], URL: BaseUrl): +def check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): # go to admin page page = admin_context.new_page() page.goto(URL.get()) @@ -49,7 +49,7 @@ def check_if_user_exists(admin_context: BrowserContext, dotenv_config: dict[str, return user.is_visible() -def create_invite_link(admin_context: BrowserContext, dotenv_config: dict[str, str], URL: BaseUrl): +def create_invite_link(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): # go to admin page page = admin_context.new_page() page.goto(URL.get()) @@ -98,19 +98,19 @@ def create_user(user_context: BrowserContext, invitelink): expect(page.locator("ak-library")).to_be_visible() -def setup_user_state(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager, URL: BaseUrl): +def setup_user_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl): # load admin cookies to context state_file = DIR.STATES / "authentik_admin_state.json" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) - if check_if_user_exists(context, dotenv_config, URL): + if check_if_user_exists(context, env_config, URL): # just login with user pass context.clear_cookies() else: # get invite_link - invite_link = create_invite_link(context, dotenv_config, URL) + invite_link = create_invite_link(context, env_config, URL) # create user context.clear_cookies() create_user(context, invite_link) diff --git a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py index 79b5050..0cb466c 100644 --- a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py @@ -1,8 +1,4 @@ -from pytest_abra.runner import Runner, Test - - -def condition_always_false(dotenv_config: dict[str, str]) -> bool: - return False +from pytest_abra import Runner, Test class RunnerNextcloud(Runner): diff --git a/recipes/nextcloud/tests_nextcloud/tests_nextcloud.py b/recipes/nextcloud/tests_nextcloud/tests_nextcloud.py index b79150f..496b3ae 100644 --- a/recipes/nextcloud/tests_nextcloud/tests_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/tests_nextcloud.py @@ -4,9 +4,9 @@ import pytest from playwright.sync_api import Page, expect -def test_nextcloud_quota(nextcloud_admin_page: Page, dotenv_config: dict[str, str]): - """Test Nextcloud""" - if dotenv_config.get("DEFAULT_QUOTA"): +def test_nextcloud_quota(nextcloud_admin_page: Page, env_config: dict[str, str]): + """Tests if the quota set in .env file matches the actual quota shown on the page within 10%""" + if env_config.get("DEFAULT_QUOTA"): # get quota from website quota_string = nextcloud_admin_page.get_by_text( re.compile(r"\d*,\d .* \d*,\d") @@ -17,7 +17,7 @@ def test_nextcloud_quota(nextcloud_admin_page: Page, dotenv_config: dict[str, st quota_website = float(out_number) # get quota from env - quota_config_string = dotenv_config["DEFAULT_QUOTA"] # "100 MB" + quota_config_string = env_config["DEFAULT_QUOTA"] # "100 MB" assert "MB" in quota_config_string quota_config = float(quota_config_string.strip("MB")) @@ -27,6 +27,6 @@ def test_nextcloud_quota(nextcloud_admin_page: Page, dotenv_config: dict[str, st @pytest.mark.skip -def test_nextcloud_apps(nextcloud_admin_page: Page, dotenv_config: dict[str, str]): - for app in dotenv_config["nc_apps"]: +def test_nextcloud_apps(nextcloud_admin_page: Page, env_config: dict[str, str]): + for app in env_config["nc_apps"]: expect(nextcloud_admin_page.get_by_role("link", name=app)).to_be_visible() diff --git a/recipes/wordpress/tests_wordpress/conftest.py b/recipes/wordpress/tests_wordpress/conftest.py index 77977a9..69fa4cf 100644 --- a/recipes/wordpress/tests_wordpress/conftest.py +++ b/recipes/wordpress/tests_wordpress/conftest.py @@ -1,10 +1,9 @@ import json import pytest -from dotenv import dotenv_values from playwright.sync_api import BrowserContext, Page -from pytest_abra.dir_manager import DirManager +from pytest_abra import BaseUrl, DirManager pytest_plugins = "authentik.tests_authentik.fixtures_authentik" @@ -18,10 +17,7 @@ def wordpress_admin_context(context: BrowserContext, DIR: DirManager) -> Browser @pytest.fixture -def wordpress_admin_page(wordpress_admin_context: BrowserContext, DIR: DirManager) -> Page: +def wordpress_admin_page(wordpress_admin_context: BrowserContext, URL: BaseUrl) -> Page: page = wordpress_admin_context.new_page() - env_file = DIR.ENV_FILES / "wordpress" - config: dict[str, str] = dotenv_values(env_file) # type: ignore - url = "https://" + config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) return page diff --git a/recipes/wordpress/tests_wordpress/runner_wordpress.py b/recipes/wordpress/tests_wordpress/runner_wordpress.py index eae53ee..d58bd74 100644 --- a/recipes/wordpress/tests_wordpress/runner_wordpress.py +++ b/recipes/wordpress/tests_wordpress/runner_wordpress.py @@ -1,26 +1,21 @@ -from pytest_abra.runner import Runner, Test +from pytest_abra import ConditionArgs, Runner, Test -def condition_always_true(dotenv_config: dict[str, str]) -> bool: - return True - - -def condition_always_false(dotenv_config: dict[str, str]) -> bool: - return False - - -def condition_has_locale(dotenv_config: dict[str, str]) -> bool: - if "LOCALE" in dotenv_config: - if "de" in dotenv_config["LOCALE"]: - return True +def condition_has_locale(args: ConditionArgs) -> bool: + env_config = args.env_config + if "de" in env_config.get("LOCALE", ""): + return True return False class RunnerWordpress(Runner): env_type = "wordpress" dependencies = ["authentik"] - setups = [Test(test_file="setup_wordpress.py")] - tests = [ - Test(test_file="test_wordpress.py"), - Test(condition=condition_has_locale, test_file="test_wordpress_localization.py"), + setups = [ + Test(test_file="setup_wordpress.py"), + Test(test_file="setup_wordpress_trigger_email.py"), + ] + tests = [ + Test(test_file="test_wordpress_receive_email.py", prevent_skip=True), + # Test(condition=condition_has_locale, test_file="test_wordpress_localization.py"), ] diff --git a/recipes/wordpress/tests_wordpress/setup_wordpress.py b/recipes/wordpress/tests_wordpress/setup_wordpress.py index b9fd773..4c44799 100644 --- a/recipes/wordpress/tests_wordpress/setup_wordpress.py +++ b/recipes/wordpress/tests_wordpress/setup_wordpress.py @@ -4,10 +4,10 @@ from playwright.sync_api import BrowserContext, Page, expect from pytest_abra.dir_manager import DirManager -def test_visit_from_domain(authentik_admin_context: BrowserContext, dotenv_config: dict[str, str]): +def test_visit_from_domain(authentik_admin_context: BrowserContext, env_config: dict[str, str]): """visit wordpress directly with admin_session, expect not to be logged in""" page = authentik_admin_context.new_page() - url = "https://" + dotenv_config["DOMAIN"] + url = "https://" + env_config["DOMAIN"] page.goto(url) with pytest.raises(AssertionError): # look for admin bar diff --git a/recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py b/recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py new file mode 100644 index 0000000..33afb6b --- /dev/null +++ b/recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py @@ -0,0 +1,19 @@ +import os + +from playwright.sync_api import Page, expect + +from pytest_abra import BaseUrl + + +def setup_trigger_email(wordpress_admin_page: Page, URL: BaseUrl): + """change profile email to EMAIL to trigger email""" + page = wordpress_admin_page + page.goto(URL.get("wp-admin/profile.php")) + EMAIL = os.environ["IMAP_EMAIL"] + print(EMAIL) + # breakpoint() + page.pause() + page.locator("input[id='email']").fill(EMAIL) + page.locator("input[id='submit']").click() + + expect(page.locator("div.notice").get_by_text(EMAIL)).to_be_visible() diff --git a/recipes/wordpress/tests_wordpress/test_wordpress.py b/recipes/wordpress/tests_wordpress/test_wordpress.py deleted file mode 100644 index e69de29..0000000 diff --git a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py index a0c76ba..51601cb 100644 --- a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py +++ b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py @@ -5,11 +5,11 @@ from playwright.sync_api import BrowserContext, expect from pytest_abra.dir_manager import DirManager -def test_welcome_message(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager): +def test_welcome_message(context: BrowserContext, env_config: dict[str, str], DIR: DirManager): page = context.new_page() - url = "https://" + dotenv_config["DOMAIN"] + url = "https://" + env_config["DOMAIN"] page.goto(url) expect(page.locator(".wp-block-heading")).to_be_visible() - if "locale" in dotenv_config and "de" in dotenv_config["locale"]: + if "locale" in env_config and "de" in env_config["locale"]: expect(page.get_by_role("heading")).to_have_text("Willkommen bei WordPress!") diff --git a/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py b/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py new file mode 100644 index 0000000..736f18d --- /dev/null +++ b/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py @@ -0,0 +1,13 @@ +from icecream import ic + +from pytest_abra.custom_fixtures import Message + + +def test_demo(imap_recent_messages: list[Message]): + for message in imap_recent_messages: + print(dir(message)) + ic(message.subject) + ic(message.body["plain"]) + + exit() + assert False diff --git a/tests/test_env_resolution.py b/tests/test_env_resolution.py index 4b11941..ddb5fc4 100644 --- a/tests/test_env_resolution.py +++ b/tests/test_env_resolution.py @@ -20,7 +20,7 @@ def test_complex_sorting() -> None: ] demo_types = ["a", "b", "c", "d", "e", "f", "g"] - env_files = [EnvFile(env_type=t, env_path=Path(), config=dict()) for t in demo_types] + env_files = [EnvFile(env_type=t, env_path=Path(), env_config=dict()) for t in demo_types] EnvManager.sort_env_files_by_rule sorted_env_files = EnvManager.sort_env_files_by_rule(env_files, demo_rules) @@ -36,7 +36,7 @@ def test_circular_import() -> None: ] demo_types = ["a", "b", "c"] - env_files = [EnvFile(env_type=t, env_path=Path(), config=dict()) for t in demo_types] + env_files = [EnvFile(env_type=t, env_path=Path(), env_config=dict()) for t in demo_types] with pytest.raises(ValueError): EnvManager.sort_env_files_by_rule(env_files, demo_rules) From 873bf73ae84d93c591145bc9cadde813203dac35 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:34:25 +0100 Subject: [PATCH 14/18] add api testing (#14) * add fixture to make api calls with authentification * add authentik test that checks the status of all blueprints * add option to append any kind of data to html report Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/14 Co-authored-by: Daniel Co-committed-by: Daniel --- .../{structure.md => documentation.md} | 26 ++++++++++++- pyproject.toml | 3 +- pytest_abra/__init__.py | 2 + pytest_abra/custom_fixtures.py | 21 +++++++--- pytest_abra/runner.py | 9 +++-- .../tests_authentik/runner_authentik.py | 2 +- .../test_authentik_blueprint_api.py | 39 +++++++++++++++++++ .../tests_authentik/test_authentik_dummy.py | 6 --- 8 files changed, 89 insertions(+), 19 deletions(-) rename prototyping/{structure.md => documentation.md} (75%) create mode 100644 recipes/authentik/tests_authentik/test_authentik_blueprint_api.py delete mode 100644 recipes/authentik/tests_authentik/test_authentik_dummy.py diff --git a/prototyping/structure.md b/prototyping/documentation.md similarity index 75% rename from prototyping/structure.md rename to prototyping/documentation.md index 15c820f..e27c359 100644 --- a/prototyping/structure.md +++ b/prototyping/documentation.md @@ -1,3 +1,10 @@ +# RUN + + + + + + Abratest has 3 required inputs, but most importantly the test configuration is done through the .env files given with the --env_paths argument. So let's say we want to run abratest with these 3 .env files: @@ -60,11 +67,14 @@ Furthermore, some `Runner` classes can depend on others. For example, `RunnerWor | 9. | Wordpress-2 | cleanups | +# Create a custom Runner -To comprehend this process, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. +To comprehend the process of creating a new subclass of `Runner`, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. ```python +from pytest_abra import Runner, Test + class RunnerWordpress(Runner): env_type = "wordpress" dependencies = ["authentik"] @@ -78,3 +88,17 @@ class RunnerWordpress(Runner): ] cleanups = [] ``` + +The signature of condition functions can be seen below. The function takes one `NamedTuple` and returns of type `bool`. You can learn about the contents of the input by looking up the class `ConditionArgs`. Generally speaking, it provides access to all of the .env files, especially the one related to the current Runner. + +```python +def condition_function(args: ConditionArgs) -> bool: + ... +``` + + +# Create custom Tests + +The test files are written in the same way as any other pytest test file. The only difference is that pytest-abra provides custom fixtures that make it easy to get the configuration by the provided .env files and to deal with URLS etc. + +# todo: add example \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9bf9361..a63e9dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "imbox == 0.9.8", "hatchling == 1.18.0", "icecream", + "tabulate", ] [project.entry-points.pytest11] @@ -49,6 +50,6 @@ line-length = 120 target-version = "py311" [tool.pytest.ini_options] -python_functions = "test_* setup_*" +python_functions = "setup_* test_* cleanup_*" norecursedirs = ".* previous-work recipes" testpaths = "tests" \ No newline at end of file diff --git a/pytest_abra/__init__.py b/pytest_abra/__init__.py index 18988b8..0b134ec 100644 --- a/pytest_abra/__init__.py +++ b/pytest_abra/__init__.py @@ -1,5 +1,6 @@ from pytest_abra.coordinator import Coordinator from pytest_abra.dir_manager import DirManager +from pytest_abra.env_manager import EnvFile from pytest_abra.runner import ConditionArgs, Runner, Test from pytest_abra.utils import BaseUrl @@ -10,4 +11,5 @@ __all__ = [ "Test", "DirManager", "BaseUrl", + "EnvFile", ] diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index 2c343a9..82bc0bd 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -5,18 +5,16 @@ import os import re from datetime import datetime, timedelta from pathlib import Path -from typing import Protocol, TypedDict +from typing import Generator, Protocol, TypedDict import pytest from dotenv import dotenv_values -from icecream import ic +from icecream import ic # type: ignore from imbox import Imbox # type: ignore -from playwright.sync_api import BrowserContext, expect +from playwright.sync_api import APIRequestContext, BrowserContext, Playwright, expect from pytest import Parser -from pytest_abra.dir_manager import DirManager -from pytest_abra.env_manager import EnvFile -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager, EnvFile def pytest_addoption(parser: Parser): @@ -152,3 +150,14 @@ def imap_recent_messages(imap_client: Imbox) -> list[Message]: messages.append(message) return messages + + +@pytest.fixture(scope="session") +def api_request_context( + playwright: Playwright, + DIR: DirManager, +) -> Generator[APIRequestContext, None, None]: + state_file = DIR.STATES / "authentik_admin_state.json" + request_context = playwright.request.new_context(storage_state=state_file) + yield request_context + request_context.dispose() diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 1a75d27..cb86d66 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -1,3 +1,4 @@ +import os from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Callable, NamedTuple @@ -134,7 +135,6 @@ class Runner: # command_arguments.append("--traceconfig") command_arguments.append("-v") - # command_arguments.append("-rx") command_arguments.append(str(full_test_path)) command_arguments.append("--runner_index") @@ -158,12 +158,13 @@ class Runner: command_arguments.append(str(self.DIR.RECORDS / "traces" / full_test_path.stem)) # tracing - command_arguments.append("--tracing") + command_arguments.append("--tracing") # "on", "off", "retain-on-failure" command_arguments.append("retain-on-failure") - # command_arguments.append("on") # Disable capturing. With -s set, prints will go to console as if pytest is not there. - # command_arguments.append("-s") + if os.environ.get("PWDEBUG") == "1": + command_arguments.append("-s") + command_arguments.append("-s") # headed # command_arguments.append("--headed") diff --git a/recipes/authentik/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py index 0570db7..2543dc6 100644 --- a/recipes/authentik/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -4,4 +4,4 @@ from pytest_abra import Runner, Test class RunnerAuthentik(Runner): env_type = "authentik" setups = [Test(test_file="setup_authentik.py")] - # tests = [Test(test_file="test_authentik_dummy.py")] + tests = [Test(test_file="test_authentik_blueprint_api.py")] diff --git a/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py new file mode 100644 index 0000000..87eb452 --- /dev/null +++ b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py @@ -0,0 +1,39 @@ +# api testing +# https://playwright.dev/python/docs/api-testing + +import pytest_html # type: ignore +from icecream import ic # type: ignore +from playwright.sync_api import APIRequestContext +from tabulate import tabulate # type: ignore + +from pytest_abra import BaseUrl + + +def test_authentik_blueprint_status( + api_request_context: APIRequestContext, + URL: BaseUrl, + extras, +) -> None: + blueprints = api_request_context.get(URL.get("api/v3/managed/blueprints")) + assert blueprints.ok + blueprints_data = blueprints.json() + ic(blueprints_data) + + # fake failed blueprint + # blueprints_data["results"][10]["status"] = "failed" + + table_data_all = [] + table_data_failed = [] + for item in blueprints_data["results"]: + row = [item["name"], item["enabled"], item["status"]] + table_data_all.append(row) + if item["status"] != "successful": + table_data_failed.append(row) + + table = tabulate(table_data_all, headers=["name", "enabled", "status"]) + extras.append(pytest_html.extras.text(table, name="Authentik Blueprint Status")) + + # with pytest -v (verbose) the failed blueprints will be visible in the traceback + assert ( + table_data_failed == [] + ), "One or more blueprints were not successful. See Authentik Blueprint Status in html report" diff --git a/recipes/authentik/tests_authentik/test_authentik_dummy.py b/recipes/authentik/tests_authentik/test_authentik_dummy.py deleted file mode 100644 index bddb561..0000000 --- a/recipes/authentik/tests_authentik/test_authentik_dummy.py +++ /dev/null @@ -1,6 +0,0 @@ -def test_true(): - assert 1 + 1 == 2 - - -def test_not_true(): - assert 1 + 1 == 3 From 016b88a68d7992b7acc897d9d788186e87e4cf43 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 15:28:48 +0100 Subject: [PATCH 15/18] testing-html-merge (#15) * add tests for merge_html_reports function Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/15 Co-authored-by: Daniel Co-committed-by: Daniel --- README.md | 1 + pyproject.toml | 5 +- pytest_abra/coordinator.py | 8 +- pytest_abra/html_helper.py | 35 +- pytest_abra/runner.py | 10 +- ....py__test_should_create_bug_report_0_0.txt | 33 + tests/assets/html_merge/assets/style.css | 319 ++++++++ tests/assets/html_merge/setup_wordpress.html | 770 ++++++++++++++++++ .../test_authentik_blueprint_api.html | 770 ++++++++++++++++++ .../test_wordpress_receive_email.html | 770 ++++++++++++++++++ tests/test_html_merge.py | 60 ++ 11 files changed, 2762 insertions(+), 19 deletions(-) create mode 100644 tests/assets/html_merge/assets/recipes_authentik_tests_authentik_test_authentik_blueprint_api.py__test_should_create_bug_report_0_0.txt create mode 100644 tests/assets/html_merge/assets/style.css create mode 100644 tests/assets/html_merge/setup_wordpress.html create mode 100644 tests/assets/html_merge/test_authentik_blueprint_api.html create mode 100644 tests/assets/html_merge/test_wordpress_receive_email.html create mode 100644 tests/test_html_merge.py diff --git a/README.md b/README.md index 159e16e..7fb9702 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ playwright codegen demo.playwright.dev/todomvc # visit given url in codegen mod ```bash pytest # test pytest-abra +pytest -m "not slow" # test pytest-abra without slow tests pytest --collect-only # debug test pytest-abra docker compose run --rm app pytest # run pytest-abra ``` diff --git a/pyproject.toml b/pyproject.toml index a63e9dd..d436714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,4 +52,7 @@ target-version = "py311" [tool.pytest.ini_options] python_functions = "setup_* test_* cleanup_*" norecursedirs = ".* previous-work recipes" -testpaths = "tests" \ No newline at end of file +testpaths = "tests" +markers = [ + "slow: marks tests as slow", +] \ No newline at end of file diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 86cecdf..52f5e77 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -7,7 +7,7 @@ from loguru import logger from pytest_abra.dir_manager import DirManager from pytest_abra.env_manager import EnvFile, EnvManager -from pytest_abra.html_helper import merge_html_files +from pytest_abra.html_helper import merge_html_reports from pytest_abra.runner import Runner from pytest_abra.utils import rmtree @@ -58,10 +58,10 @@ class Coordinator: def combine_html(self) -> None: """combines all generated pytest html reports into one""" - in_path = str(self.DIR.RECORDS / "html") - out_path = str(self.DIR.RECORDS / "full-report.html") + in_dir_path = str(self.DIR.RECORDS / "html") + out_file_path = str(self.DIR.RECORDS / "full-report.html") title = "combined.html" - merge_html_files(in_path, out_path, title) + merge_html_reports(in_dir_path, out_file_path, title) def collect_traces(self): """moves all traces into SESSION/RECORDS dir diff --git a/pytest_abra/html_helper.py b/pytest_abra/html_helper.py index 6e81c7e..6215350 100644 --- a/pytest_abra/html_helper.py +++ b/pytest_abra/html_helper.py @@ -5,19 +5,36 @@ import json import os import pathlib import re +import shutil -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup # type: ignore from packaging import version CHECKBOX_REGEX = r"^(?P0|[1-9]\d*) (?P.*)" -def merge_html_files(in_path: str, out_path: str, title: str): - paths = get_html_files(in_path, out_path) - if not paths: - raise RuntimeError(f"Unable to find html files in {in_path}") +def custom_copy_assets(assets_dir_path: str, out_file_path: str): + """custom function added for pytest_abra - assets_dir_path = get_assets_path(in_path) + copies every asset to asset folder. Exclude style.css as this is already handled by pytest_html_merger""" + + assets_source_dir = pathlib.Path(assets_dir_path) + assets_source_files = [p for p in assets_source_dir.glob("*") if p.is_file() and p.name != "style.css"] + out_dir_path = pathlib.Path(out_file_path).parent + assets_target_dir = out_dir_path / "assets" + assets_target_dir.mkdir(exist_ok=True) + for asset in assets_source_files: + shutil.copy(asset, assets_target_dir / asset.name) + + +def merge_html_reports(in_dir_path: str, out_file_path: str, report_title: str): + paths = get_html_files(in_dir_path, out_file_path) + if not paths: + raise RuntimeError(f"Unable to find html files in {in_dir_path}") + + assets_dir_path = get_assets_path(in_dir_path) + + custom_copy_assets(assets_dir_path, out_file_path) first_file = BeautifulSoup("".join(open(paths[0])), features="html.parser") paths.pop(0) @@ -30,7 +47,7 @@ def merge_html_files(in_path: str, out_path: str, title: str): if assets_dir_path is None: print( f"Will assume css is embedded in the reports. If this is not the case, " - f"Please make sure that you have 'assets' directory inside {in_path} " + f"Please make sure that you have 'assets' directory inside {in_dir_path} " f"which contains css files generated by pytest-html." ) else: @@ -42,7 +59,7 @@ def merge_html_files(in_path: str, out_path: str, title: str): head.style.append(content) h = first_file.find("h1") - h.string = title or os.path.basename(out_path) + h.string = report_title or os.path.basename(out_file_path) ps = first_file.find_all("p") pytest_version = ps[0].text.split(" ")[-1] @@ -106,7 +123,7 @@ def merge_html_files(in_path: str, out_path: str, title: str): for cb_type in cb_types: set_checkbox_value(first_file, cb_type, cb_types[cb_type]) - with open(out_path, "w") as f: + with open(out_file_path, "w") as f: f.write(str(first_file)) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index cb86d66..3d75562 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -57,7 +57,7 @@ class Runner: """runs the main test script and if available and sub test scripts if their running condition is met""" # check if required dependencies have passed if not self._dependencies_passed(): - logger.warning(f"skipping run_tests() of {self.env_type}, because some dependencies have not passed") + logger.warning(f"skipping run_tests() of {self.env_type} (one or more dependencies have not passed)") return for test in test_list: @@ -79,16 +79,16 @@ class Runner: # check if test aleady passed if self._is_test_passed(identifier_string, remove_existing=True): if test.prevent_skip: - logger.info(f"continuing , test {identifier_string} has passed but prevent_skip=True") + logger.info(f"continuing {identifier_string} (passed before but prevent_skip=True)") else: - logger.info(f"skipping {identifier_string}, test has passed") + logger.info(f"skipping {identifier_string} (test has passed)") return if test.condition: condition_result = self._run_condition(test.condition) if not condition_result: # test condition is defined but not met - logger.info(f"skipping {identifier_string}, test condition is not met") + logger.info(f"skipping {identifier_string} (test condition is not met)") return # test condition is undefined or not met @@ -126,7 +126,7 @@ class Runner: return already_passed def _call_pytest(self, full_test_path: Path) -> int: - """runs pytest programmatically on a specific file + """runs pytest programmatically with a specific file all tests in the file [full_test_path] will be run along with command line arguments""" diff --git a/tests/assets/html_merge/assets/recipes_authentik_tests_authentik_test_authentik_blueprint_api.py__test_should_create_bug_report_0_0.txt b/tests/assets/html_merge/assets/recipes_authentik_tests_authentik_test_authentik_blueprint_api.py__test_should_create_bug_report_0_0.txt new file mode 100644 index 0000000..eba0ff3 --- /dev/null +++ b/tests/assets/html_merge/assets/recipes_authentik_tests_authentik_test_authentik_blueprint_api.py__test_should_create_bug_report_0_0.txt @@ -0,0 +1,33 @@ +name enabled status +-------------------------------------------------------- --------- ---------- +Custom Invalidation Flow True successful +System - SCIM Provider - Mappings True successful +System - OAuth2 Provider - Scopes True successful +System - SAML Provider - Mappings True successful +System - LDAP Source - Mappings True successful +Migration - Remove old prompt fields True successful +Default - Events Transport & Rules True successful +Default - Source pre-authentication flow True successful +Default - TOTP MFA setup flow True successful +Default - WebAuthn MFA setup flow True successful +Default - Provider authorization flow (explicit consent) True failed +Default - Source authentication flow True successful +Default - Provider authorization flow (implicit consent) True successful +Default - Static MFA setup flow True successful +matrix True successful +Custom System Tenant True successful +Nextcloud True successful +Wordpress True successful +Custom Authentication Flow True successful +wekan True successful +Default - Invalidation flow True successful +Default - Tenant True successful +Flow Translations True successful +Default - User settings flow False successful +Default - Source enrollment flow False successful +Invitation Enrollment Flow True successful +vikunja True successful +Default - Password change flow False successful +Default - Authentication flow False successful +Recovery with email verification True successful +System - Proxy Provider - Scopes True successful \ No newline at end of file diff --git a/tests/assets/html_merge/assets/style.css b/tests/assets/html_merge/assets/style.css new file mode 100644 index 0000000..561524c --- /dev/null +++ b/tests/assets/html_merge/assets/style.css @@ -0,0 +1,319 @@ +body { + font-family: Helvetica, Arial, sans-serif; + font-size: 12px; + /* do not increase min-width as some may use split screens */ + min-width: 800px; + color: #999; +} + +h1 { + font-size: 24px; + color: black; +} + +h2 { + font-size: 16px; + color: black; +} + +p { + color: black; +} + +a { + color: #999; +} + +table { + border-collapse: collapse; +} + +/****************************** + * SUMMARY INFORMATION + ******************************/ +#environment td { + padding: 5px; + border: 1px solid #e6e6e6; + vertical-align: top; +} +#environment tr:nth-child(odd) { + background-color: #f6f6f6; +} +#environment ul { + margin: 0; + padding: 0 20px; +} + +/****************************** + * TEST RESULT COLORS + ******************************/ +span.passed, +.passed .col-result { + color: green; +} + +span.skipped, +span.xfailed, +span.rerun, +.skipped .col-result, +.xfailed .col-result, +.rerun .col-result { + color: orange; +} + +span.error, +span.failed, +span.xpassed, +.error .col-result, +.failed .col-result, +.xpassed .col-result { + color: red; +} + +.col-links__extra { + margin-right: 3px; +} + +/****************************** + * RESULTS TABLE + * + * 1. Table Layout + * 2. Extra + * 3. Sorting items + * + ******************************/ +/*------------------ + * 1. Table Layout + *------------------*/ +#results-table { + border: 1px solid #e6e6e6; + color: #999; + font-size: 12px; + width: 100%; +} +#results-table th, +#results-table td { + padding: 5px; + border: 1px solid #e6e6e6; + text-align: left; +} +#results-table th { + font-weight: bold; +} + +/*------------------ + * 2. Extra + *------------------*/ +.logwrapper { + max-height: 230px; + overflow-y: scroll; + background-color: #e6e6e6; +} +.logwrapper.expanded { + max-height: none; +} +.logwrapper.expanded .logexpander:after { + content: "collapse [-]"; +} +.logwrapper .logexpander { + z-index: 1; + position: sticky; + top: 10px; + width: max-content; + border: 1px solid; + border-radius: 3px; + padding: 5px 7px; + margin: 10px 0 10px calc(100% - 80px); + cursor: pointer; + background-color: #e6e6e6; +} +.logwrapper .logexpander:after { + content: "expand [+]"; +} +.logwrapper .logexpander:hover { + color: #000; + border-color: #000; +} +.logwrapper .log { + min-height: 40px; + position: relative; + top: -50px; + height: calc(100% + 50px); + border: 1px solid #e6e6e6; + color: black; + display: block; + font-family: "Courier New", Courier, monospace; + padding: 5px; + padding-right: 80px; + white-space: pre-wrap; +} + +div.media { + border: 1px solid #e6e6e6; + float: right; + height: 240px; + margin: 0 5px; + overflow: hidden; + width: 320px; +} + +.media-container { + display: grid; + grid-template-columns: 25px auto 25px; + align-items: center; + flex: 1 1; + overflow: hidden; + height: 200px; +} + +.media-container--fullscreen { + grid-template-columns: 0px auto 0px; +} + +.media-container__nav--right, +.media-container__nav--left { + text-align: center; + cursor: pointer; +} + +.media-container__viewport { + cursor: pointer; + text-align: center; + height: inherit; +} +.media-container__viewport img, +.media-container__viewport video { + object-fit: cover; + width: 100%; + max-height: 100%; +} + +.media__name, +.media__counter { + display: flex; + flex-direction: row; + justify-content: space-around; + flex: 0 0 25px; + align-items: center; +} + +.collapsible td:not(.col-links) { + cursor: pointer; +} +.collapsible td:not(.col-links):hover::after { + color: #bbb; + font-style: italic; + cursor: pointer; +} + +.col-result { + width: 130px; +} +.col-result:hover::after { + content: " (hide details)"; +} + +.col-result.collapsed:hover::after { + content: " (show details)"; +} + +#environment-header h2:hover::after { + content: " (hide details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; +} + +#environment-header.collapsed h2:hover::after { + content: " (show details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; +} + +/*------------------ + * 3. Sorting items + *------------------*/ +.sortable { + cursor: pointer; +} +.sortable.desc:after { + content: " "; + position: relative; + left: 5px; + bottom: -12.5px; + border: 10px solid #4caf50; + border-bottom: 0; + border-left-color: transparent; + border-right-color: transparent; +} +.sortable.asc:after { + content: " "; + position: relative; + left: 5px; + bottom: 12.5px; + border: 10px solid #4caf50; + border-top: 0; + border-left-color: transparent; + border-right-color: transparent; +} + +.hidden, .summary__reload__button.hidden { + display: none; +} + +.summary__data { + flex: 0 0 550px; +} +.summary__reload { + flex: 1 1; + display: flex; + justify-content: center; +} +.summary__reload__button { + flex: 0 0 300px; + display: flex; + color: white; + font-weight: bold; + background-color: #4caf50; + text-align: center; + justify-content: center; + align-items: center; + border-radius: 3px; + cursor: pointer; +} +.summary__reload__button:hover { + background-color: #46a049; +} +.summary__spacer { + flex: 0 0 550px; +} + +.controls { + display: flex; + justify-content: space-between; +} + +.filters, +.collapse { + display: flex; + align-items: center; +} +.filters button, +.collapse button { + color: #999; + border: none; + background: none; + cursor: pointer; + text-decoration: underline; +} +.filters button:hover, +.collapse button:hover { + color: #ccc; +} + +.filter__label { + margin-right: 10px; +} diff --git a/tests/assets/html_merge/setup_wordpress.html b/tests/assets/html_merge/setup_wordpress.html new file mode 100644 index 0000000..43180e8 --- /dev/null +++ b/tests/assets/html_merge/setup_wordpress.html @@ -0,0 +1,770 @@ + + + + + setup_wordpress.html + + + +

setup_wordpress.html

+

Report generated on 08-Dec-2023 at 14:55:57 by pytest-html + v4.1.1

+
+

Environment

+
+
+ + + + + +
+
+

Summary

+
+
+

2 tests took 00:00:11.

+

(Un)check the boxes to filter the results.

+
+ +
+
+
+
+ + 0 Failed, + + 2 Passed, + + 0 Skipped, + + 0 Expected failures, + + 0 Unexpected passes, + + 0 Errors, + + 0 Reruns +
+
+  /  +
+
+
+
+
+
+
+
+ + + + + + + + + +
ResultTestDurationLinks
+ +
+
+ +
+ \ No newline at end of file diff --git a/tests/assets/html_merge/test_authentik_blueprint_api.html b/tests/assets/html_merge/test_authentik_blueprint_api.html new file mode 100644 index 0000000..37897c3 --- /dev/null +++ b/tests/assets/html_merge/test_authentik_blueprint_api.html @@ -0,0 +1,770 @@ + + + + + test_authentik_blueprint_api.html + + + +

test_authentik_blueprint_api.html

+

Report generated on 09-Dec-2023 at 12:22:45 by pytest-html + v4.1.1

+
+

Environment

+
+
+ + + + + +
+
+

Summary

+
+
+

1 test took 00:00:01.

+

(Un)check the boxes to filter the results.

+
+ +
+
+
+
+ + 1 Failed, + + 0 Passed, + + 0 Skipped, + + 0 Expected failures, + + 0 Unexpected passes, + + 0 Errors, + + 0 Reruns +
+
+  /  +
+
+
+
+
+
+
+
+ + + + + + + + + +
ResultTestDurationLinks
+ +
+
+ +
+ \ No newline at end of file diff --git a/tests/assets/html_merge/test_wordpress_receive_email.html b/tests/assets/html_merge/test_wordpress_receive_email.html new file mode 100644 index 0000000..b29f75a --- /dev/null +++ b/tests/assets/html_merge/test_wordpress_receive_email.html @@ -0,0 +1,770 @@ + + + + + test_wordpress_receive_email.html + + + +

test_wordpress_receive_email.html

+

Report generated on 08-Dec-2023 at 16:00:41 by pytest-html + v4.1.1

+
+

Environment

+
+
+ + + + + +
+
+

Summary

+
+
+

1 test took 946 ms.

+

(Un)check the boxes to filter the results.

+
+ +
+
+
+
+ + 1 Failed, + + 0 Passed, + + 0 Skipped, + + 0 Expected failures, + + 0 Unexpected passes, + + 0 Errors, + + 0 Reruns +
+
+  /  +
+
+
+
+
+
+
+
+ + + + + + + + + +
ResultTestDurationLinks
+ +
+
+ +
+ \ No newline at end of file diff --git a/tests/test_html_merge.py b/tests/test_html_merge.py new file mode 100644 index 0000000..d97ab83 --- /dev/null +++ b/tests/test_html_merge.py @@ -0,0 +1,60 @@ +# tmp_path fixture: +# https://docs.pytest.org/en/6.2.x/tmpdir.html + +from pathlib import Path + +import pytest +from icecream import ic # type: ignore +from playwright.sync_api import BrowserContext, expect + +from pytest_abra import BaseUrl +from pytest_abra.html_helper import merge_html_reports + + +@pytest.fixture(scope="session") +def session_tmp_path(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("html_test") + + +def test_merge_html(session_tmp_path: Path): + """combines all generated pytest html reports into one""" + + in_dir_path = Path(__file__).parent / "assets" / "html_merge" + in_dir_path = in_dir_path.resolve() + ic(in_dir_path) + + out_file_path = session_tmp_path / "test.html" + out_assets_dir = session_tmp_path / "assets" + + merge_html_reports(in_dir_path.as_posix(), out_file_path.as_posix(), "combined.html") + + assert out_file_path.is_file() + assert out_assets_dir.is_dir() + assert next(out_assets_dir.glob("*")) + + +@pytest.mark.slow +def test_check_result_with_playwright(session_tmp_path, context: BrowserContext): + html_file = session_tmp_path / "test.html" + file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() + page = context.new_page() + page.goto(file_url) + + # check if combined is correct + expect(page.get_by_text("2 Passed,")).to_be_visible() + expect(page.get_by_text("2 Failed,")).to_be_visible() + expect(page.get_by_text("tests ran in 12.946 seconds")).to_be_visible() + + # check if heading is correct + expect(page.get_by_role("heading", name="combined.html")).to_be_visible() + + # check if traceback is included + expect(page.get_by_text("E AssertionError: One or more")).to_be_visible() + + # check if asset works + with page.expect_popup() as page1_info: + page.get_by_role("link", name="Authentik Blueprint Status").click() + page1 = page1_info.value + + # see if content of txt file is correct + expect(page1.get_by_text("failed")).to_be_visible() From 2dd765a9749e1dc7bf299b6bba6d5120c3b19e5c Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 14:03:58 +0100 Subject: [PATCH 16/18] various (#16) * add full integration test of cli / pytest_abra with all tests * save path of runner_*.py in runner subclass to improve test discovery -> allows for same test name in two different runners * reorganize output dir names * use URL fixture everywhere * rework coordinator interface * add --session_id to cli args * add log results table * plenty of refactoring * add assert messages * add plenty of tests * add /docs dir with plenty of documentation * fix authentik setup * add authentik cleanup, remove test user * add random test user credential generation and integrate into test routine. random creds are saved to STATES Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/16 Co-authored-by: Daniel Co-committed-by: Daniel --- .gitignore | 3 +- README.md | 154 ++------ credentials-example.json | 9 + docs/documentation.md | 329 ++++++++++++++++++ main.py | 16 +- prototyping/documentation.md | 104 ------ pyproject.toml | 10 +- pytest_abra/__init__.py | 3 +- pytest_abra/cli.py | 26 +- pytest_abra/coordinator.py | 63 +++- pytest_abra/custom_fixtures.py | 35 +- pytest_abra/dir_manager.py | 32 +- pytest_abra/env_manager.py | 30 +- pytest_abra/runner.py | 126 +++---- pytest_abra/shared_types.py | 16 + pytest_abra/utils.py | 51 ++- .../tests_authentik/cleanup_authentik.py | 40 +++ recipes/authentik/tests_authentik/conftest.py | 46 +++ .../tests_authentik/fixtures_authentik.py | 5 +- .../tests_authentik/runner_authentik.py | 1 + .../tests_authentik/setup_authentik.py | 42 +-- .../test_authentik_blueprint_api.py | 2 +- recipes/nextcloud/tests_nextcloud/conftest.py | 3 +- .../tests_nextcloud/setup_nextcloud.py | 3 +- .../tests_wordpress/runner_wordpress.py | 11 +- .../tests_wordpress/setup_wordpress.py | 7 +- .../test_wordpress_localization.py | 9 +- .../test_wordpress_receive_email.py | 2 + tests/test_cli.py | 54 +++ tests/test_cli_full_integration.py | 68 ++++ tests/test_coordinator.py | 43 +++ tests/test_dir_manager.py | 30 ++ tests/test_env_manager.py | 137 ++++++++ tests/test_env_resolution.py | 14 + tests/test_html_merge.py | 24 +- tests/test_runner.py | 29 ++ 36 files changed, 1145 insertions(+), 432 deletions(-) create mode 100644 credentials-example.json create mode 100644 docs/documentation.md delete mode 100644 prototyping/documentation.md create mode 100644 pytest_abra/shared_types.py create mode 100644 recipes/authentik/tests_authentik/cleanup_authentik.py create mode 100644 recipes/authentik/tests_authentik/conftest.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_cli_full_integration.py create mode 100644 tests/test_coordinator.py create mode 100644 tests/test_dir_manager.py create mode 100644 tests/test_env_manager.py create mode 100644 tests/test_runner.py diff --git a/.gitignore b/.gitignore index 78cd629..f541c4b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ TestResults/ *.json *.zip *.egg-info -credentials* \ No newline at end of file +credentials* +!credentials-example.json \ No newline at end of file diff --git a/README.md b/README.md index 7fb9702..2a1b8f0 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,18 @@ # pytest-abra -Pytest-Abra is an installable python package design to test instances created with [abra](https://docs.coopcloud.tech/abra/). After installation, you will have two things: +Pytest-Abra is an installable python package baed on pytest, designed to test instances created with [abra](https://docs.coopcloud.tech/abra/). After installation, you will have two things: - `abratest` CLI command. *Used to initialize the testing.* -- `pytest-abra` Pytest plugin. *Automatically loads custom fixtures in any pytest (see `pytest_abra/custom_fixtures.py`)* - -## CLI (`abratest`) - -`abratest` can be called via terminal: - -```bash -abratest [arguments] -``` - -To run successfully, very specific arguments are required. The easiest way to use abratest is with the helper script in `main.py`. Of yourse you can implement a similar helper script in the language of your liking. The cli command `abratest` has 3 **required arguments**: - -- `--env_paths`: list of the .env files used in the test -- `--recipes_dir`: directory of all available abra recipes -- `--output_dir`: target directory for all test results - -### env_paths [string] - -The variable env_paths consists of one or more paths pointing at .env files. The paths are separated with ";". These .env files are actually configuration files for `abra` recipes, but `pytest-abra` uses the same files for test configuration. - -To run `abratest` with these `.env` configuration files - -``` -/path/to/config_1.env -/path/to/config_2.env -/path/to/config_3.env -``` - -we simply call - -``` -abratest --env_paths /path/to/config_1.env;/path/to/config_2.env;/path/to/config_3.env -``` - -Under the hood, each `.env` file in `--env_paths` will create one instance of a `Runner` subclass. Let's say we have `wordpress_configuration.env` containing `TYPE=wordpress`. This will create an instance of `RunnerWordpress`. This class has to be imported from `recipes_dir`. - -### recipes_dir [string] - -The required argument `--recipes_dir` has to point to the directory, where all the abra recipes are stored. We can call `abratest` with - -``` -abratest --recipes_dir /path/to/abra/recipes -``` - -The expected dir structure inside of `recipes_dir` is as follows: - -``` -DIR recipes_dir [contains abra recipes] -│ -├── DIR authentik [authentik recipe] -│ ├── [files from authentik recipe] -│ └── DIR tests_authentik [pytest tests for authentik] -│ ├── FILE runner_authentik.py # containing RunnerAuthentik class -│ └── [pytest_files] -│ -└── DIR wordpress [wordpress recipe] - ├── [files from wordpress recipe] - └── DIR tests_wordpress [pytest tests for wordpress] - ├── FILE runner_wordpress.py # containing RunnerWordpress class - └── [pytest_files] -``` - -The class `RunnerWordpress` will be automatically imported using `importlib` library, which is equivalent to the code below. Note that `recipes_dir` will be added to sys.path automatically for the import to work and that every `Runner` class matching `recipes_dir.rglob("*/runner*.py")` will be imported. - -```python -from wordpress.tests_wordpress.runner_wordpress import RunnerWordpress -``` - -### output_dir [string] - -Path to the directory where all test outputs are stored (test report, tracebacks, playwright traces etc.) - -``` -abratest --output_dir /path/to/output -``` +- `pytest-abra` Pytest plugin. *Automatically loads custom fixtures in any pytest run (see `pytest_abra/custom_fixtures.py`)* # Usage -To use pytest-abra, follow these steps: +Pytest-abra can easily be installed on any system but also offers a Docker image. To use pytest-abra, follow these steps: -## 1. GIT clone [with & without Docker] +## Usage [without Docker] + +### Installation [without Docker] To clone with submodules, use these git commands: @@ -95,14 +23,6 @@ git submodule update --init // add submodule after normal cloning git submodule update --remote // update submodules ``` -## Run - -You can run pytest-abra with and without Docker. Choose now and follow the steps accordingly: - -## 2.1 Run without Docker - -### Installation - Create a python environment and install all dependencies via ```bash @@ -110,46 +30,40 @@ pip install -e . playwright install ``` -Run the script with +### Run [without Docker] + +Run the helper script or directly use the cli command (see docs) ```bash -python main.py +python main.py # run pytest-abra +abratest [options] ``` -## 2.2 Run with Docker +## Usage [with docker] + +### Installation [with docker] + +To clone with submodules, use these git commands: + +```bash +git clone --recurse-submodules +// optional: +git submodule update --init // add submodule after normal cloning +git submodule update --remote // update submodules +``` + +Build the image ```bash docker compose build # build the image +docker compose build --no-cache # Force rebuild without cache +``` + +### Run [with docker] + +Run the script + +```bash docker compose run --rm app python main.py # run pytest-abra -docker compose run --rm -it app /bin/bash # debug the container -``` - -Force rebuild with cache - -```bash -docker-compose up --build -``` - -Force rebuild without cache - -```bash -docker-compose build --no-cache -``` - -## Playwright Debug & Codegen - -Use playwright debug mode or codegen to create testing code easily by recording browser actions https://playwright.dev/python/docs/codegen - -```bash -abratest --debug # launch your tests in debug mode -playwright codegen demo.playwright.dev/todomvc # visit given url in codegen mode -``` - -## Development - -```bash -pytest # test pytest-abra -pytest -m "not slow" # test pytest-abra without slow tests -pytest --collect-only # debug test pytest-abra -docker compose run --rm app pytest # run pytest-abra +docker compose run --rm -it app /bin/bash # use the container interactively ``` diff --git a/credentials-example.json b/credentials-example.json new file mode 100644 index 0000000..fa0f58b --- /dev/null +++ b/credentials-example.json @@ -0,0 +1,9 @@ +{ + "ADMIN_USER": "admin", + "ADMIN_PASS": "password", + "IMAP_EMAIL": "test@domain.com", + "IMAP_HOST": "mail.domain.com", + "IMAP_PORT": "993", + "IMAP_USER": "imap_user", + "IMAP_PASS": "password" +} \ No newline at end of file diff --git a/docs/documentation.md b/docs/documentation.md new file mode 100644 index 0000000..b84c7da --- /dev/null +++ b/docs/documentation.md @@ -0,0 +1,329 @@ +# pytest-abra + +Pytest-Abra is an installable python package baed on pytest, designed to test instances created with [abra](https://docs.coopcloud.tech/abra/). After installation, you will have two things: + +- `abratest` CLI command. *Used to initialize the testing.* + +- `pytest-abra` Pytest plugin. *Automatically loads custom fixtures in any pytest run (see `pytest_abra/custom_fixtures.py`)* + +# Getting Started + +Pytest-abra can easily be installed on any system but also offers a Docker image. To use pytest-abra, follow these steps: + +## Usage [without Docker] + +### Installation [without Docker] + +To clone with submodules, use these git commands: + +```bash +git clone --recurse-submodules +// optional: +git submodule update --init // add submodule after normal cloning +git submodule update --remote // update submodules +``` + +Create a python environment and install all dependencies via + +```bash +pip install -e . +playwright install +``` + +### Run [without Docker] + +Run the helper script or directly use the cli command (see docs) + +```bash +python main.py # run pytest-abra +abratest [options] +``` + +## Usage [with docker] + +### Installation [with docker] + +To clone with submodules, use these git commands: + +```bash +git clone --recurse-submodules +// optional: +git submodule update --init // add submodule after normal cloning +git submodule update --remote // update submodules +``` + +Build the image + +```bash +docker compose build # build the image +docker compose build --no-cache # Force rebuild without cache +``` + +### Run [with docker] + +Run the script + +```bash +docker compose run --rm app python main.py # run pytest-abra +docker compose run --rm -it app /bin/bash # use the container interactively +``` + +# Documentation + +After Installation, `abratest` can be called via terminal: + +```bash +abratest [arguments] +``` + +To run successfully, very specific arguments are required. The easiest way to use `abratest` is with the helper script `main.py`. Of yourse you can implement a similar helper script in the language of your liking. + +## CLI Interface + +The cli command `abratest` has 3 **required arguments**: + +- `--env_paths ENV_PATHS`: list of the .env files used in the test +- `--recipes_dir RECIPES_DIR`: directory of all available abra recipes +- `--output_dir OUTPUT_DIR`: target directory for all test results + +Furtheremore, there are these optional arguments: + +- `--resume`: `abratest` will take the directory in `output_dir` with the most recent creation date and resume the tests there. +- `--session_id SESSION_ID`: Instead of generating a new session_id, the given session_id is used to run or resume the test. Overwrites --resume to False. +- `--debug`: enables playwright debug mode, see docs [here](https://playwright.dev/python/docs/running-tests#debugging-tests) +- `--timeout`: will overwrite the default playwright timeouts in [ms], see docs [here](https://playwright.dev/python/docs/api/class-browsercontext#browser-context-set-default-timeout) and [here](https://playwright.dev/python/docs/test-assertions#global-timeout). In our current setup, some tests can fail at 10s but will pass with 20s. + +### env_paths [required | string] + +The .env files provied through the `--env_paths` argument are the most important input to abratest, as they serve as configuration for the tests. One or more paths pointing at .env files can be provided, multiple paths are separated with ";". These .env files are actually the same files that are used to configure the `abra` recipes for instance creation. + +To run `abratest` with these `.env` configuration files + +- `/path/config_1.env` [of TYPE authentik] +- `/path/config_2.env` [of TYPE wordpress] +- `/path/config_3.env` [of TYPE wordpress] + +we simply call + +``` +abratest --env_paths /path/config_1.env;/path/config_2.env;/path/config_3.env [...other args] +``` + +Under the hood, each `.env` file in `--env_paths` will create one instance of a `Runner` subclass. Let's say we have `config_2.env` containing `TYPE=wordpress`. This will create an instance of `RunnerWordpress`. This class has to be imported from `recipes_dir`. + +### recipes_dir [required | string] + +The required argument `--recipes_dir` has to point to the directory, where all the abra recipes are stored. We can call `abratest` with + +``` +abratest --recipes_dir /path/to/abra/recipes +``` + +The expected dir structure inside of `recipes_dir` is as follows: + +``` +DIR recipes_dir [contains abra recipes] +│ +├── DIR authentik [authentik recipe] +│ ├── [files from authentik recipe] +│ └── DIR tests_authentik [pytest tests for authentik] +│ ├── FILE runner_authentik.py # containing RunnerAuthentik class +│ └── [pytest_files] +│ +└── DIR wordpress [wordpress recipe] + ├── [files from wordpress recipe] + └── DIR tests_wordpress [pytest tests for wordpress] + ├── FILE runner_wordpress.py # containing RunnerWordpress class + └── [pytest_files] +``` + +The class `RunnerWordpress` will be automatically imported using `importlib` library, which is equivalent to the code below. Note that `recipes_dir` will be added to sys.path automatically for the import to work and that every `Runner` class matching `recipes_dir.rglob("*/runner*.py")` will be imported. + +```python +from wordpress.tests_wordpress.runner_wordpress import RunnerWordpress +``` + +### output_dir [required | string] + +Path to the directory where all test outputs are stored (test report, tracebacks, playwright traces etc.) + +``` +abratest --output_dir /path/to/output +``` + +# Functionality + +Abratest has 3 required inputs, but most importantly the test configuration is done through the .env files given with the --env_paths argument. So let's say we want to run abratest with these 3 .env files: + +- `config1.env` [of TYPE authentik] + +- config2.env [of TYPE wordpress] + +- config3.env [of TYPE wordpress] + +Now we run + +```bash +abratest --env_paths path/config1.env;path/config2.env;path/config3.env [...other args] +``` + + +``` +abratest -> create Coordinator() instance +└── Coordinator() -> create Runner() subclass instances + ├── RunnerAuthentik() [based on config1.env, loaded + │ │ from abra/recipes/authentik] + │ │ # RunnerAuthentik with 3 test files: + │ ├── RUN pytest path/setup_authentik.py + │ ├── RUN pytest path/test_authentik_1.py + │ └── RUN pytest path/test_authentik_2.py + ├── RunnerWordpress() [based on config2.env, loaded + │ │ from abra/recipes/wordpress] + │ │ # RunnerWordpress with 1 test file + │ ├── RUN pytest path/setup_authentik.py + │ ├── RUN pytest path/test_authentik_1.py + │ └── RUN pytest path/test_authentik_2.py + └── RunnerWordpress() [based on config3.env, loaded + │ from abra/recipes/wordpress] + │ # RunnerWordpress with 1 test file + ├── RUN pytest path/setup_authentik.py + ├── RUN pytest path/test_authentik_1.py + └── RUN pytest path/test_authentik_2.py + + +``` + +Coordinator will take care of the correct order of the tests. In general, tests are placed in one of 3 categories: `setups`, `tests` and `cleanups`. To associate a test with one of these categories, place the Test in the corresponding list of the Runner class, i.e. Runner.setups = [test] or Runner.tests = [test]. The execution order will be. + +> [setups] ➔ [tests] ➔ [cleanups] + + +Furthermore, some `Runner` classes can depend on others. For example, `RunnerWordpress` depends on `RunnerAuthentik`. Therefore, `Coordinator` will make sure that `RunnerAuthentik` runs before `RunnerWordpress`. We will end up with with this order: + +| # | Runner | Type | +| --- | -------------- | -------- | +| 1. | Authentik | setups | +| 2. | Wordpress-1 | setups | +| 3. | Wordpress-2 | setups | +| 4. | Authentik | tests | +| 5. | Wordpress-1 | tests | +| 6. | Wordpress-2 | tests | +| 7. | Authentik | cleanups | +| 8. | Wordpress-1 | cleanups | +| 9. | Wordpress-2 | cleanups | + + +# Create a test suite for a recipe + +todo + +To understand how a test suite is built, let's have a look at the files + +runner_authentik.py -> required, defines the Runner subclass (see below) +conftest.py -> not required. special file for pytest. is automatically discovered and loaded. convenient place to define fixtures and functions to be used in more than one test routine +setup_authentik.py -> not required. can hold setup routine for authentik, has to be registered in runner_authentik.py + +# Create a custom Runner + +To comprehend the process of creating a new subclass of `Runner`, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. + + +```python +from pytest_abra import Runner, Test + +class RunnerWordpress(Runner): + env_type = "wordpress" + dependencies = ["authentik"] + setups = [ + Test(test_file="setup_wordpress_1.py"), + Test(test_file="setup_wordpress_2.py"), + ] + tests = [ + Test(test_file="test_wordpress.py"), + Test(condition=condition_function, test_file="test_wordpress_conditional.py"), + ] + cleanups = [] +``` + +The signature of condition functions can be seen below. The function takes one `NamedTuple` and returns of type `bool`. You can learn about the contents of the input by looking up the class `ConditionArgs`. Generally speaking, it provides access to all of the .env files, especially the one related to the current Runner. + +```python +def condition_function(args: ConditionArgs) -> bool: + ... +``` + +## Discovery of `Runners` and `Tests` + +- Runners will be discovered, if they are defined in a moduled of name `runner_*.py` including a class of name `Runner*`. + +- Tests will be discovered by filename as long as they are placed in the parent dir of `runner_*.py` or in any subdirectory. + +``` +DIR parent_dir +├── FILE runner_*.py +├── FILE test1.py +└── DIR subdir + ├── DIR subsubdir + │ └── test2.py + └── test3.py +``` + +# Create custom Tests + +The test files are written in the same way as any other pytest test file. The only difference is that pytest-abra provides custom fixtures that make it easy to get the configuration by the provided .env files and to deal with URLS etc. + + +### Step 1) Add new Test + +Create a new testfile `new_test.py` in the same directory or a subdirectory of `runner_wordpress.py`. +Register `new_test.py` as a test in the `RunnerWordpress` class. +Set prevent_skip=True, so that you can run your new test over and over again for debugging, without it being skipped + +```python +# runner_wordpress.py +from pytest_abra import Runner, Test + +class RunnerWordpress(Runner): + env_type = "wordpress" + tests = [ + Test(test_file="working_test.py"), + Test(test_file="new_test.py", prevent_skip=True), + ] +``` + +```python +# new_test.py + +def test_new(): + ... +``` + +### Step 2) Call abratest + +Call abratest with `--debug` to enable playwright debug mode and either `--session_id` or `--resume`. + +```bash +abratest [required-options] --debug --session_id debug_session +``` + +This could be done by modifying `main.py`. The first time you run abratest, all tests will be executed as usual. The second time, all tests will be skipped as they have passed already. Only your new test will be run again and again, as the prevent_skip option is enabled. So you can run all tests once and then skip all tests besides your new test you want to debug. + +# todo: add example + +# Playwright Debug & Codegen + +Use playwright debug mode or codegen to create testing code easily by recording browser actions https://playwright.dev/python/docs/codegen + +```bash +abratest --debug # launch your tests in debug mode +playwright codegen demo.playwright.dev/todomvc # visit given url in codegen mode +``` + +## Development + +```bash +pytest # test pytest-abra +pytest -m "not slow" # test pytest-abra without slow tests +pytest --collect-only # debug test pytest-abra +docker compose run --rm app pytest # run pytest-abra +``` diff --git a/main.py b/main.py index aa809bc..9d3ac37 100644 --- a/main.py +++ b/main.py @@ -1,16 +1,12 @@ -import json -import os import subprocess from pathlib import Path +from pytest_abra.utils import load_json_to_environ + # --------------------- load credentials to env variables -------------------- # cred_file = Path("credentials.json") -with open(cred_file, "r") as f: - CREDENTIALS = json.load(f) - -for key, value in CREDENTIALS.items(): - os.environ[key] = value +load_json_to_environ(cred_file) # --------------------------------- env files -------------------------------- # @@ -18,7 +14,7 @@ for key, value in CREDENTIALS.items(): # triggers the execution of one test Runner and provides configuration to the # tests inside the runner. -ENV_FILES_ROOT = Path("../envfiles").resolve() +ENV_FILES_ROOT = Path("./envfiles").resolve() ENV_FILES = [ ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env", # authentik ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env", # wordpress @@ -28,7 +24,7 @@ ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES]) # ----------------------------------- dirs ----------------------------------- # -RECIPES_DIR = Path("../recipes").resolve() +RECIPES_DIR = Path("./recipes").resolve() OUTPUT_DIR = Path("./test-output").resolve() # ------------------------------------ run ----------------------------------- # @@ -44,5 +40,7 @@ subprocess.run( OUTPUT_DIR, "--resume", # "--debug", + # "--session_id", + # "abc", ] ) diff --git a/prototyping/documentation.md b/prototyping/documentation.md deleted file mode 100644 index e27c359..0000000 --- a/prototyping/documentation.md +++ /dev/null @@ -1,104 +0,0 @@ -# RUN - - - - - - - - -Abratest has 3 required inputs, but most importantly the test configuration is done through the .env files given with the --env_paths argument. So let's say we want to run abratest with these 3 .env files: - -- config1.env [of TYPE authentik] - -- config2.env [of TYPE wordpress] - -- config3.env [of TYPE wordpress] - -Now we run - -```bash -abratest --env_paths path/config1.env;path/config2.env;path/config3.env [...other args] -``` - - -``` -abratest -> create Coordinator() instance -└── Coordinator() -> create Runner() subclass instances - ├── RunnerAuthentik() [based on config1.env, loaded - │ │ from abra/recipes/authentik] - │ │ # RunnerAuthentik with 3 test files: - │ ├── RUN pytest path/setup_authentik.py - │ ├── RUN pytest path/test_authentik_1.py - │ └── RUN pytest path/test_authentik_2.py - ├── RunnerWordpress() [based on config2.env, loaded - │ │ from abra/recipes/wordpress] - │ │ # RunnerWordpress with 1 test file - │ ├── RUN pytest path/setup_authentik.py - │ ├── RUN pytest path/test_authentik_1.py - │ └── RUN pytest path/test_authentik_2.py - └── RunnerWordpress() [based on config3.env, loaded - │ from abra/recipes/wordpress] - │ # RunnerWordpress with 1 test file - ├── RUN pytest path/setup_authentik.py - ├── RUN pytest path/test_authentik_1.py - └── RUN pytest path/test_authentik_2.py - - -``` - -Coordinator will take care of the correct order of the tests. In general, tests are placed in one of 3 categories: `setups`, `tests` and `cleanups`. To associate a test with one of these categories, place the Test in the corresponding list of the Runner class, i.e. Runner.setups = [test] or Runner.tests = [test]. The execution order will be. - -> [setups] ➔ [tests] ➔ [cleanups] - - -Furthermore, some `Runner` classes can depend on others. For example, `RunnerWordpress` depends on `RunnerAuthentik`. Therefore, `Coordinator` will make sure that `RunnerAuthentik` runs before `RunnerWordpress`. We will end up with with this order: - -| # | Runner | Type | -| --- | -------------- | -------- | -| 1. | Authentik | setups | -| 2. | Wordpress-1 | setups | -| 3. | Wordpress-2 | setups | -| 4. | Authentik | tests | -| 5. | Wordpress-1 | tests | -| 6. | Wordpress-2 | tests | -| 7. | Authentik | cleanups | -| 8. | Wordpress-1 | cleanups | -| 9. | Wordpress-2 | cleanups | - - -# Create a custom Runner - -To comprehend the process of creating a new subclass of `Runner`, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. - - -```python -from pytest_abra import Runner, Test - -class RunnerWordpress(Runner): - env_type = "wordpress" - dependencies = ["authentik"] - setups = [ - Test(test_file="setup_wordpress_1.py"), - Test(test_file="setup_wordpress_2.py"), - ] - tests = [ - Test(test_file="test_wordpress.py"), - Test(condition=condition_function, test_file="test_wordpress_conditional.py"), - ] - cleanups = [] -``` - -The signature of condition functions can be seen below. The function takes one `NamedTuple` and returns of type `bool`. You can learn about the contents of the input by looking up the class `ConditionArgs`. Generally speaking, it provides access to all of the .env files, especially the one related to the current Runner. - -```python -def condition_function(args: ConditionArgs) -> bool: - ... -``` - - -# Create custom Tests - -The test files are written in the same way as any other pytest test file. The only difference is that pytest-abra provides custom fixtures that make it easy to get the configuration by the provided .env files and to deal with URLS etc. - -# todo: add example \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d436714..8625d38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,9 +21,15 @@ dependencies = [ "loguru == 0.7.2", "beautifulsoup4 == 4.12.2", "imbox == 0.9.8", + "tabulate == 0.9.0", "hatchling == 1.18.0", - "icecream", - "tabulate", + "icecream == 2.1.3", +] + +[project.optional-dependencies] +dev = [ + "mypy", + "ruff", ] [project.entry-points.pytest11] diff --git a/pytest_abra/__init__.py b/pytest_abra/__init__.py index 0b134ec..5ec5233 100644 --- a/pytest_abra/__init__.py +++ b/pytest_abra/__init__.py @@ -1,6 +1,6 @@ from pytest_abra.coordinator import Coordinator from pytest_abra.dir_manager import DirManager -from pytest_abra.env_manager import EnvFile +from pytest_abra.env_manager import EnvFile, EnvManager from pytest_abra.runner import ConditionArgs, Runner, Test from pytest_abra.utils import BaseUrl @@ -12,4 +12,5 @@ __all__ = [ "DirManager", "BaseUrl", "EnvFile", + "EnvManager", ] diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 1ebf852..ca39cf2 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -2,21 +2,29 @@ import argparse import os from pathlib import Path +import pkg_resources # type: ignore from loguru import logger from pytest_abra import Coordinator from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import get_datetime_string +from pytest_abra.utils import get_session_id + + +def get_version(): + return pkg_resources.get_distribution("pytest_abra").version def run(): parser = argparse.ArgumentParser() + parser.add_argument("--version", "-V", action="version", version=get_version(), help="output the version number") parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;", required=True) - parser.add_argument("--recipes_dir", type=Path, help="List of loaded env files separated with ;", required=True) - parser.add_argument("--output_dir", type=Path, help="List of loaded env files separated with ;", required=True) + parser.add_argument("--recipes_dir", type=Path, help="Dir of abra recipes and respective runners", required=True) + parser.add_argument("--output_dir", type=Path, help="Dir of test outputs", required=True) parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=20_000) parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode") parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests") + parser.add_argument("--session_id", help="Session dir name (inside output_dir). Overwrites --resume") + args = parser.parse_args() env_paths = [Path(s) for s in args.env_paths.split(";")] @@ -27,17 +35,13 @@ def run(): # ----------------------------- define session_id ---------------------------- # - session_id = "test-" + get_datetime_string() - if args.resume: - latest_session_id = DirManager.get_latest_session_id(args.output_dir) - if latest_session_id: - session_id = DirManager.get_latest_session_id(args.output_dir) + session_id = get_session_id(args.output_dir, args.resume, args.session_id) # ------------------------------- setup logging ------------------------------ # # todo: move to Coordinator DIR = DirManager(output_dir=args.output_dir, session_id=session_id) - log_file = DIR.RECORDS / "coordinator.log" + log_file = DIR.RESULTS / "coordinator.log" logger.add(log_file) # ---------------------------- initialize and run ---------------------------- # @@ -49,7 +53,7 @@ def run(): recipes_dir=args.recipes_dir, timeout=args.timeout, ) - coordinator.setup_test() - coordinator.run_test() + coordinator.prepare_tests() + coordinator.run_tests() coordinator.combine_html() coordinator.collect_traces() diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 52f5e77..6704360 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -1,15 +1,18 @@ import importlib +import json import re import sys from pathlib import Path from loguru import logger +from tabulate import tabulate # type: ignore from pytest_abra.dir_manager import DirManager from pytest_abra.env_manager import EnvFile, EnvManager from pytest_abra.html_helper import merge_html_reports from pytest_abra.runner import Runner -from pytest_abra.utils import rmtree +from pytest_abra.shared_types import TestResult +from pytest_abra.utils import generate_random_string, load_json_to_environ, rmtree class Coordinator: @@ -32,21 +35,24 @@ class Coordinator: self.ENV = EnvManager(env_paths=env_paths, RUNNER_DICT=self.RUNNER_DICT) self.TIMEOUT = timeout - def setup_test(self) -> None: - logger.info("calling setup_test()") + def prepare_tests(self) -> None: + logger.info("calling prepare_tests()") self.DIR.create_all_dirs() - self.ENV.copy_env_files(self.DIR) + self.ENV.copy_env_files(self.ENV.env_files, self.DIR) + self.load_test_credentials(self.DIR) - def run_test(self) -> None: - logger.info("calling run_test()") + def run_tests(self) -> None: + logger.info("calling run_tests()") self.runners: list[Runner] = self._load_runners(self.ENV.env_files) + status_list: list[TestResult] = [] for runner in self.runners: - runner.run_setups() + status_list.extend(runner.run_setups()) for runner in self.runners: - runner.run_tests() + status_list.extend(runner.run_tests()) for runner in self.runners: - runner.run_cleanups() - logger.info("run_test() finished") + status_list.extend(runner.run_cleanups()) + status_table = tabulate([[t.test_name, t.status] for t in status_list], headers=["name", "status"]) + logger.info(f"run_tests() finished\n{status_table}") def _load_runners(self, env_files: list[EnvFile]) -> list[Runner]: """Creates an instance of the correct Runner class for each given env file""" @@ -58,13 +64,13 @@ class Coordinator: def combine_html(self) -> None: """combines all generated pytest html reports into one""" - in_dir_path = str(self.DIR.RECORDS / "html") - out_file_path = str(self.DIR.RECORDS / "full-report.html") + in_dir_path = str(self.DIR.RESULTS / "html") + out_file_path = str(self.DIR.RESULTS / "full-report.html") title = "combined.html" merge_html_reports(in_dir_path, out_file_path, title) def collect_traces(self): - """moves all traces into SESSION/RECORDS dir + """moves all traces into SESSION/RESULTS dir if tests are rerun and generate another trace, the new trace will get a unique name such as tracename-0 @@ -80,14 +86,34 @@ class Coordinator: index += 1 return get_new_path(root_dir, base_name, index=index) - trace_root_dir = self.DIR.RECORDS / "traces" + trace_root_dir = self.DIR.RESULTS / "traces" for f in trace_root_dir.rglob("*/trace.zip"): - new_path = get_new_path(self.DIR.RECORDS, f.parent.name) + new_path = get_new_path(self.DIR.RESULTS, f.parent.name) f.parent.rename(new_path) rmtree(trace_root_dir) @staticmethod - def create_runner_dict(recipes_dir: Path) -> dict[str, type["Runner"]]: + def load_test_credentials(DIR: DirManager): + """Load test user credentials. If not available, create them randomly. + + Test users are created during testing but should be deleted after the test. In case test + users are not deleted after tests by accident, the user credentials are not known to an + attacker.""" + + test_credentials_path = DIR.STATES / "credentials_test.json" + if not test_credentials_path.is_file(): + test_credentials = { + "TEST_USER": "test-" + generate_random_string(6), + "TEST_PASS": generate_random_string(12, punctuation=True), + } + + with open(test_credentials_path, "w") as json_file: + json.dump(test_credentials, json_file) + + load_json_to_environ(test_credentials_path) + + @staticmethod + def create_runner_dict(recipes_dir: Path) -> dict[str, type[Runner]]: """Creates a dictionary holding all the RunnerClasses that can be discovered in recipes_dir example: @@ -101,18 +127,19 @@ class Coordinator: because recipes_dir is added to sys.path. """ - RUNNER_DICT: dict[str, type["Runner"]] = dict() + RUNNER_DICT: dict[str, type[Runner]] = dict() runner_discovery_pattern = re.compile("Runner.+") # make it possible to import modules from recipes_dir sys.path.append(recipes_dir.as_posix()) - for module_path in recipes_dir.rglob("*/runner*.py"): + for module_path in recipes_dir.rglob("*/runner_*.py"): rel_path = module_path.relative_to(recipes_dir).as_posix().replace("/", ".").replace(".py", "") module = importlib.import_module(rel_path) runner_class_names = [name for name in dir(module) if runner_discovery_pattern.match(name)] assert len(runner_class_names) == 1 runner_class_name = runner_class_names[0] RunnerClass: type[Runner] = getattr(module, runner_class_name) + RunnerClass._tests_path = module_path.parent RUNNER_DICT[RunnerClass.env_type] = RunnerClass return RUNNER_DICT diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index 82bc0bd..0f0a5f0 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -3,7 +3,8 @@ import os import re -from datetime import datetime, timedelta + +# from datetime import datetime, timedelta from pathlib import Path from typing import Generator, Protocol, TypedDict @@ -11,7 +12,7 @@ import pytest from dotenv import dotenv_values from icecream import ic # type: ignore from imbox import Imbox # type: ignore -from playwright.sync_api import APIRequestContext, BrowserContext, Playwright, expect +from playwright.sync_api import BrowserContext, expect from pytest import Parser from pytest_abra import BaseUrl, DirManager, EnvFile @@ -49,9 +50,9 @@ def DIR(request) -> DirManager: DIR.OUTPUT DIR.SESSION - DIR.RECORDS DIR.STATES - DIR.RESULTS""" + DIR.RESULTS + DIR.STATUS""" output_dir = request.config.getoption("--output_dir") assert output_dir, "pytest argument --output_dir not set" @@ -93,13 +94,13 @@ def URL(env_config: dict[str, str]) -> BaseUrl: @pytest.fixture(scope="session") -def imap_client() -> None: +def imap_client() -> Generator[Imbox, None, None]: """imap email client using credentials from environment variables""" - assert os.environ["IMAP_HOST"] - assert os.environ["IMAP_PORT"] - assert os.environ["IMAP_USER"] - assert os.environ["IMAP_PASS"] + assert os.environ["IMAP_HOST"], "required environment variable is undefined" + assert os.environ["IMAP_PORT"], "required environment variable is undefined" + assert os.environ["IMAP_USER"], "required environment variable is undefined" + assert os.environ["IMAP_PASS"], "required environment variable is undefined" imbox = Imbox( hostname=os.environ["IMAP_HOST"], @@ -138,9 +139,8 @@ def imap_recent_messages(imap_client: Imbox) -> list[Message]: for uid, message in messages: print(uid, message.subject, message.date)""" - N_MINUTES = 30 - - n_minutes_ago = datetime.now() - timedelta(minutes=N_MINUTES) + # N_MINUTES = 30 + # n_minutes_ago = datetime.now() - timedelta(minutes=N_MINUTES) uids: list[bytes] = [] messages: list[Message] = [] # for uid, message in imap_client.messages(date__gt=n_minutes_ago): @@ -150,14 +150,3 @@ def imap_recent_messages(imap_client: Imbox) -> list[Message]: messages.append(message) return messages - - -@pytest.fixture(scope="session") -def api_request_context( - playwright: Playwright, - DIR: DirManager, -) -> Generator[APIRequestContext, None, None]: - state_file = DIR.STATES / "authentik_admin_state.json" - request_context = playwright.request.new_context(storage_state=state_file) - yield request_context - request_context.dispose() diff --git a/pytest_abra/dir_manager.py b/pytest_abra/dir_manager.py index 1ff4c11..b5ca29e 100644 --- a/pytest_abra/dir_manager.py +++ b/pytest_abra/dir_manager.py @@ -11,11 +11,11 @@ class DirManager: The structures is as follows: tests dir/ session_id-1/ - records results states + status session_id-2/ - records + results ... """ @@ -32,11 +32,11 @@ class DirManager: dirs: list[Path] = [ self.OUTPUT_DIR, self.SESSION, - self.RECORDS, - self.HTML, self.STATES, self.ENV_FILES, self.RESULTS, + self.HTML, + self.STATUS, ] for d in dirs: d.mkdir(exist_ok=True) @@ -49,14 +49,6 @@ class DirManager: def SESSION(self): return self.OUTPUT_DIR / self.session_id - @property - def RECORDS(self): - return self.SESSION / "records" - - @property - def HTML(self): - return self.RECORDS / "html" - @property def STATES(self): return self.SESSION / "states" @@ -69,6 +61,14 @@ class DirManager: def RESULTS(self): return self.SESSION / "results" + @property + def HTML(self): + return self.RESULTS / "html" + + @property + def STATUS(self): + return self.SESSION / "status" + @property def RECIPES(self): return self.recipes_dir @@ -80,7 +80,13 @@ class DirManager: @staticmethod def get_latest_session_id(output_dir: Path) -> Optional[str]: - """returns the name of the newest dir inside of output_dir""" + """returns the name of the newest dir inside of output_dir + + if output_dir does not exists or is empty, None is returned""" + + if not output_dir.is_dir(): + return None + all_dirs = [d for d in output_dir.iterdir() if d.is_dir()] if all_dirs: newest_dir: Path = max(all_dirs, key=lambda x: x.stat().st_ctime) diff --git a/pytest_abra/env_manager.py b/pytest_abra/env_manager.py index f9d79cd..7076268 100644 --- a/pytest_abra/env_manager.py +++ b/pytest_abra/env_manager.py @@ -4,9 +4,10 @@ from typing import TYPE_CHECKING, NamedTuple from dotenv import dotenv_values +from pytest_abra.utils import files_are_same + if TYPE_CHECKING: - from pytest_abra.dir_manager import DirManager - from pytest_abra.runner import Runner + from pytest_abra import DirManager, Runner class EnvFile(NamedTuple): @@ -45,6 +46,7 @@ class EnvManager: def _get_dependency_rules(env_files: list[EnvFile], RUNNER_DICT: dict[str, type["Runner"]]) -> list[DependencyRule]: dependency_rules: list[DependencyRule] = [] for env_file in env_files: + assert env_file.env_type in RUNNER_DICT, f"no runner for env_type={env_file.env_type} found in RUNNER_DICT" child_runner_class = RUNNER_DICT[env_file.env_type] for dependency in child_runner_class.dependencies: dependency_rule = DependencyRule(child=child_runner_class.env_type, dependency=dependency) @@ -93,11 +95,25 @@ class EnvManager: "Could not resolve test order. This is possibly due to a circular dependency (a on b, b on c, c on a)" ) - def copy_env_files(self, DIR: "DirManager") -> None: - """Copies all env files to STATES/env_files. Files will be renamed to - -- - 00-authentik-login.test.dev.local-it.cloud.env""" + @staticmethod + def copy_env_files(env_files: list[EnvFile], DIR: "DirManager") -> None: + """Copies all env files to STATES/env_files. - for index, env_file in enumerate(self.env_files): + Files will be renamed to --. Example: + 00-authentik-login.test.dev.local-it.cloud.env + + Does nothing when called twice with same env_files. Throws an AssertionError if either + contents or filenames of env_files have changed (probably test rerun with different input)""" + + dir_was_not_empty = len(list(DIR.ENV_FILES.iterdir())) > 0 + + for index, env_file in enumerate(env_files): file_name = "-".join([str(index).zfill(2), env_file.env_type, env_file.env_path.name]) + if dir_was_not_empty: + # check that the copied env files have not changed + present_files = [f.name for f in DIR.ENV_FILES.iterdir()] + assert ( + file_name in present_files and files_are_same(env_file.env_path, DIR.ENV_FILES / file_name) + ), "It appears that you are resuming a test while the input env files have changed. Start a new test instead" + shutil.copy(env_file.env_path, DIR.ENV_FILES / file_name) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 3d75562..c9eb650 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Callable, NamedTuple import pytest from loguru import logger +from pytest_abra.shared_types import STATUS, TestResult + if TYPE_CHECKING: - from pytest_abra.coordinator import Coordinator - from pytest_abra.env_manager import EnvFile + from pytest_abra import Coordinator, DirManager, EnvFile class ConditionArgs(NamedTuple): @@ -30,6 +31,7 @@ class Runner: tests: list[Test] = [] cleanups: list[Test] = [] dependencies: list[str] = [] + _tests_path: Path = Path() def __init__(self, coordinator: "Coordinator", runner_index: int): self.coordinator = coordinator @@ -41,62 +43,58 @@ class Runner: logger.info(f"creating instance of {self.__class__.__name__}") - def run_setups(self): + def run_setups(self) -> list[TestResult]: """runs the setup scripts if available""" - self._execute_test_list(self.setups) + return self._execute_tests_list(self.setups) - def run_tests(self): + def run_tests(self) -> list[TestResult]: """runs the test scripts if available""" - self._execute_test_list(self.tests) + return self._execute_tests_list(self.tests) - def run_cleanups(self): + def run_cleanups(self) -> list[TestResult]: """runs the cleanup scripts if available""" - self._execute_test_list(self.cleanups) + return self._execute_tests_list(self.cleanups) - def _execute_test_list(self, test_list: list[Test]): - """runs the main test script and if available and sub test scripts if their running condition is met""" + def _execute_tests_list(self, test_list: list[Test]) -> list[TestResult]: + """Runs all tests given in the list. If condition is defined, it is also checked.""" # check if required dependencies have passed if not self._dependencies_passed(): logger.warning(f"skipping run_tests() of {self.env_type} (one or more dependencies have not passed)") - return + return [TestResult("skipped_dep", test.test_file) for test in test_list] - for test in test_list: - self._run_test_with_checks(test) - - def _run_test_with_checks(self, test: Test): - # dependency passed: true / false - # already_passed: true / false - # prevent_skip: true / false - # condition_available: true / pass - # condition_met: true / false + return [self._run_test_with_checks(test) for test in test_list] + def _run_test_with_checks(self, test: Test) -> TestResult: identifier_string = self.combine_names(self.env_type, test.test_file) - results = list(self.DIR.RECIPES.rglob(test.test_file)) - assert len(results) == 1, f"{test.test_file} should exist exactly 1 time, but found {len(results)} times" - full_test_path = results[0] + test_files = list(self._tests_path.rglob(test.test_file)) + assert len(test_files) == 1, f"{test.test_file} should exist exactly once, but found {len(test_files)} times" + full_test_path = test_files[0] # check if test aleady passed - if self._is_test_passed(identifier_string, remove_existing=True): + if self._is_test_passed(self.DIR, identifier_string): if test.prevent_skip: logger.info(f"continuing {identifier_string} (passed before but prevent_skip=True)") else: logger.info(f"skipping {identifier_string} (test has passed)") - return + return TestResult("skipped_pas", test.test_file) if test.condition: - condition_result = self._run_condition(test.condition) + condition_result = self._call_condition_function(test.condition) if not condition_result: # test condition is defined but not met logger.info(f"skipping {identifier_string} (test condition is not met)") - return + self._create_status_file(self.DIR, status="skipped_con", identifier_string=identifier_string) + return TestResult("skipped_con", test.test_file) # test condition is undefined or not met logger.info(f"running {identifier_string}") - result = self._call_pytest(full_test_path) - self._create_result_file(result=result, identifier_string=identifier_string) + exit_code = self._call_pytest(full_test_path) + status = self.exit_code_to_str(exit_code) + self._create_status_file(self.DIR, status=status, identifier_string=identifier_string) + return TestResult(status, test.test_file) - def _run_condition(self, condition_function: Callable[[ConditionArgs], bool]): + def _call_condition_function(self, condition_function: Callable[[ConditionArgs], bool]): """run the test condition function with multiple arguments""" # more arguments can be added later without changing the function signature conditon_args = ConditionArgs( @@ -106,24 +104,40 @@ class Runner: ) return condition_function(conditon_args) - def _is_test_passed(self, identifier_string: str, remove_existing: bool = False) -> bool: - """returns True if the selected test matching identifier_string already passed + @classmethod + def _create_status_file( + cls, + DIR: "DirManager", + status: STATUS, + identifier_string: str, + ): + """create result file to indicated passed/failed/skipped test""" - This is determined by the presence of a specific output file in the RESULTS folder that - matches identifier_string + # remove matching files + for status_file in cls._get_status_files(DIR, identifier_string): + status_file.unlink() - remove_existing: If True, result files matching identifier_string with a status - other than 'passed' will be deleted""" + full_name = cls.combine_names(status, identifier_string) + file_path = DIR.STATUS / full_name + with open(file_path, "w") as _: + pass # create empty file - already_passed = False - for result in self.DIR.RESULTS.glob("*"): - if identifier_string in result.name: - # process any result file (passed / failed / skipped) if it exists - if "passed" in result.name: - already_passed = True - elif remove_existing: - result.unlink() - return already_passed + @staticmethod + def _get_status_files(DIR: "DirManager", identifier_string: str) -> list[Path]: + return [f for f in DIR.STATUS.glob("*") if identifier_string in f.name] + + @classmethod + def _is_test_passed(cls, DIR: "DirManager", identifier_string: str) -> bool: + """returns True if the selected test matching identifier_string already passed""" + + matching_files = cls._get_status_files(DIR, identifier_string) + if len(matching_files) == 1: + status_file = matching_files[0] + if "passed" in status_file.name: + return True + elif len(matching_files) > 1: + logger.warning("more than one matching status file found") + return False def _call_pytest(self, full_test_path: Path) -> int: """runs pytest programmatically with a specific file @@ -155,7 +169,7 @@ class Runner: # --output only works with the given context and page fixture # folder needs to be unique! traces will not appear, if every pytest run has same output dir command_arguments.append("--output") - command_arguments.append(str(self.DIR.RECORDS / "traces" / full_test_path.stem)) + command_arguments.append(str(self.DIR.RESULTS / "traces" / full_test_path.stem)) # tracing command_arguments.append("--tracing") # "on", "off", "retain-on-failure" @@ -170,28 +184,16 @@ class Runner: # command_arguments.append("--headed") # html report. Will be combined into one file later. - command_arguments.append(f"--html={self.DIR.RECORDS / 'html' / full_test_path.with_suffix('.html').name}") + command_arguments.append(f"--html={self.DIR.RESULTS / 'html' / full_test_path.with_suffix('.html').name}") return pytest.main(command_arguments) - def _create_result_file( - self, - result: int, - identifier_string: str, - ): - """create result file to indicated passed/failed or skipped test""" - - full_name = self.combine_names(self.result_int_to_str(result), identifier_string) - file_path = self.DIR.RESULTS / full_name - with open(file_path, "w") as _: - pass # create empty file - def _dependencies_passed(self): """returns true if all setups of each dependency have passed""" # todo: what about conditional setups? - passed_tests = [r.name for r in self.DIR.RESULTS.glob("*") if "passed" in r.name] + passed_tests = [r.name for r in self.DIR.STATUS.glob("*") if "passed" in r.name] results = [] for dependency in self.dependencies: dependency_runner = self.coordinator.RUNNER_DICT[dependency] @@ -201,11 +203,9 @@ class Runner: return all(results) @staticmethod - def result_int_to_str(result_int: int) -> str: + def exit_code_to_str(result_int: int) -> STATUS: """converts the pytest exit code (int) into a meaningful string""" match result_int: - case -1: - return "skipped" case 0: return "passed" case _: diff --git a/pytest_abra/shared_types.py b/pytest_abra/shared_types.py new file mode 100644 index 0000000..ff0e147 --- /dev/null +++ b/pytest_abra/shared_types.py @@ -0,0 +1,16 @@ +from typing import Literal, NamedTuple + +""" +passed: test passed +failed: test failed +skipped_con: test skipped because condition was not met +skipped_dep: test skipped because dependencies did not finish +skipped_pas: test skipped because it passed before +""" + +STATUS = Literal["passed", "failed", "skipped_con", "skipped_dep", "skipped_pas"] + + +class TestResult(NamedTuple): + status: STATUS + test_name: str diff --git a/pytest_abra/utils.py b/pytest_abra/utils.py index 828e6b9..0dfdb5d 100644 --- a/pytest_abra/utils.py +++ b/pytest_abra/utils.py @@ -1,8 +1,17 @@ +import json +import os +import random +import string from dataclasses import dataclass from datetime import datetime from pathlib import Path +from typing import Optional from urllib.parse import urlunparse +from loguru import logger + +from pytest_abra.dir_manager import DirManager + @dataclass class BaseUrl: @@ -24,7 +33,7 @@ def get_datetime_string() -> str: return current_datetime.strftime("%Y-%m-%d-%H-%M-%S") -def rmtree(root_dir: Path): +def rmtree(root_dir: Path) -> None: """removes a folder with content recursively""" if not root_dir.is_dir(): return @@ -35,3 +44,43 @@ def rmtree(root_dir: Path): child.unlink() root_dir.rmdir() + + +def generate_random_string(length: int, punctuation=False) -> str: + """returns a random string of the given length""" + characters = string.ascii_letters + string.digits + if punctuation: + characters += string.punctuation + random_string = "".join(random.choice(characters) for _ in range(length)) + return random_string + + +def load_json_to_environ(cred_file: Path) -> None: + """Load the contents of a json file directly into os.environ. Variable names are inherited""" + + if not cred_file.is_file(): + logger.warning(f"{cred_file} could not be found, no credentials loaded") + return + + with open(cred_file, "r") as f: + CREDENTIALS = json.load(f) + + for key, value in CREDENTIALS.items(): + os.environ[key] = value + + +def get_session_id(args_output_dir: Path, args_resume: bool, args_session_id: Optional[str]) -> str: + """converts the cli arguments to the correct session_id""" + session_id = args_session_id + if not session_id: + session_id = "test-" + get_datetime_string() + if args_resume: + latest_session_id = DirManager.get_latest_session_id(args_output_dir) + if latest_session_id: + session_id = latest_session_id + return session_id + + +def files_are_same(file1: Path, file2: Path) -> bool: + with open(file1, "r") as f1, open(file2, "r") as f2: + return f1.read() == f2.read() diff --git a/recipes/authentik/tests_authentik/cleanup_authentik.py b/recipes/authentik/tests_authentik/cleanup_authentik.py new file mode 100644 index 0000000..b556ea4 --- /dev/null +++ b/recipes/authentik/tests_authentik/cleanup_authentik.py @@ -0,0 +1,40 @@ +import json +import os +import re + +from playwright.sync_api import BrowserContext + +from pytest_abra import BaseUrl, DirManager + +ADMIN_USER = os.environ["ADMIN_USER"] +ADMIN_PASS = os.environ["ADMIN_PASS"] +TEST_USER = os.environ["TEST_USER"] +TEST_PASS = os.environ["TEST_PASS"] + + +def remove_user(admin_context: BrowserContext, URL: BaseUrl): + """removes TEST_USER account from authentik""" + page = admin_context.new_page() + page.goto(URL.get()) + page.get_by_role("link", name="Admin Interface").click() + nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) + nav.click() + nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() + + name_pattern = re.compile(TEST_USER) + page.get_by_role("row", name=name_pattern).get_by_label("").check() + page.get_by_role("button", name=re.compile(r"Löschen|Delete")).click() + page.get_by_role("dialog").get_by_role("button", name=re.compile(r"Löschen|Delete")).click() + + +def cleanup_delete_user( + context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists +): + # load admin cookies to context + state_file = DIR.STATES / "authentik_admin_state.json" + storage_state = json.loads(state_file.read_bytes()) + context.add_cookies(storage_state["cookies"]) + + if check_if_user_exists(context, env_config, URL): + remove_user(context, URL) + assert not check_if_user_exists(context, env_config, URL) diff --git a/recipes/authentik/tests_authentik/conftest.py b/recipes/authentik/tests_authentik/conftest.py new file mode 100644 index 0000000..df919d9 --- /dev/null +++ b/recipes/authentik/tests_authentik/conftest.py @@ -0,0 +1,46 @@ +import os +import re +from typing import Callable, Generator + +import pytest +from playwright.sync_api import APIRequestContext, BrowserContext, Playwright, TimeoutError + +from pytest_abra import BaseUrl, DirManager + + +@pytest.fixture(scope="session") +def api_request_context( + playwright: Playwright, + DIR: DirManager, +) -> Generator[APIRequestContext, None, None]: + state_file = DIR.STATES / "authentik_admin_state.json" + request_context = playwright.request.new_context(storage_state=state_file) + yield request_context + request_context.dispose() + + +@pytest.fixture +def check_if_user_exists() -> Callable[[BrowserContext, dict[str, str], BaseUrl], bool]: + """This is actually a normal function supplied by a fixture. We do this, because imports from + tests_authentik are difficult as it is not part of the python environment. We expect + from X import function + to fail here. However, pytest handles the loading of fixtures from conftest.py automatically, + hence we use that to load functions too.""" + + def inner_check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl) -> bool: + # go to admin page + page = admin_context.new_page() + page.goto(URL.get()) + page.get_by_role("link", name="Admin Interface").click() + nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) + nav.click() + nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() + + user = page.get_by_text(os.environ["TEST_USER"]) + try: + user.wait_for(state="visible", timeout=5_000) + return True + except TimeoutError: + return False + + return inner_check_if_user_exists diff --git a/recipes/authentik/tests_authentik/fixtures_authentik.py b/recipes/authentik/tests_authentik/fixtures_authentik.py index fa50e3f..fdba975 100644 --- a/recipes/authentik/tests_authentik/fixtures_authentik.py +++ b/recipes/authentik/tests_authentik/fixtures_authentik.py @@ -3,13 +3,13 @@ import json import pytest from playwright.sync_api import BrowserContext, Page -from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager @pytest.fixture def authentik_admin_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "authentik_admin_state.json" + assert state_file.is_file(), "authentik setup did not finish successfully" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) return context @@ -27,6 +27,7 @@ def authentik_admin_page(authentik_admin_context: BrowserContext, DIR: DirManage @pytest.fixture def authentik_user_context(context: BrowserContext, DIR: DirManager) -> BrowserContext: state_file = DIR.STATES / "authentik_user_state.json" + assert state_file.is_file(), "authentik setup did not finish successfully" storage_state = json.loads(state_file.read_bytes()) context.add_cookies(storage_state["cookies"]) return context diff --git a/recipes/authentik/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py index 2543dc6..4e9b81a 100644 --- a/recipes/authentik/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -5,3 +5,4 @@ class RunnerAuthentik(Runner): env_type = "authentik" setups = [Test(test_file="setup_authentik.py")] tests = [Test(test_file="test_authentik_blueprint_api.py")] + cleanups = [Test(test_file="cleanup_authentik.py")] diff --git a/recipes/authentik/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py index 9aa6e3b..b66b41d 100644 --- a/recipes/authentik/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -4,21 +4,18 @@ import re from playwright.sync_api import BrowserContext, expect -from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager ADMIN_USER = os.environ["ADMIN_USER"] ADMIN_PASS = os.environ["ADMIN_PASS"] +TEST_USER = os.environ["TEST_USER"] +TEST_PASS = os.environ["TEST_PASS"] -TESTUSER = {"username": "testuser", "name": "Test User", "password": "test123", "email": "test@example.com"} - - -def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager): +def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl): # go to page page = context.new_page() - url = "https://" + env_config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) # check welcome message welcome_message = env_config.get("welcome_message") @@ -35,20 +32,6 @@ def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: context.storage_state(path=DIR.STATES / "authentik_admin_state.json") -def check_if_user_exists(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): - # go to admin page - page = admin_context.new_page() - page.goto(URL.get()) - page.get_by_role("link", name="Admin Interface").click() - nav = page.locator("ak-sidebar-item", has_text=re.compile(r"Directory|Verzeichnis")) - nav.click() - nav.get_by_role("link", name=re.compile(r"Users|Benutzer")).click() - - user = page.get_by_text(TESTUSER["username"]) - user.wait_for(state="visible") - return user.is_visible() - - def create_invite_link(admin_context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): # go to admin page page = admin_context.new_page() @@ -85,20 +68,23 @@ def create_user(user_context: BrowserContext, invitelink): page = user_context.new_page() page.goto(invitelink) page.get_by_placeholder("Benutzername").click() - page.get_by_placeholder("Benutzername").fill(TESTUSER["username"]) + page.get_by_placeholder("Benutzername").fill(TEST_USER) page.locator('input[name="name"]').click() - page.locator('input[name="name"]').fill(TESTUSER["name"]) + page.locator('input[name="name"]').fill("name") page.locator('input[name="email"]').click() - page.locator('input[name="email"]').fill(TESTUSER["email"]) + email = os.environ["IMAP_EMAIL"] if "IMAP_EMAIL" in os.environ else "test@domain.com" + page.locator('input[name="email"]').fill(email) page.get_by_placeholder("Passwort", exact=True).click() - page.get_by_placeholder("Passwort", exact=True).fill(TESTUSER["password"]) + page.get_by_placeholder("Passwort", exact=True).fill(TEST_PASS) page.get_by_placeholder("Passwort (wiederholen)").click() - page.get_by_placeholder("Passwort (wiederholen)").fill(TESTUSER["password"]) + page.get_by_placeholder("Passwort (wiederholen)").fill(TEST_PASS) page.get_by_role("button", name="Weiter").click() expect(page.locator("ak-library")).to_be_visible() -def setup_user_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl): +def setup_user_state( + context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists +): # load admin cookies to context state_file = DIR.STATES / "authentik_admin_state.json" storage_state = json.loads(state_file.read_bytes()) diff --git a/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py index 87eb452..cf5914b 100644 --- a/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py +++ b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py @@ -17,7 +17,7 @@ def test_authentik_blueprint_status( blueprints = api_request_context.get(URL.get("api/v3/managed/blueprints")) assert blueprints.ok blueprints_data = blueprints.json() - ic(blueprints_data) + # ic(blueprints_data) # fake failed blueprint # blueprints_data["results"][10]["status"] = "failed" diff --git a/recipes/nextcloud/tests_nextcloud/conftest.py b/recipes/nextcloud/tests_nextcloud/conftest.py index ce56379..4151c7a 100644 --- a/recipes/nextcloud/tests_nextcloud/conftest.py +++ b/recipes/nextcloud/tests_nextcloud/conftest.py @@ -4,8 +4,7 @@ import os import pytest from playwright.sync_api import BrowserContext, Page -from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager pytest_plugins = "authentik.tests_authentik.fixtures_authentik" diff --git a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py index 64c8e94..3f18b2c 100644 --- a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py @@ -2,8 +2,7 @@ import re from playwright.sync_api import Page, expect -from pytest_abra.dir_manager import DirManager -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager # url dashboard # https://files.test.dev.local-it.cloud/apps/dashboard/ diff --git a/recipes/wordpress/tests_wordpress/runner_wordpress.py b/recipes/wordpress/tests_wordpress/runner_wordpress.py index d58bd74..aef1273 100644 --- a/recipes/wordpress/tests_wordpress/runner_wordpress.py +++ b/recipes/wordpress/tests_wordpress/runner_wordpress.py @@ -1,11 +1,12 @@ from pytest_abra import ConditionArgs, Runner, Test -def condition_has_locale(args: ConditionArgs) -> bool: +def env_config_has_locale(args: ConditionArgs) -> bool: env_config = args.env_config - if "de" in env_config.get("LOCALE", ""): + if "LOCALE" in env_config: return True - return False + else: + return False class RunnerWordpress(Runner): @@ -16,6 +17,6 @@ class RunnerWordpress(Runner): Test(test_file="setup_wordpress_trigger_email.py"), ] tests = [ - Test(test_file="test_wordpress_receive_email.py", prevent_skip=True), - # Test(condition=condition_has_locale, test_file="test_wordpress_localization.py"), + # Test(test_file="test_wordpress_receive_email.py", prevent_skip=True), + Test(condition=env_config_has_locale, test_file="test_wordpress_localization.py"), ] diff --git a/recipes/wordpress/tests_wordpress/setup_wordpress.py b/recipes/wordpress/tests_wordpress/setup_wordpress.py index 4c44799..a65d8fe 100644 --- a/recipes/wordpress/tests_wordpress/setup_wordpress.py +++ b/recipes/wordpress/tests_wordpress/setup_wordpress.py @@ -1,14 +1,13 @@ import pytest from playwright.sync_api import BrowserContext, Page, expect -from pytest_abra.dir_manager import DirManager +from pytest_abra import BaseUrl, DirManager -def test_visit_from_domain(authentik_admin_context: BrowserContext, env_config: dict[str, str]): +def test_visit_from_domain(authentik_admin_context: BrowserContext, URL: BaseUrl): """visit wordpress directly with admin_session, expect not to be logged in""" page = authentik_admin_context.new_page() - url = "https://" + env_config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) with pytest.raises(AssertionError): # look for admin bar expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000) diff --git a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py index 51601cb..2e01d79 100644 --- a/recipes/wordpress/tests_wordpress/test_wordpress_localization.py +++ b/recipes/wordpress/tests_wordpress/test_wordpress_localization.py @@ -2,14 +2,13 @@ from playwright.sync_api import BrowserContext, expect -from pytest_abra.dir_manager import DirManager +from pytest_abra import BaseUrl -def test_welcome_message(context: BrowserContext, env_config: dict[str, str], DIR: DirManager): +def test_de_welcome_message(context: BrowserContext, env_config: dict[str, str], URL: BaseUrl): page = context.new_page() - url = "https://" + env_config["DOMAIN"] - page.goto(url) + page.goto(URL.get()) expect(page.locator(".wp-block-heading")).to_be_visible() - if "locale" in env_config and "de" in env_config["locale"]: + if "de" in env_config.get("locale", ""): expect(page.get_by_role("heading")).to_have_text("Willkommen bei WordPress!") diff --git a/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py b/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py index 736f18d..f806d13 100644 --- a/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py +++ b/recipes/wordpress/tests_wordpress/test_wordpress_receive_email.py @@ -1,8 +1,10 @@ +import pytest from icecream import ic from pytest_abra.custom_fixtures import Message +@pytest.mark.skip def test_demo(imap_recent_messages: list[Message]): for message in imap_recent_messages: print(dir(message)) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..8a94ee6 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,54 @@ +import re +import time +from pathlib import Path + +import pytest + +from pytest_abra import DirManager +from pytest_abra.utils import get_session_id + + +def test_get_session_id_random(tmp_path: Path): + args_output_dir = tmp_path + args_resume = False + args_session_id = None + session_id = get_session_id(args_output_dir, args_resume, args_session_id) + assert re.search(r"\d+-\d+-\d+", session_id) + + +def test_get_session_id_explicit1(tmp_path: Path): + args_output_dir = tmp_path + args_resume = False + args_session_id = "abc" + session_id = get_session_id(args_output_dir, args_resume, args_session_id) + assert session_id == "abc" + + +def test_get_session_id_explicit2(tmp_path: Path): + args_output_dir = tmp_path + args_resume = True + args_session_id = "abc" + session_id = get_session_id(args_output_dir, args_resume, args_session_id) + assert session_id == "abc" + + +@pytest.mark.slow +def test_get_session_id_integration(tmp_path: Path): + assert len(list(tmp_path.iterdir())) == 0 + session_id_1 = get_session_id(args_output_dir=tmp_path, args_resume=False, args_session_id=None) + + DIR = DirManager(output_dir=tmp_path, session_id=session_id_1) + DIR.create_all_dirs() + assert len(list(tmp_path.iterdir())) == 1 + + time.sleep(1.1) # get_session_id won't be unique if called without time passed + session_id_2 = get_session_id(args_output_dir=tmp_path, args_resume=False, args_session_id=None) + DIR = DirManager(output_dir=tmp_path, session_id=session_id_2) + DIR.create_all_dirs() + assert len(list(tmp_path.iterdir())) == 2 + + session_id_3 = get_session_id(args_output_dir=tmp_path, args_resume=True, args_session_id=None) + assert session_id_2 == session_id_3 + + session_id_4 = get_session_id(args_output_dir=tmp_path, args_resume=True, args_session_id="abc") + assert session_id_4 == "abc" diff --git a/tests/test_cli_full_integration.py b/tests/test_cli_full_integration.py new file mode 100644 index 0000000..e2d8520 --- /dev/null +++ b/tests/test_cli_full_integration.py @@ -0,0 +1,68 @@ +import subprocess +from pathlib import Path + +import pytest + +from pytest_abra import DirManager +from pytest_abra.utils import load_json_to_environ + + +@pytest.fixture(scope="session") +def session_tmp_path_testout(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("test_out") + + +@pytest.mark.slow +def test_abratest_cli_full_integration(session_tmp_path_testout: Path): + """run abratest against the dev instance""" + + # --------------------- load credentials to env variables -------------------- # + + cred_file = Path("credentials.json") + load_json_to_environ(cred_file) + + # --------------------------------- env files -------------------------------- # + + ENV_FILES_ROOT = Path("./envfiles").resolve() + ENV_FILES = [ + ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env", # authentik + ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env", # wordpress + ENV_FILES_ROOT / "files.test.dev.local-it.cloud.env", # nextcloud + ] + ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES]) + + # ----------------------------------- dirs ----------------------------------- # + + RECIPES_DIR = Path("./recipes").resolve() + # OUTPUT_DIR = Path("./test-output").resolve() + OUTPUT_DIR = session_tmp_path_testout.resolve() + + # ------------------------------------ run ----------------------------------- # + + result = subprocess.run( + [ + "abratest", + "--env_paths", + ENV_PATHS, + "--recipes_dir", + RECIPES_DIR, + "--output_dir", + OUTPUT_DIR, + "--session_id", + "abc", + ] + ) + + assert result.returncode == 0 + + +@pytest.mark.slow +def test_results_abra(session_tmp_path_testout: Path): + OUTPUT_DIR = session_tmp_path_testout.resolve() + + DIR = DirManager(output_dir=OUTPUT_DIR, session_id="abc") + all_files = [f.name for f in DIR.STATUS.rglob("*")] + passed_files = [f.name for f in DIR.STATUS.rglob("passed-*")] + failed_files = set(all_files) - set(passed_files) + assert len(all_files) > 0 + assert not failed_files, failed_files diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py new file mode 100644 index 0000000..3c00ed6 --- /dev/null +++ b/tests/test_coordinator.py @@ -0,0 +1,43 @@ +import os +import shutil +from pathlib import Path + +import pytest + +from pytest_abra.coordinator import Coordinator +from pytest_abra.dir_manager import DirManager + + +def test_load_test_credentials(tmp_path: Path): + assert "TEST_USER" not in os.environ + + DIR = DirManager(output_dir=tmp_path, session_id="abc") + DIR.create_all_dirs() + + Coordinator.load_test_credentials(DIR) + assert (DIR.STATES / "credentials_test.json").is_file() + + assert "TEST_USER" in os.environ + test_user_before = os.environ["TEST_USER"] + + # os.environ.clear() # this breaks pytest! + del os.environ["TEST_USER"] + assert "TEST_USER" not in os.environ + + Coordinator.load_test_credentials(DIR) + assert test_user_before == os.environ["TEST_USER"] + + +@pytest.fixture(scope="session") +def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path: + tmp_recipes_target = tmp_path_factory.mktemp("recipes") + recipes_dir_source = Path("recipes") + shutil.copytree(recipes_dir_source, tmp_recipes_target, dirs_exist_ok=True) + return tmp_recipes_target + + +def test_runner_runner_dict_import(tmp_recipes: Path): + """import from recipes dict should work, because create_runner_dict has sys.path.append""" + + RUNNER_DICT = Coordinator.create_runner_dict(tmp_recipes) + assert len(RUNNER_DICT.keys()) > 0 diff --git a/tests/test_dir_manager.py b/tests/test_dir_manager.py new file mode 100644 index 0000000..8dd78b1 --- /dev/null +++ b/tests/test_dir_manager.py @@ -0,0 +1,30 @@ + +import time +import pytest +from pytest_abra.dir_manager import DirManager +from pathlib import Path + +def test_get_latest_session_id_from_non_existing_dir(tmp_path: Path): + out = DirManager.get_latest_session_id(tmp_path / "not_exist") + assert out is None + +def test_get_latest_session_id_from_empty_dir(tmp_path: Path): + out = DirManager.get_latest_session_id(tmp_path) + assert out is None + +def test_get_latest_session_id_single(tmp_path: Path): + (tmp_path / "a").mkdir() + out = DirManager.get_latest_session_id(tmp_path) + assert out == "a" + + + +@pytest.mark.slow +def test_get_latest_session_id(tmp_path: Path): + (tmp_path / "a").mkdir() + time.sleep(1.1) + (tmp_path / "b").mkdir() + out = DirManager.get_latest_session_id(tmp_path) + assert out == "b" + + \ No newline at end of file diff --git a/tests/test_env_manager.py b/tests/test_env_manager.py new file mode 100644 index 0000000..05255f4 --- /dev/null +++ b/tests/test_env_manager.py @@ -0,0 +1,137 @@ +import shutil +from pathlib import Path + +import pytest + +from pytest_abra.dir_manager import DirManager +from pytest_abra.env_manager import EnvManager +from pytest_abra.utils import files_are_same + +ENV_PATHS = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik +] + + +@pytest.fixture +def tmp_output(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("output") + + +@pytest.fixture +def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("recipes") + + +def test_copy_env_files(tmp_output: Path, tmp_recipes: Path): + # create dirs in output + DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes) + DIR.create_all_dirs() + + # confirm dir is empty + assert len(list(DIR.ENV_FILES.iterdir())) == 0 + + # copy env files + env_files = EnvManager._get_env_files(ENV_PATHS) + EnvManager.copy_env_files(env_files, DIR) + + # check that each env file is present in DIR.ENV_FILES with correct contents + assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files) + for index, env_path in enumerate(ENV_PATHS): + matching_files = [f for f in DIR.ENV_FILES.iterdir() if index == int(f.name.split("-")[0])] + assert len(matching_files) == 1 + assert files_are_same(env_path, matching_files[0]) + + +def test_copy_env_files_twice(tmp_output: Path, tmp_recipes: Path): + """Copy the same env files twice""" + # create dirs in output + DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes) + DIR.create_all_dirs() + + # confirm dir is empty + assert len(list(DIR.ENV_FILES.iterdir())) == 0 + + # copy env files + env_files = EnvManager._get_env_files(ENV_PATHS) + EnvManager.copy_env_files(env_files, DIR) + + # check that each env file is present in DIR.ENV_FILES with correct contents + assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files) + + # copy env files again + EnvManager.copy_env_files(env_files, DIR) + + for index, env_path in enumerate(ENV_PATHS): + matching_files = [f for f in DIR.ENV_FILES.iterdir() if index == int(f.name.split("-")[0])] + assert len(matching_files) == 1 + assert files_are_same(env_path, matching_files[0]) + + +def test_copy_env_files_twice_with_content_change(tmp_output: Path, tmp_recipes: Path, tmp_path: Path): + # copy env files to tmp_path + assert len(list(tmp_path.iterdir())) == 0 + for f in ENV_PATHS: + shutil.copy(f, tmp_path / f.name) + ENV_PATHS_NEW = list(tmp_path.iterdir()) + assert len(ENV_PATHS_NEW) > 0 + + # create dirs in output + DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes) + DIR.create_all_dirs() + + # confirm dir is empty + assert len(list(DIR.ENV_FILES.iterdir())) == 0 + + # copy env files from tmp_path to tmp_output + env_files = EnvManager._get_env_files(ENV_PATHS_NEW) + EnvManager.copy_env_files(env_files, DIR) + + # check that each env file is present in DIR.ENV_FILES with correct contents + assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files) + + # change content of one env_file in tmp_path + file_path = next(tmp_path.iterdir()) + with open(file_path, "w") as file: + file.write("This is the new content") + + # copy env files again + with pytest.raises(AssertionError) as excinfo: + EnvManager.copy_env_files(env_files, DIR) + + assert "input env files have changed" in str(excinfo.value) + + +def test_copy_env_files_twice_with_name_change(tmp_output: Path, tmp_recipes: Path, tmp_path: Path): + # copy env files to tmp_path + assert len(list(tmp_path.iterdir())) == 0 + for f in ENV_PATHS: + shutil.copy(f, tmp_path / f.name) + ENV_PATHS_NEW = list(tmp_path.iterdir()) + assert len(ENV_PATHS_NEW) > 0 + + # create dirs in output + DIR = DirManager(output_dir=tmp_output, session_id="abc", recipes_dir=tmp_recipes) + DIR.create_all_dirs() + + # confirm dir is empty + assert len(list(DIR.ENV_FILES.iterdir())) == 0 + + # copy env files from tmp_path to tmp_output + env_files = EnvManager._get_env_files(ENV_PATHS_NEW) + EnvManager.copy_env_files(env_files, DIR) + + # check that each env file is present in DIR.ENV_FILES with correct contents + assert len(list(DIR.ENV_FILES.iterdir())) == len(env_files) + + # change name of one env_file in tmp_path + file_path = next(tmp_path.iterdir()) + file_path.rename(file_path.parent / (file_path.stem + "-other" + file_path.suffix)) + + # copy env files from tmp_path to tmp_output again + with pytest.raises(AssertionError) as excinfo: + env_files = EnvManager._get_env_files(list(tmp_path.iterdir())) + EnvManager.copy_env_files(env_files, DIR) + + assert "input env files have changed" in str(excinfo.value) diff --git a/tests/test_env_resolution.py b/tests/test_env_resolution.py index ddb5fc4..bbdf8f0 100644 --- a/tests/test_env_resolution.py +++ b/tests/test_env_resolution.py @@ -102,3 +102,17 @@ def test_env_manager() -> None: assert ENV.env_files[0].env_type == "authentik" assert ENV.env_files[1].env_type == "authentik" assert ENV.env_files[2].env_type == "wordpress" + + +def test_RUNNER_DICT_missing_key() -> None: + """RUNNER_DICT missing wordpress key while .env file with TYPE=wordpress given""" + env_paths_list = [ + Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik + ] + RUNNER_DICT_COPY = RUNNER_DICT.copy() + del RUNNER_DICT_COPY["wordpress"] + with pytest.raises(AssertionError) as excinfo: + EnvManager(env_paths_list, RUNNER_DICT_COPY) + assert "no runner for" in str(excinfo.value) diff --git a/tests/test_html_merge.py b/tests/test_html_merge.py index d97ab83..a216dfa 100644 --- a/tests/test_html_merge.py +++ b/tests/test_html_merge.py @@ -16,27 +16,31 @@ def session_tmp_path(tmp_path_factory: pytest.TempPathFactory) -> Path: return tmp_path_factory.mktemp("html_test") -def test_merge_html(session_tmp_path: Path): +@pytest.fixture(scope="session") +def html_file(session_tmp_path: Path) -> Path: """combines all generated pytest html reports into one""" in_dir_path = Path(__file__).parent / "assets" / "html_merge" in_dir_path = in_dir_path.resolve() - ic(in_dir_path) - out_file_path = session_tmp_path / "test.html" - out_assets_dir = session_tmp_path / "assets" + html_file = session_tmp_path / "test.html" - merge_html_reports(in_dir_path.as_posix(), out_file_path.as_posix(), "combined.html") + merge_html_reports(in_dir_path.as_posix(), html_file.as_posix(), "combined.html") + return html_file - assert out_file_path.is_file() - assert out_assets_dir.is_dir() - assert next(out_assets_dir.glob("*")) + +def test_merge_html(html_file: Path): + assert html_file.is_file() + assert html_file.parent.is_dir() + assert next(html_file.parent.glob("*")) @pytest.mark.slow -def test_check_result_with_playwright(session_tmp_path, context: BrowserContext): - html_file = session_tmp_path / "test.html" +def test_check_result_with_playwright(html_file: Path, context: BrowserContext): + assert html_file.is_file() + file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() + page = context.new_page() page.goto(file_url) diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..24f9bf6 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,29 @@ +from pathlib import Path + +from pytest_abra import DirManager, Runner + + +def test_runner_create_status_file(tmp_path: Path): + """check if _create_status_file prevents duplicates""" + + DIR = DirManager(output_dir=tmp_path, session_id="temp") + DIR.create_all_dirs() + assert len(list(DIR.STATUS.iterdir())) == 0 + + # create first status file + Runner._create_status_file(DIR, "passed", "identifier-a") + assert len(list(DIR.STATUS.iterdir())) == 1 + + # create second status file + Runner._create_status_file(DIR, "passed", "identifier-b") + assert len(list(DIR.STATUS.iterdir())) == 2 + + # check if _get_status_files finds only the correct status file + result = Runner._get_status_files(DIR, "identifier-a") + assert len(result) == 1 + + # overwrite first status file + Runner._create_status_file(DIR, "failed", "identifier-a") + assert len(list(DIR.STATUS.iterdir())) == 2 + + assert Runner._is_test_passed(DIR, "identifier-a") is False From 8b9dd47f9e872e2c8241f5d6c4efe5a5894c9895 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 14 Dec 2023 14:18:53 +0100 Subject: [PATCH 17/18] hatch-dynamic-versioning (#17) - remove pkg_resources requirement to output project version - project version is not globally defined in pytest_abra/__init__.py - use hatch dynamic to get version in pyproject.toml Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/17 Co-authored-by: Daniel Co-committed-by: Daniel --- pyproject.toml | 5 ++++- pytest_abra/__init__.py | 2 ++ pytest_abra/cli.py | 5 ++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8625d38..6708c04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "pytest-abra" description = "A pytest plugin to test instances of abra recipes" +dynamic = ["version"] authors = [{name = "Local-IT e.V."}] readme = "README.md" -version = "0.2.0" requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", @@ -42,6 +42,9 @@ abratest = "pytest_abra.cli:run" requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.version] +path = "pytest_abra/__init__.py" + [tool.hatch.build] include = [ "pytest_abra/*.py", diff --git a/pytest_abra/__init__.py b/pytest_abra/__init__.py index 5ec5233..76af3df 100644 --- a/pytest_abra/__init__.py +++ b/pytest_abra/__init__.py @@ -14,3 +14,5 @@ __all__ = [ "EnvFile", "EnvManager", ] + +__version__ = "0.3.0" diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index ca39cf2..680ecd1 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -2,16 +2,15 @@ import argparse import os from pathlib import Path -import pkg_resources # type: ignore from loguru import logger -from pytest_abra import Coordinator +from pytest_abra import Coordinator, __version__ from pytest_abra.dir_manager import DirManager from pytest_abra.utils import get_session_id def get_version(): - return pkg_resources.get_distribution("pytest_abra").version + return __version__ def run(): From 0fafa2227237f0f278752a4fad39c5b99692487d Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 15 Dec 2023 17:57:48 +0100 Subject: [PATCH 18/18] cleanup (#18) - remove demo runner - improve docs - rename all tests to test_* (previously, also setup_* and cleanup_* existed) to improve stability as it is not guaranteed that pytest.ini is loaded. - improve logging formatting - improve full integration test Reviewed-on: https://git.local-it.org/local-it-infrastructure/e2e_tests/pulls/18 Co-authored-by: Daniel Co-committed-by: Daniel --- docs/documentation.md | 1 + pyproject.toml | 1 - pytest_abra/cli.py | 8 +++-- pytest_abra/custom_fixtures.py | 2 +- pytest_abra/runner.py | 1 + .../tests_authentik/cleanup_authentik.py | 2 +- .../tests_authentik/setup_authentik.py | 7 +++-- recipes/demo/tests_demo/__init__.py | 0 recipes/demo/tests_demo/fixtures_demo.py | 26 ---------------- recipes/demo/tests_demo/runner_demo.py | 24 -------------- recipes/demo/tests_demo/setup_demo.py | 3 -- .../tests_nextcloud/runner_nextcloud.py | 2 +- .../tests_nextcloud/setup_nextcloud.py | 2 +- .../tests_wordpress/setup_wordpress.py | 2 +- .../setup_wordpress_trigger_email.py | 2 +- tests/test_cli_full_integration.py | 31 +++++++++++++------ tests/test_coordinator.py | 12 ++++++- tests/test_dir_manager.py | 13 ++++---- 18 files changed, 58 insertions(+), 81 deletions(-) delete mode 100644 recipes/demo/tests_demo/__init__.py delete mode 100644 recipes/demo/tests_demo/fixtures_demo.py delete mode 100644 recipes/demo/tests_demo/runner_demo.py delete mode 100644 recipes/demo/tests_demo/setup_demo.py diff --git a/docs/documentation.md b/docs/documentation.md index b84c7da..421dcb2 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -222,6 +222,7 @@ To understand how a test suite is built, let's have a look at the files runner_authentik.py -> required, defines the Runner subclass (see below) conftest.py -> not required. special file for pytest. is automatically discovered and loaded. convenient place to define fixtures and functions to be used in more than one test routine setup_authentik.py -> not required. can hold setup routine for authentik, has to be registered in runner_authentik.py +fixtures_authentik.py -> not required. holds fixtures that are meant to be imported by other test modules that depend on authentik. # Create a custom Runner diff --git a/pyproject.toml b/pyproject.toml index 6708c04..0a2ed00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,6 @@ line-length = 120 target-version = "py311" [tool.pytest.ini_options] -python_functions = "setup_* test_* cleanup_*" norecursedirs = ".* previous-work recipes" testpaths = "tests" markers = [ diff --git a/pytest_abra/cli.py b/pytest_abra/cli.py index 680ecd1..ab4d791 100644 --- a/pytest_abra/cli.py +++ b/pytest_abra/cli.py @@ -1,5 +1,6 @@ import argparse import os +import sys from pathlib import Path from loguru import logger @@ -19,10 +20,11 @@ def run(): parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;", required=True) parser.add_argument("--recipes_dir", type=Path, help="Dir of abra recipes and respective runners", required=True) parser.add_argument("--output_dir", type=Path, help="Dir of test outputs", required=True) - parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=20_000) + parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=30_000) parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode") parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests") parser.add_argument("--session_id", help="Session dir name (inside output_dir). Overwrites --resume") + parser.add_argument("--cleanup", help="Force test cleanup. Should not be necessary") args = parser.parse_args() env_paths = [Path(s) for s in args.env_paths.split(";")] @@ -41,7 +43,9 @@ def run(): # todo: move to Coordinator DIR = DirManager(output_dir=args.output_dir, session_id=session_id) log_file = DIR.RESULTS / "coordinator.log" - logger.add(log_file) + logger.remove() + logger.add(log_file, format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}") + logger.add(sys.stdout, colorize=True, format="{time:YYYY-MM-DD HH:mm:ss} {message}") # ---------------------------- initialize and run ---------------------------- # diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index 0f0a5f0..4c07127 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -22,7 +22,7 @@ def pytest_addoption(parser: Parser): parser.addoption("--runner_index", action="store", type=int) parser.addoption("--output_dir", action="store", type=Path) parser.addoption("--session_id", action="store", type=str) - parser.addoption("--timeout", action="store", type=int, default=20_000) + parser.addoption("--timeout", action="store", type=int, default=30_000) @pytest.fixture(autouse=True) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index c9eb650..6676560 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -149,6 +149,7 @@ class Runner: # command_arguments.append("--traceconfig") command_arguments.append("-v") + command_arguments.append(str(full_test_path)) command_arguments.append("--runner_index") diff --git a/recipes/authentik/tests_authentik/cleanup_authentik.py b/recipes/authentik/tests_authentik/cleanup_authentik.py index b556ea4..3f65dfd 100644 --- a/recipes/authentik/tests_authentik/cleanup_authentik.py +++ b/recipes/authentik/tests_authentik/cleanup_authentik.py @@ -27,7 +27,7 @@ def remove_user(admin_context: BrowserContext, URL: BaseUrl): page.get_by_role("dialog").get_by_role("button", name=re.compile(r"Löschen|Delete")).click() -def cleanup_delete_user( +def test_cleanup_delete_user( context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists ): # load admin cookies to context diff --git a/recipes/authentik/tests_authentik/setup_authentik.py b/recipes/authentik/tests_authentik/setup_authentik.py index b66b41d..9dd6aee 100644 --- a/recipes/authentik/tests_authentik/setup_authentik.py +++ b/recipes/authentik/tests_authentik/setup_authentik.py @@ -12,7 +12,7 @@ TEST_USER = os.environ["TEST_USER"] TEST_PASS = os.environ["TEST_PASS"] -def setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl): +def test_setup_admin_state(context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl): # go to page page = context.new_page() page.goto(URL.get()) @@ -50,7 +50,8 @@ def create_invite_link(admin_context: BrowserContext, env_config: dict[str, str] page.locator('input[name="name"]').click() linkname = "test_link_123" page.locator('input[name="name"]').fill(linkname) - page.get_by_placeholder("Wählen Sie ein Objekt aus.").click() + placeholder_pattern = re.compile(r"Wählen Sie ein|Select an") + page.get_by_placeholder(placeholder_pattern).click() page.get_by_role("option", name=re.compile(r"invitation-enrollment-flow")).click() # force, because else we get "intercepts pointer events" @@ -82,7 +83,7 @@ def create_user(user_context: BrowserContext, invitelink): expect(page.locator("ak-library")).to_be_visible() -def setup_user_state( +def test_setup_user_state( context: BrowserContext, env_config: dict[str, str], DIR: DirManager, URL: BaseUrl, check_if_user_exists ): # load admin cookies to context diff --git a/recipes/demo/tests_demo/__init__.py b/recipes/demo/tests_demo/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/recipes/demo/tests_demo/fixtures_demo.py b/recipes/demo/tests_demo/fixtures_demo.py deleted file mode 100644 index c650202..0000000 --- a/recipes/demo/tests_demo/fixtures_demo.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -This file can be used to define fixtures thate are then used by other tests which -depend on [demo]. For this to work - -1. the Runner class of the other test needs to define the depencency as seen - by referencing RunnerDemo in the dependencies list: - -from pytest_abra.tests_demo.runner_demo import RunnerDemo - -class RunnerOther(Runner): - dependencies = [RunnerDemo] - - -2. the specific tests that rely on these fixtures need to import the fixtures. - To globally import for all tests in 'other', the import should be done in conftest: - -in 'conftest.py' in 'test_other' dir: -from pytest_abra.tests_demo.fixtures_demo import demo_fixture -""" - -import pytest - - -@pytest.fixture -def demo_fixture(): - return "" diff --git a/recipes/demo/tests_demo/runner_demo.py b/recipes/demo/tests_demo/runner_demo.py deleted file mode 100644 index e735e5f..0000000 --- a/recipes/demo/tests_demo/runner_demo.py +++ /dev/null @@ -1,24 +0,0 @@ -from pytest_abra.runner import Runner, Test - - -class RunnerDemo(Runner): - """Every env file has a corresponding runner class""" - - env_type = "demo" # name of the test, used for logging / output naming - - # this indicates that tests from RunnerDemo depend on the setup from RunnerAuthentik. - # RunnerDemo will only execute, when setup_authentik.py has finished successfully. - # For example, setup_authentik.py generates session states, that can be used as fixtures - # that can be loaded from fixtures_authentik.py - dependencies: list[str] = ["authentik"] - - # todo: update these comments - # Filename of Demo setup. If defined, it will run 1st by executing pytest - # Filename of Demo test. This file contains unconditional tests that will be run in any - # case. If defined, it will run 2nd by executing pytest - # this list can hold many more tests from RunnerDemo that run conditional. The condition - # and the test file can be defined by creating a ConditionalTest instance: - # ConditionalTest(condition: Callable, test_file: str) - setups: list[Test] = [] - tests: list[Test] = [] - cleanups: list[Test] = [] diff --git a/recipes/demo/tests_demo/setup_demo.py b/recipes/demo/tests_demo/setup_demo.py deleted file mode 100644 index d46a9d6..0000000 --- a/recipes/demo/tests_demo/setup_demo.py +++ /dev/null @@ -1,3 +0,0 @@ -# Define functions here that are specifically meant for setup, not for testing. This means -# all actions that simply are required for other tests from 'demo' to run. Runs before all -# tests from 'demo'. diff --git a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py index 0cb466c..fe56587 100644 --- a/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/runner_nextcloud.py @@ -6,7 +6,7 @@ class RunnerNextcloud(Runner): dependencies = ["authentik"] setups = [Test(test_file="setup_nextcloud.py", prevent_skip=False)] tests = [ - Test(test_file="tests_nextcloud.py", prevent_skip=True), + Test(test_file="tests_nextcloud.py"), # Test(condition=condition_always_false, test_file="tests_nextcloud_onlyoffice.py"), ] # cleanups = [Test(test_file="cleanup_nextcloud.py")] diff --git a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py index 3f18b2c..378c1db 100644 --- a/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py +++ b/recipes/nextcloud/tests_nextcloud/setup_nextcloud.py @@ -10,7 +10,7 @@ from pytest_abra import BaseUrl, DirManager # https://files.test.dev.local-it.cloud/apps/files/ -def setup_nextcloud_admin_session(authentik_admin_page: Page, DIR: DirManager, URL: BaseUrl): +def test_setup_nextcloud_admin_session(authentik_admin_page: Page, DIR: DirManager, URL: BaseUrl): """visit nextcloud from authentik with admin_session to create wordpress_admin_session""" with authentik_admin_page.expect_popup() as event_context: authentik_admin_page.get_by_role("link", name="Nextcloud").click() diff --git a/recipes/wordpress/tests_wordpress/setup_wordpress.py b/recipes/wordpress/tests_wordpress/setup_wordpress.py index a65d8fe..3438db7 100644 --- a/recipes/wordpress/tests_wordpress/setup_wordpress.py +++ b/recipes/wordpress/tests_wordpress/setup_wordpress.py @@ -13,7 +13,7 @@ def test_visit_from_domain(authentik_admin_context: BrowserContext, URL: BaseUrl expect(page.locator("#wpadminbar")).to_be_visible(timeout=3_000) -def setup_wordpress_admin_session(authentik_admin_page: Page, DIR: DirManager): +def test_setup_wordpress_admin_session(authentik_admin_page: Page, DIR: DirManager): """visit wordpress from authentik with admin_session to create wordpress_admin_session""" with authentik_admin_page.expect_popup() as event_context: authentik_admin_page.get_by_role("link", name="Wordpress").click() diff --git a/recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py b/recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py index 33afb6b..9ae1e28 100644 --- a/recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py +++ b/recipes/wordpress/tests_wordpress/setup_wordpress_trigger_email.py @@ -5,7 +5,7 @@ from playwright.sync_api import Page, expect from pytest_abra import BaseUrl -def setup_trigger_email(wordpress_admin_page: Page, URL: BaseUrl): +def test_setup_trigger_email(wordpress_admin_page: Page, URL: BaseUrl): """change profile email to EMAIL to trigger email""" page = wordpress_admin_page page.goto(URL.get("wp-admin/profile.php")) diff --git a/tests/test_cli_full_integration.py b/tests/test_cli_full_integration.py index e2d8520..fcf39b9 100644 --- a/tests/test_cli_full_integration.py +++ b/tests/test_cli_full_integration.py @@ -1,3 +1,4 @@ +import shutil import subprocess from pathlib import Path @@ -8,13 +9,25 @@ from pytest_abra.utils import load_json_to_environ @pytest.fixture(scope="session") -def session_tmp_path_testout(tmp_path_factory: pytest.TempPathFactory) -> Path: - return tmp_path_factory.mktemp("test_out") +def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path: + tmp_recipes_target = tmp_path_factory.mktemp("recipes") + recipes_dir_source = Path("recipes") + shutil.copytree(recipes_dir_source, tmp_recipes_target, dirs_exist_ok=True) + return tmp_recipes_target + + +@pytest.fixture(scope="session") +def tmp_output(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("output") @pytest.mark.slow -def test_abratest_cli_full_integration(session_tmp_path_testout: Path): - """run abratest against the dev instance""" +def test_abratest_cli_full_integration(tmp_output: Path, tmp_recipes: Path): + """Full integration test of abratest against the dev instance. Recipes dir not in path + + this test is hard to debug as the output dir is in tmp. If required, try + pytest -s + or find the tmp dir to look into test outputs""" # --------------------- load credentials to env variables -------------------- # @@ -33,9 +46,9 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path): # ----------------------------------- dirs ----------------------------------- # - RECIPES_DIR = Path("./recipes").resolve() - # OUTPUT_DIR = Path("./test-output").resolve() - OUTPUT_DIR = session_tmp_path_testout.resolve() + RECIPES_DIR = tmp_recipes.resolve() + # RECIPES_DIR = Path("recipes") + OUTPUT_DIR = tmp_output.resolve() # ------------------------------------ run ----------------------------------- # @@ -57,8 +70,8 @@ def test_abratest_cli_full_integration(session_tmp_path_testout: Path): @pytest.mark.slow -def test_results_abra(session_tmp_path_testout: Path): - OUTPUT_DIR = session_tmp_path_testout.resolve() +def test_full_integration_results(tmp_output: Path): + OUTPUT_DIR = tmp_output.resolve() DIR = DirManager(output_dir=OUTPUT_DIR, session_id="abc") all_files = [f.name for f in DIR.STATUS.rglob("*")] diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py index 3c00ed6..08b7975 100644 --- a/tests/test_coordinator.py +++ b/tests/test_coordinator.py @@ -1,5 +1,6 @@ import os import shutil +import sys from pathlib import Path import pytest @@ -36,7 +37,16 @@ def tmp_recipes(tmp_path_factory: pytest.TempPathFactory) -> Path: return tmp_recipes_target -def test_runner_runner_dict_import(tmp_recipes: Path): +@pytest.fixture +def clear_sys_path(): + """clear sys.path before test, restore after""" + syspath_copy = sys.path.copy() + sys.path.clear() + yield + sys.path.extend(syspath_copy) + + +def test_runner_runner_dict_import(tmp_recipes: Path, clear_sys_path): """import from recipes dict should work, because create_runner_dict has sys.path.append""" RUNNER_DICT = Coordinator.create_runner_dict(tmp_recipes) diff --git a/tests/test_dir_manager.py b/tests/test_dir_manager.py index 8dd78b1..caee6fa 100644 --- a/tests/test_dir_manager.py +++ b/tests/test_dir_manager.py @@ -1,23 +1,26 @@ - import time -import pytest -from pytest_abra.dir_manager import DirManager from pathlib import Path +import pytest + +from pytest_abra.dir_manager import DirManager + + def test_get_latest_session_id_from_non_existing_dir(tmp_path: Path): out = DirManager.get_latest_session_id(tmp_path / "not_exist") assert out is None + def test_get_latest_session_id_from_empty_dir(tmp_path: Path): out = DirManager.get_latest_session_id(tmp_path) assert out is None + def test_get_latest_session_id_single(tmp_path: Path): (tmp_path / "a").mkdir() out = DirManager.get_latest_session_id(tmp_path) assert out == "a" - @pytest.mark.slow def test_get_latest_session_id(tmp_path: Path): @@ -26,5 +29,3 @@ def test_get_latest_session_id(tmp_path: Path): (tmp_path / "b").mkdir() out = DirManager.get_latest_session_id(tmp_path) assert out == "b" - - \ No newline at end of file