From b0fb608e05e6565decc412fdae5fe103b134601d Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 Dec 2023 18:31:58 +0100 Subject: [PATCH 01/12] rename --- prototyping/{structure.md => documentation.md} | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) rename prototyping/{structure.md => documentation.md} (90%) diff --git a/prototyping/structure.md b/prototyping/documentation.md similarity index 90% rename from prototyping/structure.md rename to prototyping/documentation.md index 15c820f..8581223 100644 --- a/prototyping/structure.md +++ b/prototyping/documentation.md @@ -1,3 +1,10 @@ +# RUN + + + + + + Abratest has 3 required inputs, but most importantly the test configuration is done through the .env files given with the --env_paths argument. So let's say we want to run abratest with these 3 .env files: @@ -60,11 +67,14 @@ Furthermore, some `Runner` classes can depend on others. For example, `RunnerWor | 9. | Wordpress-2 | cleanups | +# Create a Runner -To comprehend this process, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. +To comprehend the process of creating a new subclass of `Runner`, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. ```python +from pytest_abra import Runner, Test + class RunnerWordpress(Runner): env_type = "wordpress" dependencies = ["authentik"] -- 2.47.2 From 47a829044c90a8220d997517257abf5e9d1484e4 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 Dec 2023 18:43:01 +0100 Subject: [PATCH 02/12] WIP --- prototyping/documentation.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/prototyping/documentation.md b/prototyping/documentation.md index 8581223..e27c359 100644 --- a/prototyping/documentation.md +++ b/prototyping/documentation.md @@ -67,7 +67,7 @@ Furthermore, some `Runner` classes can depend on others. For example, `RunnerWor | 9. | Wordpress-2 | cleanups | -# Create a Runner +# Create a custom Runner To comprehend the process of creating a new subclass of `Runner`, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally. @@ -88,3 +88,17 @@ class RunnerWordpress(Runner): ] cleanups = [] ``` + +The signature of condition functions can be seen below. The function takes one `NamedTuple` and returns of type `bool`. You can learn about the contents of the input by looking up the class `ConditionArgs`. Generally speaking, it provides access to all of the .env files, especially the one related to the current Runner. + +```python +def condition_function(args: ConditionArgs) -> bool: + ... +``` + + +# Create custom Tests + +The test files are written in the same way as any other pytest test file. The only difference is that pytest-abra provides custom fixtures that make it easy to get the configuration by the provided .env files and to deal with URLS etc. + +# todo: add example \ No newline at end of file -- 2.47.2 From 15be273c0824c6b663cba968063c031e83dc5c32 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 Dec 2023 18:43:11 +0100 Subject: [PATCH 03/12] also find files of name cleanup_ --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9bf9361..9bd455e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,6 @@ line-length = 120 target-version = "py311" [tool.pytest.ini_options] -python_functions = "test_* setup_*" +python_functions = "setup_* test_* cleanup_*" norecursedirs = ".* previous-work recipes" testpaths = "tests" \ No newline at end of file -- 2.47.2 From c9f8a4b2a9173d66b0782ee96c14568fe3ae52d3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 Dec 2023 19:02:16 +0100 Subject: [PATCH 04/12] add EnvFile --- pytest_abra/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest_abra/__init__.py b/pytest_abra/__init__.py index 18988b8..0b134ec 100644 --- a/pytest_abra/__init__.py +++ b/pytest_abra/__init__.py @@ -1,5 +1,6 @@ from pytest_abra.coordinator import Coordinator from pytest_abra.dir_manager import DirManager +from pytest_abra.env_manager import EnvFile from pytest_abra.runner import ConditionArgs, Runner, Test from pytest_abra.utils import BaseUrl @@ -10,4 +11,5 @@ __all__ = [ "Test", "DirManager", "BaseUrl", + "EnvFile", ] -- 2.47.2 From 6cfc23eb44e9d4d0eecfc26e92766fb6ed626652 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 Dec 2023 19:02:26 +0100 Subject: [PATCH 05/12] add api fixture --- pytest_abra/custom_fixtures.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index 2c343a9..2047ac9 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -5,18 +5,16 @@ import os import re from datetime import datetime, timedelta from pathlib import Path -from typing import Protocol, TypedDict +from typing import Generator, Protocol, TypedDict import pytest from dotenv import dotenv_values -from icecream import ic +from icecream import ic # type: ignore from imbox import Imbox # type: ignore -from playwright.sync_api import BrowserContext, expect +from playwright.sync_api import APIRequestContext, BrowserContext, Playwright, expect from pytest import Parser -from pytest_abra.dir_manager import DirManager -from pytest_abra.env_manager import EnvFile -from pytest_abra.utils import BaseUrl +from pytest_abra import BaseUrl, DirManager, EnvFile def pytest_addoption(parser: Parser): @@ -152,3 +150,15 @@ def imap_recent_messages(imap_client: Imbox) -> list[Message]: messages.append(message) return messages + + +@pytest.fixture(scope="session") +def api_request_context( + playwright: Playwright, + DIR: DirManager, + URL: BaseUrl, +) -> Generator[APIRequestContext, None, None]: + state_file = DIR.STATES / "authentik_admin_state.json" + request_context = playwright.request.new_context(storage_state=state_file, base_url=URL.get()) + yield request_context + request_context.dispose() -- 2.47.2 From ee05c07feba71378e2900c87a2d1438fcd9343f6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 Dec 2023 19:03:51 +0100 Subject: [PATCH 06/12] prepare authentik test for blueprint api --- recipes/authentik/tests_authentik/runner_authentik.py | 2 +- .../tests_authentik/test_authentik_blueprint_api.py | 2 ++ recipes/authentik/tests_authentik/test_authentik_dummy.py | 6 ------ 3 files changed, 3 insertions(+), 7 deletions(-) create mode 100644 recipes/authentik/tests_authentik/test_authentik_blueprint_api.py delete mode 100644 recipes/authentik/tests_authentik/test_authentik_dummy.py diff --git a/recipes/authentik/tests_authentik/runner_authentik.py b/recipes/authentik/tests_authentik/runner_authentik.py index 0570db7..2543dc6 100644 --- a/recipes/authentik/tests_authentik/runner_authentik.py +++ b/recipes/authentik/tests_authentik/runner_authentik.py @@ -4,4 +4,4 @@ from pytest_abra import Runner, Test class RunnerAuthentik(Runner): env_type = "authentik" setups = [Test(test_file="setup_authentik.py")] - # tests = [Test(test_file="test_authentik_dummy.py")] + tests = [Test(test_file="test_authentik_blueprint_api.py")] diff --git a/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py new file mode 100644 index 0000000..009560a --- /dev/null +++ b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py @@ -0,0 +1,2 @@ +# api testing +# https://playwright.dev/python/docs/api-testing diff --git a/recipes/authentik/tests_authentik/test_authentik_dummy.py b/recipes/authentik/tests_authentik/test_authentik_dummy.py deleted file mode 100644 index bddb561..0000000 --- a/recipes/authentik/tests_authentik/test_authentik_dummy.py +++ /dev/null @@ -1,6 +0,0 @@ -def test_true(): - assert 1 + 1 == 2 - - -def test_not_true(): - assert 1 + 1 == 3 -- 2.47.2 From 9080840c3ca266777edffbc4cea3dffc97040b1e Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:04:30 +0100 Subject: [PATCH 07/12] use command_arguments.append("-s") if debug --- pytest_abra/runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 1a75d27..b2f83ff 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -1,3 +1,4 @@ +import os from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Callable, NamedTuple @@ -133,7 +134,7 @@ class Runner: # command_arguments.append("--traceconfig") - command_arguments.append("-v") + # command_arguments.append("-v") # command_arguments.append("-rx") command_arguments.append(str(full_test_path)) @@ -163,7 +164,8 @@ class Runner: # command_arguments.append("on") # Disable capturing. With -s set, prints will go to console as if pytest is not there. - # command_arguments.append("-s") + if os.environ["PWDEBUG"] == "1": + command_arguments.append("-s") # headed # command_arguments.append("--headed") -- 2.47.2 From 14a5369a36aa32e5ea8485ebefa0242ac1cf4983 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:05:46 +0100 Subject: [PATCH 08/12] fixup --- pytest_abra/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index b2f83ff..1e310ca 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -164,7 +164,7 @@ class Runner: # command_arguments.append("on") # Disable capturing. With -s set, prints will go to console as if pytest is not there. - if os.environ["PWDEBUG"] == "1": + if os.environ.get("PWDEBUG") == "1": command_arguments.append("-s") # headed -- 2.47.2 From cb13efdc3135da51bc553665e4f10b0888f07750 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:26:48 +0100 Subject: [PATCH 09/12] add tabulate --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9bd455e..a63e9dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "imbox == 0.9.8", "hatchling == 1.18.0", "icecream", + "tabulate", ] [project.entry-points.pytest11] -- 2.47.2 From a3a0ea328d70378d5441b08f1a8d8f74d0bf2386 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:29:54 +0100 Subject: [PATCH 10/12] remobe baseurl from api_request_context --- pytest_abra/custom_fixtures.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytest_abra/custom_fixtures.py b/pytest_abra/custom_fixtures.py index 2047ac9..82bc0bd 100644 --- a/pytest_abra/custom_fixtures.py +++ b/pytest_abra/custom_fixtures.py @@ -156,9 +156,8 @@ def imap_recent_messages(imap_client: Imbox) -> list[Message]: def api_request_context( playwright: Playwright, DIR: DirManager, - URL: BaseUrl, ) -> Generator[APIRequestContext, None, None]: state_file = DIR.STATES / "authentik_admin_state.json" - request_context = playwright.request.new_context(storage_state=state_file, base_url=URL.get()) + request_context = playwright.request.new_context(storage_state=state_file) yield request_context request_context.dispose() -- 2.47.2 From 57008bc3d26e572311df1364080368ebe3cdfd2f Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:30:06 +0100 Subject: [PATCH 11/12] improve pytest arguments --- pytest_abra/runner.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index 1e310ca..cb86d66 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -134,8 +134,7 @@ class Runner: # command_arguments.append("--traceconfig") - # command_arguments.append("-v") - # command_arguments.append("-rx") + command_arguments.append("-v") command_arguments.append(str(full_test_path)) command_arguments.append("--runner_index") @@ -159,13 +158,13 @@ class Runner: command_arguments.append(str(self.DIR.RECORDS / "traces" / full_test_path.stem)) # tracing - command_arguments.append("--tracing") + command_arguments.append("--tracing") # "on", "off", "retain-on-failure" command_arguments.append("retain-on-failure") - # command_arguments.append("on") # Disable capturing. With -s set, prints will go to console as if pytest is not there. if os.environ.get("PWDEBUG") == "1": command_arguments.append("-s") + command_arguments.append("-s") # headed # command_arguments.append("--headed") -- 2.47.2 From 3ff3cb87cfbb1fd51f070f4b7b7ec74c011d8793 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:30:37 +0100 Subject: [PATCH 12/12] add test_authentik_blueprint_status --- .../test_authentik_blueprint_api.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py index 009560a..87eb452 100644 --- a/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py +++ b/recipes/authentik/tests_authentik/test_authentik_blueprint_api.py @@ -1,2 +1,39 @@ # api testing # https://playwright.dev/python/docs/api-testing + +import pytest_html # type: ignore +from icecream import ic # type: ignore +from playwright.sync_api import APIRequestContext +from tabulate import tabulate # type: ignore + +from pytest_abra import BaseUrl + + +def test_authentik_blueprint_status( + api_request_context: APIRequestContext, + URL: BaseUrl, + extras, +) -> None: + blueprints = api_request_context.get(URL.get("api/v3/managed/blueprints")) + assert blueprints.ok + blueprints_data = blueprints.json() + ic(blueprints_data) + + # fake failed blueprint + # blueprints_data["results"][10]["status"] = "failed" + + table_data_all = [] + table_data_failed = [] + for item in blueprints_data["results"]: + row = [item["name"], item["enabled"], item["status"]] + table_data_all.append(row) + if item["status"] != "successful": + table_data_failed.append(row) + + table = tabulate(table_data_all, headers=["name", "enabled", "status"]) + extras.append(pytest_html.extras.text(table, name="Authentik Blueprint Status")) + + # with pytest -v (verbose) the failed blueprints will be visible in the traceback + assert ( + table_data_failed == [] + ), "One or more blueprints were not successful. See Authentik Blueprint Status in html report" -- 2.47.2