add api testing (#14)

* add fixture to make api calls with authentification

* add authentik test that checks the status of all blueprints

* add option to append any kind of data to html report

Reviewed-on: local-it-infrastructure/e2e_tests#14
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>

prototyping/documentation.md

@@ -1,3 +1,10 @@
+# RUN
+
+
+
+
+
+
 
 
 Abratest has 3 required inputs, but most importantly the test configuration is done through the .env files given with the --env_paths argument. So let's say we want to run abratest with these 3 .env files:
@@ -60,11 +67,14 @@ Furthermore, some `Runner` classes can depend on others. For example, `RunnerWor
 | 9.  | Wordpress-2    | cleanups |
 
 
+# Create a custom Runner
 
-To comprehend this process, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally.
+To comprehend the process of creating a new subclass of `Runner`, let's examine a simplified rendition of the `RunnerWordpress` class. Within it, there exist two setup scripts and two test scripts, one of which operates conditionally.
 
 
 ```python
+from pytest_abra import Runner, Test
+
 class RunnerWordpress(Runner):
     env_type = "wordpress"
     dependencies = ["authentik"]
@@ -78,3 +88,17 @@ class RunnerWordpress(Runner):
     ]
     cleanups = []
 ```
+
+The signature of condition functions can be seen below. The function takes one `NamedTuple` and returns of type `bool`. You can learn about the contents of the input by looking up the class `ConditionArgs`. Generally speaking, it provides access to all of the .env files, especially the one related to the current Runner.
+
+```python
+def condition_function(args: ConditionArgs) -> bool:
+    ...
+```
+
+
+# Create custom Tests
+
+The test files are written in the same way as any other pytest test file. The only difference is that pytest-abra provides custom fixtures that make it easy to get the configuration by the provided .env files and to deal with URLS etc.
+
+# todo: add example

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "imbox == 0.9.8",
     "hatchling == 1.18.0",
     "icecream",
+    "tabulate",
 ]
 
 [project.entry-points.pytest11]
@@ -49,6 +50,6 @@ line-length = 120
 target-version = "py311"
 
 [tool.pytest.ini_options]
-python_functions = "test_* setup_*"
+python_functions = "setup_* test_* cleanup_*"
 norecursedirs = ".* previous-work recipes"
 testpaths = "tests"

pytest_abra/__init__.py

@@ -1,5 +1,6 @@
 from pytest_abra.coordinator import Coordinator
 from pytest_abra.dir_manager import DirManager
+from pytest_abra.env_manager import EnvFile
 from pytest_abra.runner import ConditionArgs, Runner, Test
 from pytest_abra.utils import BaseUrl
 
@@ -10,4 +11,5 @@ __all__ = [
     "Test",
     "DirManager",
     "BaseUrl",
+    "EnvFile",
 ]

pytest_abra/custom_fixtures.py

@@ -5,18 +5,16 @@ import os
 import re
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Protocol, TypedDict
+from typing import Generator, Protocol, TypedDict
 
 import pytest
 from dotenv import dotenv_values
-from icecream import ic
+from icecream import ic  # type: ignore
 from imbox import Imbox  # type: ignore
-from playwright.sync_api import BrowserContext, expect
+from playwright.sync_api import APIRequestContext, BrowserContext, Playwright, expect
 from pytest import Parser
 
-from pytest_abra.dir_manager import DirManager
-from pytest_abra.env_manager import EnvFile
-from pytest_abra.utils import BaseUrl
+from pytest_abra import BaseUrl, DirManager, EnvFile
 
 
 def pytest_addoption(parser: Parser):
@@ -152,3 +150,14 @@ def imap_recent_messages(imap_client: Imbox) -> list[Message]:
         messages.append(message)
 
     return messages
+
+
+@pytest.fixture(scope="session")
+def api_request_context(
+    playwright: Playwright,
+    DIR: DirManager,
+) -> Generator[APIRequestContext, None, None]:
+    state_file = DIR.STATES / "authentik_admin_state.json"
+    request_context = playwright.request.new_context(storage_state=state_file)
+    yield request_context
+    request_context.dispose()

pytest_abra/runner.py

@@ -1,3 +1,4 @@
+import os
 from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Callable, NamedTuple
@@ -134,7 +135,6 @@ class Runner:
         # command_arguments.append("--traceconfig")
 
         command_arguments.append("-v")
-        # command_arguments.append("-rx")
         command_arguments.append(str(full_test_path))
 
         command_arguments.append("--runner_index")
@@ -158,12 +158,13 @@ class Runner:
         command_arguments.append(str(self.DIR.RECORDS / "traces" / full_test_path.stem))
 
         # tracing
-        command_arguments.append("--tracing")
+        command_arguments.append("--tracing")  # "on", "off", "retain-on-failure"
         command_arguments.append("retain-on-failure")
-        # command_arguments.append("on")
 
         # Disable capturing. With -s set, prints will go to console as if pytest is not there.
-        # command_arguments.append("-s")
+        if os.environ.get("PWDEBUG") == "1":
+            command_arguments.append("-s")
+        command_arguments.append("-s")
 
         # headed
         # command_arguments.append("--headed")

recipes/authentik/tests_authentik/runner_authentik.py

@@ -4,4 +4,4 @@ from pytest_abra import Runner, Test
 class RunnerAuthentik(Runner):
     env_type = "authentik"
     setups = [Test(test_file="setup_authentik.py")]
-    # tests = [Test(test_file="test_authentik_dummy.py")]
+    tests = [Test(test_file="test_authentik_blueprint_api.py")]

recipes/authentik/tests_authentik/test_authentik_blueprint_api.py

@@ -0,0 +1,39 @@
+# api testing
+# https://playwright.dev/python/docs/api-testing
+
+import pytest_html  # type: ignore
+from icecream import ic  # type: ignore
+from playwright.sync_api import APIRequestContext
+from tabulate import tabulate  # type: ignore
+
+from pytest_abra import BaseUrl
+
+
+def test_authentik_blueprint_status(
+    api_request_context: APIRequestContext,
+    URL: BaseUrl,
+    extras,
+) -> None:
+    blueprints = api_request_context.get(URL.get("api/v3/managed/blueprints"))
+    assert blueprints.ok
+    blueprints_data = blueprints.json()
+    ic(blueprints_data)
+
+    # fake failed blueprint
+    # blueprints_data["results"][10]["status"] = "failed"
+
+    table_data_all = []
+    table_data_failed = []
+    for item in blueprints_data["results"]:
+        row = [item["name"], item["enabled"], item["status"]]
+        table_data_all.append(row)
+        if item["status"] != "successful":
+            table_data_failed.append(row)
+
+    table = tabulate(table_data_all, headers=["name", "enabled", "status"])
+    extras.append(pytest_html.extras.text(table, name="Authentik Blueprint Status"))
+
+    # with pytest -v (verbose) the failed blueprints will be visible in the traceback
+    assert (
+        table_data_failed == []
+    ), "One or more blueprints were not successful. See Authentik Blueprint Status in html report"

recipes/authentik/tests_authentik/test_authentik_dummy.py

@@ -1,6 +0,0 @@
-def test_true():
-    assert 1 + 1 == 2
-
-
-def test_not_true():
-    assert 1 + 1 == 3