installable package (#9)

* turn repo into installable package (pip install -e .)

* add hatchling build packend

* call it pytest-abra

* add pytest entrypoint, so that it gets loaded automatically if installed (and pytest is run)

* make fixtures optional, so that pytest can still be used in other context

* add cli script -> you can now directly run "pytest-abra" in console

Reviewed-on: local-it-infrastructure/e2e_tests#9
Co-authored-by: Daniel <d.brummerloh@gmail.com>
Co-committed-by: Daniel <d.brummerloh@gmail.com>
This commit is contained in:
Daniel 2023-12-07 11:32:01 +01:00 committed by dan
parent 4c5a470a70
commit 8685688698
33 changed files with 294 additions and 210 deletions

1
.gitignore vendored
View file

@ -5,4 +5,5 @@ TestResults/
*.pyc
*.json
*.zip
*.egg-info
credentials*

View file

@ -1,9 +1,9 @@
# AbraTest
# pytest-abra
...description...
# Usage
To use AbraTest, follow these steps:
To use pytest-abra, follow these steps:
## 1. GIT Clone
@ -17,7 +17,7 @@ git submodule update --remote // update submodules
## Run
You can run AbraTest with and without Docker. Choose now and follow the steps accordingly:
You can run pytest-abra with and without Docker. Choose now and follow the steps accordingly:
## 2.1 Run without Docker
@ -33,17 +33,14 @@ playwright install
Run the script with
```bash
python main.py # run abratest
pytest # test abratest
pytest --collect-only # debug test abratest
python main.py
```
# 2.2 Run with Docker
```bash
docker compose build # build the image
docker compose run --rm app ./run_abratest.sh # run AbraTest
docker compose run --rm app ./test_abratest.sh # test AbraTest
docker compose run --rm app ./run_pytest-abra.sh # run pytest-abra
```
Force rebuild with cache
@ -64,4 +61,12 @@ Use playwright codegen to create code for new testes easily https://playwright.d
```bash
playwright codegen demo.playwright.dev/todomvc
```
```
## Development
```bash
pytest # test pytest-abra
pytest --collect-only # debug test pytest-abra
docker compose run --rm app ./test_pytest-abra.sh # test pytest-abra
```

View file

@ -1,94 +0,0 @@
# regarding conftest:
# If you have conftest.py files which do not reside in a python package directory
# (i.e. one containing an __init__.py) then “import conftest” can be ambiguous
# because there might be other conftest.py files as well on your PYTHONPATH or
# sys.path. It is thus good practise for projects to either put conftest.py under
# a package scope or to never import anything from a conftest.py file.
import os
from imaplib import IMAP4_SSL
from pathlib import Path
import pytest
from dotenv import dotenv_values
from playwright.sync_api import BrowserContext, expect
from pytest import Parser
from abratest.dir_manager import DirManager
from abratest.utils import BaseUrl
# global timeout and LOCALE
LOCALE = {"Accept-Language": "de_DE"}
TIMEOUT = 20_000
expect.set_options(timeout=TIMEOUT)
@pytest.fixture
def context(context: BrowserContext) -> BrowserContext:
context.set_default_timeout(TIMEOUT)
context.set_extra_http_headers(LOCALE)
return context
def pytest_addoption(parser: Parser):
parser.addoption(
"--env_file",
action="store",
required=True,
)
parser.addoption(
"--output_dir",
action="store",
required=True,
)
parser.addoption(
"--session_id",
action="store",
required=True,
)
@pytest.fixture(scope="session", autouse=True)
def DIR(request) -> DirManager:
"""Fixture holding test directories
DIR.OUTPUT
DIR.SESSION
DIR.RECORDS
DIR.STATES
DIR.RESULTS"""
output_dir = request.config.getoption("--output_dir")
output_dir = Path(output_dir)
session_id = request.config.getoption("--session_id")
dirmanager = DirManager(output_dir=output_dir, session_id=session_id)
dirmanager.create_all_dirs()
return dirmanager
@pytest.fixture(scope="session", autouse=True)
def dotenv_config(request) -> dict[str, str]:
dotenv_path = request.config.getoption("--env_file")
dotenv_path = Path(dotenv_path)
assert dotenv_path.is_file()
return dotenv_values(dotenv_path) # type: ignore
@pytest.fixture(scope="session", autouse=True)
def URL(dotenv_config: dict[str, str]) -> BaseUrl:
return BaseUrl(netloc=dotenv_config["DOMAIN"])
@pytest.fixture(scope="session")
def imap_ssl_email_client() -> None:
assert os.environ["IMAP_HOST"]
assert os.environ["IMAP_PORT"]
assert os.environ["IMAP_USER"]
assert os.environ["IMAP_PASS"]
port = int(os.environ["IMAP_PORT"])
imap_client = IMAP4_SSL(host=os.environ["IMAP_HOST"], port=port)
imap_client.login(os.environ["IMAP_USER"], os.environ["IMAP_PASS"])
imap_client.select("INBOX")
yield imap_client
imap_client.close()
imap_client.logout()

View file

@ -1,2 +0,0 @@
# this file exists so that tests inside /tests always find /src imports,
# because this will cause the root (/) to be added to sys.path

92
main.py
View file

@ -1,42 +1,10 @@
import json
import os
import subprocess
from pathlib import Path
from loguru import logger
from abratest.coordinator import Coordinator
from abratest.dir_manager import DirManager
from abratest.utils import get_session_id
# ----------------------------- lookup env files ----------------------------- #
# This list of env files is the input to testing framework. each env file
# triggers the execution of one test Runner and provides configuration to the
# tests inside the runner. There can be dependencies, for example wordpress
# requires that authentik ran first to create the admin session and the user
# session. At the moment, wrong ordering results in unsuccessful test
# (wrong ordering would be wordpress env file is before authentik env file).
# At the moment, functionailty is only guaranteed if each env file use
# a unique TYPE var.
ENV_FILES = [
Path("envfiles/login.test.dev.local-it.cloud.env"), # authentik
Path("envfiles/blog.test.dev.local-it.cloud.env"), # wordpress
# Path("envfiles/files.test.dev.local-it.cloud.env"), # nextcloud
]
# ----------------------------- define ouptut dir ---------------------------- #
OUTPUT_DIR = Path("./test-output").resolve()
RECIPES_DIR = Path("./recipes").resolve()
# --------------------- load credentials to env variables -------------------- #
cred_file = Path("credentials.json")
with open(cred_file, "r") as f:
CREDENTIALS = json.load(f)
@ -44,37 +12,35 @@ with open(cred_file, "r") as f:
for key, value in CREDENTIALS.items():
os.environ[key] = value
# --------------------------------- env files -------------------------------- #
# -------------------------- enable playwright debug ------------------------- #
# This list of env files is the input to testing framework. each env file
# triggers the execution of one test Runner and provides configuration to the
# tests inside the runner.
ENV_FILES_ROOT = Path("../envfiles").resolve()
ENV_FILES = [
ENV_FILES_ROOT / "login.test.dev.local-it.cloud.env", # authentik
ENV_FILES_ROOT / "blog.test.dev.local-it.cloud.env", # wordpress
ENV_FILES_ROOT / "files.test.dev.local-it.cloud.env", # nextcloud
]
ENV_PATHS = ";".join([x.as_posix() for x in ENV_FILES])
# ----------------------------------- dirs ----------------------------------- #
RECIPES_DIR = Path("../recipes").resolve()
OUTPUT_DIR = Path("./test-output").resolve()
# add abra-testing dir
os.environ["PYTEST_PLUGINS"] = "abratest.plugin-abra" # "abratest.plugin,abratest.other"
# os.environ["PWDEBUG"] = "1"
# ----------------------------- define session_id ---------------------------- #
session_id = get_session_id()
# session_id = "abc"
# ------------------------------- setup logging ------------------------------ #
DIR = DirManager(output_dir=OUTPUT_DIR, session_id=session_id)
log_file = DIR.RECORDS / "coordinator.log"
logger.add(log_file)
# ---------------------------- initialize and run ---------------------------- #
coordinator = Coordinator(
env_paths_list=ENV_FILES, output_dir=OUTPUT_DIR, session_id=session_id, recipes_dir=RECIPES_DIR
subprocess.run(
[
"abratest",
"--env_paths",
ENV_PATHS,
"--recipes_dir",
RECIPES_DIR,
"--output_dir",
OUTPUT_DIR,
# "--debug",
]
)
coordinator.setup_test()
coordinator.run_test()
coordinator.combine_html()
coordinator.collect_traces()

View file

@ -1,6 +1,6 @@
from playwright.sync_api import BrowserContext, expect
from abratest.dir_manager import DirManager
from pytest_abra.dir_manager import DirManager
def test_wordpress(admin_session: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager):

View file

@ -0,0 +1,17 @@
import os
import subprocess
# Set an environment variable in the parent process
os.environ["PARENT_VARIABLE"] = "12345s"
# Spawn a subprocess and modify the environment variable
subprocess.run(
[
"python",
"-c",
"import os; print('b', os.environ['PARENT_VARIABLE']); os.environ['PARENT_VARIABLE'] = 'modified_value'; print('c', os.environ['PARENT_VARIABLE'])",
]
)
# Check if the modification in the subprocess affected the parent process
print("a", os.environ["PARENT_VARIABLE"]) # This will print 'parent_value', not 'modified_value'

View file

@ -1,18 +1,50 @@
[project]
name = "abratest"
name = "pytest-abra"
description = "A pytest plugin to test instances of abra recipes"
authors = [{name = "Local-IT e.V."}]
readme = "README.md"
version = "0.2.0"
requires-python = "~=3.11"
requires-python = ">=3.10"
classifiers = [
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Framework :: Pytest",
]
dependencies = [
"pytest == 7.4.3",
"playwright == 1.40",
"pytest-html == 4.1.1",
"pytest-playwright == 0.4.3",
"python-dotenv == 1.0.0",
"loguru == 0.7.2",
"beautifulsoup4 == 4.12.2",
"imbox == 0.9.8",
"hatchling == 1.18.0",
"icecream",
]
[project.optional-dependencies]
dev = [
"ruff >= 0.1.7",
[project.entry_points]
pytest11 = [
"pytest_abra = pytest_abra.pytest_abra",
]
[project.scripts]
abratest = "pytest_abra.cli:run"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build]
include = [
"pytest_abra/*.py",
]
exclude = [
"*.json",
]
[tool.setuptools]
package-dir = {"" = "abratest"}
[tool.ruff]
line-length = 120

56
pytest_abra/cli.py Normal file
View file

@ -0,0 +1,56 @@
import argparse
import os
from pathlib import Path
from loguru import logger
from pytest_abra.coordinator import Coordinator
from pytest_abra.dir_manager import DirManager
from pytest_abra.utils import get_datetime_string
def run():
parser = argparse.ArgumentParser()
parser.add_argument("--env_paths", type=str, help="List of loaded env files separated with ;")
parser.add_argument("--recipes_dir", type=Path, help="List of loaded env files separated with ;")
parser.add_argument("--output_dir", type=Path, help="List of loaded env files separated with ;")
parser.add_argument("--timeout", type=int, help="Set Playwright timeout in ms", default=20_000)
parser.add_argument("--debug", action="store_true", help="Enable Playwright debug mode")
parser.add_argument("--resume", action="store_true", help="Re-run the most recent test, skipping passed tests")
args = parser.parse_args()
ENV_FILES = [Path(s) for s in args.env_paths.split(";")]
# -------------------------- enable playwright debug ------------------------- #
if args.debug:
os.environ["PWDEBUG"] = "1"
# ----------------------------- define session_id ---------------------------- #
session_id = "test-" + get_datetime_string()
if args.resume:
# look for previous session_id
pass
# session_id = "abc"
# ------------------------------- setup logging ------------------------------ #
# todo: move to Coordinator
DIR = DirManager(output_dir=args.output_dir, session_id=session_id)
log_file = DIR.RECORDS / "coordinator.log"
logger.add(log_file)
# ---------------------------- initialize and run ---------------------------- #
coordinator = Coordinator(
env_paths_list=ENV_FILES,
output_dir=args.output_dir,
session_id=session_id,
recipes_dir=args.recipes_dir,
timeout=args.timeout,
)
coordinator.setup_test()
coordinator.run_test()
coordinator.combine_html()
coordinator.collect_traces()

View file

@ -4,15 +4,17 @@ from pathlib import Path
from loguru import logger
from abratest.dir_manager import DirManager
from abratest.env_manager import EnvFile, EnvManager
from abratest.html_helper import merge_html_files
from abratest.runner import Runner
from abratest.utils import rmtree
from pytest_abra.dir_manager import DirManager
from pytest_abra.env_manager import EnvFile, EnvManager
from pytest_abra.html_helper import merge_html_files
from pytest_abra.runner import Runner
from pytest_abra.utils import rmtree
class Coordinator:
def __init__(self, env_paths_list: list[Path], output_dir: Path, session_id: str, recipes_dir: Path) -> None:
def __init__(
self, env_paths_list: list[Path], output_dir: Path, session_id: str, recipes_dir: Path, timeout: int
) -> None:
# logging
out_string = "".join([e.name + "\n" for e in env_paths_list])
out_string += f"output_dir = {output_dir}\n"
@ -22,6 +24,7 @@ class Coordinator:
self.RUNNER_DICT = self.create_runner_dict(recipes_dir)
self.DIR = DirManager(output_dir=output_dir, session_id=session_id, recipes_dir=recipes_dir)
self.ENV = EnvManager(env_paths_list, self.RUNNER_DICT)
self.TIMEOUT = timeout
def setup_test(self) -> None:
logger.info("calling setup_test()")

1
pytest_abra/demo.py Normal file
View file

@ -0,0 +1 @@
print("wooooorking")

View file

@ -46,7 +46,7 @@ class DirManager:
@property
def SESSION(self):
return self.OUTPUT_DIR / f"test-{self.session_id}"
return self.OUTPUT_DIR / self.session_id
@property
def RECORDS(self):

View file

@ -4,8 +4,8 @@ from typing import NamedTuple
from dotenv import dotenv_values
from abratest.dir_manager import DirManager
from abratest.runner import Runner
from pytest_abra.dir_manager import DirManager
from pytest_abra.runner import Runner
class EnvFile(NamedTuple):

View file

@ -0,0 +1,95 @@
# This file is registered as a pytest plugin, meaning it will automatically loaded.
# All fixtures in this file will be available without manual loading.
import os
import re
from imaplib import IMAP4_SSL
from pathlib import Path
import pytest
from dotenv import dotenv_values
from playwright.sync_api import BrowserContext, expect
from pytest import Parser
from pytest_abra.dir_manager import DirManager
from pytest_abra.env_manager import EnvFile
from pytest_abra.utils import BaseUrl
def pytest_addoption(parser: Parser):
parser.addoption("--runner_index", action="store", type=int)
parser.addoption("--output_dir", action="store", type=Path)
parser.addoption("--session_id", action="store", type=str)
parser.addoption("--timeout", action="store", type=int, default=20_000)
@pytest.fixture
def context(context: BrowserContext, request) -> BrowserContext:
# note: because this has the existing context fixture as an argument, it is ensured
# that the original fixture is called first and then overwritten by this custom one.
TIMEOUT = request.config.getoption("--timeout")
LOCALE = {"Accept-Language": "de_DE"}
context.set_default_timeout(TIMEOUT)
context.set_extra_http_headers(LOCALE)
expect.set_options(timeout=TIMEOUT)
return context
@pytest.fixture(scope="session")
def DIR(request) -> DirManager:
"""Fixture holding test directories
DIR.OUTPUT
DIR.SESSION
DIR.RECORDS
DIR.STATES
DIR.RESULTS"""
output_dir = request.config.getoption("--output_dir")
assert output_dir, "pytest argument --output_dir not set"
session_id = request.config.getoption("--session_id")
assert session_id, "pytest argument --session_id not set"
dirmanager = DirManager(output_dir=output_dir, session_id=session_id)
dirmanager.create_all_dirs()
return dirmanager
@pytest.fixture(scope="session")
def ENV_FILES(DIR: DirManager) -> dict[int, EnvFile]:
out: dict[int, EnvFile] = dict()
for env_path in DIR.ENV_FILES.glob("*.env"):
config: dict[str, str] = dotenv_values(env_path) # type: ignore
env_type = config["TYPE"]
result = re.search(r"(\d+)-*", env_path.name)
assert result
runner_index = int(result[1])
out[runner_index] = EnvFile(env_path=env_path, config=config, env_type=env_type)
return out
@pytest.fixture(scope="session")
def dotenv_config(request, ENV_FILES: dict[int, EnvFile]) -> dict[str, str]:
runner_index = request.config.getoption("--runner_index")
return ENV_FILES[runner_index].config
@pytest.fixture(scope="session")
def URL(dotenv_config: dict[str, str]) -> BaseUrl:
return BaseUrl(netloc=dotenv_config["DOMAIN"])
@pytest.fixture(scope="session")
def imap_ssl_email_client() -> None:
assert os.environ["IMAP_HOST"]
assert os.environ["IMAP_PORT"]
assert os.environ["IMAP_USER"]
assert os.environ["IMAP_PASS"]
port = int(os.environ["IMAP_PORT"])
imap_client = IMAP4_SSL(host=os.environ["IMAP_HOST"], port=port)
imap_client.login(os.environ["IMAP_USER"], os.environ["IMAP_PASS"])
imap_client.select("INBOX")
yield imap_client
imap_client.close()
imap_client.logout()

View file

@ -6,8 +6,8 @@ import pytest
from loguru import logger
if TYPE_CHECKING:
from abratest.coordinator import Coordinator
from abratest.env_manager import EnvFile
from pytest_abra.coordinator import Coordinator
from pytest_abra.env_manager import EnvFile
@dataclass
@ -123,8 +123,8 @@ class Runner:
# command_arguments.append("-rx")
command_arguments.append(str(full_test_path))
command_arguments.append("--env_file")
command_arguments.append(str(self.dotenv_path))
command_arguments.append("--runner_index")
command_arguments.append(str(self.runner_index))
# set root dir for tests output (used in DirManager). this is our custom argument
command_arguments.append("--output_dir")
@ -133,6 +133,9 @@ class Runner:
command_arguments.append("--session_id")
command_arguments.append(self.DIR.session_id)
command_arguments.append("--timeout")
command_arguments.append(str(self.coordinator.TIMEOUT))
# artifacts dir from pytest
# warning: https://github.com/microsoft/playwright-pytest/issues/111
# --output only works with the given context and page fixture

View file

@ -17,7 +17,7 @@ class BaseUrl:
return urlunparse((self.scheme, self.netloc, path, self.params, self.query, self.fragment))
def get_session_id() -> str:
def get_datetime_string() -> str:
current_datetime = datetime.now()
return current_datetime.strftime("%Y-%m-%d-%H-%M-%S")

View file

@ -3,8 +3,8 @@ import json
import pytest
from playwright.sync_api import BrowserContext, Page
from abratest.dir_manager import DirManager
from abratest.utils import BaseUrl
from pytest_abra.dir_manager import DirManager
from pytest_abra.utils import BaseUrl
@pytest.fixture

View file

@ -1,4 +1,4 @@
from abratest.runner import Runner, Test
from pytest_abra.runner import Runner, Test
def condition_always_true(dotenv_config: dict[str, str]) -> bool:

View file

@ -4,8 +4,8 @@ import re
from playwright.sync_api import BrowserContext, expect
from abratest.dir_manager import DirManager
from abratest.utils import BaseUrl
from pytest_abra.dir_manager import DirManager
from pytest_abra.utils import BaseUrl
ADMIN_USER = os.environ["ADMIN_USER"]
ADMIN_PASS = os.environ["ADMIN_PASS"]

View file

@ -5,7 +5,7 @@ depend on [demo]. For this to work
1. the Runner class of the other test needs to define the depencency as seen
by referencing RunnerDemo in the dependencies list:
from abratest.tests_demo.runner_demo import RunnerDemo
from pytest_abra.tests_demo.runner_demo import RunnerDemo
class RunnerOther(Runner):
dependencies = [RunnerDemo]
@ -15,7 +15,7 @@ class RunnerOther(Runner):
To globally import for all tests in 'other', the import should be done in conftest:
in 'conftest.py' in 'test_other' dir:
from abratest.tests_demo.fixtures_demo import demo_fixture
from pytest_abra.tests_demo.fixtures_demo import demo_fixture
"""
import pytest

View file

@ -1,4 +1,4 @@
from abratest.runner import Runner, Test
from pytest_abra.runner import Runner, Test
class RunnerDemo(Runner):

View file

@ -4,8 +4,8 @@ import os
import pytest
from playwright.sync_api import BrowserContext, Page
from abratest.dir_manager import DirManager
from abratest.utils import BaseUrl
from pytest_abra.dir_manager import DirManager
from pytest_abra.utils import BaseUrl
pytest_plugins = "authentik.tests_authentik.fixtures_authentik"

View file

@ -1,4 +1,4 @@
from abratest.runner import Runner, Test
from pytest_abra.runner import Runner, Test
def condition_always_false(dotenv_config: dict[str, str]) -> bool:

View file

@ -1,7 +1,7 @@
from playwright.sync_api import Page, expect
from abratest.dir_manager import DirManager
from abratest.utils import BaseUrl
from pytest_abra.dir_manager import DirManager
from pytest_abra.utils import BaseUrl
# url dashboard
# https://files.test.dev.local-it.cloud/apps/dashboard/

View file

@ -4,7 +4,7 @@ import pytest
from dotenv import dotenv_values
from playwright.sync_api import BrowserContext, Page
from abratest.dir_manager import DirManager
from pytest_abra.dir_manager import DirManager
pytest_plugins = "authentik.tests_authentik.fixtures_authentik"

View file

@ -1,4 +1,4 @@
from abratest.runner import Runner, Test
from pytest_abra.runner import Runner, Test
def condition_always_true(dotenv_config: dict[str, str]) -> bool:

View file

@ -1,7 +1,7 @@
import pytest
from playwright.sync_api import BrowserContext, Page, expect
from abratest.dir_manager import DirManager
from pytest_abra.dir_manager import DirManager
def test_visit_from_domain(authentik_admin_context: BrowserContext, dotenv_config: dict[str, str]):

View file

@ -2,7 +2,7 @@
from playwright.sync_api import BrowserContext, expect
from abratest.dir_manager import DirManager
from pytest_abra.dir_manager import DirManager
def test_welcome_message(context: BrowserContext, dotenv_config: dict[str, str], DIR: DirManager):

View file

@ -6,3 +6,4 @@ icecream
loguru
beautifulsoup4
imbox
hatchling

View file

@ -2,8 +2,8 @@ from pathlib import Path
import pytest
from abratest.coordinator import Coordinator
from abratest.env_manager import DependencyRule, EnvFile, EnvManager
from pytest_abra.coordinator import Coordinator
from pytest_abra.env_manager import DependencyRule, EnvFile, EnvManager
RECIPES_DIR = Path("./recipes").resolve()
RUNNER_DICT = Coordinator.create_runner_dict(RECIPES_DIR)

View file

@ -1,4 +1,4 @@
from abratest.utils import BaseUrl
from pytest_abra.utils import BaseUrl
url_input = {
"netloc": "blog.dev.local-it.cloud",