From a2f81aa1901cb8e40a38102897645a0c4e73d202 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:48:21 +0100 Subject: [PATCH 1/8] rename html merge function --- pytest_abra/coordinator.py | 4 ++-- pytest_abra/html_helper.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 86cecdf..70706e8 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -7,7 +7,7 @@ from loguru import logger from pytest_abra.dir_manager import DirManager from pytest_abra.env_manager import EnvFile, EnvManager -from pytest_abra.html_helper import merge_html_files +from pytest_abra.html_helper import merge_html_reports from pytest_abra.runner import Runner from pytest_abra.utils import rmtree @@ -61,7 +61,7 @@ class Coordinator: in_path = str(self.DIR.RECORDS / "html") out_path = str(self.DIR.RECORDS / "full-report.html") title = "combined.html" - merge_html_files(in_path, out_path, title) + merge_html_reports(in_path, out_path, title) def collect_traces(self): """moves all traces into SESSION/RECORDS dir diff --git a/pytest_abra/html_helper.py b/pytest_abra/html_helper.py index 6e81c7e..6080b2f 100644 --- a/pytest_abra/html_helper.py +++ b/pytest_abra/html_helper.py @@ -12,7 +12,7 @@ from packaging import version CHECKBOX_REGEX = r"^(?P0|[1-9]\d*) (?P.*)" -def merge_html_files(in_path: str, out_path: str, title: str): +def merge_html_reports(in_path: str, out_path: str, title: str): paths = get_html_files(in_path, out_path) if not paths: raise RuntimeError(f"Unable to find html files in {in_path}") -- 2.47.2 From 7519014416fc5f7c8d84118d377deb542bbec49d Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:55:32 +0100 Subject: [PATCH 2/8] improve logging messages --- pytest_abra/runner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pytest_abra/runner.py b/pytest_abra/runner.py index cb86d66..3d75562 100644 --- a/pytest_abra/runner.py +++ b/pytest_abra/runner.py @@ -57,7 +57,7 @@ class Runner: """runs the main test script and if available and sub test scripts if their running condition is met""" # check if required dependencies have passed if not self._dependencies_passed(): - logger.warning(f"skipping run_tests() of {self.env_type}, because some dependencies have not passed") + logger.warning(f"skipping run_tests() of {self.env_type} (one or more dependencies have not passed)") return for test in test_list: @@ -79,16 +79,16 @@ class Runner: # check if test aleady passed if self._is_test_passed(identifier_string, remove_existing=True): if test.prevent_skip: - logger.info(f"continuing , test {identifier_string} has passed but prevent_skip=True") + logger.info(f"continuing {identifier_string} (passed before but prevent_skip=True)") else: - logger.info(f"skipping {identifier_string}, test has passed") + logger.info(f"skipping {identifier_string} (test has passed)") return if test.condition: condition_result = self._run_condition(test.condition) if not condition_result: # test condition is defined but not met - logger.info(f"skipping {identifier_string}, test condition is not met") + logger.info(f"skipping {identifier_string} (test condition is not met)") return # test condition is undefined or not met @@ -126,7 +126,7 @@ class Runner: return already_passed def _call_pytest(self, full_test_path: Path) -> int: - """runs pytest programmatically on a specific file + """runs pytest programmatically with a specific file all tests in the file [full_test_path] will be run along with command line arguments""" -- 2.47.2 From ad8f3a389119733ae10ac8c20b09f6813cf91ed3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 12:58:02 +0100 Subject: [PATCH 3/8] add test assets --- ....py__test_should_create_bug_report_0_0.txt | 33 + tests/assets/html_merge/assets/style.css | 319 ++++++++ tests/assets/html_merge/setup_wordpress.html | 770 ++++++++++++++++++ .../test_authentik_blueprint_api.html | 770 ++++++++++++++++++ .../test_wordpress_receive_email.html | 770 ++++++++++++++++++ 5 files changed, 2662 insertions(+) create mode 100644 tests/assets/html_merge/assets/recipes_authentik_tests_authentik_test_authentik_blueprint_api.py__test_should_create_bug_report_0_0.txt create mode 100644 tests/assets/html_merge/assets/style.css create mode 100644 tests/assets/html_merge/setup_wordpress.html create mode 100644 tests/assets/html_merge/test_authentik_blueprint_api.html create mode 100644 tests/assets/html_merge/test_wordpress_receive_email.html diff --git a/tests/assets/html_merge/assets/recipes_authentik_tests_authentik_test_authentik_blueprint_api.py__test_should_create_bug_report_0_0.txt b/tests/assets/html_merge/assets/recipes_authentik_tests_authentik_test_authentik_blueprint_api.py__test_should_create_bug_report_0_0.txt new file mode 100644 index 0000000..eba0ff3 --- /dev/null +++ b/tests/assets/html_merge/assets/recipes_authentik_tests_authentik_test_authentik_blueprint_api.py__test_should_create_bug_report_0_0.txt @@ -0,0 +1,33 @@ +name enabled status +-------------------------------------------------------- --------- ---------- +Custom Invalidation Flow True successful +System - SCIM Provider - Mappings True successful +System - OAuth2 Provider - Scopes True successful +System - SAML Provider - Mappings True successful +System - LDAP Source - Mappings True successful +Migration - Remove old prompt fields True successful +Default - Events Transport & Rules True successful +Default - Source pre-authentication flow True successful +Default - TOTP MFA setup flow True successful +Default - WebAuthn MFA setup flow True successful +Default - Provider authorization flow (explicit consent) True failed +Default - Source authentication flow True successful +Default - Provider authorization flow (implicit consent) True successful +Default - Static MFA setup flow True successful +matrix True successful +Custom System Tenant True successful +Nextcloud True successful +Wordpress True successful +Custom Authentication Flow True successful +wekan True successful +Default - Invalidation flow True successful +Default - Tenant True successful +Flow Translations True successful +Default - User settings flow False successful +Default - Source enrollment flow False successful +Invitation Enrollment Flow True successful +vikunja True successful +Default - Password change flow False successful +Default - Authentication flow False successful +Recovery with email verification True successful +System - Proxy Provider - Scopes True successful \ No newline at end of file diff --git a/tests/assets/html_merge/assets/style.css b/tests/assets/html_merge/assets/style.css new file mode 100644 index 0000000..561524c --- /dev/null +++ b/tests/assets/html_merge/assets/style.css @@ -0,0 +1,319 @@ +body { + font-family: Helvetica, Arial, sans-serif; + font-size: 12px; + /* do not increase min-width as some may use split screens */ + min-width: 800px; + color: #999; +} + +h1 { + font-size: 24px; + color: black; +} + +h2 { + font-size: 16px; + color: black; +} + +p { + color: black; +} + +a { + color: #999; +} + +table { + border-collapse: collapse; +} + +/****************************** + * SUMMARY INFORMATION + ******************************/ +#environment td { + padding: 5px; + border: 1px solid #e6e6e6; + vertical-align: top; +} +#environment tr:nth-child(odd) { + background-color: #f6f6f6; +} +#environment ul { + margin: 0; + padding: 0 20px; +} + +/****************************** + * TEST RESULT COLORS + ******************************/ +span.passed, +.passed .col-result { + color: green; +} + +span.skipped, +span.xfailed, +span.rerun, +.skipped .col-result, +.xfailed .col-result, +.rerun .col-result { + color: orange; +} + +span.error, +span.failed, +span.xpassed, +.error .col-result, +.failed .col-result, +.xpassed .col-result { + color: red; +} + +.col-links__extra { + margin-right: 3px; +} + +/****************************** + * RESULTS TABLE + * + * 1. Table Layout + * 2. Extra + * 3. Sorting items + * + ******************************/ +/*------------------ + * 1. Table Layout + *------------------*/ +#results-table { + border: 1px solid #e6e6e6; + color: #999; + font-size: 12px; + width: 100%; +} +#results-table th, +#results-table td { + padding: 5px; + border: 1px solid #e6e6e6; + text-align: left; +} +#results-table th { + font-weight: bold; +} + +/*------------------ + * 2. Extra + *------------------*/ +.logwrapper { + max-height: 230px; + overflow-y: scroll; + background-color: #e6e6e6; +} +.logwrapper.expanded { + max-height: none; +} +.logwrapper.expanded .logexpander:after { + content: "collapse [-]"; +} +.logwrapper .logexpander { + z-index: 1; + position: sticky; + top: 10px; + width: max-content; + border: 1px solid; + border-radius: 3px; + padding: 5px 7px; + margin: 10px 0 10px calc(100% - 80px); + cursor: pointer; + background-color: #e6e6e6; +} +.logwrapper .logexpander:after { + content: "expand [+]"; +} +.logwrapper .logexpander:hover { + color: #000; + border-color: #000; +} +.logwrapper .log { + min-height: 40px; + position: relative; + top: -50px; + height: calc(100% + 50px); + border: 1px solid #e6e6e6; + color: black; + display: block; + font-family: "Courier New", Courier, monospace; + padding: 5px; + padding-right: 80px; + white-space: pre-wrap; +} + +div.media { + border: 1px solid #e6e6e6; + float: right; + height: 240px; + margin: 0 5px; + overflow: hidden; + width: 320px; +} + +.media-container { + display: grid; + grid-template-columns: 25px auto 25px; + align-items: center; + flex: 1 1; + overflow: hidden; + height: 200px; +} + +.media-container--fullscreen { + grid-template-columns: 0px auto 0px; +} + +.media-container__nav--right, +.media-container__nav--left { + text-align: center; + cursor: pointer; +} + +.media-container__viewport { + cursor: pointer; + text-align: center; + height: inherit; +} +.media-container__viewport img, +.media-container__viewport video { + object-fit: cover; + width: 100%; + max-height: 100%; +} + +.media__name, +.media__counter { + display: flex; + flex-direction: row; + justify-content: space-around; + flex: 0 0 25px; + align-items: center; +} + +.collapsible td:not(.col-links) { + cursor: pointer; +} +.collapsible td:not(.col-links):hover::after { + color: #bbb; + font-style: italic; + cursor: pointer; +} + +.col-result { + width: 130px; +} +.col-result:hover::after { + content: " (hide details)"; +} + +.col-result.collapsed:hover::after { + content: " (show details)"; +} + +#environment-header h2:hover::after { + content: " (hide details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; +} + +#environment-header.collapsed h2:hover::after { + content: " (show details)"; + color: #bbb; + font-style: italic; + cursor: pointer; + font-size: 12px; +} + +/*------------------ + * 3. Sorting items + *------------------*/ +.sortable { + cursor: pointer; +} +.sortable.desc:after { + content: " "; + position: relative; + left: 5px; + bottom: -12.5px; + border: 10px solid #4caf50; + border-bottom: 0; + border-left-color: transparent; + border-right-color: transparent; +} +.sortable.asc:after { + content: " "; + position: relative; + left: 5px; + bottom: 12.5px; + border: 10px solid #4caf50; + border-top: 0; + border-left-color: transparent; + border-right-color: transparent; +} + +.hidden, .summary__reload__button.hidden { + display: none; +} + +.summary__data { + flex: 0 0 550px; +} +.summary__reload { + flex: 1 1; + display: flex; + justify-content: center; +} +.summary__reload__button { + flex: 0 0 300px; + display: flex; + color: white; + font-weight: bold; + background-color: #4caf50; + text-align: center; + justify-content: center; + align-items: center; + border-radius: 3px; + cursor: pointer; +} +.summary__reload__button:hover { + background-color: #46a049; +} +.summary__spacer { + flex: 0 0 550px; +} + +.controls { + display: flex; + justify-content: space-between; +} + +.filters, +.collapse { + display: flex; + align-items: center; +} +.filters button, +.collapse button { + color: #999; + border: none; + background: none; + cursor: pointer; + text-decoration: underline; +} +.filters button:hover, +.collapse button:hover { + color: #ccc; +} + +.filter__label { + margin-right: 10px; +} diff --git a/tests/assets/html_merge/setup_wordpress.html b/tests/assets/html_merge/setup_wordpress.html new file mode 100644 index 0000000..43180e8 --- /dev/null +++ b/tests/assets/html_merge/setup_wordpress.html @@ -0,0 +1,770 @@ + + + + + setup_wordpress.html + + + +

setup_wordpress.html

+

Report generated on 08-Dec-2023 at 14:55:57 by pytest-html + v4.1.1

+
+

Environment

+
+
+ + + + + +
+
+

Summary

+
+
+

2 tests took 00:00:11.

+

(Un)check the boxes to filter the results.

+
+ +
+
+
+
+ + 0 Failed, + + 2 Passed, + + 0 Skipped, + + 0 Expected failures, + + 0 Unexpected passes, + + 0 Errors, + + 0 Reruns +
+
+  /  +
+
+
+
+
+
+
+
+ + + + + + + + + +
ResultTestDurationLinks
+ +
+
+ +
+ \ No newline at end of file diff --git a/tests/assets/html_merge/test_authentik_blueprint_api.html b/tests/assets/html_merge/test_authentik_blueprint_api.html new file mode 100644 index 0000000..37897c3 --- /dev/null +++ b/tests/assets/html_merge/test_authentik_blueprint_api.html @@ -0,0 +1,770 @@ + + + + + test_authentik_blueprint_api.html + + + +

test_authentik_blueprint_api.html

+

Report generated on 09-Dec-2023 at 12:22:45 by pytest-html + v4.1.1

+
+

Environment

+
+
+ + + + + +
+
+

Summary

+
+
+

1 test took 00:00:01.

+

(Un)check the boxes to filter the results.

+
+ +
+
+
+
+ + 1 Failed, + + 0 Passed, + + 0 Skipped, + + 0 Expected failures, + + 0 Unexpected passes, + + 0 Errors, + + 0 Reruns +
+
+  /  +
+
+
+
+
+
+
+
+ + + + + + + + + +
ResultTestDurationLinks
+ +
+
+ +
+ \ No newline at end of file diff --git a/tests/assets/html_merge/test_wordpress_receive_email.html b/tests/assets/html_merge/test_wordpress_receive_email.html new file mode 100644 index 0000000..b29f75a --- /dev/null +++ b/tests/assets/html_merge/test_wordpress_receive_email.html @@ -0,0 +1,770 @@ + + + + + test_wordpress_receive_email.html + + + +

test_wordpress_receive_email.html

+

Report generated on 08-Dec-2023 at 16:00:41 by pytest-html + v4.1.1

+
+

Environment

+
+
+ + + + + +
+
+

Summary

+
+
+

1 test took 946 ms.

+

(Un)check the boxes to filter the results.

+
+ +
+
+
+
+ + 1 Failed, + + 0 Passed, + + 0 Skipped, + + 0 Expected failures, + + 0 Unexpected passes, + + 0 Errors, + + 0 Reruns +
+
+  /  +
+
+
+
+
+
+
+
+ + + + + + + + + +
ResultTestDurationLinks
+ +
+
+ +
+ \ No newline at end of file -- 2.47.2 From cb6b1661314bdb8e58d37cdd6558b4f050441f09 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 14:45:56 +0100 Subject: [PATCH 4/8] add custom_copy_assets --- pytest_abra/html_helper.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/pytest_abra/html_helper.py b/pytest_abra/html_helper.py index 6080b2f..6215350 100644 --- a/pytest_abra/html_helper.py +++ b/pytest_abra/html_helper.py @@ -5,19 +5,36 @@ import json import os import pathlib import re +import shutil -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup # type: ignore from packaging import version CHECKBOX_REGEX = r"^(?P0|[1-9]\d*) (?P.*)" -def merge_html_reports(in_path: str, out_path: str, title: str): - paths = get_html_files(in_path, out_path) - if not paths: - raise RuntimeError(f"Unable to find html files in {in_path}") +def custom_copy_assets(assets_dir_path: str, out_file_path: str): + """custom function added for pytest_abra - assets_dir_path = get_assets_path(in_path) + copies every asset to asset folder. Exclude style.css as this is already handled by pytest_html_merger""" + + assets_source_dir = pathlib.Path(assets_dir_path) + assets_source_files = [p for p in assets_source_dir.glob("*") if p.is_file() and p.name != "style.css"] + out_dir_path = pathlib.Path(out_file_path).parent + assets_target_dir = out_dir_path / "assets" + assets_target_dir.mkdir(exist_ok=True) + for asset in assets_source_files: + shutil.copy(asset, assets_target_dir / asset.name) + + +def merge_html_reports(in_dir_path: str, out_file_path: str, report_title: str): + paths = get_html_files(in_dir_path, out_file_path) + if not paths: + raise RuntimeError(f"Unable to find html files in {in_dir_path}") + + assets_dir_path = get_assets_path(in_dir_path) + + custom_copy_assets(assets_dir_path, out_file_path) first_file = BeautifulSoup("".join(open(paths[0])), features="html.parser") paths.pop(0) @@ -30,7 +47,7 @@ def merge_html_reports(in_path: str, out_path: str, title: str): if assets_dir_path is None: print( f"Will assume css is embedded in the reports. If this is not the case, " - f"Please make sure that you have 'assets' directory inside {in_path} " + f"Please make sure that you have 'assets' directory inside {in_dir_path} " f"which contains css files generated by pytest-html." ) else: @@ -42,7 +59,7 @@ def merge_html_reports(in_path: str, out_path: str, title: str): head.style.append(content) h = first_file.find("h1") - h.string = title or os.path.basename(out_path) + h.string = report_title or os.path.basename(out_file_path) ps = first_file.find_all("p") pytest_version = ps[0].text.split(" ")[-1] @@ -106,7 +123,7 @@ def merge_html_reports(in_path: str, out_path: str, title: str): for cb_type in cb_types: set_checkbox_value(first_file, cb_type, cb_types[cb_type]) - with open(out_path, "w") as f: + with open(out_file_path, "w") as f: f.write(str(first_file)) -- 2.47.2 From 6ee6055a301482f359b65154dce2cdb06cb5f23f Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 15:17:52 +0100 Subject: [PATCH 5/8] update interface of merge_html_reports --- pytest_abra/coordinator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest_abra/coordinator.py b/pytest_abra/coordinator.py index 70706e8..52f5e77 100644 --- a/pytest_abra/coordinator.py +++ b/pytest_abra/coordinator.py @@ -58,10 +58,10 @@ class Coordinator: def combine_html(self) -> None: """combines all generated pytest html reports into one""" - in_path = str(self.DIR.RECORDS / "html") - out_path = str(self.DIR.RECORDS / "full-report.html") + in_dir_path = str(self.DIR.RECORDS / "html") + out_file_path = str(self.DIR.RECORDS / "full-report.html") title = "combined.html" - merge_html_reports(in_path, out_path, title) + merge_html_reports(in_dir_path, out_file_path, title) def collect_traces(self): """moves all traces into SESSION/RECORDS dir -- 2.47.2 From c1ebcf84c2cb93fa75d907a34fe4353139649d9e Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 15:18:04 +0100 Subject: [PATCH 6/8] add testing for merge_html_reports --- tests/test_html_merge.py | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/test_html_merge.py diff --git a/tests/test_html_merge.py b/tests/test_html_merge.py new file mode 100644 index 0000000..b453e8b --- /dev/null +++ b/tests/test_html_merge.py @@ -0,0 +1,60 @@ +# tmp_path fixture: +# https://docs.pytest.org/en/6.2.x/tmpdir.html + +import os +from pathlib import Path + +import pytest +from icecream import ic # type: ignore +from playwright.sync_api import BrowserContext, expect + +from pytest_abra import BaseUrl +from pytest_abra.html_helper import merge_html_reports + + +@pytest.fixture(scope="session") +def session_tmp_path(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("html_test") + + +def test_merge_html(session_tmp_path: Path): + """combines all generated pytest html reports into one""" + + in_dir_path = Path(__file__).parent / "assets" / "html_merge" + in_dir_path = in_dir_path.resolve() + ic(in_dir_path) + + out_file_path = session_tmp_path / "test.html" + out_assets_dir = session_tmp_path / "assets" + + merge_html_reports(in_dir_path.as_posix(), out_file_path.as_posix(), "combined.html") + + assert out_file_path.is_file() + assert out_assets_dir.is_dir() + assert next(out_assets_dir.glob("*")) + + +def test_check_result_with_playwright(session_tmp_path, context: BrowserContext): + html_file = session_tmp_path / "test.html" + file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() + page = context.new_page() + page.goto(file_url) + + # check if combined is correct + expect(page.get_by_text("2 Passed,")).to_be_visible() + expect(page.get_by_text("2 Failed,")).to_be_visible() + expect(page.get_by_text("tests ran in 12.946 seconds")).to_be_visible() + + # check if heading is correct + expect(page.get_by_role("heading", name="combined.html")).to_be_visible() + + # check if traceback is included + expect(page.get_by_text("E AssertionError: One or more")).to_be_visible() + + # check if asset works + with page.expect_popup() as page1_info: + page.get_by_role("link", name="Authentik Blueprint Status").click() + page1 = page1_info.value + + # see if content of txt file is correct + expect(page1.get_by_text("failed")).to_be_visible() -- 2.47.2 From b041145c03ca56841067301b8fe51f67b519c007 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 15:26:24 +0100 Subject: [PATCH 7/8] add slow marker --- pyproject.toml | 5 ++++- tests/test_html_merge.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a63e9dd..d436714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,4 +52,7 @@ target-version = "py311" [tool.pytest.ini_options] python_functions = "setup_* test_* cleanup_*" norecursedirs = ".* previous-work recipes" -testpaths = "tests" \ No newline at end of file +testpaths = "tests" +markers = [ + "slow: marks tests as slow", +] \ No newline at end of file diff --git a/tests/test_html_merge.py b/tests/test_html_merge.py index b453e8b..d97ab83 100644 --- a/tests/test_html_merge.py +++ b/tests/test_html_merge.py @@ -1,7 +1,6 @@ # tmp_path fixture: # https://docs.pytest.org/en/6.2.x/tmpdir.html -import os from pathlib import Path import pytest @@ -34,6 +33,7 @@ def test_merge_html(session_tmp_path: Path): assert next(out_assets_dir.glob("*")) +@pytest.mark.slow def test_check_result_with_playwright(session_tmp_path, context: BrowserContext): html_file = session_tmp_path / "test.html" file_url = BaseUrl(netloc=html_file.as_posix(), scheme="file").get() -- 2.47.2 From 38504d63d752c022df6561aa83161b8dea767ae3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 9 Dec 2023 15:27:02 +0100 Subject: [PATCH 8/8] add pytest -m "not slow" to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 159e16e..7fb9702 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ playwright codegen demo.playwright.dev/todomvc # visit given url in codegen mod ```bash pytest # test pytest-abra +pytest -m "not slow" # test pytest-abra without slow tests pytest --collect-only # debug test pytest-abra docker compose run --rm app pytest # run pytest-abra ``` -- 2.47.2