feat: add playwright and a11y audit and example test

Justfile

@@ -35,8 +35,11 @@ audit:
     mix deps.audit
     mix hex.audit
 
+# first run unit test and after that run e2e test (especially for accessibility)
 test: install-dependencies start-database
-    mix test
+    mix test.unit
+    mix test.e2e
+
 
 format:
     mix format

assets/package-lock.json

@@ -0,0 +1,59 @@
+{
+  "name": "assets",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "devDependencies": {
+        "playwright": "^1.55.0"
+      }
+    },
+    "node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
+    "node_modules/playwright": {
+      "version": "1.55.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.55.0.tgz",
+      "integrity": "sha512-sdCWStblvV1YU909Xqx0DhOjPZE4/5lJsIS84IfN9dAZfcl/CIZ5O8l3o0j7hPMjDvqoTF8ZUcc+i/GL5erstA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.55.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.55.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.55.0.tgz",
+      "integrity": "sha512-GvZs4vU3U5ro2nZpeiwyb0zuFaqb9sUiAJuyrWpcGouD8y9/HLgGbNRjIph7zU9D3hnPaisMl9zG9CgFi/biIg==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    }
+  }
+}

assets/package.json

@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "playwright": "^1.55.0"
+  }
+}

config/test.exs

@@ -19,7 +19,8 @@ config :mv, Mv.Repo,
 config :mv, MvWeb.Endpoint,
   http: [ip: {127, 0, 0, 1}, port: 4002],
   secret_key_base: "Qbc/hcosiQzgfgMMPVs2slKjY2oqiqhpQHsV3twL9dN5GVDzsmsMWC1L/BZAU3Fd",
-  server: false
+  # Set to true for playwright
+  server: true
 
 # In test we don't send emails
 config :mv, Mv.Mailer, adapter: Swoosh.Adapters.Test
@@ -45,3 +46,16 @@ config :mv, :token_signing_secret, "test_secret_key_for_ash_authentication_token
 config :mv, :session_identifier, :unsafe
 
 config :mv, :require_token_presence_for_authentication, false
+
+# Playwright config
+config :phoenix_test,
+  endpoint: MvWeb.Endpoint,
+  otp_app: :mv,
+  playwright: [
+    browser: :firefox, #:chromium
+    headless: System.get_env("PW_HEADLESS", "true") in ~w(t true),
+    js_logger: false,
+    screenshot: System.get_env("PW_SCREENSHOT", "false") in ~w(t true),
+    trace: System.get_env("PW_TRACE", "false") in ~w(t true),
+    browser_launch_timeout: 10_000
+  ]

mix.exs

@@ -11,11 +11,24 @@ defmodule Mv.MixProject do
       consolidate_protocols: Mix.env() != :dev,
       compilers: [:phoenix_live_view] ++ Mix.compilers(),
       aliases: aliases(),
+      preferred_cli_env: preferred_cli_env(),
       deps: deps(),
       listeners: [Phoenix.CodeReloader]
     ]
   end
 
+  # Specifies which environment to be set for which alias / tasks
+  defp preferred_cli_env do
+    [
+      # Standard‑Mix‑Task
+      test: :test,
+
+      # Aliases
+      "test.unit": :test,
+      "test.e2e":  :test,
+    ]
+  end
+
   # Configuration for the OTP application.
   #
   # Type `mix help compile.app` for more information.
@@ -75,6 +88,8 @@ defmodule Mv.MixProject do
       {:mix_audit, "~> 2.1", only: [:dev, :test], runtime: false},
       {:sobelow, "~> 0.14", only: [:dev, :test], runtime: false},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
+      {:phoenix_test_playwright, "~> 0.4", only: :test, runtime: false},
+      {:a11y_audit, "~> 0.2.3", only: :test},
       {:ecto_commons, "~> 0.3"}
     ]
   end
@@ -91,6 +106,8 @@ defmodule Mv.MixProject do
       "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
       "ecto.reset": ["ecto.drop", "ecto.setup"],
       test: ["ash.setup --quiet", "test"],
+      "test.unit": ["ash.setup --quiet","test test/unit"],
+      "test.e2e":  ["test test/e2e"],
       "assets.setup": ["tailwind.install --if-missing", "esbuild.install --if-missing"],
       "assets.build": ["tailwind mv", "esbuild mv"],
       "assets.deploy": [

mix.lock

@@ -1,4 +1,5 @@
 %{
+  "a11y_audit": {:hex, :a11y_audit, "0.2.3", "4a041eeeb9ae87967b30526b8bb9e4a1a3b0136b6e0f1324a029c47cc0938a21", [:mix], [{:hound, "~> 1.1", [hex: :hound, repo: "hexpm", optional: true]}, {:wallaby, "~> 0.30", [hex: :wallaby, repo: "hexpm", optional: true]}], "hexpm", "748ed5dcca3c4a20db2b4a71b3ca2130c66dcacb48cfabf1d70d4fde80eddc27"},
   "ash": {:hex, :ash, "3.5.34", "e79e82dc3e3e66fb54a598eeba5feca2d1c3af6a0e752a3378cbad8d7a47dc6f", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.7", [hex: :ecto, repo: "hexpm", optional: false]}, {:ets, "~> 0.8", [hex: :ets, repo: "hexpm", optional: false]}, {:igniter, ">= 0.6.4 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:picosat_elixir, "~> 0.2", [hex: :picosat_elixir, repo: "hexpm", optional: true]}, {:plug, ">= 0.0.0", [hex: :plug, repo: "hexpm", optional: true]}, {:reactor, "~> 0.11", [hex: :reactor, repo: "hexpm", optional: false]}, {:simple_sat, ">= 0.1.1 and < 1.0.0-0", [hex: :simple_sat, repo: "hexpm", optional: true]}, {:spark, ">= 2.2.65 and < 3.0.0-0", [hex: :spark, repo: "hexpm", optional: false]}, {:splode, ">= 0.2.6 and < 1.0.0-0", [hex: :splode, repo: "hexpm", optional: false]}, {:stream_data, "~> 1.0", [hex: :stream_data, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5cbf0a4d0ec1b6525b0782e4f5509c55dad446d657c635ceffe55f78a59132cd"},
   "ash_admin": {:hex, :ash_admin, "0.13.16", "6b30487e88b0a47b2da1c508b157be6d86b954ba464a01d412e6d5e047a53ad5", [:mix], [{:ash, ">= 3.4.63 and < 4.0.0-0", [hex: :ash, repo: "hexpm", optional: false]}, {:ash_phoenix, ">= 2.1.8 and < 3.0.0-0", [hex: :ash_phoenix, repo: "hexpm", optional: false]}, {:gettext, "~> 0.26", [hex: :gettext, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.7", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 1.1-rc", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}], "hexpm", "07a03d761b2029d8b1fefad815eb3cc525532ae9d440e7ca3f5c9f4c1ecb5d17"},
   "ash_authentication": {:hex, :ash_authentication, "4.9.9", "23ec61bedc3157c258ece622c6f0f6a7645df275ff5e794d513cc6e8798471eb", [:mix], [{:argon2_elixir, "~> 4.0", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:ash, ">= 3.4.29 and < 4.0.0-0", [hex: :ash, repo: "hexpm", optional: false]}, {:ash_postgres, ">= 2.6.8 and < 3.0.0-0", [hex: :ash_postgres, repo: "hexpm", optional: true]}, {:assent, "~> 0.2.13", [hex: :assent, repo: "hexpm", optional: false]}, {:bcrypt_elixir, "~> 3.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: false]}, {:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:finch, "~> 0.19", [hex: :finch, repo: "hexpm", optional: false]}, {:igniter, "~> 0.4", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:joken, "~> 2.5", [hex: :joken, repo: "hexpm", optional: false]}, {:plug, "~> 1.13", [hex: :plug, repo: "hexpm", optional: false]}, {:spark, "~> 2.0", [hex: :spark, repo: "hexpm", optional: false]}, {:splode, "~> 0.2", [hex: :splode, repo: "hexpm", optional: false]}], "hexpm", "ab8bd1277ff570425346dcf22dd14a059d9bbce0c28d24964b60e51fabaddda8"},
@@ -29,6 +30,7 @@
   "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"},
   "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"},
   "fine": {:hex, :fine, "0.1.4", "b19a89c1476c7c57afb5f9314aed5960b5bc95d5277de4cb5ee8e1d1616ce379", [:mix], [], "hexpm", "be3324cc454a42d80951cf6023b9954e9ff27c6daa255483b3e8d608670303f5"},
+  "floki": {:hex, :floki, "0.38.0", "62b642386fa3f2f90713f6e231da0fa3256e41ef1089f83b6ceac7a3fd3abf33", [:mix], [], "hexpm", "a5943ee91e93fb2d635b612caf5508e36d37548e84928463ef9dd986f0d1abd9"},
   "gettext": {:hex, :gettext, "0.26.2", "5978aa7b21fada6deabf1f6341ddba50bc69c999e812211903b169799208f2a8", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "aa978504bcf76511efdc22d580ba08e2279caab1066b76bb9aa81c4a1e0a32a5"},
   "glob_ex": {:hex, :glob_ex, "0.1.11", "cb50d3f1ef53f6ca04d6252c7fde09fd7a1cf63387714fe96f340a1349e62c93", [:mix], [], "hexpm", "342729363056e3145e61766b416769984c329e4378f1d558b63e341020525de4"},
   "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "0435d4ca364a608cc75e2f8683d374e55abbae26", [tag: "v2.2.0", sparse: "optimized", depth: 1]},
@@ -57,6 +59,8 @@
   "phoenix_live_view": {:hex, :phoenix_live_view, "1.1.8", "d283d5e047e6c013182a3833e99ff33942e3a8076f9f984c337ea04cc53e8206", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0-rc", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6184cf1e82fe6627d40cfa62236133099438513710d30358f4c085c16ecb84b4"},
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},
   "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"},
+  "phoenix_test": {:hex, :phoenix_test, "0.7.1", "0de2b8a7b7cb5ca3bf422211eb544b15cef1ed7c62ac9fb6806a304cee2624a7", [:mix], [{:floki, ">= 0.30.0", [hex: :floki, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, ">= 1.0.0", [hex: :mime, repo: "hexpm", optional: true]}, {:phoenix, ">= 1.7.10", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "e5de115d48f22af9d9e9a31ffcf063407adf07163a20abb02d61180975f46622"},
+  "phoenix_test_playwright": {:hex, :phoenix_test_playwright, "0.7.1", "20992d444992f94e5b824d388cb9d849c6e42246a7568b604f8df40854b9e17d", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.7", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_ecto, "~> 4.5", [hex: :phoenix_ecto, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:phoenix_test, "~> 0.6", [hex: :phoenix_test, repo: "hexpm", optional: false]}], "hexpm", "90266ce5b5d72a244e9a3bd00462e2450fcb0499f683b2ea0e59223e586fb6de"},
   "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"},
   "plug": {:hex, :plug, "1.18.1", "5067f26f7745b7e31bc3368bc1a2b818b9779faa959b49c934c17730efc911cf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "57a57db70df2b422b564437d2d33cf8d33cd16339c1edb190cd11b1a3a546cc2"},
   "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"},

test-results/.last-run.json

@@ -0,0 +1,4 @@
+{
+  "status": "failed",
+  "failedTests": []
+}

test/e2e/a11y/a11y_test.exs

@@ -0,0 +1,23 @@
+defmodule MvWeb.A11yTest do
+  use PhoenixTest.Playwright.Case, async: true
+
+  alias PhoenixTest.Playwright.Frame
+
+  test "is accessible", %{conn: conn} do
+    conn = conn_with_oidc_user(conn)
+    conn = Plug.Test.init_test_session(conn, locale: "de")
+    {:ok, _view, html} = live(conn, "/members")
+    conn
+    #|> visit("/members")
+    |> unwrap(&assert_a11y/1)
+  end
+
+  defp assert_a11y(%{frame_id: frame_id}) do
+    Frame.evaluate(frame_id, A11yAudit.JS.axe_core())
+
+    frame_id
+    |> Frame.evaluate("axe.run()")
+    |> A11yAudit.Results.from_json()
+    |> A11yAudit.Assertions.assert_no_violations()
+  end
+end

test/test_helper.exs

@@ -1,2 +1,3 @@
 ExUnit.start()
 Ecto.Adapters.SQL.Sandbox.mode(Mv.Repo, :manual)
+Application.put_env(:phoenix_test, :base_url, MvWeb.Endpoint.url())

test/unit/mv_web/member_live/index_test.exs

@@ -2,6 +2,7 @@ defmodule MvWeb.MemberLive.IndexTest do
   use MvWeb.ConnCase, async: true
   import Phoenix.LiveViewTest
 
+
   test "shows translated title in German", %{conn: conn} do
     conn = conn_with_oidc_user(conn)
     conn = Plug.Test.init_test_session(conn, locale: "de")