diff --git a/.drone.yml b/.drone.yml index e207ca0..ebc74e7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -83,8 +83,8 @@ steps: - mix local.hex --force # Fetch dependencies - mix deps.get - # Run tests - - mix test + # Run fast tests (excludes slow/performance tests) + - mix test --exclude slow - name: rebuild-cache image: drillster/drone-volume-cache @@ -178,3 +178,52 @@ steps: - unset GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL - renovate-config-validator - renovate + +--- +kind: pipeline +type: docker +name: nightly-tests + +trigger: + event: + - cron + cron: + - "0 2 * * *" # Run at 2 AM daily + +services: + - name: postgres + image: docker.io/library/postgres:18.1 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + +steps: + - name: wait_for_postgres + image: docker.io/library/postgres:18.1 + commands: + # Wait for postgres to become available + - | + for i in {1..20}; do + if pg_isready -h postgres -U postgres; then + exit 0 + else + true + fi + sleep 2 + done + echo "Postgres did not become available, aborting." + exit 1 + + - name: test-slow + image: docker.io/library/elixir:1.18.3-otp-27 + environment: + MIX_ENV: test + TEST_POSTGRES_HOST: postgres + TEST_POSTGRES_PORT: 5432 + commands: + # Install hex package manager + - mix local.hex --force + # Fetch dependencies + - mix deps.get + # Run only all tests + - mix test diff --git a/CODE_GUIDELINES.md b/CODE_GUIDELINES.md index 28c454b..b11a369 100644 --- a/CODE_GUIDELINES.md +++ b/CODE_GUIDELINES.md @@ -1654,6 +1654,28 @@ end - Mock external services - Use fixtures efficiently +**Performance Tests:** + +Performance tests that explicitly validate performance characteristics should be tagged with `@tag :slow` or `@moduletag :slow` to exclude them from standard test runs. This improves developer feedback loops while maintaining comprehensive coverage. + +**When to Tag as `:slow`:** + +- Tests that explicitly measure execution time or validate performance characteristics +- Tests that use large datasets (e.g., 50+ records) to test scalability +- Tests that validate query optimization (N+1 prevention, index usage) + +**Running Tests:** + +```bash +# Fast tests only (default) +mix test --exclude slow + +# Performance tests only +mix test --only slow + +# All tests +mix test +``` --- ## 5. Security Guidelines diff --git a/Justfile b/Justfile index f25041c..6337d7c 100644 --- a/Justfile +++ b/Justfile @@ -22,7 +22,7 @@ seed-database: start-database: docker compose up -d -ci-dev: lint audit test +ci-dev: lint audit test-fast gettext: mix gettext.extract @@ -44,6 +44,18 @@ audit: test *args: install-dependencies mix test {{args}} +# Run only fast tests (excludes slow/performance tests) +test-fast *args: install-dependencies + mix test --exclude slow {{args}} + +# Run only slow/performance tests +test-slow *args: install-dependencies + mix test --only slow {{args}} + +# Run all tests (fast + slow) +test-all *args: install-dependencies + mix test {{args}} + format: mix format diff --git a/docs/test-optimization-summary.md b/docs/test-optimization-summary.md index ae50081..001a235 100644 --- a/docs/test-optimization-summary.md +++ b/docs/test-optimization-summary.md @@ -116,8 +116,67 @@ Based on the initial analysis (`mix test --slowest 20`), the following test file 1. ✅ **Done:** Optimize seeds tests (this document) 2. ⏳ **Next:** Review and optimize LiveView tests with large data setups 3. ⏳ **Next:** Implement shared fixtures for policy tests -4. ⏳ **Next:** Tag performance tests as `:slow` for conditional execution -5. ⏳ **Future:** Consider nightly integration test suite for comprehensive coverage +4. ✅ **Done:** Tag performance tests as `:slow` for conditional execution +5. ✅ **Done:** Nightly integration test suite for comprehensive coverage + +--- + +## Slow Test Suite + +Performance tests have been tagged with `@tag :slow` and separated into a dedicated test suite. + +### Structure + +- **Performance Tests:** Explicit tests that validate performance characteristics (e.g., N+1 query prevention, filter performance with large datasets) +- **Tagging:** All performance tests are tagged with `@tag :slow` or `@moduletag :slow` +- **Execution:** Standard test runs exclude slow tests, but they can be executed on demand + +### Execution + +**Fast Tests (Default):** +```bash +just test-fast +# or +mix test --exclude slow + +# With specific files or options +just test-fast test/membership/member_test.exs +just test-fast --seed 123 +``` + +**Performance Tests Only:** +```bash +just test-slow +# or +mix test --only slow + +# With specific files or options +just test-slow test/mv_web/member_live/index_test.exs +just test-slow --seed 123 +``` + +**All Tests:** +```bash +just test +# or +mix test + +# With specific files or options +just test-all test/mv_web/ +just test-all --max-failures 5 +``` + +**Note:** All suite commands (`test-fast`, `test-slow`, `test-all`) support additional arguments. The suite semantics (tags) are always preserved - additional arguments are appended to the command. + +### CI/CD Integration + +- **Standard CI:** Runs `mix test --exclude slow` for faster feedback loops +- **Nightly Builds:** Separate pipeline (`nightly-tests`) runs daily at 2 AM and executes all performance tests +- **Manual Execution:** Performance tests can be executed anytime with `just test-slow` + +### Further Details + +See [test-slow-suite.md](test-slow-suite.md) for complete documentation of the Slow Test Suite. --- @@ -148,6 +207,7 @@ Based on the initial analysis (`mix test --slowest 20`), the following test file ## References - **Detailed Documentation:** `docs/test-optimization-seeds.md` +- **Slow Test Suite:** `docs/test-slow-suite.md` - **Test File:** `test/seeds_test.exs` - **Original Analysis:** Internal benchmarking session (2026-01-28) - **Related Guidelines:** `CODE_GUIDELINES.md` - Section 4 (Testing Standards) diff --git a/docs/test-slow-suite.md b/docs/test-slow-suite.md new file mode 100644 index 0000000..e4cfcc2 --- /dev/null +++ b/docs/test-slow-suite.md @@ -0,0 +1,280 @@ +# Slow Test Suite Documentation + +**Date:** 2026-01-28 +**Status:** ✅ Active + +--- + +## Overview + +The Slow Test Suite contains performance tests that explicitly validate performance characteristics of the application. These tests are intentionally slower because they use larger datasets or test performance-critical paths (e.g., N+1 query prevention, filter performance with many records). + +These tests are marked with `@tag :slow` or `@moduletag :slow` and are excluded from standard test runs to improve developer feedback loops while maintaining comprehensive coverage. + +--- + +## Purpose + +Performance tests serve to: + +1. **Validate Performance Characteristics:** Ensure queries and operations perform within acceptable time limits +2. **Prevent Regressions:** Catch performance regressions before they reach production +3. **Test Scalability:** Verify that the application handles larger datasets efficiently +4. **N+1 Query Prevention:** Ensure proper preloading and query optimization + +--- + +## Identified Performance Tests + +### 1. Member LiveView - Boolean Filter Performance + +**File:** `test/mv_web/member_live/index_test.exs` +**Test:** `"boolean filter performance with 150 members"` +**Duration:** ~3.8 seconds +**Purpose:** Validates that boolean custom field filtering performs efficiently with 150 members + +**What it tests:** +- Creates 150 members (75 with `true`, 75 with `false` for a boolean custom field) +- Tests filter performance (< 1 second requirement) +- Verifies correct filtering behavior + +### 2. Group LiveView - Index Performance + +**File:** `test/mv_web/live/group_live/index_test.exs` +**Describe Block:** `"performance"` +**Tests:** 2 tests +**Purpose:** Validates efficient page loading and member count calculation + +**What it tests:** +- Page loads efficiently with many groups (no N+1 queries) +- Member count calculation is efficient + +### 3. Group LiveView - Show Performance + +**File:** `test/mv_web/live/group_live/show_test.exs` +**Describe Block:** `"performance"` +**Tests:** 2 tests +**Purpose:** Validates efficient member list loading and slug lookup + +**What it tests:** +- Member list is loaded efficiently (no N+1 queries) +- Slug lookup uses unique_slug index efficiently + +### 4. Member LiveView - Membership Fee Status Performance + +**File:** `test/mv_web/member_live/index_membership_fee_status_test.exs` +**Describe Block:** `"performance"` +**Tests:** 1 test +**Purpose:** Validates efficient cycle loading without N+1 queries + +**What it tests:** +- Cycles are loaded efficiently without N+1 queries +- Multiple members with cycles render without performance issues + +### 5. Group Integration - Query Performance + +**File:** `test/membership/group_integration_test.exs` +**Describe Block:** `"Query Performance"` +**Tests:** 1 test +**Purpose:** Validates N+1 query prevention for group-member relationships + +**What it tests:** +- Preloading groups with members avoids N+1 queries +- Query optimization for many-to-many relationships + +--- + +## Running Slow Tests + +### Local Development + +**Run only fast tests (default):** +```bash +just test-fast +# or +mix test --exclude slow + +# With specific files or options +just test-fast test/membership/member_test.exs +just test-fast --seed 123 +``` + +**Run only performance tests:** +```bash +just test-slow +# or +mix test --only slow + +# With specific files or options +just test-slow test/mv_web/member_live/index_test.exs +just test-slow --seed 123 +``` + +**Run all tests (fast + slow):** +```bash +just test +# or +mix test + +# With specific files or options +just test-all test/mv_web/ +just test-all --max-failures 5 +``` + +**Note:** All suite commands (`test-fast`, `test-slow`, `test-all`) support additional arguments. The suite semantics (tags) are always preserved - additional arguments are appended to the command. + +### CI/CD + +**Standard CI Pipeline:** +- Runs `mix test --exclude slow` for faster feedback +- Executes on every push to any branch + +**Nightly Pipeline:** +- Runs `mix test --only slow` for comprehensive performance coverage +- Executes daily at 2 AM via cron trigger +- Pipeline name: `nightly-tests` + +--- + +## Best Practices for New Performance Tests + +### When to Tag as `:slow` + +Tag tests as `:slow` when they: + +1. **Explicitly test performance:** Tests that measure execution time or validate performance characteristics +2. **Use large datasets:** Tests that create many records (e.g., 50+ members, 100+ records) +3. **Test scalability:** Tests that verify the application handles larger workloads +4. **Validate query optimization:** Tests that ensure N+1 queries are prevented + +### When NOT to Tag as `:slow` + +Do NOT tag tests as `:slow` if they are: + +1. **Simply slow by accident:** Tests that are slow due to inefficient setup, not intentional performance testing +2. **Slow due to bugs:** Tests that are slow because of actual performance bugs (fix the bug instead) +3. **Integration tests:** Integration tests should be tagged separately if needed (`@tag :integration`) + +### Tagging Guidelines + +**For single tests:** +```elixir +@tag :slow +test "boolean filter performance with 150 members" do + # test implementation +end +``` + +**For describe blocks (all tests in block):** +```elixir +@moduletag :slow +describe "performance" do + test "page loads efficiently" do + # test implementation + end + + test "member count is efficient" do + # test implementation + end +end +``` + +--- + +## Performance Test Structure + +### Recommended Structure + +```elixir +defmodule MvWeb.SomeLiveViewTest do + use MvWeb.ConnCase, async: true + + # Regular tests here (not tagged) + + @moduletag :slow + describe "performance" do + test "loads efficiently without N+1 queries" do + # Create test data + # Measure/validate performance + # Assert correct behavior + end + + test "handles large datasets efficiently" do + # Create large dataset + # Measure performance + # Assert performance requirements + end + end +end +``` + +### Performance Assertions + +Performance tests should include explicit performance assertions: + +```elixir +# Example: Time-based assertion +start_time = System.monotonic_time(:millisecond) +# ... perform operation ... +end_time = System.monotonic_time(:millisecond) +duration = end_time - start_time + +assert duration < 1000, "Operation took #{duration}ms, expected < 1000ms" +``` + +```elixir +# Example: Query count assertion (using Ecto query logging) +# Verify no N+1 queries by checking query count +``` + +--- + +## Monitoring and Maintenance + +### Regular Review + +- **Quarterly Review:** Review slow tests quarterly to ensure they're still relevant +- **Performance Baselines:** Update performance assertions if legitimate performance improvements occur +- **Test Cleanup:** Remove or optimize tests that become redundant + +### Success Metrics + +- ✅ Performance tests catch regressions before production +- ✅ Standard test runs complete in < 3 minutes +- ✅ Nightly builds complete successfully +- ✅ No false positives in performance tests + +### Troubleshooting + +**If a performance test fails:** + +1. **Check if it's a real regression:** Compare with previous runs +2. **Check CI environment:** Ensure CI has adequate resources +3. **Review test data:** Ensure test data setup is correct +4. **Check for flakiness:** Run test multiple times to verify consistency + +**If a performance test is too slow:** + +1. **Review test implementation:** Look for inefficiencies in test setup +2. **Consider reducing dataset size:** If still representative +3. **Split into smaller tests:** If testing multiple concerns + +--- + +## Related Documentation + +- **Test Optimization Summary:** `docs/test-optimization-summary.md` +- **Seeds Test Optimization:** `docs/test-optimization-seeds.md` +- **Testing Standards:** `CODE_GUIDELINES.md` - Section 4 (Testing Standards) +- **CI/CD Configuration:** `.drone.yml` + +--- + +## Changelog + +### 2026-01-28: Initial Setup +- Marked 5 performance test suites with `@tag :slow` or `@moduletag :slow` +- Added `test-fast`, `test-slow`, and `test-all` commands to Justfile +- Updated CI to exclude slow tests from standard runs +- Added nightly CI pipeline for slow tests +- Created this documentation diff --git a/test/membership/group_integration_test.exs b/test/membership/group_integration_test.exs index 2333a66..8e36119 100644 --- a/test/membership/group_integration_test.exs +++ b/test/membership/group_integration_test.exs @@ -79,6 +79,7 @@ defmodule Mv.Membership.GroupIntegrationTest do end end + @moduletag :slow describe "Query Performance" do test "preloading groups with members avoids N+1 queries", %{actor: actor} do # Create test data diff --git a/test/mv_web/live/group_live/index_test.exs b/test/mv_web/live/group_live/index_test.exs index 7dabcab..e844bcc 100644 --- a/test/mv_web/live/group_live/index_test.exs +++ b/test/mv_web/live/group_live/index_test.exs @@ -114,6 +114,7 @@ defmodule MvWeb.GroupLive.IndexTest do end end + @moduletag :slow describe "performance" do test "page loads efficiently with many groups", %{conn: conn} do # Create multiple groups diff --git a/test/mv_web/live/group_live/show_test.exs b/test/mv_web/live/group_live/show_test.exs index af298fd..177318e 100644 --- a/test/mv_web/live/group_live/show_test.exs +++ b/test/mv_web/live/group_live/show_test.exs @@ -219,6 +219,7 @@ defmodule MvWeb.GroupLive.ShowTest do end end + @moduletag :slow describe "performance" do test "member list is loaded efficiently (no N+1 queries)", %{conn: conn} do group = Fixtures.group_fixture() diff --git a/test/mv_web/member_live/index_membership_fee_status_test.exs b/test/mv_web/member_live/index_membership_fee_status_test.exs index 043c5cb..f868e96 100644 --- a/test/mv_web/member_live/index_membership_fee_status_test.exs +++ b/test/mv_web/member_live/index_membership_fee_status_test.exs @@ -238,6 +238,7 @@ defmodule MvWeb.MemberLive.IndexMembershipFeeStatusTest do end end + @moduletag :slow describe "performance" do test "loads cycles efficiently without N+1 queries", %{conn: conn} do fee_type = create_fee_type(%{interval: :yearly}) diff --git a/test/mv_web/member_live/index_test.exs b/test/mv_web/member_live/index_test.exs index 0624c77..388f70d 100644 --- a/test/mv_web/member_live/index_test.exs +++ b/test/mv_web/member_live/index_test.exs @@ -1775,6 +1775,7 @@ defmodule MvWeb.MemberLive.IndexTest do assert Enum.any?(boolean_fields_after, &(&1.name == "Newly Added Field")) end + @tag :slow test "boolean filter performance with 150 members", %{conn: conn} do system_actor = Mv.Helpers.SystemActor.get_system_actor() conn = conn_with_oidc_user(conn)