Minor test refactoring to improve on performance closes #383 #384

Merged
simon merged 23 commits from test-performance-optimization into main 2026-01-29 15:44:01 +01:00
4 changed files with 193 additions and 88 deletions
Showing only changes of commit 17974d7a12 - Show all commits

View file

@ -1,6 +1,6 @@
kind: pipeline
type: docker
name: check
name: check-fast
services:
- name: postgres
@ -12,6 +12,7 @@ services:
trigger:
event:
- push
- pull_request
steps:
- name: compute cache key
@ -72,7 +73,7 @@ steps:
echo "Postgres did not become available, aborting."
exit 1
- name: test
- name: test-fast
image: docker.io/library/elixir:1.18.3-otp-27
environment:
MIX_ENV: test
@ -83,8 +84,115 @@ steps:
- mix local.hex --force
# Fetch dependencies
- mix deps.get
# Run fast tests (excludes slow/performance tests)
- mix test --exclude slow
# Run fast tests (excludes slow/performance and UI tests)
- mix test --exclude slow --exclude ui
- name: rebuild-cache
image: drillster/drone-volume-cache
settings:
rebuild: true
mount:
- ./deps
- ./_build
volumes:
- name: cache
path: /cache
volumes:
- name: cache
host:
path: /tmp/drone_cache
---
kind: pipeline
type: docker
name: check-full
services:
- name: postgres
image: docker.io/library/postgres:18.1
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
trigger:
event:
- promote
target:
- production
steps:
- name: compute cache key
image: docker.io/library/elixir:1.18.3-otp-27
commands:
- mix_lock_hash=$(sha256sum mix.lock | cut -d ' ' -f 1)
- echo "$DRONE_REPO_OWNER/$DRONE_REPO_NAME/$mix_lock_hash" >> .cache_key
# Print cache key for debugging
- cat .cache_key
- name: restore-cache
image: drillster/drone-volume-cache
settings:
restore: true
mount:
- ./deps
- ./_build
ttl: 30
volumes:
- name: cache
path: /cache
- name: lint
image: docker.io/library/elixir:1.18.3-otp-27
commands:
# Install hex package manager
- mix local.hex --force
# Fetch dependencies
- mix deps.get
# Check for compilation errors & warnings
- mix compile --warnings-as-errors
# Check formatting
- mix format --check-formatted
# Security checks
- mix sobelow --config
# Check dependencies for known vulnerabilities
- mix deps.audit
# Check for dependencies that are not maintained anymore
- mix hex.audit
# Provide hints for improving code quality
- mix credo
# Check that translations are up to date
- mix gettext.extract --check-up-to-date
- name: wait_for_postgres
image: docker.io/library/postgres:18.1
commands:
# Wait for postgres to become available
- |
for i in {1..20}; do
if pg_isready -h postgres -U postgres; then
exit 0
else
true
fi
sleep 2
done
echo "Postgres did not become available, aborting."
exit 1
- name: test-all
image: docker.io/library/elixir:1.18.3-otp-27
environment:
MIX_ENV: test
TEST_POSTGRES_HOST: postgres
TEST_POSTGRES_PORT: 5432
commands:
# Install hex package manager
- mix local.hex --force
# Fetch dependencies
- mix deps.get
# Run all tests (including slow/performance and UI tests)
- mix test
- name: rebuild-cache
image: drillster/drone-volume-cache
@ -147,7 +255,7 @@ steps:
- push
depends_on:
- check
- check-fast
---
kind: pipeline
@ -178,55 +286,3 @@ steps:
- unset GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL
- renovate-config-validator
- renovate
---
kind: pipeline
type: docker
name: nightly-tests
trigger:
event:
- cron
- custom # Allows manual triggering
cron:
- nightly-tests # Cron job name configured in Drone UI/CLI
branch:
- main
services:
- name: postgres
image: docker.io/library/postgres:18.1
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
steps:
- name: wait_for_postgres
image: docker.io/library/postgres:18.1
commands:
# Wait for postgres to become available
- |
for i in {1..20}; do
if pg_isready -h postgres -U postgres; then
exit 0
else
true
fi
sleep 2
done
echo "Postgres did not become available, aborting."
exit 1
- name: test-all
image: docker.io/library/elixir:1.18.3-otp-27
environment:
MIX_ENV: test
TEST_POSTGRES_HOST: postgres
TEST_POSTGRES_PORT: 5432
commands:
# Install hex package manager
- mix local.hex --force
# Fetch dependencies
- mix deps.get
# Run all tests (including slow/performance tests)
- mix test

48
.forgejo/README.md Normal file
View file

@ -0,0 +1,48 @@
# Forgejo Configuration
This directory contains configuration files for Forgejo (self-hosted Git service).
## Pull Request Template
The `pull_request_template.md` is automatically loaded when creating a new Pull Request. It provides a checklist and instructions for the PR workflow, including how to run the full test suite before merging.
## Branch Protection Setup
To enforce the full test suite before merging to `main`, configure branch protection in Forgejo:
### Steps:
1. Go to **Repository Settings****Branches** → **Protected Branches**
2. Add a new rule for branch: `main`
3. Configure the following settings:
- ☑️ **Enable Branch Protection**
- ☑️ **Require status checks to pass before merging**
- Add required check: `check-full`
- ☐ **Require approvals** (optional, based on team preference)
- ☑️ **Block if there are outstanding requests** (optional)
### What this does:
- The **"Merge"** button in PRs will only be enabled after `check-full` passes
- `check-full` is triggered by **promoting** a build in Drone CI (see PR template)
- This ensures all tests (including slow and UI tests) run before merging
## Workflow
1. **Create PR** → Fast test suite (`check-fast`) runs automatically
2. **Development** → Fast tests run on every push for quick feedback
3. **Ready to merge:**
- Remove `WIP:` from PR title
- Go to Drone CI and **promote** the build to `production`
- This triggers `check-full` (full test suite)
4. **After full tests pass** → Merge button becomes available
5. **Merge to main** → Container is built and published
## Secrets Required
Make sure the following secrets are configured in Drone CI:
- `DRONE_REGISTRY_USERNAME` - For container registry
- `DRONE_REGISTRY_TOKEN` - For container registry
- `RENOVATE_TOKEN` - For Renovate bot
- `GITHUB_COM_TOKEN` - For Renovate bot (GitHub dependencies)

View file

@ -1703,10 +1703,10 @@ just test-all
**Test Organization Best Practices:**
- **Fast Tests (Standard CI):** Business logic, validations, data persistence, edge cases
- **UI Tests (Nightly CI):** Basic HTML rendering, navigation, translations, UI state
- **Performance Tests (Nightly CI):** Query optimization, large datasets, timing assertions
- **UI Tests (Full Test Suite):** Basic HTML rendering, navigation, translations, UI state
- **Performance Tests (Full Test Suite):** Query optimization, large datasets, timing assertions
This organization ensures fast feedback in standard CI while maintaining comprehensive coverage in nightly runs.
This organization ensures fast feedback in standard CI while maintaining comprehensive coverage via promotion before merge.
---
## 5. Security Guidelines

View file

@ -27,7 +27,7 @@ This document provides a comprehensive overview of test performance optimization
| Seeds tests reduction | 13 → 4 tests | ~10-16s | ✅ Completed |
| Performance tests tagging | 9 tests | ~3-4s per run | ✅ Completed |
| Critical test query filtering | 1 test | ~8-10s | ✅ Completed |
| Nightly suite tagging | 25 tests | ~77s per run | ✅ Completed |
| Full test suite via promotion | 25 tests | ~77s per run | ✅ Completed |
| **Total Saved** | | **~98-107s** | |
---
@ -80,14 +80,14 @@ Critical deployment requirements are still covered:
---
### 2. Nightly Suite Implementation (`@tag :slow`)
### 2. Full Test Suite via Promotion (`@tag :slow`)
**Date:** 2026-01-28
**Status:** ✅ Completed
#### What Changed
Tests with **low risk** and **execution time >1 second** are now tagged with `@tag :slow` and excluded from standard test runs. These tests are important but not critical for every commit and can be run in nightly CI builds.
Tests with **low risk** and **execution time >1 second** are now tagged with `@tag :slow` and excluded from standard test runs. These tests are important but not critical for every commit and are run via promotion before merging to `main`.
#### Tagging Criteria
@ -105,7 +105,7 @@ Tests with **low risk** and **execution time >1 second** are now tagged with `@t
- ❌ Email Synchronization
- ❌ Representative tests per Permission Set + Action
#### Identified Nightly Suite Tests (25 tests)
#### Identified Tests for Full Test Suite (25 tests)
**1. Seeds Tests (2 tests) - 18.1s**
- `"runs successfully and creates basic data"` (9.0s)
@ -155,7 +155,7 @@ just test-fast
mix test --exclude slow
```
**Slow/Nightly Tests Only:**
**Slow Tests Only:**
```bash
just test-slow
# or
@ -171,10 +171,10 @@ mix test
#### CI/CD Integration
- **Standard CI:** Runs `mix test --exclude slow` for faster feedback loops (~6 minutes)
- **Nightly Builds:** Separate pipeline runs daily and executes `mix test` (all tests, including slow) for comprehensive coverage (~7.4 minutes)
- **Pre-Merge:** Full test suite (`mix test`) runs before merging to main
- **Manual Execution:** Can be triggered via Drone CLI or Web UI
- **Standard CI (`check-fast`):** Runs `mix test --exclude slow --exclude ui` for faster feedback loops (~6 minutes)
- **Full Test Suite (`check-full`):** Triggered via promotion before merge, executes `mix test` (all tests, including slow and UI) for comprehensive coverage (~7.4 minutes)
- **Pre-Merge:** Full test suite (`mix test`) runs via promotion before merging to main
- **Manual Execution:** Promote build to `production` in Drone CI to trigger full test suite
#### Risk Assessment
@ -183,7 +183,7 @@ mix test
- All tagged tests have **low risk** - they don't catch critical regressions
- Core functionality remains tested (CRUD, Auth, Bootstrap)
- Standard test runs are faster (~6 minutes vs ~7.4 minutes)
- Nightly builds ensure comprehensive coverage
- Full test suite runs via promotion before merge ensures comprehensive coverage
- No functionality is lost, only execution timing changed
**Critical Tests Remain in Fast Suite:**
@ -243,14 +243,14 @@ The test loaded **all members** from the database, not just the 2 members from t
---
### 3. Nightly Suite Analysis and Categorization
### 3. Full Test Suite Analysis and Categorization
**Date:** 2026-01-28
**Status:** ✅ Completed
#### Analysis Methodology
A comprehensive analysis was performed to identify tests suitable for the nightly suite based on:
A comprehensive analysis was performed to identify tests suitable for the full test suite (via promotion) based on:
- **Execution time:** Tests taking >1 second
- **Risk assessment:** Tests that don't catch critical regressions
- **Test category:** UI/Display, workflow details, edge cases
@ -264,7 +264,7 @@ A comprehensive analysis was performed to identify tests suitable for the nightl
- Email Synchronization
- Representative Policy Tests (one per Permission Set + Action)
**🟡 LOW RISK - Moved to Nightly Suite:**
**🟡 LOW RISK - Moved to Full Test Suite (via Promotion):**
- Seeds Tests (non-critical: smoke test, idempotency)
- LiveView Display/Formatting Tests
- UserLive.ShowTest (core functionality covered by Index/Form)
@ -288,9 +288,9 @@ A comprehensive analysis was performed to identify tests suitable for the nightl
**Overall Risk:** ⚠️ **Low** - All moved tests have low risk and don't catch critical regressions. Core functionality remains fully tested.
#### Tests Excluded from Nightly Suite
#### Tests Excluded from Full Test Suite
The following tests were **NOT** moved to nightly suite despite being slow:
The following tests were **NOT** moved to full test suite (via promotion) despite being slow:
- **Policy Tests:** Medium risk - kept in fast suite (representative tests remain)
- **UserLive.FormTest:** Medium risk - core CRUD functionality
@ -303,7 +303,7 @@ The following tests were **NOT** moved to nightly suite despite being slow:
### Top 20 Slowest Tests (without `:slow`)
After implementing the nightly suite, the remaining slowest tests are:
After implementing the full test suite via promotion, the remaining slowest tests are:
| Rank | Test | File | Time | Category |
|------|------|------|------|----------|
@ -463,7 +463,7 @@ All optimizations maintain test coverage while improving performance:
- ✅ No regression in authorization or membership fee bugs
- ✅ Top 20 slowest tests: < 60 seconds (currently ~44s)
- ✅ Total execution time (without `:slow`): < 10 minutes (currently 6.1 min)
- ⏳ Nightly suite execution time: < 2 minutes (currently ~1.3 min)
- ⏳ Slow tests execution time: < 2 minutes (currently ~1.3 min)
#### What to Watch For
@ -543,10 +543,11 @@ mix test test/mv_web/member_live/index_member_fields_display_test.exs --slowest
- Command: `mix test --only slow` or `just test-slow`
- Excluded from standard CI runs
**Nightly CI Builds:**
**Full Test Suite (via Promotion):**
- Triggered by promoting a build to `production` in Drone CI
- Runs all tests (`mix test`) for comprehensive coverage
- Execution time: ~7.4 minutes
- Ensures full test coverage including slow/performance tests
- Required before merging to `main` (enforced via branch protection)
**All Tests:**
- Includes both fast and slow tests
@ -636,16 +637,16 @@ test/
- ✅ Optimized critical test with query filtering
- ✅ Created slow test suite infrastructure
- ✅ Updated CI/CD to exclude slow tests from standard runs
- ✅ Added nightly CI pipeline for slow tests
- ✅ Added promotion-based full test suite pipeline (`check-full`)
**Time Saved:** ~21-30 seconds per test run
### 2026-01-28: Nightly Suite Implementation
### 2026-01-28: Full Test Suite via Promotion Implementation
**Completed:**
- ✅ Analyzed all tests for nightly suite candidates
- ✅ Analyzed all tests for full test suite candidates
- ✅ Identified 36 tests with low risk and >1s execution time
- ✅ Tagged 25 tests with `@tag :slow` for nightly suite
- ✅ Tagged 25 tests with `@tag :slow` for full test suite (via promotion)
- ✅ Categorized tests by risk level and execution time
- ✅ Documented tagging criteria and guidelines
@ -667,7 +668,7 @@ test/
- **Time saved:** ~77 seconds (17% reduction)
**Next Steps:**
- ⏳ Monitor nightly suite execution in CI
- ⏳ Monitor full test suite execution via promotion in CI
- ⏳ Optimize remaining slow tests (Policy tests, etc.)
- ⏳ Further optimize Seeds tests (Priority 3)
@ -702,11 +703,11 @@ A: Tag it with `@tag :slow` for individual tests or `@describetag :slow` for des
**Q: Can I run slow tests locally?**
A: Yes, use `just test-slow` or `mix test --only slow`. They're excluded from standard runs for faster feedback.
**Q: What is the "nightly suite"?**
A: The "nightly suite" refers to the nightly CI pipeline that runs **all tests** (`mix test`), including slow tests. Tests tagged with `@tag :slow` or `@describetag :slow` are excluded from standard CI runs for faster feedback, but are included in the nightly full test suite for comprehensive coverage.
**Q: What is the "full test suite"?**
A: The full test suite runs **all tests** (`mix test`), including slow and UI tests. Tests tagged with `@tag :slow` or `@describetag :slow` are excluded from standard CI runs (`check-fast`) for faster feedback, but are included when promoting a build to `production` (`check-full`) before merging to `main`.
**Q: Which tests should I tag as `:slow`?**
A: Tag tests with `@tag :slow` if they: (1) take >1 second, (2) have low risk (not critical for catching regressions), and (3) test UI/Display/Formatting or workflow details. See "Test Tagging Guidelines" section for details.
**Q: What if a slow test fails in nightly builds?**
A: If a test in the nightly suite fails, investigate the failure. If it indicates a critical regression, consider moving it back to the fast suite. If it's a flaky test, fix the test itself.
**Q: What if a slow test fails in the full test suite?**
A: If a test in the full test suite fails, investigate the failure. If it indicates a critical regression, consider moving it back to the fast suite. If it's a flaky test, fix the test itself. The merge will be blocked until all tests pass.