chore: change pr merge workflow
This commit is contained in:
parent
1019914d50
commit
17974d7a12
4 changed files with 193 additions and 88 deletions
170
.drone.yml
170
.drone.yml
|
|
@ -1,6 +1,6 @@
|
||||||
kind: pipeline
|
kind: pipeline
|
||||||
type: docker
|
type: docker
|
||||||
name: check
|
name: check-fast
|
||||||
|
|
||||||
services:
|
services:
|
||||||
- name: postgres
|
- name: postgres
|
||||||
|
|
@ -12,6 +12,7 @@ services:
|
||||||
trigger:
|
trigger:
|
||||||
event:
|
event:
|
||||||
- push
|
- push
|
||||||
|
- pull_request
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: compute cache key
|
- name: compute cache key
|
||||||
|
|
@ -72,7 +73,7 @@ steps:
|
||||||
echo "Postgres did not become available, aborting."
|
echo "Postgres did not become available, aborting."
|
||||||
exit 1
|
exit 1
|
||||||
|
|
||||||
- name: test
|
- name: test-fast
|
||||||
image: docker.io/library/elixir:1.18.3-otp-27
|
image: docker.io/library/elixir:1.18.3-otp-27
|
||||||
environment:
|
environment:
|
||||||
MIX_ENV: test
|
MIX_ENV: test
|
||||||
|
|
@ -83,8 +84,115 @@ steps:
|
||||||
- mix local.hex --force
|
- mix local.hex --force
|
||||||
# Fetch dependencies
|
# Fetch dependencies
|
||||||
- mix deps.get
|
- mix deps.get
|
||||||
# Run fast tests (excludes slow/performance tests)
|
# Run fast tests (excludes slow/performance and UI tests)
|
||||||
- mix test --exclude slow
|
- mix test --exclude slow --exclude ui
|
||||||
|
|
||||||
|
- name: rebuild-cache
|
||||||
|
image: drillster/drone-volume-cache
|
||||||
|
settings:
|
||||||
|
rebuild: true
|
||||||
|
mount:
|
||||||
|
- ./deps
|
||||||
|
- ./_build
|
||||||
|
volumes:
|
||||||
|
- name: cache
|
||||||
|
path: /cache
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- name: cache
|
||||||
|
host:
|
||||||
|
path: /tmp/drone_cache
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: pipeline
|
||||||
|
type: docker
|
||||||
|
name: check-full
|
||||||
|
|
||||||
|
services:
|
||||||
|
- name: postgres
|
||||||
|
image: docker.io/library/postgres:18.1
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: postgres
|
||||||
|
POSTGRES_PASSWORD: postgres
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
event:
|
||||||
|
- promote
|
||||||
|
target:
|
||||||
|
- production
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: compute cache key
|
||||||
|
image: docker.io/library/elixir:1.18.3-otp-27
|
||||||
|
commands:
|
||||||
|
- mix_lock_hash=$(sha256sum mix.lock | cut -d ' ' -f 1)
|
||||||
|
- echo "$DRONE_REPO_OWNER/$DRONE_REPO_NAME/$mix_lock_hash" >> .cache_key
|
||||||
|
# Print cache key for debugging
|
||||||
|
- cat .cache_key
|
||||||
|
|
||||||
|
- name: restore-cache
|
||||||
|
image: drillster/drone-volume-cache
|
||||||
|
settings:
|
||||||
|
restore: true
|
||||||
|
mount:
|
||||||
|
- ./deps
|
||||||
|
- ./_build
|
||||||
|
ttl: 30
|
||||||
|
volumes:
|
||||||
|
- name: cache
|
||||||
|
path: /cache
|
||||||
|
|
||||||
|
- name: lint
|
||||||
|
image: docker.io/library/elixir:1.18.3-otp-27
|
||||||
|
commands:
|
||||||
|
# Install hex package manager
|
||||||
|
- mix local.hex --force
|
||||||
|
# Fetch dependencies
|
||||||
|
- mix deps.get
|
||||||
|
# Check for compilation errors & warnings
|
||||||
|
- mix compile --warnings-as-errors
|
||||||
|
# Check formatting
|
||||||
|
- mix format --check-formatted
|
||||||
|
# Security checks
|
||||||
|
- mix sobelow --config
|
||||||
|
# Check dependencies for known vulnerabilities
|
||||||
|
- mix deps.audit
|
||||||
|
# Check for dependencies that are not maintained anymore
|
||||||
|
- mix hex.audit
|
||||||
|
# Provide hints for improving code quality
|
||||||
|
- mix credo
|
||||||
|
# Check that translations are up to date
|
||||||
|
- mix gettext.extract --check-up-to-date
|
||||||
|
|
||||||
|
- name: wait_for_postgres
|
||||||
|
image: docker.io/library/postgres:18.1
|
||||||
|
commands:
|
||||||
|
# Wait for postgres to become available
|
||||||
|
- |
|
||||||
|
for i in {1..20}; do
|
||||||
|
if pg_isready -h postgres -U postgres; then
|
||||||
|
exit 0
|
||||||
|
else
|
||||||
|
true
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
echo "Postgres did not become available, aborting."
|
||||||
|
exit 1
|
||||||
|
|
||||||
|
- name: test-all
|
||||||
|
image: docker.io/library/elixir:1.18.3-otp-27
|
||||||
|
environment:
|
||||||
|
MIX_ENV: test
|
||||||
|
TEST_POSTGRES_HOST: postgres
|
||||||
|
TEST_POSTGRES_PORT: 5432
|
||||||
|
commands:
|
||||||
|
# Install hex package manager
|
||||||
|
- mix local.hex --force
|
||||||
|
# Fetch dependencies
|
||||||
|
- mix deps.get
|
||||||
|
# Run all tests (including slow/performance and UI tests)
|
||||||
|
- mix test
|
||||||
|
|
||||||
- name: rebuild-cache
|
- name: rebuild-cache
|
||||||
image: drillster/drone-volume-cache
|
image: drillster/drone-volume-cache
|
||||||
|
|
@ -147,7 +255,7 @@ steps:
|
||||||
- push
|
- push
|
||||||
|
|
||||||
depends_on:
|
depends_on:
|
||||||
- check
|
- check-fast
|
||||||
|
|
||||||
---
|
---
|
||||||
kind: pipeline
|
kind: pipeline
|
||||||
|
|
@ -178,55 +286,3 @@ steps:
|
||||||
- unset GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL
|
- unset GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL
|
||||||
- renovate-config-validator
|
- renovate-config-validator
|
||||||
- renovate
|
- renovate
|
||||||
|
|
||||||
---
|
|
||||||
kind: pipeline
|
|
||||||
type: docker
|
|
||||||
name: nightly-tests
|
|
||||||
|
|
||||||
trigger:
|
|
||||||
event:
|
|
||||||
- cron
|
|
||||||
- custom # Allows manual triggering
|
|
||||||
cron:
|
|
||||||
- nightly-tests # Cron job name configured in Drone UI/CLI
|
|
||||||
branch:
|
|
||||||
- main
|
|
||||||
|
|
||||||
services:
|
|
||||||
- name: postgres
|
|
||||||
image: docker.io/library/postgres:18.1
|
|
||||||
environment:
|
|
||||||
POSTGRES_USER: postgres
|
|
||||||
POSTGRES_PASSWORD: postgres
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: wait_for_postgres
|
|
||||||
image: docker.io/library/postgres:18.1
|
|
||||||
commands:
|
|
||||||
# Wait for postgres to become available
|
|
||||||
- |
|
|
||||||
for i in {1..20}; do
|
|
||||||
if pg_isready -h postgres -U postgres; then
|
|
||||||
exit 0
|
|
||||||
else
|
|
||||||
true
|
|
||||||
fi
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
echo "Postgres did not become available, aborting."
|
|
||||||
exit 1
|
|
||||||
|
|
||||||
- name: test-all
|
|
||||||
image: docker.io/library/elixir:1.18.3-otp-27
|
|
||||||
environment:
|
|
||||||
MIX_ENV: test
|
|
||||||
TEST_POSTGRES_HOST: postgres
|
|
||||||
TEST_POSTGRES_PORT: 5432
|
|
||||||
commands:
|
|
||||||
# Install hex package manager
|
|
||||||
- mix local.hex --force
|
|
||||||
# Fetch dependencies
|
|
||||||
- mix deps.get
|
|
||||||
# Run all tests (including slow/performance tests)
|
|
||||||
- mix test
|
|
||||||
|
|
|
||||||
48
.forgejo/README.md
Normal file
48
.forgejo/README.md
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
# Forgejo Configuration
|
||||||
|
|
||||||
|
This directory contains configuration files for Forgejo (self-hosted Git service).
|
||||||
|
|
||||||
|
## Pull Request Template
|
||||||
|
|
||||||
|
The `pull_request_template.md` is automatically loaded when creating a new Pull Request. It provides a checklist and instructions for the PR workflow, including how to run the full test suite before merging.
|
||||||
|
|
||||||
|
## Branch Protection Setup
|
||||||
|
|
||||||
|
To enforce the full test suite before merging to `main`, configure branch protection in Forgejo:
|
||||||
|
|
||||||
|
### Steps:
|
||||||
|
|
||||||
|
1. Go to **Repository Settings** → **Branches** → **Protected Branches**
|
||||||
|
2. Add a new rule for branch: `main`
|
||||||
|
3. Configure the following settings:
|
||||||
|
- ☑️ **Enable Branch Protection**
|
||||||
|
- ☑️ **Require status checks to pass before merging**
|
||||||
|
- Add required check: `check-full`
|
||||||
|
- ☐ **Require approvals** (optional, based on team preference)
|
||||||
|
- ☑️ **Block if there are outstanding requests** (optional)
|
||||||
|
|
||||||
|
### What this does:
|
||||||
|
|
||||||
|
- The **"Merge"** button in PRs will only be enabled after `check-full` passes
|
||||||
|
- `check-full` is triggered by **promoting** a build in Drone CI (see PR template)
|
||||||
|
- This ensures all tests (including slow and UI tests) run before merging
|
||||||
|
|
||||||
|
## Workflow
|
||||||
|
|
||||||
|
1. **Create PR** → Fast test suite (`check-fast`) runs automatically
|
||||||
|
2. **Development** → Fast tests run on every push for quick feedback
|
||||||
|
3. **Ready to merge:**
|
||||||
|
- Remove `WIP:` from PR title
|
||||||
|
- Go to Drone CI and **promote** the build to `production`
|
||||||
|
- This triggers `check-full` (full test suite)
|
||||||
|
4. **After full tests pass** → Merge button becomes available
|
||||||
|
5. **Merge to main** → Container is built and published
|
||||||
|
|
||||||
|
## Secrets Required
|
||||||
|
|
||||||
|
Make sure the following secrets are configured in Drone CI:
|
||||||
|
|
||||||
|
- `DRONE_REGISTRY_USERNAME` - For container registry
|
||||||
|
- `DRONE_REGISTRY_TOKEN` - For container registry
|
||||||
|
- `RENOVATE_TOKEN` - For Renovate bot
|
||||||
|
- `GITHUB_COM_TOKEN` - For Renovate bot (GitHub dependencies)
|
||||||
|
|
@ -1703,10 +1703,10 @@ just test-all
|
||||||
**Test Organization Best Practices:**
|
**Test Organization Best Practices:**
|
||||||
|
|
||||||
- **Fast Tests (Standard CI):** Business logic, validations, data persistence, edge cases
|
- **Fast Tests (Standard CI):** Business logic, validations, data persistence, edge cases
|
||||||
- **UI Tests (Nightly CI):** Basic HTML rendering, navigation, translations, UI state
|
- **UI Tests (Full Test Suite):** Basic HTML rendering, navigation, translations, UI state
|
||||||
- **Performance Tests (Nightly CI):** Query optimization, large datasets, timing assertions
|
- **Performance Tests (Full Test Suite):** Query optimization, large datasets, timing assertions
|
||||||
|
|
||||||
This organization ensures fast feedback in standard CI while maintaining comprehensive coverage in nightly runs.
|
This organization ensures fast feedback in standard CI while maintaining comprehensive coverage via promotion before merge.
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5. Security Guidelines
|
## 5. Security Guidelines
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ This document provides a comprehensive overview of test performance optimization
|
||||||
| Seeds tests reduction | 13 → 4 tests | ~10-16s | ✅ Completed |
|
| Seeds tests reduction | 13 → 4 tests | ~10-16s | ✅ Completed |
|
||||||
| Performance tests tagging | 9 tests | ~3-4s per run | ✅ Completed |
|
| Performance tests tagging | 9 tests | ~3-4s per run | ✅ Completed |
|
||||||
| Critical test query filtering | 1 test | ~8-10s | ✅ Completed |
|
| Critical test query filtering | 1 test | ~8-10s | ✅ Completed |
|
||||||
| Nightly suite tagging | 25 tests | ~77s per run | ✅ Completed |
|
| Full test suite via promotion | 25 tests | ~77s per run | ✅ Completed |
|
||||||
| **Total Saved** | | **~98-107s** | |
|
| **Total Saved** | | **~98-107s** | |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
@ -80,14 +80,14 @@ Critical deployment requirements are still covered:
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 2. Nightly Suite Implementation (`@tag :slow`)
|
### 2. Full Test Suite via Promotion (`@tag :slow`)
|
||||||
|
|
||||||
**Date:** 2026-01-28
|
**Date:** 2026-01-28
|
||||||
**Status:** ✅ Completed
|
**Status:** ✅ Completed
|
||||||
|
|
||||||
#### What Changed
|
#### What Changed
|
||||||
|
|
||||||
Tests with **low risk** and **execution time >1 second** are now tagged with `@tag :slow` and excluded from standard test runs. These tests are important but not critical for every commit and can be run in nightly CI builds.
|
Tests with **low risk** and **execution time >1 second** are now tagged with `@tag :slow` and excluded from standard test runs. These tests are important but not critical for every commit and are run via promotion before merging to `main`.
|
||||||
|
|
||||||
#### Tagging Criteria
|
#### Tagging Criteria
|
||||||
|
|
||||||
|
|
@ -105,7 +105,7 @@ Tests with **low risk** and **execution time >1 second** are now tagged with `@t
|
||||||
- ❌ Email Synchronization
|
- ❌ Email Synchronization
|
||||||
- ❌ Representative tests per Permission Set + Action
|
- ❌ Representative tests per Permission Set + Action
|
||||||
|
|
||||||
#### Identified Nightly Suite Tests (25 tests)
|
#### Identified Tests for Full Test Suite (25 tests)
|
||||||
|
|
||||||
**1. Seeds Tests (2 tests) - 18.1s**
|
**1. Seeds Tests (2 tests) - 18.1s**
|
||||||
- `"runs successfully and creates basic data"` (9.0s)
|
- `"runs successfully and creates basic data"` (9.0s)
|
||||||
|
|
@ -155,7 +155,7 @@ just test-fast
|
||||||
mix test --exclude slow
|
mix test --exclude slow
|
||||||
```
|
```
|
||||||
|
|
||||||
**Slow/Nightly Tests Only:**
|
**Slow Tests Only:**
|
||||||
```bash
|
```bash
|
||||||
just test-slow
|
just test-slow
|
||||||
# or
|
# or
|
||||||
|
|
@ -171,10 +171,10 @@ mix test
|
||||||
|
|
||||||
#### CI/CD Integration
|
#### CI/CD Integration
|
||||||
|
|
||||||
- **Standard CI:** Runs `mix test --exclude slow` for faster feedback loops (~6 minutes)
|
- **Standard CI (`check-fast`):** Runs `mix test --exclude slow --exclude ui` for faster feedback loops (~6 minutes)
|
||||||
- **Nightly Builds:** Separate pipeline runs daily and executes `mix test` (all tests, including slow) for comprehensive coverage (~7.4 minutes)
|
- **Full Test Suite (`check-full`):** Triggered via promotion before merge, executes `mix test` (all tests, including slow and UI) for comprehensive coverage (~7.4 minutes)
|
||||||
- **Pre-Merge:** Full test suite (`mix test`) runs before merging to main
|
- **Pre-Merge:** Full test suite (`mix test`) runs via promotion before merging to main
|
||||||
- **Manual Execution:** Can be triggered via Drone CLI or Web UI
|
- **Manual Execution:** Promote build to `production` in Drone CI to trigger full test suite
|
||||||
|
|
||||||
#### Risk Assessment
|
#### Risk Assessment
|
||||||
|
|
||||||
|
|
@ -183,7 +183,7 @@ mix test
|
||||||
- All tagged tests have **low risk** - they don't catch critical regressions
|
- All tagged tests have **low risk** - they don't catch critical regressions
|
||||||
- Core functionality remains tested (CRUD, Auth, Bootstrap)
|
- Core functionality remains tested (CRUD, Auth, Bootstrap)
|
||||||
- Standard test runs are faster (~6 minutes vs ~7.4 minutes)
|
- Standard test runs are faster (~6 minutes vs ~7.4 minutes)
|
||||||
- Nightly builds ensure comprehensive coverage
|
- Full test suite runs via promotion before merge ensures comprehensive coverage
|
||||||
- No functionality is lost, only execution timing changed
|
- No functionality is lost, only execution timing changed
|
||||||
|
|
||||||
**Critical Tests Remain in Fast Suite:**
|
**Critical Tests Remain in Fast Suite:**
|
||||||
|
|
@ -243,14 +243,14 @@ The test loaded **all members** from the database, not just the 2 members from t
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 3. Nightly Suite Analysis and Categorization
|
### 3. Full Test Suite Analysis and Categorization
|
||||||
|
|
||||||
**Date:** 2026-01-28
|
**Date:** 2026-01-28
|
||||||
**Status:** ✅ Completed
|
**Status:** ✅ Completed
|
||||||
|
|
||||||
#### Analysis Methodology
|
#### Analysis Methodology
|
||||||
|
|
||||||
A comprehensive analysis was performed to identify tests suitable for the nightly suite based on:
|
A comprehensive analysis was performed to identify tests suitable for the full test suite (via promotion) based on:
|
||||||
- **Execution time:** Tests taking >1 second
|
- **Execution time:** Tests taking >1 second
|
||||||
- **Risk assessment:** Tests that don't catch critical regressions
|
- **Risk assessment:** Tests that don't catch critical regressions
|
||||||
- **Test category:** UI/Display, workflow details, edge cases
|
- **Test category:** UI/Display, workflow details, edge cases
|
||||||
|
|
@ -264,7 +264,7 @@ A comprehensive analysis was performed to identify tests suitable for the nightl
|
||||||
- Email Synchronization
|
- Email Synchronization
|
||||||
- Representative Policy Tests (one per Permission Set + Action)
|
- Representative Policy Tests (one per Permission Set + Action)
|
||||||
|
|
||||||
**🟡 LOW RISK - Moved to Nightly Suite:**
|
**🟡 LOW RISK - Moved to Full Test Suite (via Promotion):**
|
||||||
- Seeds Tests (non-critical: smoke test, idempotency)
|
- Seeds Tests (non-critical: smoke test, idempotency)
|
||||||
- LiveView Display/Formatting Tests
|
- LiveView Display/Formatting Tests
|
||||||
- UserLive.ShowTest (core functionality covered by Index/Form)
|
- UserLive.ShowTest (core functionality covered by Index/Form)
|
||||||
|
|
@ -288,9 +288,9 @@ A comprehensive analysis was performed to identify tests suitable for the nightl
|
||||||
|
|
||||||
**Overall Risk:** ⚠️ **Low** - All moved tests have low risk and don't catch critical regressions. Core functionality remains fully tested.
|
**Overall Risk:** ⚠️ **Low** - All moved tests have low risk and don't catch critical regressions. Core functionality remains fully tested.
|
||||||
|
|
||||||
#### Tests Excluded from Nightly Suite
|
#### Tests Excluded from Full Test Suite
|
||||||
|
|
||||||
The following tests were **NOT** moved to nightly suite despite being slow:
|
The following tests were **NOT** moved to full test suite (via promotion) despite being slow:
|
||||||
|
|
||||||
- **Policy Tests:** Medium risk - kept in fast suite (representative tests remain)
|
- **Policy Tests:** Medium risk - kept in fast suite (representative tests remain)
|
||||||
- **UserLive.FormTest:** Medium risk - core CRUD functionality
|
- **UserLive.FormTest:** Medium risk - core CRUD functionality
|
||||||
|
|
@ -303,7 +303,7 @@ The following tests were **NOT** moved to nightly suite despite being slow:
|
||||||
|
|
||||||
### Top 20 Slowest Tests (without `:slow`)
|
### Top 20 Slowest Tests (without `:slow`)
|
||||||
|
|
||||||
After implementing the nightly suite, the remaining slowest tests are:
|
After implementing the full test suite via promotion, the remaining slowest tests are:
|
||||||
|
|
||||||
| Rank | Test | File | Time | Category |
|
| Rank | Test | File | Time | Category |
|
||||||
|------|------|------|------|----------|
|
|------|------|------|------|----------|
|
||||||
|
|
@ -463,7 +463,7 @@ All optimizations maintain test coverage while improving performance:
|
||||||
- ✅ No regression in authorization or membership fee bugs
|
- ✅ No regression in authorization or membership fee bugs
|
||||||
- ✅ Top 20 slowest tests: < 60 seconds (currently ~44s)
|
- ✅ Top 20 slowest tests: < 60 seconds (currently ~44s)
|
||||||
- ✅ Total execution time (without `:slow`): < 10 minutes (currently 6.1 min)
|
- ✅ Total execution time (without `:slow`): < 10 minutes (currently 6.1 min)
|
||||||
- ⏳ Nightly suite execution time: < 2 minutes (currently ~1.3 min)
|
- ⏳ Slow tests execution time: < 2 minutes (currently ~1.3 min)
|
||||||
|
|
||||||
#### What to Watch For
|
#### What to Watch For
|
||||||
|
|
||||||
|
|
@ -543,10 +543,11 @@ mix test test/mv_web/member_live/index_member_fields_display_test.exs --slowest
|
||||||
- Command: `mix test --only slow` or `just test-slow`
|
- Command: `mix test --only slow` or `just test-slow`
|
||||||
- Excluded from standard CI runs
|
- Excluded from standard CI runs
|
||||||
|
|
||||||
**Nightly CI Builds:**
|
**Full Test Suite (via Promotion):**
|
||||||
|
- Triggered by promoting a build to `production` in Drone CI
|
||||||
- Runs all tests (`mix test`) for comprehensive coverage
|
- Runs all tests (`mix test`) for comprehensive coverage
|
||||||
- Execution time: ~7.4 minutes
|
- Execution time: ~7.4 minutes
|
||||||
- Ensures full test coverage including slow/performance tests
|
- Required before merging to `main` (enforced via branch protection)
|
||||||
|
|
||||||
**All Tests:**
|
**All Tests:**
|
||||||
- Includes both fast and slow tests
|
- Includes both fast and slow tests
|
||||||
|
|
@ -636,16 +637,16 @@ test/
|
||||||
- ✅ Optimized critical test with query filtering
|
- ✅ Optimized critical test with query filtering
|
||||||
- ✅ Created slow test suite infrastructure
|
- ✅ Created slow test suite infrastructure
|
||||||
- ✅ Updated CI/CD to exclude slow tests from standard runs
|
- ✅ Updated CI/CD to exclude slow tests from standard runs
|
||||||
- ✅ Added nightly CI pipeline for slow tests
|
- ✅ Added promotion-based full test suite pipeline (`check-full`)
|
||||||
|
|
||||||
**Time Saved:** ~21-30 seconds per test run
|
**Time Saved:** ~21-30 seconds per test run
|
||||||
|
|
||||||
### 2026-01-28: Nightly Suite Implementation
|
### 2026-01-28: Full Test Suite via Promotion Implementation
|
||||||
|
|
||||||
**Completed:**
|
**Completed:**
|
||||||
- ✅ Analyzed all tests for nightly suite candidates
|
- ✅ Analyzed all tests for full test suite candidates
|
||||||
- ✅ Identified 36 tests with low risk and >1s execution time
|
- ✅ Identified 36 tests with low risk and >1s execution time
|
||||||
- ✅ Tagged 25 tests with `@tag :slow` for nightly suite
|
- ✅ Tagged 25 tests with `@tag :slow` for full test suite (via promotion)
|
||||||
- ✅ Categorized tests by risk level and execution time
|
- ✅ Categorized tests by risk level and execution time
|
||||||
- ✅ Documented tagging criteria and guidelines
|
- ✅ Documented tagging criteria and guidelines
|
||||||
|
|
||||||
|
|
@ -667,7 +668,7 @@ test/
|
||||||
- **Time saved:** ~77 seconds (17% reduction)
|
- **Time saved:** ~77 seconds (17% reduction)
|
||||||
|
|
||||||
**Next Steps:**
|
**Next Steps:**
|
||||||
- ⏳ Monitor nightly suite execution in CI
|
- ⏳ Monitor full test suite execution via promotion in CI
|
||||||
- ⏳ Optimize remaining slow tests (Policy tests, etc.)
|
- ⏳ Optimize remaining slow tests (Policy tests, etc.)
|
||||||
- ⏳ Further optimize Seeds tests (Priority 3)
|
- ⏳ Further optimize Seeds tests (Priority 3)
|
||||||
|
|
||||||
|
|
@ -702,11 +703,11 @@ A: Tag it with `@tag :slow` for individual tests or `@describetag :slow` for des
|
||||||
**Q: Can I run slow tests locally?**
|
**Q: Can I run slow tests locally?**
|
||||||
A: Yes, use `just test-slow` or `mix test --only slow`. They're excluded from standard runs for faster feedback.
|
A: Yes, use `just test-slow` or `mix test --only slow`. They're excluded from standard runs for faster feedback.
|
||||||
|
|
||||||
**Q: What is the "nightly suite"?**
|
**Q: What is the "full test suite"?**
|
||||||
A: The "nightly suite" refers to the nightly CI pipeline that runs **all tests** (`mix test`), including slow tests. Tests tagged with `@tag :slow` or `@describetag :slow` are excluded from standard CI runs for faster feedback, but are included in the nightly full test suite for comprehensive coverage.
|
A: The full test suite runs **all tests** (`mix test`), including slow and UI tests. Tests tagged with `@tag :slow` or `@describetag :slow` are excluded from standard CI runs (`check-fast`) for faster feedback, but are included when promoting a build to `production` (`check-full`) before merging to `main`.
|
||||||
|
|
||||||
**Q: Which tests should I tag as `:slow`?**
|
**Q: Which tests should I tag as `:slow`?**
|
||||||
A: Tag tests with `@tag :slow` if they: (1) take >1 second, (2) have low risk (not critical for catching regressions), and (3) test UI/Display/Formatting or workflow details. See "Test Tagging Guidelines" section for details.
|
A: Tag tests with `@tag :slow` if they: (1) take >1 second, (2) have low risk (not critical for catching regressions), and (3) test UI/Display/Formatting or workflow details. See "Test Tagging Guidelines" section for details.
|
||||||
|
|
||||||
**Q: What if a slow test fails in nightly builds?**
|
**Q: What if a slow test fails in the full test suite?**
|
||||||
A: If a test in the nightly suite fails, investigate the failure. If it indicates a critical regression, consider moving it back to the fast suite. If it's a flaky test, fix the test itself.
|
A: If a test in the full test suite fails, investigate the failure. If it indicates a critical regression, consider moving it back to the fast suite. If it's a flaky test, fix the test itself. The merge will be blocked until all tests pass.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue