From 17974d7a127f16e352f0fe47ebe369a5cb6c2a34 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 29 Jan 2026 14:30:09 +0100 Subject: [PATCH] chore: change pr merge workflow --- .drone.yml | 170 +++++++++++++++++--------- .forgejo/README.md | 48 ++++++++ CODE_GUIDELINES.md | 6 +- docs/test-performance-optimization.md | 57 ++++----- 4 files changed, 193 insertions(+), 88 deletions(-) create mode 100644 .forgejo/README.md diff --git a/.drone.yml b/.drone.yml index 755eed3..a35cdbc 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,6 +1,6 @@ kind: pipeline type: docker -name: check +name: check-fast services: - name: postgres @@ -12,6 +12,7 @@ services: trigger: event: - push + - pull_request steps: - name: compute cache key @@ -72,7 +73,7 @@ steps: echo "Postgres did not become available, aborting." exit 1 - - name: test + - name: test-fast image: docker.io/library/elixir:1.18.3-otp-27 environment: MIX_ENV: test @@ -83,8 +84,115 @@ steps: - mix local.hex --force # Fetch dependencies - mix deps.get - # Run fast tests (excludes slow/performance tests) - - mix test --exclude slow + # Run fast tests (excludes slow/performance and UI tests) + - mix test --exclude slow --exclude ui + + - name: rebuild-cache + image: drillster/drone-volume-cache + settings: + rebuild: true + mount: + - ./deps + - ./_build + volumes: + - name: cache + path: /cache + +volumes: + - name: cache + host: + path: /tmp/drone_cache + +--- +kind: pipeline +type: docker +name: check-full + +services: + - name: postgres + image: docker.io/library/postgres:18.1 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + +trigger: + event: + - promote + target: + - production + +steps: + - name: compute cache key + image: docker.io/library/elixir:1.18.3-otp-27 + commands: + - mix_lock_hash=$(sha256sum mix.lock | cut -d ' ' -f 1) + - echo "$DRONE_REPO_OWNER/$DRONE_REPO_NAME/$mix_lock_hash" >> .cache_key + # Print cache key for debugging + - cat .cache_key + + - name: restore-cache + image: drillster/drone-volume-cache + settings: + restore: true + mount: + - ./deps + - ./_build + ttl: 30 + volumes: + - name: cache + path: /cache + + - name: lint + image: docker.io/library/elixir:1.18.3-otp-27 + commands: + # Install hex package manager + - mix local.hex --force + # Fetch dependencies + - mix deps.get + # Check for compilation errors & warnings + - mix compile --warnings-as-errors + # Check formatting + - mix format --check-formatted + # Security checks + - mix sobelow --config + # Check dependencies for known vulnerabilities + - mix deps.audit + # Check for dependencies that are not maintained anymore + - mix hex.audit + # Provide hints for improving code quality + - mix credo + # Check that translations are up to date + - mix gettext.extract --check-up-to-date + + - name: wait_for_postgres + image: docker.io/library/postgres:18.1 + commands: + # Wait for postgres to become available + - | + for i in {1..20}; do + if pg_isready -h postgres -U postgres; then + exit 0 + else + true + fi + sleep 2 + done + echo "Postgres did not become available, aborting." + exit 1 + + - name: test-all + image: docker.io/library/elixir:1.18.3-otp-27 + environment: + MIX_ENV: test + TEST_POSTGRES_HOST: postgres + TEST_POSTGRES_PORT: 5432 + commands: + # Install hex package manager + - mix local.hex --force + # Fetch dependencies + - mix deps.get + # Run all tests (including slow/performance and UI tests) + - mix test - name: rebuild-cache image: drillster/drone-volume-cache @@ -147,7 +255,7 @@ steps: - push depends_on: - - check + - check-fast --- kind: pipeline @@ -178,55 +286,3 @@ steps: - unset GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL - renovate-config-validator - renovate - ---- -kind: pipeline -type: docker -name: nightly-tests - -trigger: - event: - - cron - - custom # Allows manual triggering - cron: - - nightly-tests # Cron job name configured in Drone UI/CLI - branch: - - main - -services: - - name: postgres - image: docker.io/library/postgres:18.1 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - -steps: - - name: wait_for_postgres - image: docker.io/library/postgres:18.1 - commands: - # Wait for postgres to become available - - | - for i in {1..20}; do - if pg_isready -h postgres -U postgres; then - exit 0 - else - true - fi - sleep 2 - done - echo "Postgres did not become available, aborting." - exit 1 - - - name: test-all - image: docker.io/library/elixir:1.18.3-otp-27 - environment: - MIX_ENV: test - TEST_POSTGRES_HOST: postgres - TEST_POSTGRES_PORT: 5432 - commands: - # Install hex package manager - - mix local.hex --force - # Fetch dependencies - - mix deps.get - # Run all tests (including slow/performance tests) - - mix test diff --git a/.forgejo/README.md b/.forgejo/README.md new file mode 100644 index 0000000..6fa6d72 --- /dev/null +++ b/.forgejo/README.md @@ -0,0 +1,48 @@ +# Forgejo Configuration + +This directory contains configuration files for Forgejo (self-hosted Git service). + +## Pull Request Template + +The `pull_request_template.md` is automatically loaded when creating a new Pull Request. It provides a checklist and instructions for the PR workflow, including how to run the full test suite before merging. + +## Branch Protection Setup + +To enforce the full test suite before merging to `main`, configure branch protection in Forgejo: + +### Steps: + +1. Go to **Repository Settings** → **Branches** → **Protected Branches** +2. Add a new rule for branch: `main` +3. Configure the following settings: + - ☑️ **Enable Branch Protection** + - ☑️ **Require status checks to pass before merging** + - Add required check: `check-full` + - ☐ **Require approvals** (optional, based on team preference) + - ☑️ **Block if there are outstanding requests** (optional) + +### What this does: + +- The **"Merge"** button in PRs will only be enabled after `check-full` passes +- `check-full` is triggered by **promoting** a build in Drone CI (see PR template) +- This ensures all tests (including slow and UI tests) run before merging + +## Workflow + +1. **Create PR** → Fast test suite (`check-fast`) runs automatically +2. **Development** → Fast tests run on every push for quick feedback +3. **Ready to merge:** + - Remove `WIP:` from PR title + - Go to Drone CI and **promote** the build to `production` + - This triggers `check-full` (full test suite) +4. **After full tests pass** → Merge button becomes available +5. **Merge to main** → Container is built and published + +## Secrets Required + +Make sure the following secrets are configured in Drone CI: + +- `DRONE_REGISTRY_USERNAME` - For container registry +- `DRONE_REGISTRY_TOKEN` - For container registry +- `RENOVATE_TOKEN` - For Renovate bot +- `GITHUB_COM_TOKEN` - For Renovate bot (GitHub dependencies) diff --git a/CODE_GUIDELINES.md b/CODE_GUIDELINES.md index d5a437e..8c4b650 100644 --- a/CODE_GUIDELINES.md +++ b/CODE_GUIDELINES.md @@ -1703,10 +1703,10 @@ just test-all **Test Organization Best Practices:** - **Fast Tests (Standard CI):** Business logic, validations, data persistence, edge cases -- **UI Tests (Nightly CI):** Basic HTML rendering, navigation, translations, UI state -- **Performance Tests (Nightly CI):** Query optimization, large datasets, timing assertions +- **UI Tests (Full Test Suite):** Basic HTML rendering, navigation, translations, UI state +- **Performance Tests (Full Test Suite):** Query optimization, large datasets, timing assertions -This organization ensures fast feedback in standard CI while maintaining comprehensive coverage in nightly runs. +This organization ensures fast feedback in standard CI while maintaining comprehensive coverage via promotion before merge. --- ## 5. Security Guidelines diff --git a/docs/test-performance-optimization.md b/docs/test-performance-optimization.md index 6002f16..807fb90 100644 --- a/docs/test-performance-optimization.md +++ b/docs/test-performance-optimization.md @@ -27,7 +27,7 @@ This document provides a comprehensive overview of test performance optimization | Seeds tests reduction | 13 → 4 tests | ~10-16s | ✅ Completed | | Performance tests tagging | 9 tests | ~3-4s per run | ✅ Completed | | Critical test query filtering | 1 test | ~8-10s | ✅ Completed | -| Nightly suite tagging | 25 tests | ~77s per run | ✅ Completed | +| Full test suite via promotion | 25 tests | ~77s per run | ✅ Completed | | **Total Saved** | | **~98-107s** | | --- @@ -80,14 +80,14 @@ Critical deployment requirements are still covered: --- -### 2. Nightly Suite Implementation (`@tag :slow`) +### 2. Full Test Suite via Promotion (`@tag :slow`) **Date:** 2026-01-28 **Status:** ✅ Completed #### What Changed -Tests with **low risk** and **execution time >1 second** are now tagged with `@tag :slow` and excluded from standard test runs. These tests are important but not critical for every commit and can be run in nightly CI builds. +Tests with **low risk** and **execution time >1 second** are now tagged with `@tag :slow` and excluded from standard test runs. These tests are important but not critical for every commit and are run via promotion before merging to `main`. #### Tagging Criteria @@ -105,7 +105,7 @@ Tests with **low risk** and **execution time >1 second** are now tagged with `@t - ❌ Email Synchronization - ❌ Representative tests per Permission Set + Action -#### Identified Nightly Suite Tests (25 tests) +#### Identified Tests for Full Test Suite (25 tests) **1. Seeds Tests (2 tests) - 18.1s** - `"runs successfully and creates basic data"` (9.0s) @@ -155,7 +155,7 @@ just test-fast mix test --exclude slow ``` -**Slow/Nightly Tests Only:** +**Slow Tests Only:** ```bash just test-slow # or @@ -171,10 +171,10 @@ mix test #### CI/CD Integration -- **Standard CI:** Runs `mix test --exclude slow` for faster feedback loops (~6 minutes) -- **Nightly Builds:** Separate pipeline runs daily and executes `mix test` (all tests, including slow) for comprehensive coverage (~7.4 minutes) -- **Pre-Merge:** Full test suite (`mix test`) runs before merging to main -- **Manual Execution:** Can be triggered via Drone CLI or Web UI +- **Standard CI (`check-fast`):** Runs `mix test --exclude slow --exclude ui` for faster feedback loops (~6 minutes) +- **Full Test Suite (`check-full`):** Triggered via promotion before merge, executes `mix test` (all tests, including slow and UI) for comprehensive coverage (~7.4 minutes) +- **Pre-Merge:** Full test suite (`mix test`) runs via promotion before merging to main +- **Manual Execution:** Promote build to `production` in Drone CI to trigger full test suite #### Risk Assessment @@ -183,7 +183,7 @@ mix test - All tagged tests have **low risk** - they don't catch critical regressions - Core functionality remains tested (CRUD, Auth, Bootstrap) - Standard test runs are faster (~6 minutes vs ~7.4 minutes) -- Nightly builds ensure comprehensive coverage +- Full test suite runs via promotion before merge ensures comprehensive coverage - No functionality is lost, only execution timing changed **Critical Tests Remain in Fast Suite:** @@ -243,14 +243,14 @@ The test loaded **all members** from the database, not just the 2 members from t --- -### 3. Nightly Suite Analysis and Categorization +### 3. Full Test Suite Analysis and Categorization **Date:** 2026-01-28 **Status:** ✅ Completed #### Analysis Methodology -A comprehensive analysis was performed to identify tests suitable for the nightly suite based on: +A comprehensive analysis was performed to identify tests suitable for the full test suite (via promotion) based on: - **Execution time:** Tests taking >1 second - **Risk assessment:** Tests that don't catch critical regressions - **Test category:** UI/Display, workflow details, edge cases @@ -264,7 +264,7 @@ A comprehensive analysis was performed to identify tests suitable for the nightl - Email Synchronization - Representative Policy Tests (one per Permission Set + Action) -**🟡 LOW RISK - Moved to Nightly Suite:** +**🟡 LOW RISK - Moved to Full Test Suite (via Promotion):** - Seeds Tests (non-critical: smoke test, idempotency) - LiveView Display/Formatting Tests - UserLive.ShowTest (core functionality covered by Index/Form) @@ -288,9 +288,9 @@ A comprehensive analysis was performed to identify tests suitable for the nightl **Overall Risk:** ⚠️ **Low** - All moved tests have low risk and don't catch critical regressions. Core functionality remains fully tested. -#### Tests Excluded from Nightly Suite +#### Tests Excluded from Full Test Suite -The following tests were **NOT** moved to nightly suite despite being slow: +The following tests were **NOT** moved to full test suite (via promotion) despite being slow: - **Policy Tests:** Medium risk - kept in fast suite (representative tests remain) - **UserLive.FormTest:** Medium risk - core CRUD functionality @@ -303,7 +303,7 @@ The following tests were **NOT** moved to nightly suite despite being slow: ### Top 20 Slowest Tests (without `:slow`) -After implementing the nightly suite, the remaining slowest tests are: +After implementing the full test suite via promotion, the remaining slowest tests are: | Rank | Test | File | Time | Category | |------|------|------|------|----------| @@ -463,7 +463,7 @@ All optimizations maintain test coverage while improving performance: - ✅ No regression in authorization or membership fee bugs - ✅ Top 20 slowest tests: < 60 seconds (currently ~44s) - ✅ Total execution time (without `:slow`): < 10 minutes (currently 6.1 min) -- ⏳ Nightly suite execution time: < 2 minutes (currently ~1.3 min) +- ⏳ Slow tests execution time: < 2 minutes (currently ~1.3 min) #### What to Watch For @@ -543,10 +543,11 @@ mix test test/mv_web/member_live/index_member_fields_display_test.exs --slowest - Command: `mix test --only slow` or `just test-slow` - Excluded from standard CI runs -**Nightly CI Builds:** +**Full Test Suite (via Promotion):** +- Triggered by promoting a build to `production` in Drone CI - Runs all tests (`mix test`) for comprehensive coverage - Execution time: ~7.4 minutes -- Ensures full test coverage including slow/performance tests +- Required before merging to `main` (enforced via branch protection) **All Tests:** - Includes both fast and slow tests @@ -636,16 +637,16 @@ test/ - ✅ Optimized critical test with query filtering - ✅ Created slow test suite infrastructure - ✅ Updated CI/CD to exclude slow tests from standard runs -- ✅ Added nightly CI pipeline for slow tests +- ✅ Added promotion-based full test suite pipeline (`check-full`) **Time Saved:** ~21-30 seconds per test run -### 2026-01-28: Nightly Suite Implementation +### 2026-01-28: Full Test Suite via Promotion Implementation **Completed:** -- ✅ Analyzed all tests for nightly suite candidates +- ✅ Analyzed all tests for full test suite candidates - ✅ Identified 36 tests with low risk and >1s execution time -- ✅ Tagged 25 tests with `@tag :slow` for nightly suite +- ✅ Tagged 25 tests with `@tag :slow` for full test suite (via promotion) - ✅ Categorized tests by risk level and execution time - ✅ Documented tagging criteria and guidelines @@ -667,7 +668,7 @@ test/ - **Time saved:** ~77 seconds (17% reduction) **Next Steps:** -- ⏳ Monitor nightly suite execution in CI +- ⏳ Monitor full test suite execution via promotion in CI - ⏳ Optimize remaining slow tests (Policy tests, etc.) - ⏳ Further optimize Seeds tests (Priority 3) @@ -702,11 +703,11 @@ A: Tag it with `@tag :slow` for individual tests or `@describetag :slow` for des **Q: Can I run slow tests locally?** A: Yes, use `just test-slow` or `mix test --only slow`. They're excluded from standard runs for faster feedback. -**Q: What is the "nightly suite"?** -A: The "nightly suite" refers to the nightly CI pipeline that runs **all tests** (`mix test`), including slow tests. Tests tagged with `@tag :slow` or `@describetag :slow` are excluded from standard CI runs for faster feedback, but are included in the nightly full test suite for comprehensive coverage. +**Q: What is the "full test suite"?** +A: The full test suite runs **all tests** (`mix test`), including slow and UI tests. Tests tagged with `@tag :slow` or `@describetag :slow` are excluded from standard CI runs (`check-fast`) for faster feedback, but are included when promoting a build to `production` (`check-full`) before merging to `main`. **Q: Which tests should I tag as `:slow`?** A: Tag tests with `@tag :slow` if they: (1) take >1 second, (2) have low risk (not critical for catching regressions), and (3) test UI/Display/Formatting or workflow details. See "Test Tagging Guidelines" section for details. -**Q: What if a slow test fails in nightly builds?** -A: If a test in the nightly suite fails, investigate the failure. If it indicates a critical regression, consider moving it back to the fast suite. If it's a flaky test, fix the test itself. +**Q: What if a slow test fails in the full test suite?** +A: If a test in the full test suite fails, investigate the failure. If it indicates a critical regression, consider moving it back to the fast suite. If it's a flaky test, fix the test itself. The merge will be blocked until all tests pass.