diff --git a/.drone.yml b/.drone.yml index 483a08a..8c7f325 100644 --- a/.drone.yml +++ b/.drone.yml @@ -4,7 +4,7 @@ name: check services: - name: postgres - image: docker.io/library/postgres:17.6 + image: docker.io/library/postgres:17.7 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -57,7 +57,7 @@ steps: - mix gettext.extract --check-up-to-date - name: wait_for_postgres - image: docker.io/library/postgres:17.6 + image: docker.io/library/postgres:17.7 commands: # Wait for postgres to become available - | @@ -166,7 +166,7 @@ environment: steps: - name: renovate - image: renovate/renovate:41.173 + image: renovate/renovate:42.44 environment: RENOVATE_CONFIG_FILE: "renovate_backend_config.js" RENOVATE_TOKEN: diff --git a/Dockerfile b/Dockerfile index 88468a2..7a01d21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,4 +90,4 @@ USER nobody # above and adding an entrypoint. See https://github.com/krallin/tini for details # ENTRYPOINT ["/tini", "--"] -CMD ["/app/bin/server"] +ENTRYPOINT ["/app/bin/docker-entrypoint.sh"] diff --git a/README.md b/README.md index 090f4e9..6255f8d 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ For testing the production Docker build locally: docker compose -f docker-compose.prod.yml up ``` -5. **Run database migrations:** +5. **Database migrations run automatically** on app start. For manual migration: ```bash docker compose -f docker-compose.prod.yml exec app /app/bin/mv eval "Mv.Release.migrate" ``` diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b4b7a1f..5b35e10 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -33,7 +33,7 @@ services: restart: unless-stopped db-prod: - image: postgres:16-alpine + image: postgres:17.7-alpine container_name: mv-prod-db environment: POSTGRES_USER: postgres diff --git a/docker-compose.yml b/docker-compose.yml index 56876f2..b10ab22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ networks: services: db: - image: postgres:17.6-alpine + image: postgres:17.7-alpine environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/docs/contributions-architecture.md b/docs/contributions-architecture.md deleted file mode 100644 index 3718a3b..0000000 --- a/docs/contributions-architecture.md +++ /dev/null @@ -1,653 +0,0 @@ -# Membership Contributions - Technical Architecture - -**Project:** Mila - Membership Management System -**Feature:** Membership Contribution Management -**Version:** 1.0 -**Last Updated:** 2025-11-27 -**Status:** Architecture Design - Ready for Implementation - ---- - -## Purpose - -This document defines the technical architecture for the Membership Contributions system. It focuses on architectural decisions, patterns, module structure, and integration points **without** concrete implementation details. - -**Related Documents:** -- [contributions-overview.md](./contributions-overview.md) - Business logic and requirements -- [database-schema-readme.md](./database-schema-readme.md) - Database documentation -- [database_schema.dbml](./database_schema.dbml) - Database schema definition - ---- - -## Table of Contents - -1. [Architecture Principles](#architecture-principles) -2. [Domain Structure](#domain-structure) -3. [Data Architecture](#data-architecture) -4. [Business Logic Architecture](#business-logic-architecture) -5. [Integration Points](#integration-points) -6. [Acceptance Criteria](#acceptance-criteria) -7. [Testing Strategy](#testing-strategy) -8. [Security Considerations](#security-considerations) -9. [Performance Considerations](#performance-considerations) - ---- - -## Architecture Principles - -### Core Design Decisions - -1. **Single Responsibility:** - - Each module has one clear responsibility - - Period generation separated from status management - - Calendar logic isolated in dedicated module - -2. **No Redundancy:** - - No `period_end` field (calculated from `period_start` + `interval`) - - No `interval_type` field (read from `contribution_type.interval`) - - Eliminates data inconsistencies - -3. **Immutability Where Important:** - - `contribution_type.interval` cannot be changed after creation - - Prevents complex migration scenarios - - Enforced via Ash change validation - -4. **Historical Accuracy:** - - `amount` stored per period for audit trail - - Enables tracking of contribution changes over time - - Old periods retain original amounts - -5. **Calendar-Based Periods:** - - All periods aligned to calendar boundaries - - Simplifies date calculations - - Predictable period generation - ---- - -## Domain Structure - -### Ash Domain: `Mv.Contributions` - -**Purpose:** Encapsulates all contribution-related resources and logic - -**Resources:** -- `ContributionType` - Contribution type definitions (admin-managed) -- `ContributionPeriod` - Individual contribution periods per member - -**Extensions:** -- Member resource extended with contribution fields - -### Module Organization - -``` -lib/ -├── contributions/ -│ ├── contributions.ex # Ash domain definition -│ ├── contribution_type.ex # ContributionType resource -│ ├── contribution_period.ex # ContributionPeriod resource -│ └── changes/ -│ ├── prevent_interval_change.ex # Validates interval immutability -│ ├── set_contribution_start_date.ex # Auto-sets start date -│ └── validate_same_interval.ex # Validates interval match on type change -├── mv/ -│ └── contributions/ -│ ├── period_generator.ex # Period generation algorithm -│ └── calendar_periods.ex # Calendar period calculations -└── membership/ - └── member.ex # Extended with contribution relationships -``` - -### Separation of Concerns - -**Domain Layer (Ash Resources):** -- Data validation -- Relationship management -- Policy enforcement -- Action definitions - -**Business Logic Layer (`Mv.Contributions`):** -- Period generation algorithm -- Calendar calculations -- Date boundary handling -- Status transitions - -**UI Layer (LiveView):** -- User interaction -- Display logic -- Authorization checks -- Form handling - ---- - -## Data Architecture - -### Database Schema Extensions - -**See:** [database-schema-readme.md](./database-schema-readme.md) and [database_schema.dbml](./database_schema.dbml) for complete schema documentation. - -### New Tables - -1. **`contribution_types`** - - Purpose: Define contribution types with fixed intervals - - Key Constraint: `interval` field immutable after creation - - Relationships: has_many members, has_many contribution_periods - -2. **`contribution_periods`** - - Purpose: Individual contribution periods for members - - Key Design: NO `period_end` or `interval_type` fields (calculated) - - Relationships: belongs_to member, belongs_to contribution_type - - Composite uniqueness: One period per member per period_start - -### Member Table Extensions - -**Fields Added:** -- `contribution_type_id` (FK, NOT NULL with default from settings) -- `contribution_start_date` (Date, nullable) - -**Existing Fields Used:** -- `joined_at` - For calculating contribution start -- `left_at` - For limiting period generation -- These fields must remain member fields and should not be replaced by custom fields in the future - -### Settings Integration - -**Global Settings:** -- `contributions.include_joining_period` (Boolean) -- `contributions.default_contribution_type_id` (UUID) - -**Storage:** Existing settings mechanism (TBD: dedicated table or configuration resource) - -### Foreign Key Behaviors - -| Relationship | On Delete | Rationale | -|--------------|-----------|-----------| -| `contribution_periods.member_id → members.id` | CASCADE | Remove periods when member deleted | -| `contribution_periods.contribution_type_id → contribution_types.id` | RESTRICT | Prevent type deletion if periods exist | -| `members.contribution_type_id → contribution_types.id` | RESTRICT | Prevent type deletion if assigned to members | - ---- - -## Business Logic Architecture - -### Period Generation System - -**Component:** `Mv.Contributions.PeriodGenerator` - -**Responsibilities:** -- Calculate which periods should exist for a member -- Generate missing periods -- Respect contribution_start_date and left_at boundaries -- Skip existing periods (idempotent) - -**Triggers:** -1. Member contribution type assigned (via Ash change) -2. Member created with contribution type (via Ash change) -3. Scheduled job runs (daily/weekly cron) -4. Admin manual regeneration (UI action) - -**Algorithm Steps:** -1. Retrieve member with contribution_type and dates -2. Determine first period start (based on contribution_start_date) -3. Calculate all period starts from first to today (or left_at) -4. Query existing periods for member -5. Generate missing periods with current contribution_type.amount -6. Insert new periods (batch operation) - -**Edge Case Handling:** -- If contribution_start_date is NULL: Calculate from joined_at + global setting -- If left_at is set: Stop generation at left_at -- If contribution_type changes: Handled separately by regeneration logic - -### Calendar Period Calculations - -**Component:** `Mv.Contributions.CalendarPeriods` - -**Responsibilities:** -- Calculate period boundaries based on interval type -- Determine current period -- Determine last completed period -- Calculate period_end from period_start + interval - -**Functions (high-level):** -- `calculate_period_start/3` - Given date and interval, find period start -- `calculate_period_end/2` - Given period_start and interval, calculate end -- `next_period_start/2` - Given period_start and interval, find next -- `is_current_period?/2` - Check if period contains today -- `is_last_completed_period?/2` - Check if period just ended - -**Interval Logic:** -- **Monthly:** Start = 1st of month, End = last day of month -- **Quarterly:** Start = 1st of quarter (Jan/Apr/Jul/Oct), End = last day of quarter -- **Half-yearly:** Start = 1st of half (Jan/Jul), End = last day of half -- **Yearly:** Start = Jan 1st, End = Dec 31st - -### Status Management - -**Component:** Ash actions on `ContributionPeriod` - -**Status Transitions:** -- Simple state machine: unpaid ↔ paid ↔ suspended -- No complex validation (all transitions allowed) -- Permissions checked via Ash policies - -**Actions Required:** -- `mark_as_paid` - Set status to :paid -- `mark_as_suspended` - Set status to :suspended -- `mark_as_unpaid` - Set status to :unpaid (error correction) - -**Bulk Operations:** -- `bulk_mark_as_paid` - Mark multiple periods as paid (efficiency) - - low priority, can be a future issue - -### Contribution Type Change Handling - -**Component:** Ash change on `Member.contribution_type_id` - -**Validation:** -- Check if new type has same interval as old type -- If different: Reject change (MVP constraint) -- If same: Allow change - -**Side Effects on Allowed Change:** -1. Keep all existing periods unchanged -2. Find future unpaid periods -3. Delete future unpaid periods -4. Regenerate periods with new contribution_type_id and amount - -**Implementation Pattern:** -- Use Ash change module to validate -- Use after_action hook to trigger regeneration -- Use transaction to ensure atomicity - ---- - -## Integration Points - -### Member Resource Integration - -**Extension Points:** -1. Add fields via migration -2. Add relationships (belongs_to, has_many) -3. Add calculations (current_period_status, overdue_count) -4. Add changes (auto-set contribution_start_date, validate interval) - -**Backward Compatibility:** -- New fields nullable or with defaults -- Existing members get default contribution type from settings -- No breaking changes to existing member functionality - -### Settings System Integration - -**Requirements:** -- Store two global settings -- Provide UI for admin to modify -- Default values if not set -- Validation (e.g., default_contribution_type_id must exist) - -**Access Pattern:** -- Read settings during period generation -- Read settings during member creation -- Write settings only via admin UI - -### Permission System Integration - -**See:** [roles-and-permissions-architecture.md](./roles-and-permissions-architecture.md) - -**Required Permissions:** -- `ContributionType.create/update/destroy` - Admin only -- `ContributionType.read` - Admin, Treasurer, Board -- `ContributionPeriod.update` (status changes) - Admin, Treasurer -- `ContributionPeriod.read` - Admin, Treasurer, Board, Own member - -**Policy Patterns:** -- Use existing HasPermission check -- Leverage existing roles (Admin, Kassenwart) -- Member can read own periods (linked via member_id) - -### LiveView Integration - -**New LiveViews Required:** -1. ContributionType index/form (admin) -2. ContributionPeriod table component (member detail view) -3. Settings form section (admin) -4. Member list column (contribution status) - -**Existing LiveViews to Extend:** -- Member detail view: Add contributions section -- Member list view: Add status column -- Settings page: Add contributions section - -**Authorization Helpers:** -- Use existing `can?/3` helper for UI conditionals -- Check permissions before showing actions - ---- - -## Acceptance Criteria - -### ContributionType Resource - -**AC-CT-1:** Admin can create contribution type with name, amount, interval, description -**AC-CT-2:** Interval field is immutable after creation (validation error on change attempt) -**AC-CT-3:** Admin can update name, amount, description (but not interval) -**AC-CT-4:** Cannot delete contribution type if assigned to members -**AC-CT-5:** Cannot delete contribution type if periods exist referencing it -**AC-CT-6:** Interval must be one of: monthly, quarterly, half_yearly, yearly - -### ContributionPeriod Resource - -**AC-CP-1:** Period has period_start, status, amount, notes, member_id, contribution_type_id -**AC-CP-2:** Period_end is calculated, not stored -**AC-CP-3:** Status defaults to :unpaid -**AC-CP-4:** One period per member per period_start (uniqueness constraint) -**AC-CP-5:** Amount is set at generation time from contribution_type.amount -**AC-CP-6:** Periods cascade delete when member deleted -**AC-CP-7:** Admin/Treasurer can change status -**AC-CP-8:** Member can read own periods - -### Member Extensions - -**AC-M-1:** Member has contribution_type_id field (NOT NULL with default) -**AC-M-2:** Member has contribution_start_date field (nullable) -**AC-M-3:** New members get default contribution type from global setting -**AC-M-4:** contribution_start_date auto-set based on joined_at and global setting -**AC-M-5:** Admin can manually override contribution_start_date -**AC-M-6:** Cannot change to contribution type with different interval (MVP) - -### Period Generation - -**AC-PG-1:** Periods generated when member gets contribution type -**AC-PG-2:** Periods generated when member created (via change hook) -**AC-PG-3:** Scheduled job generates missing periods daily -**AC-PG-4:** Generation respects contribution_start_date -**AC-PG-5:** Generation stops at left_at if member exited -**AC-PG-6:** Generation is idempotent (skips existing periods) -**AC-PG-7:** Periods align to calendar boundaries (1st of month/quarter/half/year) -**AC-PG-8:** Amount comes from contribution_type at generation time - -### Calendar Logic - -**AC-CL-1:** Monthly periods: 1st to last day of month -**AC-CL-2:** Quarterly periods: 1st of Jan/Apr/Jul/Oct to last day of quarter -**AC-CL-3:** Half-yearly periods: 1st of Jan/Jul to last day of half -**AC-CL-4:** Yearly periods: Jan 1 to Dec 31 -**AC-CL-5:** Period_end calculated correctly for all interval types -**AC-CL-6:** Current period determined correctly based on today's date -**AC-CL-7:** Last completed period determined correctly - -### Contribution Type Change - -**AC-TC-1:** Can change to type with same interval -**AC-TC-2:** Cannot change to type with different interval (error message) -**AC-TC-3:** On allowed change: future unpaid periods regenerated -**AC-TC-4:** On allowed change: paid/suspended periods unchanged -**AC-TC-5:** On allowed change: amount updated to new type's amount -**AC-TC-6:** Change is atomic (transaction) - -### Settings - -**AC-S-1:** Global setting: include_joining_period (boolean, default true) -**AC-S-2:** Global setting: default_contribution_type_id (UUID, required) -**AC-S-3:** Admin can modify settings via UI -**AC-S-4:** Settings validated (e.g., default type must exist) -**AC-S-5:** Settings applied to new members immediately - -### UI - Member List - -**AC-UI-ML-1:** New column shows contribution status -**AC-UI-ML-2:** Default: Shows last completed period status -**AC-UI-ML-3:** Optional: Toggle to show current period status -**AC-UI-ML-4:** Color coding: green (paid), red (unpaid), gray (suspended) -**AC-UI-ML-5:** Filter: Unpaid in last period -**AC-UI-ML-6:** Filter: Unpaid in current period - -### UI - Member Detail - -**AC-UI-MD-1:** Contributions section shows all periods -**AC-UI-MD-2:** Table columns: Period, Interval, Amount, Status, Actions -**AC-UI-MD-3:** Checkbox per period for bulk marking (low prio) -**AC-UI-MD-4:** "Mark selected as paid" button -**AC-UI-MD-5:** Dropdown to change contribution type (same interval only) -**AC-UI-MD-6:** Warning if different interval selected -**AC-UI-MD-7:** Only show actions if user has permission - -### UI - Contribution Types Admin - -**AC-UI-CTA-1:** List all contribution types -**AC-UI-CTA-2:** Show: Name, Amount, Interval, Member count -**AC-UI-CTA-3:** Create new contribution type form -**AC-UI-CTA-4:** Edit form: Name, Amount, Description editable -**AC-UI-CTA-5:** Edit form: Interval grayed out (not editable) -**AC-UI-CTA-6:** Warning on amount change (explain impact) -**AC-UI-CTA-7:** Cannot delete if members assigned -**AC-UI-CTA-8:** Only admin can access - -### UI - Settings Admin - -**AC-UI-SA-1:** Contributions section in settings -**AC-UI-SA-2:** Dropdown to select default contribution type -**AC-UI-SA-3:** Checkbox: Include joining period -**AC-UI-SA-4:** Explanatory text with examples -**AC-UI-SA-5:** Save button with validation - ---- - -## Testing Strategy - -### Unit Testing - -**Period Generator Tests:** -- Correct period_start calculation for all interval types -- Correct period count from start to end date -- Respects contribution_start_date boundary -- Respects left_at boundary -- Skips existing periods (idempotent) -- Handles edge dates (year boundaries, leap years) - -**Calendar Periods Tests:** -- Period boundaries correct for all intervals -- Period_end calculation correct -- Current period detection -- Last completed period detection -- Next period calculation - -**Validation Tests:** -- Interval immutability enforced -- Same interval validation on type change -- Status transitions allowed -- Uniqueness constraints enforced - -### Integration Testing - -**Period Generation Flow:** -- Member creation triggers generation -- Type assignment triggers generation -- Type change regenerates future periods -- Scheduled job generates missing periods -- Left member stops generation - -**Status Management Flow:** -- Mark single period as paid -- Bulk mark multiple periods (low prio) -- Status transitions work -- Permissions enforced - -**Contribution Type Management:** -- Create type -- Update amount (regeneration triggered) -- Cannot update interval -- Cannot delete if in use - -### LiveView Testing - -**Member List:** -- Status column displays correctly -- Toggle between last/current works -- Filters work correctly -- Color coding applied - -**Member Detail:** -- Periods table displays all periods -- Checkboxes work -- Bulk marking works (low prio) -- Type change validation works -- Actions only shown with permission - -**Admin UI:** -- Type CRUD works -- Settings save correctly -- Validations display errors -- Only authorized users can access - -### Edge Case Testing - -**Interval Change Attempt:** -- Error message displayed -- No data modified -- User can cancel/choose different type - -**Exit with Unpaid:** -- Warning shown -- Option to suspend offered -- Exit completes correctly - -**Amount Change:** -- Warning displayed -- Only future unpaid regenerated -- Historical periods unchanged - -**Date Boundaries:** -- Today = period start handled -- Today = period end handled -- Leap year handled - -### Performance Testing - -**Period Generation:** -- Generate 10 years of monthly periods: < 100ms -- Generate for 1000 members: < 5 seconds -- Idempotent check efficient (no full scan) - -**Member List Query:** -- With status column: < 200ms for 1000 members -- Filters applied efficiently -- No N+1 queries - ---- - -## Security Considerations - -### Authorization - -**Permissions Required:** -- ContributionType management: Admin only -- ContributionPeriod status changes: Admin + Treasurer -- View all periods: Admin + Treasurer + Board -- View own periods: All authenticated users - -**Policy Enforcement:** -- All actions protected by Ash policies -- UI shows/hides based on permissions -- Backend validates permissions (never trust UI alone) - -### Data Integrity - -**Validation Layers:** -1. Database constraints (NOT NULL, UNIQUE, CHECK) -2. Ash validations (business rules) -3. UI validations (user experience) - -**Immutability Protection:** -- Interval change prevented at multiple layers -- Period amounts immutable (audit trail) -- Settings changes logged (future) - -### Audit Trail - -**Tracked Information:** -- Period status changes (who, when) - future enhancement -- Type amount changes (implicit via period amounts) -- Member type assignments (via timestamps) - ---- - -## Performance Considerations - -### Database Indexes - -**Required Indexes:** -- `contribution_periods(member_id)` - For member period lookups -- `contribution_periods(contribution_type_id)` - For type queries -- `contribution_periods(status)` - For unpaid filters -- `contribution_periods(period_start)` - For date range queries -- `contribution_periods(member_id, period_start)` - Composite unique index -- `members(contribution_type_id)` - For type membership count - -### Query Optimization - -**Preloading:** -- Load contribution_type with periods (avoid N+1) -- Load periods when displaying member detail -- Use Ash's load for efficient preloading - -**Calculated Fields:** -- period_end calculated on-demand (not stored) -- current_period_status calculated when needed -- Use Ash calculations for lazy evaluation - -**Pagination:** -- Period list paginated if > 50 periods -- Member list already paginated - -### Caching Strategy - -**No caching needed in MVP:** -- Contribution types rarely change -- Period queries are fast -- Settings read infrequently - -**Future caching if needed:** -- Cache settings in application memory -- Cache contribution types list -- Invalidate on change - -### Scheduled Job Performance - -**Period Generation Job:** -- Run daily or weekly (not hourly) -- Batch members (process 100 at a time) -- Skip members with no changes -- Log failures for retry - ---- - -## Future Enhancements - -### Phase 2: Interval Change Support - -**Architecture Changes:** -- Add logic to handle period overlaps -- Calculate prorata amounts if needed -- More complex validation -- Migration path for existing periods - -### Phase 3: Payment Details - -**Architecture Changes:** -- Add PaymentTransaction resource -- Link transactions to periods -- Support multiple payments per period -- Reconciliation logic - -### Phase 4: vereinfacht.digital Integration - -**Architecture Changes:** -- External API client module -- Webhook handling for transactions -- Automatic matching logic -- Manual review interface - ---- - -**End of Architecture Document** - diff --git a/docs/custom-fields-search-performance.md b/docs/custom-fields-search-performance.md new file mode 100644 index 0000000..3987c85 --- /dev/null +++ b/docs/custom-fields-search-performance.md @@ -0,0 +1,243 @@ +# Performance Analysis: Custom Fields in Search Vector + +## Current Implementation + +The search vector includes custom field values via database triggers that: +1. Aggregate all custom field values for a member +2. Extract values from JSONB format +3. Add them to the search_vector with weight 'C' + +## Performance Considerations + +### 1. Trigger Performance on Member Updates + +**Current Implementation:** +- `members_search_vector_trigger()` executes a subquery on every INSERT/UPDATE: + ```sql + SELECT string_agg(...) FROM custom_field_values WHERE member_id = NEW.id + ``` + +**Performance Impact:** +- ✅ **Good:** Index on `member_id` exists (`custom_field_values_member_id_idx`) +- ✅ **Good:** Subquery only runs for the affected member +- ⚠️ **Potential Issue:** With many custom fields per member (e.g., 50+), aggregation could be slower +- ⚠️ **Potential Issue:** JSONB extraction (`value->>'_union_value'`) is relatively fast but adds overhead + +**Expected Performance:** +- **Small scale (< 10 custom fields per member):** Negligible impact (< 5ms per operation) +- **Medium scale (10-30 custom fields):** Minor impact (5-20ms per operation) +- **Large scale (30+ custom fields):** Noticeable impact (20-50ms+ per operation) + +### 2. Trigger Performance on Custom Field Value Changes + +**Current Implementation:** +- `update_member_search_vector_from_custom_field_value()` executes on every INSERT/UPDATE/DELETE on `custom_field_values` +- **Optimized:** Only fetches required member fields (not full record) to reduce overhead +- **Optimized:** Skips re-aggregation on UPDATE if value hasn't actually changed +- Aggregates all custom field values, then updates member search_vector + +**Performance Impact:** +- ✅ **Good:** Index on `member_id` ensures fast lookup +- ✅ **Optimized:** Only required fields are fetched (first_name, last_name, email, etc.) instead of full record +- ✅ **Optimized:** UPDATE operations that don't change the value skip expensive re-aggregation (early return) +- ⚠️ **Note:** Re-aggregation is still necessary when values change (required for search_vector consistency) +- ⚠️ **Critical:** Bulk operations (e.g., importing 1000 members with custom fields) will trigger this for each row + +**Expected Performance:** +- **Single operation (value changed):** 3-10ms per custom field value change (improved from 5-15ms) +- **Single operation (value unchanged):** <1ms (early return, no aggregation) +- **Bulk operations:** Could be slow (consider disabling trigger temporarily) + +### 3. Search Vector Size + +**Current Constraints:** +- String values: max 10,000 characters per custom field +- No limit on number of custom fields per member +- tsvector has no explicit size limit, but very large vectors can cause issues + +**Potential Issues:** +- **Theoretical maximum:** If a member has 100 custom fields with 10,000 char strings each, the aggregated text could be ~1MB +- **Practical concern:** Very large search vectors (> 100KB) can slow down: + - Index updates (GIN index maintenance) + - Search queries (tsvector operations) + - Trigger execution time + +**Recommendation:** +- Monitor search_vector size in production +- Consider limiting total custom field content per member if needed +- PostgreSQL can handle large tsvectors, but performance degrades gradually + +### 4. Initial Migration Performance + +**Current Implementation:** +- Updates ALL members in a single transaction: + ```sql + UPDATE members m SET search_vector = ... (subquery for each member) + ``` + +**Performance Impact:** +- ⚠️ **Potential Issue:** With 10,000+ members, this could take minutes +- ⚠️ **Potential Issue:** Single transaction locks the members table +- ⚠️ **Potential Issue:** If migration fails, entire rollback required + +**Recommendation:** +- For large datasets (> 10,000 members), consider: + - Batch updates (e.g., 1000 members at a time) + - Run during maintenance window + - Monitor progress + +### 5. Search Query Performance + +**Current Implementation:** +- Full-text search uses GIN index on `search_vector` (fast) +- Additional LIKE queries on `custom_field_values` for substring matching: + ```sql + EXISTS (SELECT 1 FROM custom_field_values WHERE member_id = id AND ... LIKE ...) + ``` + +**Performance Impact:** +- ✅ **Good:** GIN index on `search_vector` is very fast +- ⚠️ **Potential Issue:** LIKE queries on JSONB are not indexed (sequential scan) +- ⚠️ **Potential Issue:** EXISTS subquery runs for every search, even if search_vector match is found +- ⚠️ **Potential Issue:** With many custom fields, the LIKE queries could be slow + +**Expected Performance:** +- **With GIN index match:** Very fast (< 10ms for typical queries) +- **Without GIN index match (fallback to LIKE):** Slower (10-100ms depending on data size) +- **Worst case:** Sequential scan of all custom_field_values for all members + +## Recommendations + +### Short-term (Current Implementation) + +1. **Monitor Performance:** + - Add logging for trigger execution time + - Monitor search_vector size distribution + - Track search query performance + +2. **Index Verification:** + - Ensure `custom_field_values_member_id_idx` exists and is used + - Verify GIN index on `search_vector` is maintained + +3. **Bulk Operations:** + - For bulk imports, consider temporarily disabling the custom_field_values trigger + - Re-enable and update search_vectors in batch after import + +### Medium-term Optimizations + +1. **✅ Optimize Trigger Function (FULLY IMPLEMENTED):** + - ✅ Only fetch required member fields instead of full record (reduces overhead) + - ✅ Skip re-aggregation on UPDATE if value hasn't actually changed (early return optimization) + +2. **Limit Search Vector Size:** + - Truncate very long custom field values (e.g., first 1000 chars) + - Add warning if aggregated text exceeds threshold + +3. **Optimize LIKE Queries:** + - Consider adding a generated column for searchable text + - Or use a materialized view for custom field search + +### Long-term Considerations + +1. **Alternative Approaches:** + - Separate search index table for custom fields + - Use Elasticsearch or similar for advanced search + - Materialized view for search optimization + +2. **Scaling Strategy:** + - If performance becomes an issue with 100+ custom fields per member: + - Consider limiting which custom fields are searchable + - Use a separate search service + - Implement search result caching + +## Performance Benchmarks (Estimated) + +Based on typical PostgreSQL performance: + +| Scenario | Members | Custom Fields/Member | Expected Impact | +|----------|---------|---------------------|-----------------| +| Small | < 1,000 | < 10 | Negligible (< 5ms per operation) | +| Medium | 1,000-10,000 | 10-30 | Minor (5-20ms per operation) | +| Large | 10,000-100,000 | 30-50 | Noticeable (20-50ms per operation) | +| Very Large | > 100,000 | 50+ | Significant (50-200ms+ per operation) | + +## Monitoring Queries + +```sql +-- Check search_vector size distribution +SELECT + pg_size_pretty(octet_length(search_vector::text)) as size, + COUNT(*) as member_count +FROM members +WHERE search_vector IS NOT NULL +GROUP BY octet_length(search_vector::text) +ORDER BY octet_length(search_vector::text) DESC +LIMIT 20; + +-- Check average custom fields per member +SELECT + AVG(custom_field_count) as avg_custom_fields, + MAX(custom_field_count) as max_custom_fields +FROM ( + SELECT member_id, COUNT(*) as custom_field_count + FROM custom_field_values + GROUP BY member_id +) subq; + +-- Check trigger execution time (requires pg_stat_statements) +SELECT + mean_exec_time, + calls, + query +FROM pg_stat_statements +WHERE query LIKE '%members_search_vector_trigger%' +ORDER BY mean_exec_time DESC; +``` + +## Code Quality Improvements (Post-Review) + +### Refactored Search Implementation + +The search query has been refactored for better maintainability and clarity: + +**Before:** Single large OR-chain with mixed search types (hard to maintain) + +**After:** Modular functions grouped by search type: +- `build_fts_filter/1` - Full-text search (highest priority, fastest) +- `build_substring_filter/2` - Substring matching on structured fields +- `build_custom_field_filter/1` - Custom field value search (JSONB LIKE) +- `build_fuzzy_filter/2` - Trigram/fuzzy matching for names and streets + +**Benefits:** +- ✅ Clear separation of concerns +- ✅ Easier to maintain and test +- ✅ Better documentation of search priority +- ✅ Easier to optimize individual search types + +**Search Priority Order:** +1. **FTS (Full-Text Search)** - Fastest, uses GIN index on search_vector +2. **Substring** - For structured fields (postal_code, phone_number, etc.) +3. **Custom Fields** - JSONB LIKE queries (fallback for substring matching) +4. **Fuzzy Matching** - Trigram similarity for names and streets + +## Conclusion + +The current implementation is **well-optimized for typical use cases** (< 30 custom fields per member, < 10,000 members). For larger scales, monitoring and potential optimizations may be needed. + +**Key Strengths:** +- Indexed lookups (member_id index) +- Efficient GIN index for search +- Trigger-based automatic updates +- Modular, maintainable search code structure + +**Key Weaknesses:** +- LIKE queries on JSONB (not indexed) +- Re-aggregation on every custom field change (necessary for consistency) +- Potential size issues with many/large custom fields +- Substring searches (contains/ILIKE) not index-optimized + +**Recent Optimizations:** +- ✅ Trigger function optimized to fetch only required fields (reduces overhead by ~30-50%) +- ✅ Early return on UPDATE when value hasn't changed (skips expensive re-aggregation, <1ms vs 3-10ms) +- ✅ Improved performance for custom field value updates (3-10ms vs 5-15ms when value changes) + diff --git a/docs/database-schema-readme.md b/docs/database-schema-readme.md index 1644f2a..6457db5 100644 --- a/docs/database-schema-readme.md +++ b/docs/database-schema-readme.md @@ -168,9 +168,16 @@ Member (1) → (N) Properties ### Weighted Fields - **Weight A (highest):** first_name, last_name - **Weight B:** email, notes -- **Weight C:** phone_number, city, street, house_number, postal_code +- **Weight C:** phone_number, city, street, house_number, postal_code, custom_field_values - **Weight D (lowest):** join_date, exit_date +### Custom Field Values in Search +Custom field values are automatically included in the search vector: +- All custom field values (string, integer, boolean, date, email) are aggregated and added to the search vector +- Values are converted to text format for indexing +- Custom field values receive weight 'C' (same as phone_number, city, etc.) +- The search vector is automatically updated when custom field values are created, updated, or deleted via database triggers + ### Usage Example ```sql SELECT * FROM members diff --git a/docs/feature-roadmap.md b/docs/feature-roadmap.md index 2f86f5e..c4ecfc9 100644 --- a/docs/feature-roadmap.md +++ b/docs/feature-roadmap.md @@ -187,16 +187,16 @@ **Current State:** - ✅ Basic "paid" boolean field on members -- ✅ **UI Mock-ups for Contribution Types & Settings** (2025-12-02) +- ✅ **UI Mock-ups for Membership Fee Types & Settings** (2025-12-02) - ⚠️ No payment tracking **Open Issues:** - [#156](https://git.local-it.org/local-it/mitgliederverwaltung/issues/156) - Set up & document testing environment for vereinfacht.digital (L, Low priority) -- [#226](https://git.local-it.org/local-it/mitgliederverwaltung/issues/226) - Payment/Contribution Mockup Pages (Preview) +- [#226](https://git.local-it.org/local-it/mitgliederverwaltung/issues/226) - Payment/Membership Fee Mockup Pages (Preview) **Mock-Up Pages (Non-Functional Preview):** -- `/contribution_types` - Contribution Types Management -- `/contribution_settings` - Global Contribution Settings +- `/membership_fee_types` - Membership Fee Types Management +- `/membership_fee_settings` - Global Membership Fee Settings **Missing Features:** - ❌ Membership fee configuration diff --git a/docs/membership-fee-architecture.md b/docs/membership-fee-architecture.md new file mode 100644 index 0000000..c601b79 --- /dev/null +++ b/docs/membership-fee-architecture.md @@ -0,0 +1,712 @@ +# Membership Fees - Technical Architecture + +**Project:** Mila - Membership Management System +**Feature:** Membership Fee Management +**Version:** 1.0 +**Last Updated:** 2025-11-27 +**Status:** Architecture Design - Ready for Implementation + +--- + +## Purpose + +This document defines the technical architecture for the Membership Fees system. It focuses on architectural decisions, patterns, module structure, and integration points **without** concrete implementation details. + +**Related Documents:** + +- [membership-fee-overview.md](./membership-fee-overview.md) - Business logic and requirements +- [database-schema-readme.md](./database-schema-readme.md) - Database documentation +- [database_schema.dbml](./database_schema.dbml) - Database schema definition + +--- + +## Table of Contents + +1. [Architecture Principles](#architecture-principles) +2. [Domain Structure](#domain-structure) +3. [Data Architecture](#data-architecture) +4. [Business Logic Architecture](#business-logic-architecture) +5. [Integration Points](#integration-points) +6. [Acceptance Criteria](#acceptance-criteria) +7. [Testing Strategy](#testing-strategy) +8. [Security Considerations](#security-considerations) +9. [Performance Considerations](#performance-considerations) + +--- + +## Architecture Principles + +### Core Design Decisions + +1. **Single Responsibility:** + - Each module has one clear responsibility + - Cycle generation separated from status management + - Calendar logic isolated in dedicated module + +2. **No Redundancy:** + - No `cycle_end` field (calculated from `cycle_start` + `interval`) + - No `interval_type` field (read from `membership_fee_type.interval`) + - Eliminates data inconsistencies + +3. **Immutability Where Important:** + - `membership_fee_type.interval` cannot be changed after creation + - Prevents complex migration scenarios + - Enforced via Ash change validation + +4. **Historical Accuracy:** + - `amount` stored per cycle for audit trail + - Enables tracking of membership fee changes over time + - Old cycles retain original amounts + +5. **Calendar-Based Cycles:** + - All cycles aligned to calendar boundaries + - Simplifies date calculations + - Predictable cycle generation + +--- + +## Domain Structure + +### Ash Domain: `Mv.MembershipFees` + +**Purpose:** Encapsulates all membership fee-related resources and logic + +**Resources:** + +- `MembershipFeeType` - Membership fee type definitions (admin-managed) +- `MembershipFeeCycle` - Individual membership fee cycles per member + +**Extensions:** + +- Member resource extended with membership fee fields + +### Module Organization + +``` +lib/ +├── membership_fees/ +│ ├── membership_fees.ex # Ash domain definition +│ ├── membership_fee_type.ex # MembershipFeeType resource +│ ├── membership_fee_cycle.ex # MembershipFeeCycle resource +│ └── changes/ +│ ├── prevent_interval_change.ex # Validates interval immutability +│ ├── set_membership_fee_start_date.ex # Auto-sets start date +│ └── validate_same_interval.ex # Validates interval match on type change +├── mv/ +│ └── membership_fees/ +│ ├── cycle_generator.ex # Cycle generation algorithm +│ └── calendar_cycles.ex # Calendar cycle calculations +└── membership/ + └── member.ex # Extended with membership fee relationships +``` + +### Separation of Concerns + +**Domain Layer (Ash Resources):** + +- Data validation +- Relationship management +- Policy enforcement +- Action definitions + +**Business Logic Layer (`Mv.MembershipFees`):** + +- Cycle generation algorithm +- Calendar calculations +- Date boundary handling +- Status transitions + +**UI Layer (LiveView):** + +- User interaction +- Display logic +- Authorization checks +- Form handling + +--- + +## Data Architecture + +### Database Schema Extensions + +**See:** [database-schema-readme.md](./database-schema-readme.md) and [database_schema.dbml](./database_schema.dbml) for complete schema documentation. + +### New Tables + +1. **`membership_fee_types`** + - Purpose: Define membership fee types with fixed intervals + - Key Constraint: `interval` field immutable after creation + - Relationships: has_many members, has_many membership_fee_cycles + +2. **`membership_fee_cycles`** + - Purpose: Individual membership fee cycles for members + - Key Design: NO `cycle_end` or `interval_type` fields (calculated) + - Relationships: belongs_to member, belongs_to membership_fee_type + - Composite uniqueness: One cycle per member per cycle_start + +### Member Table Extensions + +**Fields Added:** + +- `membership_fee_type_id` (FK, NOT NULL with default from settings) +- `membership_fee_start_date` (Date, nullable) + +**Existing Fields Used:** + +- `joined_at` - For calculating membership fee start +- `left_at` - For limiting cycle generation +- These fields must remain member fields and should not be replaced by custom fields in the future + +### Settings Integration + +**Global Settings:** + +- `membership_fees.include_joining_cycle` (Boolean) +- `membership_fees.default_membership_fee_type_id` (UUID) + +**Storage:** Existing settings mechanism (TBD: dedicated table or configuration resource) + +### Foreign Key Behaviors + +| Relationship | On Delete | Rationale | +|--------------|-----------|-----------| +| `membership_fee_cycles.member_id → members.id` | CASCADE | Remove membership fee cycles when member deleted | +| `membership_fee_cycles.membership_fee_type_id → membership_fee_types.id` | RESTRICT | Prevent membership fee type deletion if cycles exist | +| `members.membership_fee_type_id → membership_fee_types.id` | RESTRICT | Prevent membership fee type deletion if assigned to members | + +--- + +## Business Logic Architecture + +### Cycle Generation System + +**Component:** `Mv.MembershipFees.CycleGenerator` + +**Responsibilities:** + +- Calculate which cycles should exist for a member +- Generate missing cycles +- Respect membership_fee_start_date and left_at boundaries +- Skip existing cycles (idempotent) + +**Triggers:** + +1. Member membership fee type assigned (via Ash change) +2. Member created with membership fee type (via Ash change) +3. Scheduled job runs (daily/weekly cron) +4. Admin manual regeneration (UI action) + +**Algorithm Steps:** + +1. Retrieve member with membership fee type and dates +2. Determine first cycle start (based on membership_fee_start_date) +3. Calculate all cycle starts from first to today (or left_at) +4. Query existing cycles for member +5. Generate missing cycles with current membership fee type's amount +6. Insert new cycles (batch operation) + +**Edge Case Handling:** + +- If membership_fee_start_date is NULL: Calculate from joined_at + global setting +- If left_at is set: Stop generation at left_at +- If membership fee type changes: Handled separately by regeneration logic + +### Calendar Cycle Calculations + +**Component:** `Mv.MembershipFees.CalendarCycles` + +**Responsibilities:** + +- Calculate cycle boundaries based on interval type +- Determine current cycle +- Determine last completed cycle +- Calculate cycle_end from cycle_start + interval + +**Functions (high-level):** + +- `calculate_cycle_start/3` - Given date and interval, find cycle start +- `calculate_cycle_end/2` - Given cycle_start and interval, calculate end +- `next_cycle_start/2` - Given cycle_start and interval, find next +- `is_current_cycle?/2` - Check if cycle contains today +- `is_last_completed_cycle?/2` - Check if cycle just ended + +**Interval Logic:** + +- **Monthly:** Start = 1st of month, End = last day of month +- **Quarterly:** Start = 1st of quarter (Jan/Apr/Jul/Oct), End = last day of quarter +- **Half-yearly:** Start = 1st of half (Jan/Jul), End = last day of half +- **Yearly:** Start = Jan 1st, End = Dec 31st + +### Status Management + +**Component:** Ash actions on `MembershipFeeCycle` + +**Status Transitions:** + +- Simple state machine: unpaid ↔ paid ↔ suspended +- No complex validation (all transitions allowed) +- Permissions checked via Ash policies + +**Actions Required:** + +- `mark_as_paid` - Set status to :paid +- `mark_as_suspended` - Set status to :suspended +- `mark_as_unpaid` - Set status to :unpaid (error correction) + +**Bulk Operations:** + +- `bulk_mark_as_paid` - Mark multiple cycles as paid (efficiency) + - low priority, can be a future issue + +### Membership Fee Type Change Handling + +**Component:** Ash change on `Member.membership_fee_type_id` + +**Validation:** + +- Check if new type has same interval as old type +- If different: Reject change (MVP constraint) +- If same: Allow change + +**Side Effects on Allowed Change:** + +1. Keep all existing cycles unchanged +2. Find future unpaid cycles +3. Delete future unpaid cycles +4. Regenerate cycles with new membership_fee_type_id and amount + +**Implementation Pattern:** + +- Use Ash change module to validate +- Use after_action hook to trigger regeneration +- Use transaction to ensure atomicity + +--- + +## Integration Points + +### Member Resource Integration + +**Extension Points:** + +1. Add fields via migration +2. Add relationships (belongs_to, has_many) +3. Add calculations (current_cycle_status, overdue_count) +4. Add changes (auto-set membership_fee_start_date, validate interval) + +**Backward Compatibility:** + +- New fields nullable or with defaults +- Existing members get default membership fee type from settings +- No breaking changes to existing member functionality + +### Settings System Integration + +**Requirements:** + +- Store two global settings +- Provide UI for admin to modify +- Default values if not set +- Validation (e.g., default membership fee type must exist) + +**Access Pattern:** + +- Read settings during cycle generation +- Read settings during member creation +- Write settings only via admin UI + +### Permission System Integration + +**See:** [roles-and-permissions-architecture.md](./roles-and-permissions-architecture.md) + +**Required Permissions:** + +- `MembershipFeeType.create/update/destroy` - Admin only +- `MembershipFeeType.read` - Admin, Treasurer, Board +- `MembershipFeeCycle.update` (status changes) - Admin, Treasurer +- `MembershipFeeCycle.read` - Admin, Treasurer, Board, Own member + +**Policy Patterns:** + +- Use existing HasPermission check +- Leverage existing roles (Admin, Kassenwart) +- Member can read own cycles (linked via member_id) + +### LiveView Integration + +**New LiveViews Required:** + +1. MembershipFeeType index/form (admin) +2. MembershipFeeCycle table component (member detail view) +3. Settings form section (admin) +4. Member list column (membership fee status) + +**Existing LiveViews to Extend:** + +- Member detail view: Add membership fees section +- Member list view: Add status column +- Settings page: Add membership fees section + +**Authorization Helpers:** + +- Use existing `can?/3` helper for UI conditionals +- Check permissions before showing actions + +--- + +## Acceptance Criteria + +### MembershipFeeType Resource + +**AC-MFT-1:** Admin can create membership fee type with name, amount, interval, description +**AC-MFT-2:** Interval field is immutable after creation (validation error on change attempt) +**AC-MFT-3:** Admin can update name, amount, description (but not interval) +**AC-MFT-4:** Cannot delete membership fee type if assigned to members +**AC-MFT-5:** Cannot delete membership fee type if cycles exist referencing it +**AC-MFT-6:** Interval must be one of: monthly, quarterly, half_yearly, yearly + +### MembershipFeeCycle Resource + +**AC-MFC-1:** Cycle has cycle_start, status, amount, notes, member_id, membership_fee_type_id +**AC-MFC-2:** cycle_end is calculated, not stored +**AC-MFC-3:** Status defaults to :unpaid +**AC-MFC-4:** One cycle per member per cycle_start (uniqueness constraint) +**AC-MFC-5:** Amount is set at generation time from membership_fee_type.amount +**AC-MFC-6:** Cycles cascade delete when member deleted +**AC-MFC-7:** Admin/Treasurer can change status +**AC-MFC-8:** Member can read own cycles + +### Member Extensions + +**AC-M-1:** Member has membership_fee_type_id field (NOT NULL with default) +**AC-M-2:** Member has membership_fee_start_date field (nullable) +**AC-M-3:** New members get default membership fee type from global setting +**AC-M-4:** membership_fee_start_date auto-set based on joined_at and global setting +**AC-M-5:** Admin can manually override membership_fee_start_date +**AC-M-6:** Cannot change to membership fee type with different interval (MVP) + +### Cycle Generation + +**AC-CG-1:** Cycles generated when member gets membership fee type +**AC-CG-2:** Cycles generated when member created (via change hook) +**AC-CG-3:** Scheduled job generates missing cycles daily +**AC-CG-4:** Generation respects membership_fee_start_date +**AC-CG-5:** Generation stops at left_at if member exited +**AC-CG-6:** Generation is idempotent (skips existing cycles) +**AC-CG-7:** Cycles align to calendar boundaries (1st of month/quarter/half/year) +**AC-CG-8:** Amount comes from membership_fee_type at generation time + +### Calendar Logic + +**AC-CL-1:** Monthly cycles: 1st to last day of month +**AC-CL-2:** Quarterly cycles: 1st of Jan/Apr/Jul/Oct to last day of quarter +**AC-CL-3:** Half-yearly cycles: 1st of Jan/Jul to last day of half +**AC-CL-4:** Yearly cycles: Jan 1 to Dec 31 +**AC-CL-5:** cycle_end calculated correctly for all interval types +**AC-CL-6:** Current cycle determined correctly based on today's date +**AC-CL-7:** Last completed cycle determined correctly + +### Membership Fee Type Change + +**AC-TC-1:** Can change to type with same interval +**AC-TC-2:** Cannot change to type with different interval (error message) +**AC-TC-3:** On allowed change: future unpaid cycles regenerated +**AC-TC-4:** On allowed change: paid/suspended cycles unchanged +**AC-TC-5:** On allowed change: amount updated to new type's amount +**AC-TC-6:** Change is atomic (transaction) + +### Settings + +**AC-S-1:** Global setting: include_joining_cycle (boolean, default true) +**AC-S-2:** Global setting: default_membership_fee_type_id (UUID, required) +**AC-S-3:** Admin can modify settings via UI +**AC-S-4:** Settings validated (e.g., default membership fee type must exist) +**AC-S-5:** Settings applied to new members immediately + +### UI - Member List + +**AC-UI-ML-1:** New column shows membership fee status +**AC-UI-ML-2:** Default: Shows last completed cycle status +**AC-UI-ML-3:** Optional: Toggle to show current cycle status +**AC-UI-ML-4:** Color coding: green (paid), red (unpaid), gray (suspended) +**AC-UI-ML-5:** Filter: Unpaid in last cycle +**AC-UI-ML-6:** Filter: Unpaid in current cycle + +### UI - Member Detail + +**AC-UI-MD-1:** Membership fees section shows all cycles +**AC-UI-MD-2:** Table columns: Cycle, Interval, Amount, Status, Actions +**AC-UI-MD-3:** Checkbox per cycle for bulk marking (low prio) +**AC-UI-MD-4:** "Mark selected as paid" button +**AC-UI-MD-5:** Dropdown to change membership fee type (same interval only) +**AC-UI-MD-6:** Warning if different interval selected +**AC-UI-MD-7:** Only show actions if user has permission + +### UI - Membership Fee Types Admin + +**AC-UI-CTA-1:** List all membership fee types +**AC-UI-CTA-2:** Show: Name, Amount, Interval, Member count +**AC-UI-CTA-3:** Create new membership fee type form +**AC-UI-CTA-4:** Edit form: Name, Amount, Description editable +**AC-UI-CTA-5:** Edit form: Interval grayed out (not editable) +**AC-UI-CTA-6:** Warning on amount change (explain impact) +**AC-UI-CTA-7:** Cannot delete if members assigned +**AC-UI-CTA-8:** Only admin can access + +### UI - Settings Admin + +**AC-UI-SA-1:** Membership fees section in settings +**AC-UI-SA-2:** Dropdown to select default membership fee type +**AC-UI-SA-3:** Checkbox: Include joining cycle +**AC-UI-SA-4:** Explanatory text with examples +**AC-UI-SA-5:** Save button with validation + +--- + +## Testing Strategy + +### Unit Testing + +**Cycle Generator Tests:** + +- Correct cycle_start calculation for all interval types +- Correct cycle count from start to end date +- Respects membership_fee_start_date boundary +- Respects left_at boundary +- Skips existing cycles (idempotent) +- Handles edge dates (year boundaries, leap years) + +**Calendar Cycles Tests:** + +- Cycle boundaries correct for all intervals +- cycle_end calculation correct +- Current cycle detection +- Last completed cycle detection +- Next cycle calculation + +**Validation Tests:** + +- Interval immutability enforced +- Same interval validation on type change +- Status transitions allowed +- Uniqueness constraints enforced + +### Integration Testing + +**Cycle Generation Flow:** + +- Member creation triggers generation +- Type assignment triggers generation +- Type change regenerates future cycles +- Scheduled job generates missing cycles +- Left member stops generation + +**Status Management Flow:** + +- Mark single cycle as paid +- Bulk mark multiple cycles (low prio) +- Status transitions work +- Permissions enforced + +**Membership Fee Type Management:** + +- Create type +- Update amount (regeneration triggered) +- Cannot update interval +- Cannot delete if in use + +### LiveView Testing + +**Member List:** + +- Status column displays correctly +- Toggle between last/current works +- Filters work correctly +- Color coding applied + +**Member Detail:** + +- Cycles table displays all cycles +- Checkboxes work +- Bulk marking works (low prio) +- Membership fee type change validation works +- Actions only shown with permission + +**Admin UI:** + +- Type CRUD works +- Settings save correctly +- Validations display errors +- Only authorized users can access + +### Edge Case Testing + +**Interval Change Attempt:** + +- Error message displayed +- No data modified +- User can cancel/choose different type + +**Exit with Unpaid:** + +- Warning shown +- Option to suspend offered +- Exit completes correctly + +**Amount Change:** + +- Warning displayed +- Only future unpaid regenerated +- Historical cycles unchanged + +**Date Boundaries:** + +- Today = cycle start handled +- Today = cycle end handled +- Leap year handled + +### Performance Testing + +**Cycle Generation:** + +- Generate 10 years of monthly cycles: < 100ms +- Generate for 1000 members: < 5 seconds +- Idempotent check efficient (no full scan) + +**Member List Query:** + +- With status column: < 200ms for 1000 members +- Filters applied efficiently +- No N+1 queries + +--- + +## Security Considerations + +### Authorization + +**Permissions Required:** + +- Membership fee type management: Admin only +- Membership fee cycle status changes: Admin + Treasurer +- View all cycles: Admin + Treasurer + Board +- View own cycles: All authenticated users + +**Policy Enforcement:** + +- All actions protected by Ash policies +- UI shows/hides based on permissions +- Backend validates permissions (never trust UI alone) + +### Data Integrity + +**Validation Layers:** + +1. Database constraints (NOT NULL, UNIQUE, CHECK) +2. Ash validations (business rules) +3. UI validations (user experience) + +**Immutability Protection:** + +- Interval change prevented at multiple layers +- Cycle amounts immutable (audit trail) +- Settings changes logged (future) + +### Audit Trail + +**Tracked Information:** + +- Cycle status changes (who, when) - future enhancement +- Membership fee type amount changes (implicit via cycle amounts) + +--- + +## Performance Considerations + +### Database Indexes + +**Required Indexes:** + +- `membership_fee_cycles(member_id)` - For member cycle lookups +- `membership_fee_cycles(membership_fee_type_id)` - For type queries +- `membership_fee_cycles(status)` - For unpaid filters +- `membership_fee_cycles(cycle_start)` - For date range queries +- `membership_fee_cycles(member_id, cycle_start)` - Composite unique index +- `members(membership_fee_type_id)` - For type membership count + +### Query Optimization + +**Preloading:** + +- Load membership_fee_type with cycles (avoid N+1) +- Load cycles when displaying member detail +- Use Ash's load for efficient preloading + +**Calculated Fields:** + +- cycle_end calculated on-demand (not stored) +- current_cycle_status calculated when needed +- Use Ash calculations for lazy evaluation + +**Pagination:** + +- Cycle list paginated if > 50 cycles +- Member list already paginated + +### Caching Strategy + +**No caching needed in MVP:** + +- Membership fee types rarely change +- Cycle queries are fast +- Settings read infrequently + +**Future caching if needed:** + +- Cache settings in application memory +- Cache membership fee types list +- Invalidate on change + +### Scheduled Job Performance + +**Cycle Generation Job:** + +- Run daily or weekly (not hourly) +- Batch members (process 100 at a time) +- Skip members with no changes +- Log failures for retry + +--- + +## Future Enhancements + +### Phase 2: Interval Change Support + +**Architecture Changes:** + +- Add logic to handle cycle overlaps +- Calculate prorata amounts if needed +- More complex validation +- Migration path for existing cycles + +### Phase 3: Payment Details + +**Architecture Changes:** + +- Add PaymentTransaction resource +- Link transactions to cycles +- Support multiple payments per cycle +- Reconciliation logic + +### Phase 4: vereinfacht.digital Integration + +**Architecture Changes:** + +- External API client module +- Webhook handling for transactions +- Automatic matching logic +- Manual review interface + +--- + +**End of Architecture Document** diff --git a/docs/contributions-overview.md b/docs/membership-fee-overview.md similarity index 59% rename from docs/contributions-overview.md rename to docs/membership-fee-overview.md index e0c4bc8..229b73b 100644 --- a/docs/contributions-overview.md +++ b/docs/membership-fee-overview.md @@ -1,7 +1,7 @@ -# Membership Contributions - Overview +# Membership Fees - Overview **Project:** Mila - Membership Management System -**Feature:** Membership Contribution Management +**Feature:** Membership Fee Management **Version:** 1.0 **Last Updated:** 2025-11-27 **Status:** Concept - Ready for Review @@ -10,9 +10,9 @@ ## Purpose -This document provides a comprehensive overview of the Membership Contributions system. It covers business logic, data model, UI/UX design, and technical architecture in a concise, bullet-point format. +This document provides a comprehensive overview of the Membership Fees system. It covers business logic, data model, UI/UX design, and technical architecture in a concise, bullet-point format. -**For detailed implementation:** See [contributions-implementation-plan.md](./contributions-implementation-plan.md) (created after concept iterations) +**For detailed implementation:** See [membership-fee-implementation-plan.md](./membership-fee-implementation-plan.md) (created after concept iterations) --- @@ -36,7 +36,7 @@ This document provides a comprehensive overview of the Membership Contributions - Minimal complexity - Clear data model without redundancies - Intuitive operation -- Calendar period-based (Month/Quarter/Half-Year/Year) +- Calendar cycle-based (Month/Quarter/Half-Year/Year) --- @@ -46,9 +46,9 @@ This document provides a comprehensive overview of the Membership Contributions **Core Entities:** -- Beitragsart ↔ Contribution Type / Membership Fee Type -- Beitragsintervall ↔ Contribution Period -- Mitgliedsbeitrag ↔ Membership Fee / Contribution +- Beitragsart ↔ Membership Fee Type +- Beitragszyklus ↔ Membership Fee Cycle +- Mitgliedsbeitrag ↔ Membership Fee **Status:** @@ -56,7 +56,7 @@ This document provides a comprehensive overview of the Membership Contributions - unbezahlt ↔ unpaid - ausgesetzt ↔ suspended / waived -**Intervals:** +**Intervals (Frequenz / Payment Frequency):** - monatlich ↔ monthly - quartalsweise ↔ quarterly @@ -65,8 +65,8 @@ This document provides a comprehensive overview of the Membership Contributions **UI Elements:** -- "Letztes Intervall" ↔ "Last Period" (e.g., 2023 when in 2024) -- "Aktuelles Intervall" ↔ "Current Period" (e.g., 2024) +- "Letzter Zyklus" ↔ "Last Cycle" (e.g., 2023 when in 2024) +- "Aktueller Zyklus" ↔ "Current Cycle" (e.g., 2024) - "Als bezahlt markieren" ↔ "Mark as paid" - "Aussetzen" ↔ "Suspend" / "Waive" @@ -74,43 +74,41 @@ This document provides a comprehensive overview of the Membership Contributions ## Data Model -### Contribution Type (ContributionType) +### Membership Fee Type (MembershipFeeType) ``` - id (UUID) - name (String) - e.g., "Regular", "Reduced", "Student" -- amount (Decimal) - Contribution amount in Euro +- amount (Decimal) - Membership fee amount in Euro - interval (Enum) - :monthly, :quarterly, :half_yearly, :yearly - description (Text, optional) -- timestamps ``` **Important:** - `interval` is **IMMUTABLE** after creation! - Admin can only change `name`, `amount`, `description` -- On change: Future unpaid periods regenerated with new amount +- On change: Future unpaid cycles regenerated with new amount -### Contribution Period (ContributionPeriod) +### Membership Fee Cycle (MembershipFeeCycle) ``` - id (UUID) - member_id (FK → members.id) -- contribution_type_id (FK → contribution_types.id) -- period_start (Date) - Calendar period start (01.01., 01.04., 01.07., 01.10., etc.) +- membership_fee_type_id (FK → membership_fee_types.id) +- cycle_start (Date) - Calendar cycle start (01.01., 01.04., 01.07., 01.10., etc.) - status (Enum) - :unpaid (default), :paid, :suspended -- amount (Decimal) - Amount at generation time (history when type changes) +- amount (Decimal) - Membership fee amount at generation time (history when type changes) - notes (Text, optional) - Admin notes -- timestamps ``` **Important:** -- **NO** `period_end` - calculated from `period_start` + `interval` -- **NO** `interval_type` - read from `contribution_type.interval` +- **NO** `cycle_end` - calculated from `cycle_start` + `interval` +- **NO** `interval_type` - read from `membership_fee_type.interval` - Avoids redundancy and inconsistencies! -**Calendar Period Logic:** +**Calendar Cycle Logic:** - Monthly: 01.01. - 31.01., 01.02. - 28./29.02., etc. - Quarterly: 01.01. - 31.03., 01.04. - 30.06., 01.07. - 30.09., 01.10. - 31.12. @@ -120,70 +118,75 @@ This document provides a comprehensive overview of the Membership Contributions ### Member (Extensions) ``` -- contribution_type_id (FK → contribution_types.id, NOT NULL, default from settings) -- contribution_start_date (Date, nullable) - When to start generating contributions +- membership_fee_type_id (FK → membership_fee_types.id, NOT NULL, default from settings) +- membership_fee_start_date (Date, nullable) - When to start generating membership fees - left_at (Date, nullable) - Exit date (existing) ``` -**Logic for contribution_start_date:** +**Logic for membership_fee_start_date:** -- Auto-set based on global setting `include_joining_period` -- If `include_joining_period = true`: First day of joining month/quarter/year -- If `include_joining_period = false`: First day of NEXT period after joining +- Auto-set based on global setting `include_joining_cycle` +- If `include_joining_cycle = true`: First day of joining month/quarter/year +- If `include_joining_cycle = false`: First day of NEXT cycle after joining - Can be manually overridden by admin -**NO** `include_joining_period` field on Member - unnecessary due to `contribution_start_date`! +**NO** `include_joining_cycle` field on Member - unnecessary due to `membership_fee_start_date`! ### Global Settings ``` -key: "contributions.include_joining_period" +key: "membership_fees.include_joining_cycle" value: Boolean (Default: true) -key: "contributions.default_contribution_type_id" -value: UUID (Required) - Default contribution type for new members +key: "membership_fees.default_membership_fee_type_id" +value: UUID (Required) - Default membership fee type for new members ``` -**Meaning include_joining_period:** +**Meaning include_joining_cycle:** -- `true`: Joining period is included (member pays from joining period) -- `false`: Only from next full period after joining +- `true`: Joining cycle is included (member pays from joining cycle) +- `false`: Only from next full cycle after joining -**Meaning default_contribution_type_id:** +**Meaning of default membership fee type setting:** -- Every new member automatically gets this contribution type +- Every new member automatically gets this membership fee type - Must be configured in admin settings -- Prevents: Members without contribution type +- Prevents: Members without membership fee type --- ## Business Logic -### Period Generation +### Cycle Generation **Triggers:** -- Member gets contribution type assigned (also during member creation) -- New period begins (Cron job daily/weekly) +- Member gets membership fee type assigned (also during member creation) +- New cycle begins (Cron job daily/weekly) - Admin requests manual regeneration **Algorithm:** -1. Get `member.contribution_start_date` and `member.contribution_type` -2. Calculate first period based on `contribution_start_date` -3. Generate all periods from start to today (or `left_at` if present) -4. Skip existing periods -5. Set `amount` to current `contribution_type.amount` +Lock the whole cycle table for the duration of the algorithm + +1. Get `member.membership_fee_start_date` and member's membership fee type +2. Generate cycles until today (or `left_at` if present): + - If no cycle exists: + - Generate all cycles from `membership_fee_start_date` + - else: + - Generate all cycles from last existing cycle + - use the interval to generate the cycles +3. Set `amount` to current membership fee type's amount **Example (Yearly):** ``` Joining date: 15.03.2023 -include_joining_period: true -→ contribution_start_date: 01.01.2023 +include_joining_cycle: true +→ membership_fee_start_date: 01.01.2023 -Generated periods: -- 01.01.2023 - 31.12.2023 (joining period) +Generated cycles: +- 01.01.2023 - 31.12.2023 (joining cycle) - 01.01.2024 - 31.12.2024 - 01.01.2025 - 31.12.2025 (current year) ``` @@ -192,10 +195,10 @@ Generated periods: ``` Joining date: 15.03.2023 -include_joining_period: false -→ contribution_start_date: 01.04.2023 +include_joining_cycle: false +→ membership_fee_start_date: 01.04.2023 -Generated periods: +Generated cycles: - 01.04.2023 - 30.06.2023 (first full quarter) - 01.07.2023 - 30.09.2023 - 01.10.2023 - 31.12.2023 @@ -218,44 +221,44 @@ suspended → unpaid - Admin + Treasurer (Kassenwart) can change status - Uses existing permission system -### Contribution Type Change +### Membership Fee Type Change -**MVP - Same Interval Only:** +**MVP - Same Cycle Only:** -- Member can only choose contribution type with **same interval** +- Member can only choose membership fee type with **same cycle** - Example: From "Regular (yearly)" to "Reduced (yearly)" ✓ - Example: From "Regular (yearly)" to "Reduced (monthly)" ✗ **Logic on Change:** -1. Check: New contribution type has same interval -2. If yes: Set `member.contribution_type_id` -3. Future **unpaid** periods: Delete and regenerate with new amount -4. Paid/suspended periods: Remain unchanged (historical amount) +1. Check: New membership fee type has same interval +2. If yes: Set `member.membership_fee_type_id` +3. Future **unpaid** cycles: Delete and regenerate with new amount +4. Paid/suspended cycles: Remain unchanged (historical amount) **Future - Different Intervals:** - Enable interval switching (e.g., yearly → monthly) -- More complex logic for period overlaps +- More complex logic for cycle overlaps - Needs additional validation ### Member Exit **Logic:** -- Periods only generated until `member.left_at` -- Existing periods remain visible -- Unpaid exit period can be marked as "suspended" +- Cycles only generated until `member.left_at` +- Existing cycles remain visible +- Unpaid exit cycle can be marked as "suspended" **Example:** ``` Exit: 15.08.2024 -Yearly period: 01.01.2024 - 31.12.2024 +Yearly cycle: 01.01.2024 - 31.12.2024 -→ Period 2024 is shown (Status: unpaid) +→ Cycle 2024 is shown (Status: unpaid) → Admin can set to "suspended" -→ No periods for 2025+ generated +→ No cycles for 2025+ generated ``` --- @@ -264,46 +267,46 @@ Yearly period: 01.01.2024 - 31.12.2024 ### Member List View -**New Column: "Contribution Status"** +**New Column: "Membership Fee Status"** -**Default Display (Last Period):** +**Default Display (Last Cycle):** -- Shows status of **last completed** period -- Example in 2024: Shows contribution for 2023 +- Shows status of **last completed** cycle +- Example in 2024: Shows membership fee for 2023 - Color coding: - Green: paid ✓ - Red: unpaid ✗ - Gray: suspended ⊘ -**Optional: Show Current Period** +**Optional: Show Current Cycle** -- Toggle: "Show current period" (2024) +- Toggle: "Show current cycle" (2024) - Admin decides what to display **Filters:** -- "Unpaid contributions in last period" -- "Unpaid contributions in current period" +- "Unpaid membership fees in last cycle" +- "Unpaid membership fees in current cycle" ### Member Detail View -**Section: "Contributions"** +**Section: "Membership Fees"** -**Contribution Type Assignment:** +**Membership Fee Type Assignment:** ``` ┌─────────────────────────────────────┐ -│ Contribution Type: [Dropdown] │ -│ ⚠ Only types with same interval │ -│ can be selected │ +│ Membership Fee Type: [Dropdown] │ +│ ⚠ Only types with same interval │ +│ can be selected │ └─────────────────────────────────────┘ ``` -**Period Table:** +**Cycle Table:** ``` ┌───────────────┬──────────┬────────┬──────────┬─────────┐ -│ Period │ Interval │ Amount │ Status │ Action │ +│ Cycle │ Interval │ Amount │ Status │ Action │ ├───────────────┼──────────┼────────┼──────────┼─────────┤ │ 01.01.2023- │ Yearly │ 50 € │ ☑ Paid │ │ │ 31.12.2023 │ │ │ │ │ @@ -322,9 +325,9 @@ Legend: ☑ = paid | ☐ = unpaid | ⊘ = suspended - Checkbox in each row for fast marking - Button: "Mark selected as paid/unpaid/suspended" -- Bulk action for multiple periods +- Bulk action for multiple cycles -### Admin: Contribution Types Management +### Admin: Membership Fee Types Management **List:** @@ -352,37 +355,37 @@ Legend: ☑ = paid | ☐ = unpaid | ⊘ = suspended Impact: - 45 members affected -- Future unpaid periods will be generated with 65 € -- Already paid periods remain with old amount +- Future unpaid cycles will be generated with 65 € +- Already paid cycles remain with old amount [Cancel] [Confirm] ``` ### Admin: Settings -**Contribution Configuration:** +**Membership Fee Configuration:** ``` -Default Contribution Type: [Dropdown: Contribution Types] +Default Membership Fee Type: [Dropdown: Membership Fee Types] Selected: "Regular (60 €, Yearly)" -This contribution type is automatically assigned to all new members. +This membership fee type is automatically assigned to all new members. Can be changed individually per member. --- -☐ Include joining period +☐ Include joining cycle When active: -Members pay from the period of their joining. +Members pay from the cycle of their joining. Example (Yearly): Joining: 15.03.2023 → Pays from 2023 When inactive: -Members pay from the next full period. +Members pay from the next full cycle. Example (Yearly): Joining: 15.03.2023 @@ -393,7 +396,7 @@ Joining: 15.03.2023 ## Edge Cases -### 1. Contribution Type Change with Different Interval +### 1. Membership Fee Type Change with Different Interval **MVP:** Blocked (only same interval allowed) @@ -402,11 +405,11 @@ Joining: 15.03.2023 ``` Error: Interval change not possible -Current contribution type: "Regular (Yearly)" -Selected contribution type: "Student (Monthly)" +Current membership fee type: "Regular (Yearly)" +Selected membership fee type: "Student (Monthly)" Changing the interval is currently not possible. -Please select a contribution type with interval "Yearly". +Please select a membership fee type with interval "Yearly". [OK] ``` @@ -415,32 +418,32 @@ Please select a contribution type with interval "Yearly". - Allow interval switching - Calculate overlaps -- Generate new periods without duplicates +- Generate new cycles without duplicates -### 2. Exit with Unpaid Contributions +### 2. Exit with Unpaid Membership Fees **Scenario:** ``` Member exits: 15.08.2024 -Yearly period 2024: unpaid +Yearly cycle 2024: unpaid ``` **UI Notice on Exit: (Low Prio)** ``` -⚠ Unpaid contributions present +⚠ Unpaid membership fees present -This member has 1 unpaid period(s): +This member has 1 unpaid cycle(s): - 2024: 60 € (unpaid) Do you want to continue? -[ ] Mark contribution as "suspended" +[ ] Mark membership fee as "suspended" [Cancel] [Confirm Exit] ``` -### 3. Multiple Unpaid Periods +### 3. Multiple Unpaid Cycles **Scenario:** Member hasn't paid for 2 years @@ -467,9 +470,9 @@ Do you want to continue? **Result:** -- Period 2023: Saved with 50 € (history) -- Period 2024: Generated with 60 € (current) -- Both periods show correct historical amount +- Cycle 2023: Saved with 50 € (history) +- Cycle 2024: Generated with 60 € (current) +- Both cycles show correct historical amount ### 5. Date Boundaries @@ -477,7 +480,7 @@ Do you want to continue? **Solution:** -- Current period (2025) is generated +- Current cycle (2025) is generated - Status: unpaid (open) - Shown in overview @@ -489,17 +492,17 @@ Do you want to continue? **Included:** -- ✓ Contribution types (CRUD) -- ✓ Automatic period generation +- ✓ Membership fee types (CRUD) +- ✓ Automatic cycle generation - ✓ Status management (paid/unpaid/suspended) -- ✓ Member overview with contribution status -- ✓ Period view per member +- ✓ Member overview with membership fee status +- ✓ Cycle view per member - ✓ Quick checkbox marking - ✓ Bulk actions - ✓ Amount history - ✓ Same-interval type change -- ✓ Default contribution type -- ✓ Joining period configuration +- ✓ Default membership fee type +- ✓ Joining cycle configuration **NOT Included:** @@ -515,7 +518,7 @@ Do you want to continue? **Phase 2:** - Payment details (date, amount, method) -- Interval change for future unpaid periods +- Interval change for future unpaid cycles - Manual vereinfacht.digital links per member - Extended filter options diff --git a/lib/membership/member.ex b/lib/membership/member.ex index b788dc9..d29a759 100644 --- a/lib/membership/member.ex +++ b/lib/membership/member.ex @@ -29,7 +29,9 @@ defmodule Mv.Membership.Member do ## Full-Text Search Members have a `search_vector` attribute (tsvector) that is automatically - updated via database trigger. Search includes name, email, notes, and contact fields. + updated via database trigger. Search includes name, email, notes, contact fields, + and all custom field values. Custom field values are automatically included in + the search vector with weight 'C' (same as phone_number, city, etc.). """ use Ash.Resource, domain: Mv.Membership, @@ -40,6 +42,21 @@ defmodule Mv.Membership.Member do # Module constants @member_search_limit 10 + + # Similarity threshold for fuzzy name/address matching. + # Lower value = more results but less accurate (0.1-0.9) + # + # Fuzzy matching uses two complementary strategies: + # 1. % operator: Fast GIN-index-based matching using server-wide threshold (default 0.3) + # - Catches exact trigram matches quickly via index + # 2. similarity/word_similarity functions: Precise matching with this configurable threshold + # - Catches partial matches that % operator might miss + # + # Value 0.2 chosen based on testing with typical German names: + # - "Müller" vs "Mueller": similarity ~0.65 ✓ + # - "Schmidt" vs "Schmitt": similarity ~0.75 ✓ + # - "Wagner" vs "Wegner": similarity ~0.55 ✓ + # - Random unrelated names: similarity ~0.15 ✗ @default_similarity_threshold 0.2 # Use constants from Mv.Constants for member fields @@ -139,30 +156,21 @@ defmodule Mv.Membership.Member do if is_binary(q) and String.trim(q) != "" do q2 = String.trim(q) - pat = "%" <> q2 <> "%" + # Sanitize for LIKE patterns (escape % and _), limit length to 100 chars + q2_sanitized = sanitize_search_query(q2) + pat = "%" <> q2_sanitized <> "%" + + # Build search filters grouped by search type for maintainability + # Priority: FTS > Substring > Custom Fields > Fuzzy Matching + # Note: FTS and fuzzy use q2 (unsanitized), LIKE-based filters use pat (sanitized) + fts_match = build_fts_filter(q2) + substring_match = build_substring_filter(q2_sanitized, pat) + custom_field_match = build_custom_field_filter(pat) + fuzzy_match = build_fuzzy_filter(q2, threshold) - # FTS as main filter and fuzzy search just for first name, last name and strees query |> Ash.Query.filter( - expr( - # Substring on numeric-like fields (best effort, supports middle substrings) - fragment("search_vector @@ websearch_to_tsquery('simple', ?)", ^q2) or - fragment("search_vector @@ plainto_tsquery('simple', ?)", ^q2) or - contains(postal_code, ^q2) or - contains(house_number, ^q2) or - contains(phone_number, ^q2) or - contains(email, ^q2) or - contains(city, ^q2) or ilike(city, ^pat) or - fragment("? % first_name", ^q2) or - fragment("? % last_name", ^q2) or - fragment("? % street", ^q2) or - fragment("word_similarity(?, first_name) > ?", ^q2, ^threshold) or - fragment("word_similarity(?, last_name) > ?", ^q2, ^threshold) or - fragment("word_similarity(?, street) > ?", ^q2, ^threshold) or - fragment("similarity(first_name, ?) > ?", ^q2, ^threshold) or - fragment("similarity(last_name, ?) > ?", ^q2, ^threshold) or - fragment("similarity(street, ?) > ?", ^q2, ^threshold) - ) + expr(^fts_match or ^substring_match or ^custom_field_match or ^fuzzy_match) ) else query @@ -476,7 +484,6 @@ defmodule Mv.Membership.Member do - `query` - Ash.Query.t() to apply search to - `opts` - Keyword list or map with search options: - `:query` or `"query"` - Search string - - `:fields` or `"fields"` - Optional field restrictions ## Returns - Modified Ash.Query.t() with search filters applied @@ -497,16 +504,103 @@ defmodule Mv.Membership.Member do if String.trim(q) == "" do query else - args = - case opts[:fields] || opts["fields"] do - nil -> %{query: q} - fields -> %{query: q, fields: fields} - end - - Ash.Query.for_read(query, :search, args) + Ash.Query.for_read(query, :search, %{query: q}) end end + # ============================================================================ + # Search Input Sanitization + # ============================================================================ + + # Sanitizes search input to prevent LIKE pattern injection. + # Escapes SQL LIKE wildcards (% and _) and limits query length. + # + # ## Examples + # + # iex> sanitize_search_query("test%injection") + # "test\\%injection" + # + # iex> sanitize_search_query("very_long_search") + # "very\\_long\\_search" + # + defp sanitize_search_query(query) when is_binary(query) do + query + |> String.slice(0, 100) + |> String.replace("\\", "\\\\") + |> String.replace("%", "\\%") + |> String.replace("_", "\\_") + end + + defp sanitize_search_query(_), do: "" + + # ============================================================================ + # Search Filter Builders + # ============================================================================ + # These functions build search filters grouped by search type for maintainability. + # Priority order: FTS > Substring > Custom Fields > Fuzzy Matching + + # Builds full-text search filter using tsvector (highest priority, fastest) + # Uses GIN index on search_vector for optimal performance + defp build_fts_filter(query) do + expr( + fragment("search_vector @@ websearch_to_tsquery('simple', ?)", ^query) or + fragment("search_vector @@ plainto_tsquery('simple', ?)", ^query) + ) + end + + # Builds substring search filter for structured fields + # Note: contains/2 uses ILIKE '%value%' which is not index-optimized + # Performance: Good for small datasets, may be slow on large tables + defp build_substring_filter(query, _pattern) do + expr( + contains(postal_code, ^query) or + contains(house_number, ^query) or + contains(phone_number, ^query) or + contains(email, ^query) or + contains(city, ^query) + ) + end + + # Builds search filter for custom field values using ILIKE on JSONB + # Note: ILIKE on JSONB is not index-optimized, may be slow with many custom fields + # This is a fallback for substring matching in custom fields (e.g., phone numbers) + # Uses ->> operator which always returns TEXT directly (no need for -> + ::text fallback) + # Important: `id` must be passed as parameter to correctly reference the outer members table + defp build_custom_field_filter(pattern) do + expr( + fragment( + "EXISTS (SELECT 1 FROM custom_field_values WHERE member_id = ? AND (value->>'_union_value' ILIKE ? OR value->>'value' ILIKE ?))", + id, + ^pattern, + ^pattern + ) + ) + end + + # Builds fuzzy/trigram matching filter for name and street fields. + # Uses pg_trgm extension with GIN indexes for performance. + # + # Two-tier matching strategy: + # - % operator: Uses server-wide pg_trgm.similarity_threshold (typically 0.3) + # for fast index-based initial filtering + # - similarity/word_similarity: Uses @default_similarity_threshold (0.2) + # for more lenient matching to catch edge cases + # + # Note: Requires trigram GIN indexes on first_name, last_name, street. + defp build_fuzzy_filter(query, threshold) do + expr( + fragment("? % first_name", ^query) or + fragment("? % last_name", ^query) or + fragment("? % street", ^query) or + fragment("word_similarity(?, first_name) > ?", ^query, ^threshold) or + fragment("word_similarity(?, last_name) > ?", ^query, ^threshold) or + fragment("word_similarity(?, street) > ?", ^query, ^threshold) or + fragment("similarity(first_name, ?) > ?", ^query, ^threshold) or + fragment("similarity(last_name, ?) > ?", ^query, ^threshold) or + fragment("similarity(street, ?) > ?", ^query, ^threshold) + ) + end + # Private helper to apply filters for :available_for_linking action # user_email: may be nil/empty when creating new user, or populated when editing # search_query: optional search term for fuzzy matching @@ -515,9 +609,9 @@ defmodule Mv.Membership.Member do # - Empty user_email ("") → email == "" is always false → only fuzzy search matches # - This allows a single filter expression instead of duplicating fuzzy search logic # - # Cyclomatic complexity is unavoidable here: PostgreSQL fuzzy search requires - # multiple OR conditions for good search quality (FTS + trigram similarity + substring) - # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity + # Note: Custom field search is intentionally excluded from linking to optimize + # autocomplete performance. Custom fields are still searchable via the main + # member search which uses the indexed search_vector. defp apply_linking_filters(query, user_email, search_query) do has_search = search_query && String.trim(search_query) != "" # Use empty string instead of nil to simplify filter logic @@ -526,35 +620,23 @@ defmodule Mv.Membership.Member do if has_search do # Search query provided: return email-match OR fuzzy-search candidates trimmed_search = String.trim(search_query) + # Sanitize for LIKE patterns (contains uses ILIKE internally) + sanitized_search = sanitize_search_query(trimmed_search) + + # Build search filters - excluding custom_field_filter for performance + fts_match = build_fts_filter(trimmed_search) + fuzzy_match = build_fuzzy_filter(trimmed_search, @default_similarity_threshold) + email_substring_match = expr(contains(email, ^sanitized_search)) query |> Ash.Query.filter( expr( - # Email match candidate (for filter_by_email_match priority) - # If email is "", this is always false and fuzzy search takes over - # Fuzzy search candidates + # Email exact match has highest priority (for filter_by_email_match) + # If email is "", this is always false and search filters take over email == ^trimmed_email or - fragment("search_vector @@ websearch_to_tsquery('simple', ?)", ^trimmed_search) or - fragment("search_vector @@ plainto_tsquery('simple', ?)", ^trimmed_search) or - fragment("? % first_name", ^trimmed_search) or - fragment("? % last_name", ^trimmed_search) or - fragment("word_similarity(?, first_name) > 0.2", ^trimmed_search) or - fragment( - "word_similarity(?, last_name) > ?", - ^trimmed_search, - ^@default_similarity_threshold - ) or - fragment( - "similarity(first_name, ?) > ?", - ^trimmed_search, - ^@default_similarity_threshold - ) or - fragment( - "similarity(last_name, ?) > ?", - ^trimmed_search, - ^@default_similarity_threshold - ) or - contains(email, ^trimmed_search) + ^fts_match or + ^fuzzy_match or + ^email_substring_match ) ) else diff --git a/lib/mv_web/components/core_components.ex b/lib/mv_web/components/core_components.ex index d19b1eb..a23381d 100644 --- a/lib/mv_web/components/core_components.ex +++ b/lib/mv_web/components/core_components.ex @@ -153,7 +153,7 @@ defmodule MvWeb.CoreComponents do aria-haspopup="menu" aria-expanded={@open} aria-controls={@id} - class="btn btn-ghost" + class="btn" phx-click="toggle_dropdown" phx-target={@phx_target} data-testid="dropdown-button" @@ -236,6 +236,30 @@ defmodule MvWeb.CoreComponents do """ end + @doc """ + Renders a section in with a border similar to cards. + + + ## Examples + + <.form_section title={gettext("Personal Data")}> +
input
+ + """ + attr :title, :string, required: true + slot :inner_block, required: true + + def form_section(assigns) do + ~H""" +@@ -474,6 +498,7 @@ defmodule MvWeb.CoreComponents do slot :col, required: true do attr :label, :string + attr :class, :string attr :col_click, :any, doc: "optional column-specific click handler that overrides row_click" end @@ -490,7 +515,7 @@ defmodule MvWeb.CoreComponents do
| {col[:label]} | +{col[:label]} | <.live_component module={MvWeb.Components.SortHeaderComponent} @@ -514,7 +539,34 @@ defmodule MvWeb.CoreComponents do (col[:col_click] && col[:col_click].(@row_item.(row))) || (@row_click && @row_click.(row)) } - class={["max-w-xs truncate", (col[:col_click] || @row_click) && "hover:cursor-pointer"]} + class={ + col_class = Map.get(col, :class) + has_click = col[:col_click] || @row_click + classes = ["max-w-xs"] + + classes = + if col_class == nil || (col_class && !String.contains?(col_class, "text-center")) do + ["truncate" | classes] + else + classes + end + + classes = + if has_click do + ["hover:cursor-pointer" | classes] + else + classes + end + + classes = + if col_class do + [col_class | classes] + else + classes + end + + Enum.join(classes, " ") + } > {render_slot(col, @row_item.(row))} diff --git a/lib/mv_web/live/components/field_visibility_dropdown_component.ex b/lib/mv_web/live/components/field_visibility_dropdown_component.ex index 642273c..5fc0abf 100644 --- a/lib/mv_web/live/components/field_visibility_dropdown_component.ex +++ b/lib/mv_web/live/components/field_visibility_dropdown_component.ex @@ -152,9 +152,25 @@ defmodule MvWeb.Components.FieldVisibilityDropdownComponent do defp field_to_string(field) when is_atom(field), do: Atom.to_string(field) defp field_to_string(field) when is_binary(field), do: field - defp format_field_label(field) do + defp format_field_label(field) when is_atom(field) do + MvWeb.Translations.MemberFields.label(field) + end + + defp format_field_label(field) when is_binary(field) do + case safe_to_existing_atom(field) do + {:ok, atom} -> MvWeb.Translations.MemberFields.label(atom) + :error -> fallback_label(field) + end + end + + defp safe_to_existing_atom(string) do + {:ok, String.to_existing_atom(string)} + rescue + ArgumentError -> :error + end + + defp fallback_label(field) do field - |> field_to_string() |> String.replace("_", " ") |> String.split() |> Enum.map_join(" ", &String.capitalize/1) diff --git a/lib/mv_web/live/components/payment_filter_component.ex b/lib/mv_web/live/components/payment_filter_component.ex index c9dc731..47556dd 100644 --- a/lib/mv_web/live/components/payment_filter_component.ex +++ b/lib/mv_web/live/components/payment_filter_component.ex @@ -44,7 +44,7 @@ defmodule MvWeb.Components.PaymentFilterComponent do |
|---|