Compare commits

..

20 commits

Author SHA1 Message Date
Renovate Bot
b48529113c chore(deps): update mix dependencies
Some checks failed
continuous-integration/drone/push Build is failing
continuous-integration/drone/promote/production Build is failing
2026-02-03 00:15:03 +00:00
c998d14b95 Merge pull request 'Implements custom field CSV import closes #338' (#395) from feature/338_import_custom_fields into main
All checks were successful
continuous-integration/drone/push Build is passing
Reviewed-on: #395
2026-02-02 17:05:29 +01:00
960506d16a refactoring
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/promote/production Build is passing
2026-02-02 16:56:07 +01:00
aef3aa299f fix test
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/promote/production Build is passing
2026-02-02 15:04:07 +01:00
b21c3df7ef refactoring 2026-02-02 14:34:12 +01:00
71db9cf3c1 formatting
Some checks failed
continuous-integration/drone/push Build is failing
2026-02-02 13:54:27 +01:00
9e27de84cb Merge branch 'main' into feature/338_import_custom_fields
Some checks failed
continuous-integration/drone/push Build is failing
2026-02-02 13:46:05 +01:00
c56ca68922 docs: update docs
Some checks failed
continuous-integration/drone/push Build is failing
2026-02-02 13:42:24 +01:00
f5591c392a i18n: add translation 2026-02-02 13:42:16 +01:00
aab5666f46 Merge pull request 'Adds config for import limits closes #336' (#394) from feature/336_import_auth into main
All checks were successful
continuous-integration/drone/push Build is passing
Reviewed-on: #394
2026-02-02 13:15:22 +01:00
12715f3d85 refactoring 2026-02-02 13:07:08 +01:00
86a3c4e50e tests: add tests for import 2026-02-02 13:07:00 +01:00
3f8797c356 feat: import custom fields via CSV 2026-02-02 11:42:07 +01:00
ce6240133d i18n: update translations
Some checks failed
continuous-integration/drone/push Build is passing
continuous-integration/drone/promote/production Build is failing
2026-02-02 10:23:49 +01:00
4997819c73 feat: validate config 2026-02-02 10:22:21 +01:00
b6d53d2826 refactor: add test to seperate async false module 2026-02-02 10:22:05 +01:00
e74154581c feat: changes UI info based on config for limits 2026-02-02 10:10:02 +01:00
d61a939deb formatting
All checks were successful
continuous-integration/drone/push Build is passing
2026-02-02 09:50:47 +01:00
3f551c5f8d feat: add configs for impor tlimits
Some checks failed
continuous-integration/drone/push Build is failing
2026-02-02 09:49:13 +01:00
9fd617e45a tests: add tests for config 2026-02-02 09:48:37 +01:00
12 changed files with 1027 additions and 250 deletions

View file

@ -51,6 +51,13 @@ config :mv,
generators: [timestamp_type: :utc_datetime],
ash_domains: [Mv.Membership, Mv.Accounts, Mv.MembershipFees, Mv.Authorization]
# CSV Import configuration
config :mv,
csv_import: [
max_file_size_mb: 10,
max_rows: 1000
]
# Configures the endpoint
config :mv, MvWeb.Endpoint,
url: [host: "localhost"],

View file

@ -2,7 +2,7 @@
**Version:** 1.0
**Last Updated:** 2026-01-13
**Status:** In Progress (Backend Complete, UI Pending)
**Status:** In Progress (Backend Complete, UI Complete, Tests Pending)
**Related Documents:**
- [Feature Roadmap](./feature-roadmap.md) - Overall feature planning
@ -15,15 +15,15 @@
- ✅ Issue #4: Header Normalization + Per-Header Mapping
- ✅ Issue #5: Validation (Required Fields) + Error Formatting
- ✅ Issue #6: Persistence via Ash Create + Per-Row Error Capture (with Error-Capping)
- ✅ Issue #11: Custom Field Import (Backend)
- ✅ Issue #7: Admin Global Settings LiveView UI (Upload + Start Import + Results + Template Links)
- ✅ Issue #8: Authorization + Limits
- ✅ Issue #11: Custom Field Import (Backend + UI)
**In Progress / Pending:**
- ⏳ Issue #7: Admin Global Settings LiveView UI (Upload + Start Import + Results)
- ⏳ Issue #8: Authorization + Limits
- ⏳ Issue #9: End-to-End LiveView Tests + Fixtures
- ⏳ Issue #10: Documentation Polish
**Latest Update:** Error-Capping in `process_chunk/4` implemented (2025-01-XX)
**Latest Update:** CSV Import UI fully implemented in GlobalSettingsLive with chunk processing, progress tracking, error display, and custom field support (2026-01-13)
---
@ -161,6 +161,13 @@ A **basic CSV member import feature** that allows administrators to upload a CSV
- Any custom field column using the custom field's **name** as the header (e.g., `membership_number`, `birth_date`)
- **Important:** Custom fields must be created in Mila before importing. The CSV header must match the custom field name exactly (same normalization as member fields).
- **Behavior:** If the CSV contains custom field columns that don't exist in Mila, a warning message will be shown and those columns will be ignored during import.
- **Value Validation:** Custom field values are validated according to the custom field type:
- **string**: Any text value (trimmed)
- **integer**: Must be a valid integer (e.g., `42`, `-10`). Invalid values will cause a row error with the custom field name and reason.
- **boolean**: Accepts `true`, `false`, `1`, `0`, `yes`, `no`, `ja`, `nein` (case-insensitive). Invalid values will cause a row error.
- **date**: Must be in ISO-8601 format (YYYY-MM-DD, e.g., `2024-01-15`). Invalid values will cause a row error.
- **email**: Must be a valid email format (contains `@`, 5-254 characters, valid format). Invalid values will cause a row error.
- **Error Messages:** Custom field validation errors are included in the import error list with format: `custom_field: <name> <reason>` (e.g., `custom_field: Alter expected integer, got: abc`)
**Member Field Header Mapping:**
@ -496,36 +503,51 @@ Use `Mv.Authorization.PermissionSets` (preferred) instead of hard-coded string c
**Dependencies:** Issue #6
**Status:** ✅ **COMPLETED**
**Goal:** UI section with upload, progress, results, and template links.
**Tasks:**
- [ ] Render import section only for admins
- [ ] **Add prominent UI notice about custom fields:**
- [x] Render import section only for admins
- [x] **Add prominent UI notice about custom fields:**
- Display alert/info box: "Custom fields must be created in Mila before importing CSV files with custom field columns"
- Explain: "Use the custom field name as the CSV column header (same normalization as member fields applies)"
- Add link to custom fields management section
- [ ] Configure `allow_upload/3`:
- `.csv` only, `max_entries: 1`, `max_file_size: 10MB`, `auto_upload: false`
- [ ] `handle_event("start_import", ...)`:
- [x] Configure `allow_upload/3`:
- `.csv` only, `max_entries: 1`, `max_file_size: 10MB`, `auto_upload: true` (auto-upload enabled for better UX)
- [x] `handle_event("start_import", ...)`:
- Admin permission check
- Consume upload -> read file content
- Call `MemberCSV.prepare/2`
- Store `import_state` in assigns (chunks + column_map + metadata)
- Initialize progress assigns
- `send(self(), {:process_chunk, 0})`
- [ ] `handle_info({:process_chunk, idx}, socket)`:
- [x] `handle_info({:process_chunk, idx}, socket)`:
- Fetch chunk from `import_state`
- Call `MemberCSV.process_chunk/3`
- Call `MemberCSV.process_chunk/4` with error capping support
- Merge counts/errors into progress assigns (cap errors at 50 overall)
- Schedule next chunk (or finish and show results)
- [ ] Results UI:
- Async task processing with SQL sandbox support for tests
- [x] Results UI:
- Success count
- Failure count
- Error list (line number + message + field)
- **Warning messages for unknown custom field columns** (non-existent names) shown in results
- Progress indicator during import
- Error truncation notice when errors exceed limit
**Template links:**
- Link `/templates/member_import_en.csv` and `/templates/member_import_de.csv` via Phoenix static path helpers.
- [x] Link `/templates/member_import_en.csv` and `/templates/member_import_de.csv` via Phoenix static path helpers.
**Definition of Done:**
- [x] Upload area with drag & drop support
- [x] Template download links (EN/DE)
- [x] Progress tracking during import
- [x] Results display with success/error counts
- [x] Error list with line numbers and field information
- [x] Warning display for unknown custom field columns
- [x] Admin-only access control
- [x] Async chunk processing with proper error handling
---
@ -533,19 +555,32 @@ Use `Mv.Authorization.PermissionSets` (preferred) instead of hard-coded string c
**Dependencies:** None (can be parallelized)
**Status:** ✅ **COMPLETED**
**Goal:** Ensure admin-only access and enforce limits.
**Tasks:**
- [ ] Admin check in start import event handler
- [ ] File size enforced in upload config
- [ ] Row limit enforced in `MemberCSV.prepare/2` (max_rows from config)
- [ ] Configuration:
```elixir
config :mv, csv_import: [
max_file_size_mb: 10,
max_rows: 1000
]
```
- [x] Admin check in start import event handler (via `Authorization.can?/3`)
- [x] File size enforced in upload config (`max_file_size: 10MB`)
- [x] Row limit enforced in `MemberCSV.prepare/2` (max_rows: 1000, configurable via opts)
- [x] Chunk size limit (200 rows per chunk)
- [x] Error limit (50 errors per import)
- [x] UI-level authorization check (import section only visible to admins)
- [x] Event-level authorization check (prevents unauthorized import attempts)
**Implementation Notes:**
- File size limit: 10 MB (10,485,760 bytes) enforced via `allow_upload/3`
- Row limit: 1,000 rows (excluding header) enforced in `MemberCSV.prepare/2`
- Chunk size: 200 rows per chunk (configurable via opts)
- Error limit: 50 errors per import (configurable via `@max_errors`)
- Authorization uses `MvWeb.Authorization.can?/3` with `:create` permission on `Mv.Membership.Member`
**Definition of Done:**
- [x] Admin-only access enforced at UI and event level
- [x] File size limit enforced
- [x] Row count limit enforced
- [x] Chunk processing with size limits
- [x] Error capping implemented
---
@ -589,7 +624,7 @@ Use `Mv.Authorization.PermissionSets` (preferred) instead of hard-coded string c
**Priority:** High (Core v1 Feature)
**Status:** ✅ **COMPLETED** (Backend Implementation)
**Status:** ✅ **COMPLETED** (Backend + UI Implementation)
**Goal:** Support importing custom field values from CSV columns. Custom fields should exist in Mila before import for best results.
@ -604,23 +639,26 @@ Use `Mv.Authorization.PermissionSets` (preferred) instead of hard-coded string c
- [x] Query existing custom fields during `prepare/2` to map custom field columns
- [x] Collect unknown custom field columns and add warning messages (don't fail import)
- [x] Map custom field CSV values to `CustomFieldValue` creation in `process_chunk/4`
- [x] Handle custom field type validation (string, integer, boolean, date, email)
- [x] Handle custom field type validation (string, integer, boolean, date, email) with proper error messages
- [x] Create `CustomFieldValue` records linked to members during import
- [ ] Update error messages to include custom field validation errors (if needed)
- [ ] Add UI help text explaining custom field requirements (pending Issue #7):
- [x] Validate custom field values and return structured errors with custom field name and reason
- [x] UI help text and link to custom field management (implemented in Issue #7)
- [x] Update error messages to include custom field validation errors (format: `custom_field: <name> expected <type>, got: <value>`)
- [x] Add UI help text explaining custom field requirements (completed in Issue #7):
- "Custom fields must be created in Mila before importing"
- "Use the custom field name as the CSV column header (same normalization as member fields)"
- Link to custom fields management section
- [ ] Update CSV templates documentation to explain custom field columns (pending Issue #1)
- [x] Update CSV templates documentation to explain custom field columns (documented in Issue #1)
- [x] Add tests for custom field import (valid, invalid name, type validation, warning for unknown)
**Definition of Done:**
- [x] Custom field columns are recognized by name (with normalization)
- [x] Warning messages shown for unknown custom field columns (import continues)
- [x] Custom field values are created and linked to members
- [x] Type validation works for all custom field types
- [ ] UI clearly explains custom field requirements (pending Issue #7)
- [x] Type validation works for all custom field types (string, integer, boolean, date, email)
- [x] UI clearly explains custom field requirements (completed in Issue #7)
- [x] Tests cover custom field import scenarios (including warning for unknown names)
- [x] Error messages include custom field validation errors with proper formatting
**Implementation Notes:**
- Custom field lookup is built in `prepare/2` and passed via `custom_field_lookup` in opts

View file

@ -21,4 +21,99 @@ defmodule Mv.Config do
def sql_sandbox? do
Application.get_env(:mv, :sql_sandbox, false)
end
@doc """
Returns the maximum file size for CSV imports in bytes.
Reads the `max_file_size_mb` value from the CSV import configuration
and converts it to bytes.
## Returns
- Maximum file size in bytes (default: 10_485_760 bytes = 10 MB)
## Examples
iex> Mv.Config.csv_import_max_file_size_bytes()
10_485_760
"""
@spec csv_import_max_file_size_bytes() :: non_neg_integer()
def csv_import_max_file_size_bytes do
max_file_size_mb = get_csv_import_config(:max_file_size_mb, 10)
max_file_size_mb * 1024 * 1024
end
@doc """
Returns the maximum number of rows allowed in CSV imports.
Reads the `max_rows` value from the CSV import configuration.
## Returns
- Maximum number of rows (default: 1000)
## Examples
iex> Mv.Config.csv_import_max_rows()
1000
"""
@spec csv_import_max_rows() :: pos_integer()
def csv_import_max_rows do
get_csv_import_config(:max_rows, 1000)
end
@doc """
Returns the maximum file size for CSV imports in megabytes.
Reads the `max_file_size_mb` value from the CSV import configuration.
## Returns
- Maximum file size in megabytes (default: 10)
## Examples
iex> Mv.Config.csv_import_max_file_size_mb()
10
"""
@spec csv_import_max_file_size_mb() :: pos_integer()
def csv_import_max_file_size_mb do
get_csv_import_config(:max_file_size_mb, 10)
end
# Helper function to get CSV import config values
defp get_csv_import_config(key, default) do
Application.get_env(:mv, :csv_import, [])
|> Keyword.get(key, default)
|> parse_and_validate_integer(default)
end
# Parses and validates integer configuration values.
#
# Accepts:
# - Integer values (passed through)
# - String integers (e.g., "1000") - parsed to integer
# - Invalid values (e.g., "abc", nil) - falls back to default
#
# Always clamps the result to a minimum of 1 to ensure positive values.
#
# Note: We don't log warnings for unparseable values because:
# - These functions may be called frequently (e.g., on every request)
# - Logging would create excessive log spam
# - The fallback to default provides a safe behavior
# - Configuration errors should be caught during deployment/testing
defp parse_and_validate_integer(value, _default) when is_integer(value) do
max(1, value)
end
defp parse_and_validate_integer(value, default) when is_binary(value) do
case Integer.parse(value) do
{int, _remainder} -> max(1, int)
:error -> default
end
end
defp parse_and_validate_integer(_value, default) do
default
end
end

View file

@ -63,7 +63,9 @@ defmodule Mv.Membership.Import.MemberCSV do
chunks: list(list({pos_integer(), map()})),
column_map: %{atom() => non_neg_integer()},
custom_field_map: %{String.t() => non_neg_integer()},
custom_field_lookup: %{String.t() => %{id: String.t(), value_type: atom()}},
custom_field_lookup: %{
String.t() => %{id: String.t(), value_type: atom(), name: String.t()}
},
warnings: list(String.t())
}
@ -79,6 +81,11 @@ defmodule Mv.Membership.Import.MemberCSV do
use Gettext, backend: MvWeb.Gettext
alias Mv.Helpers.SystemActor
# Import FieldTypes for human-readable type labels
alias MvWeb.Translations.FieldTypes
# Configuration constants
@default_max_errors 50
@default_chunk_size 200
@ -102,6 +109,7 @@ defmodule Mv.Membership.Import.MemberCSV do
- `opts` - Optional keyword list:
- `:max_rows` - Maximum number of data rows allowed (default: 1000)
- `:chunk_size` - Number of rows per chunk (default: 200)
- `:actor` - Actor for authorization (default: system actor for systemic operations)
## Returns
@ -120,9 +128,10 @@ defmodule Mv.Membership.Import.MemberCSV do
def prepare(file_content, opts \\ []) do
max_rows = Keyword.get(opts, :max_rows, @default_max_rows)
chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size)
actor = Keyword.get(opts, :actor, SystemActor.get_system_actor())
with {:ok, headers, rows} <- CsvParser.parse(file_content),
{:ok, custom_fields} <- load_custom_fields(),
{:ok, custom_fields} <- load_custom_fields(actor),
{:ok, maps, warnings} <- build_header_maps(headers, custom_fields),
:ok <- validate_row_count(rows, max_rows) do
chunks = chunk_rows(rows, maps, chunk_size)
@ -142,10 +151,10 @@ defmodule Mv.Membership.Import.MemberCSV do
end
# Loads all custom fields from the database
defp load_custom_fields do
defp load_custom_fields(actor) do
custom_fields =
Mv.Membership.CustomField
|> Ash.read!()
|> Ash.read!(actor: actor)
{:ok, custom_fields}
rescue
@ -158,7 +167,7 @@ defmodule Mv.Membership.Import.MemberCSV do
custom_fields
|> Enum.reduce(%{}, fn cf, acc ->
id_str = to_string(cf.id)
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type})
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type, name: cf.name})
end)
end
@ -182,8 +191,10 @@ defmodule Mv.Membership.Import.MemberCSV do
normalized != "" && not member_field?(normalized)
end)
|> Enum.map(fn header ->
"Unknown column '#{header}' will be ignored. " <>
"If this is a custom field, create it in Mila before importing."
gettext(
"Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing.",
header: header
)
end)
{:ok, %{member: member_map, custom: custom_map}, warnings}
@ -224,17 +235,20 @@ defmodule Mv.Membership.Import.MemberCSV do
# Builds a row map from raw row values using column maps
defp build_row_map(row_values, maps) do
row_tuple = List.to_tuple(row_values)
tuple_size = tuple_size(row_tuple)
member_map =
maps.member
|> Enum.reduce(%{}, fn {field, index}, acc ->
value = Enum.at(row_values, index, "")
value = if index < tuple_size, do: elem(row_tuple, index), else: ""
Map.put(acc, field, value)
end)
custom_map =
maps.custom
|> Enum.reduce(%{}, fn {custom_field_id, index}, acc ->
value = Enum.at(row_values, index, "")
value = if index < tuple_size, do: elem(row_tuple, index), else: ""
Map.put(acc, custom_field_id, value)
end)
@ -299,7 +313,7 @@ defmodule Mv.Membership.Import.MemberCSV do
custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{})
existing_error_count = Keyword.get(opts, :existing_error_count, 0)
max_errors = Keyword.get(opts, :max_errors, @default_max_errors)
actor = Keyword.fetch!(opts, :actor)
actor = Keyword.get(opts, :actor, SystemActor.get_system_actor())
{inserted, failed, errors, _collected_error_count, truncated?} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
@ -508,8 +522,33 @@ defmodule Mv.Membership.Import.MemberCSV do
{:ok, %{member: trimmed_member_attrs, custom: custom_attrs}} ->
# Prepare custom field values for Ash
custom_field_values = prepare_custom_field_values(custom_attrs, custom_field_lookup)
case prepare_custom_field_values(custom_attrs, custom_field_lookup) do
{:error, validation_errors} ->
# Custom field validation errors - return first error
first_error = List.first(validation_errors)
{:error, %Error{csv_line_number: line_number, field: nil, message: first_error}}
{:ok, custom_field_values} ->
create_member_with_custom_fields(
trimmed_member_attrs,
custom_field_values,
line_number,
actor
)
end
end
rescue
e ->
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)}}
end
# Creates a member with custom field values, handling errors appropriately
defp create_member_with_custom_fields(
trimmed_member_attrs,
custom_field_values,
line_number,
actor
) do
# Create member with custom field values
member_attrs_with_cf =
trimmed_member_attrs
@ -536,76 +575,208 @@ defmodule Mv.Membership.Import.MemberCSV do
{:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}}
end
end
rescue
e ->
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)}}
end
# Prepares custom field values from row map for Ash
# Returns {:ok, [custom_field_value_maps]} or {:error, [validation_errors]}
defp prepare_custom_field_values(custom_attrs, custom_field_lookup) when is_map(custom_attrs) do
{values, errors} =
custom_attrs
|> Enum.filter(fn {_id, value} -> value != nil && value != "" end)
|> Enum.map(fn {custom_field_id_str, value} ->
|> Enum.reduce({[], []}, fn {custom_field_id_str, value}, {acc_values, acc_errors} ->
process_single_custom_field(
custom_field_id_str,
value,
custom_field_lookup,
acc_values,
acc_errors
)
end)
if Enum.empty?(errors) do
{:ok, Enum.reverse(values)}
else
{:error, Enum.reverse(errors)}
end
end
defp prepare_custom_field_values(_, _), do: {:ok, []}
# Processes a single custom field value and returns updated accumulator
defp process_single_custom_field(
custom_field_id_str,
value,
custom_field_lookup,
acc_values,
acc_errors
) do
# Trim value early and skip if empty
trimmed_value = if is_binary(value), do: String.trim(value), else: value
# Skip empty values (after trimming) - don't create CFV
if trimmed_value == "" or trimmed_value == nil do
{acc_values, acc_errors}
else
process_non_empty_custom_field(
custom_field_id_str,
trimmed_value,
custom_field_lookup,
acc_values,
acc_errors
)
end
end
# Processes a non-empty custom field value
defp process_non_empty_custom_field(
custom_field_id_str,
trimmed_value,
custom_field_lookup,
acc_values,
acc_errors
) do
case Map.get(custom_field_lookup, custom_field_id_str) do
nil ->
# Custom field not found, skip
nil
{acc_values, acc_errors}
%{id: custom_field_id, value_type: value_type} ->
%{
%{id: custom_field_id, value_type: value_type, name: custom_field_name} ->
case format_custom_field_value(trimmed_value, value_type, custom_field_name) do
{:ok, formatted_value} ->
value_map = %{
"custom_field_id" => to_string(custom_field_id),
"value" => format_custom_field_value(value, value_type)
"value" => formatted_value
}
end
end)
|> Enum.filter(&(&1 != nil))
end
defp prepare_custom_field_values(_, _), do: []
{[value_map | acc_values], acc_errors}
{:error, reason} ->
{acc_values, [reason | acc_errors]}
end
end
end
# Formats a custom field value according to its type
# Uses _union_type and _union_value format as expected by Ash
defp format_custom_field_value(value, :string) when is_binary(value) do
%{"_union_type" => "string", "_union_value" => String.trim(value)}
# Returns {:ok, formatted_value} or {:error, error_message}
defp format_custom_field_value(value, :string, _custom_field_name) when is_binary(value) do
{:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}}
end
defp format_custom_field_value(value, :integer) when is_binary(value) do
case Integer.parse(value) do
{int_value, _} -> %{"_union_type" => "integer", "_union_value" => int_value}
:error -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
defp format_custom_field_value(value, :integer, custom_field_name) when is_binary(value) do
trimmed = String.trim(value)
case Integer.parse(trimmed) do
{int_value, ""} ->
# Fully consumed - valid integer
{:ok, %{"_union_type" => "integer", "_union_value" => int_value}}
{_int_value, _remaining} ->
# Not fully consumed - invalid
{:error, format_custom_field_error(custom_field_name, :integer, trimmed)}
:error ->
{:error, format_custom_field_error(custom_field_name, :integer, trimmed)}
end
end
defp format_custom_field_value(value, :boolean) when is_binary(value) do
bool_value =
value
|> String.trim()
|> String.downcase()
|> case do
"true" -> true
"1" -> true
"yes" -> true
"ja" -> true
_ -> false
end
defp format_custom_field_value(value, :boolean, custom_field_name) when is_binary(value) do
trimmed = String.trim(value)
%{"_union_type" => "boolean", "_union_value" => bool_value}
end
case parse_boolean_value(trimmed) do
{:ok, bool_value} ->
{:ok, %{"_union_type" => "boolean", "_union_value" => bool_value}}
defp format_custom_field_value(value, :date) when is_binary(value) do
case Date.from_iso8601(String.trim(value)) do
{:ok, date} -> %{"_union_type" => "date", "_union_value" => date}
{:error, _} -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
:error ->
{:error,
format_custom_field_error_with_details(
custom_field_name,
:boolean,
trimmed,
gettext("(true/false/1/0/yes/no/ja/nein)")
)}
end
end
defp format_custom_field_value(value, :email) when is_binary(value) do
%{"_union_type" => "email", "_union_value" => String.trim(value)}
defp format_custom_field_value(value, :date, custom_field_name) when is_binary(value) do
trimmed = String.trim(value)
case Date.from_iso8601(trimmed) do
{:ok, date} ->
{:ok, %{"_union_type" => "date", "_union_value" => date}}
{:error, _} ->
{:error,
format_custom_field_error_with_details(
custom_field_name,
:date,
trimmed,
gettext("(ISO-8601 format: YYYY-MM-DD)")
)}
end
end
defp format_custom_field_value(value, _type) when is_binary(value) do
defp format_custom_field_value(value, :email, custom_field_name) when is_binary(value) do
trimmed = String.trim(value)
# Use EctoCommons.EmailValidator for consistency with Member email validation
changeset =
{%{}, %{email: :string}}
|> Ecto.Changeset.cast(%{email: trimmed}, [:email])
|> EctoCommons.EmailValidator.validate_email(:email,
checks: Mv.Constants.email_validator_checks()
)
if changeset.valid? do
{:ok, %{"_union_type" => "email", "_union_value" => trimmed}}
else
{:error, format_custom_field_error(custom_field_name, :email, trimmed)}
end
end
defp format_custom_field_value(value, _type, _custom_field_name) when is_binary(value) do
# Default to string if type is unknown
%{"_union_type" => "string", "_union_value" => String.trim(value)}
{:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}}
end
# Parses a boolean value from a string, supporting multiple formats
defp parse_boolean_value(value) when is_binary(value) do
lower = String.downcase(value)
parse_boolean_value_lower(lower)
end
# Helper function with pattern matching for boolean values
defp parse_boolean_value_lower("true"), do: {:ok, true}
defp parse_boolean_value_lower("1"), do: {:ok, true}
defp parse_boolean_value_lower("yes"), do: {:ok, true}
defp parse_boolean_value_lower("ja"), do: {:ok, true}
defp parse_boolean_value_lower("false"), do: {:ok, false}
defp parse_boolean_value_lower("0"), do: {:ok, false}
defp parse_boolean_value_lower("no"), do: {:ok, false}
defp parse_boolean_value_lower("nein"), do: {:ok, false}
defp parse_boolean_value_lower(_), do: :error
# Generates a consistent error message for custom field validation failures
# Uses human-readable field type labels (e.g., "Number" instead of "integer")
defp format_custom_field_error(custom_field_name, value_type, value) do
type_label = FieldTypes.label(value_type)
gettext("custom_field: %{name} expected %{type}, got: %{value}",
name: custom_field_name,
type: type_label,
value: value
)
end
# Generates an error message with additional details (e.g., format hints)
defp format_custom_field_error_with_details(custom_field_name, value_type, value, details) do
type_label = FieldTypes.label(value_type)
gettext("custom_field: %{name} expected %{type} %{details}, got: %{value}",
name: custom_field_name,
type: type_label,
details: details,
value: value
)
end
# Trims all string values in member attributes

View file

@ -50,9 +50,9 @@ defmodule MvWeb.CustomFieldLive.IndexComponent do
</div>
<%!-- Hide table when form is visible --%>
<div :if={!@show_form} id="custom_fields">
<.table
:if={!@show_form}
id="custom_fields"
id="custom_fields_table"
rows={@streams.custom_fields}
row_click={
fn {_id, custom_field} ->
@ -105,11 +105,14 @@ defmodule MvWeb.CustomFieldLive.IndexComponent do
</:action>
<:action :let={{_id, custom_field}}>
<.link phx-click={JS.push("prepare_delete", value: %{id: custom_field.id}, target: @myself)}>
<.link phx-click={
JS.push("prepare_delete", value: %{id: custom_field.id}, target: @myself)
}>
{gettext("Delete")}
</.link>
</:action>
</.table>
</div>
<%!-- Delete Confirmation Modal --%>
<dialog :if={@show_delete_modal} id="delete-custom-field-modal" class="modal modal-open">

View file

@ -34,8 +34,8 @@ defmodule MvWeb.GlobalSettingsLive do
### Limits
- Maximum file size: 10 MB
- Maximum rows: 1,000 rows (excluding header)
- Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
- Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
- Processing: chunks of 200 rows
- Errors: capped at 50 per import
@ -54,8 +54,6 @@ defmodule MvWeb.GlobalSettingsLive do
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
# CSV Import configuration constants
# 10 MB
@max_file_size_bytes 10_485_760
@max_errors 50
@impl true
@ -76,13 +74,15 @@ defmodule MvWeb.GlobalSettingsLive do
|> assign(:import_status, :idle)
|> assign(:locale, locale)
|> assign(:max_errors, @max_errors)
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
|> assign_form()
# Configure file upload with auto-upload enabled
# Files are uploaded automatically when selected, no need for manual trigger
|> allow_upload(:csv_file,
accept: ~w(.csv),
max_entries: 1,
max_file_size: @max_file_size_bytes,
max_file_size: Config.csv_import_max_file_size_bytes(),
auto_upload: true
)
@ -138,16 +138,21 @@ defmodule MvWeb.GlobalSettingsLive do
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
<.form_section title={gettext("Import Members (CSV)")}>
<div role="note" class="alert alert-info mb-4">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<p class="font-semibold">
<p class="text-sm mb-2">
{gettext(
"Custom fields must be created in Mila before importing CSV files with custom field columns"
"Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
)}
</p>
<p class="text-sm mt-2">
{gettext(
"Use the custom field name as the CSV column header (same normalization as member fields applies)"
)}
<p class="text-sm">
<.link
href="#custom_fields"
class="link"
data-testid="custom-fields-link"
>
{gettext("Manage Memberdata")}
</.link>
</p>
</div>
</div>
@ -200,7 +205,7 @@ defmodule MvWeb.GlobalSettingsLive do
/>
<label class="label" id="csv_file_help">
<span class="label-text-alt">
{gettext("CSV files only, maximum 10 MB")}
{gettext("CSV files only, maximum %{size} MB", size: @csv_import_max_file_size_mb)}
</span>
</label>
</div>
@ -408,8 +413,11 @@ defmodule MvWeb.GlobalSettingsLive do
# Processes CSV upload and starts import
defp process_csv_upload(socket) do
actor = MvWeb.LiveHelpers.current_actor(socket)
with {:ok, content} <- consume_and_read_csv(socket),
{:ok, import_state} <- MemberCSV.prepare(content) do
{:ok, import_state} <-
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
start_import(socket, import_state)
else
{:error, reason} when is_binary(reason) ->

View file

@ -1980,16 +1980,6 @@ msgstr " (Datenfeld: %{field})"
msgid "CSV File"
msgstr "CSV Datei"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum 10 MB"
msgstr "Nur CSV Dateien, maximal 10 MB"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
msgstr "Individuelle Datenfelder müssen zuerst in Mila angelegt werden bevor das Importieren von diesen Feldern mit CSV Dateien mölich ist."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Download CSV templates:"
@ -2120,11 +2110,6 @@ msgstr "Erfolgreich eingefügt: %{count} Mitglied(er)"
msgid "Summary"
msgstr "Zusammenfassung"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
msgstr "Verwenden Sie den Namen des benutzerdefinierten Feldes als CSV-Spaltenüberschrift (gleiche Normalisierung wie bei Mitgliedsfeldern)"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Warnings"
@ -2272,3 +2257,58 @@ msgstr "Nicht berechtigt."
#, elixir-autogen, elixir-format
msgid "Could not load data fields. Please check your permissions."
msgstr "Datenfelder konnten nicht geladen werden. Bitte überprüfen Sie Ihre Berechtigungen."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "CSV files only, maximum %{size} MB"
msgstr "Nur CSV Dateien, maximal %{size} MB"
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "(ISO-8601 format: YYYY-MM-DD)"
msgstr "(ISO-8601 Format: JJJJ-MM-TT)"
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "(true/false/1/0/yes/no/ja/nein)"
msgstr "(true/false/1/0/yes/no/ja/nein)"
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "custom_field: %{name} expected %{type} %{details}, got: %{value}"
msgstr "Datenfeld: %{name} erwartet %{type} %{details}, erhalten: %{value}"
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "custom_field: %{name} expected %{type}, got: %{value}"
msgstr "Datenfeld: %{name} erwartet %{type}, erhalten: %{value}"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Manage Memberdata"
msgstr "Mitgliederdaten verwalten"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
msgstr "Verwende die Namen der Datenfelder als Spaltennamen in der CSV Datei. Datenfelder müssen in Mila bereits angelegt sein, damit sie importiert werden können. Sie müssen in der Liste der Mitgliederdaten als Datenfeld enthalten sein (z.B. E-Mail). Spalten mit unbekannten Spaltenüberschriften werden mit einer Warnung ignoriert."
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing."
msgstr "Unbekannte Spalte '%{header}' wird ignoriert. Falls dies ein Datenfeld ist, erstellen Sie es in Mila vor dem Import."
#~ #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Custom Fields in CSV Import"
#~ msgstr "Benutzerdefinierte Felder"
#~ #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Individual data fields must be created in Mila before importing. Use the field name as the CSV column header. Unknown custom field columns will be ignored with a warning."
#~ msgstr "Individuelle Datenfelder müssen in Mila erstellt werden, bevor sie importiert werden können. Verwenden Sie den Namen des Datenfeldes als CSV-Spaltenüberschrift. Unbekannte Spaltenüberschriften werden mit einer Warnung ignoriert."
#~ #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Manage Custom Fields"
#~ msgstr "Benutzerdefinierte Felder verwalten"

View file

@ -1981,16 +1981,6 @@ msgstr ""
msgid "CSV File"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum 10 MB"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Download CSV templates:"
@ -2121,11 +2111,6 @@ msgstr ""
msgid "Summary"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Warnings"
@ -2273,3 +2258,43 @@ msgstr ""
#, elixir-autogen, elixir-format
msgid "Could not load data fields. Please check your permissions."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum %{size} MB"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "(ISO-8601 format: YYYY-MM-DD)"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "(true/false/1/0/yes/no/ja/nein)"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "custom_field: %{name} expected %{type} %{details}, got: %{value}"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "custom_field: %{name} expected %{type}, got: %{value}"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Manage Memberdata"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing."
msgstr ""

View file

@ -1981,16 +1981,6 @@ msgstr ""
msgid "CSV File"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum 10 MB"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Download CSV templates:"
@ -2121,11 +2111,6 @@ msgstr ""
msgid "Summary"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Warnings"
@ -2273,3 +2258,58 @@ msgstr ""
#, elixir-autogen, elixir-format
msgid "Could not load data fields. Please check your permissions."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "CSV files only, maximum %{size} MB"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "(ISO-8601 format: YYYY-MM-DD)"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "(true/false/1/0/yes/no/ja/nein)"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "custom_field: %{name} expected %{type} %{details}, got: %{value}"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "custom_field: %{name} expected %{type}, got: %{value}"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Manage Memberdata"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing."
msgstr "Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing."
#~ #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Custom Fields in CSV Import"
#~ msgstr ""
#~ #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Individual data fields must be created in Mila before importing. Use the field name as the CSV column header. Unknown custom field columns will be ignored with a warning."
#~ msgstr ""
#~ #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Manage Custom Fields"
#~ msgstr ""

View file

@ -1,5 +1,5 @@
defmodule Mv.Membership.Import.MemberCSVTest do
use Mv.DataCase, async: false
use Mv.DataCase, async: true
alias Mv.Membership.Import.MemberCSV
@ -35,11 +35,10 @@ defmodule Mv.Membership.Import.MemberCSVTest do
end
describe "prepare/2" do
test "function exists and accepts file_content and opts" do
test "accepts file_content and opts and returns tagged tuple" do
file_content = "email\njohn@example.com"
opts = []
# This will fail until the function is implemented
result = MemberCSV.prepare(file_content, opts)
assert match?({:ok, _}, result) or match?({:error, _}, result)
end
@ -65,11 +64,6 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert {:error, _reason} = MemberCSV.prepare(file_content, opts)
end
test "function has documentation" do
# Check that @doc exists by reading the module
assert function_exported?(MemberCSV, :prepare, 2)
end
end
describe "process_chunk/4" do
@ -78,7 +72,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
%{actor: system_actor}
end
test "function exists and accepts chunk_rows_with_lines, column_map, custom_field_map, and opts",
test "accepts chunk_rows_with_lines, column_map, custom_field_map, and opts and returns tagged tuple",
%{
actor: actor
} do
@ -87,7 +81,6 @@ defmodule Mv.Membership.Import.MemberCSVTest do
custom_field_map = %{}
opts = [actor: actor]
# This will fail until the function is implemented
result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
assert match?({:ok, _}, result) or match?({:error, _}, result)
end
@ -231,7 +224,11 @@ defmodule Mv.Membership.Import.MemberCSVTest do
custom_field_map = %{to_string(custom_field.id) => 1}
custom_field_lookup = %{
to_string(custom_field.id) => %{id: custom_field.id, value_type: custom_field.value_type}
to_string(custom_field.id) => %{
id: custom_field.id,
value_type: custom_field.value_type,
name: custom_field.name
}
}
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
@ -332,11 +329,6 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert chunk_result.errors == []
end
test "function has documentation" do
# Check that @doc exists by reading the module
assert function_exported?(MemberCSV, :process_chunk, 4)
end
test "error capping collects exactly 50 errors", %{actor: actor} do
# Create 50 rows with invalid emails
chunk_rows_with_lines =
@ -611,15 +603,300 @@ defmodule Mv.Membership.Import.MemberCSVTest do
end
end
describe "module documentation" do
test "module has @moduledoc" do
# Check that the module exists and has documentation
assert Code.ensure_loaded?(MemberCSV)
describe "custom field import" do
setup do
system_actor = Mv.Helpers.SystemActor.get_system_actor()
%{actor: system_actor}
end
# Try to get the module documentation
{:docs_v1, _, _, _, %{"en" => moduledoc}, _, _} = Code.fetch_docs(MemberCSV)
assert is_binary(moduledoc)
assert String.length(moduledoc) > 0
test "creates member with valid integer custom field value", %{actor: actor} do
# Create integer custom field
{:ok, custom_field} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{
name: "Alter",
value_type: :integer
})
|> Ash.create(actor: actor)
chunk_rows_with_lines = [
{2,
%{
member: %{email: "withage@example.com"},
custom: %{to_string(custom_field.id) => "25"}
}}
]
column_map = %{email: 0}
custom_field_map = %{to_string(custom_field.id) => 1}
custom_field_lookup = %{
to_string(custom_field.id) => %{
id: custom_field.id,
value_type: custom_field.value_type,
name: custom_field.name
}
}
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
assert chunk_result.inserted == 1
assert chunk_result.failed == 0
# Verify member and custom field value were created
members = Mv.Membership.list_members!(actor: actor)
member = Enum.find(members, &(&1.email == "withage@example.com"))
assert member != nil
{:ok, member_with_cf} = Ash.load(member, :custom_field_values, actor: actor)
assert length(member_with_cf.custom_field_values) == 1
cfv = List.first(member_with_cf.custom_field_values)
assert cfv.custom_field_id == custom_field.id
assert cfv.value.value == 25
assert cfv.value.type == :integer
end
test "returns error for invalid integer custom field value", %{actor: actor} do
# Create integer custom field
{:ok, custom_field} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{
name: "Alter",
value_type: :integer
})
|> Ash.create(actor: actor)
chunk_rows_with_lines = [
{2,
%{
member: %{email: "invalidage@example.com"},
custom: %{to_string(custom_field.id) => "abc"}
}}
]
column_map = %{email: 0}
custom_field_map = %{to_string(custom_field.id) => 1}
custom_field_lookup = %{
to_string(custom_field.id) => %{
id: custom_field.id,
value_type: custom_field.value_type,
name: custom_field.name
}
}
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
assert chunk_result.inserted == 0
assert chunk_result.failed == 1
assert length(chunk_result.errors) == 1
error = List.first(chunk_result.errors)
assert error.csv_line_number == 2
assert error.message =~ "custom_field: Alter"
assert error.message =~ "Number"
assert error.message =~ "abc"
end
test "returns error for invalid date custom field value", %{actor: actor} do
# Create date custom field
{:ok, custom_field} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{
name: "Geburtstag",
value_type: :date
})
|> Ash.create(actor: actor)
chunk_rows_with_lines = [
{3,
%{
member: %{email: "invaliddate@example.com"},
custom: %{to_string(custom_field.id) => "not-a-date"}
}}
]
column_map = %{email: 0}
custom_field_map = %{to_string(custom_field.id) => 1}
custom_field_lookup = %{
to_string(custom_field.id) => %{
id: custom_field.id,
value_type: custom_field.value_type,
name: custom_field.name
}
}
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
assert chunk_result.inserted == 0
assert chunk_result.failed == 1
assert length(chunk_result.errors) == 1
error = List.first(chunk_result.errors)
assert error.csv_line_number == 3
assert error.message =~ "custom_field: Geburtstag"
assert error.message =~ "Date"
end
test "returns error for invalid email custom field value", %{actor: actor} do
# Create email custom field
{:ok, custom_field} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{
name: "Work Email",
value_type: :email
})
|> Ash.create(actor: actor)
chunk_rows_with_lines = [
{4,
%{
member: %{email: "invalidemailcf@example.com"},
custom: %{to_string(custom_field.id) => "not-an-email"}
}}
]
column_map = %{email: 0}
custom_field_map = %{to_string(custom_field.id) => 1}
custom_field_lookup = %{
to_string(custom_field.id) => %{
id: custom_field.id,
value_type: custom_field.value_type,
name: custom_field.name
}
}
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
assert chunk_result.inserted == 0
assert chunk_result.failed == 1
assert length(chunk_result.errors) == 1
error = List.first(chunk_result.errors)
assert error.csv_line_number == 4
assert error.message =~ "custom_field: Work Email"
assert error.message =~ "E-Mail"
end
test "returns error for invalid boolean custom field value", %{actor: actor} do
# Create boolean custom field
{:ok, custom_field} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{
name: "Is Active",
value_type: :boolean
})
|> Ash.create(actor: actor)
chunk_rows_with_lines = [
{5,
%{
member: %{email: "invalidbool@example.com"},
custom: %{to_string(custom_field.id) => "maybe"}
}}
]
column_map = %{email: 0}
custom_field_map = %{to_string(custom_field.id) => 1}
custom_field_lookup = %{
to_string(custom_field.id) => %{
id: custom_field.id,
value_type: custom_field.value_type,
name: custom_field.name
}
}
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
assert chunk_result.inserted == 0
assert chunk_result.failed == 1
assert length(chunk_result.errors) == 1
error = List.first(chunk_result.errors)
assert error.csv_line_number == 5
assert error.message =~ "custom_field: Is Active"
# Error message should indicate boolean/Yes-No validation failure
assert String.contains?(error.message, "Yes/No") ||
String.contains?(error.message, "true/false") ||
String.contains?(error.message, "boolean")
end
end
describe "prepare/2 with custom fields" do
setup do
system_actor = Mv.Helpers.SystemActor.get_system_actor()
# Create a custom field
{:ok, custom_field} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{
name: "Membership Number",
value_type: :string
})
|> Ash.create(actor: system_actor)
%{actor: system_actor, custom_field: custom_field}
end
test "includes custom field in custom_field_map when header matches", %{
custom_field: custom_field
} do
# CSV with custom field column
csv_content = "email;Membership Number\njohn@example.com;12345"
assert {:ok, import_state} = MemberCSV.prepare(csv_content)
# Check that custom field is mapped
assert Map.has_key?(import_state.custom_field_map, to_string(custom_field.id))
assert import_state.column_map[:email] == 0
end
test "includes warning for unknown custom field column", %{custom_field: _custom_field} do
# CSV with unknown custom field column (not matching any existing custom field)
csv_content = "email;NichtExistierend\njohn@example.com;value"
assert {:ok, import_state} = MemberCSV.prepare(csv_content)
# Check that warning is present
assert import_state.warnings != []
warning = List.first(import_state.warnings)
assert warning =~ "NichtExistierend"
assert warning =~ "ignored"
assert warning =~ "custom field"
# Check that unknown column is not in custom_field_map
assert import_state.custom_field_map == %{}
# Member import should still succeed
assert import_state.column_map[:email] == 0
end
test "import succeeds even with unknown custom field columns", %{custom_field: _custom_field} do
# CSV with unknown custom field column
csv_content = "email;UnknownField\njohn@example.com;value"
assert {:ok, import_state} = MemberCSV.prepare(csv_content)
# Import state should be valid
assert import_state.column_map[:email] == 0
assert import_state.chunks != []
end
end
end

View file

@ -0,0 +1,73 @@
defmodule MvWeb.GlobalSettingsLiveConfigTest do
@moduledoc """
Tests for GlobalSettingsLive that modify global Application configuration.
These tests run with `async: false` to prevent race conditions when
modifying global Application environment variables (Application.put_env).
This follows the same pattern as Mv.ConfigTest.
"""
use MvWeb.ConnCase, async: false
import Phoenix.LiveViewTest
# Helper function to upload CSV file in tests
defp upload_csv_file(view, csv_content, filename) do
view
|> file_input("#csv-upload-form", :csv_file, [
%{
last_modified: System.system_time(:second),
name: filename,
content: csv_content,
size: byte_size(csv_content),
type: "text/csv"
}
])
|> render_upload(filename)
end
describe "CSV Import - Configuration Tests" do
setup %{conn: conn} do
# Ensure admin user
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
conn = MvWeb.ConnCase.conn_with_password_user(conn, admin_user)
{:ok, conn: conn, admin_user: admin_user}
end
test "configured row limit is enforced", %{conn: conn} do
# Business rule: CSV import respects configured row limits
# Test that a custom limit (500) is enforced, not just the default (1000)
original_config = Application.get_env(:mv, :csv_import, [])
try do
Application.put_env(:mv, :csv_import, max_rows: 500)
{:ok, view, _html} = live(conn, ~p"/settings")
# Generate CSV with 501 rows (exceeding custom limit of 500)
header = "first_name;last_name;email;street;postal_code;city\n"
rows =
for i <- 1..501 do
"Row#{i};Last#{i};email#{i}@example.com;Street#{i};12345;City#{i}\n"
end
large_csv = header <> Enum.join(rows)
# Simulate file upload using helper function
upload_csv_file(view, large_csv, "too_many_rows_custom.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
html = render(view)
# Business rule: import should be rejected when exceeding configured limit
assert html =~ "exceeds" or html =~ "maximum" or html =~ "limit" or
html =~ "Failed to prepare"
after
# Restore original config
Application.put_env(:mv, :csv_import, original_config)
end
end
end
end

View file

@ -110,7 +110,7 @@ defmodule MvWeb.GlobalSettingsLiveTest do
{:ok, _view, html} = live(conn, ~p"/settings")
# Check for custom fields notice text
assert html =~ "Custom fields" or html =~ "custom field"
assert html =~ "Use the data field name"
end
test "admin user sees template download links", %{conn: conn} do