Compare commits
1 commit
b48529113c
...
8938067195
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8938067195 |
12 changed files with 249 additions and 1026 deletions
|
|
@ -51,13 +51,6 @@ config :mv,
|
|||
generators: [timestamp_type: :utc_datetime],
|
||||
ash_domains: [Mv.Membership, Mv.Accounts, Mv.MembershipFees, Mv.Authorization]
|
||||
|
||||
# CSV Import configuration
|
||||
config :mv,
|
||||
csv_import: [
|
||||
max_file_size_mb: 10,
|
||||
max_rows: 1000
|
||||
]
|
||||
|
||||
# Configures the endpoint
|
||||
config :mv, MvWeb.Endpoint,
|
||||
url: [host: "localhost"],
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
**Version:** 1.0
|
||||
**Last Updated:** 2026-01-13
|
||||
**Status:** In Progress (Backend Complete, UI Complete, Tests Pending)
|
||||
**Status:** In Progress (Backend Complete, UI Pending)
|
||||
**Related Documents:**
|
||||
- [Feature Roadmap](./feature-roadmap.md) - Overall feature planning
|
||||
|
||||
|
|
@ -15,15 +15,15 @@
|
|||
- ✅ Issue #4: Header Normalization + Per-Header Mapping
|
||||
- ✅ Issue #5: Validation (Required Fields) + Error Formatting
|
||||
- ✅ Issue #6: Persistence via Ash Create + Per-Row Error Capture (with Error-Capping)
|
||||
- ✅ Issue #7: Admin Global Settings LiveView UI (Upload + Start Import + Results + Template Links)
|
||||
- ✅ Issue #8: Authorization + Limits
|
||||
- ✅ Issue #11: Custom Field Import (Backend + UI)
|
||||
- ✅ Issue #11: Custom Field Import (Backend)
|
||||
|
||||
**In Progress / Pending:**
|
||||
- ⏳ Issue #7: Admin Global Settings LiveView UI (Upload + Start Import + Results)
|
||||
- ⏳ Issue #8: Authorization + Limits
|
||||
- ⏳ Issue #9: End-to-End LiveView Tests + Fixtures
|
||||
- ⏳ Issue #10: Documentation Polish
|
||||
|
||||
**Latest Update:** CSV Import UI fully implemented in GlobalSettingsLive with chunk processing, progress tracking, error display, and custom field support (2026-01-13)
|
||||
**Latest Update:** Error-Capping in `process_chunk/4` implemented (2025-01-XX)
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -161,13 +161,6 @@ A **basic CSV member import feature** that allows administrators to upload a CSV
|
|||
- Any custom field column using the custom field's **name** as the header (e.g., `membership_number`, `birth_date`)
|
||||
- **Important:** Custom fields must be created in Mila before importing. The CSV header must match the custom field name exactly (same normalization as member fields).
|
||||
- **Behavior:** If the CSV contains custom field columns that don't exist in Mila, a warning message will be shown and those columns will be ignored during import.
|
||||
- **Value Validation:** Custom field values are validated according to the custom field type:
|
||||
- **string**: Any text value (trimmed)
|
||||
- **integer**: Must be a valid integer (e.g., `42`, `-10`). Invalid values will cause a row error with the custom field name and reason.
|
||||
- **boolean**: Accepts `true`, `false`, `1`, `0`, `yes`, `no`, `ja`, `nein` (case-insensitive). Invalid values will cause a row error.
|
||||
- **date**: Must be in ISO-8601 format (YYYY-MM-DD, e.g., `2024-01-15`). Invalid values will cause a row error.
|
||||
- **email**: Must be a valid email format (contains `@`, 5-254 characters, valid format). Invalid values will cause a row error.
|
||||
- **Error Messages:** Custom field validation errors are included in the import error list with format: `custom_field: <name> – <reason>` (e.g., `custom_field: Alter – expected integer, got: abc`)
|
||||
|
||||
**Member Field Header Mapping:**
|
||||
|
||||
|
|
@ -503,51 +496,36 @@ Use `Mv.Authorization.PermissionSets` (preferred) instead of hard-coded string c
|
|||
|
||||
**Dependencies:** Issue #6
|
||||
|
||||
**Status:** ✅ **COMPLETED**
|
||||
|
||||
**Goal:** UI section with upload, progress, results, and template links.
|
||||
|
||||
**Tasks:**
|
||||
- [x] Render import section only for admins
|
||||
- [x] **Add prominent UI notice about custom fields:**
|
||||
- [ ] Render import section only for admins
|
||||
- [ ] **Add prominent UI notice about custom fields:**
|
||||
- Display alert/info box: "Custom fields must be created in Mila before importing CSV files with custom field columns"
|
||||
- Explain: "Use the custom field name as the CSV column header (same normalization as member fields applies)"
|
||||
- Add link to custom fields management section
|
||||
- [x] Configure `allow_upload/3`:
|
||||
- `.csv` only, `max_entries: 1`, `max_file_size: 10MB`, `auto_upload: true` (auto-upload enabled for better UX)
|
||||
- [x] `handle_event("start_import", ...)`:
|
||||
- [ ] Configure `allow_upload/3`:
|
||||
- `.csv` only, `max_entries: 1`, `max_file_size: 10MB`, `auto_upload: false`
|
||||
- [ ] `handle_event("start_import", ...)`:
|
||||
- Admin permission check
|
||||
- Consume upload -> read file content
|
||||
- Call `MemberCSV.prepare/2`
|
||||
- Store `import_state` in assigns (chunks + column_map + metadata)
|
||||
- Initialize progress assigns
|
||||
- `send(self(), {:process_chunk, 0})`
|
||||
- [x] `handle_info({:process_chunk, idx}, socket)`:
|
||||
- [ ] `handle_info({:process_chunk, idx}, socket)`:
|
||||
- Fetch chunk from `import_state`
|
||||
- Call `MemberCSV.process_chunk/4` with error capping support
|
||||
- Call `MemberCSV.process_chunk/3`
|
||||
- Merge counts/errors into progress assigns (cap errors at 50 overall)
|
||||
- Schedule next chunk (or finish and show results)
|
||||
- Async task processing with SQL sandbox support for tests
|
||||
- [x] Results UI:
|
||||
- [ ] Results UI:
|
||||
- Success count
|
||||
- Failure count
|
||||
- Error list (line number + message + field)
|
||||
- **Warning messages for unknown custom field columns** (non-existent names) shown in results
|
||||
- Progress indicator during import
|
||||
- Error truncation notice when errors exceed limit
|
||||
|
||||
**Template links:**
|
||||
- [x] Link `/templates/member_import_en.csv` and `/templates/member_import_de.csv` via Phoenix static path helpers.
|
||||
|
||||
**Definition of Done:**
|
||||
- [x] Upload area with drag & drop support
|
||||
- [x] Template download links (EN/DE)
|
||||
- [x] Progress tracking during import
|
||||
- [x] Results display with success/error counts
|
||||
- [x] Error list with line numbers and field information
|
||||
- [x] Warning display for unknown custom field columns
|
||||
- [x] Admin-only access control
|
||||
- [x] Async chunk processing with proper error handling
|
||||
- Link `/templates/member_import_en.csv` and `/templates/member_import_de.csv` via Phoenix static path helpers.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -555,32 +533,19 @@ Use `Mv.Authorization.PermissionSets` (preferred) instead of hard-coded string c
|
|||
|
||||
**Dependencies:** None (can be parallelized)
|
||||
|
||||
**Status:** ✅ **COMPLETED**
|
||||
|
||||
**Goal:** Ensure admin-only access and enforce limits.
|
||||
|
||||
**Tasks:**
|
||||
- [x] Admin check in start import event handler (via `Authorization.can?/3`)
|
||||
- [x] File size enforced in upload config (`max_file_size: 10MB`)
|
||||
- [x] Row limit enforced in `MemberCSV.prepare/2` (max_rows: 1000, configurable via opts)
|
||||
- [x] Chunk size limit (200 rows per chunk)
|
||||
- [x] Error limit (50 errors per import)
|
||||
- [x] UI-level authorization check (import section only visible to admins)
|
||||
- [x] Event-level authorization check (prevents unauthorized import attempts)
|
||||
|
||||
**Implementation Notes:**
|
||||
- File size limit: 10 MB (10,485,760 bytes) enforced via `allow_upload/3`
|
||||
- Row limit: 1,000 rows (excluding header) enforced in `MemberCSV.prepare/2`
|
||||
- Chunk size: 200 rows per chunk (configurable via opts)
|
||||
- Error limit: 50 errors per import (configurable via `@max_errors`)
|
||||
- Authorization uses `MvWeb.Authorization.can?/3` with `:create` permission on `Mv.Membership.Member`
|
||||
|
||||
**Definition of Done:**
|
||||
- [x] Admin-only access enforced at UI and event level
|
||||
- [x] File size limit enforced
|
||||
- [x] Row count limit enforced
|
||||
- [x] Chunk processing with size limits
|
||||
- [x] Error capping implemented
|
||||
- [ ] Admin check in start import event handler
|
||||
- [ ] File size enforced in upload config
|
||||
- [ ] Row limit enforced in `MemberCSV.prepare/2` (max_rows from config)
|
||||
- [ ] Configuration:
|
||||
```elixir
|
||||
config :mv, csv_import: [
|
||||
max_file_size_mb: 10,
|
||||
max_rows: 1000
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -624,7 +589,7 @@ Use `Mv.Authorization.PermissionSets` (preferred) instead of hard-coded string c
|
|||
|
||||
**Priority:** High (Core v1 Feature)
|
||||
|
||||
**Status:** ✅ **COMPLETED** (Backend + UI Implementation)
|
||||
**Status:** ✅ **COMPLETED** (Backend Implementation)
|
||||
|
||||
**Goal:** Support importing custom field values from CSV columns. Custom fields should exist in Mila before import for best results.
|
||||
|
||||
|
|
@ -639,26 +604,23 @@ Use `Mv.Authorization.PermissionSets` (preferred) instead of hard-coded string c
|
|||
- [x] Query existing custom fields during `prepare/2` to map custom field columns
|
||||
- [x] Collect unknown custom field columns and add warning messages (don't fail import)
|
||||
- [x] Map custom field CSV values to `CustomFieldValue` creation in `process_chunk/4`
|
||||
- [x] Handle custom field type validation (string, integer, boolean, date, email) with proper error messages
|
||||
- [x] Handle custom field type validation (string, integer, boolean, date, email)
|
||||
- [x] Create `CustomFieldValue` records linked to members during import
|
||||
- [x] Validate custom field values and return structured errors with custom field name and reason
|
||||
- [x] UI help text and link to custom field management (implemented in Issue #7)
|
||||
- [x] Update error messages to include custom field validation errors (format: `custom_field: <name> – expected <type>, got: <value>`)
|
||||
- [x] Add UI help text explaining custom field requirements (completed in Issue #7):
|
||||
- [ ] Update error messages to include custom field validation errors (if needed)
|
||||
- [ ] Add UI help text explaining custom field requirements (pending Issue #7):
|
||||
- "Custom fields must be created in Mila before importing"
|
||||
- "Use the custom field name as the CSV column header (same normalization as member fields)"
|
||||
- Link to custom fields management section
|
||||
- [x] Update CSV templates documentation to explain custom field columns (documented in Issue #1)
|
||||
- [ ] Update CSV templates documentation to explain custom field columns (pending Issue #1)
|
||||
- [x] Add tests for custom field import (valid, invalid name, type validation, warning for unknown)
|
||||
|
||||
**Definition of Done:**
|
||||
- [x] Custom field columns are recognized by name (with normalization)
|
||||
- [x] Warning messages shown for unknown custom field columns (import continues)
|
||||
- [x] Custom field values are created and linked to members
|
||||
- [x] Type validation works for all custom field types (string, integer, boolean, date, email)
|
||||
- [x] UI clearly explains custom field requirements (completed in Issue #7)
|
||||
- [x] Type validation works for all custom field types
|
||||
- [ ] UI clearly explains custom field requirements (pending Issue #7)
|
||||
- [x] Tests cover custom field import scenarios (including warning for unknown names)
|
||||
- [x] Error messages include custom field validation errors with proper formatting
|
||||
|
||||
**Implementation Notes:**
|
||||
- Custom field lookup is built in `prepare/2` and passed via `custom_field_lookup` in opts
|
||||
|
|
|
|||
|
|
@ -21,99 +21,4 @@ defmodule Mv.Config do
|
|||
def sql_sandbox? do
|
||||
Application.get_env(:mv, :sql_sandbox, false)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the maximum file size for CSV imports in bytes.
|
||||
|
||||
Reads the `max_file_size_mb` value from the CSV import configuration
|
||||
and converts it to bytes.
|
||||
|
||||
## Returns
|
||||
|
||||
- Maximum file size in bytes (default: 10_485_760 bytes = 10 MB)
|
||||
|
||||
## Examples
|
||||
|
||||
iex> Mv.Config.csv_import_max_file_size_bytes()
|
||||
10_485_760
|
||||
"""
|
||||
@spec csv_import_max_file_size_bytes() :: non_neg_integer()
|
||||
def csv_import_max_file_size_bytes do
|
||||
max_file_size_mb = get_csv_import_config(:max_file_size_mb, 10)
|
||||
max_file_size_mb * 1024 * 1024
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the maximum number of rows allowed in CSV imports.
|
||||
|
||||
Reads the `max_rows` value from the CSV import configuration.
|
||||
|
||||
## Returns
|
||||
|
||||
- Maximum number of rows (default: 1000)
|
||||
|
||||
## Examples
|
||||
|
||||
iex> Mv.Config.csv_import_max_rows()
|
||||
1000
|
||||
"""
|
||||
@spec csv_import_max_rows() :: pos_integer()
|
||||
def csv_import_max_rows do
|
||||
get_csv_import_config(:max_rows, 1000)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the maximum file size for CSV imports in megabytes.
|
||||
|
||||
Reads the `max_file_size_mb` value from the CSV import configuration.
|
||||
|
||||
## Returns
|
||||
|
||||
- Maximum file size in megabytes (default: 10)
|
||||
|
||||
## Examples
|
||||
|
||||
iex> Mv.Config.csv_import_max_file_size_mb()
|
||||
10
|
||||
"""
|
||||
@spec csv_import_max_file_size_mb() :: pos_integer()
|
||||
def csv_import_max_file_size_mb do
|
||||
get_csv_import_config(:max_file_size_mb, 10)
|
||||
end
|
||||
|
||||
# Helper function to get CSV import config values
|
||||
defp get_csv_import_config(key, default) do
|
||||
Application.get_env(:mv, :csv_import, [])
|
||||
|> Keyword.get(key, default)
|
||||
|> parse_and_validate_integer(default)
|
||||
end
|
||||
|
||||
# Parses and validates integer configuration values.
|
||||
#
|
||||
# Accepts:
|
||||
# - Integer values (passed through)
|
||||
# - String integers (e.g., "1000") - parsed to integer
|
||||
# - Invalid values (e.g., "abc", nil) - falls back to default
|
||||
#
|
||||
# Always clamps the result to a minimum of 1 to ensure positive values.
|
||||
#
|
||||
# Note: We don't log warnings for unparseable values because:
|
||||
# - These functions may be called frequently (e.g., on every request)
|
||||
# - Logging would create excessive log spam
|
||||
# - The fallback to default provides a safe behavior
|
||||
# - Configuration errors should be caught during deployment/testing
|
||||
defp parse_and_validate_integer(value, _default) when is_integer(value) do
|
||||
max(1, value)
|
||||
end
|
||||
|
||||
defp parse_and_validate_integer(value, default) when is_binary(value) do
|
||||
case Integer.parse(value) do
|
||||
{int, _remainder} -> max(1, int)
|
||||
:error -> default
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_and_validate_integer(_value, default) do
|
||||
default
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -63,9 +63,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
chunks: list(list({pos_integer(), map()})),
|
||||
column_map: %{atom() => non_neg_integer()},
|
||||
custom_field_map: %{String.t() => non_neg_integer()},
|
||||
custom_field_lookup: %{
|
||||
String.t() => %{id: String.t(), value_type: atom(), name: String.t()}
|
||||
},
|
||||
custom_field_lookup: %{String.t() => %{id: String.t(), value_type: atom()}},
|
||||
warnings: list(String.t())
|
||||
}
|
||||
|
||||
|
|
@ -81,11 +79,6 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
use Gettext, backend: MvWeb.Gettext
|
||||
|
||||
alias Mv.Helpers.SystemActor
|
||||
|
||||
# Import FieldTypes for human-readable type labels
|
||||
alias MvWeb.Translations.FieldTypes
|
||||
|
||||
# Configuration constants
|
||||
@default_max_errors 50
|
||||
@default_chunk_size 200
|
||||
|
|
@ -109,7 +102,6 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
- `opts` - Optional keyword list:
|
||||
- `:max_rows` - Maximum number of data rows allowed (default: 1000)
|
||||
- `:chunk_size` - Number of rows per chunk (default: 200)
|
||||
- `:actor` - Actor for authorization (default: system actor for systemic operations)
|
||||
|
||||
## Returns
|
||||
|
||||
|
|
@ -128,10 +120,9 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
def prepare(file_content, opts \\ []) do
|
||||
max_rows = Keyword.get(opts, :max_rows, @default_max_rows)
|
||||
chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size)
|
||||
actor = Keyword.get(opts, :actor, SystemActor.get_system_actor())
|
||||
|
||||
with {:ok, headers, rows} <- CsvParser.parse(file_content),
|
||||
{:ok, custom_fields} <- load_custom_fields(actor),
|
||||
{:ok, custom_fields} <- load_custom_fields(),
|
||||
{:ok, maps, warnings} <- build_header_maps(headers, custom_fields),
|
||||
:ok <- validate_row_count(rows, max_rows) do
|
||||
chunks = chunk_rows(rows, maps, chunk_size)
|
||||
|
|
@ -151,10 +142,10 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
end
|
||||
|
||||
# Loads all custom fields from the database
|
||||
defp load_custom_fields(actor) do
|
||||
defp load_custom_fields do
|
||||
custom_fields =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.read!(actor: actor)
|
||||
|> Ash.read!()
|
||||
|
||||
{:ok, custom_fields}
|
||||
rescue
|
||||
|
|
@ -167,7 +158,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
custom_fields
|
||||
|> Enum.reduce(%{}, fn cf, acc ->
|
||||
id_str = to_string(cf.id)
|
||||
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type, name: cf.name})
|
||||
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type})
|
||||
end)
|
||||
end
|
||||
|
||||
|
|
@ -191,10 +182,8 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
normalized != "" && not member_field?(normalized)
|
||||
end)
|
||||
|> Enum.map(fn header ->
|
||||
gettext(
|
||||
"Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing.",
|
||||
header: header
|
||||
)
|
||||
"Unknown column '#{header}' will be ignored. " <>
|
||||
"If this is a custom field, create it in Mila before importing."
|
||||
end)
|
||||
|
||||
{:ok, %{member: member_map, custom: custom_map}, warnings}
|
||||
|
|
@ -235,20 +224,17 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
# Builds a row map from raw row values using column maps
|
||||
defp build_row_map(row_values, maps) do
|
||||
row_tuple = List.to_tuple(row_values)
|
||||
tuple_size = tuple_size(row_tuple)
|
||||
|
||||
member_map =
|
||||
maps.member
|
||||
|> Enum.reduce(%{}, fn {field, index}, acc ->
|
||||
value = if index < tuple_size, do: elem(row_tuple, index), else: ""
|
||||
value = Enum.at(row_values, index, "")
|
||||
Map.put(acc, field, value)
|
||||
end)
|
||||
|
||||
custom_map =
|
||||
maps.custom
|
||||
|> Enum.reduce(%{}, fn {custom_field_id, index}, acc ->
|
||||
value = if index < tuple_size, do: elem(row_tuple, index), else: ""
|
||||
value = Enum.at(row_values, index, "")
|
||||
Map.put(acc, custom_field_id, value)
|
||||
end)
|
||||
|
||||
|
|
@ -313,7 +299,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{})
|
||||
existing_error_count = Keyword.get(opts, :existing_error_count, 0)
|
||||
max_errors = Keyword.get(opts, :max_errors, @default_max_errors)
|
||||
actor = Keyword.get(opts, :actor, SystemActor.get_system_actor())
|
||||
actor = Keyword.fetch!(opts, :actor)
|
||||
|
||||
{inserted, failed, errors, _collected_error_count, truncated?} =
|
||||
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
|
||||
|
|
@ -522,33 +508,8 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
{:ok, %{member: trimmed_member_attrs, custom: custom_attrs}} ->
|
||||
# Prepare custom field values for Ash
|
||||
case prepare_custom_field_values(custom_attrs, custom_field_lookup) do
|
||||
{:error, validation_errors} ->
|
||||
# Custom field validation errors - return first error
|
||||
first_error = List.first(validation_errors)
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: first_error}}
|
||||
custom_field_values = prepare_custom_field_values(custom_attrs, custom_field_lookup)
|
||||
|
||||
{:ok, custom_field_values} ->
|
||||
create_member_with_custom_fields(
|
||||
trimmed_member_attrs,
|
||||
custom_field_values,
|
||||
line_number,
|
||||
actor
|
||||
)
|
||||
end
|
||||
end
|
||||
rescue
|
||||
e ->
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)}}
|
||||
end
|
||||
|
||||
# Creates a member with custom field values, handling errors appropriately
|
||||
defp create_member_with_custom_fields(
|
||||
trimmed_member_attrs,
|
||||
custom_field_values,
|
||||
line_number,
|
||||
actor
|
||||
) do
|
||||
# Create member with custom field values
|
||||
member_attrs_with_cf =
|
||||
trimmed_member_attrs
|
||||
|
|
@ -575,208 +536,76 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
{:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}}
|
||||
end
|
||||
end
|
||||
rescue
|
||||
e ->
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)}}
|
||||
end
|
||||
|
||||
# Prepares custom field values from row map for Ash
|
||||
# Returns {:ok, [custom_field_value_maps]} or {:error, [validation_errors]}
|
||||
defp prepare_custom_field_values(custom_attrs, custom_field_lookup) when is_map(custom_attrs) do
|
||||
{values, errors} =
|
||||
custom_attrs
|
||||
|> Enum.filter(fn {_id, value} -> value != nil && value != "" end)
|
||||
|> Enum.reduce({[], []}, fn {custom_field_id_str, value}, {acc_values, acc_errors} ->
|
||||
process_single_custom_field(
|
||||
custom_field_id_str,
|
||||
value,
|
||||
custom_field_lookup,
|
||||
acc_values,
|
||||
acc_errors
|
||||
)
|
||||
end)
|
||||
|
||||
if Enum.empty?(errors) do
|
||||
{:ok, Enum.reverse(values)}
|
||||
else
|
||||
{:error, Enum.reverse(errors)}
|
||||
end
|
||||
end
|
||||
|
||||
defp prepare_custom_field_values(_, _), do: {:ok, []}
|
||||
|
||||
# Processes a single custom field value and returns updated accumulator
|
||||
defp process_single_custom_field(
|
||||
custom_field_id_str,
|
||||
value,
|
||||
custom_field_lookup,
|
||||
acc_values,
|
||||
acc_errors
|
||||
) do
|
||||
# Trim value early and skip if empty
|
||||
trimmed_value = if is_binary(value), do: String.trim(value), else: value
|
||||
|
||||
# Skip empty values (after trimming) - don't create CFV
|
||||
if trimmed_value == "" or trimmed_value == nil do
|
||||
{acc_values, acc_errors}
|
||||
else
|
||||
process_non_empty_custom_field(
|
||||
custom_field_id_str,
|
||||
trimmed_value,
|
||||
custom_field_lookup,
|
||||
acc_values,
|
||||
acc_errors
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
# Processes a non-empty custom field value
|
||||
defp process_non_empty_custom_field(
|
||||
custom_field_id_str,
|
||||
trimmed_value,
|
||||
custom_field_lookup,
|
||||
acc_values,
|
||||
acc_errors
|
||||
) do
|
||||
|> Enum.map(fn {custom_field_id_str, value} ->
|
||||
case Map.get(custom_field_lookup, custom_field_id_str) do
|
||||
nil ->
|
||||
# Custom field not found, skip
|
||||
{acc_values, acc_errors}
|
||||
nil
|
||||
|
||||
%{id: custom_field_id, value_type: value_type, name: custom_field_name} ->
|
||||
case format_custom_field_value(trimmed_value, value_type, custom_field_name) do
|
||||
{:ok, formatted_value} ->
|
||||
value_map = %{
|
||||
%{id: custom_field_id, value_type: value_type} ->
|
||||
%{
|
||||
"custom_field_id" => to_string(custom_field_id),
|
||||
"value" => formatted_value
|
||||
"value" => format_custom_field_value(value, value_type)
|
||||
}
|
||||
end
|
||||
end)
|
||||
|> Enum.filter(&(&1 != nil))
|
||||
end
|
||||
|
||||
{[value_map | acc_values], acc_errors}
|
||||
|
||||
{:error, reason} ->
|
||||
{acc_values, [reason | acc_errors]}
|
||||
end
|
||||
end
|
||||
end
|
||||
defp prepare_custom_field_values(_, _), do: []
|
||||
|
||||
# Formats a custom field value according to its type
|
||||
# Uses _union_type and _union_value format as expected by Ash
|
||||
# Returns {:ok, formatted_value} or {:error, error_message}
|
||||
defp format_custom_field_value(value, :string, _custom_field_name) when is_binary(value) do
|
||||
{:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}}
|
||||
defp format_custom_field_value(value, :string) when is_binary(value) do
|
||||
%{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :integer, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
case Integer.parse(trimmed) do
|
||||
{int_value, ""} ->
|
||||
# Fully consumed - valid integer
|
||||
{:ok, %{"_union_type" => "integer", "_union_value" => int_value}}
|
||||
|
||||
{_int_value, _remaining} ->
|
||||
# Not fully consumed - invalid
|
||||
{:error, format_custom_field_error(custom_field_name, :integer, trimmed)}
|
||||
|
||||
:error ->
|
||||
{:error, format_custom_field_error(custom_field_name, :integer, trimmed)}
|
||||
defp format_custom_field_value(value, :integer) when is_binary(value) do
|
||||
case Integer.parse(value) do
|
||||
{int_value, _} -> %{"_union_type" => "integer", "_union_value" => int_value}
|
||||
:error -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :boolean, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
defp format_custom_field_value(value, :boolean) when is_binary(value) do
|
||||
bool_value =
|
||||
value
|
||||
|> String.trim()
|
||||
|> String.downcase()
|
||||
|> case do
|
||||
"true" -> true
|
||||
"1" -> true
|
||||
"yes" -> true
|
||||
"ja" -> true
|
||||
_ -> false
|
||||
end
|
||||
|
||||
case parse_boolean_value(trimmed) do
|
||||
{:ok, bool_value} ->
|
||||
{:ok, %{"_union_type" => "boolean", "_union_value" => bool_value}}
|
||||
%{"_union_type" => "boolean", "_union_value" => bool_value}
|
||||
end
|
||||
|
||||
:error ->
|
||||
{:error,
|
||||
format_custom_field_error_with_details(
|
||||
custom_field_name,
|
||||
:boolean,
|
||||
trimmed,
|
||||
gettext("(true/false/1/0/yes/no/ja/nein)")
|
||||
)}
|
||||
defp format_custom_field_value(value, :date) when is_binary(value) do
|
||||
case Date.from_iso8601(String.trim(value)) do
|
||||
{:ok, date} -> %{"_union_type" => "date", "_union_value" => date}
|
||||
{:error, _} -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :date, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
case Date.from_iso8601(trimmed) do
|
||||
{:ok, date} ->
|
||||
{:ok, %{"_union_type" => "date", "_union_value" => date}}
|
||||
|
||||
{:error, _} ->
|
||||
{:error,
|
||||
format_custom_field_error_with_details(
|
||||
custom_field_name,
|
||||
:date,
|
||||
trimmed,
|
||||
gettext("(ISO-8601 format: YYYY-MM-DD)")
|
||||
)}
|
||||
end
|
||||
defp format_custom_field_value(value, :email) when is_binary(value) do
|
||||
%{"_union_type" => "email", "_union_value" => String.trim(value)}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :email, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
# Use EctoCommons.EmailValidator for consistency with Member email validation
|
||||
changeset =
|
||||
{%{}, %{email: :string}}
|
||||
|> Ecto.Changeset.cast(%{email: trimmed}, [:email])
|
||||
|> EctoCommons.EmailValidator.validate_email(:email,
|
||||
checks: Mv.Constants.email_validator_checks()
|
||||
)
|
||||
|
||||
if changeset.valid? do
|
||||
{:ok, %{"_union_type" => "email", "_union_value" => trimmed}}
|
||||
else
|
||||
{:error, format_custom_field_error(custom_field_name, :email, trimmed)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, _type, _custom_field_name) when is_binary(value) do
|
||||
defp format_custom_field_value(value, _type) when is_binary(value) do
|
||||
# Default to string if type is unknown
|
||||
{:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}}
|
||||
end
|
||||
|
||||
# Parses a boolean value from a string, supporting multiple formats
|
||||
defp parse_boolean_value(value) when is_binary(value) do
|
||||
lower = String.downcase(value)
|
||||
parse_boolean_value_lower(lower)
|
||||
end
|
||||
|
||||
# Helper function with pattern matching for boolean values
|
||||
defp parse_boolean_value_lower("true"), do: {:ok, true}
|
||||
defp parse_boolean_value_lower("1"), do: {:ok, true}
|
||||
defp parse_boolean_value_lower("yes"), do: {:ok, true}
|
||||
defp parse_boolean_value_lower("ja"), do: {:ok, true}
|
||||
defp parse_boolean_value_lower("false"), do: {:ok, false}
|
||||
defp parse_boolean_value_lower("0"), do: {:ok, false}
|
||||
defp parse_boolean_value_lower("no"), do: {:ok, false}
|
||||
defp parse_boolean_value_lower("nein"), do: {:ok, false}
|
||||
defp parse_boolean_value_lower(_), do: :error
|
||||
|
||||
# Generates a consistent error message for custom field validation failures
|
||||
# Uses human-readable field type labels (e.g., "Number" instead of "integer")
|
||||
defp format_custom_field_error(custom_field_name, value_type, value) do
|
||||
type_label = FieldTypes.label(value_type)
|
||||
|
||||
gettext("custom_field: %{name} – expected %{type}, got: %{value}",
|
||||
name: custom_field_name,
|
||||
type: type_label,
|
||||
value: value
|
||||
)
|
||||
end
|
||||
|
||||
# Generates an error message with additional details (e.g., format hints)
|
||||
defp format_custom_field_error_with_details(custom_field_name, value_type, value, details) do
|
||||
type_label = FieldTypes.label(value_type)
|
||||
|
||||
gettext("custom_field: %{name} – expected %{type} %{details}, got: %{value}",
|
||||
name: custom_field_name,
|
||||
type: type_label,
|
||||
details: details,
|
||||
value: value
|
||||
)
|
||||
%{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
|
||||
# Trims all string values in member attributes
|
||||
|
|
|
|||
|
|
@ -50,9 +50,9 @@ defmodule MvWeb.CustomFieldLive.IndexComponent do
|
|||
</div>
|
||||
|
||||
<%!-- Hide table when form is visible --%>
|
||||
<div :if={!@show_form} id="custom_fields">
|
||||
<.table
|
||||
id="custom_fields_table"
|
||||
:if={!@show_form}
|
||||
id="custom_fields"
|
||||
rows={@streams.custom_fields}
|
||||
row_click={
|
||||
fn {_id, custom_field} ->
|
||||
|
|
@ -105,14 +105,11 @@ defmodule MvWeb.CustomFieldLive.IndexComponent do
|
|||
</:action>
|
||||
|
||||
<:action :let={{_id, custom_field}}>
|
||||
<.link phx-click={
|
||||
JS.push("prepare_delete", value: %{id: custom_field.id}, target: @myself)
|
||||
}>
|
||||
<.link phx-click={JS.push("prepare_delete", value: %{id: custom_field.id}, target: @myself)}>
|
||||
{gettext("Delete")}
|
||||
</.link>
|
||||
</:action>
|
||||
</.table>
|
||||
</div>
|
||||
|
||||
<%!-- Delete Confirmation Modal --%>
|
||||
<dialog :if={@show_delete_modal} id="delete-custom-field-modal" class="modal modal-open">
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
|
||||
### Limits
|
||||
|
||||
- Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
|
||||
- Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
|
||||
- Maximum file size: 10 MB
|
||||
- Maximum rows: 1,000 rows (excluding header)
|
||||
- Processing: chunks of 200 rows
|
||||
- Errors: capped at 50 per import
|
||||
|
||||
|
|
@ -54,6 +54,8 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
|
||||
|
||||
# CSV Import configuration constants
|
||||
# 10 MB
|
||||
@max_file_size_bytes 10_485_760
|
||||
@max_errors 50
|
||||
|
||||
@impl true
|
||||
|
|
@ -74,15 +76,13 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
|> assign(:import_status, :idle)
|
||||
|> assign(:locale, locale)
|
||||
|> assign(:max_errors, @max_errors)
|
||||
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|
||||
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
|
||||
|> assign_form()
|
||||
# Configure file upload with auto-upload enabled
|
||||
# Files are uploaded automatically when selected, no need for manual trigger
|
||||
|> allow_upload(:csv_file,
|
||||
accept: ~w(.csv),
|
||||
max_entries: 1,
|
||||
max_file_size: Config.csv_import_max_file_size_bytes(),
|
||||
max_file_size: @max_file_size_bytes,
|
||||
auto_upload: true
|
||||
)
|
||||
|
||||
|
|
@ -138,21 +138,16 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
|
||||
<.form_section title={gettext("Import Members (CSV)")}>
|
||||
<div role="note" class="alert alert-info mb-4">
|
||||
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
|
||||
<div>
|
||||
<p class="text-sm mb-2">
|
||||
<p class="font-semibold">
|
||||
{gettext(
|
||||
"Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
|
||||
"Custom fields must be created in Mila before importing CSV files with custom field columns"
|
||||
)}
|
||||
</p>
|
||||
<p class="text-sm">
|
||||
<.link
|
||||
href="#custom_fields"
|
||||
class="link"
|
||||
data-testid="custom-fields-link"
|
||||
>
|
||||
{gettext("Manage Memberdata")}
|
||||
</.link>
|
||||
<p class="text-sm mt-2">
|
||||
{gettext(
|
||||
"Use the custom field name as the CSV column header (same normalization as member fields applies)"
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -205,7 +200,7 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
/>
|
||||
<label class="label" id="csv_file_help">
|
||||
<span class="label-text-alt">
|
||||
{gettext("CSV files only, maximum %{size} MB", size: @csv_import_max_file_size_mb)}
|
||||
{gettext("CSV files only, maximum 10 MB")}
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
|
|
@ -413,11 +408,8 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
|
||||
# Processes CSV upload and starts import
|
||||
defp process_csv_upload(socket) do
|
||||
actor = MvWeb.LiveHelpers.current_actor(socket)
|
||||
|
||||
with {:ok, content} <- consume_and_read_csv(socket),
|
||||
{:ok, import_state} <-
|
||||
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
|
||||
{:ok, import_state} <- MemberCSV.prepare(content) do
|
||||
start_import(socket, import_state)
|
||||
else
|
||||
{:error, reason} when is_binary(reason) ->
|
||||
|
|
|
|||
|
|
@ -1980,6 +1980,16 @@ msgstr " (Datenfeld: %{field})"
|
|||
msgid "CSV File"
|
||||
msgstr "CSV Datei"
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "CSV files only, maximum 10 MB"
|
||||
msgstr "Nur CSV Dateien, maximal 10 MB"
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
|
||||
msgstr "Individuelle Datenfelder müssen zuerst in Mila angelegt werden bevor das Importieren von diesen Feldern mit CSV Dateien mölich ist."
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Download CSV templates:"
|
||||
|
|
@ -2110,6 +2120,11 @@ msgstr "Erfolgreich eingefügt: %{count} Mitglied(er)"
|
|||
msgid "Summary"
|
||||
msgstr "Zusammenfassung"
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
|
||||
msgstr "Verwenden Sie den Namen des benutzerdefinierten Feldes als CSV-Spaltenüberschrift (gleiche Normalisierung wie bei Mitgliedsfeldern)"
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format, fuzzy
|
||||
msgid "Warnings"
|
||||
|
|
@ -2257,58 +2272,3 @@ msgstr "Nicht berechtigt."
|
|||
#, elixir-autogen, elixir-format
|
||||
msgid "Could not load data fields. Please check your permissions."
|
||||
msgstr "Datenfelder konnten nicht geladen werden. Bitte überprüfen Sie Ihre Berechtigungen."
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format, fuzzy
|
||||
msgid "CSV files only, maximum %{size} MB"
|
||||
msgstr "Nur CSV Dateien, maximal %{size} MB"
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(ISO-8601 format: YYYY-MM-DD)"
|
||||
msgstr "(ISO-8601 Format: JJJJ-MM-TT)"
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(true/false/1/0/yes/no/ja/nein)"
|
||||
msgstr "(true/false/1/0/yes/no/ja/nein)"
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "custom_field: %{name} – expected %{type} %{details}, got: %{value}"
|
||||
msgstr "Datenfeld: %{name} – erwartet %{type} %{details}, erhalten: %{value}"
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "custom_field: %{name} – expected %{type}, got: %{value}"
|
||||
msgstr "Datenfeld: %{name} – erwartet %{type}, erhalten: %{value}"
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Manage Memberdata"
|
||||
msgstr "Mitgliederdaten verwalten"
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format, fuzzy
|
||||
msgid "Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
|
||||
msgstr "Verwende die Namen der Datenfelder als Spaltennamen in der CSV Datei. Datenfelder müssen in Mila bereits angelegt sein, damit sie importiert werden können. Sie müssen in der Liste der Mitgliederdaten als Datenfeld enthalten sein (z.B. E-Mail). Spalten mit unbekannten Spaltenüberschriften werden mit einer Warnung ignoriert."
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing."
|
||||
msgstr "Unbekannte Spalte '%{header}' wird ignoriert. Falls dies ein Datenfeld ist, erstellen Sie es in Mila vor dem Import."
|
||||
|
||||
#~ #: lib/mv_web/live/global_settings_live.ex
|
||||
#~ #, elixir-autogen, elixir-format, fuzzy
|
||||
#~ msgid "Custom Fields in CSV Import"
|
||||
#~ msgstr "Benutzerdefinierte Felder"
|
||||
|
||||
#~ #: lib/mv_web/live/global_settings_live.ex
|
||||
#~ #, elixir-autogen, elixir-format, fuzzy
|
||||
#~ msgid "Individual data fields must be created in Mila before importing. Use the field name as the CSV column header. Unknown custom field columns will be ignored with a warning."
|
||||
#~ msgstr "Individuelle Datenfelder müssen in Mila erstellt werden, bevor sie importiert werden können. Verwenden Sie den Namen des Datenfeldes als CSV-Spaltenüberschrift. Unbekannte Spaltenüberschriften werden mit einer Warnung ignoriert."
|
||||
|
||||
#~ #: lib/mv_web/live/global_settings_live.ex
|
||||
#~ #, elixir-autogen, elixir-format
|
||||
#~ msgid "Manage Custom Fields"
|
||||
#~ msgstr "Benutzerdefinierte Felder verwalten"
|
||||
|
|
|
|||
|
|
@ -1981,6 +1981,16 @@ msgstr ""
|
|||
msgid "CSV File"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "CSV files only, maximum 10 MB"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Download CSV templates:"
|
||||
|
|
@ -2111,6 +2121,11 @@ msgstr ""
|
|||
msgid "Summary"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Warnings"
|
||||
|
|
@ -2258,43 +2273,3 @@ msgstr ""
|
|||
#, elixir-autogen, elixir-format
|
||||
msgid "Could not load data fields. Please check your permissions."
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "CSV files only, maximum %{size} MB"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(ISO-8601 format: YYYY-MM-DD)"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(true/false/1/0/yes/no/ja/nein)"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "custom_field: %{name} – expected %{type} %{details}, got: %{value}"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "custom_field: %{name} – expected %{type}, got: %{value}"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Manage Memberdata"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing."
|
||||
msgstr ""
|
||||
|
|
|
|||
|
|
@ -1981,6 +1981,16 @@ msgstr ""
|
|||
msgid "CSV File"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "CSV files only, maximum 10 MB"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Download CSV templates:"
|
||||
|
|
@ -2111,6 +2121,11 @@ msgstr ""
|
|||
msgid "Summary"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format, fuzzy
|
||||
msgid "Warnings"
|
||||
|
|
@ -2258,58 +2273,3 @@ msgstr ""
|
|||
#, elixir-autogen, elixir-format
|
||||
msgid "Could not load data fields. Please check your permissions."
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format, fuzzy
|
||||
msgid "CSV files only, maximum %{size} MB"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(ISO-8601 format: YYYY-MM-DD)"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(true/false/1/0/yes/no/ja/nein)"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "custom_field: %{name} – expected %{type} %{details}, got: %{value}"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "custom_field: %{name} – expected %{type}, got: %{value}"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Manage Memberdata"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format, fuzzy
|
||||
msgid "Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing."
|
||||
msgstr "Unknown column '%{header}' will be ignored. If this is a custom field, create it in Mila before importing."
|
||||
|
||||
#~ #: lib/mv_web/live/global_settings_live.ex
|
||||
#~ #, elixir-autogen, elixir-format, fuzzy
|
||||
#~ msgid "Custom Fields in CSV Import"
|
||||
#~ msgstr ""
|
||||
|
||||
#~ #: lib/mv_web/live/global_settings_live.ex
|
||||
#~ #, elixir-autogen, elixir-format, fuzzy
|
||||
#~ msgid "Individual data fields must be created in Mila before importing. Use the field name as the CSV column header. Unknown custom field columns will be ignored with a warning."
|
||||
#~ msgstr ""
|
||||
|
||||
#~ #: lib/mv_web/live/global_settings_live.ex
|
||||
#~ #, elixir-autogen, elixir-format
|
||||
#~ msgid "Manage Custom Fields"
|
||||
#~ msgstr ""
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
defmodule Mv.Membership.Import.MemberCSVTest do
|
||||
use Mv.DataCase, async: true
|
||||
use Mv.DataCase, async: false
|
||||
|
||||
alias Mv.Membership.Import.MemberCSV
|
||||
|
||||
|
|
@ -35,10 +35,11 @@ defmodule Mv.Membership.Import.MemberCSVTest do
|
|||
end
|
||||
|
||||
describe "prepare/2" do
|
||||
test "accepts file_content and opts and returns tagged tuple" do
|
||||
test "function exists and accepts file_content and opts" do
|
||||
file_content = "email\njohn@example.com"
|
||||
opts = []
|
||||
|
||||
# This will fail until the function is implemented
|
||||
result = MemberCSV.prepare(file_content, opts)
|
||||
assert match?({:ok, _}, result) or match?({:error, _}, result)
|
||||
end
|
||||
|
|
@ -64,6 +65,11 @@ defmodule Mv.Membership.Import.MemberCSVTest do
|
|||
|
||||
assert {:error, _reason} = MemberCSV.prepare(file_content, opts)
|
||||
end
|
||||
|
||||
test "function has documentation" do
|
||||
# Check that @doc exists by reading the module
|
||||
assert function_exported?(MemberCSV, :prepare, 2)
|
||||
end
|
||||
end
|
||||
|
||||
describe "process_chunk/4" do
|
||||
|
|
@ -72,7 +78,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
|
|||
%{actor: system_actor}
|
||||
end
|
||||
|
||||
test "accepts chunk_rows_with_lines, column_map, custom_field_map, and opts and returns tagged tuple",
|
||||
test "function exists and accepts chunk_rows_with_lines, column_map, custom_field_map, and opts",
|
||||
%{
|
||||
actor: actor
|
||||
} do
|
||||
|
|
@ -81,6 +87,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
|
|||
custom_field_map = %{}
|
||||
opts = [actor: actor]
|
||||
|
||||
# This will fail until the function is implemented
|
||||
result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
|
||||
assert match?({:ok, _}, result) or match?({:error, _}, result)
|
||||
end
|
||||
|
|
@ -224,11 +231,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
|
|||
custom_field_map = %{to_string(custom_field.id) => 1}
|
||||
|
||||
custom_field_lookup = %{
|
||||
to_string(custom_field.id) => %{
|
||||
id: custom_field.id,
|
||||
value_type: custom_field.value_type,
|
||||
name: custom_field.name
|
||||
}
|
||||
to_string(custom_field.id) => %{id: custom_field.id, value_type: custom_field.value_type}
|
||||
}
|
||||
|
||||
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
|
||||
|
|
@ -329,6 +332,11 @@ defmodule Mv.Membership.Import.MemberCSVTest do
|
|||
assert chunk_result.errors == []
|
||||
end
|
||||
|
||||
test "function has documentation" do
|
||||
# Check that @doc exists by reading the module
|
||||
assert function_exported?(MemberCSV, :process_chunk, 4)
|
||||
end
|
||||
|
||||
test "error capping collects exactly 50 errors", %{actor: actor} do
|
||||
# Create 50 rows with invalid emails
|
||||
chunk_rows_with_lines =
|
||||
|
|
@ -603,300 +611,15 @@ defmodule Mv.Membership.Import.MemberCSVTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "custom field import" do
|
||||
setup do
|
||||
system_actor = Mv.Helpers.SystemActor.get_system_actor()
|
||||
%{actor: system_actor}
|
||||
end
|
||||
describe "module documentation" do
|
||||
test "module has @moduledoc" do
|
||||
# Check that the module exists and has documentation
|
||||
assert Code.ensure_loaded?(MemberCSV)
|
||||
|
||||
test "creates member with valid integer custom field value", %{actor: actor} do
|
||||
# Create integer custom field
|
||||
{:ok, custom_field} =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.Changeset.for_create(:create, %{
|
||||
name: "Alter",
|
||||
value_type: :integer
|
||||
})
|
||||
|> Ash.create(actor: actor)
|
||||
|
||||
chunk_rows_with_lines = [
|
||||
{2,
|
||||
%{
|
||||
member: %{email: "withage@example.com"},
|
||||
custom: %{to_string(custom_field.id) => "25"}
|
||||
}}
|
||||
]
|
||||
|
||||
column_map = %{email: 0}
|
||||
custom_field_map = %{to_string(custom_field.id) => 1}
|
||||
|
||||
custom_field_lookup = %{
|
||||
to_string(custom_field.id) => %{
|
||||
id: custom_field.id,
|
||||
value_type: custom_field.value_type,
|
||||
name: custom_field.name
|
||||
}
|
||||
}
|
||||
|
||||
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
|
||||
|
||||
assert {:ok, chunk_result} =
|
||||
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
|
||||
|
||||
assert chunk_result.inserted == 1
|
||||
assert chunk_result.failed == 0
|
||||
|
||||
# Verify member and custom field value were created
|
||||
members = Mv.Membership.list_members!(actor: actor)
|
||||
member = Enum.find(members, &(&1.email == "withage@example.com"))
|
||||
assert member != nil
|
||||
|
||||
{:ok, member_with_cf} = Ash.load(member, :custom_field_values, actor: actor)
|
||||
assert length(member_with_cf.custom_field_values) == 1
|
||||
cfv = List.first(member_with_cf.custom_field_values)
|
||||
assert cfv.custom_field_id == custom_field.id
|
||||
assert cfv.value.value == 25
|
||||
assert cfv.value.type == :integer
|
||||
end
|
||||
|
||||
test "returns error for invalid integer custom field value", %{actor: actor} do
|
||||
# Create integer custom field
|
||||
{:ok, custom_field} =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.Changeset.for_create(:create, %{
|
||||
name: "Alter",
|
||||
value_type: :integer
|
||||
})
|
||||
|> Ash.create(actor: actor)
|
||||
|
||||
chunk_rows_with_lines = [
|
||||
{2,
|
||||
%{
|
||||
member: %{email: "invalidage@example.com"},
|
||||
custom: %{to_string(custom_field.id) => "abc"}
|
||||
}}
|
||||
]
|
||||
|
||||
column_map = %{email: 0}
|
||||
custom_field_map = %{to_string(custom_field.id) => 1}
|
||||
|
||||
custom_field_lookup = %{
|
||||
to_string(custom_field.id) => %{
|
||||
id: custom_field.id,
|
||||
value_type: custom_field.value_type,
|
||||
name: custom_field.name
|
||||
}
|
||||
}
|
||||
|
||||
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
|
||||
|
||||
assert {:ok, chunk_result} =
|
||||
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
|
||||
|
||||
assert chunk_result.inserted == 0
|
||||
assert chunk_result.failed == 1
|
||||
assert length(chunk_result.errors) == 1
|
||||
|
||||
error = List.first(chunk_result.errors)
|
||||
assert error.csv_line_number == 2
|
||||
assert error.message =~ "custom_field: Alter"
|
||||
assert error.message =~ "Number"
|
||||
assert error.message =~ "abc"
|
||||
end
|
||||
|
||||
test "returns error for invalid date custom field value", %{actor: actor} do
|
||||
# Create date custom field
|
||||
{:ok, custom_field} =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.Changeset.for_create(:create, %{
|
||||
name: "Geburtstag",
|
||||
value_type: :date
|
||||
})
|
||||
|> Ash.create(actor: actor)
|
||||
|
||||
chunk_rows_with_lines = [
|
||||
{3,
|
||||
%{
|
||||
member: %{email: "invaliddate@example.com"},
|
||||
custom: %{to_string(custom_field.id) => "not-a-date"}
|
||||
}}
|
||||
]
|
||||
|
||||
column_map = %{email: 0}
|
||||
custom_field_map = %{to_string(custom_field.id) => 1}
|
||||
|
||||
custom_field_lookup = %{
|
||||
to_string(custom_field.id) => %{
|
||||
id: custom_field.id,
|
||||
value_type: custom_field.value_type,
|
||||
name: custom_field.name
|
||||
}
|
||||
}
|
||||
|
||||
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
|
||||
|
||||
assert {:ok, chunk_result} =
|
||||
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
|
||||
|
||||
assert chunk_result.inserted == 0
|
||||
assert chunk_result.failed == 1
|
||||
assert length(chunk_result.errors) == 1
|
||||
|
||||
error = List.first(chunk_result.errors)
|
||||
assert error.csv_line_number == 3
|
||||
assert error.message =~ "custom_field: Geburtstag"
|
||||
assert error.message =~ "Date"
|
||||
end
|
||||
|
||||
test "returns error for invalid email custom field value", %{actor: actor} do
|
||||
# Create email custom field
|
||||
{:ok, custom_field} =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.Changeset.for_create(:create, %{
|
||||
name: "Work Email",
|
||||
value_type: :email
|
||||
})
|
||||
|> Ash.create(actor: actor)
|
||||
|
||||
chunk_rows_with_lines = [
|
||||
{4,
|
||||
%{
|
||||
member: %{email: "invalidemailcf@example.com"},
|
||||
custom: %{to_string(custom_field.id) => "not-an-email"}
|
||||
}}
|
||||
]
|
||||
|
||||
column_map = %{email: 0}
|
||||
custom_field_map = %{to_string(custom_field.id) => 1}
|
||||
|
||||
custom_field_lookup = %{
|
||||
to_string(custom_field.id) => %{
|
||||
id: custom_field.id,
|
||||
value_type: custom_field.value_type,
|
||||
name: custom_field.name
|
||||
}
|
||||
}
|
||||
|
||||
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
|
||||
|
||||
assert {:ok, chunk_result} =
|
||||
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
|
||||
|
||||
assert chunk_result.inserted == 0
|
||||
assert chunk_result.failed == 1
|
||||
assert length(chunk_result.errors) == 1
|
||||
|
||||
error = List.first(chunk_result.errors)
|
||||
assert error.csv_line_number == 4
|
||||
assert error.message =~ "custom_field: Work Email"
|
||||
assert error.message =~ "E-Mail"
|
||||
end
|
||||
|
||||
test "returns error for invalid boolean custom field value", %{actor: actor} do
|
||||
# Create boolean custom field
|
||||
{:ok, custom_field} =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.Changeset.for_create(:create, %{
|
||||
name: "Is Active",
|
||||
value_type: :boolean
|
||||
})
|
||||
|> Ash.create(actor: actor)
|
||||
|
||||
chunk_rows_with_lines = [
|
||||
{5,
|
||||
%{
|
||||
member: %{email: "invalidbool@example.com"},
|
||||
custom: %{to_string(custom_field.id) => "maybe"}
|
||||
}}
|
||||
]
|
||||
|
||||
column_map = %{email: 0}
|
||||
custom_field_map = %{to_string(custom_field.id) => 1}
|
||||
|
||||
custom_field_lookup = %{
|
||||
to_string(custom_field.id) => %{
|
||||
id: custom_field.id,
|
||||
value_type: custom_field.value_type,
|
||||
name: custom_field.name
|
||||
}
|
||||
}
|
||||
|
||||
opts = [custom_field_lookup: custom_field_lookup, actor: actor]
|
||||
|
||||
assert {:ok, chunk_result} =
|
||||
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
|
||||
|
||||
assert chunk_result.inserted == 0
|
||||
assert chunk_result.failed == 1
|
||||
assert length(chunk_result.errors) == 1
|
||||
|
||||
error = List.first(chunk_result.errors)
|
||||
assert error.csv_line_number == 5
|
||||
assert error.message =~ "custom_field: Is Active"
|
||||
# Error message should indicate boolean/Yes-No validation failure
|
||||
assert String.contains?(error.message, "Yes/No") ||
|
||||
String.contains?(error.message, "true/false") ||
|
||||
String.contains?(error.message, "boolean")
|
||||
end
|
||||
end
|
||||
|
||||
describe "prepare/2 with custom fields" do
|
||||
setup do
|
||||
system_actor = Mv.Helpers.SystemActor.get_system_actor()
|
||||
|
||||
# Create a custom field
|
||||
{:ok, custom_field} =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.Changeset.for_create(:create, %{
|
||||
name: "Membership Number",
|
||||
value_type: :string
|
||||
})
|
||||
|> Ash.create(actor: system_actor)
|
||||
|
||||
%{actor: system_actor, custom_field: custom_field}
|
||||
end
|
||||
|
||||
test "includes custom field in custom_field_map when header matches", %{
|
||||
custom_field: custom_field
|
||||
} do
|
||||
# CSV with custom field column
|
||||
csv_content = "email;Membership Number\njohn@example.com;12345"
|
||||
|
||||
assert {:ok, import_state} = MemberCSV.prepare(csv_content)
|
||||
|
||||
# Check that custom field is mapped
|
||||
assert Map.has_key?(import_state.custom_field_map, to_string(custom_field.id))
|
||||
assert import_state.column_map[:email] == 0
|
||||
end
|
||||
|
||||
test "includes warning for unknown custom field column", %{custom_field: _custom_field} do
|
||||
# CSV with unknown custom field column (not matching any existing custom field)
|
||||
csv_content = "email;NichtExistierend\njohn@example.com;value"
|
||||
|
||||
assert {:ok, import_state} = MemberCSV.prepare(csv_content)
|
||||
|
||||
# Check that warning is present
|
||||
assert import_state.warnings != []
|
||||
warning = List.first(import_state.warnings)
|
||||
assert warning =~ "NichtExistierend"
|
||||
assert warning =~ "ignored"
|
||||
assert warning =~ "custom field"
|
||||
|
||||
# Check that unknown column is not in custom_field_map
|
||||
assert import_state.custom_field_map == %{}
|
||||
# Member import should still succeed
|
||||
assert import_state.column_map[:email] == 0
|
||||
end
|
||||
|
||||
test "import succeeds even with unknown custom field columns", %{custom_field: _custom_field} do
|
||||
# CSV with unknown custom field column
|
||||
csv_content = "email;UnknownField\njohn@example.com;value"
|
||||
|
||||
assert {:ok, import_state} = MemberCSV.prepare(csv_content)
|
||||
|
||||
# Import state should be valid
|
||||
assert import_state.column_map[:email] == 0
|
||||
assert import_state.chunks != []
|
||||
# Try to get the module documentation
|
||||
{:docs_v1, _, _, _, %{"en" => moduledoc}, _, _} = Code.fetch_docs(MemberCSV)
|
||||
assert is_binary(moduledoc)
|
||||
assert String.length(moduledoc) > 0
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
defmodule MvWeb.GlobalSettingsLiveConfigTest do
|
||||
@moduledoc """
|
||||
Tests for GlobalSettingsLive that modify global Application configuration.
|
||||
|
||||
These tests run with `async: false` to prevent race conditions when
|
||||
modifying global Application environment variables (Application.put_env).
|
||||
This follows the same pattern as Mv.ConfigTest.
|
||||
"""
|
||||
use MvWeb.ConnCase, async: false
|
||||
import Phoenix.LiveViewTest
|
||||
|
||||
# Helper function to upload CSV file in tests
|
||||
defp upload_csv_file(view, csv_content, filename) do
|
||||
view
|
||||
|> file_input("#csv-upload-form", :csv_file, [
|
||||
%{
|
||||
last_modified: System.system_time(:second),
|
||||
name: filename,
|
||||
content: csv_content,
|
||||
size: byte_size(csv_content),
|
||||
type: "text/csv"
|
||||
}
|
||||
])
|
||||
|> render_upload(filename)
|
||||
end
|
||||
|
||||
describe "CSV Import - Configuration Tests" do
|
||||
setup %{conn: conn} do
|
||||
# Ensure admin user
|
||||
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
|
||||
conn = MvWeb.ConnCase.conn_with_password_user(conn, admin_user)
|
||||
|
||||
{:ok, conn: conn, admin_user: admin_user}
|
||||
end
|
||||
|
||||
test "configured row limit is enforced", %{conn: conn} do
|
||||
# Business rule: CSV import respects configured row limits
|
||||
# Test that a custom limit (500) is enforced, not just the default (1000)
|
||||
original_config = Application.get_env(:mv, :csv_import, [])
|
||||
|
||||
try do
|
||||
Application.put_env(:mv, :csv_import, max_rows: 500)
|
||||
|
||||
{:ok, view, _html} = live(conn, ~p"/settings")
|
||||
|
||||
# Generate CSV with 501 rows (exceeding custom limit of 500)
|
||||
header = "first_name;last_name;email;street;postal_code;city\n"
|
||||
|
||||
rows =
|
||||
for i <- 1..501 do
|
||||
"Row#{i};Last#{i};email#{i}@example.com;Street#{i};12345;City#{i}\n"
|
||||
end
|
||||
|
||||
large_csv = header <> Enum.join(rows)
|
||||
|
||||
# Simulate file upload using helper function
|
||||
upload_csv_file(view, large_csv, "too_many_rows_custom.csv")
|
||||
|
||||
view
|
||||
|> form("#csv-upload-form", %{})
|
||||
|> render_submit()
|
||||
|
||||
html = render(view)
|
||||
# Business rule: import should be rejected when exceeding configured limit
|
||||
assert html =~ "exceeds" or html =~ "maximum" or html =~ "limit" or
|
||||
html =~ "Failed to prepare"
|
||||
after
|
||||
# Restore original config
|
||||
Application.put_env(:mv, :csv_import, original_config)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -110,7 +110,7 @@ defmodule MvWeb.GlobalSettingsLiveTest do
|
|||
{:ok, _view, html} = live(conn, ~p"/settings")
|
||||
|
||||
# Check for custom fields notice text
|
||||
assert html =~ "Use the data field name"
|
||||
assert html =~ "Custom fields" or html =~ "custom field"
|
||||
end
|
||||
|
||||
test "admin user sees template download links", %{conn: conn} do
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue