local-it/mitgliederverwaltung
2
3
Fork 0
Code Issues 67 Pull requests 6 Projects 3 Releases Packages 1 Activity Actions

Merge pull request 'ImplementsCSV Import UI closes #335' (#359) from feature/335_csv_import_ui into main
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse source 
Reviewed-on: #359
This commit is contained in:
moritz 2026-01-25 18:45:07 +01:00
commit d1f70e2877
13 changed files with 1850 additions and 373 deletions
Download patch file Download diff file Expand all files Collapse all files

57
lib/mv/membership/import/header_mapper.ex
View file

@ -97,31 +97,48 @@ defmodule Mv.Membership.Import.HeaderMapper do
} }
# Build reverse map: normalized_variant -> canonical_field # Build reverse map: normalized_variant -> canonical_field
# Cached on first access for performance # Computed on each access - the map is small enough that recomputing is fast
# This avoids Module.get_attribute issues while maintaining simplicity
defp normalized_to_canonical do defp normalized_to_canonical do
cached = Process.get({__MODULE__, :normalized_to_canonical})
if cached do
cached
else
map = build_normalized_to_canonical_map()
Process.put({__MODULE__, :normalized_to_canonical}, map)
map
end
end
# Builds the normalized variant -> canonical field map
defp build_normalized_to_canonical_map do
@member_field_variants_raw @member_field_variants_raw
|> Enum.flat_map(&map_variants_to_normalized/1) |> Enum.flat_map(fn {canonical, variants} ->
|> Map.new()
end
# Maps a canonical field and its variants to normalized tuples
defp map_variants_to_normalized({canonical, variants}) do
Enum.map(variants, fn variant -> Enum.map(variants, fn variant ->
{normalize_header(variant), canonical} {normalize_header(variant), canonical}
end) end)
end)
|> Map.new()
end
@doc """
Returns a MapSet of normalized member field names.
This is the single source of truth for known member fields.
Used to distinguish between member fields and custom fields.
## Returns
- `MapSet.t(String.t())` - Set of normalized member field names
## Examples
iex> HeaderMapper.known_member_fields()
#MapSet<["email", "firstname", "lastname", "street", "postalcode", "city"]>
"""
# Known member fields computed at compile-time for performance and determinism
@known_member_fields @member_field_variants_raw
|> Map.keys()
|> Enum.map(fn canonical ->
# Normalize the canonical field name (e.g., :first_name -> "firstname")
canonical
|> Atom.to_string()
|> String.replace("_", "")
|> String.downcase()
end)
|> MapSet.new()
@spec known_member_fields() :: MapSet.t(String.t())
def known_member_fields do
@known_member_fields
end end
@doc """ @doc """

76
lib/mv/membership/import/member_csv.ex
View file

@ -79,6 +79,11 @@ defmodule Mv.Membership.Import.MemberCSV do
use Gettext, backend: MvWeb.Gettext use Gettext, backend: MvWeb.Gettext
# Configuration constants
@default_max_errors 50
@default_chunk_size 200
@default_max_rows 1000
@doc """ @doc """
Prepares CSV content for import by parsing, mapping headers, and validating limits. Prepares CSV content for import by parsing, mapping headers, and validating limits.
@ -113,8 +118,8 @@ defmodule Mv.Membership.Import.MemberCSV do
""" """
@spec prepare(String.t(), keyword()) :: {:ok, import_state()} | {:error, String.t()} @spec prepare(String.t(), keyword()) :: {:ok, import_state()} | {:error, String.t()}
def prepare(file_content, opts \\ []) do def prepare(file_content, opts \\ []) do
max_rows = Keyword.get(opts, :max_rows, 1000) max_rows = Keyword.get(opts, :max_rows, @default_max_rows)
chunk_size = Keyword.get(opts, :chunk_size, 200) chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size)
with {:ok, headers, rows} <- CsvParser.parse(file_content), with {:ok, headers, rows} <- CsvParser.parse(file_content),
{:ok, custom_fields} <- load_custom_fields(), {:ok, custom_fields} <- load_custom_fields(),
@ -189,18 +194,12 @@ defmodule Mv.Membership.Import.MemberCSV do
end end
# Checks if a normalized header matches a member field # Checks if a normalized header matches a member field
# Uses HeaderMapper's internal logic to check if header would map to a member field # Uses HeaderMapper.known_member_fields/0 as single source of truth
defp member_field?(normalized) do defp member_field?(normalized) when is_binary(normalized) do
# Try to build maps with just this header - if it maps to a member field, it's a member field MapSet.member?(HeaderMapper.known_member_fields(), normalized)
case HeaderMapper.build_maps([normalized], []) do end
{:ok, %{member: member_map}} ->
# If member_map is not empty, it's a member field
map_size(member_map) > 0
_ -> defp member_field?(_), do: false
false
end
end
# Validates that row count doesn't exceed limit # Validates that row count doesn't exceed limit
defp validate_row_count(rows, max_rows) do defp validate_row_count(rows, max_rows) do
@ -299,18 +298,29 @@ defmodule Mv.Membership.Import.MemberCSV do
def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, opts \\ []) do def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, opts \\ []) do
custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{}) custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{})
existing_error_count = Keyword.get(opts, :existing_error_count, 0) existing_error_count = Keyword.get(opts, :existing_error_count, 0)
max_errors = Keyword.get(opts, :max_errors, 50) max_errors = Keyword.get(opts, :max_errors, @default_max_errors)
actor = Keyword.fetch!(opts, :actor)
{inserted, failed, errors, _collected_error_count, truncated?} = {inserted, failed, errors, _collected_error_count, truncated?} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map}, acc -> Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
current_error_count = existing_error_count + elem(acc, 3) {acc_inserted, acc_failed,
acc_errors, acc_error_count,
acc_truncated?} ->
current_error_count = existing_error_count + acc_error_count
case process_row(row_map, line_number, custom_field_lookup) do case process_row(row_map, line_number, custom_field_lookup, actor) do
{:ok, _member} -> {:ok, _member} ->
update_inserted(acc) update_inserted(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?}
)
{:error, error} -> {:error, error} ->
handle_row_error(acc, error, current_error_count, max_errors) handle_row_error(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?},
error,
current_error_count,
max_errors
)
end end
end) end)
@ -487,7 +497,8 @@ defmodule Mv.Membership.Import.MemberCSV do
defp process_row( defp process_row(
row_map, row_map,
line_number, line_number,
custom_field_lookup custom_field_lookup,
actor
) do ) do
# Validate row before database insertion # Validate row before database insertion
case validate_row(row_map, line_number, []) do case validate_row(row_map, line_number, []) do
@ -512,15 +523,14 @@ defmodule Mv.Membership.Import.MemberCSV do
member_attrs_with_cf member_attrs_with_cf
end end
# Use system_actor for CSV imports (systemic operation) case Mv.Membership.create_member(final_attrs, actor: actor) do
system_actor = Mv.Helpers.SystemActor.get_system_actor()
case Mv.Membership.create_member(final_attrs, actor: system_actor) do
{:ok, member} -> {:ok, member} ->
{:ok, member} {:ok, member}
{:error, %Ash.Error.Invalid{} = error} -> {:error, %Ash.Error.Invalid{} = error} ->
{:error, format_ash_error(error, line_number)} # Extract email from final_attrs for better error messages
email = Map.get(final_attrs, :email) || Map.get(trimmed_member_attrs, :email)
{:error, format_ash_error(error, line_number, email)}
{:error, error} -> {:error, error} ->
{:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}} {:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}}
@ -613,7 +623,7 @@ defmodule Mv.Membership.Import.MemberCSV do
end end
# Formats Ash errors into MemberCSV.Error structs # Formats Ash errors into MemberCSV.Error structs
defp format_ash_error(%Ash.Error.Invalid{errors: errors}, line_number) do defp format_ash_error(%Ash.Error.Invalid{errors: errors}, line_number, email) do
# Try to find email-related errors first (for better error messages) # Try to find email-related errors first (for better error messages)
email_error = email_error =
Enum.find(errors, fn error -> Enum.find(errors, fn error ->
@ -628,35 +638,37 @@ defmodule Mv.Membership.Import.MemberCSV do
%Error{ %Error{
csv_line_number: line_number, csv_line_number: line_number,
field: field, field: field,
message: format_error_message(message, field) message: format_error_message(message, field, email)
} }
%{message: message} -> %{message: message} ->
%Error{ %Error{
csv_line_number: line_number, csv_line_number: line_number,
field: nil, field: nil,
message: format_error_message(message, nil) message: format_error_message(message, nil, email)
} }
_ -> _ ->
%Error{ %Error{
csv_line_number: line_number, csv_line_number: line_number,
field: nil, field: nil,
message: "Validation failed" message: gettext("Validation failed")
} }
end end
end end
# Formats error messages, handling common cases like email uniqueness # Formats error messages, handling common cases like email uniqueness
defp format_error_message(message, field) when is_binary(message) do defp format_error_message(message, field, email) when is_binary(message) do
if email_uniqueness_error?(message, field) do if email_uniqueness_error?(message, field) do
"email has already been taken" # Include email in error message for better user feedback
email_str = if email, do: to_string(email), else: gettext("email")
gettext("email %{email} has already been taken", email: email_str)
else else
message message
end end
end end
defp format_error_message(message, _field), do: to_string(message) defp format_error_message(message, _field, _email), do: to_string(message)
# Checks if error message indicates email uniqueness constraint violation # Checks if error message indicates email uniqueness constraint violation
defp email_uniqueness_error?(message, :email) do defp email_uniqueness_error?(message, :email) do

566
lib/mv_web/live/global_settings_live.ex
View file

@ -7,6 +7,7 @@ defmodule MvWeb.GlobalSettingsLive do
- Manage custom fields - Manage custom fields
- Real-time form validation - Real-time form validation
- Success/error feedback - Success/error feedback
- CSV member import (admin only)
## Settings ## Settings
- `club_name` - The name of the association/club (required) - `club_name` - The name of the association/club (required)
@ -14,6 +15,29 @@ defmodule MvWeb.GlobalSettingsLive do
## Events ## Events
- `validate` - Real-time form validation - `validate` - Real-time form validation
- `save` - Save settings changes - `save` - Save settings changes
- `start_import` - Start CSV member import (admin only)
## CSV Import
The CSV import feature allows administrators to upload CSV files and import members.
### File Upload
Files are uploaded automatically when selected (`auto_upload: true`). No manual
upload trigger is required.
### Rate Limiting
Currently, there is no rate limiting for CSV imports. Administrators can start
multiple imports in quick succession. This is intentional for bulk data migration
scenarios, but should be monitored in production.
### Limits
- Maximum file size: 10 MB
- Maximum rows: 1,000 rows (excluding header)
- Processing: chunks of 200 rows
- Errors: capped at 50 per import
## Note ## Note
Settings is a singleton resource - there is only one settings record. Settings is a singleton resource - there is only one settings record.
@ -21,18 +45,48 @@ defmodule MvWeb.GlobalSettingsLive do
""" """
use MvWeb, :live_view use MvWeb, :live_view
alias Mv.Authorization.Actor
alias Mv.Config
alias Mv.Membership alias Mv.Membership
alias Mv.Membership.Import.MemberCSV
alias MvWeb.Authorization
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
# CSV Import configuration constants
# 10 MB
@max_file_size_bytes 10_485_760
@max_errors 50
@impl true @impl true
def mount(_params, _session, socket) do def mount(_params, session, socket) do
{:ok, settings} = Membership.get_settings() {:ok, settings} = Membership.get_settings()
{:ok, # Get locale from session for translations
locale = session["locale"] || "de"
Gettext.put_locale(MvWeb.Gettext, locale)
socket =
socket socket
|> assign(:page_title, gettext("Settings")) |> assign(:page_title, gettext("Settings"))
|> assign(:settings, settings) |> assign(:settings, settings)
|> assign(:active_editing_section, nil) |> assign(:active_editing_section, nil)
|> assign_form()} |> assign(:import_state, nil)
|> assign(:import_progress, nil)
|> assign(:import_status, :idle)
|> assign(:locale, locale)
|> assign(:max_errors, @max_errors)
|> assign_form()
# Configure file upload with auto-upload enabled
# Files are uploaded automatically when selected, no need for manual trigger
|> allow_upload(:csv_file,
accept: ~w(.csv),
max_entries: 1,
max_file_size: @max_file_size_bytes,
auto_upload: true
)
{:ok, socket}
end end
@impl true @impl true
@ -78,6 +132,206 @@ defmodule MvWeb.GlobalSettingsLive do
id="custom-fields-component" id="custom-fields-component"
/> />
</.form_section> </.form_section>
<%!-- CSV Import Section (Admin only) --%>
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
<.form_section title={gettext("Import Members (CSV)")}>
<div role="note" class="alert alert-info mb-4">
<div>
<p class="font-semibold">
{gettext(
"Custom fields must be created in Mila before importing CSV files with custom field columns"
)}
</p>
<p class="text-sm mt-2">
{gettext(
"Use the custom field name as the CSV column header (same normalization as member fields applies)"
)}
</p>
</div>
</div>
<div class="mb-4">
<p class="text-sm text-base-content/70 mb-2">
{gettext("Download CSV templates:")}
</p>
<ul class="list-disc list-inside space-y-1">
<li>
<.link
href={~p"/templates/member_import_en.csv"}
download="member_import_en.csv"
class="link link-primary"
>
{gettext("English Template")}
</.link>
</li>
<li>
<.link
href={~p"/templates/member_import_de.csv"}
download="member_import_de.csv"
class="link link-primary"
>
{gettext("German Template")}
</.link>
</li>
</ul>
</div>
<.form
id="csv-upload-form"
for={%{}}
multipart={true}
phx-change="validate_csv_upload"
phx-submit="start_import"
data-testid="csv-upload-form"
>
<div class="form-control">
<label for="csv_file" class="label">
<span class="label-text">
{gettext("CSV File")}
</span>
</label>
<.live_file_input
upload={@uploads.csv_file}
id="csv_file"
class="file-input file-input-bordered w-full"
aria-describedby="csv_file_help"
/>
<label class="label" id="csv_file_help">
<span class="label-text-alt">
{gettext("CSV files only, maximum 10 MB")}
</span>
</label>
</div>
<.button
type="submit"
phx-disable-with={gettext("Starting import...")}
variant="primary"
disabled={
@import_status == :running or
Enum.empty?(@uploads.csv_file.entries) or
@uploads.csv_file.entries |> List.first() |> then(&(&1 && not &1.done?))
}
data-testid="start-import-button"
>
{gettext("Start Import")}
</.button>
</.form>
<%= if @import_status == :running or @import_status == :done do %>
<%= if @import_progress do %>
<div
role="status"
aria-live="polite"
class="mt-4"
data-testid="import-progress-container"
>
<%= if @import_progress.status == :running do %>
<p class="text-sm" data-testid="import-progress-text">
{gettext("Processing chunk %{current} of %{total}...",
current: @import_progress.current_chunk,
total: @import_progress.total_chunks
)}
</p>
<% end %>
<%= if @import_progress.status == :done do %>
<section class="space-y-4" data-testid="import-results-panel">
<h2 class="text-lg font-semibold">
{gettext("Import Results")}
</h2>
<div class="space-y-4">
<div>
<h3 class="text-sm font-semibold mb-2">
{gettext("Summary")}
</h3>
<div class="text-sm space-y-2">
<p>
<.icon
name="hero-check-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Successfully inserted: %{count} member(s)",
count: @import_progress.inserted
)}
</p>
<%= if @import_progress.failed > 0 do %>
<p>
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Failed: %{count} row(s)", count: @import_progress.failed)}
</p>
<% end %>
<%= if @import_progress.errors_truncated? do %>
<p>
<.icon
name="hero-information-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Error list truncated to %{count} entries",
count: @max_errors
)}
</p>
<% end %>
</div>
</div>
<%= if length(@import_progress.errors) > 0 do %>
<div data-testid="import-error-list">
<h3 class="text-sm font-semibold mb-2">
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Errors")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for error <- @import_progress.errors do %>
<li>
{gettext("Line %{line}: %{message}",
line: error.csv_line_number || "?",
message: error.message || gettext("Unknown error")
)}
<%= if error.field do %>
{gettext(" (Field: %{field})", field: error.field)}
<% end %>
</li>
<% end %>
</ul>
</div>
<% end %>
<%= if length(@import_progress.warnings) > 0 do %>
<div class="alert alert-warning">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<h3 class="font-semibold mb-2">
{gettext("Warnings")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for warning <- @import_progress.warnings do %>
<li>{warning}</li>
<% end %>
</ul>
</div>
</div>
<% end %>
</div>
</section>
<% end %>
</div>
<% end %>
<% end %>
</.form_section>
<% end %>
</Layouts.app> </Layouts.app>
""" """
end end
@ -110,6 +364,112 @@ defmodule MvWeb.GlobalSettingsLive do
end end
end end
@impl true
def handle_event("validate_csv_upload", _params, socket) do
{:noreply, socket}
end
@impl true
def handle_event("start_import", _params, socket) do
case check_import_prerequisites(socket) do
{:error, message} ->
{:noreply, put_flash(socket, :error, message)}
:ok ->
process_csv_upload(socket)
end
end
# Checks if import can be started (admin permission, status, upload ready)
defp check_import_prerequisites(socket) do
# Ensure user role is loaded before authorization check
user = socket.assigns[:current_user]
user_with_role = Actor.ensure_loaded(user)
cond do
not Authorization.can?(user_with_role, :create, Mv.Membership.Member) ->
{:error, gettext("Only administrators can import members from CSV files.")}
socket.assigns.import_status == :running ->
{:error, gettext("Import is already running. Please wait for it to complete.")}
Enum.empty?(socket.assigns.uploads.csv_file.entries) ->
{:error, gettext("Please select a CSV file to import.")}
not List.first(socket.assigns.uploads.csv_file.entries).done? ->
{:error,
gettext("Please wait for the file upload to complete before starting the import.")}
true ->
:ok
end
end
# Processes CSV upload and starts import
defp process_csv_upload(socket) do
with {:ok, content} <- consume_and_read_csv(socket),
{:ok, import_state} <- MemberCSV.prepare(content) do
start_import(socket, import_state)
else
{:error, reason} when is_binary(reason) ->
{:noreply,
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{reason}", reason: reason)
)}
{:error, error} ->
error_message = format_error_message(error)
{:noreply,
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{error}", error: error_message)
)}
end
end
# Starts the import process
defp start_import(socket, import_state) do
progress = initialize_import_progress(import_state)
socket =
socket
|> assign(:import_state, import_state)
|> assign(:import_progress, progress)
|> assign(:import_status, :running)
send(self(), {:process_chunk, 0})
{:noreply, socket}
end
# Initializes import progress structure
defp initialize_import_progress(import_state) do
%{
inserted: 0,
failed: 0,
errors: [],
warnings: import_state.warnings || [],
status: :running,
current_chunk: 0,
total_chunks: length(import_state.chunks),
errors_truncated?: false
}
end
# Formats error messages for display
defp format_error_message(error) do
case error do
%{message: msg} when is_binary(msg) -> msg
%{errors: errors} when is_list(errors) -> inspect(errors)
reason when is_binary(reason) -> reason
other -> inspect(other)
end
end
@impl true @impl true
def handle_info({:custom_field_saved, _custom_field, action}, socket) do def handle_info({:custom_field_saved, _custom_field, action}, socket) do
send_update(MvWeb.CustomFieldLive.IndexComponent, send_update(MvWeb.CustomFieldLive.IndexComponent,
@ -180,6 +540,139 @@ defmodule MvWeb.GlobalSettingsLive do
{:noreply, assign(socket, :settings, updated_settings)} {:noreply, assign(socket, :settings, updated_settings)}
end end
@impl true
def handle_info({:process_chunk, idx}, socket) do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
if idx >= 0 and idx < length(import_state.chunks) do
start_chunk_processing_task(socket, import_state, progress, idx)
else
handle_chunk_error(socket, :invalid_index, idx)
end
_ ->
# Missing required assigns - mark as error
handle_chunk_error(socket, :missing_state, idx)
end
end
@impl true
def handle_info({:chunk_done, idx, result}, socket) do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
handle_chunk_result(socket, import_state, progress, idx, result)
_ ->
# Missing required assigns - mark as error
handle_chunk_error(socket, :missing_state, idx)
end
end
@impl true
def handle_info({:chunk_error, idx, reason}, socket) do
handle_chunk_error(socket, :processing_failed, idx, reason)
end
# Starts async task to process a chunk
# In tests (SQL sandbox mode), runs synchronously to avoid Ecto Sandbox issues
defp start_chunk_processing_task(socket, import_state, progress, idx) do
chunk = Enum.at(import_state.chunks, idx)
# Ensure user role is loaded before using as actor
user = socket.assigns[:current_user]
actor = Actor.ensure_loaded(user)
live_view_pid = self()
# Process chunk with existing error count for capping
opts = [
custom_field_lookup: import_state.custom_field_lookup,
existing_error_count: length(progress.errors),
max_errors: @max_errors,
actor: actor
]
# Get locale from socket for translations in background tasks
locale = socket.assigns[:locale] || "de"
Gettext.put_locale(MvWeb.Gettext, locale)
if Config.sql_sandbox?() do
# Run synchronously in tests to avoid Ecto Sandbox issues with async tasks
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
# In test mode, send the message - it will be processed when render() is called
# in the test. The test helper wait_for_import_completion() handles message processing
send(live_view_pid, {:chunk_done, idx, chunk_result})
else
# Start async task to process chunk in production
# Use start_child for fire-and-forget: no monitor, no Task messages
# We only use our own send/2 messages for communication
Task.Supervisor.start_child(Mv.TaskSupervisor, fn ->
# Set locale in task process for translations
Gettext.put_locale(MvWeb.Gettext, locale)
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
send(live_view_pid, {:chunk_done, idx, chunk_result})
end)
end
{:noreply, socket}
end
# Handles chunk processing result from async task
defp handle_chunk_result(socket, import_state, progress, idx, chunk_result) do
# Merge progress
new_progress = merge_progress(progress, chunk_result, idx)
socket =
socket
|> assign(:import_progress, new_progress)
|> assign(:import_status, new_progress.status)
# Schedule next chunk or mark as done
socket = schedule_next_chunk(socket, idx, length(import_state.chunks))
{:noreply, socket}
end
# Handles chunk processing errors
defp handle_chunk_error(socket, error_type, idx, reason \\ nil) do
error_message =
case error_type do
:invalid_index ->
gettext("Invalid chunk index: %{idx}", idx: idx)
:missing_state ->
gettext("Import state is missing. Cannot process chunk %{idx}.", idx: idx)
:processing_failed ->
gettext("Failed to process chunk %{idx}: %{reason}",
idx: idx,
reason: inspect(reason)
)
end
socket =
socket
|> assign(:import_status, :error)
|> put_flash(:error, error_message)
{:noreply, socket}
end
defp assign_form(%{assigns: %{settings: settings}} = socket) do defp assign_form(%{assigns: %{settings: settings}} = socket) do
form = form =
AshPhoenix.Form.for_update( AshPhoenix.Form.for_update(
@ -192,4 +685,71 @@ defmodule MvWeb.GlobalSettingsLive do
assign(socket, form: to_form(form)) assign(socket, form: to_form(form))
end end
defp consume_and_read_csv(socket) do
result =
consume_uploaded_entries(socket, :csv_file, fn %{path: path}, _entry ->
case File.read(path) do
{:ok, content} -> {:ok, content}
{:error, reason} -> {:error, Exception.message(reason)}
end
end)
result
|> case do
[content] when is_binary(content) ->
{:ok, content}
[{:ok, content}] when is_binary(content) ->
{:ok, content}
[{:error, reason}] ->
{:error, gettext("Failed to read file: %{reason}", reason: reason)}
[] ->
{:error, gettext("No file was uploaded")}
_other ->
{:error, gettext("Failed to read uploaded file")}
end
end
defp merge_progress(progress, chunk_result, current_chunk_idx) do
# Merge errors with cap of @max_errors overall
all_errors = progress.errors ++ chunk_result.errors
new_errors = Enum.take(all_errors, @max_errors)
errors_truncated? = length(all_errors) > @max_errors
# Merge warnings (optional dedupe - simple append for now)
new_warnings = progress.warnings ++ Map.get(chunk_result, :warnings, [])
# Update status based on whether we're done
# current_chunk_idx is 0-based, so after processing chunk 0, we've processed 1 chunk
chunks_processed = current_chunk_idx + 1
new_status = if chunks_processed >= progress.total_chunks, do: :done, else: :running
%{
inserted: progress.inserted + chunk_result.inserted,
failed: progress.failed + chunk_result.failed,
errors: new_errors,
warnings: new_warnings,
status: new_status,
current_chunk: chunks_processed,
total_chunks: progress.total_chunks,
errors_truncated?: errors_truncated? || chunk_result.errors_truncated?
}
end
defp schedule_next_chunk(socket, current_idx, total_chunks) do
next_idx = current_idx + 1
if next_idx < total_chunks do
# Schedule next chunk
send(self(), {:process_chunk, next_idx})
socket
else
# All chunks processed - status already set to :done in merge_progress
socket
end
end
end end

176
priv/gettext/de/LC_MESSAGES/default.po
View file

@ -1825,6 +1825,7 @@ msgstr "erstellt"
msgid "updated" msgid "updated"
msgstr "aktualisiert" msgstr "aktualisiert"
#: lib/mv_web/live/global_settings_live.ex
#: lib/mv_web/live/user_live/form.ex #: lib/mv_web/live/user_live/form.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "Unknown error" msgid "Unknown error"
@ -1949,3 +1950,178 @@ msgstr "Zurücksetzen"
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "Only administrators can regenerate cycles" msgid "Only administrators can regenerate cycles"
msgstr "Nur Administrator*innen können Zyklen regenerieren" msgstr "Nur Administrator*innen können Zyklen regenerieren"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid " (Field: %{field})"
msgstr " (Datenfeld: %{field})"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV File"
msgstr "CSV Datei"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum 10 MB"
msgstr "Nur CSV Dateien, maximal 10 MB"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
msgstr "Individuelle Datenfelder müssen zuerst in Mila angelegt werden bevor das Importieren von diesen Feldern mit CSV Dateien mölich ist."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Download CSV templates:"
msgstr "CSV Vorlagen herunterladen:"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "English Template"
msgstr "Englische Vorlage"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Error list truncated to %{count} entries"
msgstr "Liste der Fehler auf %{count} Einträge reduziert"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Errors"
msgstr "Fehler"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to prepare CSV import: %{error}"
msgstr "Das Vorbereiten des CSV Imports ist gescheitert: %{error}"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to prepare CSV import: %{reason}"
msgstr "Das Vorbereiten des CSV Imports ist gescheitert: %{reason}"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to process chunk %{idx}: %{reason}"
msgstr "Das Importieren von %{idx} ist gescheitert: %{reason}"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Failed to read file: %{reason}"
msgstr "Fehler beim Lesen der Datei: %{reason}"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to read uploaded file"
msgstr "Fehler beim Lesen der hochgeladenen Datei"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed: %{count} row(s)"
msgstr "Fehlgeschlagen: %{count} Zeile(n)"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "German Template"
msgstr "Deutsche Vorlage"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Import Members (CSV)"
msgstr "Mitglieder importieren (CSV)"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Import Results"
msgstr "Import-Ergebnisse"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Import is already running. Please wait for it to complete."
msgstr "Import läuft bereits. Bitte warten Sie, bis er abgeschlossen ist."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Import state is missing. Cannot process chunk %{idx}."
msgstr "Import-Status fehlt. Chunk %{idx} kann nicht verarbeitet werden."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Invalid chunk index: %{idx}"
msgstr "Ungültiger Chunk-Index: %{idx}"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Line %{line}: %{message}"
msgstr "Zeile %{line}: %{message}"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "No file was uploaded"
msgstr "Es wurde keine Datei hochgeladen"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Only administrators can import members from CSV files."
msgstr "Nur Administrator*innen können Mitglieder aus CSV-Dateien importieren."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Please select a CSV file to import."
msgstr "Bitte wählen Sie eine CSV-Datei zum Importieren."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Please wait for the file upload to complete before starting the import."
msgstr "Bitte warten Sie, bis der Datei-Upload abgeschlossen ist, bevor Sie den Import starten."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Processing chunk %{current} of %{total}..."
msgstr "Verarbeite Chunk %{current} von %{total}..."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Start Import"
msgstr "Import starten"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Starting import..."
msgstr "Import wird gestartet..."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Successfully inserted: %{count} member(s)"
msgstr "Erfolgreich eingefügt: %{count} Mitglied(er)"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Summary"
msgstr "Zusammenfassung"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
msgstr "Verwenden Sie den Namen des benutzerdefinierten Feldes als CSV-Spaltenüberschrift (gleiche Normalisierung wie bei Mitgliedsfeldern)"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Warnings"
msgstr "Warnungen"
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "Validation failed"
msgstr "Validierung fehlgeschlagen: %{message}"
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "email"
msgstr "E-Mail"
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "email %{email} has already been taken"
msgstr "E-Mail %{email} wurde bereits verwendet"

176
priv/gettext/default.pot
View file

@ -1826,6 +1826,7 @@ msgstr ""
msgid "updated" msgid "updated"
msgstr "" msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#: lib/mv_web/live/user_live/form.ex #: lib/mv_web/live/user_live/form.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "Unknown error" msgid "Unknown error"
@ -1950,3 +1951,178 @@ msgstr ""
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "Only administrators can regenerate cycles" msgid "Only administrators can regenerate cycles"
msgstr "" msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid " (Field: %{field})"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV File"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum 10 MB"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Download CSV templates:"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "English Template"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Error list truncated to %{count} entries"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Errors"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to prepare CSV import: %{error}"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to prepare CSV import: %{reason}"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to process chunk %{idx}: %{reason}"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to read file: %{reason}"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed to read uploaded file"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Failed: %{count} row(s)"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "German Template"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Import Members (CSV)"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Import Results"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Import is already running. Please wait for it to complete."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Import state is missing. Cannot process chunk %{idx}."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Invalid chunk index: %{idx}"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Line %{line}: %{message}"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "No file was uploaded"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Only administrators can import members from CSV files."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Please select a CSV file to import."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Please wait for the file upload to complete before starting the import."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Processing chunk %{current} of %{total}..."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Start Import"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Starting import..."
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Successfully inserted: %{count} member(s)"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Summary"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "Warnings"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "Validation failed"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "email"
msgstr ""
#: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format
msgid "email %{email} has already been taken"
msgstr ""

413
priv/gettext/en/LC_MESSAGES/default.po
View file

@ -1826,6 +1826,7 @@ msgstr ""
msgid "updated" msgid "updated"
msgstr "" msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#: lib/mv_web/live/user_live/form.ex #: lib/mv_web/live/user_live/form.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "Unknown error" msgid "Unknown error"
@ -1951,309 +1952,177 @@ msgstr ""
msgid "Only administrators can regenerate cycles" msgid "Only administrators can regenerate cycles"
msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy #, elixir-autogen, elixir-format
#~ msgid "Use this form to manage Custom Field Value records in your database." msgid " (Field: %{field})"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Choose a custom field" msgid "CSV File"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Joining year - reduced to 0" msgid "CSV files only, maximum 10 MB"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/components/layouts/sidebar.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy #, elixir-autogen, elixir-format
#~ msgid "Admin" msgid "Custom fields must be created in Mila before importing CSV files with custom field columns"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #: lib/mv_web/live/contribution_type_live/index.ex #, elixir-autogen, elixir-format
#~ #, elixir-autogen, elixir-format msgid "Download CSV templates:"
#~ msgid "Regular" msgstr ""
#~ msgstr ""
#~ #: lib/mv_web/live/components/member_filter_component.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy #, elixir-autogen, elixir-format
#~ msgid "Payment" msgid "English Template"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Current" msgid "Error list truncated to %{count} entries"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Paid via bank transfer" msgid "Errors"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Mark as Unpaid" msgid "Failed to prepare CSV import: %{error}"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Half-yearly contribution for supporting members" msgid "Failed to prepare CSV import: %{reason}"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Reduced fee for unemployed, pensioners, or low income" msgid "Failed to process chunk %{idx}: %{reason}"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Custom field value not found" msgid "Failed to read file: %{reason}"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Supporting Member" msgid "Failed to read uploaded file"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Monthly fee for students and trainees" msgid "Failed: %{count} row(s)"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/components/payment_filter_component.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Filter by payment status" msgid "German Template"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Custom field value %{action} successfully" msgid "Import Members (CSV)"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Total Contributions" msgid "Import Results"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Manage contribution types for membership fees." msgid "Import is already running. Please wait for it to complete."
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Change Contribution Type" msgid "Import state is missing. Cannot process chunk %{idx}."
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "New Contribution Type" msgid "Invalid chunk index: %{idx}"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Time Period" msgid "Line %{line}: %{message}"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy #, elixir-autogen, elixir-format
#~ msgid "Custom field value deleted successfully" msgid "No file was uploaded"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy #, elixir-autogen, elixir-format
#~ msgid "You do not have permission to access this custom field value" msgid "Only administrators can import members from CSV files."
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Cannot delete - members assigned" msgid "Please select a CSV file to import."
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #: lib/mv_web/live/contribution_type_live/index.ex #, elixir-autogen, elixir-format
#~ #, elixir-autogen, elixir-format msgid "Please wait for the file upload to complete before starting the import."
#~ msgid "Preview Mockup" msgstr ""
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Contribution Types" msgid "Processing chunk %{current} of %{total}..."
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #: lib/mv_web/live/contribution_type_live/index.ex #, elixir-autogen, elixir-format
#~ #, elixir-autogen, elixir-format msgid "Start Import"
#~ msgid "This page is not functional and only displays the planned features." msgstr ""
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format, fuzzy #, elixir-autogen, elixir-format
#~ msgid "Member since" msgid "Starting import..."
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Unsupported value type: %{type}" msgid "Successfully inserted: %{count} member(s)"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Custom field" msgid "Summary"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "Mark as Paid" msgid "Use the custom field name as the CSV column header (same normalization as member fields applies)"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv_web/live/global_settings_live.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Contribution type" msgid "Warnings"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/components/layouts/sidebar.ex #: lib/mv/membership/import/member_csv.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Contributions" msgid "Validation failed"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex #: lib/mv/membership/import/member_csv.ex
#~ #: lib/mv_web/live/contribution_type_live/index.ex #, elixir-autogen, elixir-format, fuzzy
#~ #, elixir-autogen, elixir-format msgid "email"
#~ msgid "Reduced" msgstr ""
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex #: lib/mv/membership/import/member_csv.ex
#~ #, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
#~ msgid "No fee for honorary members" msgid "email %{email} has already been taken"
#~ msgstr "" msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/index.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "You do not have permission to delete this custom field value"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "%{count} period selected"
#~ msgid_plural "%{count} periods selected"
#~ msgstr[0] ""
#~ msgstr[1] ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Mark as Suspended"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Contribution types define different membership fee structures. Each type has a fixed cycle (monthly, quarterly, half-yearly, yearly) that cannot be changed after creation."
#~ msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Choose a member"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Suspend"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Reopen"
#~ msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Value"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Why are not all contribution types shown?"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Contribution Start"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Standard membership fee for regular members"
#~ msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Save Custom Field Value"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #: lib/mv_web/live/contribution_type_live/index.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Honorary"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Contributions for %{name}"
#~ msgstr ""
#~ #: lib/mv_web/live/components/member_filter_component.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "Payment status filter"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Family"
#~ msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/index.ex
#~ #, elixir-autogen, elixir-format, fuzzy
#~ msgid "You do not have permission to view custom field values"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Student"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Quarterly fee for family memberships"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Members can only switch between contribution types with the same payment interval (e.g., yearly to yearly). This prevents complex period overlaps."
#~ msgstr ""
#~ #: lib/mv_web/live/custom_field_value_live/form.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Please select a custom field first"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Open Contributions"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_period_live/show.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Member Contributions"
#~ msgstr ""
#~ #: lib/mv_web/live/contribution_type_live/index.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "About Contribution Types"
#~ msgstr ""
#~ #: lib/mv_web/live/components/member_filter_component.ex
#~ #, elixir-autogen, elixir-format
#~ msgid "Filter by %{name}"
#~ msgstr ""

8
test/fixtures/csv_with_bom_semicolon.csv vendored Normal file
View file

@ -0,0 +1,8 @@
first_name;last_name;email;street;postal_code;city
Alice;Smith;alice.smith@example.com;Main Street 1;12345;Berlin
1 first_name last_name email street postal_code city
2 Alice Smith alice.smith@example.com Main Street 1 12345 Berlin

11
test/fixtures/csv_with_empty_lines.csv vendored Normal file
View file

@ -0,0 +1,11 @@
first_name;last_name;email;street;postal_code;city
Alice;Smith;alice.smith@example.com;Main Street 1;12345;Berlin
Bob;Johnson;invalid-email;Park Avenue 2;54321;Munich
1 first_name last_name email street postal_code city
2 Alice Smith alice.smith@example.com Main Street 1 12345 Berlin
3 Bob Johnson invalid-email Park Avenue 2 54321 Munich

10
test/fixtures/csv_with_unknown_custom_field.csv vendored Normal file
View file

@ -0,0 +1,10 @@
first_name;last_name;email;street;postal_code;city;UnknownCustomField
Alice;Smith;alice.smith@example.com;Main Street 1;12345;Berlin;SomeValue
Bob;Johnson;bob.johnson@example.com;Park Avenue 2;54321;Munich;AnotherValue
1 first_name last_name email street postal_code city UnknownCustomField
2 Alice Smith alice.smith@example.com Main Street 1 12345 Berlin SomeValue
3 Bob Johnson bob.johnson@example.com Park Avenue 2 54321 Munich AnotherValue

10
test/fixtures/invalid_member_import.csv vendored Normal file
View file

@ -0,0 +1,10 @@
first_name;last_name;email;street;postal_code;city
Alice;Smith;invalid-email;Main Street 1;12345;Berlin
Bob;Johnson;;Park Avenue 2;54321;Munich
1 first_name last_name email street postal_code city
2 Alice Smith invalid-email Main Street 1 12345 Berlin
3 Bob Johnson Park Avenue 2 54321 Munich

10
test/fixtures/valid_member_import.csv vendored Normal file
View file

@ -0,0 +1,10 @@
first_name;last_name;email;street;postal_code;city
Alice;Smith;alice.smith@example.com;Main Street 1;12345;Berlin
Bob;Johnson;bob.johnson@example.com;Park Avenue 2;54321;Munich
1 first_name last_name email street postal_code city
2 Alice Smith alice.smith@example.com Main Street 1 12345 Berlin
3 Bob Johnson bob.johnson@example.com Park Avenue 2 54321 Munich

89
test/mv/membership/import/member_csv_test.exs
View file

@ -73,25 +73,33 @@ defmodule Mv.Membership.Import.MemberCSVTest do
end end
describe "process_chunk/4" do describe "process_chunk/4" do
test "function exists and accepts chunk_rows_with_lines, column_map, custom_field_map, and opts" do setup do
system_actor = Mv.Helpers.SystemActor.get_system_actor()
%{actor: system_actor}
end
test "function exists and accepts chunk_rows_with_lines, column_map, custom_field_map, and opts",
%{
actor: actor
} do
chunk_rows_with_lines = [{2, %{member: %{email: "john@example.com"}, custom: %{}}}] chunk_rows_with_lines = [{2, %{member: %{email: "john@example.com"}, custom: %{}}}]
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
# This will fail until the function is implemented # This will fail until the function is implemented
result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
assert match?({:ok, _}, result) or match?({:error, _}, result) assert match?({:ok, _}, result) or match?({:error, _}, result)
end end
test "creates member successfully with valid data" do test "creates member successfully with valid data", %{actor: actor} do
chunk_rows_with_lines = [ chunk_rows_with_lines = [
{2, %{member: %{email: "john@example.com", first_name: "John"}, custom: %{}}} {2, %{member: %{email: "john@example.com", first_name: "John"}, custom: %{}}}
] ]
column_map = %{email: 0, first_name: 1} column_map = %{email: 0, first_name: 1}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -106,14 +114,14 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert Enum.any?(members, &(&1.email == "john@example.com")) assert Enum.any?(members, &(&1.email == "john@example.com"))
end end
test "returns error for invalid email" do test "returns error for invalid email", %{actor: actor} do
chunk_rows_with_lines = [ chunk_rows_with_lines = [
{2, %{member: %{email: "invalid-email"}, custom: %{}}} {2, %{member: %{email: "invalid-email"}, custom: %{}}}
] ]
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -130,14 +138,14 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert error.message != "" assert error.message != ""
end end
test "returns error for missing email" do test "returns error for missing email", %{actor: actor} do
chunk_rows_with_lines = [ chunk_rows_with_lines = [
{2, %{member: %{}, custom: %{}}} {2, %{member: %{}, custom: %{}}}
] ]
column_map = %{} column_map = %{}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -152,14 +160,14 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert is_binary(error.message) assert is_binary(error.message)
end end
test "returns error for whitespace-only email" do test "returns error for whitespace-only email", %{actor: actor} do
chunk_rows_with_lines = [ chunk_rows_with_lines = [
{3, %{member: %{email: " "}, custom: %{}}} {3, %{member: %{email: " "}, custom: %{}}}
] ]
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -173,13 +181,11 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert error.field == :email assert error.field == :email
end end
test "returns error for duplicate email" do test "returns error for duplicate email", %{actor: actor} do
# Create existing member first # Create existing member first
system_actor = Mv.Helpers.SystemActor.get_system_actor()
{:ok, _existing} = {:ok, _existing} =
Mv.Membership.create_member(%{email: "duplicate@example.com", first_name: "Existing"}, Mv.Membership.create_member(%{email: "duplicate@example.com", first_name: "Existing"},
actor: system_actor actor: actor
) )
chunk_rows_with_lines = [ chunk_rows_with_lines = [
@ -188,7 +194,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0, first_name: 1} column_map = %{email: 0, first_name: 1}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -203,9 +209,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert error.message =~ "email" or error.message =~ "duplicate" or error.message =~ "unique" assert error.message =~ "email" or error.message =~ "duplicate" or error.message =~ "unique"
end end
test "creates member with custom field values" do test "creates member with custom field values", %{actor: actor} do
system_actor = Mv.Helpers.SystemActor.get_system_actor()
# Create custom field first # Create custom field first
{:ok, custom_field} = {:ok, custom_field} =
Mv.Membership.CustomField Mv.Membership.CustomField
@ -213,7 +217,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
name: "Phone", name: "Phone",
value_type: :string value_type: :string
}) })
|> Ash.create(actor: system_actor) |> Ash.create(actor: actor)
chunk_rows_with_lines = [ chunk_rows_with_lines = [
{2, {2,
@ -230,7 +234,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
to_string(custom_field.id) => %{id: custom_field.id, value_type: custom_field.value_type} to_string(custom_field.id) => %{id: custom_field.id, value_type: custom_field.value_type}
} }
opts = [custom_field_lookup: custom_field_lookup] opts = [custom_field_lookup: custom_field_lookup, actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -239,8 +243,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert chunk_result.failed == 0 assert chunk_result.failed == 0
# Verify member and custom field value were created # Verify member and custom field value were created
system_actor = Mv.Helpers.SystemActor.get_system_actor() members = Mv.Membership.list_members!(actor: actor)
members = Mv.Membership.list_members!(actor: system_actor)
member = Enum.find(members, &(&1.email == "withcustom@example.com")) member = Enum.find(members, &(&1.email == "withcustom@example.com"))
assert member != nil assert member != nil
@ -251,7 +254,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert cfv.value.value == "123-456-7890" assert cfv.value.value == "123-456-7890"
end end
test "handles multiple rows with mixed success and failure" do test "handles multiple rows with mixed success and failure", %{actor: actor} do
chunk_rows_with_lines = [ chunk_rows_with_lines = [
{2, %{member: %{email: "valid1@example.com"}, custom: %{}}}, {2, %{member: %{email: "valid1@example.com"}, custom: %{}}},
{3, %{member: %{email: "invalid-email"}, custom: %{}}}, {3, %{member: %{email: "invalid-email"}, custom: %{}}},
@ -260,7 +263,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -276,7 +279,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert is_binary(error.message) assert is_binary(error.message)
end end
test "preserves CSV line numbers in errors" do test "preserves CSV line numbers in errors", %{actor: actor} do
chunk_rows_with_lines = [ chunk_rows_with_lines = [
{5, %{member: %{email: "invalid"}, custom: %{}}}, {5, %{member: %{email: "invalid"}, custom: %{}}},
{10, %{member: %{email: "also-invalid"}, custom: %{}}} {10, %{member: %{email: "also-invalid"}, custom: %{}}}
@ -284,7 +287,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -297,11 +300,11 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert 10 in line_numbers assert 10 in line_numbers
end end
test "returns {:ok, chunk_result} on success" do test "returns {:ok, chunk_result} on success", %{actor: actor} do
chunk_rows_with_lines = [{2, %{member: %{email: "test@example.com"}, custom: %{}}}] chunk_rows_with_lines = [{2, %{member: %{email: "test@example.com"}, custom: %{}}}]
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -315,11 +318,11 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert is_list(chunk_result.errors) assert is_list(chunk_result.errors)
end end
test "returns {:ok, _} with zero counts for empty chunk" do test "returns {:ok, _} with zero counts for empty chunk", %{actor: actor} do
chunk_rows_with_lines = [] chunk_rows_with_lines = []
column_map = %{} column_map = %{}
custom_field_map = %{} custom_field_map = %{}
opts = [] opts = [actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -334,7 +337,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert function_exported?(MemberCSV, :process_chunk, 4) assert function_exported?(MemberCSV, :process_chunk, 4)
end end
test "error capping collects exactly 50 errors" do test "error capping collects exactly 50 errors", %{actor: actor} do
# Create 50 rows with invalid emails # Create 50 rows with invalid emails
chunk_rows_with_lines = chunk_rows_with_lines =
1..50 1..50
@ -344,7 +347,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [existing_error_count: 0, max_errors: 50] opts = [existing_error_count: 0, max_errors: 50, actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -354,7 +357,9 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert length(chunk_result.errors) == 50 assert length(chunk_result.errors) == 50
end end
test "error capping collects only first 50 errors when more than 50 errors occur" do test "error capping collects only first 50 errors when more than 50 errors occur", %{
actor: actor
} do
# Create 60 rows with invalid emails # Create 60 rows with invalid emails
chunk_rows_with_lines = chunk_rows_with_lines =
1..60 1..60
@ -364,7 +369,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [existing_error_count: 0, max_errors: 50] opts = [existing_error_count: 0, max_errors: 50, actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -374,7 +379,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert length(chunk_result.errors) == 50 assert length(chunk_result.errors) == 50
end end
test "error capping respects existing_error_count" do test "error capping respects existing_error_count", %{actor: actor} do
# Create 30 rows with invalid emails # Create 30 rows with invalid emails
chunk_rows_with_lines = chunk_rows_with_lines =
1..30 1..30
@ -384,7 +389,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [existing_error_count: 25, max_errors: 50] opts = [existing_error_count: 25, max_errors: 50, actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -395,7 +400,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert length(chunk_result.errors) == 25 assert length(chunk_result.errors) == 25
end end
test "error capping collects no errors when limit already reached" do test "error capping collects no errors when limit already reached", %{actor: actor} do
# Create 10 rows with invalid emails # Create 10 rows with invalid emails
chunk_rows_with_lines = chunk_rows_with_lines =
1..10 1..10
@ -405,7 +410,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [existing_error_count: 50, max_errors: 50] opts = [existing_error_count: 50, max_errors: 50, actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -415,7 +420,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert chunk_result.errors == [] assert chunk_result.errors == []
end end
test "error capping with mixed success and failure" do test "error capping with mixed success and failure", %{actor: actor} do
# Create 100 rows: 30 valid, 70 invalid # Create 100 rows: 30 valid, 70 invalid
valid_rows = valid_rows =
1..30 1..30
@ -433,7 +438,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [existing_error_count: 0, max_errors: 50] opts = [existing_error_count: 0, max_errors: 50, actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
@ -444,7 +449,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert length(chunk_result.errors) == 50 assert length(chunk_result.errors) == 50
end end
test "error capping with custom max_errors" do test "error capping with custom max_errors", %{actor: actor} do
# Create 20 rows with invalid emails # Create 20 rows with invalid emails
chunk_rows_with_lines = chunk_rows_with_lines =
1..20 1..20
@ -454,7 +459,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0} column_map = %{email: 0}
custom_field_map = %{} custom_field_map = %{}
opts = [existing_error_count: 0, max_errors: 10] opts = [existing_error_count: 0, max_errors: 10, actor: actor]
assert {:ok, chunk_result} = assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)

613
test/mv_web/live/global_settings_live_test.exs
View file

@ -3,6 +3,22 @@ defmodule MvWeb.GlobalSettingsLiveTest do
import Phoenix.LiveViewTest import Phoenix.LiveViewTest
alias Mv.Membership alias Mv.Membership
# Helper function to upload CSV file in tests
# Reduces code duplication across multiple test cases
defp upload_csv_file(view, csv_content, filename \\ "test_import.csv") do
view
|> file_input("#csv-upload-form", :csv_file, [
%{
last_modified: System.system_time(:second),
name: filename,
content: csv_content,
size: byte_size(csv_content),
type: "text/csv"
}
])
|> render_upload(filename)
end
describe "Global Settings LiveView" do describe "Global Settings LiveView" do
setup %{conn: conn} do setup %{conn: conn} do
user = create_test_user(%{email: "admin@example.com"}) user = create_test_user(%{email: "admin@example.com"})
@ -81,4 +97,601 @@ defmodule MvWeb.GlobalSettingsLiveTest do
assert render(view) =~ "updated" or render(view) =~ "success" assert render(view) =~ "updated" or render(view) =~ "success"
end end
end end
describe "CSV Import Section" do
test "admin user sees import section", %{conn: conn} do
{:ok, _view, html} = live(conn, ~p"/settings")
# Check for import section heading or identifier
assert html =~ "Import" or html =~ "CSV" or html =~ "member_import"
end
test "admin user sees custom fields notice", %{conn: conn} do
{:ok, _view, html} = live(conn, ~p"/settings")
# Check for custom fields notice text
assert html =~ "Custom fields" or html =~ "custom field"
end
test "admin user sees template download links", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
html = render(view)
# Check for English template link
assert html =~ "member_import_en.csv" or html =~ "/templates/member_import_en.csv"
# Check for German template link
assert html =~ "member_import_de.csv" or html =~ "/templates/member_import_de.csv"
end
test "template links use static path helper", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
html = render(view)
# Check that links contain the static path pattern
# Static paths typically start with /templates/ or contain the full path
assert html =~ "/templates/member_import_en.csv" or
html =~ ~r/href=["'][^"']*member_import_en\.csv["']/
assert html =~ "/templates/member_import_de.csv" or
html =~ ~r/href=["'][^"']*member_import_de\.csv["']/
end
test "admin user sees file upload input", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
html = render(view)
# Check for file input element
assert html =~ ~r/type=["']file["']/i or html =~ "phx-hook" or html =~ "upload"
end
test "file upload has CSV-only restriction", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
html = render(view)
# Check for CSV file type restriction in help text or accept attribute
assert html =~ ~r/\.csv/i or html =~ "CSV" or html =~ ~r/accept=["'][^"']*csv["']/i
end
test "non-admin user does not see import section", %{conn: conn} do
# Create non-admin user (member role)
member_user = Mv.Fixtures.user_with_role_fixture("own_data")
conn = MvWeb.ConnCase.conn_with_password_user(conn, member_user)
{:ok, _view, html} = live(conn, ~p"/settings")
# Import section should not be visible
refute html =~ "Import Members" or html =~ "CSV Import" or
(html =~ "Import" and html =~ "CSV")
end
end
describe "CSV Import - Import" do
setup %{conn: conn} do
# Ensure admin user
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
conn = MvWeb.ConnCase.conn_with_password_user(conn, admin_user)
# Read valid CSV fixture
csv_content =
Path.join([__DIR__, "..", "..", "fixtures", "valid_member_import.csv"])
|> File.read!()
{:ok, conn: conn, admin_user: admin_user, csv_content: csv_content}
end
test "admin can upload CSV and start import", %{conn: conn, csv_content: csv_content} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Simulate file upload using helper function
upload_csv_file(view, csv_content)
# Trigger start_import event via form submit
assert view
|> form("#csv-upload-form", %{})
|> render_submit()
# Check that import has started or shows appropriate message
html = render(view)
# Either import started successfully OR we see a specific error (not admin error)
import_started = html =~ "Import in progress" or html =~ "running" or html =~ "progress"
no_admin_error = not (html =~ "Only administrators can import")
# If import failed, it should be a CSV parsing error, not an admin error
if html =~ "Failed to prepare CSV import" do
# This is acceptable - CSV might have issues, but admin check passed
assert no_admin_error
else
# Import should have started
assert import_started or html =~ "CSV File"
end
end
test "admin import initializes progress correctly", %{conn: conn, csv_content: csv_content} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Simulate file upload using helper function
upload_csv_file(view, csv_content)
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Check that import has started or shows appropriate message
html = render(view)
# Either import started successfully OR we see a specific error (not admin error)
import_started = html =~ "Import in progress" or html =~ "running" or html =~ "progress"
no_admin_error = not (html =~ "Only administrators can import")
# If import failed, it should be a CSV parsing error, not an admin error
if html =~ "Failed to prepare CSV import" do
# This is acceptable - CSV might have issues, but admin check passed
assert no_admin_error
else
# Import should have started
assert import_started or html =~ "CSV File"
end
end
test "non-admin cannot start import", %{conn: conn} do
# Create non-admin user
member_user = Mv.Fixtures.user_with_role_fixture("own_data")
conn = MvWeb.ConnCase.conn_with_password_user(conn, member_user)
{:ok, view, _html} = live(conn, ~p"/settings")
# Since non-admin shouldn't see the section, we check that import section is not visible
html = render(view)
refute html =~ "Import Members" or html =~ "CSV Import" or html =~ "start_import"
end
test "invalid CSV shows user-friendly error", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Create invalid CSV (missing required fields)
invalid_csv = "invalid_header\nincomplete_row"
# Simulate file upload using helper function
upload_csv_file(view, invalid_csv, "invalid.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Check for error message (flash)
html = render(view)
assert html =~ "error" or html =~ "failed" or html =~ "Failed to prepare"
end
@tag :skip
test "empty CSV shows error", %{conn: conn} do
# Skip this test - Phoenix LiveView has issues with empty file uploads in tests
# The error is handled correctly in production, but test framework has limitations
{:ok, view, _html} = live(conn, ~p"/settings")
empty_csv = " "
csv_path = Path.join([System.tmp_dir!(), "empty_#{System.unique_integer()}.csv"])
File.write!(csv_path, empty_csv)
view
|> file_input("#csv-upload-form", :csv_file, [
%{
last_modified: System.system_time(:second),
name: "empty.csv",
content: empty_csv,
size: byte_size(empty_csv),
type: "text/csv"
}
])
|> render_upload("empty.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Check for error message
html = render(view)
assert html =~ "error" or html =~ "empty" or html =~ "failed" or html =~ "Failed to prepare"
end
end
describe "CSV Import - Step 3: Chunk Processing" do
setup %{conn: conn} do
# Ensure admin user
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
conn = MvWeb.ConnCase.conn_with_password_user(conn, admin_user)
# Read valid CSV fixture
valid_csv_content =
Path.join([__DIR__, "..", "..", "fixtures", "valid_member_import.csv"])
|> File.read!()
# Read invalid CSV fixture
invalid_csv_content =
Path.join([__DIR__, "..", "..", "fixtures", "invalid_member_import.csv"])
|> File.read!()
{:ok,
conn: conn,
admin_user: admin_user,
valid_csv_content: valid_csv_content,
invalid_csv_content: invalid_csv_content}
end
test "happy path: valid CSV processes all chunks and shows done status", %{
conn: conn,
valid_csv_content: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Simulate file upload using helper function
upload_csv_file(view, csv_content)
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait for processing to complete
# In test mode, chunks are processed synchronously and messages are sent via send/2
# render(view) processes handle_info messages, so we call it multiple times
# to ensure all messages are processed
# Use the same approach as "success rendering" test which works
Process.sleep(1000)
html = render(view)
# Should show success count (inserted count)
assert html =~ "Inserted" or html =~ "inserted" or html =~ "2"
# Should show completed status
assert html =~ "completed" or html =~ "done" or html =~ "Import completed" or
has_element?(view, "[data-testid='import-results-panel']")
end
test "error handling: invalid CSV shows errors with line numbers", %{
conn: conn,
invalid_csv_content: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Simulate file upload using helper function
upload_csv_file(view, csv_content, "invalid_import.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait for chunk processing
Process.sleep(500)
html = render(view)
# Should show failure count > 0
assert html =~ "failed" or html =~ "error" or html =~ "Failed"
# Should show line numbers in errors (from service, not recalculated)
# Line numbers should be 2, 3 (header is line 1)
assert html =~ "2" or html =~ "3" or html =~ "line"
end
test "error cap: many failing rows caps errors at 50", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Generate CSV with 100 invalid rows (all missing email)
header = "first_name;last_name;email;street;postal_code;city\n"
invalid_rows = for i <- 1..100, do: "Row#{i};Last#{i};;Street#{i};12345;City#{i}\n"
large_invalid_csv = header <> Enum.join(invalid_rows)
# Simulate file upload using helper function
upload_csv_file(view, large_invalid_csv, "large_invalid.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait for chunk processing
Process.sleep(1000)
html = render(view)
# Should show failed count == 100
assert html =~ "100" or html =~ "failed"
# Errors should be capped at 50 (but we can't easily check exact count in HTML)
# The important thing is that processing completes without crashing
assert html =~ "done" or html =~ "complete" or html =~ "finished"
end
test "chunk scheduling: progress updates show chunk processing", %{
conn: conn,
valid_csv_content: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Simulate file upload using helper function
upload_csv_file(view, csv_content)
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait a bit for processing to start
Process.sleep(200)
# Check that status area exists (with aria-live for accessibility)
html = render(view)
assert html =~ "aria-live" or html =~ "status" or html =~ "progress" or
html =~ "Processing" or html =~ "chunk"
# Final state should be :done
Process.sleep(500)
final_html = render(view)
assert final_html =~ "done" or final_html =~ "complete" or final_html =~ "finished"
end
end
describe "CSV Import - Step 4: Results UI" do
setup %{conn: conn} do
# Ensure admin user
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
conn = MvWeb.ConnCase.conn_with_password_user(conn, admin_user)
# Read valid CSV fixture
valid_csv_content =
Path.join([__DIR__, "..", "..", "fixtures", "valid_member_import.csv"])
|> File.read!()
# Read invalid CSV fixture
invalid_csv_content =
Path.join([__DIR__, "..", "..", "fixtures", "invalid_member_import.csv"])
|> File.read!()
# Read CSV with unknown custom field
unknown_custom_field_csv =
Path.join([__DIR__, "..", "..", "fixtures", "csv_with_unknown_custom_field.csv"])
|> File.read!()
{:ok,
conn: conn,
admin_user: admin_user,
valid_csv_content: valid_csv_content,
invalid_csv_content: invalid_csv_content,
unknown_custom_field_csv: unknown_custom_field_csv}
end
test "success rendering: valid CSV shows success count", %{
conn: conn,
valid_csv_content: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Simulate file upload using helper function
upload_csv_file(view, csv_content)
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait for processing to complete
Process.sleep(1000)
html = render(view)
# Should show success count (inserted count)
assert html =~ "Inserted" or html =~ "inserted" or html =~ "2"
# Should show completed status
assert html =~ "completed" or html =~ "done" or html =~ "Import completed"
end
test "error rendering: invalid CSV shows failure count and error list with line numbers", %{
conn: conn,
invalid_csv_content: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Simulate file upload using helper function
upload_csv_file(view, csv_content, "invalid_import.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait for processing
Process.sleep(1000)
html = render(view)
# Should show failure count
assert html =~ "Failed" or html =~ "failed"
# Should show error list with line numbers (from service, not recalculated)
assert html =~ "Line" or html =~ "line" or html =~ "2" or html =~ "3"
# Should show error messages
assert html =~ "error" or html =~ "Error" or html =~ "Errors"
end
test "warning rendering: CSV with unknown custom field shows warnings block", %{
conn: conn,
unknown_custom_field_csv: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/settings")
csv_path =
Path.join([System.tmp_dir!(), "unknown_custom_#{System.unique_integer()}.csv"])
File.write!(csv_path, csv_content)
view
|> file_input("#csv-upload-form", :csv_file, [
%{
last_modified: System.system_time(:second),
name: "unknown_custom.csv",
content: csv_content,
size: byte_size(csv_content),
type: "text/csv"
}
])
|> render_upload("unknown_custom.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait for processing
Process.sleep(1000)
html = render(view)
# Should show warnings block (if warnings were generated)
# Warnings are generated when unknown custom field columns are detected
# Check if warnings section exists OR if import completed successfully
has_warnings = html =~ "Warning" or html =~ "warning" or html =~ "Warnings"
import_completed = html =~ "completed" or html =~ "done" or html =~ "Import Results"
# If warnings exist, they should contain the column name
if has_warnings do
assert html =~ "UnknownCustomField" or html =~ "unknown" or html =~ "Unknown column" or
html =~ "will be ignored"
end
# Import should complete (either with or without warnings)
assert import_completed
end
test "A11y: file input has label", %{conn: conn} do
{:ok, _view, html} = live(conn, ~p"/settings")
# Check for label associated with file input
assert html =~ ~r/<label[^>]*for=["']csv_file["']/i or
html =~ ~r/<label[^>]*>.*CSV File/i
end
test "A11y: status/progress container has aria-live", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
html = render(view)
# Check for aria-live attribute in status area
assert html =~ ~r/aria-live=["']polite["']/i
end
test "A11y: links have descriptive text", %{conn: conn} do
{:ok, _view, html} = live(conn, ~p"/settings")
# Check that links have descriptive text (not just "click here")
# Template links should have text like "English Template" or "German Template"
assert html =~ "English Template" or html =~ "German Template" or
html =~ "English" or html =~ "German"
# Custom Fields section should have descriptive text (Data Field button)
# The component uses "New Data Field" button, not a link
assert html =~ "Data Field" or html =~ "New Data Field"
end
end
describe "CSV Import - Step 5: Edge Cases" do
setup %{conn: conn} do
# Ensure admin user
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
conn = MvWeb.ConnCase.conn_with_password_user(conn, admin_user)
{:ok, conn: conn, admin_user: admin_user}
end
test "BOM + semicolon delimiter: import succeeds", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Read CSV with BOM
csv_content =
Path.join([__DIR__, "..", "..", "fixtures", "csv_with_bom_semicolon.csv"])
|> File.read!()
# Simulate file upload using helper function
upload_csv_file(view, csv_content, "bom_import.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait for processing
Process.sleep(1000)
html = render(view)
# Should succeed (BOM is stripped automatically)
assert html =~ "completed" or html =~ "done" or html =~ "Inserted"
# Should not show error about BOM
refute html =~ "BOM" or html =~ "encoding"
end
test "empty lines: line numbers in errors correspond to physical CSV lines", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
# CSV with empty line: header (line 1), valid row (line 2), empty (line 3), invalid (line 4)
csv_content =
Path.join([__DIR__, "..", "..", "fixtures", "csv_with_empty_lines.csv"])
|> File.read!()
# Simulate file upload using helper function
upload_csv_file(view, csv_content, "empty_lines.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
# Wait for processing
Process.sleep(1000)
html = render(view)
# Should show error with correct line number (line 4, not line 3)
# The error should be on the line with invalid email, which is after the empty line
assert html =~ "Line 4" or html =~ "line 4" or html =~ "4"
# Should show error message
assert html =~ "error" or html =~ "Error" or html =~ "invalid"
end
test "too many rows (1001): import is rejected with user-friendly error", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Generate CSV with 1001 rows dynamically
header = "first_name;last_name;email;street;postal_code;city\n"
rows =
for i <- 1..1001 do
"Row#{i};Last#{i};email#{i}@example.com;Street#{i};12345;City#{i}\n"
end
large_csv = header <> Enum.join(rows)
# Simulate file upload using helper function
upload_csv_file(view, large_csv, "too_many_rows.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
html = render(view)
# Should show user-friendly error about row limit
assert html =~ "exceeds" or html =~ "maximum" or html =~ "limit" or html =~ "1000" or
html =~ "Failed to prepare"
end
test "wrong file type (.txt): upload shows error", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/settings")
# Create .txt file (not .csv)
txt_content = "This is not a CSV file\nJust some text\n"
txt_path = Path.join([System.tmp_dir!(), "wrong_type_#{System.unique_integer()}.txt"])
File.write!(txt_path, txt_content)
# Try to upload .txt file
# Note: allow_upload is configured to accept only .csv, so this should fail
# In tests, we can't easily simulate file type rejection, but we can check
# that the UI shows appropriate help text
html = render(view)
# Should show CSV-only restriction in help text
assert html =~ "CSV" or html =~ "csv" or html =~ ".csv"
end
test "file input has correct accept attribute for CSV only", %{conn: conn} do
{:ok, _view, html} = live(conn, ~p"/settings")
# Check that file input has accept attribute for CSV
assert html =~ ~r/accept=["'][^"']*csv["']/i or html =~ "CSV files only"
end
end
end end