local-it/mitgliederverwaltung
2
3
Fork 0
Code Issues 66 Pull requests 6 Projects 3 Releases Packages 1 Activity Actions

feat: move import/export to own section

Browse source
This commit is contained in:
carla 2026-02-03 14:19:36 +01:00
parent c998d14b95
commit 6aba54df68
4 changed files with 634 additions and 559 deletions
Download patch file Download diff file Expand all files Collapse all files

1
lib/mv_web/components/layouts/sidebar.ex
View file

@ -91,6 +91,7 @@ defmodule MvWeb.Layouts.Sidebar do
href={~p"/membership_fee_settings"}
label={gettext("Fee Settings")}
/>
<.menu_subitem href={~p"/admin/import-export"} label={gettext("Import/Export")} />
<.menu_subitem href={~p"/settings"} label={gettext("Settings")} />
</.menu_group>
</ul>

561
lib/mv_web/live/global_settings_live.ex
View file

@ -7,7 +7,6 @@ defmodule MvWeb.GlobalSettingsLive do
- Manage custom fields
- Real-time form validation
- Success/error feedback
- CSV member import (admin only)
## Settings
- `club_name` - The name of the association/club (required)
@ -15,47 +14,19 @@ defmodule MvWeb.GlobalSettingsLive do
## Events
- `validate` - Real-time form validation
- `save` - Save settings changes
- `start_import` - Start CSV member import (admin only)
## CSV Import
The CSV import feature allows administrators to upload CSV files and import members.
### File Upload
Files are uploaded automatically when selected (`auto_upload: true`). No manual
upload trigger is required.
### Rate Limiting
Currently, there is no rate limiting for CSV imports. Administrators can start
multiple imports in quick succession. This is intentional for bulk data migration
scenarios, but should be monitored in production.
### Limits
- Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
- Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
- Processing: chunks of 200 rows
- Errors: capped at 50 per import
## Note
Settings is a singleton resource - there is only one settings record.
The club_name can also be set via the `ASSOCIATION_NAME` environment variable.
CSV member import has been moved to the Import/Export page (`/admin/import-export`).
"""
use MvWeb, :live_view
alias Mv.Authorization.Actor
alias Mv.Config
alias Mv.Membership
alias Mv.Membership.Import.MemberCSV
alias MvWeb.Authorization
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
# CSV Import configuration constants
@max_errors 50
@impl true
def mount(_params, session, socket) do
{:ok, settings} = Membership.get_settings()
@ -69,22 +40,8 @@ defmodule MvWeb.GlobalSettingsLive do
|> assign(:page_title, gettext("Settings"))
|> assign(:settings, settings)
|> assign(:active_editing_section, nil)
|> assign(:import_state, nil)
|> assign(:import_progress, nil)
|> assign(:import_status, :idle)
|> assign(:locale, locale)
|> assign(:max_errors, @max_errors)
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
|> assign_form()
# Configure file upload with auto-upload enabled
# Files are uploaded automatically when selected, no need for manual trigger
|> allow_upload(:csv_file,
accept: ~w(.csv),
max_entries: 1,
max_file_size: Config.csv_import_max_file_size_bytes(),
auto_upload: true
)
{:ok, socket}
end
@ -133,211 +90,6 @@ defmodule MvWeb.GlobalSettingsLive do
actor={@current_user}
/>
</.form_section>
<%!-- CSV Import Section (Admin only) --%>
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
<.form_section title={gettext("Import Members (CSV)")}>
<div role="note" class="alert alert-info mb-4">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<p class="text-sm mb-2">
{gettext(
"Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
)}
</p>
<p class="text-sm">
<.link
href="#custom_fields"
class="link"
data-testid="custom-fields-link"
>
{gettext("Manage Memberdata")}
</.link>
</p>
</div>
</div>
<div class="mb-4">
<p class="text-sm text-base-content/70 mb-2">
{gettext("Download CSV templates:")}
</p>
<ul class="list-disc list-inside space-y-1">
<li>
<.link
href={~p"/templates/member_import_en.csv"}
download="member_import_en.csv"
class="link link-primary"
>
{gettext("English Template")}
</.link>
</li>
<li>
<.link
href={~p"/templates/member_import_de.csv"}
download="member_import_de.csv"
class="link link-primary"
>
{gettext("German Template")}
</.link>
</li>
</ul>
</div>
<.form
id="csv-upload-form"
for={%{}}
multipart={true}
phx-change="validate_csv_upload"
phx-submit="start_import"
data-testid="csv-upload-form"
>
<div class="form-control">
<label for="csv_file" class="label">
<span class="label-text">
{gettext("CSV File")}
</span>
</label>
<.live_file_input
upload={@uploads.csv_file}
id="csv_file"
class="file-input file-input-bordered w-full"
aria-describedby="csv_file_help"
/>
<label class="label" id="csv_file_help">
<span class="label-text-alt">
{gettext("CSV files only, maximum %{size} MB", size: @csv_import_max_file_size_mb)}
</span>
</label>
</div>
<.button
type="submit"
phx-disable-with={gettext("Starting import...")}
variant="primary"
disabled={
@import_status == :running or
Enum.empty?(@uploads.csv_file.entries) or
@uploads.csv_file.entries |> List.first() |> then(&(&1 && not &1.done?))
}
data-testid="start-import-button"
>
{gettext("Start Import")}
</.button>
</.form>
<%= if @import_status == :running or @import_status == :done do %>
<%= if @import_progress do %>
<div
role="status"
aria-live="polite"
class="mt-4"
data-testid="import-progress-container"
>
<%= if @import_progress.status == :running do %>
<p class="text-sm" data-testid="import-progress-text">
{gettext("Processing chunk %{current} of %{total}...",
current: @import_progress.current_chunk,
total: @import_progress.total_chunks
)}
</p>
<% end %>
<%= if @import_progress.status == :done do %>
<section class="space-y-4" data-testid="import-results-panel">
<h2 class="text-lg font-semibold">
{gettext("Import Results")}
</h2>
<div class="space-y-4">
<div>
<h3 class="text-sm font-semibold mb-2">
{gettext("Summary")}
</h3>
<div class="text-sm space-y-2">
<p>
<.icon
name="hero-check-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Successfully inserted: %{count} member(s)",
count: @import_progress.inserted
)}
</p>
<%= if @import_progress.failed > 0 do %>
<p>
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Failed: %{count} row(s)", count: @import_progress.failed)}
</p>
<% end %>
<%= if @import_progress.errors_truncated? do %>
<p>
<.icon
name="hero-information-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Error list truncated to %{count} entries",
count: @max_errors
)}
</p>
<% end %>
</div>
</div>
<%= if length(@import_progress.errors) > 0 do %>
<div data-testid="import-error-list">
<h3 class="text-sm font-semibold mb-2">
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Errors")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for error <- @import_progress.errors do %>
<li>
{gettext("Line %{line}: %{message}",
line: error.csv_line_number || "?",
message: error.message || gettext("Unknown error")
)}
<%= if error.field do %>
{gettext(" (Field: %{field})", field: error.field)}
<% end %>
</li>
<% end %>
</ul>
</div>
<% end %>
<%= if length(@import_progress.warnings) > 0 do %>
<div class="alert alert-warning">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<h3 class="font-semibold mb-2">
{gettext("Warnings")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for warning <- @import_progress.warnings do %>
<li>{warning}</li>
<% end %>
</ul>
</div>
</div>
<% end %>
</div>
</section>
<% end %>
</div>
<% end %>
<% end %>
</.form_section>
<% end %>
</Layouts.app>
"""
end
@ -370,115 +122,6 @@ defmodule MvWeb.GlobalSettingsLive do
end
end
@impl true
def handle_event("validate_csv_upload", _params, socket) do
{:noreply, socket}
end
@impl true
def handle_event("start_import", _params, socket) do
case check_import_prerequisites(socket) do
{:error, message} ->
{:noreply, put_flash(socket, :error, message)}
:ok ->
process_csv_upload(socket)
end
end
# Checks if import can be started (admin permission, status, upload ready)
defp check_import_prerequisites(socket) do
# Ensure user role is loaded before authorization check
user = socket.assigns[:current_user]
user_with_role = Actor.ensure_loaded(user)
cond do
not Authorization.can?(user_with_role, :create, Mv.Membership.Member) ->
{:error, gettext("Only administrators can import members from CSV files.")}
socket.assigns.import_status == :running ->
{:error, gettext("Import is already running. Please wait for it to complete.")}
Enum.empty?(socket.assigns.uploads.csv_file.entries) ->
{:error, gettext("Please select a CSV file to import.")}
not List.first(socket.assigns.uploads.csv_file.entries).done? ->
{:error,
gettext("Please wait for the file upload to complete before starting the import.")}
true ->
:ok
end
end
# Processes CSV upload and starts import
defp process_csv_upload(socket) do
actor = MvWeb.LiveHelpers.current_actor(socket)
with {:ok, content} <- consume_and_read_csv(socket),
{:ok, import_state} <-
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
start_import(socket, import_state)
else
{:error, reason} when is_binary(reason) ->
{:noreply,
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{reason}", reason: reason)
)}
{:error, error} ->
error_message = format_error_message(error)
{:noreply,
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{error}", error: error_message)
)}
end
end
# Starts the import process
defp start_import(socket, import_state) do
progress = initialize_import_progress(import_state)
socket =
socket
|> assign(:import_state, import_state)
|> assign(:import_progress, progress)
|> assign(:import_status, :running)
send(self(), {:process_chunk, 0})
{:noreply, socket}
end
# Initializes import progress structure
defp initialize_import_progress(import_state) do
%{
inserted: 0,
failed: 0,
errors: [],
warnings: import_state.warnings || [],
status: :running,
current_chunk: 0,
total_chunks: length(import_state.chunks),
errors_truncated?: false
}
end
# Formats error messages for display
defp format_error_message(error) do
case error do
%{message: msg} when is_binary(msg) -> msg
%{errors: errors} when is_list(errors) -> inspect(errors)
reason when is_binary(reason) -> reason
other -> inspect(other)
end
end
@impl true
def handle_info({:custom_field_saved, _custom_field, action}, socket) do
send_update(MvWeb.CustomFieldLive.IndexComponent,
@ -558,139 +201,6 @@ defmodule MvWeb.GlobalSettingsLive do
{:noreply, assign(socket, :settings, updated_settings)}
end
@impl true
def handle_info({:process_chunk, idx}, socket) do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
if idx >= 0 and idx < length(import_state.chunks) do
start_chunk_processing_task(socket, import_state, progress, idx)
else
handle_chunk_error(socket, :invalid_index, idx)
end
_ ->
# Missing required assigns - mark as error
handle_chunk_error(socket, :missing_state, idx)
end
end
@impl true
def handle_info({:chunk_done, idx, result}, socket) do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
handle_chunk_result(socket, import_state, progress, idx, result)
_ ->
# Missing required assigns - mark as error
handle_chunk_error(socket, :missing_state, idx)
end
end
@impl true
def handle_info({:chunk_error, idx, reason}, socket) do
handle_chunk_error(socket, :processing_failed, idx, reason)
end
# Starts async task to process a chunk
# In tests (SQL sandbox mode), runs synchronously to avoid Ecto Sandbox issues
defp start_chunk_processing_task(socket, import_state, progress, idx) do
chunk = Enum.at(import_state.chunks, idx)
# Ensure user role is loaded before using as actor
user = socket.assigns[:current_user]
actor = Actor.ensure_loaded(user)
live_view_pid = self()
# Process chunk with existing error count for capping
opts = [
custom_field_lookup: import_state.custom_field_lookup,
existing_error_count: length(progress.errors),
max_errors: @max_errors,
actor: actor
]
# Get locale from socket for translations in background tasks
locale = socket.assigns[:locale] || "de"
Gettext.put_locale(MvWeb.Gettext, locale)
if Config.sql_sandbox?() do
# Run synchronously in tests to avoid Ecto Sandbox issues with async tasks
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
# In test mode, send the message - it will be processed when render() is called
# in the test. The test helper wait_for_import_completion() handles message processing
send(live_view_pid, {:chunk_done, idx, chunk_result})
else
# Start async task to process chunk in production
# Use start_child for fire-and-forget: no monitor, no Task messages
# We only use our own send/2 messages for communication
Task.Supervisor.start_child(Mv.TaskSupervisor, fn ->
# Set locale in task process for translations
Gettext.put_locale(MvWeb.Gettext, locale)
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
send(live_view_pid, {:chunk_done, idx, chunk_result})
end)
end
{:noreply, socket}
end
# Handles chunk processing result from async task
defp handle_chunk_result(socket, import_state, progress, idx, chunk_result) do
# Merge progress
new_progress = merge_progress(progress, chunk_result, idx)
socket =
socket
|> assign(:import_progress, new_progress)
|> assign(:import_status, new_progress.status)
# Schedule next chunk or mark as done
socket = schedule_next_chunk(socket, idx, length(import_state.chunks))
{:noreply, socket}
end
# Handles chunk processing errors
defp handle_chunk_error(socket, error_type, idx, reason \\ nil) do
error_message =
case error_type do
:invalid_index ->
gettext("Invalid chunk index: %{idx}", idx: idx)
:missing_state ->
gettext("Import state is missing. Cannot process chunk %{idx}.", idx: idx)
:processing_failed ->
gettext("Failed to process chunk %{idx}: %{reason}",
idx: idx,
reason: inspect(reason)
)
end
socket =
socket
|> assign(:import_status, :error)
|> put_flash(:error, error_message)
{:noreply, socket}
end
defp assign_form(%{assigns: %{settings: settings}} = socket) do
form =
AshPhoenix.Form.for_update(
@ -703,71 +213,4 @@ defmodule MvWeb.GlobalSettingsLive do
assign(socket, form: to_form(form))
end
defp consume_and_read_csv(socket) do
result =
consume_uploaded_entries(socket, :csv_file, fn %{path: path}, _entry ->
case File.read(path) do
{:ok, content} -> {:ok, content}
{:error, reason} -> {:error, Exception.message(reason)}
end
end)
result
|> case do
[content] when is_binary(content) ->
{:ok, content}
[{:ok, content}] when is_binary(content) ->
{:ok, content}
[{:error, reason}] ->
{:error, gettext("Failed to read file: %{reason}", reason: reason)}
[] ->
{:error, gettext("No file was uploaded")}
_other ->
{:error, gettext("Failed to read uploaded file")}
end
end
defp merge_progress(progress, chunk_result, current_chunk_idx) do
# Merge errors with cap of @max_errors overall
all_errors = progress.errors ++ chunk_result.errors
new_errors = Enum.take(all_errors, @max_errors)
errors_truncated? = length(all_errors) > @max_errors
# Merge warnings (optional dedupe - simple append for now)
new_warnings = progress.warnings ++ Map.get(chunk_result, :warnings, [])
# Update status based on whether we're done
# current_chunk_idx is 0-based, so after processing chunk 0, we've processed 1 chunk
chunks_processed = current_chunk_idx + 1
new_status = if chunks_processed >= progress.total_chunks, do: :done, else: :running
%{
inserted: progress.inserted + chunk_result.inserted,
failed: progress.failed + chunk_result.failed,
errors: new_errors,
warnings: new_warnings,
status: new_status,
current_chunk: chunks_processed,
total_chunks: progress.total_chunks,
errors_truncated?: errors_truncated? || chunk_result.errors_truncated?
}
end
defp schedule_next_chunk(socket, current_idx, total_chunks) do
next_idx = current_idx + 1
if next_idx < total_chunks do
# Schedule next chunk
send(self(), {:process_chunk, next_idx})
socket
else
# All chunks processed - status already set to :done in merge_progress
socket
end
end
end

628
lib/mv_web/live/import_export_live.ex Normal file
View file

@ -0,0 +1,628 @@
defmodule MvWeb.ImportExportLive do
@moduledoc """
LiveView for importing and exporting members via CSV.
## Features
- CSV member import (admin only)
- Real-time import progress tracking
- Error and warning reporting
- Custom fields support
## CSV Import
The CSV import feature allows administrators to upload CSV files and import members.
### File Upload
Files are uploaded automatically when selected (`auto_upload: true`). No manual
upload trigger is required.
### Rate Limiting
Currently, there is no rate limiting for CSV imports. Administrators can start
multiple imports in quick succession. This is intentional for bulk data migration
scenarios, but should be monitored in production.
### Limits
- Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
- Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
- Processing: chunks of 200 rows
- Errors: capped at 50 per import
"""
use MvWeb, :live_view
alias Mv.Authorization.Actor
alias Mv.Config
alias Mv.Membership
alias Mv.Membership.Import.MemberCSV
alias MvWeb.Authorization
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
# CSV Import configuration constants
@max_errors 50
@impl true
def mount(_params, session, socket) do
# Get locale from session for translations
locale = session["locale"] || "de"
Gettext.put_locale(MvWeb.Gettext, locale)
# Get club name from settings
club_name =
case Membership.get_settings() do
{:ok, settings} -> settings.club_name
_ -> "Mitgliederverwaltung"
end
socket =
socket
|> assign(:page_title, gettext("Import/Export"))
|> assign(:club_name, club_name)
|> assign(:import_state, nil)
|> assign(:import_progress, nil)
|> assign(:import_status, :idle)
|> assign(:locale, locale)
|> assign(:max_errors, @max_errors)
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
# Configure file upload with auto-upload enabled
# Files are uploaded automatically when selected, no need for manual trigger
|> allow_upload(:csv_file,
accept: ~w(.csv),
max_entries: 1,
max_file_size: Config.csv_import_max_file_size_bytes(),
auto_upload: true
)
{:ok, socket}
end
@impl true
def render(assigns) do
~H"""
<Layouts.app flash={@flash} current_user={@current_user} club_name={@club_name}>
<.header>
{gettext("Import/Export")}
<:subtitle>
{gettext("Import members from CSV files or export member data.")}
</:subtitle>
</.header>
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
<%!-- CSV Import Section --%>
<.form_section title={gettext("Import Members (CSV)")}>
<div role="note" class="alert alert-info mb-4">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<p class="text-sm mb-2">
{gettext(
"Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
)}
</p>
<p class="text-sm">
<.link
href={~p"/settings#custom_fields"}
class="link"
data-testid="custom-fields-link"
>
{gettext("Manage Memberdata")}
</.link>
</p>
</div>
</div>
<div class="mb-4">
<p class="text-sm text-base-content/70 mb-2">
{gettext("Download CSV templates:")}
</p>
<ul class="list-disc list-inside space-y-1">
<li>
<.link
href={~p"/templates/member_import_en.csv"}
download="member_import_en.csv"
class="link link-primary"
>
{gettext("English Template")}
</.link>
</li>
<li>
<.link
href={~p"/templates/member_import_de.csv"}
download="member_import_de.csv"
class="link link-primary"
>
{gettext("German Template")}
</.link>
</li>
</ul>
</div>
<.form
id="csv-upload-form"
for={%{}}
multipart={true}
phx-change="validate_csv_upload"
phx-submit="start_import"
data-testid="csv-upload-form"
>
<div class="form-control">
<label for="csv_file" class="label">
<span class="label-text">
{gettext("CSV File")}
</span>
</label>
<.live_file_input
upload={@uploads.csv_file}
id="csv_file"
class="file-input file-input-bordered w-full"
aria-describedby="csv_file_help"
/>
<label class="label" id="csv_file_help">
<span class="label-text-alt">
{gettext("CSV files only, maximum %{size} MB", size: @csv_import_max_file_size_mb)}
</span>
</label>
</div>
<.button
type="submit"
phx-disable-with={gettext("Starting import...")}
variant="primary"
disabled={
@import_status == :running or
Enum.empty?(@uploads.csv_file.entries) or
@uploads.csv_file.entries |> List.first() |> then(&(&1 && not &1.done?))
}
data-testid="start-import-button"
>
{gettext("Start Import")}
</.button>
</.form>
<%= if @import_status == :running or @import_status == :done do %>
<%= if @import_progress do %>
<div
role="status"
aria-live="polite"
class="mt-4"
data-testid="import-progress-container"
>
<%= if @import_progress.status == :running do %>
<p class="text-sm" data-testid="import-progress-text">
{gettext("Processing chunk %{current} of %{total}...",
current: @import_progress.current_chunk,
total: @import_progress.total_chunks
)}
</p>
<% end %>
<%= if @import_progress.status == :done do %>
<section class="space-y-4" data-testid="import-results-panel">
<h2 class="text-lg font-semibold">
{gettext("Import Results")}
</h2>
<div class="space-y-4">
<div>
<h3 class="text-sm font-semibold mb-2">
{gettext("Summary")}
</h3>
<div class="text-sm space-y-2">
<p>
<.icon
name="hero-check-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Successfully inserted: %{count} member(s)",
count: @import_progress.inserted
)}
</p>
<%= if @import_progress.failed > 0 do %>
<p>
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Failed: %{count} row(s)", count: @import_progress.failed)}
</p>
<% end %>
<%= if @import_progress.errors_truncated? do %>
<p>
<.icon
name="hero-information-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Error list truncated to %{count} entries",
count: @max_errors
)}
</p>
<% end %>
</div>
</div>
<%= if length(@import_progress.errors) > 0 do %>
<div data-testid="import-error-list">
<h3 class="text-sm font-semibold mb-2">
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Errors")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for error <- @import_progress.errors do %>
<li>
{gettext("Line %{line}: %{message}",
line: error.csv_line_number || "?",
message: error.message || gettext("Unknown error")
)}
<%= if error.field do %>
{gettext(" (Field: %{field})", field: error.field)}
<% end %>
</li>
<% end %>
</ul>
</div>
<% end %>
<%= if length(@import_progress.warnings) > 0 do %>
<div class="alert alert-warning">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<h3 class="font-semibold mb-2">
{gettext("Warnings")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for warning <- @import_progress.warnings do %>
<li>{warning}</li>
<% end %>
</ul>
</div>
</div>
<% end %>
</div>
</section>
<% end %>
</div>
<% end %>
<% end %>
</.form_section>
<%!-- Export Section (Placeholder) --%>
<.form_section title={gettext("Export Members (CSV)")}>
<div role="note" class="alert alert-info">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<p class="text-sm">
{gettext("Export functionality will be available in a future release.")}
</p>
</div>
</div>
</.form_section>
<% else %>
<div role="alert" class="alert alert-error">
<.icon name="hero-exclamation-circle" class="size-5" aria-hidden="true" />
<div>
<p>{gettext("You do not have permission to access this page.")}</p>
</div>
</div>
<% end %>
</Layouts.app>
"""
end
@impl true
def handle_event("validate_csv_upload", _params, socket) do
{:noreply, socket}
end
@impl true
def handle_event("start_import", _params, socket) do
case check_import_prerequisites(socket) do
{:error, message} ->
{:noreply, put_flash(socket, :error, message)}
:ok ->
process_csv_upload(socket)
end
end
# Checks if import can be started (admin permission, status, upload ready)
defp check_import_prerequisites(socket) do
# Ensure user role is loaded before authorization check
user = socket.assigns[:current_user]
user_with_role = Actor.ensure_loaded(user)
cond do
not Authorization.can?(user_with_role, :create, Mv.Membership.Member) ->
{:error, gettext("Only administrators can import members from CSV files.")}
socket.assigns.import_status == :running ->
{:error, gettext("Import is already running. Please wait for it to complete.")}
Enum.empty?(socket.assigns.uploads.csv_file.entries) ->
{:error, gettext("Please select a CSV file to import.")}
not List.first(socket.assigns.uploads.csv_file.entries).done? ->
{:error,
gettext("Please wait for the file upload to complete before starting the import.")}
true ->
:ok
end
end
# Processes CSV upload and starts import
defp process_csv_upload(socket) do
actor = MvWeb.LiveHelpers.current_actor(socket)
with {:ok, content} <- consume_and_read_csv(socket),
{:ok, import_state} <-
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
start_import(socket, import_state)
else
{:error, reason} when is_binary(reason) ->
{:noreply,
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{reason}", reason: reason)
)}
{:error, error} ->
error_message = format_error_message(error)
{:noreply,
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{error}", error: error_message)
)}
end
end
# Starts the import process
defp start_import(socket, import_state) do
progress = initialize_import_progress(import_state)
socket =
socket
|> assign(:import_state, import_state)
|> assign(:import_progress, progress)
|> assign(:import_status, :running)
send(self(), {:process_chunk, 0})
{:noreply, socket}
end
# Initializes import progress structure
defp initialize_import_progress(import_state) do
%{
inserted: 0,
failed: 0,
errors: [],
warnings: import_state.warnings || [],
status: :running,
current_chunk: 0,
total_chunks: length(import_state.chunks),
errors_truncated?: false
}
end
# Formats error messages for display
defp format_error_message(error) do
case error do
%{message: msg} when is_binary(msg) -> msg
%{errors: errors} when is_list(errors) -> inspect(errors)
reason when is_binary(reason) -> reason
other -> inspect(other)
end
end
@impl true
def handle_info({:process_chunk, idx}, socket) do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
if idx >= 0 and idx < length(import_state.chunks) do
start_chunk_processing_task(socket, import_state, progress, idx)
else
handle_chunk_error(socket, :invalid_index, idx)
end
_ ->
# Missing required assigns - mark as error
handle_chunk_error(socket, :missing_state, idx)
end
end
@impl true
def handle_info({:chunk_done, idx, result}, socket) do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
handle_chunk_result(socket, import_state, progress, idx, result)
_ ->
# Missing required assigns - mark as error
handle_chunk_error(socket, :missing_state, idx)
end
end
@impl true
def handle_info({:chunk_error, idx, reason}, socket) do
handle_chunk_error(socket, :processing_failed, idx, reason)
end
# Starts async task to process a chunk
# In tests (SQL sandbox mode), runs synchronously to avoid Ecto Sandbox issues
defp start_chunk_processing_task(socket, import_state, progress, idx) do
chunk = Enum.at(import_state.chunks, idx)
# Ensure user role is loaded before using as actor
user = socket.assigns[:current_user]
actor = Actor.ensure_loaded(user)
live_view_pid = self()
# Process chunk with existing error count for capping
opts = [
custom_field_lookup: import_state.custom_field_lookup,
existing_error_count: length(progress.errors),
max_errors: @max_errors,
actor: actor
]
# Get locale from socket for translations in background tasks
locale = socket.assigns[:locale] || "de"
Gettext.put_locale(MvWeb.Gettext, locale)
if Config.sql_sandbox?() do
# Run synchronously in tests to avoid Ecto Sandbox issues with async tasks
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
# In test mode, send the message - it will be processed when render() is called
# in the test. The test helper wait_for_import_completion() handles message processing
send(live_view_pid, {:chunk_done, idx, chunk_result})
else
# Start async task to process chunk in production
# Use start_child for fire-and-forget: no monitor, no Task messages
# We only use our own send/2 messages for communication
Task.Supervisor.start_child(Mv.TaskSupervisor, fn ->
# Set locale in task process for translations
Gettext.put_locale(MvWeb.Gettext, locale)
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
send(live_view_pid, {:chunk_done, idx, chunk_result})
end)
end
{:noreply, socket}
end
# Handles chunk processing result from async task
defp handle_chunk_result(socket, import_state, progress, idx, chunk_result) do
# Merge progress
new_progress = merge_progress(progress, chunk_result, idx)
socket =
socket
|> assign(:import_progress, new_progress)
|> assign(:import_status, new_progress.status)
# Schedule next chunk or mark as done
socket = schedule_next_chunk(socket, idx, length(import_state.chunks))
{:noreply, socket}
end
# Handles chunk processing errors
defp handle_chunk_error(socket, error_type, idx, reason \\ nil) do
error_message =
case error_type do
:invalid_index ->
gettext("Invalid chunk index: %{idx}", idx: idx)
:missing_state ->
gettext("Import state is missing. Cannot process chunk %{idx}.", idx: idx)
:processing_failed ->
gettext("Failed to process chunk %{idx}: %{reason}",
idx: idx,
reason: inspect(reason)
)
end
socket =
socket
|> assign(:import_status, :error)
|> put_flash(:error, error_message)
{:noreply, socket}
end
defp consume_and_read_csv(socket) do
result =
consume_uploaded_entries(socket, :csv_file, fn %{path: path}, _entry ->
case File.read(path) do
{:ok, content} -> {:ok, content}
{:error, reason} -> {:error, Exception.message(reason)}
end
end)
result
|> case do
[content] when is_binary(content) ->
{:ok, content}
[{:ok, content}] when is_binary(content) ->
{:ok, content}
[{:error, reason}] ->
{:error, gettext("Failed to read file: %{reason}", reason: reason)}
[] ->
{:error, gettext("No file was uploaded")}
_other ->
{:error, gettext("Failed to read uploaded file")}
end
end
defp merge_progress(progress, chunk_result, current_chunk_idx) do
# Merge errors with cap of @max_errors overall
all_errors = progress.errors ++ chunk_result.errors
new_errors = Enum.take(all_errors, @max_errors)
errors_truncated? = length(all_errors) > @max_errors
# Merge warnings (optional dedupe - simple append for now)
new_warnings = progress.warnings ++ Map.get(chunk_result, :warnings, [])
# Update status based on whether we're done
# current_chunk_idx is 0-based, so after processing chunk 0, we've processed 1 chunk
chunks_processed = current_chunk_idx + 1
new_status = if chunks_processed >= progress.total_chunks, do: :done, else: :running
%{
inserted: progress.inserted + chunk_result.inserted,
failed: progress.failed + chunk_result.failed,
errors: new_errors,
warnings: new_warnings,
status: new_status,
current_chunk: chunks_processed,
total_chunks: progress.total_chunks,
errors_truncated?: errors_truncated? || chunk_result.errors_truncated?
}
end
defp schedule_next_chunk(socket, current_idx, total_chunks) do
next_idx = current_idx + 1
if next_idx < total_chunks do
# Schedule next chunk
send(self(), {:process_chunk, next_idx})
socket
else
# All chunks processed - status already set to :done in merge_progress
socket
end
end
end

3
lib/mv_web/router.ex
View file

@ -88,6 +88,9 @@ defmodule MvWeb.Router do
live "/admin/roles/:id", RoleLive.Show, :show
live "/admin/roles/:id/edit", RoleLive.Form, :edit
# Import/Export (Admin only)
live "/admin/import-export", ImportExportLive
post "/set_locale", LocaleController, :set_locale
end