mitgliederverwaltung/lib/mv_web/live/import_export_live.ex

464 lines
15 KiB
Elixir

defmodule MvWeb.ImportExportLive do
@moduledoc """
LiveView for importing and exporting members via CSV.
## Features
- CSV member import (admin only)
- Real-time import progress tracking
- Error and warning reporting
- Custom fields support
## CSV Import
The CSV import feature allows administrators to upload CSV files and import members.
### File Upload
Files are uploaded automatically when selected (`auto_upload: true`). No manual
upload trigger is required.
### Rate Limiting
Currently, there is no rate limiting for CSV imports. Administrators can start
multiple imports in quick succession. This is intentional for bulk data migration
scenarios, but should be monitored in production.
### Limits
- Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
- Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
- Processing: chunks of 200 rows
- Errors: capped at 50 per import
"""
use MvWeb, :live_view
alias Mv.Authorization.Actor
alias Mv.Config
alias Mv.Membership
alias Mv.Membership.Import.ImportRunner
alias Mv.Membership.Import.MemberCSV
alias MvWeb.Authorization
alias MvWeb.ImportExportLive.Components
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
# Maximum number of errors to collect per import to prevent memory issues
# and keep error display manageable. Additional errors are silently dropped
# after this limit is reached.
@max_errors 50
# Maximum length for error messages before truncation
@max_error_message_length 200
@impl true
def mount(_params, session, socket) do
# Get locale from session for translations
locale = session["locale"] || "de"
Gettext.put_locale(MvWeb.Gettext, locale)
# Get club name from settings
club_name =
case Membership.get_settings() do
{:ok, settings} -> settings.club_name
_ -> "Mitgliederverwaltung"
end
socket =
socket
|> assign(:page_title, gettext("Import/Export"))
|> assign(:club_name, club_name)
|> assign(:import_state, nil)
|> assign(:import_progress, nil)
|> assign(:import_status, :idle)
|> assign(:locale, locale)
|> assign(:max_errors, @max_errors)
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
# Configure file upload with auto-upload enabled
# Files are uploaded automatically when selected, no need for manual trigger
|> allow_upload(:csv_file,
accept: ~w(.csv),
max_entries: 1,
max_file_size: Config.csv_import_max_file_size_bytes(),
auto_upload: true
)
{:ok, socket}
end
@impl true
def render(assigns) do
~H"""
<Layouts.app flash={@flash} current_user={@current_user} club_name={@club_name}>
<.header>
{gettext("Import/Export")}
<:subtitle>
{gettext("Import members from CSV files or export member data.")}
</:subtitle>
</.header>
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
<%!-- CSV Import Section --%>
<.form_section title={gettext("Import Members (CSV)")}>
<Components.custom_fields_notice {assigns} />
<Components.template_links {assigns} />
<Components.import_form {assigns} />
<%= if @import_status == :running or @import_status == :done or @import_status == :error do %>
<Components.import_progress {assigns} />
<% end %>
</.form_section>
<%!-- Export Section (Placeholder) --%>
<.form_section title={gettext("Export Members (CSV)")}>
<div role="note" class="alert alert-info">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<p class="text-sm">
{gettext("Export functionality will be available in a future release.")}
</p>
</div>
</div>
</.form_section>
<% else %>
<div role="alert" class="alert alert-error">
<.icon name="hero-exclamation-circle" class="size-5" aria-hidden="true" />
<div>
<p>{gettext("You do not have permission to access this page.")}</p>
</div>
</div>
<% end %>
</Layouts.app>
"""
end
@impl true
def handle_event("validate_csv_upload", _params, socket) do
{:noreply, socket}
end
@impl true
def handle_event("start_import", _params, socket) do
case check_import_prerequisites(socket) do
{:error, message} ->
{:noreply, put_flash(socket, :error, message)}
:ok ->
process_csv_upload(socket)
end
end
# Checks if all prerequisites for starting an import are met.
#
# Validates:
# - User has admin permissions
# - No import is currently running
# - CSV file is uploaded and ready
#
# Returns `:ok` if all checks pass, `{:error, message}` otherwise.
#
# Note: on_mount :ensure_user_role_loaded already guarantees the role is loaded,
# so ensure_actor_loaded is primarily for clarity.
@spec check_import_prerequisites(Phoenix.LiveView.Socket.t()) ::
:ok | {:error, String.t()}
defp check_import_prerequisites(socket) do
# on_mount already ensures role is loaded, but we keep this for clarity
user_with_role = ensure_actor_loaded(socket)
cond do
not Authorization.can?(user_with_role, :create, Mv.Membership.Member) ->
{:error, gettext("Only administrators can import members from CSV files.")}
socket.assigns.import_status == :running ->
{:error, gettext("Import is already running. Please wait for it to complete.")}
Enum.empty?(socket.assigns.uploads.csv_file.entries) ->
{:error, gettext("Please select a CSV file to import.")}
not List.first(socket.assigns.uploads.csv_file.entries).done? ->
{:error,
gettext("Please wait for the file upload to complete before starting the import.")}
true ->
:ok
end
end
# Processes CSV upload and starts import process.
#
# Reads the uploaded CSV file, prepares it for import, and initiates
# the chunked processing workflow.
@spec process_csv_upload(Phoenix.LiveView.Socket.t()) ::
{:noreply, Phoenix.LiveView.Socket.t()}
defp process_csv_upload(socket) do
actor = MvWeb.LiveHelpers.current_actor(socket)
with {:ok, content} <- consume_and_read_csv(socket),
{:ok, import_state} <-
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
start_import(socket, import_state)
else
{:error, reason} when is_binary(reason) ->
{:noreply,
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{reason}", reason: reason)
)}
{:error, error} ->
error_message = format_error_message(error)
{:noreply,
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{reason}", reason: error_message)
)}
end
end
# Starts the import process by initializing progress tracking and scheduling the first chunk.
@spec start_import(Phoenix.LiveView.Socket.t(), map()) ::
{:noreply, Phoenix.LiveView.Socket.t()}
defp start_import(socket, import_state) do
progress = ImportRunner.initial_progress(import_state, max_errors: @max_errors)
socket =
socket
|> assign(:import_state, import_state)
|> assign(:import_progress, progress)
|> assign(:import_status, :running)
send(self(), {:process_chunk, 0})
{:noreply, socket}
end
# Formats error messages for user-friendly display.
#
# Handles various error types including Ash errors, maps with message fields,
# lists of errors, and fallback formatting for unknown types.
@spec format_error_message(any()) :: String.t()
defp format_error_message(error) do
case error do
%Ash.Error.Invalid{} = ash_error ->
format_ash_error(ash_error)
%{message: msg} when is_binary(msg) ->
msg
%{errors: errors} when is_list(errors) ->
format_error_list(errors)
reason when is_binary(reason) ->
reason
other ->
format_unknown_error(other)
end
end
# Formats Ash validation errors for display
defp format_ash_error(%Ash.Error.Invalid{errors: errors}) when is_list(errors) do
Enum.map_join(errors, ", ", &format_single_error/1)
end
defp format_ash_error(error) do
format_unknown_error(error)
end
# Formats a list of errors into a readable string
defp format_error_list(errors) do
Enum.map_join(errors, ", ", &format_single_error/1)
end
# Formats a single error item
defp format_single_error(error) when is_map(error) do
Map.get(error, :message) || Map.get(error, :field) || inspect(error, limit: :infinity)
end
defp format_single_error(error) do
to_string(error)
end
# Formats unknown error types with truncation for very long messages
defp format_unknown_error(other) do
error_str = inspect(other, limit: :infinity, pretty: true)
if String.length(error_str) > @max_error_message_length do
String.slice(error_str, 0, @max_error_message_length - 3) <> "..."
else
error_str
end
end
@impl true
def handle_info({:process_chunk, idx}, socket) do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
if idx < length(import_state.chunks) do
start_chunk_processing_task(socket, import_state, progress, idx)
else
handle_chunk_error(socket, :invalid_index, idx)
end
_ ->
# Missing required assigns - mark as error
handle_chunk_error(socket, :missing_state, idx)
end
end
@impl true
def handle_info({:chunk_done, idx, result}, socket) do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
handle_chunk_result(socket, import_state, progress, idx, result)
_ ->
# Missing required assigns - mark as error
handle_chunk_error(socket, :missing_state, idx)
end
end
@impl true
def handle_info({:chunk_error, idx, reason}, socket) do
handle_chunk_error(socket, :processing_failed, idx, reason)
end
# Starts async task to process a chunk of CSV rows (or runs synchronously in test sandbox).
# Locale must be set in the process that runs the chunk (Gettext is process-local); see run_chunk_with_locale/7.
@spec start_chunk_processing_task(
Phoenix.LiveView.Socket.t(),
map(),
map(),
non_neg_integer()
) :: {:noreply, Phoenix.LiveView.Socket.t()}
defp start_chunk_processing_task(socket, import_state, progress, idx) do
chunk = Enum.at(import_state.chunks, idx)
actor = ensure_actor_loaded(socket)
live_view_pid = self()
locale = socket.assigns[:locale] || "de"
opts = [
custom_field_lookup: import_state.custom_field_lookup,
existing_error_count: length(progress.errors),
max_errors: @max_errors,
actor: actor
]
if Config.sql_sandbox?() do
run_chunk_with_locale(
locale,
chunk,
import_state.column_map,
import_state.custom_field_map,
opts,
live_view_pid,
idx
)
else
Task.Supervisor.start_child(
Mv.TaskSupervisor,
fn ->
run_chunk_with_locale(
locale,
chunk,
import_state.column_map,
import_state.custom_field_map,
opts,
live_view_pid,
idx
)
end
)
end
{:noreply, socket}
end
# Sets Gettext locale in the current process, then processes the chunk.
# Must be called in the process that runs the chunk (sync: LiveView process; async: Task process).
defp run_chunk_with_locale(
locale,
chunk,
column_map,
custom_field_map,
opts,
live_view_pid,
idx
) do
Gettext.put_locale(MvWeb.Gettext, locale)
ImportRunner.process_chunk(chunk, column_map, custom_field_map, opts, live_view_pid, idx)
end
# Handles chunk processing result from async task and schedules the next chunk.
@spec handle_chunk_result(
Phoenix.LiveView.Socket.t(),
map(),
map(),
non_neg_integer(),
map()
) :: {:noreply, Phoenix.LiveView.Socket.t()}
defp handle_chunk_result(socket, import_state, progress, idx, chunk_result) do
new_progress =
ImportRunner.merge_progress(progress, chunk_result, idx, max_errors: @max_errors)
socket =
socket
|> assign(:import_progress, new_progress)
|> assign(:import_status, new_progress.status)
|> maybe_send_next_chunk(idx, length(import_state.chunks))
{:noreply, socket}
end
defp maybe_send_next_chunk(socket, current_idx, total_chunks) do
case ImportRunner.next_chunk_action(current_idx, total_chunks) do
{:send_chunk, next_idx} ->
send(self(), {:process_chunk, next_idx})
socket
:done ->
socket
end
end
# Handles chunk processing errors and updates socket with error status.
@spec handle_chunk_error(
Phoenix.LiveView.Socket.t(),
:invalid_index | :missing_state | :processing_failed,
non_neg_integer(),
any()
) :: {:noreply, Phoenix.LiveView.Socket.t()}
defp handle_chunk_error(socket, error_type, idx, reason \\ nil) do
message = ImportRunner.format_chunk_error(error_type, idx, reason)
socket =
socket
|> assign(:import_status, :error)
|> put_flash(:error, message)
{:noreply, socket}
end
# Consumes uploaded CSV file entries and reads the file content.
@spec consume_and_read_csv(Phoenix.LiveView.Socket.t()) ::
{:ok, String.t()} | {:error, String.t()}
defp consume_and_read_csv(socket) do
raw = consume_uploaded_entries(socket, :csv_file, &ImportRunner.read_file_entry/2)
ImportRunner.parse_consume_result(raw)
end
# Ensures the actor (user with role) is loaded from socket assigns.
#
# Note: on_mount :ensure_user_role_loaded already guarantees the role is loaded,
# so this is primarily for clarity and defensive programming.
@spec ensure_actor_loaded(Phoenix.LiveView.Socket.t()) :: Mv.Accounts.User.t() | nil
defp ensure_actor_loaded(socket) do
user = socket.assigns[:current_user]
# on_mount already ensures role is loaded, but we keep this for clarity
Actor.ensure_loaded(user)
end
end