464 lines
15 KiB
Elixir
464 lines
15 KiB
Elixir
defmodule MvWeb.ImportExportLive do
|
|
@moduledoc """
|
|
LiveView for importing and exporting members via CSV.
|
|
|
|
## Features
|
|
- CSV member import (admin only)
|
|
- Real-time import progress tracking
|
|
- Error and warning reporting
|
|
- Custom fields support
|
|
|
|
## CSV Import
|
|
|
|
The CSV import feature allows administrators to upload CSV files and import members.
|
|
|
|
### File Upload
|
|
|
|
Files are uploaded automatically when selected (`auto_upload: true`). No manual
|
|
upload trigger is required.
|
|
|
|
### Rate Limiting
|
|
|
|
Currently, there is no rate limiting for CSV imports. Administrators can start
|
|
multiple imports in quick succession. This is intentional for bulk data migration
|
|
scenarios, but should be monitored in production.
|
|
|
|
### Limits
|
|
|
|
- Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
|
|
- Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
|
|
- Processing: chunks of 200 rows
|
|
- Errors: capped at 50 per import
|
|
"""
|
|
use MvWeb, :live_view
|
|
|
|
alias Mv.Authorization.Actor
|
|
alias Mv.Config
|
|
alias Mv.Membership
|
|
alias Mv.Membership.Import.ImportRunner
|
|
alias Mv.Membership.Import.MemberCSV
|
|
alias MvWeb.Authorization
|
|
alias MvWeb.ImportExportLive.Components
|
|
|
|
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
|
|
|
|
# Maximum number of errors to collect per import to prevent memory issues
|
|
# and keep error display manageable. Additional errors are silently dropped
|
|
# after this limit is reached.
|
|
@max_errors 50
|
|
|
|
# Maximum length for error messages before truncation
|
|
@max_error_message_length 200
|
|
|
|
@impl true
|
|
def mount(_params, session, socket) do
|
|
# Get locale from session for translations
|
|
locale = session["locale"] || "de"
|
|
Gettext.put_locale(MvWeb.Gettext, locale)
|
|
|
|
# Get club name from settings
|
|
club_name =
|
|
case Membership.get_settings() do
|
|
{:ok, settings} -> settings.club_name
|
|
_ -> "Mitgliederverwaltung"
|
|
end
|
|
|
|
socket =
|
|
socket
|
|
|> assign(:page_title, gettext("Import/Export"))
|
|
|> assign(:club_name, club_name)
|
|
|> assign(:import_state, nil)
|
|
|> assign(:import_progress, nil)
|
|
|> assign(:import_status, :idle)
|
|
|> assign(:locale, locale)
|
|
|> assign(:max_errors, @max_errors)
|
|
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|
|
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
|
|
# Configure file upload with auto-upload enabled
|
|
# Files are uploaded automatically when selected, no need for manual trigger
|
|
|> allow_upload(:csv_file,
|
|
accept: ~w(.csv),
|
|
max_entries: 1,
|
|
max_file_size: Config.csv_import_max_file_size_bytes(),
|
|
auto_upload: true
|
|
)
|
|
|
|
{:ok, socket}
|
|
end
|
|
|
|
@impl true
|
|
def render(assigns) do
|
|
~H"""
|
|
<Layouts.app flash={@flash} current_user={@current_user} club_name={@club_name}>
|
|
<.header>
|
|
{gettext("Import/Export")}
|
|
<:subtitle>
|
|
{gettext("Import members from CSV files or export member data.")}
|
|
</:subtitle>
|
|
</.header>
|
|
|
|
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
|
|
<%!-- CSV Import Section --%>
|
|
<.form_section title={gettext("Import Members (CSV)")}>
|
|
<Components.custom_fields_notice {assigns} />
|
|
<Components.template_links {assigns} />
|
|
<Components.import_form {assigns} />
|
|
<%= if @import_status == :running or @import_status == :done or @import_status == :error do %>
|
|
<Components.import_progress {assigns} />
|
|
<% end %>
|
|
</.form_section>
|
|
|
|
<%!-- Export Section (Placeholder) --%>
|
|
<.form_section title={gettext("Export Members (CSV)")}>
|
|
<div role="note" class="alert alert-info">
|
|
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
|
|
<div>
|
|
<p class="text-sm">
|
|
{gettext("Export functionality will be available in a future release.")}
|
|
</p>
|
|
</div>
|
|
</div>
|
|
</.form_section>
|
|
<% else %>
|
|
<div role="alert" class="alert alert-error">
|
|
<.icon name="hero-exclamation-circle" class="size-5" aria-hidden="true" />
|
|
<div>
|
|
<p>{gettext("You do not have permission to access this page.")}</p>
|
|
</div>
|
|
</div>
|
|
<% end %>
|
|
</Layouts.app>
|
|
"""
|
|
end
|
|
|
|
@impl true
|
|
def handle_event("validate_csv_upload", _params, socket) do
|
|
{:noreply, socket}
|
|
end
|
|
|
|
@impl true
|
|
def handle_event("start_import", _params, socket) do
|
|
case check_import_prerequisites(socket) do
|
|
{:error, message} ->
|
|
{:noreply, put_flash(socket, :error, message)}
|
|
|
|
:ok ->
|
|
process_csv_upload(socket)
|
|
end
|
|
end
|
|
|
|
# Checks if all prerequisites for starting an import are met.
|
|
#
|
|
# Validates:
|
|
# - User has admin permissions
|
|
# - No import is currently running
|
|
# - CSV file is uploaded and ready
|
|
#
|
|
# Returns `:ok` if all checks pass, `{:error, message}` otherwise.
|
|
#
|
|
# Note: on_mount :ensure_user_role_loaded already guarantees the role is loaded,
|
|
# so ensure_actor_loaded is primarily for clarity.
|
|
@spec check_import_prerequisites(Phoenix.LiveView.Socket.t()) ::
|
|
:ok | {:error, String.t()}
|
|
defp check_import_prerequisites(socket) do
|
|
# on_mount already ensures role is loaded, but we keep this for clarity
|
|
user_with_role = ensure_actor_loaded(socket)
|
|
|
|
cond do
|
|
not Authorization.can?(user_with_role, :create, Mv.Membership.Member) ->
|
|
{:error, gettext("Only administrators can import members from CSV files.")}
|
|
|
|
socket.assigns.import_status == :running ->
|
|
{:error, gettext("Import is already running. Please wait for it to complete.")}
|
|
|
|
Enum.empty?(socket.assigns.uploads.csv_file.entries) ->
|
|
{:error, gettext("Please select a CSV file to import.")}
|
|
|
|
not List.first(socket.assigns.uploads.csv_file.entries).done? ->
|
|
{:error,
|
|
gettext("Please wait for the file upload to complete before starting the import.")}
|
|
|
|
true ->
|
|
:ok
|
|
end
|
|
end
|
|
|
|
# Processes CSV upload and starts import process.
|
|
#
|
|
# Reads the uploaded CSV file, prepares it for import, and initiates
|
|
# the chunked processing workflow.
|
|
@spec process_csv_upload(Phoenix.LiveView.Socket.t()) ::
|
|
{:noreply, Phoenix.LiveView.Socket.t()}
|
|
defp process_csv_upload(socket) do
|
|
actor = MvWeb.LiveHelpers.current_actor(socket)
|
|
|
|
with {:ok, content} <- consume_and_read_csv(socket),
|
|
{:ok, import_state} <-
|
|
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
|
|
start_import(socket, import_state)
|
|
else
|
|
{:error, reason} when is_binary(reason) ->
|
|
{:noreply,
|
|
put_flash(
|
|
socket,
|
|
:error,
|
|
gettext("Failed to prepare CSV import: %{reason}", reason: reason)
|
|
)}
|
|
|
|
{:error, error} ->
|
|
error_message = format_error_message(error)
|
|
|
|
{:noreply,
|
|
put_flash(
|
|
socket,
|
|
:error,
|
|
gettext("Failed to prepare CSV import: %{reason}", reason: error_message)
|
|
)}
|
|
end
|
|
end
|
|
|
|
# Starts the import process by initializing progress tracking and scheduling the first chunk.
|
|
@spec start_import(Phoenix.LiveView.Socket.t(), map()) ::
|
|
{:noreply, Phoenix.LiveView.Socket.t()}
|
|
defp start_import(socket, import_state) do
|
|
progress = ImportRunner.initial_progress(import_state, max_errors: @max_errors)
|
|
|
|
socket =
|
|
socket
|
|
|> assign(:import_state, import_state)
|
|
|> assign(:import_progress, progress)
|
|
|> assign(:import_status, :running)
|
|
|
|
send(self(), {:process_chunk, 0})
|
|
|
|
{:noreply, socket}
|
|
end
|
|
|
|
# Formats error messages for user-friendly display.
|
|
#
|
|
# Handles various error types including Ash errors, maps with message fields,
|
|
# lists of errors, and fallback formatting for unknown types.
|
|
@spec format_error_message(any()) :: String.t()
|
|
defp format_error_message(error) do
|
|
case error do
|
|
%Ash.Error.Invalid{} = ash_error ->
|
|
format_ash_error(ash_error)
|
|
|
|
%{message: msg} when is_binary(msg) ->
|
|
msg
|
|
|
|
%{errors: errors} when is_list(errors) ->
|
|
format_error_list(errors)
|
|
|
|
reason when is_binary(reason) ->
|
|
reason
|
|
|
|
other ->
|
|
format_unknown_error(other)
|
|
end
|
|
end
|
|
|
|
# Formats Ash validation errors for display
|
|
defp format_ash_error(%Ash.Error.Invalid{errors: errors}) when is_list(errors) do
|
|
Enum.map_join(errors, ", ", &format_single_error/1)
|
|
end
|
|
|
|
defp format_ash_error(error) do
|
|
format_unknown_error(error)
|
|
end
|
|
|
|
# Formats a list of errors into a readable string
|
|
defp format_error_list(errors) do
|
|
Enum.map_join(errors, ", ", &format_single_error/1)
|
|
end
|
|
|
|
# Formats a single error item
|
|
defp format_single_error(error) when is_map(error) do
|
|
Map.get(error, :message) || Map.get(error, :field) || inspect(error, limit: :infinity)
|
|
end
|
|
|
|
defp format_single_error(error) do
|
|
to_string(error)
|
|
end
|
|
|
|
# Formats unknown error types with truncation for very long messages
|
|
defp format_unknown_error(other) do
|
|
error_str = inspect(other, limit: :infinity, pretty: true)
|
|
|
|
if String.length(error_str) > @max_error_message_length do
|
|
String.slice(error_str, 0, @max_error_message_length - 3) <> "..."
|
|
else
|
|
error_str
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def handle_info({:process_chunk, idx}, socket) do
|
|
case socket.assigns do
|
|
%{import_state: import_state, import_progress: progress}
|
|
when is_map(import_state) and is_map(progress) ->
|
|
if idx < length(import_state.chunks) do
|
|
start_chunk_processing_task(socket, import_state, progress, idx)
|
|
else
|
|
handle_chunk_error(socket, :invalid_index, idx)
|
|
end
|
|
|
|
_ ->
|
|
# Missing required assigns - mark as error
|
|
handle_chunk_error(socket, :missing_state, idx)
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def handle_info({:chunk_done, idx, result}, socket) do
|
|
case socket.assigns do
|
|
%{import_state: import_state, import_progress: progress}
|
|
when is_map(import_state) and is_map(progress) ->
|
|
handle_chunk_result(socket, import_state, progress, idx, result)
|
|
|
|
_ ->
|
|
# Missing required assigns - mark as error
|
|
handle_chunk_error(socket, :missing_state, idx)
|
|
end
|
|
end
|
|
|
|
@impl true
|
|
def handle_info({:chunk_error, idx, reason}, socket) do
|
|
handle_chunk_error(socket, :processing_failed, idx, reason)
|
|
end
|
|
|
|
# Starts async task to process a chunk of CSV rows (or runs synchronously in test sandbox).
|
|
# Locale must be set in the process that runs the chunk (Gettext is process-local); see run_chunk_with_locale/7.
|
|
@spec start_chunk_processing_task(
|
|
Phoenix.LiveView.Socket.t(),
|
|
map(),
|
|
map(),
|
|
non_neg_integer()
|
|
) :: {:noreply, Phoenix.LiveView.Socket.t()}
|
|
defp start_chunk_processing_task(socket, import_state, progress, idx) do
|
|
chunk = Enum.at(import_state.chunks, idx)
|
|
actor = ensure_actor_loaded(socket)
|
|
live_view_pid = self()
|
|
locale = socket.assigns[:locale] || "de"
|
|
|
|
opts = [
|
|
custom_field_lookup: import_state.custom_field_lookup,
|
|
existing_error_count: length(progress.errors),
|
|
max_errors: @max_errors,
|
|
actor: actor
|
|
]
|
|
|
|
if Config.sql_sandbox?() do
|
|
run_chunk_with_locale(
|
|
locale,
|
|
chunk,
|
|
import_state.column_map,
|
|
import_state.custom_field_map,
|
|
opts,
|
|
live_view_pid,
|
|
idx
|
|
)
|
|
else
|
|
Task.Supervisor.start_child(
|
|
Mv.TaskSupervisor,
|
|
fn ->
|
|
run_chunk_with_locale(
|
|
locale,
|
|
chunk,
|
|
import_state.column_map,
|
|
import_state.custom_field_map,
|
|
opts,
|
|
live_view_pid,
|
|
idx
|
|
)
|
|
end
|
|
)
|
|
end
|
|
|
|
{:noreply, socket}
|
|
end
|
|
|
|
# Sets Gettext locale in the current process, then processes the chunk.
|
|
# Must be called in the process that runs the chunk (sync: LiveView process; async: Task process).
|
|
defp run_chunk_with_locale(
|
|
locale,
|
|
chunk,
|
|
column_map,
|
|
custom_field_map,
|
|
opts,
|
|
live_view_pid,
|
|
idx
|
|
) do
|
|
Gettext.put_locale(MvWeb.Gettext, locale)
|
|
ImportRunner.process_chunk(chunk, column_map, custom_field_map, opts, live_view_pid, idx)
|
|
end
|
|
|
|
# Handles chunk processing result from async task and schedules the next chunk.
|
|
@spec handle_chunk_result(
|
|
Phoenix.LiveView.Socket.t(),
|
|
map(),
|
|
map(),
|
|
non_neg_integer(),
|
|
map()
|
|
) :: {:noreply, Phoenix.LiveView.Socket.t()}
|
|
defp handle_chunk_result(socket, import_state, progress, idx, chunk_result) do
|
|
new_progress =
|
|
ImportRunner.merge_progress(progress, chunk_result, idx, max_errors: @max_errors)
|
|
|
|
socket =
|
|
socket
|
|
|> assign(:import_progress, new_progress)
|
|
|> assign(:import_status, new_progress.status)
|
|
|> maybe_send_next_chunk(idx, length(import_state.chunks))
|
|
|
|
{:noreply, socket}
|
|
end
|
|
|
|
defp maybe_send_next_chunk(socket, current_idx, total_chunks) do
|
|
case ImportRunner.next_chunk_action(current_idx, total_chunks) do
|
|
{:send_chunk, next_idx} ->
|
|
send(self(), {:process_chunk, next_idx})
|
|
socket
|
|
|
|
:done ->
|
|
socket
|
|
end
|
|
end
|
|
|
|
# Handles chunk processing errors and updates socket with error status.
|
|
@spec handle_chunk_error(
|
|
Phoenix.LiveView.Socket.t(),
|
|
:invalid_index | :missing_state | :processing_failed,
|
|
non_neg_integer(),
|
|
any()
|
|
) :: {:noreply, Phoenix.LiveView.Socket.t()}
|
|
defp handle_chunk_error(socket, error_type, idx, reason \\ nil) do
|
|
message = ImportRunner.format_chunk_error(error_type, idx, reason)
|
|
|
|
socket =
|
|
socket
|
|
|> assign(:import_status, :error)
|
|
|> put_flash(:error, message)
|
|
|
|
{:noreply, socket}
|
|
end
|
|
|
|
# Consumes uploaded CSV file entries and reads the file content.
|
|
@spec consume_and_read_csv(Phoenix.LiveView.Socket.t()) ::
|
|
{:ok, String.t()} | {:error, String.t()}
|
|
defp consume_and_read_csv(socket) do
|
|
raw = consume_uploaded_entries(socket, :csv_file, &ImportRunner.read_file_entry/2)
|
|
ImportRunner.parse_consume_result(raw)
|
|
end
|
|
|
|
# Ensures the actor (user with role) is loaded from socket assigns.
|
|
#
|
|
# Note: on_mount :ensure_user_role_loaded already guarantees the role is loaded,
|
|
# so this is primarily for clarity and defensive programming.
|
|
@spec ensure_actor_loaded(Phoenix.LiveView.Socket.t()) :: Mv.Accounts.User.t() | nil
|
|
defp ensure_actor_loaded(socket) do
|
|
user = socket.assigns[:current_user]
|
|
# on_mount already ensures role is loaded, but we keep this for clarity
|
|
Actor.ensure_loaded(user)
|
|
end
|
|
end
|