chore: rename ImportExport module to Import
This commit is contained in:
parent
ce542eae3e
commit
b18f895939
10 changed files with 161 additions and 197 deletions
|
|
@ -1,464 +0,0 @@
|
|||
defmodule MvWeb.ImportExportLive do
|
||||
@moduledoc """
|
||||
LiveView for importing and exporting members via CSV.
|
||||
|
||||
## Features
|
||||
- CSV member import (admin only)
|
||||
- Real-time import progress tracking
|
||||
- Error and warning reporting
|
||||
- Custom fields support
|
||||
|
||||
## CSV Import
|
||||
|
||||
The CSV import feature allows administrators to upload CSV files and import members.
|
||||
|
||||
### File Upload
|
||||
|
||||
Files are uploaded automatically when selected (`auto_upload: true`). No manual
|
||||
upload trigger is required.
|
||||
|
||||
### Rate Limiting
|
||||
|
||||
Currently, there is no rate limiting for CSV imports. Administrators can start
|
||||
multiple imports in quick succession. This is intentional for bulk data migration
|
||||
scenarios, but should be monitored in production.
|
||||
|
||||
### Limits
|
||||
|
||||
- Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
|
||||
- Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
|
||||
- Processing: chunks of 200 rows
|
||||
- Errors: capped at 50 per import
|
||||
"""
|
||||
use MvWeb, :live_view
|
||||
|
||||
alias Mv.Authorization.Actor
|
||||
alias Mv.Config
|
||||
alias Mv.Membership
|
||||
alias Mv.Membership.Import.ImportRunner
|
||||
alias Mv.Membership.Import.MemberCSV
|
||||
alias MvWeb.Authorization
|
||||
alias MvWeb.ImportExportLive.Components
|
||||
|
||||
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
|
||||
|
||||
# Maximum number of errors to collect per import to prevent memory issues
|
||||
# and keep error display manageable. Additional errors are silently dropped
|
||||
# after this limit is reached.
|
||||
@max_errors 50
|
||||
|
||||
# Maximum length for error messages before truncation
|
||||
@max_error_message_length 200
|
||||
|
||||
@impl true
|
||||
def mount(_params, session, socket) do
|
||||
# Get locale from session for translations
|
||||
locale = session["locale"] || "de"
|
||||
Gettext.put_locale(MvWeb.Gettext, locale)
|
||||
|
||||
# Get club name from settings
|
||||
club_name =
|
||||
case Membership.get_settings() do
|
||||
{:ok, settings} -> settings.club_name
|
||||
_ -> "Mitgliederverwaltung"
|
||||
end
|
||||
|
||||
socket =
|
||||
socket
|
||||
|> assign(:page_title, gettext("Import/Export"))
|
||||
|> assign(:club_name, club_name)
|
||||
|> assign(:import_state, nil)
|
||||
|> assign(:import_progress, nil)
|
||||
|> assign(:import_status, :idle)
|
||||
|> assign(:locale, locale)
|
||||
|> assign(:max_errors, @max_errors)
|
||||
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|
||||
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
|
||||
# Configure file upload with auto-upload enabled
|
||||
# Files are uploaded automatically when selected, no need for manual trigger
|
||||
|> allow_upload(:csv_file,
|
||||
accept: ~w(.csv),
|
||||
max_entries: 1,
|
||||
max_file_size: Config.csv_import_max_file_size_bytes(),
|
||||
auto_upload: true
|
||||
)
|
||||
|
||||
{:ok, socket}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def render(assigns) do
|
||||
~H"""
|
||||
<Layouts.app flash={@flash} current_user={@current_user} club_name={@club_name}>
|
||||
<.header>
|
||||
{gettext("Import/Export")}
|
||||
<:subtitle>
|
||||
{gettext("Import members from CSV files or export member data.")}
|
||||
</:subtitle>
|
||||
</.header>
|
||||
|
||||
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
|
||||
<%!-- CSV Import Section --%>
|
||||
<.form_section title={gettext("Import Members (CSV)")}>
|
||||
<Components.custom_fields_notice {assigns} />
|
||||
<Components.template_links {assigns} />
|
||||
<Components.import_form {assigns} />
|
||||
<%= if @import_status == :running or @import_status == :done or @import_status == :error do %>
|
||||
<Components.import_progress {assigns} />
|
||||
<% end %>
|
||||
</.form_section>
|
||||
|
||||
<%!-- Export Section (Placeholder) --%>
|
||||
<.form_section title={gettext("Export Members (CSV)")}>
|
||||
<div role="note" class="alert alert-info">
|
||||
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
|
||||
<div>
|
||||
<p class="text-sm">
|
||||
{gettext("Export functionality will be available in a future release.")}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</.form_section>
|
||||
<% else %>
|
||||
<div role="alert" class="alert alert-error">
|
||||
<.icon name="hero-exclamation-circle" class="size-5" aria-hidden="true" />
|
||||
<div>
|
||||
<p>{gettext("You do not have permission to access this page.")}</p>
|
||||
</div>
|
||||
</div>
|
||||
<% end %>
|
||||
</Layouts.app>
|
||||
"""
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_event("validate_csv_upload", _params, socket) do
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_event("start_import", _params, socket) do
|
||||
case check_import_prerequisites(socket) do
|
||||
{:error, message} ->
|
||||
{:noreply, put_flash(socket, :error, message)}
|
||||
|
||||
:ok ->
|
||||
process_csv_upload(socket)
|
||||
end
|
||||
end
|
||||
|
||||
# Checks if all prerequisites for starting an import are met.
|
||||
#
|
||||
# Validates:
|
||||
# - User has admin permissions
|
||||
# - No import is currently running
|
||||
# - CSV file is uploaded and ready
|
||||
#
|
||||
# Returns `:ok` if all checks pass, `{:error, message}` otherwise.
|
||||
#
|
||||
# Note: on_mount :ensure_user_role_loaded already guarantees the role is loaded,
|
||||
# so ensure_actor_loaded is primarily for clarity.
|
||||
@spec check_import_prerequisites(Phoenix.LiveView.Socket.t()) ::
|
||||
:ok | {:error, String.t()}
|
||||
defp check_import_prerequisites(socket) do
|
||||
# on_mount already ensures role is loaded, but we keep this for clarity
|
||||
user_with_role = ensure_actor_loaded(socket)
|
||||
|
||||
cond do
|
||||
not Authorization.can?(user_with_role, :create, Mv.Membership.Member) ->
|
||||
{:error, gettext("Only administrators can import members from CSV files.")}
|
||||
|
||||
socket.assigns.import_status == :running ->
|
||||
{:error, gettext("Import is already running. Please wait for it to complete.")}
|
||||
|
||||
Enum.empty?(socket.assigns.uploads.csv_file.entries) ->
|
||||
{:error, gettext("Please select a CSV file to import.")}
|
||||
|
||||
not List.first(socket.assigns.uploads.csv_file.entries).done? ->
|
||||
{:error,
|
||||
gettext("Please wait for the file upload to complete before starting the import.")}
|
||||
|
||||
true ->
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
# Processes CSV upload and starts import process.
|
||||
#
|
||||
# Reads the uploaded CSV file, prepares it for import, and initiates
|
||||
# the chunked processing workflow.
|
||||
@spec process_csv_upload(Phoenix.LiveView.Socket.t()) ::
|
||||
{:noreply, Phoenix.LiveView.Socket.t()}
|
||||
defp process_csv_upload(socket) do
|
||||
actor = MvWeb.LiveHelpers.current_actor(socket)
|
||||
|
||||
with {:ok, content} <- consume_and_read_csv(socket),
|
||||
{:ok, import_state} <-
|
||||
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
|
||||
start_import(socket, import_state)
|
||||
else
|
||||
{:error, reason} when is_binary(reason) ->
|
||||
{:noreply,
|
||||
put_flash(
|
||||
socket,
|
||||
:error,
|
||||
gettext("Failed to prepare CSV import: %{reason}", reason: reason)
|
||||
)}
|
||||
|
||||
{:error, error} ->
|
||||
error_message = format_error_message(error)
|
||||
|
||||
{:noreply,
|
||||
put_flash(
|
||||
socket,
|
||||
:error,
|
||||
gettext("Failed to prepare CSV import: %{reason}", reason: error_message)
|
||||
)}
|
||||
end
|
||||
end
|
||||
|
||||
# Starts the import process by initializing progress tracking and scheduling the first chunk.
|
||||
@spec start_import(Phoenix.LiveView.Socket.t(), map()) ::
|
||||
{:noreply, Phoenix.LiveView.Socket.t()}
|
||||
defp start_import(socket, import_state) do
|
||||
progress = ImportRunner.initial_progress(import_state, max_errors: @max_errors)
|
||||
|
||||
socket =
|
||||
socket
|
||||
|> assign(:import_state, import_state)
|
||||
|> assign(:import_progress, progress)
|
||||
|> assign(:import_status, :running)
|
||||
|
||||
send(self(), {:process_chunk, 0})
|
||||
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
# Formats error messages for user-friendly display.
|
||||
#
|
||||
# Handles various error types including Ash errors, maps with message fields,
|
||||
# lists of errors, and fallback formatting for unknown types.
|
||||
@spec format_error_message(any()) :: String.t()
|
||||
defp format_error_message(error) do
|
||||
case error do
|
||||
%Ash.Error.Invalid{} = ash_error ->
|
||||
format_ash_error(ash_error)
|
||||
|
||||
%{message: msg} when is_binary(msg) ->
|
||||
msg
|
||||
|
||||
%{errors: errors} when is_list(errors) ->
|
||||
format_error_list(errors)
|
||||
|
||||
reason when is_binary(reason) ->
|
||||
reason
|
||||
|
||||
other ->
|
||||
format_unknown_error(other)
|
||||
end
|
||||
end
|
||||
|
||||
# Formats Ash validation errors for display
|
||||
defp format_ash_error(%Ash.Error.Invalid{errors: errors}) when is_list(errors) do
|
||||
Enum.map_join(errors, ", ", &format_single_error/1)
|
||||
end
|
||||
|
||||
defp format_ash_error(error) do
|
||||
format_unknown_error(error)
|
||||
end
|
||||
|
||||
# Formats a list of errors into a readable string
|
||||
defp format_error_list(errors) do
|
||||
Enum.map_join(errors, ", ", &format_single_error/1)
|
||||
end
|
||||
|
||||
# Formats a single error item
|
||||
defp format_single_error(error) when is_map(error) do
|
||||
Map.get(error, :message) || Map.get(error, :field) || inspect(error, limit: :infinity)
|
||||
end
|
||||
|
||||
defp format_single_error(error) do
|
||||
to_string(error)
|
||||
end
|
||||
|
||||
# Formats unknown error types with truncation for very long messages
|
||||
defp format_unknown_error(other) do
|
||||
error_str = inspect(other, limit: :infinity, pretty: true)
|
||||
|
||||
if String.length(error_str) > @max_error_message_length do
|
||||
String.slice(error_str, 0, @max_error_message_length - 3) <> "..."
|
||||
else
|
||||
error_str
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:process_chunk, idx}, socket) do
|
||||
case socket.assigns do
|
||||
%{import_state: import_state, import_progress: progress}
|
||||
when is_map(import_state) and is_map(progress) ->
|
||||
if idx < length(import_state.chunks) do
|
||||
start_chunk_processing_task(socket, import_state, progress, idx)
|
||||
else
|
||||
handle_chunk_error(socket, :invalid_index, idx)
|
||||
end
|
||||
|
||||
_ ->
|
||||
# Missing required assigns - mark as error
|
||||
handle_chunk_error(socket, :missing_state, idx)
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:chunk_done, idx, result}, socket) do
|
||||
case socket.assigns do
|
||||
%{import_state: import_state, import_progress: progress}
|
||||
when is_map(import_state) and is_map(progress) ->
|
||||
handle_chunk_result(socket, import_state, progress, idx, result)
|
||||
|
||||
_ ->
|
||||
# Missing required assigns - mark as error
|
||||
handle_chunk_error(socket, :missing_state, idx)
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:chunk_error, idx, reason}, socket) do
|
||||
handle_chunk_error(socket, :processing_failed, idx, reason)
|
||||
end
|
||||
|
||||
# Starts async task to process a chunk of CSV rows (or runs synchronously in test sandbox).
|
||||
# Locale must be set in the process that runs the chunk (Gettext is process-local); see run_chunk_with_locale/7.
|
||||
@spec start_chunk_processing_task(
|
||||
Phoenix.LiveView.Socket.t(),
|
||||
map(),
|
||||
map(),
|
||||
non_neg_integer()
|
||||
) :: {:noreply, Phoenix.LiveView.Socket.t()}
|
||||
defp start_chunk_processing_task(socket, import_state, progress, idx) do
|
||||
chunk = Enum.at(import_state.chunks, idx)
|
||||
actor = ensure_actor_loaded(socket)
|
||||
live_view_pid = self()
|
||||
locale = socket.assigns[:locale] || "de"
|
||||
|
||||
opts = [
|
||||
custom_field_lookup: import_state.custom_field_lookup,
|
||||
existing_error_count: length(progress.errors),
|
||||
max_errors: @max_errors,
|
||||
actor: actor
|
||||
]
|
||||
|
||||
if Config.sql_sandbox?() do
|
||||
run_chunk_with_locale(
|
||||
locale,
|
||||
chunk,
|
||||
import_state.column_map,
|
||||
import_state.custom_field_map,
|
||||
opts,
|
||||
live_view_pid,
|
||||
idx
|
||||
)
|
||||
else
|
||||
Task.Supervisor.start_child(
|
||||
Mv.TaskSupervisor,
|
||||
fn ->
|
||||
run_chunk_with_locale(
|
||||
locale,
|
||||
chunk,
|
||||
import_state.column_map,
|
||||
import_state.custom_field_map,
|
||||
opts,
|
||||
live_view_pid,
|
||||
idx
|
||||
)
|
||||
end
|
||||
)
|
||||
end
|
||||
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
# Sets Gettext locale in the current process, then processes the chunk.
|
||||
# Must be called in the process that runs the chunk (sync: LiveView process; async: Task process).
|
||||
defp run_chunk_with_locale(
|
||||
locale,
|
||||
chunk,
|
||||
column_map,
|
||||
custom_field_map,
|
||||
opts,
|
||||
live_view_pid,
|
||||
idx
|
||||
) do
|
||||
Gettext.put_locale(MvWeb.Gettext, locale)
|
||||
ImportRunner.process_chunk(chunk, column_map, custom_field_map, opts, live_view_pid, idx)
|
||||
end
|
||||
|
||||
# Handles chunk processing result from async task and schedules the next chunk.
|
||||
@spec handle_chunk_result(
|
||||
Phoenix.LiveView.Socket.t(),
|
||||
map(),
|
||||
map(),
|
||||
non_neg_integer(),
|
||||
map()
|
||||
) :: {:noreply, Phoenix.LiveView.Socket.t()}
|
||||
defp handle_chunk_result(socket, import_state, progress, idx, chunk_result) do
|
||||
new_progress =
|
||||
ImportRunner.merge_progress(progress, chunk_result, idx, max_errors: @max_errors)
|
||||
|
||||
socket =
|
||||
socket
|
||||
|> assign(:import_progress, new_progress)
|
||||
|> assign(:import_status, new_progress.status)
|
||||
|> maybe_send_next_chunk(idx, length(import_state.chunks))
|
||||
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
defp maybe_send_next_chunk(socket, current_idx, total_chunks) do
|
||||
case ImportRunner.next_chunk_action(current_idx, total_chunks) do
|
||||
{:send_chunk, next_idx} ->
|
||||
send(self(), {:process_chunk, next_idx})
|
||||
socket
|
||||
|
||||
:done ->
|
||||
socket
|
||||
end
|
||||
end
|
||||
|
||||
# Handles chunk processing errors and updates socket with error status.
|
||||
@spec handle_chunk_error(
|
||||
Phoenix.LiveView.Socket.t(),
|
||||
:invalid_index | :missing_state | :processing_failed,
|
||||
non_neg_integer(),
|
||||
any()
|
||||
) :: {:noreply, Phoenix.LiveView.Socket.t()}
|
||||
defp handle_chunk_error(socket, error_type, idx, reason \\ nil) do
|
||||
message = ImportRunner.format_chunk_error(error_type, idx, reason)
|
||||
|
||||
socket =
|
||||
socket
|
||||
|> assign(:import_status, :error)
|
||||
|> put_flash(:error, message)
|
||||
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
# Consumes uploaded CSV file entries and reads the file content.
|
||||
@spec consume_and_read_csv(Phoenix.LiveView.Socket.t()) ::
|
||||
{:ok, String.t()} | {:error, String.t()}
|
||||
defp consume_and_read_csv(socket) do
|
||||
raw = consume_uploaded_entries(socket, :csv_file, &ImportRunner.read_file_entry/2)
|
||||
ImportRunner.parse_consume_result(raw)
|
||||
end
|
||||
|
||||
# Ensures the actor (user with role) is loaded from socket assigns.
|
||||
#
|
||||
# Note: on_mount :ensure_user_role_loaded already guarantees the role is loaded,
|
||||
# so this is primarily for clarity and defensive programming.
|
||||
@spec ensure_actor_loaded(Phoenix.LiveView.Socket.t()) :: Mv.Accounts.User.t() | nil
|
||||
defp ensure_actor_loaded(socket) do
|
||||
user = socket.assigns[:current_user]
|
||||
# on_mount already ensures role is loaded, but we keep this for clarity
|
||||
Actor.ensure_loaded(user)
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue