local-it/mitgliederverwaltung
2
3
Fork 0
Code Issues 67 Pull requests 6 Projects 3 Releases Packages 1 Activity Actions

refactor

Browse source
This commit is contained in:
carla 2026-02-03 15:23:35 +01:00
parent 96daf2a089
commit 7041aa320a
2 changed files with 492 additions and 287 deletions
Download patch file Download diff file Expand all files Collapse all files

690
lib/mv_web/live/import_export_live.ex
View file

@ -40,7 +40,9 @@ defmodule MvWeb.ImportExportLive do
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
# CSV Import configuration constants
# Maximum number of errors to collect per import to prevent memory issues
# and keep error display manageable. Additional errors are silently dropped
# after this limit is reached.
@max_errors 50
@impl true
@ -93,204 +95,11 @@ defmodule MvWeb.ImportExportLive do
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
<%!-- CSV Import Section --%>
<.form_section title={gettext("Import Members (CSV)")}>
<div role="note" class="alert alert-info mb-4">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<p class="text-sm mb-2">
{gettext(
"Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of memberdate (like e-mail or first name). Unknown data field columns will be ignored with a warning."
)}
</p>
<p class="text-sm">
<.link
href={~p"/settings#custom_fields"}
class="link"
data-testid="custom-fields-link"
>
{gettext("Manage Memberdata")}
</.link>
</p>
</div>
</div>
<div class="mb-4">
<p class="text-sm text-base-content/70 mb-2">
{gettext("Download CSV templates:")}
</p>
<ul class="list-disc list-inside space-y-1">
<li>
<.link
href={~p"/templates/member_import_en.csv"}
download="member_import_en.csv"
class="link link-primary"
>
{gettext("English Template")}
</.link>
</li>
<li>
<.link
href={~p"/templates/member_import_de.csv"}
download="member_import_de.csv"
class="link link-primary"
>
{gettext("German Template")}
</.link>
</li>
</ul>
</div>
<.form
id="csv-upload-form"
for={%{}}
multipart={true}
phx-change="validate_csv_upload"
phx-submit="start_import"
data-testid="csv-upload-form"
>
<div class="form-control">
<label for="csv_file" class="label">
<span class="label-text">
{gettext("CSV File")}
</span>
</label>
<.live_file_input
upload={@uploads.csv_file}
id="csv_file"
class="file-input file-input-bordered w-full"
aria-describedby="csv_file_help"
/>
<label class="label" id="csv_file_help">
<span class="label-text-alt">
{gettext("CSV files only, maximum %{size} MB", size: @csv_import_max_file_size_mb)}
</span>
</label>
</div>
<.button
type="submit"
phx-disable-with={gettext("Starting import...")}
variant="primary"
disabled={
@import_status == :running or
Enum.empty?(@uploads.csv_file.entries) or
@uploads.csv_file.entries |> List.first() |> then(&(&1 && not &1.done?))
}
data-testid="start-import-button"
>
{gettext("Start Import")}
</.button>
</.form>
<%= import_info_box(assigns) %>
<%= template_links(assigns) %>
<%= import_form(assigns) %>
<%= if @import_status == :running or @import_status == :done do %>
<%= if @import_progress do %>
<div
role="status"
aria-live="polite"
class="mt-4"
data-testid="import-progress-container"
>
<%= if @import_progress.status == :running do %>
<p class="text-sm" data-testid="import-progress-text">
{gettext("Processing chunk %{current} of %{total}...",
current: @import_progress.current_chunk,
total: @import_progress.total_chunks
)}
</p>
<% end %>
<%= if @import_progress.status == :done do %>
<section class="space-y-4" data-testid="import-results-panel">
<h2 class="text-lg font-semibold">
{gettext("Import Results")}
</h2>
<div class="space-y-4">
<div>
<h3 class="text-sm font-semibold mb-2">
{gettext("Summary")}
</h3>
<div class="text-sm space-y-2">
<p>
<.icon
name="hero-check-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Successfully inserted: %{count} member(s)",
count: @import_progress.inserted
)}
</p>
<%= if @import_progress.failed > 0 do %>
<p>
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Failed: %{count} row(s)", count: @import_progress.failed)}
</p>
<% end %>
<%= if @import_progress.errors_truncated? do %>
<p>
<.icon
name="hero-information-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Error list truncated to %{count} entries",
count: @max_errors
)}
</p>
<% end %>
</div>
</div>
<%= if length(@import_progress.errors) > 0 do %>
<div data-testid="import-error-list">
<h3 class="text-sm font-semibold mb-2">
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Errors")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for error <- @import_progress.errors do %>
<li>
{gettext("Line %{line}: %{message}",
line: error.csv_line_number || "?",
message: error.message || gettext("Unknown error")
)}
<%= if error.field do %>
{gettext(" (Field: %{field})", field: error.field)}
<% end %>
</li>
<% end %>
</ul>
</div>
<% end %>
<%= if length(@import_progress.warnings) > 0 do %>
<div class="alert alert-warning">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<h3 class="font-semibold mb-2">
{gettext("Warnings")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for warning <- @import_progress.warnings do %>
<li>{warning}</li>
<% end %>
</ul>
</div>
</div>
<% end %>
</div>
</section>
<% end %>
</div>
<% end %>
<%= import_progress(assigns) %>
<% end %>
</.form_section>
@ -317,6 +126,223 @@ defmodule MvWeb.ImportExportLive do
"""
end
# Renders the info box explaining CSV import requirements
defp import_info_box(assigns) do
~H"""
<div role="note" class="alert alert-info mb-4">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<p class="text-sm mb-2">
{gettext(
"Use the data field name as the CSV column header in your file. Data fields must exist in Mila before importing, so they must be listed in the list of member data (like e-mail or first name). Unknown data field columns will be ignored with a warning."
)}
</p>
<p class="text-sm">
<.link
href={~p"/settings#custom_fields"}
class="link"
data-testid="custom-fields-link"
>
{gettext("Manage Member Data")}
</.link>
</p>
</div>
</div>
"""
end
# Renders template download links
defp template_links(assigns) do
~H"""
<div class="mb-4">
<p class="text-sm text-base-content/70 mb-2">
{gettext("Download CSV templates:")}
</p>
<ul class="list-disc list-inside space-y-1">
<li>
<.link
href={~p"/templates/member_import_en.csv"}
download="member_import_en.csv"
class="link link-primary"
>
{gettext("English Template")}
</.link>
</li>
<li>
<.link
href={~p"/templates/member_import_de.csv"}
download="member_import_de.csv"
class="link link-primary"
>
{gettext("German Template")}
</.link>
</li>
</ul>
</div>
"""
end
# Renders the CSV upload form
defp import_form(assigns) do
~H"""
<.form
id="csv-upload-form"
for={%{}}
multipart={true}
phx-change="validate_csv_upload"
phx-submit="start_import"
data-testid="csv-upload-form"
>
<div class="form-control">
<label for="csv_file" class="label">
<span class="label-text">
{gettext("CSV File")}
</span>
</label>
<.live_file_input
upload={@uploads.csv_file}
id="csv_file"
class="file-input file-input-bordered w-full"
aria-describedby="csv_file_help"
/>
<p class="label-text-alt mt-1" id="csv_file_help">
{gettext("CSV files only, maximum %{size} MB", size: @csv_import_max_file_size_mb)}
</p>
</div>
<.button
type="submit"
phx-disable-with={gettext("Starting import...")}
variant="primary"
disabled={import_button_disabled?(@import_status, @uploads.csv_file.entries)}
data-testid="start-import-button"
>
{gettext("Start Import")}
</.button>
</.form>
"""
end
# Renders import progress and results
defp import_progress(assigns) do
~H"""
<%= if @import_progress do %>
<div
role="status"
aria-live="polite"
class="mt-4"
data-testid="import-progress-container"
>
<%= if @import_progress.status == :running do %>
<p class="text-sm" data-testid="import-progress-text">
{gettext("Processing chunk %{current} of %{total}...",
current: @import_progress.current_chunk,
total: @import_progress.total_chunks
)}
</p>
<% end %>
<%= if @import_progress.status == :done do %>
<%= import_results(assigns) %>
<% end %>
</div>
<% end %>
"""
end
# Renders import results summary, errors, and warnings
defp import_results(assigns) do
~H"""
<section class="space-y-4" data-testid="import-results-panel">
<h2 class="text-lg font-semibold">
{gettext("Import Results")}
</h2>
<div class="space-y-4">
<div>
<h3 class="text-sm font-semibold mb-2">
{gettext("Summary")}
</h3>
<div class="text-sm space-y-2">
<p>
<.icon
name="hero-check-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Successfully inserted: %{count} member(s)",
count: @import_progress.inserted
)}
</p>
<%= if @import_progress.failed > 0 do %>
<p>
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Failed: %{count} row(s)", count: @import_progress.failed)}
</p>
<% end %>
<%= if @import_progress.errors_truncated? do %>
<p>
<.icon
name="hero-information-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Error list truncated to %{count} entries", count: @max_errors)}
</p>
<% end %>
</div>
</div>
<%= if length(@import_progress.errors) > 0 do %>
<div data-testid="import-error-list">
<h3 class="text-sm font-semibold mb-2">
<.icon
name="hero-exclamation-circle"
class="size-4 inline mr-1"
aria-hidden="true"
/>
{gettext("Errors")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for error <- @import_progress.errors do %>
<li>
{gettext("Line %{line}: %{message}",
line: error.csv_line_number || "?",
message: error.message || gettext("Unknown error")
)}
<%= if error.field do %>
{gettext(" (Field: %{field})", field: error.field)}
<% end %>
</li>
<% end %>
</ul>
</div>
<% end %>
<%= if length(@import_progress.warnings) > 0 do %>
<div class="alert alert-warning" role="alert">
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
<div>
<h3 class="font-semibold mb-2">
{gettext("Warnings")}
</h3>
<ul class="list-disc list-inside space-y-1 text-sm">
<%= for warning <- @import_progress.warnings do %>
<li>{warning}</li>
<% end %>
</ul>
</div>
</div>
<% end %>
</div>
</section>
"""
end
@impl true
def handle_event("validate_csv_upload", _params, socket) do
{:noreply, socket}
@ -333,11 +359,22 @@ defmodule MvWeb.ImportExportLive do
end
end
# Checks if import can be started (admin permission, status, upload ready)
# Checks if all prerequisites for starting an import are met.
#
# Validates:
# - User has admin permissions
# - No import is currently running
# - CSV file is uploaded and ready
#
# Returns `:ok` if all checks pass, `{:error, message}` otherwise.
#
# Note: on_mount :ensure_user_role_loaded already guarantees the role is loaded,
# so ensure_actor_loaded is primarily for clarity.
@spec check_import_prerequisites(Phoenix.LiveView.Socket.t()) ::
:ok | {:error, String.t()}
defp check_import_prerequisites(socket) do
# Ensure user role is loaded before authorization check
user = socket.assigns[:current_user]
user_with_role = Actor.ensure_loaded(user)
# on_mount already ensures role is loaded, but we keep this for clarity
user_with_role = ensure_actor_loaded(socket)
cond do
not Authorization.can?(user_with_role, :create, Mv.Membership.Member) ->
@ -358,7 +395,12 @@ defmodule MvWeb.ImportExportLive do
end
end
# Processes CSV upload and starts import
# Processes CSV upload and starts import process.
#
# Reads the uploaded CSV file, prepares it for import, and initiates
# the chunked processing workflow.
@spec process_csv_upload(Phoenix.LiveView.Socket.t()) ::
{:noreply, Phoenix.LiveView.Socket.t()}
defp process_csv_upload(socket) do
actor = MvWeb.LiveHelpers.current_actor(socket)
@ -382,12 +424,14 @@ defmodule MvWeb.ImportExportLive do
put_flash(
socket,
:error,
gettext("Failed to prepare CSV import: %{error}", error: error_message)
gettext("Failed to prepare CSV import: %{reason}", reason: error_message)
)}
end
end
# Starts the import process
# Starts the import process by initializing progress tracking and scheduling the first chunk.
@spec start_import(Phoenix.LiveView.Socket.t(), map()) ::
{:noreply, Phoenix.LiveView.Socket.t()}
defp start_import(socket, import_state) do
progress = initialize_import_progress(import_state)
@ -402,7 +446,8 @@ defmodule MvWeb.ImportExportLive do
{:noreply, socket}
end
# Initializes import progress structure
# Initializes the import progress tracking structure with default values.
@spec initialize_import_progress(map()) :: map()
defp initialize_import_progress(import_state) do
%{
inserted: 0,
@ -416,13 +461,65 @@ defmodule MvWeb.ImportExportLive do
}
end
# Formats error messages for display
# Formats error messages for user-friendly display.
#
# Handles various error types including Ash errors, maps with message fields,
# lists of errors, and fallback formatting for unknown types.
@spec format_error_message(any()) :: String.t()
defp format_error_message(error) do
case error do
%{message: msg} when is_binary(msg) -> msg
%{errors: errors} when is_list(errors) -> inspect(errors)
reason when is_binary(reason) -> reason
other -> inspect(other)
%Ash.Error.Invalid{} = ash_error ->
format_ash_error(ash_error)
%{message: msg} when is_binary(msg) ->
msg
%{errors: errors} when is_list(errors) ->
format_error_list(errors)
reason when is_binary(reason) ->
reason
other ->
format_unknown_error(other)
end
end
# Formats Ash validation errors for display
defp format_ash_error(%Ash.Error.Invalid{errors: errors}) when is_list(errors) do
errors
|> Enum.map(&format_single_error/1)
|> Enum.join(", ")
end
defp format_ash_error(error) do
format_unknown_error(error)
end
# Formats a list of errors into a readable string
defp format_error_list(errors) do
errors
|> Enum.map(&format_single_error/1)
|> Enum.join(", ")
end
# Formats a single error item
defp format_single_error(error) when is_map(error) do
Map.get(error, :message) || Map.get(error, :field) || inspect(error, limit: :infinity)
end
defp format_single_error(error) do
to_string(error)
end
# Formats unknown error types with truncation for very long messages
defp format_unknown_error(other) do
error_str = inspect(other, limit: :infinity, pretty: true)
if String.length(error_str) > 200 do
String.slice(error_str, 0, 197) <> "..."
else
error_str
end
end
@ -431,7 +528,7 @@ defmodule MvWeb.ImportExportLive do
case socket.assigns do
%{import_state: import_state, import_progress: progress}
when is_map(import_state) and is_map(progress) ->
if idx >= 0 and idx < length(import_state.chunks) do
if idx < length(import_state.chunks) do
start_chunk_processing_task(socket, import_state, progress, idx)
else
handle_chunk_error(socket, :invalid_index, idx)
@ -461,13 +558,18 @@ defmodule MvWeb.ImportExportLive do
handle_chunk_error(socket, :processing_failed, idx, reason)
end
# Starts async task to process a chunk
# In tests (SQL sandbox mode), runs synchronously to avoid Ecto Sandbox issues
# Starts async task to process a chunk of CSV rows.
#
# In tests (SQL sandbox mode), runs synchronously to avoid Ecto Sandbox issues.
@spec start_chunk_processing_task(
Phoenix.LiveView.Socket.t(),
map(),
map(),
non_neg_integer()
) :: {:noreply, Phoenix.LiveView.Socket.t()}
defp start_chunk_processing_task(socket, import_state, progress, idx) do
chunk = Enum.at(import_state.chunks, idx)
# Ensure user role is loaded before using as actor
user = socket.assigns[:current_user]
actor = Actor.ensure_loaded(user)
actor = ensure_actor_loaded(socket)
live_view_pid = self()
# Process chunk with existing error count for capping
@ -484,17 +586,33 @@ defmodule MvWeb.ImportExportLive do
if Config.sql_sandbox?() do
# Run synchronously in tests to avoid Ecto Sandbox issues with async tasks
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
result =
try do
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
rescue
e ->
{:error, Exception.message(e)}
catch
:exit, reason ->
{:error, inspect(reason)}
:throw, reason ->
{:error, inspect(reason)}
end
# In test mode, send the message - it will be processed when render() is called
# in the test. The test helper wait_for_import_completion() handles message processing
send(live_view_pid, {:chunk_done, idx, chunk_result})
case result do
{:ok, chunk_result} ->
# In test mode, send the message - it will be processed when render() is called
# in the test. The test helper wait_for_import_completion() handles message processing
send(live_view_pid, {:chunk_done, idx, chunk_result})
{:error, reason} ->
send(live_view_pid, {:chunk_error, idx, reason})
end
else
# Start async task to process chunk in production
# Use start_child for fire-and-forget: no monitor, no Task messages
@ -503,22 +621,45 @@ defmodule MvWeb.ImportExportLive do
# Set locale in task process for translations
Gettext.put_locale(MvWeb.Gettext, locale)
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
result =
try do
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
rescue
e ->
{:error, Exception.message(e)}
catch
:exit, reason ->
{:error, inspect(reason)}
:throw, reason ->
{:error, inspect(reason)}
end
send(live_view_pid, {:chunk_done, idx, chunk_result})
case result do
{:ok, chunk_result} ->
send(live_view_pid, {:chunk_done, idx, chunk_result})
{:error, reason} ->
send(live_view_pid, {:chunk_error, idx, reason})
end
end)
end
{:noreply, socket}
end
# Handles chunk processing result from async task
# Handles chunk processing result from async task and schedules the next chunk.
@spec handle_chunk_result(
Phoenix.LiveView.Socket.t(),
map(),
map(),
non_neg_integer(),
map()
) :: {:noreply, Phoenix.LiveView.Socket.t()}
defp handle_chunk_result(socket, import_state, progress, idx, chunk_result) do
# Merge progress
new_progress = merge_progress(progress, chunk_result, idx)
@ -534,7 +675,13 @@ defmodule MvWeb.ImportExportLive do
{:noreply, socket}
end
# Handles chunk processing errors
# Handles chunk processing errors and updates socket with error status.
@spec handle_chunk_error(
Phoenix.LiveView.Socket.t(),
:invalid_index | :missing_state | :processing_failed,
non_neg_integer(),
any()
) :: {:noreply, Phoenix.LiveView.Socket.t()}
defp handle_chunk_error(socket, error_type, idx, reason \\ nil) do
error_message =
case error_type do
@ -559,21 +706,14 @@ defmodule MvWeb.ImportExportLive do
{:noreply, socket}
end
# Consumes uploaded CSV file entries and reads the file content.
#
# Returns the file content as a binary string or an error tuple.
@spec consume_and_read_csv(Phoenix.LiveView.Socket.t()) ::
{:ok, String.t()} | {:error, String.t()}
defp consume_and_read_csv(socket) do
result =
consume_uploaded_entries(socket, :csv_file, fn %{path: path}, _entry ->
case File.read(path) do
{:ok, content} -> {:ok, content}
{:error, reason} -> {:error, Exception.message(reason)}
end
end)
result
|> case do
[content] when is_binary(content) ->
{:ok, content}
[{:ok, content}] when is_binary(content) ->
case consume_uploaded_entries(socket, :csv_file, &read_file_entry/2) do
[{:ok, content}] ->
{:ok, content}
[{:error, reason}] ->
@ -583,10 +723,35 @@ defmodule MvWeb.ImportExportLive do
{:error, gettext("No file was uploaded")}
_other ->
{:error, gettext("Failed to read uploaded file")}
{:error, gettext("Failed to read uploaded file: unexpected format")}
end
end
# Reads a single file entry from the uploaded path
@spec read_file_entry(map(), map()) :: {:ok, String.t()} | {:error, String.t()}
defp read_file_entry(%{path: path}, _entry) do
case File.read(path) do
{:ok, content} ->
{:ok, content}
{:error, reason} when is_atom(reason) ->
# POSIX error atoms (e.g., :enoent) need to be formatted
{:error, :file.format_error(reason)}
{:error, %File.Error{reason: reason}} ->
# File.Error struct with reason atom
{:error, :file.format_error(reason)}
{:error, reason} ->
# Fallback for other error types
{:error, Exception.message(reason)}
end
end
# Merges chunk processing results into the overall import progress.
#
# Handles error capping, warning merging, and status updates.
@spec merge_progress(map(), map(), non_neg_integer()) :: map()
defp merge_progress(progress, chunk_result, current_chunk_idx) do
# Merge errors with cap of @max_errors overall
all_errors = progress.errors ++ chunk_result.errors
@ -613,6 +778,9 @@ defmodule MvWeb.ImportExportLive do
}
end
# Schedules the next chunk for processing or marks import as complete.
@spec schedule_next_chunk(Phoenix.LiveView.Socket.t(), non_neg_integer(), non_neg_integer()) ::
Phoenix.LiveView.Socket.t()
defp schedule_next_chunk(socket, current_idx, total_chunks) do
next_idx = current_idx + 1
@ -625,4 +793,22 @@ defmodule MvWeb.ImportExportLive do
socket
end
end
# Determines if the import button should be disabled based on import status and upload state
@spec import_button_disabled?(:idle | :running | :done | :error, [map()]) :: boolean()
defp import_button_disabled?(:running, _entries), do: true
defp import_button_disabled?(_status, []), do: true
defp import_button_disabled?(_status, [entry | _]) when not entry.done?, do: true
defp import_button_disabled?(_status, _entries), do: false
# Ensures the actor (user with role) is loaded from socket assigns.
#
# Note: on_mount :ensure_user_role_loaded already guarantees the role is loaded,
# so this is primarily for clarity and defensive programming.
@spec ensure_actor_loaded(Phoenix.LiveView.Socket.t()) :: Mv.Accounts.User.t() | nil
defp ensure_actor_loaded(socket) do
user = socket.assigns[:current_user]
# on_mount already ensures role is loaded, but we keep this for clarity
Actor.ensure_loaded(user)
end
end