ImplementsCSV Import UI closes #335 #359

Merged
moritz merged 12 commits from feature/335_csv_import_ui into main 2026-01-25 18:45:08 +01:00
7 changed files with 147 additions and 126 deletions
Showing only changes of commit bf7e47ce5c - Show all commits

View file

@ -97,31 +97,16 @@ defmodule Mv.Membership.Import.HeaderMapper do
}
# Build reverse map: normalized_variant -> canonical_field
# Cached on first access for performance
# Computed on each access - the map is small enough that recomputing is fast
# This avoids Module.get_attribute issues while maintaining simplicity
defp normalized_to_canonical do
cached = Process.get({__MODULE__, :normalized_to_canonical})
if cached do
cached
else
map = build_normalized_to_canonical_map()
Process.put({__MODULE__, :normalized_to_canonical}, map)
map
end
end
# Builds the normalized variant -> canonical field map
defp build_normalized_to_canonical_map do
@member_field_variants_raw
|> Enum.flat_map(&map_variants_to_normalized/1)
|> Map.new()
end
# Maps a canonical field and its variants to normalized tuples
defp map_variants_to_normalized({canonical, variants}) do
|> Enum.flat_map(fn {canonical, variants} ->
Enum.map(variants, fn variant ->
{normalize_header(variant), canonical}
end)
end)
|> Map.new()
end
@doc """
@ -139,15 +124,8 @@ defmodule Mv.Membership.Import.HeaderMapper do
iex> HeaderMapper.known_member_fields()
#MapSet<["email", "firstname", "lastname", "street", "postalcode", "city"]>
"""
@spec known_member_fields() :: MapSet.t(String.t())
def known_member_fields do
cached = Process.get({__MODULE__, :known_member_fields})
if cached do
cached
else
fields =
@member_field_variants_raw
# Known member fields computed at compile-time for performance and determinism
@known_member_fields @member_field_variants_raw
|> Map.keys()
|> Enum.map(fn canonical ->
# Normalize the canonical field name (e.g., :first_name -> "firstname")
@ -158,9 +136,9 @@ defmodule Mv.Membership.Import.HeaderMapper do
end)
|> MapSet.new()
Process.put({__MODULE__, :known_member_fields}, fields)
fields
end
@spec known_member_fields() :: MapSet.t(String.t())
def known_member_fields do
@known_member_fields
end
@doc """

View file

@ -48,6 +48,7 @@ defmodule MvWeb.GlobalSettingsLive do
alias Mv.Membership
alias Mv.Membership.Import.MemberCSV
alias MvWeb.Authorization
alias Mv.Config
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
@ -140,14 +141,6 @@ defmodule MvWeb.GlobalSettingsLive do
"Use the custom field name as the CSV column header (same normalization as member fields applies)"
)}
</p>
<div class="mt-2">
<.link
navigate={~p"/custom_field_values"}
class="link link-primary"
>
{gettext("Manage Custom Fields")}
</.link>
</div>
</div>
</div>
@ -209,6 +202,7 @@ defmodule MvWeb.GlobalSettingsLive do
phx-disable-with={gettext("Starting import...")}
variant="primary"
disabled={
@import_status == :running or
Enum.empty?(@uploads.csv_file.entries) or
@uploads.csv_file.entries |> List.first() |> then(&(&1 && not &1.done?))
}
@ -226,7 +220,7 @@ defmodule MvWeb.GlobalSettingsLive do
class="mt-4"
data-testid="import-progress-container"
>
<%= if @import_status == :running do %>
<%= if @import_progress.status == :running do %>
<p class="text-sm" data-testid="import-progress-text">
{gettext("Processing chunk %{current} of %{total}...",
current: @import_progress.current_chunk,
@ -235,7 +229,7 @@ defmodule MvWeb.GlobalSettingsLive do
</p>
<% end %>
<%= if @import_status == :done do %>
<%= if @import_progress.status == :done do %>
<section class="space-y-4" data-testid="import-results-panel">
<h2 class="text-lg font-semibold">
{gettext("Import Results")}
@ -372,6 +366,15 @@ defmodule MvWeb.GlobalSettingsLive do
def handle_event("start_import", _params, socket) do
# Server-side admin check
if Authorization.can?(socket.assigns[:current_user], :create, Mv.Membership.Member) do
# Prevent concurrent imports
if socket.assigns.import_status == :running do
{:noreply,
put_flash(
socket,
:error,
gettext("Import is already running. Please wait for it to complete.")
)}
else
# Check if upload is completed
upload_entries = socket.assigns.uploads.csv_file.entries
@ -398,7 +401,8 @@ defmodule MvWeb.GlobalSettingsLive do
warnings: import_state.warnings || [],
status: :running,
current_chunk: 0,
total_chunks: total_chunks
total_chunks: total_chunks,
errors_truncated?: false
}
socket =
@ -460,6 +464,7 @@ defmodule MvWeb.GlobalSettingsLive do
)}
end
end
end
else
{:noreply,
put_flash(
@ -576,6 +581,7 @@ defmodule MvWeb.GlobalSettingsLive do
end
# Starts async task to process a chunk
# In tests (SQL sandbox mode), runs synchronously to avoid Ecto Sandbox issues
defp start_chunk_processing_task(socket, import_state, progress, idx) do
chunk = Enum.at(import_state.chunks, idx)
actor = socket.assigns[:current_user]
@ -589,21 +595,33 @@ defmodule MvWeb.GlobalSettingsLive do
actor: actor
]
# Start async task to process chunk
Task.Supervisor.async_nolink(Mv.TaskSupervisor, fn ->
case MemberCSV.process_chunk(
if Config.sql_sandbox?() do
# Run synchronously in tests to avoid Ecto Sandbox issues with async tasks
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
) do
{:ok, chunk_result} ->
send(live_view_pid, {:chunk_done, idx, chunk_result})
)
{:error, reason} ->
send(live_view_pid, {:chunk_error, idx, reason})
end
# In test mode, send the message - it will be processed when render() is called
# in the test. The test helper wait_for_import_completion() handles message processing
send(live_view_pid, {:chunk_done, idx, chunk_result})
else
# Start async task to process chunk in production
Task.Supervisor.async_nolink(Mv.TaskSupervisor, fn ->
{:ok, chunk_result} =
MemberCSV.process_chunk(
chunk,
import_state.column_map,
import_state.custom_field_map,
opts
)
send(live_view_pid, {:chunk_done, idx, chunk_result})
end)
end
{:noreply, socket}
end
@ -670,10 +688,10 @@ defmodule MvWeb.GlobalSettingsLive do
end
end)
|> case do
[{_name, {:ok, content}}] when is_binary(content) ->
[{:ok, content}] when is_binary(content) ->
{:ok, content}
[{_name, {:error, reason}}] ->
[{:error, reason}] ->
{:error, gettext("Failed to read file: %{reason}", reason: reason)}
[] ->

View file

@ -1,3 +1,8 @@
first_name;last_name;email;street;postal_code;city
Alice;Smith;alice.smith@example.com;Main Street 1;12345;Berlin

1 first_name last_name email street postal_code city
2 Alice Smith alice.smith@example.com Main Street 1 12345 Berlin

View file

@ -4,3 +4,8 @@ Alice;Smith;alice.smith@example.com;Main Street 1;12345;Berlin
Bob;Johnson;invalid-email;Park Avenue 2;54321;Munich

1 first_name last_name email street postal_code city
2 Alice Smith alice.smith@example.com Main Street 1 12345 Berlin
3 Bob Johnson invalid-email Park Avenue 2 54321 Munich

View file

@ -3,3 +3,8 @@ Alice;Smith;alice.smith@example.com;Main Street 1;12345;Berlin;SomeValue
Bob;Johnson;bob.johnson@example.com;Park Avenue 2;54321;Munich;AnotherValue

1 first_name last_name email street postal_code city UnknownCustomField
2 Alice Smith alice.smith@example.com Main Street 1 12345 Berlin SomeValue
3 Bob Johnson bob.johnson@example.com Park Avenue 2 54321 Munich AnotherValue

View file

@ -3,3 +3,8 @@ Alice;Smith;invalid-email;Main Street 1;12345;Berlin
Bob;Johnson;;Park Avenue 2;54321;Munich

1 first_name last_name email street postal_code city
2 Alice Smith invalid-email Main Street 1 12345 Berlin
3 Bob Johnson Park Avenue 2 54321 Munich

View file

@ -3,3 +3,8 @@ Alice;Smith;alice.smith@example.com;Main Street 1;12345;Berlin
Bob;Johnson;bob.johnson@example.com;Park Avenue 2;54321;Munich

1 first_name last_name email street postal_code city
2 Alice Smith alice.smith@example.com Main Street 1 12345 Berlin
3 Bob Johnson bob.johnson@example.com Park Avenue 2 54321 Munich