Implements custom field CSV import closes #338 #395

Merged
carla merged 10 commits from feature/338_import_custom_fields into main 2026-02-02 17:05:31 +01:00
7 changed files with 199 additions and 22 deletions
Showing only changes of commit 9e27de84cb - Show all commits

View file

@ -51,6 +51,13 @@ config :mv,
generators: [timestamp_type: :utc_datetime], generators: [timestamp_type: :utc_datetime],
ash_domains: [Mv.Membership, Mv.Accounts, Mv.MembershipFees, Mv.Authorization] ash_domains: [Mv.Membership, Mv.Accounts, Mv.MembershipFees, Mv.Authorization]
# CSV Import configuration
config :mv,
csv_import: [
max_file_size_mb: 10,
max_rows: 1000
]
# Configures the endpoint # Configures the endpoint
config :mv, MvWeb.Endpoint, config :mv, MvWeb.Endpoint,
url: [host: "localhost"], url: [host: "localhost"],

View file

@ -21,4 +21,99 @@ defmodule Mv.Config do
def sql_sandbox? do def sql_sandbox? do
Application.get_env(:mv, :sql_sandbox, false) Application.get_env(:mv, :sql_sandbox, false)
end end
@doc """
Returns the maximum file size for CSV imports in bytes.
Reads the `max_file_size_mb` value from the CSV import configuration
and converts it to bytes.
## Returns
- Maximum file size in bytes (default: 10_485_760 bytes = 10 MB)
## Examples
iex> Mv.Config.csv_import_max_file_size_bytes()
10_485_760
"""
@spec csv_import_max_file_size_bytes() :: non_neg_integer()
def csv_import_max_file_size_bytes do
max_file_size_mb = get_csv_import_config(:max_file_size_mb, 10)
max_file_size_mb * 1024 * 1024
end
@doc """
Returns the maximum number of rows allowed in CSV imports.
Reads the `max_rows` value from the CSV import configuration.
## Returns
- Maximum number of rows (default: 1000)
## Examples
iex> Mv.Config.csv_import_max_rows()
1000
"""
@spec csv_import_max_rows() :: pos_integer()
def csv_import_max_rows do
get_csv_import_config(:max_rows, 1000)
end
@doc """
Returns the maximum file size for CSV imports in megabytes.
Reads the `max_file_size_mb` value from the CSV import configuration.
## Returns
- Maximum file size in megabytes (default: 10)
## Examples
iex> Mv.Config.csv_import_max_file_size_mb()
10
"""
@spec csv_import_max_file_size_mb() :: pos_integer()
def csv_import_max_file_size_mb do
get_csv_import_config(:max_file_size_mb, 10)
end
# Helper function to get CSV import config values
defp get_csv_import_config(key, default) do
Application.get_env(:mv, :csv_import, [])
|> Keyword.get(key, default)
|> parse_and_validate_integer(default)
end
# Parses and validates integer configuration values.
#
# Accepts:
# - Integer values (passed through)
# - String integers (e.g., "1000") - parsed to integer
# - Invalid values (e.g., "abc", nil) - falls back to default
#
# Always clamps the result to a minimum of 1 to ensure positive values.
#
# Note: We don't log warnings for unparseable values because:
# - These functions may be called frequently (e.g., on every request)
# - Logging would create excessive log spam
# - The fallback to default provides a safe behavior
# - Configuration errors should be caught during deployment/testing
defp parse_and_validate_integer(value, _default) when is_integer(value) do
max(1, value)
end
defp parse_and_validate_integer(value, default) when is_binary(value) do
case Integer.parse(value) do
{int, _remainder} -> max(1, int)
:error -> default
end
end
defp parse_and_validate_integer(_value, default) do
default
end
end end

View file

@ -34,8 +34,8 @@ defmodule MvWeb.GlobalSettingsLive do
### Limits ### Limits
- Maximum file size: 10 MB - Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
- Maximum rows: 1,000 rows (excluding header) - Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
- Processing: chunks of 200 rows - Processing: chunks of 200 rows
- Errors: capped at 50 per import - Errors: capped at 50 per import
@ -54,8 +54,6 @@ defmodule MvWeb.GlobalSettingsLive do
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded} on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
# CSV Import configuration constants # CSV Import configuration constants
# 10 MB
@max_file_size_bytes 10_485_760
@max_errors 50 @max_errors 50
@impl true @impl true
@ -76,13 +74,15 @@ defmodule MvWeb.GlobalSettingsLive do
|> assign(:import_status, :idle) |> assign(:import_status, :idle)
|> assign(:locale, locale) |> assign(:locale, locale)
|> assign(:max_errors, @max_errors) |> assign(:max_errors, @max_errors)
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
|> assign_form() |> assign_form()
# Configure file upload with auto-upload enabled # Configure file upload with auto-upload enabled
# Files are uploaded automatically when selected, no need for manual trigger # Files are uploaded automatically when selected, no need for manual trigger
|> allow_upload(:csv_file, |> allow_upload(:csv_file,
accept: ~w(.csv), accept: ~w(.csv),
max_entries: 1, max_entries: 1,
max_file_size: @max_file_size_bytes, max_file_size: Config.csv_import_max_file_size_bytes(),
auto_upload: true auto_upload: true
) )
@ -206,7 +206,7 @@ defmodule MvWeb.GlobalSettingsLive do
/> />
<label class="label" id="csv_file_help"> <label class="label" id="csv_file_help">
<span class="label-text-alt"> <span class="label-text-alt">
{gettext("CSV files only, maximum 10 MB")} {gettext("CSV files only, maximum %{size} MB", size: @csv_import_max_file_size_mb)}
</span> </span>
</label> </label>
</div> </div>
@ -417,7 +417,8 @@ defmodule MvWeb.GlobalSettingsLive do
actor = MvWeb.LiveHelpers.current_actor(socket) actor = MvWeb.LiveHelpers.current_actor(socket)
with {:ok, content} <- consume_and_read_csv(socket), with {:ok, content} <- consume_and_read_csv(socket),
{:ok, import_state} <- MemberCSV.prepare(content, actor: actor) do {:ok, import_state} <-
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
start_import(socket, import_state) start_import(socket, import_state)
else else
{:error, reason} when is_binary(reason) -> {:error, reason} when is_binary(reason) ->

View file

@ -1980,11 +1980,6 @@ msgstr " (Datenfeld: %{field})"
msgid "CSV File" msgid "CSV File"
msgstr "CSV Datei" msgstr "CSV Datei"
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum 10 MB"
msgstr "Nur CSV Dateien, maximal 10 MB"
#: lib/mv_web/live/global_settings_live.ex #: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "Download CSV templates:" msgid "Download CSV templates:"
@ -2263,6 +2258,11 @@ msgstr "Nicht berechtigt."
msgid "Could not load data fields. Please check your permissions." msgid "Could not load data fields. Please check your permissions."
msgstr "Datenfelder konnten nicht geladen werden. Bitte überprüfen Sie Ihre Berechtigungen." msgstr "Datenfelder konnten nicht geladen werden. Bitte überprüfen Sie Ihre Berechtigungen."
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "CSV files only, maximum %{size} MB"
msgstr "Nur CSV Dateien, maximal %{size} MB"
#: lib/mv/membership/import/member_csv.ex #: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "(ISO-8601 format: YYYY-MM-DD)" msgid "(ISO-8601 format: YYYY-MM-DD)"

View file

@ -1981,11 +1981,6 @@ msgstr ""
msgid "CSV File" msgid "CSV File"
msgstr "" msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum 10 MB"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex #: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "Download CSV templates:" msgid "Download CSV templates:"
@ -2264,6 +2259,11 @@ msgstr ""
msgid "Could not load data fields. Please check your permissions." msgid "Could not load data fields. Please check your permissions."
msgstr "" msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum %{size} MB"
msgstr ""
#: lib/mv/membership/import/member_csv.ex #: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "(ISO-8601 format: YYYY-MM-DD)" msgid "(ISO-8601 format: YYYY-MM-DD)"

View file

@ -1981,11 +1981,6 @@ msgstr ""
msgid "CSV File" msgid "CSV File"
msgstr "" msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format
msgid "CSV files only, maximum 10 MB"
msgstr ""
#: lib/mv_web/live/global_settings_live.ex #: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "Download CSV templates:" msgid "Download CSV templates:"
@ -2264,6 +2259,12 @@ msgstr ""
msgid "Could not load data fields. Please check your permissions." msgid "Could not load data fields. Please check your permissions."
msgstr "" msgstr ""
#: lib/mv_web/live/global_settings_live.ex
#, elixir-autogen, elixir-format, fuzzy
msgid "CSV files only, maximum %{size} MB"
msgstr ""
#: lib/mv/membership/import/member_csv.ex #: lib/mv/membership/import/member_csv.ex
#, elixir-autogen, elixir-format #, elixir-autogen, elixir-format
msgid "(ISO-8601 format: YYYY-MM-DD)" msgid "(ISO-8601 format: YYYY-MM-DD)"

View file

@ -0,0 +1,73 @@
defmodule MvWeb.GlobalSettingsLiveConfigTest do
@moduledoc """
Tests for GlobalSettingsLive that modify global Application configuration.
These tests run with `async: false` to prevent race conditions when
modifying global Application environment variables (Application.put_env).
This follows the same pattern as Mv.ConfigTest.
"""
use MvWeb.ConnCase, async: false
import Phoenix.LiveViewTest
# Helper function to upload CSV file in tests
defp upload_csv_file(view, csv_content, filename \\ "test_import.csv") do
view
|> file_input("#csv-upload-form", :csv_file, [
%{
last_modified: System.system_time(:second),
name: filename,
content: csv_content,
size: byte_size(csv_content),
type: "text/csv"
}
])
|> render_upload(filename)
end
describe "CSV Import - Configuration Tests" do
setup %{conn: conn} do
# Ensure admin user
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
conn = MvWeb.ConnCase.conn_with_password_user(conn, admin_user)
{:ok, conn: conn, admin_user: admin_user}
end
test "configured row limit is enforced", %{conn: conn} do
# Business rule: CSV import respects configured row limits
# Test that a custom limit (500) is enforced, not just the default (1000)
original_config = Application.get_env(:mv, :csv_import, [])
try do
Application.put_env(:mv, :csv_import, max_rows: 500)
{:ok, view, _html} = live(conn, ~p"/settings")
# Generate CSV with 501 rows (exceeding custom limit of 500)
header = "first_name;last_name;email;street;postal_code;city\n"
rows =
for i <- 1..501 do
"Row#{i};Last#{i};email#{i}@example.com;Street#{i};12345;City#{i}\n"
end
large_csv = header <> Enum.join(rows)
# Simulate file upload using helper function
upload_csv_file(view, large_csv, "too_many_rows_custom.csv")
view
|> form("#csv-upload-form", %{})
|> render_submit()
html = render(view)
# Business rule: import should be rejected when exceeding configured limit
assert html =~ "exceeds" or html =~ "maximum" or html =~ "limit" or
html =~ "Failed to prepare"
after
# Restore original config
Application.put_env(:mv, :csv_import, original_config)
end
end
end
end