Implements custom field CSV import closes #338 #395
7 changed files with 199 additions and 22 deletions
|
|
@ -51,6 +51,13 @@ config :mv,
|
|||
generators: [timestamp_type: :utc_datetime],
|
||||
ash_domains: [Mv.Membership, Mv.Accounts, Mv.MembershipFees, Mv.Authorization]
|
||||
|
||||
# CSV Import configuration
|
||||
config :mv,
|
||||
csv_import: [
|
||||
max_file_size_mb: 10,
|
||||
max_rows: 1000
|
||||
]
|
||||
|
||||
# Configures the endpoint
|
||||
config :mv, MvWeb.Endpoint,
|
||||
url: [host: "localhost"],
|
||||
|
|
|
|||
|
|
@ -21,4 +21,99 @@ defmodule Mv.Config do
|
|||
def sql_sandbox? do
|
||||
Application.get_env(:mv, :sql_sandbox, false)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the maximum file size for CSV imports in bytes.
|
||||
|
||||
Reads the `max_file_size_mb` value from the CSV import configuration
|
||||
and converts it to bytes.
|
||||
|
||||
## Returns
|
||||
|
||||
- Maximum file size in bytes (default: 10_485_760 bytes = 10 MB)
|
||||
|
||||
## Examples
|
||||
|
||||
iex> Mv.Config.csv_import_max_file_size_bytes()
|
||||
10_485_760
|
||||
"""
|
||||
@spec csv_import_max_file_size_bytes() :: non_neg_integer()
|
||||
def csv_import_max_file_size_bytes do
|
||||
max_file_size_mb = get_csv_import_config(:max_file_size_mb, 10)
|
||||
max_file_size_mb * 1024 * 1024
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the maximum number of rows allowed in CSV imports.
|
||||
|
||||
Reads the `max_rows` value from the CSV import configuration.
|
||||
|
||||
## Returns
|
||||
|
||||
- Maximum number of rows (default: 1000)
|
||||
|
||||
## Examples
|
||||
|
||||
iex> Mv.Config.csv_import_max_rows()
|
||||
1000
|
||||
"""
|
||||
@spec csv_import_max_rows() :: pos_integer()
|
||||
def csv_import_max_rows do
|
||||
get_csv_import_config(:max_rows, 1000)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the maximum file size for CSV imports in megabytes.
|
||||
|
||||
Reads the `max_file_size_mb` value from the CSV import configuration.
|
||||
|
||||
## Returns
|
||||
|
||||
- Maximum file size in megabytes (default: 10)
|
||||
|
||||
## Examples
|
||||
|
||||
iex> Mv.Config.csv_import_max_file_size_mb()
|
||||
10
|
||||
"""
|
||||
@spec csv_import_max_file_size_mb() :: pos_integer()
|
||||
def csv_import_max_file_size_mb do
|
||||
get_csv_import_config(:max_file_size_mb, 10)
|
||||
end
|
||||
|
||||
# Helper function to get CSV import config values
|
||||
defp get_csv_import_config(key, default) do
|
||||
Application.get_env(:mv, :csv_import, [])
|
||||
|> Keyword.get(key, default)
|
||||
|> parse_and_validate_integer(default)
|
||||
end
|
||||
|
||||
# Parses and validates integer configuration values.
|
||||
#
|
||||
# Accepts:
|
||||
# - Integer values (passed through)
|
||||
# - String integers (e.g., "1000") - parsed to integer
|
||||
# - Invalid values (e.g., "abc", nil) - falls back to default
|
||||
#
|
||||
# Always clamps the result to a minimum of 1 to ensure positive values.
|
||||
#
|
||||
# Note: We don't log warnings for unparseable values because:
|
||||
# - These functions may be called frequently (e.g., on every request)
|
||||
# - Logging would create excessive log spam
|
||||
# - The fallback to default provides a safe behavior
|
||||
# - Configuration errors should be caught during deployment/testing
|
||||
defp parse_and_validate_integer(value, _default) when is_integer(value) do
|
||||
max(1, value)
|
||||
end
|
||||
|
||||
defp parse_and_validate_integer(value, default) when is_binary(value) do
|
||||
case Integer.parse(value) do
|
||||
{int, _remainder} -> max(1, int)
|
||||
:error -> default
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_and_validate_integer(_value, default) do
|
||||
default
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -34,8 +34,8 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
|
||||
### Limits
|
||||
|
||||
- Maximum file size: 10 MB
|
||||
- Maximum rows: 1,000 rows (excluding header)
|
||||
- Maximum file size: configurable via `config :mv, csv_import: [max_file_size_mb: ...]`
|
||||
- Maximum rows: configurable via `config :mv, csv_import: [max_rows: ...]` (excluding header)
|
||||
- Processing: chunks of 200 rows
|
||||
- Errors: capped at 50 per import
|
||||
|
||||
|
|
@ -54,8 +54,6 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
on_mount {MvWeb.LiveHelpers, :ensure_user_role_loaded}
|
||||
|
||||
# CSV Import configuration constants
|
||||
# 10 MB
|
||||
@max_file_size_bytes 10_485_760
|
||||
@max_errors 50
|
||||
|
||||
@impl true
|
||||
|
|
@ -76,13 +74,15 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
|> assign(:import_status, :idle)
|
||||
|> assign(:locale, locale)
|
||||
|> assign(:max_errors, @max_errors)
|
||||
|> assign(:csv_import_max_rows, Config.csv_import_max_rows())
|
||||
|> assign(:csv_import_max_file_size_mb, Config.csv_import_max_file_size_mb())
|
||||
|> assign_form()
|
||||
# Configure file upload with auto-upload enabled
|
||||
# Files are uploaded automatically when selected, no need for manual trigger
|
||||
|> allow_upload(:csv_file,
|
||||
accept: ~w(.csv),
|
||||
max_entries: 1,
|
||||
max_file_size: @max_file_size_bytes,
|
||||
max_file_size: Config.csv_import_max_file_size_bytes(),
|
||||
auto_upload: true
|
||||
)
|
||||
|
||||
|
|
@ -206,7 +206,7 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
/>
|
||||
<label class="label" id="csv_file_help">
|
||||
<span class="label-text-alt">
|
||||
{gettext("CSV files only, maximum 10 MB")}
|
||||
{gettext("CSV files only, maximum %{size} MB", size: @csv_import_max_file_size_mb)}
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
|
|
@ -417,7 +417,8 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
actor = MvWeb.LiveHelpers.current_actor(socket)
|
||||
|
||||
with {:ok, content} <- consume_and_read_csv(socket),
|
||||
{:ok, import_state} <- MemberCSV.prepare(content, actor: actor) do
|
||||
{:ok, import_state} <-
|
||||
MemberCSV.prepare(content, max_rows: Config.csv_import_max_rows(), actor: actor) do
|
||||
start_import(socket, import_state)
|
||||
else
|
||||
{:error, reason} when is_binary(reason) ->
|
||||
|
|
|
|||
|
|
@ -1980,11 +1980,6 @@ msgstr " (Datenfeld: %{field})"
|
|||
msgid "CSV File"
|
||||
msgstr "CSV Datei"
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "CSV files only, maximum 10 MB"
|
||||
msgstr "Nur CSV Dateien, maximal 10 MB"
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Download CSV templates:"
|
||||
|
|
@ -2263,6 +2258,11 @@ msgstr "Nicht berechtigt."
|
|||
msgid "Could not load data fields. Please check your permissions."
|
||||
msgstr "Datenfelder konnten nicht geladen werden. Bitte überprüfen Sie Ihre Berechtigungen."
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format, fuzzy
|
||||
msgid "CSV files only, maximum %{size} MB"
|
||||
msgstr "Nur CSV Dateien, maximal %{size} MB"
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(ISO-8601 format: YYYY-MM-DD)"
|
||||
|
|
|
|||
|
|
@ -1981,11 +1981,6 @@ msgstr ""
|
|||
msgid "CSV File"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "CSV files only, maximum 10 MB"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Download CSV templates:"
|
||||
|
|
@ -2264,6 +2259,11 @@ msgstr ""
|
|||
msgid "Could not load data fields. Please check your permissions."
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "CSV files only, maximum %{size} MB"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(ISO-8601 format: YYYY-MM-DD)"
|
||||
|
|
|
|||
|
|
@ -1981,11 +1981,6 @@ msgstr ""
|
|||
msgid "CSV File"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "CSV files only, maximum 10 MB"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "Download CSV templates:"
|
||||
|
|
@ -2264,6 +2259,12 @@ msgstr ""
|
|||
msgid "Could not load data fields. Please check your permissions."
|
||||
msgstr ""
|
||||
|
||||
|
||||
#: lib/mv_web/live/global_settings_live.ex
|
||||
#, elixir-autogen, elixir-format, fuzzy
|
||||
msgid "CSV files only, maximum %{size} MB"
|
||||
msgstr ""
|
||||
|
||||
#: lib/mv/membership/import/member_csv.ex
|
||||
#, elixir-autogen, elixir-format
|
||||
msgid "(ISO-8601 format: YYYY-MM-DD)"
|
||||
|
|
|
|||
73
test/mv_web/live/global_settings_live_config_test.exs
Normal file
73
test/mv_web/live/global_settings_live_config_test.exs
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
defmodule MvWeb.GlobalSettingsLiveConfigTest do
|
||||
@moduledoc """
|
||||
Tests for GlobalSettingsLive that modify global Application configuration.
|
||||
|
||||
These tests run with `async: false` to prevent race conditions when
|
||||
modifying global Application environment variables (Application.put_env).
|
||||
This follows the same pattern as Mv.ConfigTest.
|
||||
"""
|
||||
use MvWeb.ConnCase, async: false
|
||||
import Phoenix.LiveViewTest
|
||||
|
||||
# Helper function to upload CSV file in tests
|
||||
defp upload_csv_file(view, csv_content, filename \\ "test_import.csv") do
|
||||
view
|
||||
|> file_input("#csv-upload-form", :csv_file, [
|
||||
%{
|
||||
last_modified: System.system_time(:second),
|
||||
name: filename,
|
||||
content: csv_content,
|
||||
size: byte_size(csv_content),
|
||||
type: "text/csv"
|
||||
}
|
||||
])
|
||||
|> render_upload(filename)
|
||||
end
|
||||
|
||||
describe "CSV Import - Configuration Tests" do
|
||||
setup %{conn: conn} do
|
||||
# Ensure admin user
|
||||
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
|
||||
conn = MvWeb.ConnCase.conn_with_password_user(conn, admin_user)
|
||||
|
||||
{:ok, conn: conn, admin_user: admin_user}
|
||||
end
|
||||
|
||||
test "configured row limit is enforced", %{conn: conn} do
|
||||
# Business rule: CSV import respects configured row limits
|
||||
# Test that a custom limit (500) is enforced, not just the default (1000)
|
||||
original_config = Application.get_env(:mv, :csv_import, [])
|
||||
|
||||
try do
|
||||
Application.put_env(:mv, :csv_import, max_rows: 500)
|
||||
|
||||
{:ok, view, _html} = live(conn, ~p"/settings")
|
||||
|
||||
# Generate CSV with 501 rows (exceeding custom limit of 500)
|
||||
header = "first_name;last_name;email;street;postal_code;city\n"
|
||||
|
||||
rows =
|
||||
for i <- 1..501 do
|
||||
"Row#{i};Last#{i};email#{i}@example.com;Street#{i};12345;City#{i}\n"
|
||||
end
|
||||
|
||||
large_csv = header <> Enum.join(rows)
|
||||
|
||||
# Simulate file upload using helper function
|
||||
upload_csv_file(view, large_csv, "too_many_rows_custom.csv")
|
||||
|
||||
view
|
||||
|> form("#csv-upload-form", %{})
|
||||
|> render_submit()
|
||||
|
||||
html = render(view)
|
||||
# Business rule: import should be rejected when exceeding configured limit
|
||||
assert html =~ "exceeds" or html =~ "maximum" or html =~ "limit" or
|
||||
html =~ "Failed to prepare"
|
||||
after
|
||||
# Restore original config
|
||||
Application.put_env(:mv, :csv_import, original_config)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue