local-it/mitgliederverwaltung
2
3
Fork 0
Code Issues 72 Pull requests 3 Projects 4 Releases Packages 1 Activity Actions

refactor: reduce complexity

Browse source
This commit is contained in:
carla 2026-01-15 17:00:17 +01:00
parent 67072f0c52
commit 6dc398fa5a
3 changed files with 92 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

84
lib/mv/membership/import/header_mapper.ex
View file

@ -97,17 +97,33 @@ defmodule Mv.Membership.Import.HeaderMapper do
}
# Build reverse map: normalized_variant -> canonical_field
# This is computed at runtime on first access and cached
# Cached on first access for performance
defp normalized_to_canonical do
cached = Process.get({__MODULE__, :normalized_to_canonical})
if cached do
cached
else
map = build_normalized_to_canonical_map()
Process.put({__MODULE__, :normalized_to_canonical}, map)
map
end
end
# Builds the normalized variant -> canonical field map
defp build_normalized_to_canonical_map do
@member_field_variants_raw
|> Enum.flat_map(fn {canonical, variants} ->
Enum.map(variants, fn variant ->
{normalize_header(variant), canonical}
end)
end)
|> Enum.flat_map(&map_variants_to_normalized/1)
|> Map.new()
end
# Maps a canonical field and its variants to normalized tuples
defp map_variants_to_normalized({canonical, variants}) do
Enum.map(variants, fn variant ->
{normalize_header(variant), canonical}
end)
end
@doc """
Normalizes a CSV header string for comparison.
@ -208,9 +224,10 @@ defmodule Mv.Membership.Import.HeaderMapper do
|> String.replace(<<0x2212::utf8>>, "-")
end
# Normalizes punctuation: parentheses, slashes become spaces
# Normalizes punctuation: parentheses, slashes, underscores become spaces
defp normalize_punctuation(str) do
str
|> String.replace("_", " ")
|> String.replace(~r/[()\[\]{}]/, " ")
|> String.replace(~r/[\/\\]/, " ")
end
@ -228,18 +245,18 @@ defmodule Mv.Membership.Import.HeaderMapper do
result =
headers
|> Enum.with_index()
|> Enum.reduce_while({%{}, [], %{}}, fn {header, index}, {acc_map, acc_unknown, acc_seen} ->
|> Enum.reduce_while({%{}, []}, fn {header, index}, {acc_map, acc_unknown} ->
normalized = normalize_header(header)
case process_member_header(header, index, normalized, acc_map, acc_seen) do
case process_member_header(header, index, normalized, acc_map, %{}) do
{:error, reason} ->
{:halt, {:error, reason}}
{:ok, new_map, new_seen} ->
{:cont, {new_map, acc_unknown, new_seen}}
{:ok, new_map, _} ->
{:cont, {new_map, acc_unknown}}
{:unknown} ->
{:cont, {acc_map, [index | acc_unknown], acc_seen}}
{:cont, {acc_map, [index | acc_unknown]}}
end
end)
@ -247,7 +264,7 @@ defmodule Mv.Membership.Import.HeaderMapper do
{:error, reason} ->
{:error, reason}
{member_map, unknown_indices, _normalized_seen} ->
{member_map, unknown_indices} ->
validate_required_fields(member_map, unknown_indices)
end
end
@ -258,17 +275,17 @@ defmodule Mv.Membership.Import.HeaderMapper do
{:ok, acc_map, acc_seen}
end
defp process_member_header(_header, index, normalized, acc_map, acc_seen) do
if Map.has_key?(normalized_to_canonical(), normalized) do
canonical = normalized_to_canonical()[normalized]
defp process_member_header(_header, index, normalized, acc_map, _acc_seen) do
case Map.get(normalized_to_canonical(), normalized) do
nil ->
{:unknown}
if Map.has_key?(acc_map, canonical) do
{:error, "duplicate header for #{canonical} (normalized: #{normalized})"}
else
{:ok, Map.put(acc_map, canonical, index), Map.put(acc_seen, normalized, canonical)}
end
else
{:unknown}
canonical ->
if Map.has_key?(acc_map, canonical) do
{:error, "duplicate header for #{canonical} (normalized: #{normalized})"}
else
{:ok, Map.put(acc_map, canonical, index), %{}}
end
end
end
@ -295,7 +312,7 @@ defmodule Mv.Membership.Import.HeaderMapper do
result =
unknown_indices
|> Enum.reduce_while({%{}, [], %{}}, fn index, {acc_map, acc_unknown, acc_seen} ->
|> Enum.reduce_while({%{}, []}, fn index, {acc_map, acc_unknown} ->
header = Enum.at(headers, index)
normalized = normalize_header(header)
@ -305,16 +322,16 @@ defmodule Mv.Membership.Import.HeaderMapper do
normalized,
custom_field_lookup,
acc_map,
acc_seen
%{}
) do
{:error, reason} ->
{:halt, {:error, reason}}
{:ok, new_map, new_seen} ->
{:cont, {new_map, acc_unknown, new_seen}}
{:ok, new_map, _} ->
{:cont, {new_map, acc_unknown}}
{:unknown} ->
{:cont, {acc_map, [index | acc_unknown], acc_seen}}
{:cont, {acc_map, [index | acc_unknown]}}
end
end)
@ -322,7 +339,7 @@ defmodule Mv.Membership.Import.HeaderMapper do
{:error, reason} ->
{:error, reason}
{custom_map, remaining_unknown, _normalized_seen} ->
{custom_map, remaining_unknown} ->
{:ok, custom_map, Enum.reverse(remaining_unknown)}
end
end
@ -350,10 +367,10 @@ defmodule Mv.Membership.Import.HeaderMapper do
normalized,
_custom_field_lookup,
acc_map,
acc_seen
_acc_seen
)
when normalized == "" do
{:ok, acc_map, acc_seen}
{:ok, acc_map, %{}}
end
defp process_custom_field_header(
@ -362,7 +379,7 @@ defmodule Mv.Membership.Import.HeaderMapper do
normalized,
custom_field_lookup,
acc_map,
acc_seen
_acc_seen
) do
if Map.has_key?(custom_field_lookup, normalized) do
custom_field_id = custom_field_lookup[normalized]
@ -370,8 +387,7 @@ defmodule Mv.Membership.Import.HeaderMapper do
if Map.has_key?(acc_map, custom_field_id) do
{:error, "duplicate custom field header (normalized: #{normalized})"}
else
{:ok, Map.put(acc_map, custom_field_id, index),
Map.put(acc_seen, normalized, custom_field_id)}
{:ok, Map.put(acc_map, custom_field_id, index), %{}}
end
else
{:unknown}

77
lib/mv/membership/import/member_csv.ex
View file

@ -63,6 +63,7 @@ defmodule Mv.Membership.Import.MemberCSV do
chunks: list(list({pos_integer(), map()})),
column_map: %{atom() => non_neg_integer()},
custom_field_map: %{String.t() => non_neg_integer()},
custom_field_lookup: %{String.t() => %{id: String.t(), value_type: atom()}},
warnings: list(String.t())
}
@ -118,11 +119,15 @@ defmodule Mv.Membership.Import.MemberCSV do
:ok <- validate_row_count(rows, max_rows) do
chunks = chunk_rows(rows, maps, chunk_size)
# Build custom field lookup for efficient value processing
custom_field_lookup = build_custom_field_lookup(custom_fields)
{:ok,
%{
chunks: chunks,
column_map: maps.member,
custom_field_map: maps.custom,
custom_field_lookup: custom_field_lookup,
warnings: warnings
}}
end
@ -140,6 +145,15 @@ defmodule Mv.Membership.Import.MemberCSV do
{:error, "Failed to load custom fields: #{Exception.message(e)}"}
end
# Builds custom field lookup map for efficient value processing
defp build_custom_field_lookup(custom_fields) do
custom_fields
|> Enum.reduce(%{}, fn cf, acc ->
id_str = to_string(cf.id)
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type})
end)
end
# Builds header maps using HeaderMapper and collects warnings for unknown custom fields
defp build_header_maps(headers, custom_fields) do
# Convert custom fields to maps with id and name
@ -263,11 +277,13 @@ defmodule Mv.Membership.Import.MemberCSV do
%{String.t() => non_neg_integer()},
keyword()
) :: {:ok, chunk_result()} | {:error, String.t()}
def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, _opts \\ []) do
def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, opts \\ []) do
custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{})
{inserted, failed, errors} =
Enum.reduce(chunk_rows_with_lines, {0, 0, []}, fn {line_number, row_map},
{acc_inserted, acc_failed, acc_errors} ->
case process_row(row_map, line_number) do
case process_row(row_map, line_number, custom_field_lookup) do
{:ok, _member} ->
{acc_inserted + 1, acc_failed, acc_errors}
@ -280,9 +296,13 @@ defmodule Mv.Membership.Import.MemberCSV do
end
# Processes a single row and creates member with custom field values
defp process_row(%{member: member_attrs, custom: custom_attrs}, line_number) do
defp process_row(
%{member: member_attrs, custom: custom_attrs},
line_number,
custom_field_lookup
) do
# Prepare custom field values for Ash
custom_field_values = prepare_custom_field_values(custom_attrs)
custom_field_values = prepare_custom_field_values(custom_attrs, custom_field_lookup)
# Create member with custom field values
member_attrs_with_cf =
@ -314,53 +334,26 @@ defmodule Mv.Membership.Import.MemberCSV do
end
# Prepares custom field values from row map for Ash
defp prepare_custom_field_values(custom_attrs) when is_map(custom_attrs) do
defp prepare_custom_field_values(custom_attrs, custom_field_lookup) when is_map(custom_attrs) do
custom_attrs
|> Enum.filter(fn {_id, value} -> value != nil && value != "" end)
|> Enum.map(fn {custom_field_id_str, value} ->
# Load custom field to get value_type and ensure ID is correct
case load_custom_field_by_id(custom_field_id_str) do
{:ok, custom_field} ->
# Use the actual custom_field.id (UUID) from the database
%{
"custom_field_id" => to_string(custom_field.id),
"value" => format_custom_field_value(value, custom_field.value_type)
}
{:error, _} ->
# Skip if custom field not found
case Map.get(custom_field_lookup, custom_field_id_str) do
nil ->
# Custom field not found, skip
nil
%{id: custom_field_id, value_type: value_type} ->
%{
"custom_field_id" => to_string(custom_field_id),
"value" => format_custom_field_value(value, value_type)
}
end
end)
|> Enum.filter(&(&1 != nil))
end
defp prepare_custom_field_values(_), do: []
# Loads a custom field by ID (string or UUID)
defp load_custom_field_by_id(id) when is_binary(id) do
require Ash.Query
try do
# Try to parse as UUID first
uuid_id =
case Ecto.UUID.cast(id) do
{:ok, uuid} -> uuid
:error -> id
end
custom_field =
Mv.Membership.CustomField
|> Ash.Query.filter(id == ^uuid_id)
|> Ash.read_one!()
{:ok, custom_field}
rescue
_ -> {:error, :not_found}
end
end
defp load_custom_field_by_id(_), do: {:error, :invalid_id}
defp prepare_custom_field_values(_, _), do: []
# Formats a custom field value according to its type
# Uses _union_type and _union_value format as expected by Ash

9
test/mv/membership/import/member_csv_test.exs
View file

@ -72,7 +72,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do
end
end
describe "process_chunk/3" do
describe "process_chunk/4" do
test "function exists and accepts chunk_rows_with_lines, column_map, custom_field_map, and opts" do
chunk_rows_with_lines = [{2, %{member: %{email: "john@example.com"}, custom: %{}}}]
column_map = %{email: 0}
@ -173,7 +173,12 @@ defmodule Mv.Membership.Import.MemberCSVTest do
column_map = %{email: 0}
custom_field_map = %{to_string(custom_field.id) => 1}
opts = []
custom_field_lookup = %{
to_string(custom_field.id) => %{id: custom_field.id, value_type: custom_field.value_type}
}
opts = [custom_field_lookup: custom_field_lookup]
assert {:ok, chunk_result} =
MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)