perf(import): reuse auto-created groups across import chunks

This commit is contained in:
Moritz 2026-06-03 02:32:15 +02:00
parent 68a1a9530a
commit 118b9f8d57
5 changed files with 90 additions and 3 deletions

View file

@ -97,6 +97,20 @@ defmodule Mv.Membership.Import.ImportRunner do
}
end
@doc """
Carries the in-memory group snapshot grown by a chunk back into `import_state`
so the next chunk reuses groups created earlier instead of re-reading the
Group table. When the chunk result omits `groups_found`, the state is returned
unchanged.
"""
@spec carry_groups_forward(map(), map()) :: map()
def carry_groups_forward(import_state, chunk_result) do
case Map.fetch(chunk_result, :groups_found) do
{:ok, groups_found} -> Map.put(import_state, :groups_found, groups_found)
:error -> import_state
end
end
@doc """
Returns the next action after processing a chunk: send the next chunk index or done.
"""

View file

@ -37,6 +37,9 @@ defmodule Mv.Membership.Import.MemberCSV do
- `inserted` - Number of successfully created members
- `failed` - Number of failed member creations
- `errors` - List of `%MemberCSV.Error{}` structs (capped at 50 per import)
- `groups_found` - The in-memory group snapshot grown while processing this
chunk; thread it into the next chunk's `:groups_found` opt so groups created
in an earlier chunk are reused without re-reading the Group table
## Examples
@ -94,7 +97,8 @@ defmodule Mv.Membership.Import.MemberCSV do
failed: non_neg_integer(),
errors: list(Error.t()),
errors_truncated?: boolean(),
warnings: list(String.t())
warnings: list(String.t()),
groups_found: list(Mv.Membership.Group.t() | %{id: String.t(), name: String.t()})
}
alias Mv.Membership.Import.ColumnResolver
@ -374,7 +378,7 @@ defmodule Mv.Membership.Import.MemberCSV do
actor: actor
}
{inserted, failed, errors, _collected_error_count, truncated?, warnings, _groups_acc} =
{inserted, failed, errors, _collected_error_count, truncated?, warnings, groups_acc} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false, [], groups_found}, fn {line_number,
row_map},
{acc_inserted,
@ -417,7 +421,8 @@ defmodule Mv.Membership.Import.MemberCSV do
failed: failed,
errors: Enum.reverse(errors),
errors_truncated?: truncated?,
warnings: warnings
warnings: warnings,
groups_found: groups_acc
}}
end

View file

@ -367,8 +367,11 @@ defmodule MvWeb.ImportLive do
new_progress =
ImportRunner.merge_progress(progress, chunk_result, idx, max_errors: @max_errors)
new_import_state = ImportRunner.carry_groups_forward(import_state, chunk_result)
socket =
socket
|> assign(:import_state, new_import_state)
|> assign(:import_progress, new_progress)
|> assign(:import_status, new_progress.status)
|> maybe_send_next_chunk(idx, length(import_state.chunks))