perf(import): reuse auto-created groups across import chunks

This commit is contained in:
Moritz 2026-06-03 02:32:15 +02:00
parent 68a1a9530a
commit 118b9f8d57
5 changed files with 90 additions and 3 deletions

View file

@ -3,6 +3,24 @@ defmodule Mv.Membership.Import.ImportRunnerTest do
alias Mv.Membership.Import.ImportRunner
describe "carry_groups_forward/2" do
test "replaces import_state groups_found with the chunk's grown snapshot" do
import_state = %{groups_found: [%{id: "1", name: "A"}]}
chunk_result = %{groups_found: [%{id: "1", name: "A"}, %{id: "2", name: "B"}]}
assert ImportRunner.carry_groups_forward(import_state, chunk_result) == %{
groups_found: [%{id: "1", name: "A"}, %{id: "2", name: "B"}]
}
end
test "leaves import_state unchanged when the chunk result omits groups_found" do
import_state = %{groups_found: [%{id: "1", name: "A"}], other: :kept}
chunk_result = %{inserted: 1}
assert ImportRunner.carry_groups_forward(import_state, chunk_result) == import_state
end
end
describe "read_file_entry/2" do
test "returns {:ok, content} for a readable file" do
path =

View file

@ -1116,6 +1116,53 @@ defmodule Mv.Membership.Import.MemberCSVTest do
"Expected the group table read at most a few times, got #{reads} reads for 10 rows (N+1)."
end
test "returns the grown group snapshot so later chunks skip the table read",
%{actor: actor} do
chunk1 = [
{2, %{member: %{email: "g-xchunk-1@example.com"}, custom: %{}, groups: "Shared X"}}
]
chunk2 = [
{3, %{member: %{email: "g-xchunk-2@example.com"}, custom: %{}, groups: "Shared X"}}
]
assert {:ok, result1} =
MemberCSV.process_chunk(chunk1, %{email: 0}, %{}, actor: actor, groups_found: [])
# The chunk result must expose the accumulated snapshot, including the group
# auto-created while processing this chunk, so the LiveView can thread it
# into the next chunk's opts.
assert is_list(result1.groups_found)
assert Enum.any?(result1.groups_found, &(&1.name == "Shared X"))
group_read_count = Agent.start_link(fn -> 0 end) |> elem(1)
test_pid = self()
handler = fn _event, _measurements, metadata, _config ->
if self() == test_pid and metadata[:source] == "groups" and
is_binary(metadata[:query]) and String.starts_with?(metadata.query, "SELECT") do
Agent.update(group_read_count, &(&1 + 1))
end
end
handler_id = "test-xchunk-group-read-#{System.unique_integer([:positive])}"
:telemetry.attach(handler_id, [:mv, :repo, :query], handler, nil)
assert {:ok, %{inserted: 1}} =
MemberCSV.process_chunk(chunk2, %{email: 0}, %{},
actor: actor,
groups_found: result1.groups_found
)
reads = Agent.get(group_read_count, & &1)
:telemetry.detach(handler_id)
# The second chunk receives the snapshot grown by the first, so the shared
# group resolves from memory without any full-table read.
assert reads == 0,
"Expected no group table read in the second chunk, got #{reads} (snapshot not threaded across chunks)."
end
test "empty groups cell leaves the member without group assignment", %{actor: actor} do
chunk = [{2, %{member: %{email: "g-empty@example.com"}, custom: %{}, groups: " "}}]
opts = [actor: actor, groups_found: []]