feat(member): resolve custom-field filters and sorting in PostgreSQL

Push the boolean and date custom-field predicates and custom-field
sorting down to JSONB expressions so the overview stops classifying
custom-field values in memory. A GIN index on custom_field_values.value
keeps the boolean membership predicates index-served.

test/mv_web/member_live/index_boolean_custom_field_filter_test.exs

@@ -0,0 +1,83 @@
+defmodule MvWeb.MemberLive.IndexBooleanCustomFieldFilterTest do
+  @moduledoc """
+  §1.15 — DB-backed boolean custom-field filter (in/not_in) must match the
+  previous in-memory behaviour exactly, including a member with no stored value
+  row for the field.
+  """
+  use Mv.DataCase, async: false
+
+  import Mv.Fixtures, only: [member_fixture_with_actor: 2]
+
+  alias Mv.Membership.CustomField
+  alias Mv.Membership.CustomFieldValue
+  alias Mv.Membership.Member
+  alias MvWeb.MemberLive.Index
+  alias MvWeb.MemberLive.Index.OverviewQuery
+
+  setup do
+    actor = Mv.Helpers.SystemActor.get_system_actor()
+
+    {:ok, field} =
+      CustomField
+      |> Ash.Changeset.for_create(:create, %{
+        name: "newsletter_#{System.unique_integer([:positive])}",
+        value_type: :boolean,
+        show_in_overview: true
+      })
+      |> Ash.create(actor: actor)
+
+    yes = member_fixture_with_actor(%{}, actor)
+    no = member_fixture_with_actor(%{}, actor)
+    none = member_fixture_with_actor(%{}, actor)
+
+    set_bool(yes, field, true, actor)
+    set_bool(no, field, false, actor)
+
+    %{actor: actor, field: field, yes: yes, no: no, none: none}
+  end
+
+  defp set_bool(member, field, bool, actor) do
+    CustomFieldValue
+    |> Ash.Changeset.for_create(:create, %{
+      member_id: member.id,
+      custom_field_id: field.id,
+      value: %{"_union_type" => "boolean", "_union_value" => bool}
+    })
+    |> Ash.create!(actor: actor)
+  end
+
+  defp db_ids(filters, field, actor) do
+    OverviewQuery.build(%{
+      boolean_custom_field_filters: filters,
+      boolean_custom_fields: [field]
+    })
+    |> Ash.read!(actor: actor)
+    |> MapSet.new(& &1.id)
+  end
+
+  defp oracle_ids(filters, field, actor) do
+    members =
+      Member
+      |> Ash.Query.load(custom_field_values: [:custom_field])
+      |> Ash.read!(actor: actor)
+
+    Index.apply_boolean_custom_field_filters(members, filters, [field])
+    |> MapSet.new(& &1.id)
+  end
+
+  test "filter true matches only members with stored true value", ctx do
+    filters = %{to_string(ctx.field.id) => true}
+    assert db_ids(filters, ctx.field, ctx.actor) == oracle_ids(filters, ctx.field, ctx.actor)
+    assert ctx.yes.id in db_ids(filters, ctx.field, ctx.actor)
+    refute ctx.no.id in db_ids(filters, ctx.field, ctx.actor)
+    refute ctx.none.id in db_ids(filters, ctx.field, ctx.actor)
+  end
+
+  test "filter false matches only members with stored false value (not missing rows)", ctx do
+    filters = %{to_string(ctx.field.id) => false}
+    assert db_ids(filters, ctx.field, ctx.actor) == oracle_ids(filters, ctx.field, ctx.actor)
+    assert ctx.no.id in db_ids(filters, ctx.field, ctx.actor)
+    refute ctx.yes.id in db_ids(filters, ctx.field, ctx.actor)
+    refute ctx.none.id in db_ids(filters, ctx.field, ctx.actor)
+  end
+end

test/mv_web/member_live/index_date_custom_field_filter_test.exs

@@ -0,0 +1,91 @@
+defmodule MvWeb.MemberLive.IndexDateCustomFieldFilterTest do
+  @moduledoc """
+  §1.16 — DB-backed date custom-field range filter must match the previous
+  in-memory behaviour exactly, including inclusive boundaries and members with
+  no stored value row (excluded).
+  """
+  use Mv.DataCase, async: false
+
+  import Mv.Fixtures, only: [member_fixture_with_actor: 2]
+
+  alias Mv.Membership.CustomField
+  alias Mv.Membership.CustomFieldValue
+  alias Mv.Membership.Member
+  alias MvWeb.MemberLive.Index.DateFilter
+  alias MvWeb.MemberLive.Index.OverviewQuery
+
+  setup do
+    actor = Mv.Helpers.SystemActor.get_system_actor()
+
+    {:ok, field} =
+      CustomField
+      |> Ash.Changeset.for_create(:create, %{
+        name: "birthday_#{System.unique_integer([:positive])}",
+        value_type: :date,
+        show_in_overview: true
+      })
+      |> Ash.create(actor: actor)
+
+    on_lower = member_with_date(field, ~D[2024-01-01], actor)
+    inside = member_with_date(field, ~D[2024-06-15], actor)
+    on_upper = member_with_date(field, ~D[2024-12-31], actor)
+    below = member_with_date(field, ~D[2023-12-31], actor)
+    above = member_with_date(field, ~D[2025-01-01], actor)
+    none = member_fixture_with_actor(%{}, actor)
+
+    %{
+      actor: actor,
+      field: field,
+      on_lower: on_lower,
+      inside: inside,
+      on_upper: on_upper,
+      below: below,
+      above: above,
+      none: none
+    }
+  end
+
+  defp member_with_date(field, date, actor) do
+    member = member_fixture_with_actor(%{}, actor)
+
+    CustomFieldValue
+    |> Ash.Changeset.for_create(:create, %{
+      member_id: member.id,
+      custom_field_id: field.id,
+      value: %{"_union_type" => "date", "_union_value" => date}
+    })
+    |> Ash.create!(actor: actor)
+
+    member
+  end
+
+  defp filters(field), do: %{to_string(field.id) => %{from: ~D[2024-01-01], to: ~D[2024-12-31]}}
+
+  defp db_ids(field, actor) do
+    OverviewQuery.build(%{date_filters: filters(field), date_custom_fields: [field]})
+    |> Ash.read!(actor: actor)
+    |> MapSet.new(& &1.id)
+  end
+
+  defp oracle_ids(field, actor) do
+    members =
+      Member
+      |> Ash.Query.load(custom_field_values: [:custom_field])
+      |> Ash.read!(actor: actor)
+
+    DateFilter.apply_in_memory(members, filters(field), [field])
+    |> MapSet.new(& &1.id)
+  end
+
+  test "inclusive range matches in-memory oracle, boundaries included", ctx do
+    db = db_ids(ctx.field, ctx.actor)
+    assert db == oracle_ids(ctx.field, ctx.actor)
+
+    assert ctx.on_lower.id in db
+    assert ctx.inside.id in db
+    assert ctx.on_upper.id in db
+    refute ctx.below.id in db
+    refute ctx.above.id in db
+    refute ctx.none.id in db
+  end
+end

test/mv_web/member_live/index_filter_parity_property_test.exs

@@ -0,0 +1,245 @@
+defmodule MvWeb.MemberLive.IndexFilterParityPropertyTest do
+  @moduledoc """
+  §2.1 — Filter result-set parity.
+
+  For generated member populations × filter-parameter combinations, the set of
+  member ids returned by the DB-backed `:overview` query equals the set returned
+  by the previous in-memory implementation (the reference oracle).
+
+  The unchanged DB filters (group, fee-type) are applied identically in both
+  arms; the property isolates the filters that moved from memory to the DB
+  (boolean custom fields, date custom fields, paid/unpaid cycle status) by
+  applying them DB-side in one arm and in-memory in the other.
+  """
+  use Mv.DataCase, async: false
+  use ExUnitProperties
+
+  import Mv.Fixtures, only: [create_fee_type: 2, member_fixture_with_actor: 2, create_cycle: 4]
+
+  alias Mv.Membership.CustomField
+  alias Mv.Membership.CustomFieldValue
+  alias Mv.Membership.Member
+  alias MvWeb.MemberLive.Index
+  alias MvWeb.MemberLive.Index.DateFilter
+  alias MvWeb.MemberLive.Index.MembershipFeeStatus
+  alias MvWeb.MemberLive.Index.OverviewQuery
+
+  @today ~D[2024-07-15]
+  @cycle_starts [~D[2023-01-01], ~D[2024-01-01], ~D[2025-01-01]]
+  @date_values [~D[2023-06-01], ~D[2024-06-15], ~D[2025-06-01]]
+
+  setup do
+    actor = Mv.Helpers.SystemActor.get_system_actor()
+    ft1 = create_fee_type(%{interval: :yearly}, actor)
+    ft2 = create_fee_type(%{interval: :yearly}, actor)
+
+    {:ok, bool_cf} =
+      CustomField
+      |> Ash.Changeset.for_create(:create, %{
+        name: "flag_#{System.unique_integer([:positive])}",
+        value_type: :boolean,
+        show_in_overview: true
+      })
+      |> Ash.create(actor: actor)
+
+    {:ok, date_cf} =
+      CustomField
+      |> Ash.Changeset.for_create(:create, %{
+        name: "dt_#{System.unique_integer([:positive])}",
+        value_type: :date,
+        show_in_overview: true
+      })
+      |> Ash.create(actor: actor)
+
+    %{actor: actor, ft1: ft1, ft2: ft2, bool_cf: bool_cf, date_cf: date_cf}
+  end
+
+  # ----- generators ---------------------------------------------------------
+
+  defp status_gen, do: StreamData.member_of([:unpaid, :paid, :suspended])
+
+  defp member_spec_gen do
+    StreamData.fixed_map(%{
+      fee_type: StreamData.member_of([:ft1, :ft2]),
+      cycles:
+        StreamData.list_of(
+          StreamData.tuple({StreamData.member_of(@cycle_starts), status_gen()}),
+          max_length: 3
+        ),
+      bool: StreamData.member_of([:none, true, false]),
+      date: StreamData.member_of([:none | @date_values]),
+      in_group: StreamData.boolean()
+    })
+  end
+
+  defp filter_params_gen do
+    StreamData.fixed_map(%{
+      cycle_status: StreamData.member_of([nil, :paid, :unpaid]),
+      show_current: StreamData.boolean(),
+      bool: StreamData.member_of([nil, true, false]),
+      date_range: StreamData.member_of([nil, {~D[2024-01-01], ~D[2024-12-31]}]),
+      group: StreamData.member_of([nil, :in, :not_in]),
+      fee_type: StreamData.member_of([nil, :in, :not_in])
+    })
+  end
+
+  # ----- helpers ------------------------------------------------------------
+
+  defp clear_members(actor) do
+    Member |> Ash.read!(actor: actor) |> Enum.each(&Ash.destroy!(&1, actor: actor))
+  end
+
+  defp create_population(specs, ctx) do
+    Enum.map(specs, fn spec ->
+      ft = if spec.fee_type == :ft1, do: ctx.ft1, else: ctx.ft2
+      member = member_fixture_with_actor(%{membership_fee_type_id: ft.id}, ctx.actor)
+
+      spec.cycles
+      |> Enum.uniq_by(fn {start, _} -> start end)
+      |> Enum.each(fn {start, status} ->
+        create_cycle(member, ft, %{cycle_start: start, status: status}, ctx.actor)
+      end)
+
+      maybe_set_value(member, ctx.bool_cf, spec.bool, "boolean", ctx.actor)
+      maybe_set_value(member, ctx.date_cf, spec.date, "date", ctx.actor)
+
+      if spec.in_group, do: add_to_group(member, ctx, ctx.actor)
+      {member.id, spec}
+    end)
+  end
+
+  defp maybe_set_value(_member, _cf, :none, _type, _actor), do: :ok
+
+  defp maybe_set_value(member, cf, value, type, actor) do
+    CustomFieldValue
+    |> Ash.Changeset.for_create(:create, %{
+      member_id: member.id,
+      custom_field_id: cf.id,
+      value: %{"_union_type" => type, "_union_value" => value}
+    })
+    |> Ash.create!(actor: actor)
+  end
+
+  defp add_to_group(member, ctx, actor) do
+    Mv.Membership.create_member_group(%{member_id: member.id, group_id: ctx.group.id},
+      actor: actor
+    )
+  end
+
+  defp base_opts(params, ctx) do
+    %{}
+    |> put_group(params.group, ctx)
+    |> put_fee_type(params.fee_type, ctx)
+  end
+
+  defp put_group(opts, nil, _ctx), do: opts
+
+  defp put_group(opts, dir, ctx),
+    do: Map.merge(opts, %{group_filters: %{to_string(ctx.group.id) => dir}, groups: [ctx.group]})
+
+  defp put_fee_type(opts, nil, _ctx), do: opts
+
+  defp put_fee_type(opts, dir, ctx),
+    do:
+      Map.merge(opts, %{fee_type_filters: %{to_string(ctx.ft1.id) => dir}, fee_types: [ctx.ft1]})
+
+  defp moved_opts(params, ctx) do
+    %{today: @today}
+    |> put_cycle(params.cycle_status, params.show_current)
+    |> put_bool(params.bool, ctx)
+    |> put_date(params.date_range, ctx)
+  end
+
+  defp put_cycle(opts, nil, _show), do: opts
+
+  defp put_cycle(opts, status, show),
+    do: Map.merge(opts, %{cycle_status_filter: status, show_current_cycle: show})
+
+  defp put_bool(opts, nil, _ctx), do: opts
+
+  defp put_bool(opts, bool, ctx),
+    do:
+      Map.merge(opts, %{
+        boolean_custom_field_filters: %{to_string(ctx.bool_cf.id) => bool},
+        boolean_custom_fields: [ctx.bool_cf]
+      })
+
+  defp put_date(opts, nil, _ctx), do: opts
+
+  defp put_date(opts, {from, to}, ctx),
+    do:
+      Map.merge(opts, %{
+        date_filters: %{to_string(ctx.date_cf.id) => %{from: from, to: to}},
+        date_custom_fields: [ctx.date_cf]
+      })
+
+  defp db_ids(params, ctx) do
+    base_opts(params, ctx)
+    |> Map.merge(moved_opts(params, ctx))
+    |> OverviewQuery.build()
+    |> Ash.read!(actor: ctx.actor)
+    |> MapSet.new(& &1.id)
+  end
+
+  defp oracle_ids(params, ctx) do
+    base_members =
+      base_opts(params, ctx)
+      |> OverviewQuery.build()
+      |> Ash.Query.load([
+        :membership_fee_type,
+        {:custom_field_values, [:custom_field]},
+        {:membership_fee_cycles, [:membership_fee_type]}
+      ])
+      |> Ash.read!(actor: ctx.actor)
+
+    base_members
+    |> apply_oracle_bool(params, ctx)
+    |> apply_oracle_date(params, ctx)
+    |> apply_oracle_cycle(params)
+    |> MapSet.new(& &1.id)
+  end
+
+  defp apply_oracle_bool(members, %{bool: nil}, _ctx), do: members
+
+  defp apply_oracle_bool(members, %{bool: bool}, ctx) do
+    Index.apply_boolean_custom_field_filters(
+      members,
+      %{to_string(ctx.bool_cf.id) => bool},
+      [ctx.bool_cf]
+    )
+  end
+
+  defp apply_oracle_date(members, %{date_range: nil}, _ctx), do: members
+
+  defp apply_oracle_date(members, %{date_range: {from, to}}, ctx) do
+    DateFilter.apply_in_memory(
+      members,
+      %{to_string(ctx.date_cf.id) => %{from: from, to: to}},
+      [ctx.date_cf]
+    )
+  end
+
+  defp apply_oracle_cycle(members, %{cycle_status: nil}), do: members
+
+  defp apply_oracle_cycle(members, %{cycle_status: status, show_current: show}) do
+    Enum.filter(members, fn m ->
+      MembershipFeeStatus.get_cycle_status_for_member(m, show, @today) == status
+    end)
+  end
+
+  property "DB-backed filter result set equals the in-memory oracle", ctx do
+    check all(
+            specs <- StreamData.list_of(member_spec_gen(), min_length: 1, max_length: 5),
+            params <- filter_params_gen(),
+            max_runs: 30
+          ) do
+      clear_members(ctx.actor)
+      group = Mv.Fixtures.group_fixture(%{name: "G#{System.unique_integer([:positive])}"})
+      ctx = Map.put(ctx, :group, group)
+
+      _population = create_population(specs, ctx)
+
+      assert db_ids(params, ctx) == oracle_ids(params, ctx)
+    end
+  end
+end

test/mv_web/member_live/index_overview_query_sort_test.exs

@@ -0,0 +1,98 @@
+defmodule MvWeb.MemberLive.IndexOverviewQuerySortTest do
+  @moduledoc """
+  §2.3 — Sort determinism for custom-field sorts pushed to the DB.
+
+  The keyset property test covers standard fields; these examples pin the
+  type-aware custom-field ordering (numeric vs lexical, chronological dates) and
+  the NULLS-LAST behaviour for missing/empty values in both directions, which
+  the previous in-memory `CustomFieldSort` guaranteed.
+  """
+  use Mv.DataCase, async: false
+
+  import Mv.Fixtures, only: [member_fixture_with_actor: 2]
+
+  alias Mv.Membership.CustomField
+  alias Mv.Membership.CustomFieldValue
+  alias MvWeb.MemberLive.Index.OverviewQuery
+
+  setup do
+    %{actor: Mv.Helpers.SystemActor.get_system_actor()}
+  end
+
+  defp clear_members(actor) do
+    Mv.Membership.Member |> Ash.read!(actor: actor) |> Enum.each(&Ash.destroy!(&1, actor: actor))
+  end
+
+  defp create_field(value_type, actor) do
+    {:ok, field} =
+      CustomField
+      |> Ash.Changeset.for_create(:create, %{
+        name: "sort_#{value_type}_#{System.unique_integer([:positive])}",
+        value_type: value_type,
+        show_in_overview: true
+      })
+      |> Ash.create(actor: actor)
+
+    field
+  end
+
+  defp set_value(member, field, union_type, union_value, actor) do
+    {:ok, _} =
+      CustomFieldValue
+      |> Ash.Changeset.for_create(:create, %{
+        member_id: member.id,
+        custom_field_id: field.id,
+        value: %{"_union_type" => union_type, "_union_value" => union_value}
+      })
+      |> Ash.create(actor: actor)
+  end
+
+  defp sorted_ids(field, order, actor) do
+    OverviewQuery.build(%{
+      sort_field: "custom_field_#{field.id}",
+      sort_order: order,
+      custom_fields: [field]
+    })
+    |> Ash.read!(actor: actor, page: [limit: 100])
+    |> then(& &1.results)
+    |> Enum.map(& &1.id)
+  end
+
+  test "integer custom field sorts numerically, missing value last in both directions", %{
+    actor: actor
+  } do
+    clear_members(actor)
+    field = create_field(:integer, actor)
+    a = member_fixture_with_actor(%{first_name: "A"}, actor)
+    b = member_fixture_with_actor(%{first_name: "B"}, actor)
+    none = member_fixture_with_actor(%{first_name: "N"}, actor)
+    # 9 vs 10: lexical order would place "10" before "9"; numeric must not.
+    set_value(a, field, "integer", 10, actor)
+    set_value(b, field, "integer", 9, actor)
+
+    assert sorted_ids(field, :asc, actor) == [b.id, a.id, none.id]
+    assert sorted_ids(field, :desc, actor) == [a.id, b.id, none.id]
+  end
+
+  test "date custom field sorts chronologically", %{actor: actor} do
+    clear_members(actor)
+    field = create_field(:date, actor)
+    older = member_fixture_with_actor(%{first_name: "Older"}, actor)
+    newer = member_fixture_with_actor(%{first_name: "Newer"}, actor)
+    set_value(older, field, "date", "1981-01-29", actor)
+    set_value(newer, field, "date", "1986-07-02", actor)
+
+    assert sorted_ids(field, :asc, actor) == [older.id, newer.id]
+  end
+
+  test "empty string value sorts last like a missing value", %{actor: actor} do
+    clear_members(actor)
+    field = create_field(:string, actor)
+    filled = member_fixture_with_actor(%{first_name: "Filled"}, actor)
+    empty = member_fixture_with_actor(%{first_name: "Empty"}, actor)
+    set_value(filled, field, "string", "AAA", actor)
+    set_value(empty, field, "string", "", actor)
+
+    assert sorted_ids(field, :asc, actor) == [filled.id, empty.id]
+  end
+end