performance: improvedd ash querying

docs/development-progress-log.md

@@ -329,6 +329,11 @@ end
 
 ---
 
+**PR #208:** *Show custom fields per default in member overview* 🔧
+- added show_in_overview as attribute to custom fields
+- show custom fields in member overview per default
+- can be set to false in the settings for the specific custom field
+
 ## Implementation Decisions
 
 ### Architecture Patterns
@@ -390,6 +395,7 @@ defmodule Mv.Membership.CustomField do
   attribute :value_type, :atom       # :string, :integer, :boolean, :date, :email
   attribute :immutable, :boolean     # Can't change after creation
   attribute :required, :boolean      # All members must have this
+  attribute :show_in_overview, :boolean # "If true, this custom field will be displayed in the member overview table"
 end
 
 # CustomFieldValue stores values

lib/mv_web/live/member_live/index.ex

@@ -26,6 +26,9 @@ defmodule MvWeb.MemberLive.Index do
   """
   use MvWeb, :live_view
 
+  require Ash.Query
+  import Ash.Expr
+
   alias MvWeb.MemberLive.Index.Formatter
 
   # Prefix used in sort field names for custom fields (e.g., "custom_field_<id>")
@@ -40,9 +43,6 @@ defmodule MvWeb.MemberLive.Index do
   @impl true
   def mount(_params, _session, socket) do
     # Load custom fields that should be shown in overview
-    require Ash.Query
-    import Ash.Expr
-
     # Note: Using Ash.read! (bang version) - errors will be handled by Phoenix LiveView
     # and result in a 500 error page. This is appropriate for LiveViews where errors
     # should be visible to the user rather than silently failing.
@@ -209,8 +209,7 @@ defmodule MvWeb.MemberLive.Index do
                 ""
 
               cfv ->
-                formatted = Formatter.format_custom_field_value(cfv.value, custom_field)
-                if formatted == "", do: "", else: formatted
+                Formatter.format_custom_field_value(cfv.value, custom_field)
             end
           end
         }
@@ -296,17 +295,16 @@ defmodule MvWeb.MemberLive.Index do
   #
   # Process:
   # 1. Builds base query with selected fields
-  # 2. Loads custom field values for visible custom fields
+  # 2. Loads custom field values for visible custom fields (filtered at database level)
   # 3. Applies search filter if provided
   # 4. Applies sorting (database-level for regular fields, in-memory for custom fields)
-  # 5. Filters custom field values to only visible ones (reduces memory usage)
   #
   # Performance Considerations:
+  # - Database-level filtering: Custom field values are filtered directly in the database
+  #   using Ash relationship filters, reducing memory usage and improving performance.
   # - In-memory sorting: Custom field sorting is done in memory after loading.
   #   This is suitable for small to medium datasets (<1000 members).
   #   For larger datasets, consider implementing database-level sorting or pagination.
-  # - Memory filtering: Custom field values are filtered after loading to reduce
-  #   memory usage, but all members are still loaded into memory.
   # - No pagination: All matching members are loaded at once. For large result sets,
   #   consider implementing pagination (see Issue #165).
   #
@@ -329,8 +327,8 @@ defmodule MvWeb.MemberLive.Index do
       ])
 
     # Load custom field values for visible custom fields
-    custom_field_ids = Enum.map(socket.assigns.custom_fields_visible, & &1.id)
-    query = load_custom_field_values(query, custom_field_ids)
+    custom_field_ids_list = Enum.map(socket.assigns.custom_fields_visible, & &1.id)
+    query = load_custom_field_values(query, custom_field_ids_list)
 
     # Apply the search filter first
     query = apply_search_filter(query, search_query)
@@ -349,13 +347,8 @@ defmodule MvWeb.MemberLive.Index do
     # This is appropriate for data loading in LiveViews
     members = Ash.read!(query)
 
-    # Filter custom field values to only visible ones (reduces memory usage)
-    # Performance: This iterates through all members and their custom_field_values.
-    # For large datasets (>1000 members), this could be optimized by filtering
-    # at the database level, but requires more complex Ash queries.
-    custom_field_ids = MapSet.new(Enum.map(socket.assigns.custom_fields_visible, & &1.id))
-
-    members = filter_member_custom_field_values(members, custom_field_ids)
+    # Custom field values are already filtered at the database level in load_custom_field_values/2
+    # No need for in-memory filtering anymore
 
     # Sort in memory if needed (for custom fields)
     members =
@@ -374,38 +367,28 @@ defmodule MvWeb.MemberLive.Index do
   end
 
   # Load custom field values for the given custom field IDs
+  #
+  # Filters custom field values directly in the database using Ash relationship filters.
+  # This is more efficient than loading all values and filtering in memory.
+  #
+  # Performance: Database-level filtering reduces:
+  # - Memory usage (only visible custom field values are loaded)
+  # - Network transfer (less data from database to application)
+  # - Processing time (no need to iterate through all members and filter)
   defp load_custom_field_values(query, []) do
     query
   end
 
   defp load_custom_field_values(query, custom_field_ids) when length(custom_field_ids) > 0 do
-    # Load all custom field values with their custom_field relationship
-    # Note: We filter to visible custom fields after loading to reduce memory usage
-    # Ash loads relationships efficiently with JOINs, but we only keep visible ones
+    # Filter custom field values at the database level using Ash relationship query
+    # This ensures only visible custom field values are loaded
+    custom_field_values_query =
+      Mv.Membership.CustomFieldValue
+      |> Ash.Query.filter(expr(custom_field_id in ^custom_field_ids))
+      |> Ash.Query.load(custom_field: [:id, :name, :value_type])
+
     query
-    |> Ash.Query.load(custom_field_values: [custom_field: [:id, :name, :value_type]])
-  end
-
-  # Filters custom field values to only visible ones for all members
-  defp filter_member_custom_field_values(members, custom_field_ids) do
-    Enum.map(members, fn member ->
-      filter_single_member_custom_field_values(member, custom_field_ids)
-    end)
-  end
-
-  # Filters custom field values for a single member
-  defp filter_single_member_custom_field_values(member, _custom_field_ids)
-       when not is_list(member.custom_field_values) do
-    member
-  end
-
-  defp filter_single_member_custom_field_values(member, custom_field_ids) do
-    filtered_values =
-      Enum.filter(member.custom_field_values, fn cfv ->
-        cfv.custom_field_id in custom_field_ids
-      end)
-
-    %{member | custom_field_values: filtered_values}
+    |> Ash.Query.load(custom_field_values: custom_field_values_query)
   end
 
   # -------------------------------------------------------------

lib/mv_web/live/member_live/index/formatter.ex

@@ -45,16 +45,12 @@ defmodule MvWeb.MemberLive.Index.Formatter do
   end
 
   # Format value based on type
-  defp format_value_by_type(value, :string, _) when is_binary(value) do
-    # Return empty string if value is empty, otherwise return the value
-    if String.trim(value) == "", do: "", else: value
-  end
 
   defp format_value_by_type(value, :string, _), do: to_string(value)
 
   defp format_value_by_type(value, :integer, _), do: to_string(value)
 
-  defp format_value_by_type(value, :email, _) when is_binary(value) do
+  defp format_value_by_type(value, type, _) when type in [:string, :email] and is_binary(value) do
     # Return empty string if value is empty
     if String.trim(value) == "", do: "", else: value
   end