Merge pull request #433 from foodcoops/feature/fix-document-preloading

Speedup generating large PDFs

app/documents/order_by_articles.rb

@@ -1,22 +1,28 @@
 # encoding: utf-8
 class OrderByArticles < OrderPdf
 
+  # optimal value depends on the average number of ordergroups ordering an article
+  #   as well as the available memory
+  BATCH_SIZE = 50
+
+  attr_reader :order
+
   def filename
-    I18n.t('documents.order_by_articles.filename', :name => @order.name, :date => @order.ends.to_date) + '.pdf'
+    I18n.t('documents.order_by_articles.filename', :name => order.name, :date => order.ends.to_date) + '.pdf'
   end
 
   def title
-    I18n.t('documents.order_by_articles.title', :name => @order.name,
+    I18n.t('documents.order_by_articles.title', :name => order.name,
-      :date => @order.ends.strftime(I18n.t('date.formats.default')))
+      :date => order.ends.strftime(I18n.t('date.formats.default')))
   end
 
   def body
-    @order.order_articles.ordered.each do |order_article|
+    each_order_article do |order_article|
       down_or_page
 
       rows = []
       dimrows = []
-      for goa in order_article.group_order_articles.ordered
+      each_group_order_article_for(order_article) do |goa|
         rows << [goa.group_order.ordergroup_name,
                   "#{goa.quantity} + #{goa.tolerance}",
                  goa.result,
@@ -44,4 +50,27 @@ class OrderByArticles < OrderPdf
     end
   end
 
+  private
+
+  def order_articles
+    order.order_articles.ordered.
+      joins(:article).
+      preload(:article_price). # don't join but preload article_price, just in case it went missing
+      preload(:group_order_articles => {:group_order => :ordergroup})
+  end
+
+  def each_order_article
+    order_articles.find_each_with_order(batch_size: BATCH_SIZE) {|oa| yield oa }
+  end
+
+  def group_order_articles_for(order_article)
+    goas = order_article.group_order_articles.to_a
+    goas.sort_by! {|goa| goa.group_order.ordergroup_name }
+    goas
+  end
+
+  def each_group_order_article_for(group_order)
+    group_order_articles_for(group_order).each {|goa| yield goa }
+  end
+
 end

app/documents/order_by_groups.rb

@@ -1,26 +1,31 @@
 # encoding: utf-8
 class OrderByGroups < OrderPdf
 
+  # optimal value depends on the number of articles ordered on average by each
+  #   ordergroup as well as the available memory
+  BATCH_SIZE = 50
+
+  attr_reader :order
+
   def filename
-    I18n.t('documents.order_by_groups.filename', :name => @order.name, :date => @order.ends.to_date) + '.pdf'
+    I18n.t('documents.order_by_groups.filename', :name => order.name, :date => order.ends.to_date) + '.pdf'
   end
 
   def title
-    I18n.t('documents.order_by_groups.title', :name => @order.name,
+    I18n.t('documents.order_by_groups.title', :name => order.name,
-      :date => @order.ends.strftime(I18n.t('date.formats.default')))
+      :date => order.ends.strftime(I18n.t('date.formats.default')))
   end
 
   def body
     # Start rendering
-    @order.group_orders.ordered.each do |group_order|
+    each_group_order do |group_order|
       down_or_page 15
 
       total = 0
       rows = []
       dimrows = []
 
-      group_order_articles = group_order.group_order_articles.ordered
+      each_group_order_article_for(group_order) do |goa|
-      group_order_articles.each do |goa|
         price = goa.order_article.price.fc_price
         sub_total = price * goa.result
         total += sub_total
@@ -65,4 +70,27 @@ class OrderByGroups < OrderPdf
     end
 
   end
+
+  private
+
+  def group_orders
+    order.group_orders.ordered.
+      joins(:ordergroup).order('groups.name').
+      preload(:group_order_articles => {:order_article => [:article, :article_price]})
+  end
+
+  def each_group_order
+    group_orders.find_each_with_order(batch_size: BATCH_SIZE) {|go| yield go }
+  end
+
+  def group_order_articles_for(group_order)
+    goas = group_order.group_order_articles.to_a
+    goas.sort_by!(&:id)
+    goas
+  end
+
+  def each_group_order_article_for(group_order)
+    group_order_articles_for(group_order).each {|goa| yield goa }
+  end
+
 end

app/documents/order_fax.rb

@@ -1,8 +1,12 @@
 # encoding: utf-8
 class OrderFax < OrderPdf
 
+  BATCH_SIZE = 250
+
+  attr_reader :order
+
   def filename
-    I18n.t('documents.order_fax.filename', :name => @order.name, :date => @order.ends.to_date) + '.pdf'
+    I18n.t('documents.order_fax.filename', :name => order.name, :date => order.ends.to_date) + '.pdf'
   end
 
   def title
@@ -20,8 +24,8 @@ class OrderFax < OrderPdf
       move_down 5
       text "#{contact[:zip_code]} #{contact[:city]}", size: fontsize(9), align: :right
       move_down 5
-      unless @order.supplier.try(:customer_number).blank?
+      unless order.supplier.try(:customer_number).blank?
-        text "#{Supplier.human_attribute_name :customer_number}: #{@order.supplier[:customer_number]}", size: fontsize(9), align: :right
+        text "#{Supplier.human_attribute_name :customer_number}: #{order.supplier[:customer_number]}", size: fontsize(9), align: :right
         move_down 5
       end
       unless contact[:phone].blank?
@@ -35,12 +39,12 @@ class OrderFax < OrderPdf
 
     # Recipient
     bounding_box [margin_box.left,margin_box.top-60], width: 200 do
-      text @order.name
+      text order.name
       move_down 5
-      text @order.supplier.try(:address).to_s
+      text order.supplier.try(:address).to_s
-      unless @order.supplier.try(:fax).blank?
+      unless order.supplier.try(:fax).blank?
         move_down 5
-        text "#{Supplier.human_attribute_name :fax}: #{@order.supplier[:fax]}"
+        text "#{Supplier.human_attribute_name :fax}: #{order.supplier[:fax]}"
       end
     end
 
@@ -50,24 +54,24 @@ class OrderFax < OrderPdf
     move_down 10
     text "#{Delivery.human_attribute_name :delivered_on}:"
     move_down 10
-    unless @order.supplier.try(:contact_person).blank?
+    unless order.supplier.try(:contact_person).blank?
-      text "#{Supplier.human_attribute_name :contact_person}: #{@order.supplier[:contact_person]}"
+      text "#{Supplier.human_attribute_name :contact_person}: #{order.supplier[:contact_person]}"
       move_down 10
     end
 
     # Articles
     total = 0
     data = [I18n.t('documents.order_fax.rows')]
-    data += @order.order_articles.ordered.includes(:article).order('articles.order_number, articles.name').collect do |a|
+    each_order_article do |oa|
-      subtotal = a.units_to_order * a.price.unit_quantity * a.price.price
+      subtotal = oa.units_to_order * oa.price.unit_quantity * oa.price.price
       total += subtotal
-      [a.article.order_number,
+      data << [oa.article.order_number,
-       a.units_to_order,
+               oa.units_to_order,
-       a.article.name,
+               oa.article.name,
-       a.price.unit_quantity,
+               oa.price.unit_quantity,
-       a.article.unit,
+               oa.article.unit,
-       number_to_currency(a.price.price),
+               number_to_currency(oa.price.price),
-       number_to_currency(subtotal)]
+               number_to_currency(subtotal)]
     end
     data << [I18n.t('documents.order_fax.total'), nil, nil, nil, nil, nil, number_to_currency(total)]
     table data, cell_style: {size: fontsize(8), overflow: :shrink_to_fit} do |table|
@@ -89,4 +93,17 @@ class OrderFax < OrderPdf
               #align: {0 => :left}
   end
 
+  private
+
+  def order_articles
+    order.order_articles.ordered.
+      joins(:article).
+      order('articles.order_number').order('articles.name').
+      preload(:article, :article_price)
+  end
+
+  def each_order_article
+    order_articles.find_each_with_order(batch_size: BATCH_SIZE) {|oa| yield oa }
+  end
+
 end

app/models/concerns/find_each_with_order.rb

@@ -0,0 +1,39 @@
+# @see https://gist.github.com/virtualstaticvoid/8705533
+module FindEachWithOrder
+  extend ActiveSupport::Concern
+
+  class_methods do
+
+    def find_each_with_order(options = {})
+      find_in_batches_with_order(options) do |records|
+        records.each { |record| yield record }
+      end
+    end
+
+    # NOTE: any limit() on the query is overridden with the batch size
+    def find_in_batches_with_order(options = {})
+      options.assert_valid_keys(:batch_size)
+
+      relation = self
+
+      start = 0
+      batch_size = options.delete(:batch_size) || 1000
+
+      relation = relation.limit(batch_size)
+      records = relation.offset(start).to_a
+
+      while records.any?
+        records_size = records.size
+
+        yield records
+
+        break if records_size < batch_size
+
+        # get the next batch
+        start += batch_size
+        records = relation.offset(start).to_a
+      end
+    end
+
+  end
+end

app/models/group.rb

@@ -2,6 +2,7 @@
 # Groups organize the User.
 # A Member gets the roles from the Group
 class Group < ActiveRecord::Base
+  include FindEachWithOrder
   include MarkAsDeletedWithName
 
   has_many :memberships, dependent: :destroy

app/models/group_order.rb

@@ -1,5 +1,6 @@
 # A GroupOrder represents an Order placed by an Ordergroup.
 class GroupOrder < ActiveRecord::Base
+  include FindEachWithOrder
 
   attr_accessor :group_order_articles_attributes
 

app/models/order_article.rb

@@ -1,5 +1,6 @@
 # An OrderArticle represents a single Article that is part of an Order.
 class OrderArticle < ActiveRecord::Base
+  include FindEachWithOrder
 
   attr_reader :update_global_price
 

plugins/current_orders/app/documents/multiple_orders_by_articles.rb

@@ -2,28 +2,30 @@
 class MultipleOrdersByArticles < OrderPdf
   include OrdersHelper
 
+  # optimal value depends on the average number of ordergroups ordering an article
+  #   as well as the available memory
+  BATCH_SIZE = 50
+
+  attr_reader :order
+
   def filename
-    I18n.t('documents.multiple_orders_by_articles.filename', count: @order.count) + '.pdf'
+    I18n.t('documents.multiple_orders_by_articles.filename', count: order.count) + '.pdf'
   end
 
   def title
-    I18n.t('documents.multiple_orders_by_articles.title', count: @order.count)
+    I18n.t('documents.multiple_orders_by_articles.title', count: order.count)
-  end
-
-  def order_articles
-    @order_articles ||= OrderArticle.joins(:order, :article).where(:orders => {:id => @order}).ordered.reorder('orders.id, articles.name')
   end
 
   # @todo refactor to reduce common code with order_by_articles
   def body
-    order_articles.each do |order_article|
+    each_order_article do |order_article|
       down_or_page
 
       rows = []
       dimrows = []
       has_units_str = ''
-      for goa in order_article.group_order_articles.ordered
+      each_group_order_article_for(order_article) do |goa|
-        rows << [goa.group_order.ordergroup.name,
+        rows << [goa.group_order.ordergroup_name,
                  goa.tolerance > 0 ? "#{goa.quantity} + #{goa.tolerance}" : goa.quantity,
                  goa.result,
                  number_to_currency(goa.total_price(order_article))]
@@ -70,4 +72,26 @@ class MultipleOrdersByArticles < OrderPdf
     super 'order_by_articles'
   end
 
+  def order_articles
+    OrderArticle.where(order_id: order).ordered.
+      includes(:article).references(:article).
+      reorder('order_articles.order_id, articles.name').
+      preload(:article_price). # preload not join, just in case it went missing
+      preload(:order, :group_order_articles => {:group_order => :ordergroup})
+  end
+
+  def each_order_article
+    order_articles.find_each_with_order(batch_size: BATCH_SIZE) {|oa| yield oa }
+  end
+
+  def group_order_articles_for(order_article)
+    goas = order_article.group_order_articles.to_a
+    goas.sort_by! {|goa| goa.group_order.ordergroup_name }
+    goas
+  end
+
+  def each_group_order_article_for(group_order)
+    group_order_articles_for(group_order).each {|goa| yield goa }
+  end
+
 end

plugins/current_orders/app/documents/multiple_orders_by_groups.rb

@@ -2,36 +2,34 @@
 class MultipleOrdersByGroups < OrderPdf
   include OrdersHelper
 
+  # optimal value depends on the number of articles ordered on average by each
+  #   ordergroup as well as the available memory
+  BATCH_SIZE = 50
+
+  attr_reader :order
+
   def filename
-    I18n.t('documents.multiple_orders_by_groups.filename', count: @order.count) + '.pdf'
+    I18n.t('documents.multiple_orders_by_groups.filename', count: order.count) + '.pdf'
   end
 
   def title
-    I18n.t('documents.multiple_orders_by_groups.title', count: @order.count)
+    I18n.t('documents.multiple_orders_by_groups.title', count: order.count)
-  end
-
-  def ordergroups
-    unless @ordergroups
-      @ordergroups = Ordergroup.joins(:orders).where(orders: {id: @order}).select('distinct(groups.id)').select('groups.*').reorder(:name)
-      @ordergroups = @ordergroups.where(id: @options[:ordergroup]) if @options[:ordergroup]
-    end
-    @ordergroups
   end
 
   # @todo refactor to reduce common code with order_by_groups
   def body
     # Start rendering
-    ordergroups.each do |ordergroup|
+    each_ordergroup do |ordergroup|
       down_or_page 15
 
       total = 0
       taxes = Hash.new {0}
       rows = []
       dimrows = []
-      group_order_articles = GroupOrderArticle.ordered.joins(:group_order => :order).where(:group_orders =>{:ordergroup_id => ordergroup.id}).where(:orders => {id: @order}).includes(:order_article => :article_price).reorder('orders.id')
+      has_tolerance = false
-      has_tolerance = group_order_articles.where('article_prices.unit_quantity > 1').any?
 
-      group_order_articles.each do |goa|
+      each_group_order_article_for(ordergroup)  do |goa|
+        has_tolerance = true if goa.order_article.price.unit_quantity > 1
         price = goa.order_article.price.fc_price
         sub_total = goa.total_price
         total += sub_total
@@ -42,7 +40,7 @@ class MultipleOrdersByGroups < OrderPdf
                   goa.tolerance > 0 ? "#{goa.quantity} + #{goa.tolerance}" : goa.quantity,
                   goa.result,
                   number_to_currency(sub_total),
-                  (goa.order_article.price.unit_quantity if has_tolerance)]
+                  goa.order_article.price.unit_quantity]
         dimrows << rows.length if goa.result == 0
       end
       next if rows.length == 0
@@ -59,9 +57,12 @@ class MultipleOrdersByGroups < OrderPdf
         I18n.t('shared.articles.ordered'),
         I18n.t('shared.articles.received'),
         I18n.t('shared.articles_by.price_sum'),
-        has_tolerance ? {image: "#{Rails.root}/app/assets/images/package-bg.png", scale: 0.6, position: :center} : nil
+        {image: "#{Rails.root}/app/assets/images/package-bg.png", scale: 0.6, position: :center}
       ]
 
+      # last column showing unit_quantity is useless if they're all one
+      rows.each {|row| row[-1] = nil} unless has_tolerance
+
       text ordergroup.name, size: fontsize(13), style: :bold
       table rows, width: bounds.width, cell_style: {size: fontsize(8), overflow: :shrink_to_fit} do |table|
         # borders
@@ -103,4 +104,35 @@ class MultipleOrdersByGroups < OrderPdf
     super 'order_by_groups'
   end
 
+  def ordergroups
+    s = Ordergroup.
+          includes(:group_orders).
+          where(group_orders: {order_id: order}).
+          group(:id).
+          reorder(:name)
+    s = s.where(id: @options[:ordergroup]) if @options[:ordergroup]
+    s
+  end
+
+  def each_ordergroup
+    ordergroups.find_in_batches_with_order(batch_size: BATCH_SIZE) do |ordergroups|
+      @group_order_article_batch = GroupOrderArticle.
+        joins(:group_order).
+        where(group_orders: {order_id: order}).
+        where(group_orders: {ordergroup_id: ordergroups.map(&:id)}).
+        order('group_orders.order_id, group_order_articles.id').
+        preload(group_orders: {order: :supplier}).
+        preload(order_article: [:article, :article_price, :order])
+      ordergroups.each {|ordergroup| yield ordergroup }
+    end
+  end
+
+  def group_order_articles_for(ordergroup)
+    @group_order_article_batch.select {|goa| goa.group_order.ordergroup_id == ordergroup.id }
+  end
+
+  def each_group_order_article_for(ordergroup)
+    group_order_articles_for(ordergroup).each {|goa| yield goa }
+  end
+
 end