Merge pull request #361 from foodcoops/feature/spreadsheets

Let upload provide same functionality as shared database sync

Gemfile

@@ -42,6 +42,8 @@ gem 'ruby-units'
 gem 'attribute_normalizer'
 gem 'ice_cube', github: 'wvengen/ice_cube', branch: 'issues/50-from_ical-rebased' # fork until merged
 gem 'recurring_select'
+gem 'roo', '~> 1.13.2'
+gem 'spreadsheet'
 
 # we use the git version of acts_as_versioned, and need to include it in this Gemfile
 gem 'acts_as_versioned', github: 'technoweenie/acts_as_versioned'

Gemfile.lock

@@ -333,6 +333,10 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
+    roo (1.13.2)
+      nokogiri
+      rubyzip
+      spreadsheet (> 0.6.4)
     rspec (2.99.0)
       rspec-core (~> 2.99.0)
       rspec-expectations (~> 2.99.0)
@@ -354,6 +358,7 @@ GEM
       rspec-mocks (~> 2.99.0)
     rspec-rerun (0.3.0)
       rspec
+    ruby-ole (1.2.11.8)
     ruby-prof (0.15.6)
     ruby-units (1.4.5)
     ruby_parser (3.6.5)
@@ -395,6 +400,8 @@ GEM
       eventmachine (~> 1.0.0)
       thin (~> 1.5.0)
     slop (3.6.0)
+    spreadsheet (1.0.0)
+      ruby-ole (>= 1.0)
     sprockets (2.12.3)
       hike (~> 1.2)
       multi_json (~> 1.0)
@@ -510,6 +517,7 @@ DEPENDENCIES
   ransack
   recurring_select
   resque
+  roo (~> 1.13.2)
   rspec-core (~> 2.99)
   rspec-rails
   rspec-rerun
@@ -522,6 +530,7 @@ DEPENDENCIES
   simple-navigation-bootstrap
   simple_form
   simplecov
+  spreadsheet
   sqlite3
   therubyracer
   thin

app/controllers/articles_controller.rb

@@ -35,7 +35,7 @@ class ArticlesController < ApplicationController
     @article = @supplier.articles.build(:tax => FoodsoftConfig[:tax_default])
     render :layout => false
   end
-  
+
   def create
     @article = Article.new(params[:article])
     if @article.valid? && @article.save
@@ -44,12 +44,12 @@ class ArticlesController < ApplicationController
       render :action => 'new', :layout => false
     end
   end
-  
+
   def edit
     @article = Article.find(params[:id])
     render :action => 'new', :layout => false
   end
-  
+
   # Updates one Article and highlights the line if succeded
   def update
     @article = Article.find(params[:id])
@@ -66,8 +66,8 @@ class ArticlesController < ApplicationController
     @article = Article.find(params[:id])
     @article.mark_as_deleted unless @order = @article.in_open_order # If article is in an active Order, the Order will be returned
     render :layout => false
-  end   
+  end
-  
+
   # Renders a form for editing all articles from a supplier
   def edit_all
     @articles = @supplier.articles.undeleted
@@ -102,7 +102,7 @@ class ArticlesController < ApplicationController
       redirect_to supplier_articles_path(@supplier), notice: I18n.t('articles.controller.update_all.notice')
     end
   end
-  
+
   # makes different actions on selected articles
   def update_selected
     raise I18n.t('articles.controller.error_nosel') if params[:selected_articles].nil?
@@ -129,75 +129,83 @@ class ArticlesController < ApplicationController
     redirect_to supplier_articles_url(@supplier, :per_page => params[:per_page]),
                 :alert => I18n.t('errors.general_msg', :msg => error)
   end
- 
+
   # lets start with parsing articles from uploaded file, yeah
   # Renders the upload form
   def upload
   end
- 
+
-  # parses the articles from a csv and creates a form-table with the parsed data.
+  # Update articles from a spreadsheet
-  # the csv must have the following format:
-  # status | number | name | note | manufacturer | origin | unit | clear price | unit_quantity | tax | deposit | scale quantity | scale price | category
-  # the first line will be ignored. 
-  # field-seperator: ";"
-  # text-seperator: ""
   def parse_upload
-    begin
+    uploaded_file = params[:articles]['file'] or raise I18n.t('articles.controller.parse_upload.no_file')
-      @articles = Array.new
+    options = {filename: uploaded_file.original_filename}
-      articles, outlisted_articles = FoodsoftFile::parse(params[:articles]["file"])
+    options[:outlist_absent] = (params[:articles]['outlist_absent'] == '1')
-      no_category = ArticleCategory.new
+    options[:convert_units] = (params[:articles]['convert_units'] == '1')
-      articles.each do |row|
+    @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_from_file uploaded_file.tempfile, options
-        # fallback to Others category
+    if @updated_article_pairs.empty? && @outlisted_articles.empty? && @new_articles.empty?
-        category = (ArticleCategory.find_match(row[:category]) || no_category)
+      redirect_to supplier_articles_path(@supplier), :notice => I18n.t('articles.controller.parse_upload.notice')
-        # creates a new article and price
+    end
-        article = @supplier.articles.build(:name => row[:name], 
+    @ignored_article_count = 0
-                               :note => row[:note],
+  rescue => error
-                               :manufacturer => row[:manufacturer],
+    redirect_to upload_supplier_articles_path(@supplier), :alert => I18n.t('errors.general_msg', :msg => error.message)
-                               :origin => row[:origin],
+  end
-                               :unit => row[:unit],
+
-                               :article_category => category,
+  # sync all articles with the external database
-                               :price => row[:price],
+  # renders a form with articles, which should be updated
-                               :unit_quantity => row[:unit_quantity],
+  def sync
-                               :order_number => row[:number],
+    # check if there is an shared_supplier
-                               :deposit => row[:deposit],
+    unless @supplier.shared_supplier
-                               :tax => (row[:tax] || FoodsoftConfig[:tax_default]))
+      redirect_to supplier_articles_url(@supplier), :alert => I18n.t('articles.controller.sync.shared_alert', :supplier => @supplier.name)
-        # stop parsing, when an article isn't valid
+    end
-        unless article.valid?
+    # sync articles against external database
-          raise I18n.t('articles.controller.error_parse', :msg => article.errors.full_messages.join(", "), :line => (articles.index(row) + 2).to_s)
+    @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_all
-        end
+    if @updated_article_pairs.empty? && @outlisted_articles.empty? && @new_articles.empty?
-        @articles << article
+      redirect_to supplier_articles_path(@supplier), :notice => I18n.t('articles.controller.sync.notice')
-      end
-      flash.now[:notice] = I18n.t('articles.controller.parse_upload.notice', :count => @articles.size)
-    rescue => error
-      redirect_to upload_supplier_articles_path(@supplier), :alert => I18n.t('errors.general_msg', :msg => error.message)
     end
   end
- 
-  # creates articles from form
-  def create_from_upload
-    begin
-      Article.transaction do
-        invalid_articles = false
-        @articles = []
-        params[:articles].each do |_key, article_attributes|
-          @articles << (article = @supplier.articles.build(article_attributes))
-          invalid_articles = true unless article.save
-        end
 
-        raise I18n.t('articles.controller.error_invalid') if invalid_articles
+  # Updates, deletes articles when upload or sync form is submitted
+  def update_synchronized
+    @outlisted_articles = Article.find(params[:outlisted_articles].try(:keys)||[])
+    @updated_articles = Article.find(params[:articles].try(:keys)||[])
+    @updated_articles.map{|a| a.assign_attributes(params[:articles][a.id.to_s]) }
+    @new_articles = (params[:new_articles]||[]).map{|a| @supplier.articles.build(a) }
+
+    has_error = false
+    Article.transaction do
+      # delete articles
+      begin
+        @outlisted_articles.each(&:mark_as_deleted)
+      rescue
+        # raises an exception when used in current order
+        has_error = true
+      end
+      # Update articles
+      @updated_articles.map{|a| a.save or has_error=true }
+      # Add new articles
+      @new_articles.each do |article|
+        article.availability = true if @supplier.shared_sync_method == 'all_available'
+        article.availability = false if @supplier.shared_sync_method == 'all_unavailable'
+        article.save or has_error=true
       end
-      # Successfully done.
-      redirect_to supplier_articles_path(@supplier), notice: I18n.t('articles.controller.create_from_upload.notice', :count => @articles.size)
 
-    rescue => error
+      raise ActiveRecord::Rollback if has_error
-      # An error has occurred, transaction has been rolled back.
+    end
-      flash.now[:error] = I18n.t('errors.general_msg', :msg => error.message)
+
-      render :parse_upload
+    if !has_error
+      redirect_to supplier_articles_path(@supplier), notice: I18n.t('articles.controller.update_sync.notice')
+    else
+      @updated_article_pairs = @updated_articles.map do |article|
+        orig_article = Article.find(article.id)
+        [article, orig_article.unequal_attributes(article)]
+      end
+      flash.now.alert = I18n.t('articles.controller.error_invalid')
+      render params[:from_action] == 'sync' ? :sync : :parse_upload
     end
   end
-  
+
   # renders a view to import articles in local database
-  #   
+  #
   def shared
     # build array of keywords, required for ransack _all suffix
     params[:q][:name_cont_all] = params[:q][:name_cont_all].split(' ') if params[:q]
@@ -206,7 +214,7 @@ class ArticlesController < ApplicationController
     @articles = @search.result.page(params[:page]).per(10)
     render :layout => false
   end
-  
+
   # fills a form whith values of the selected shared_article
   # when the direct parameter is set and the article is valid, it is imported directly
   def import
@@ -218,63 +226,16 @@ class ArticlesController < ApplicationController
       render :action => 'new', :layout => false
     end
   end
-  
-  # sync all articles with the external database
-  # renders a form with articles, which should be updated
-  def sync
-    # check if there is an shared_supplier
-    unless @supplier.shared_supplier
-      redirect_to supplier_articles_url(@supplier), :alert => I18n.t('articles.controller.sync.shared_alert', :supplier => @supplier.name)
-    end
-    # sync articles against external database
-    @updated_articles, @outlisted_articles, @new_articles = @supplier.sync_all
-    # convert to db-compatible-string
-    @updated_articles.each {|a, b| a.shared_updated_on = a.shared_updated_on.to_formatted_s(:db)}
-    if @updated_articles.empty? && @outlisted_articles.empty? && @new_articles.empty?
-      redirect_to supplier_articles_path(@supplier), :notice => I18n.t('articles.controller.sync.notice')
-    end
-    @ignored_article_count = @supplier.articles.where(order_number: [nil, '']).count
-  end
 
-  # Updates, deletes articles when sync form is submitted
+  private
-  def update_synchronized
-    begin
-      Article.transaction do
-        # delete articles
-        if params[:outlisted_articles]
-          Article.find(params[:outlisted_articles].keys).each(&:mark_as_deleted)
-        end
 
-        # Update articles
+  # @return [Number] Number of articles not taken into account when syncing (having no number)
-        if params[:articles]
+  helper_method \
-          params[:articles].each do |id, attrs|
+  def ignored_article_count
-            Article.find(id).update_attributes! attrs
+    if action_name == 'sync' || params[:from_action] == 'sync'
-          end
+      @ignored_article_count ||= @supplier.articles.where(order_number: [nil, '']).count
-        end
+    else
-
+      0
-        # Add new articles
-        if params[:new_articles]
-          params[:new_articles].each do |attrs|
-            article = Article.new attrs
-            article.supplier = @supplier
-            article.availability = true if @supplier.shared_sync_method == 'all_available'
-            article.availability = false if @supplier.shared_sync_method == 'all_unavailable'
-            article.save!
-          end
-        end
-      end
-
-      # Successfully done.
-      redirect_to supplier_articles_path(@supplier), notice: I18n.t('articles.controller.update_sync.notice')
-
-    rescue ActiveRecord::RecordInvalid => invalid
-      # An error has occurred, transaction has been rolled back.
-      redirect_to supplier_articles_path(@supplier),
-                  alert: I18n.t('articles.controller.error_update', :article => invalid.record.name, :msg => invalid.record.errors.full_messages)
-
-    rescue => error
-      redirect_to supplier_articles_path(@supplier),
-                  alert: I18n.t('errors.general_msg', :msg => error.message)
     end
   end
 end

app/helpers/articles_helper.rb

@@ -3,7 +3,7 @@ module ArticlesHelper
   # useful for highlighting attributes, when synchronizing articles
   def highlight_new(unequal_attributes, attribute)
     return unless unequal_attributes
-    unequal_attributes.detect {|a| a == attribute} ? "background-color: yellow" : ""
+    unequal_attributes.has_key?(attribute) ? "background-color: yellow" : ""
   end
 
   def row_classes(article)

app/models/article.rb

@@ -62,11 +62,11 @@ class Article < ActiveRecord::Base
   #validates_uniqueness_of :name, :scope => [:supplier_id, :deleted_at, :type], if: Proc.new {|a| a.supplier.shared_sync_method.blank? or a.supplier.shared_sync_method == 'import' }
   #validates_uniqueness_of :name, :scope => [:supplier_id, :deleted_at, :type, :unit, :unit_quantity]
   validate :uniqueness_of_name
-  
+
   # Callbacks
   before_save :update_price_history
   before_destroy :check_article_in_use
-  
+
   # The financial gross, net plus tax and deposti
   def gross_price
     ((price + deposit) * (tax / 100 + 1)).round(2)
@@ -76,12 +76,12 @@ class Article < ActiveRecord::Base
   def fc_price
     (gross_price  * (FoodsoftConfig[:price_markup] / 100 + 1)).round(2)
   end
-  
+
   # Returns true if article has been updated at least 2 days ago
   def recently_updated
     updated_at > 2.days.ago
   end
-  
+
   # If the article is used in an open Order, the Order will be returned.
   def in_open_order
     @in_open_order ||= begin
@@ -90,92 +90,106 @@ class Article < ActiveRecord::Base
       order_article ? order_article.order : nil
     end
   end
-  
+
   # Returns true if the article has been ordered in the given order at least once
   def ordered_in_order?(order)
     order.order_articles.where(article_id: id).where('quantity > 0').one?
   end
-  
+
   # this method checks, if the shared_article has been changed
   # unequal attributes will returned in array
-  # if only the timestamps differ and the attributes are equal, 
+  # if only the timestamps differ and the attributes are equal,
   # false will returned and self.shared_updated_on will be updated
   def shared_article_changed?(supplier = self.supplier)
     # skip early if the timestamp hasn't changed
     shared_article = self.shared_article(supplier)
     unless shared_article.nil? || self.shared_updated_on == shared_article.updated_on
-      
+      attrs = unequal_attributes(shared_article)
-      # try to convert units
+      if attrs.empty?
-      # convert supplier's price and unit_quantity into fc-size
-      new_price, new_unit_quantity = self.convert_units
-      new_unit = self.unit
-      unless new_price && new_unit_quantity
-        # if convertion isn't possible, take shared_article-price/unit_quantity
-        new_price, new_unit_quantity, new_unit = shared_article.price, shared_article.unit_quantity, shared_article.unit
-      end
-      
-      # check if all attributes differ
-      unequal_attributes = Article.compare_attributes(
-        {
-          :name => [self.name, shared_article.name],
-          :manufacturer => [self.manufacturer, shared_article.manufacturer.to_s],
-          :origin => [self.origin, shared_article.origin],
-          :unit => [self.unit, new_unit],
-          :price => [self.price.to_f.round(2), new_price.to_f.round(2)],
-          :tax => [self.tax, shared_article.tax],
-          :deposit => [self.deposit.to_f.round(2), shared_article.deposit.to_f.round(2)],
-          # take care of different num-objects.
-          :unit_quantity => [self.unit_quantity.to_s.to_f, new_unit_quantity.to_s.to_f],
-          :note => [self.note.to_s, shared_article.note.to_s]
-        }
-      )
-      if unequal_attributes.empty?            
         # when attributes not changed, update timestamp of article
         self.update_attribute(:shared_updated_on, shared_article.updated_on)
         false
       else
-        unequal_attributes
+        attrs
       end
     end
   end
-  
+
-  # compare attributes from different articles. used for auto-synchronization
+  # Return article attributes that were changed (incl. unit conversion)
-  # returns array of symbolized unequal attributes
+  # @param new_article [Article] New article to update self
+  # @option options [Boolean] :convert_units Omit or set to +true+ to keep current unit and recompute unit quantity and price.
+  # @return [Hash<Symbol, Object>] Attributes with new values
+  def unequal_attributes(new_article, options={})
+    # try to convert different units when desired
+    if options[:convert_units] == false
+      new_price, new_unit_quantity = nil, nil
+    else
+      new_price, new_unit_quantity = convert_units(new_article)
+    end
+    if new_price && new_unit_quantity
+      new_unit = self.unit
+    else
+      new_price = new_article.price
+      new_unit_quantity = new_article.unit_quantity
+      new_unit = new_article.unit
+    end
+
+    return Article.compare_attributes(
+      {
+        :name => [self.name, new_article.name],
+        :manufacturer => [self.manufacturer, new_article.manufacturer.to_s],
+        :origin => [self.origin, new_article.origin],
+        :unit => [self.unit, new_unit],
+        :price => [self.price.to_f.round(2), new_price.to_f.round(2)],
+        :tax => [self.tax, new_article.tax],
+        :deposit => [self.deposit.to_f.round(2), new_article.deposit.to_f.round(2)],
+        # take care of different num-objects.
+        :unit_quantity => [self.unit_quantity.to_s.to_f, new_unit_quantity.to_s.to_f],
+        :note => [self.note.to_s, new_article.note.to_s]
+      }
+    )
+  end
+
+  # Compare attributes from two different articles.
+  #
+  # This is used for auto-synchronization
+  # @param attributes [Hash<Symbol, Array>] Attributes with old and new values
+  # @return [Hash<Symbol, Object>] Changed attributes with new values
   def self.compare_attributes(attributes)
     unequal_attributes = attributes.select { |name, values| values[0] != values[1] && !(values[0].blank? && values[1].blank?) }
-    unequal_attributes.collect { |pair| pair[0] }
+    Hash[unequal_attributes.to_a.map {|a| [a[0], a[1].last]}]
   end
-  
+
   # to get the correspondent shared article
   def shared_article(supplier = self.supplier)
     self.order_number.blank? and return nil
     @shared_article ||= supplier.shared_supplier.shared_articles.find_by_number(self.order_number) rescue nil
   end
-  
+
   # convert units in foodcoop-size
   # uses unit factors in app_config.yml to calc the price/unit_quantity
   # returns new price and unit_quantity in array, when calc is possible => [price, unit_quanity]
   # returns false if units aren't foodsoft-compatible
   # returns nil if units are eqal
-  def convert_units
+  def convert_units(new_article = shared_article)
-    if unit != shared_article.unit
+    if unit != new_article.unit
       # legacy, used by foodcoops in Germany
-      if shared_article.unit == "KI" && unit == "ST" # 'KI' means a box, with a different amount of items in it
+      if new_article.unit == "KI" && unit == "ST" # 'KI' means a box, with a different amount of items in it
         # try to match the size out of its name, e.g. "banana 10-12 St" => 10
-        new_unit_quantity = /[0-9\-\s]+(St)/.match(shared_article.name).to_s.to_i
+        new_unit_quantity = /[0-9\-\s]+(St)/.match(new_article.name).to_s.to_i
         if new_unit_quantity && new_unit_quantity > 0
-          new_price = (shared_article.price/new_unit_quantity.to_f).round(2)
+          new_price = (new_article.price/new_unit_quantity.to_f).round(2)
           [new_price, new_unit_quantity]
         else
           false
         end
       else # use ruby-units to convert
         fc_unit = (::Unit.new(unit) rescue nil)
-        supplier_unit = (::Unit.new(shared_article.unit) rescue nil)
+        supplier_unit = (::Unit.new(new_article.unit) rescue nil)
         if fc_unit && supplier_unit && fc_unit =~ supplier_unit
           conversion_factor = (supplier_unit / fc_unit).to_base.to_r
-          new_price = shared_article.price / conversion_factor
+          new_price = new_article.price / conversion_factor
-          new_unit_quantity = shared_article.unit_quantity * conversion_factor
+          new_unit_quantity = new_article.unit_quantity * conversion_factor
           [new_price, new_unit_quantity]
         else
           false
@@ -196,7 +210,7 @@ class Article < ActiveRecord::Base
   end
 
   protected
-  
+
   # Checks if the article is in use before it will deleted
   def check_article_in_use
     raise I18n.t('articles.model.error_in_use', :article => self.name.to_s) if self.in_open_order

app/models/supplier.rb

@@ -26,39 +26,16 @@ class Supplier < ActiveRecord::Base
   # also returns an array with outlisted_articles, which should be deleted
   # also returns an array with new articles, which should be added (depending on shared_sync_method)
   def sync_all
-    updated_articles = Array.new
+    updated_article_pairs, outlisted_articles, new_articles = [], [], []
-    outlisted_articles = Array.new
-    new_articles = Array.new
     for article in articles.undeleted
       # try to find the associated shared_article
       shared_article = article.shared_article(self)
 
       if shared_article # article will be updated
-        
         unequal_attributes = article.shared_article_changed?(self)
         unless unequal_attributes.blank? # skip if shared_article has not been changed
-          
+          article.attributes = unequal_attributes
-          # try to convert different units
+          updated_article_pairs << [article, unequal_attributes]
-          new_price, new_unit_quantity = article.convert_units
-          if new_price && new_unit_quantity
-            article.price = new_price
-            article.unit_quantity = new_unit_quantity
-          else
-            article.price = shared_article.price
-            article.unit_quantity = shared_article.unit_quantity
-            article.unit = shared_article.unit
-          end
-          # update other attributes
-          article.attributes = {
-            :name => shared_article.name,
-            :manufacturer => shared_article.manufacturer,
-            :origin => shared_article.origin,
-            :shared_updated_on => shared_article.updated_on,
-            :tax => shared_article.tax,
-            :deposit => shared_article.deposit,
-            :note => shared_article.note
-          }
-          updated_articles << [article, unequal_attributes]
         end
       # Articles with no order number can be used to put non-shared articles
       # in a shared supplier, with sync keeping them.
@@ -75,7 +52,49 @@ class Supplier < ActiveRecord::Base
         end
       end
     end
-    return [updated_articles, outlisted_articles, new_articles]
+    return [updated_article_pairs, outlisted_articles, new_articles]
+  end
+
+  # Synchronise articles with spreadsheet.
+  #
+  # @param file [File] Spreadsheet file to parse
+  # @param options [Hash] Options passed to {FoodsoftFile#parse} except when listed here.
+  # @option options [Boolean] :outlist_absent Set to +true+ to remove articles not in spreadsheet.
+  # @option options [Boolean] :convert_units Omit or set to +true+ to keep current units, recomputing unit quantity and price.
+  def sync_from_file(file, options={})
+    all_order_numbers = []
+    updated_article_pairs, outlisted_articles, new_articles = [], [], []
+    FoodsoftFile::parse file, options do |status, new_attrs, line|
+      article = articles.undeleted.where(order_number: new_attrs[:order_number]).first
+      new_attrs[:article_category] = ArticleCategory.find_match(new_attrs[:article_category])
+      new_attrs[:tax] ||= FoodsoftConfig[:tax_default]
+      new_article = articles.build(new_attrs)
+
+      if status.nil?
+        if article.nil?
+          new_articles << new_article
+        else
+          unequal_attributes = article.unequal_attributes(new_article, options.slice(:convert_units))
+          unless unequal_attributes.empty?
+            article.attributes = unequal_attributes
+            updated_article_pairs << [article, unequal_attributes]
+          end
+        end
+      elsif status == :outlisted && article.present?
+        outlisted_articles << article
+
+      # stop when there is a parsing error
+      elsif status.is_a? String
+        # @todo move I18n key to model
+        raise I18n.t('articles.model.error_parse', :msg => status, :line => line.to_s)
+      end
+
+      all_order_numbers << article.order_number if article
+    end
+    if options[:outlist_absent]
+      outlisted_articles += articles.undeleted.where.not(id: all_order_numbers+[nil])
+    end
+    return [updated_article_pairs, outlisted_articles, new_articles]
   end
 
   # default value
@@ -114,4 +133,3 @@ class Supplier < ActiveRecord::Base
     end
   end
 end
-

app/views/articles/_sync.html.haml

@@ -0,0 +1,32 @@
+- if @outlisted_articles.any?
+  %h2= t '.outlist.title'
+  %p
+    = t('.outlist.body').html_safe
+    %ul
+      - for article in @outlisted_articles
+        %li
+          = hidden_field_tag "outlisted_articles[#{article.id}]", '1'
+          = article.name
+          - if article.in_open_order
+            .alert= t '.outlist.alert_used', article: article.name
+  %hr/
+
+- if @updated_article_pairs.any?
+  %h2= t '.update.title'
+  %p
+    %i
+      = t '.update.update_msg', count: @updated_article_pairs.size
+      = t '.update.body'
+  = render 'sync_table', articles: @updated_article_pairs, field: 'articles', hidden_fields: %w(shared_updated_on)
+  %hr/
+
+- if @new_articles.any?
+  %h2= t '.upnew.title'
+  %p
+    %i= t '.upnew.body_count', count: @new_articles.length
+  = render 'sync_table', articles: @new_articles, field: 'new_articles', hidden_fields: %w(shared_updated_on order_number)
+  %hr/
+
+- if ignored_article_count > 0
+  %p
+    %i= t '.outlist.body_ignored', count: ignored_article_count

app/views/articles/_sync_table.html.haml

@@ -51,3 +51,6 @@
               = form.text_field 'deposit', class: 'input-mini', style: 'width: 45px'
           %td= form.select :article_category_id, ArticleCategory.all.map {|a| [ a.name, a.id ] },
             {include_blank: true}, class: 'input-small'
+      - unless changed_article.errors.empty?
+        %tr.alert
+          %td(colspan=11)= changed_article.errors.full_messages.join(', ')

app/views/articles/parse_upload.html.haml

@@ -1,11 +1,8 @@
 - title t('.title', supplier: @supplier.name)
-%p= t('.body').html_safe
 
-= form_tag(create_from_upload_supplier_articles_path(@supplier)) do
+= form_tag update_synchronized_supplier_articles_path(@supplier) do
-  = render layout: 'edit_all_table' do |form|
+  = hidden_field_tag :from_action, 'parse_upload'
-    = form.hidden_field :manufacturer
+  = render 'sync'
-    = form.hidden_field :origin
   .form-actions
-    = submit_tag t('.submit', supplier: @supplier.name), class: 'btn btn-primary'
+    = submit_tag t('.submit'), class: 'btn btn-primary'
     = link_to t('ui.or_cancel'), upload_supplier_articles_path(@supplier)
-      

app/views/articles/sync.html.haml

@@ -1,39 +1,8 @@
 - title t('.title')
 
 = form_tag update_synchronized_supplier_articles_path(@supplier) do
-  - if @outlisted_articles.any?
+  = hidden_field_tag :from_action, 'sync'
-    %h2= t '.outlist.title'
+  = render 'sync'
-    %p
+  .form-actions
-      = t('.outlist.body').html_safe
+    = submit_tag t('.submit'), class: 'btn btn-primary'
-      %ul
+    = link_to t('ui.or_cancel'), supplier_articles_path(@supplier)
-        - for article in @outlisted_articles
-          %li
-            = hidden_field_tag "outlisted_articles[#{article.id}]", '1'
-            = article.name
-            - if article.in_open_order
-              .alert= t '.outlist.alert_used', article: article.name
-    %hr/
-
-  - if @updated_articles.any?
-    %h2= t '.update.title'
-    %p
-      %i
-        = t '.update.update_msg', count: @updated_articles.size
-        = t '.update.body'
-    = render 'sync_table', articles: @updated_articles, field: 'articles', hidden_fields: %w(shared_updated_on)
-    %hr/
-
-  - if @new_articles.any?
-    %h2= t '.upnew.title'
-    %p
-      %i= t '.upnew.body_count', count: @new_articles.length
-    = render 'sync_table', articles: @new_articles, field: 'new_articles', hidden_fields: %w(shared_updated_on order_number)
-    %hr/
-
-  - if @ignored_article_count > 0
-    %p
-      %i= t '.outlist.body_ignored', count: @ignored_article_count
-
-  = hidden_field 'supplier', 'id'
-  = submit_tag t('.submit'), class: 'btn btn-primary'
-  = link_to t('ui.or_cancel'), supplier_articles_path(@supplier)

app/views/articles/upload.html.haml

@@ -1,24 +1,88 @@
 - title t('.title', supplier: @supplier.name)
-= t('.body').html_safe
+
-%pre
+%p= t '.text_1', supplier: @supplier.name
-  = [t('.fields.status'),
+
-     Article.human_attribute_name(:order_number),
+%table.table.table-bordered
-     Article.human_attribute_name(:name),
+  %thead
-     Article.human_attribute_name(:note),
+    %tr
-     Article.human_attribute_name(:manufacturer),
+      %th= t '.field.status'
-     Article.human_attribute_name(:origin),
+      %th= Article.human_attribute_name(:order_number)
-     Article.human_attribute_name(:unit),
+      %th= Article.human_attribute_name(:name)
-     Article.human_attribute_name(:price),
+      %th= Article.human_attribute_name(:note)
-     Article.human_attribute_name(:tax),
+      %th= Article.human_attribute_name(:manufacturer)
-     Article.human_attribute_name(:deposit),
+      %th= Article.human_attribute_name(:origin)
-     Article.human_attribute_name(:unit_quantity),
+      %th= Article.human_attribute_name(:unit)
-     t('.fields.reserved'),
+      %th= Article.human_attribute_name(:price)
-     t('.fields.reserved'),
+      %th= Article.human_attribute_name(:tax)
-     Article.human_attribute_name(:article_category)].join(" | ")
+      %th= Article.human_attribute_name(:deposit)
+      %th= Article.human_attribute_name(:unit_quantity)
+      %th.muted= t '.fields.reserved'
+      %th.muted= t '.fields.reserved'
+      %th= Article.human_attribute_name(:article_category)
+  %tbody
+    %tr
+      %td
+      %td 1234A
+      %td= t '.sample.walnuts'
+      %td
+      %td= t '.sample.supplier_1'
+      %td CA
+      %td 500 gr
+      %td 8.90
+      %td= FoodsoftConfig[:tax_default] || 6
+      %td 0
+      %td 6
+      %td
+      %td
+      %td= t '.sample.nuts'
+    %tr
+      %td x
+      %td 4321Z
+      %td= t '.sample.tomato_juice'
+      %td= t '.sample.organic'
+      %td= t '.sample.supplier_2'
+      %td IN
+      %td 1.5 l
+      %td 4.35
+      %td= FoodsoftConfig[:tax_default] || 6
+      %td 0
+      %td 1
+      %td
+      %td
+      %td= t '.sample.juices'
+    %tr
+      %td
+      %td 4322Q
+      %td= t '.sample.tomato_juice'
+      %td= t '.sample.organic'
+      %td= t '.sample.supplier_3'
+      %td TR
+      %td 1.2 l
+      %td 4.02
+      %td= FoodsoftConfig[:tax_default] || 6
+      %td 0
+      %td 2
+      %td
+      %td
+      %td= t '.sample.juices'
+
+%p= t '.text_2'
 
 = form_for :articles, :url => parse_upload_supplier_articles_path(@supplier),
-   :html => { :multipart => true } do |f|
+   :html => { multipart: true, class: "form-horizontal" } do |f|
-  %label(for="articles_file")= t '.file_label'
+
-  = f.file_field "file"
+  .control-group
+    %label(for="articles_file")= t '.file_label'
+    = f.file_field "file"
+
+  .control-group
+    %label(for="articles_outlist_absent")
+      = f.check_box "outlist_absent"
+      = t '.options.outlist_absent'
+    %label(for="articles_convert_units")
+      = f.check_box "convert_units"
+      = t '.options.convert_units'
+
   .form-actions
-    = submit_tag t('.submit'), class: 'btn'
+    = submit_tag t('.submit'), class: 'btn btn-primary'
+    = link_to t('ui.or_cancel'), supplier_articles_path(@supplier)

config/locales/en.yml

@@ -355,12 +355,12 @@ en:
         notice: "%{count} new articles were saved."
       error_invalid: There are errors in articles
       error_nosel: No articles selected
-      error_parse: "%{msg} ... in line %{line}"
       error_update: 'An error occured when updating article ''%{article}'': %{msg}'
       parse_upload:
-        notice: "%{count} articles were succesfully analysed."
+        notice: Articles are already up to date.
+        no_file: Please select a file to upload
       sync:
-        notice: Catalog is up to date
+        notice: Articles are already up to date.
         shared_alert: "%{supplier} is not linked to an external database"
       update_all:
         notice: All articles and prices were updated.
@@ -405,9 +405,10 @@ en:
     model:
       error_in_use: "%{article} can not be deleted because the article is part of a current order!"
       error_nosel: You have selected no articles
+      error_parse: "%{msg} ... in line %{line}"
     parse_upload:
       body: "<p><i>Please verify the articles.</i></p> <p><i>Warning, at the moment there is no check for duplicate articles.</i></p>"
-      submit: Upload
+      submit: Process upload
       title: Upload articles
     sync:
       outlist:
@@ -434,13 +435,26 @@ en:
           other: There are %{count} articles to add.
         title: Add new ...
     upload:
-      body: <p>The file has to be a text file with the ending <tt>.csv</tt>. The first line will be ignored when imported, fields are separated by semicolons (';'), and text may be enclosed by double quotation marks ("text..."). Default character-set is utf-8.</p> <p>Column order:</p>
       fields:
         reserved: "(Reserved)"
         status: Status (x=skip)
       file_label: Please choose a compatible file
+      options:
+        convert_units: Keep current units, recompute unit quantity and price (like synchronize)
+        outlist_absent: Delete articles not in uploaded file
+      sample:
+        juices: Juices
+        nuts: Nuts
+        organic: Organic
+        supplier_1: Nuttyfarm
+        supplier_2: Brownfields
+        supplier_3: Greenfields
+        tomato_juice: Tomato juice
+        walnuts: Walnuts
       submit: Upload file
-      title: "%{supplier} / upload articles"
+      text_1: 'Here you can upload a spreadsheet to update the articles of %{supplier}. Excel (xls, xlsx) and OpenOffice (ods) spreadsheets are accepted, as well as comma-separated files (csv, columns separated by ";" with utf-8 encoding). Only the first sheet will be imported, and columns must be in the following order:'
+      text_2: 'The rows shown here are examples. When there is an "x" in the first column, the article is outlisted and will be removed. This allows you to edit the spreadsheet and quickly remove many articles at once, for example when articles become unavailable with the supplier. The category will be matched to your Foodsoft category list (both by category name and import names).'
+      title: "Upload articles of %{supplier}"
   config:
     hints:
       applepear_url: Website where the apple and pear system for tasks is explained.

config/locales/nl.yml

@@ -404,7 +404,7 @@ nl:
       error_nosel: Je hebt geen artikelen geselecteerd
     parse_upload:
       body: "<p><i>Ingelezen artikelen graag controleren.</i></p> <p><i>Let op, momenteel vind er geen controle op dubbele artikelen plaats.</i></p>"
-      submit: Uploaden
+      submit: Upload verwerken
       title: Artikelen uploaden
     sync:
       outlist:

lib/foodsoft_file.rb

@@ -1,20 +1,25 @@
-# Module for Foodsoft-File import
+require 'roo'
-# The Foodsoft-File is a cvs-file, with columns separated by semicolons
 
-require 'csv'
+# Foodsoft-file import
+class FoodsoftFile
 
-module FoodsoftFile
-  
   # parses a string from a foodsoft-file
   # returns two arrays with articles and outlisted_articles
   # the parsed article is a simple hash
-  def self.parse(file)
+  def self.parse(file, options = {})
-    articles, outlisted_articles = Array.new, Array.new
+    filepath = file.is_a?(String) ? file : file.to_path
-    row_index = 2
+    filename = options.delete(:filename) || filepath
-    ::CSV.parse(file.read.force_encoding('utf-8'), {:col_sep => ";", :headers => true}) do |row|
+    fileext = File.extname(filename)
-      # check if the line is empty
+    options[:csv_options] = {col_sep: ';', encoding: 'utf-8'}.merge(options[:csv_options]||{})
-      unless row[2] == "" || row[2].nil?        
+    s = Roo::Spreadsheet.open(filepath, options.merge({extension: fileext}))
-        article = {:number => row[1],
+
+    row_index = 1
+    s.each do |row|
+      if row_index == 1
+        # @todo try to detect headers; for now using the index is ok
+
+      elsif !row[2].blank?
+        article = {:order_number => row[1],
                    :name => row[2],
                    :note => row[3],
                    :manufacturer => row[4],
@@ -24,20 +29,13 @@ module FoodsoftFile
                    :tax => row[8],
                    :deposit => (row[9].nil? ? "0" : row[9]),
                    :unit_quantity => row[10],
-                   :scale_quantity => row[11],
+                   :article_category => row[13]}
-                   :scale_price => row[12],
+        status = row[0] && row[0].strip.downcase == 'x' ? :outlisted : nil
-                   :category => row[13]}
+        yield status, article, row_index
-        case row[0]
-        when "x"
-          # check if the article is outlisted
-          outlisted_articles << article
-        else
-          articles << article
-        end
       end
       row_index += 1
     end
-    return [articles, outlisted_articles]
+    row_index
   end
-    
+
 end

spec/fixtures/foodsoft_file_02.csv

@@ -0,0 +1,2 @@
+state;art. nummer;name;note;producer;origin;unit;net price;vat (%);deposit;unit quantity;(reserved);(reserved);category
+;1;Tomatoes;organic;Tommy farm;Somewhere, UK;500 g;1.2;6;0;20;;;Vegetables

spec/integration/articles_spec.rb

@@ -0,0 +1,115 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+
+describe ArticlesController, :type => :feature do
+  let(:user) { create :user, groups:[create(:workgroup, role_article_meta: true)] }
+  let (:supplier) { create :supplier }
+  let!(:article_category) { create :article_category }
+  before { login user }
+
+  describe ":index", :type => :feature, :js => true do
+    before { visit supplier_articles_path(supplier) }
+
+    it 'can visit supplier articles path' do
+      expect(page).to have_content(supplier.name)
+      expect(page).to have_content(I18n.t('articles.index.edit_all'))
+    end
+
+    it 'can create a new article' do
+      click_on I18n.t('articles.index.new')
+      expect(page).to have_selector('form#new_article')
+      article = FactoryGirl.build :article, supplier: supplier, article_category: article_category
+      within('#new_article') do
+        fill_in 'article_name', :with => article.name
+        fill_in 'article_unit', :with => article.unit
+        select article.article_category.name, :from => 'article_article_category_id'
+        fill_in 'article_price', :with => article.price
+        fill_in 'article_unit_quantity', :with => article.unit_quantity
+        fill_in 'article_tax', :with => article.tax
+        fill_in 'article_deposit', :with => article.deposit
+        # "Element cannot be scrolled into view" error, js as workaround
+        #find('input[type="submit"]').click
+        page.execute_script('$("form#new_article").submit();')
+      end
+      expect(page).to have_content(article.name)
+    end
+  end
+
+  describe ":upload", :type => :feature, :js => true do
+    let(:filename) { 'foodsoft_file_02.csv' }
+    let(:file)     { Rails.root.join("spec/fixtures/#{filename}") }
+    before do
+      visit upload_supplier_articles_path(supplier)
+      attach_file 'articles_file', file
+    end
+
+    Dir.glob('spec/fixtures/foodsoft_file_01.*') do |test_file|
+      describe "can import articles from #{test_file}" do
+        let(:file) { Rails.root.join(test_file) }
+        it do
+          find('input[type="submit"]').click
+          expect(find("tr:nth-child(1) #new_articles__note").value).to eq "bio ◎"
+          expect(find("tr:nth-child(2) #new_articles__name").value).to eq "Pijnboompitten"
+
+          4.times do |i|
+            all("tr:nth-child(#{i+1}) select > option")[1].select_option
+          end
+          find('input[type="submit"]').click
+          expect(page).to have_content("Pijnboompitten")
+
+          expect(supplier.articles.count).to eq 4
+        end
+      end
+    end
+
+    describe "can update existing article" do
+      let!(:article) { create :article, supplier: supplier, name: 'Foobar', order_number: 1, unit: '250 g' }
+      it do
+        find('input[type="submit"]').click
+        expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes'
+        find('input[type="submit"]').click
+        article.reload
+        expect(article.name).to eq 'Tomatoes'
+        expect([article.unit, article.unit_quantity, article.price]).to eq ['500 g', 20, 1.2]
+      end
+    end
+
+    describe "handles missing data" do
+      it do
+        find('input[type="submit"]').click # to overview
+        find('input[type="submit"]').click # missing category, re-show form
+        expect(find('tr.alert')).to be_present
+        expect(supplier.articles.count).to eq 0
+
+        all("tr select > option")[1].select_option
+        find('input[type="submit"]').click # now it should succeed
+        expect(supplier.articles.count).to eq 1
+      end
+    end
+
+    describe "can remove an existing article" do
+      let!(:article) { create :article, supplier: supplier, name: 'Foobar', order_number: 99999 }
+      it do
+        check('articles_outlist_absent')
+        find('input[type="submit"]').click
+        expect(find("#outlisted_articles_#{article.id}", visible: :all)).to be_present
+
+        all("tr select > option")[1].select_option
+        find('input[type="submit"]').click
+        expect(article.reload.deleted?).to be true
+      end
+    end
+
+    describe "can convert units when updating" do
+      let!(:article) { create :article, supplier: supplier, order_number: 1, unit: '250 g' }
+      it do
+        check('articles_convert_units')
+        find('input[type="submit"]').click
+        expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes'
+        find('input[type="submit"]').click
+        article.reload
+        expect([article.unit, article.unit_quantity, article.price]).to eq ['250 g', 40, 0.6]
+      end
+    end
+  end
+end

spec/integration/supplier_spec.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 require_relative '../spec_helper'
 
-describe 'supplier', :type => :feature do
+describe SuppliersController, :type => :feature do
   let(:supplier) { create :supplier }
 
   describe :type => :feature, :js => true do
@@ -27,55 +27,4 @@ describe 'supplier', :type => :feature do
       expect(page).to have_content(supplier.name)
     end
   end
-
-  describe :type => :feature, :js => true do
-    let(:article_category) { create :article_category }
-    let(:user) { create :user, groups:[create(:workgroup, role_article_meta: true)] }
-    before { login user }
-
-    it 'can visit supplier articles path' do
-      visit supplier_articles_path(supplier)
-      expect(page).to have_content(supplier.name)
-      expect(page).to have_content(I18n.t('articles.index.edit_all'))
-    end
-
-    it 'can create a new article' do
-      article_category.save!
-      visit supplier_articles_path(supplier)
-      click_on I18n.t('articles.index.new')
-      expect(page).to have_selector('form#new_article')
-      article = FactoryGirl.build :article, supplier: supplier, article_category: article_category
-      within('#new_article') do
-        fill_in 'article_name', :with => article.name
-        fill_in 'article_unit', :with => article.unit
-        select article.article_category.name, :from => 'article_article_category_id'
-        fill_in 'article_price', :with => article.price
-        fill_in 'article_unit_quantity', :with => article.unit_quantity
-        fill_in 'article_tax', :with => article.tax
-        fill_in 'article_deposit', :with => article.deposit
-        # "Element cannot be scrolled into view" error, js as workaround
-        #find('input[type="submit"]').click
-        page.execute_script('$("form#new_article").submit();')
-      end
-      expect(page).to have_content(article.name)
-    end
-
-    it 'can import articles' do
-      article_category.save!
-      visit upload_supplier_articles_path(supplier)
-      attach_file 'articles_file', Rails.root.join('spec/fixtures/foodsoft_file_01.csv')
-      find('input[type="submit"]').click
-
-      expect(find("#articles_0_note").value).to eq "bio ◎"
-      expect(find("#articles_1_name").value).to eq "Pijnboompitten"
-
-      4.times do |i|
-        select article_category.name, :from => "articles_#{i}_article_category_id"
-      end
-      find('input[type="submit"]').click
-
-      expect(supplier.articles.count).to eq 4
-    end
-  end
-
 end

spec/models/article_spec.rb

@@ -103,7 +103,7 @@ describe Article do
       article.update_attributes! updated_article.attributes.reject{|k,v| k=='id' or k=='type'}
       expect(article.unit).to eq '200g'
       expect(article.unit_quantity).to eq 5
-      expect(article.price).to be_within(1e-3).of(shared_article.price/5)
+      expect(article.price).to be_within(0.005).of(shared_article.price/5)
     end
 
     it 'does not synchronise when it has no order number' do