Import multiple spreadsheet formats. Make upload work like sync.

Gemfile

@@ -42,6 +42,8 @@ gem 'ruby-units'
 gem 'attribute_normalizer'
 gem 'ice_cube', github: 'wvengen/ice_cube', branch: 'issues/50-from_ical-rebased' # fork until merged
 gem 'recurring_select'
+gem 'roo', '~> 1.13.2'
+gem 'spreadsheet'
 
 # we use the git version of acts_as_versioned, and need to include it in this Gemfile
 gem 'acts_as_versioned', github: 'technoweenie/acts_as_versioned'

Gemfile.lock

@@ -333,6 +333,10 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
+    roo (1.13.2)
+      nokogiri
+      rubyzip
+      spreadsheet (> 0.6.4)
     rspec (2.99.0)
       rspec-core (~> 2.99.0)
       rspec-expectations (~> 2.99.0)
@@ -354,6 +358,7 @@ GEM
       rspec-mocks (~> 2.99.0)
     rspec-rerun (0.3.0)
       rspec
+    ruby-ole (1.2.11.8)
     ruby-prof (0.15.6)
     ruby-units (1.4.5)
     ruby_parser (3.6.5)
@@ -395,6 +400,8 @@ GEM
       eventmachine (~> 1.0.0)
       thin (~> 1.5.0)
     slop (3.6.0)
+    spreadsheet (1.0.0)
+      ruby-ole (>= 1.0)
     sprockets (2.12.3)
       hike (~> 1.2)
       multi_json (~> 1.0)
@@ -510,6 +517,7 @@ DEPENDENCIES
   ransack
   recurring_select
   resque
+  roo (~> 1.13.2)
   rspec-core (~> 2.99)
   rspec-rails
   rspec-rerun
@@ -522,6 +530,7 @@ DEPENDENCIES
   simple-navigation-bootstrap
   simple_form
   simplecov
+  spreadsheet
   sqlite3
   therubyracer
   thin

app/controllers/articles_controller.rb

@@ -35,7 +35,7 @@ class ArticlesController < ApplicationController
     @article = @supplier.articles.build(:tax => FoodsoftConfig[:tax_default])
     render :layout => false
   end
-  
+
   def create
     @article = Article.new(params[:article])
     if @article.valid? && @article.save
@@ -44,12 +44,12 @@ class ArticlesController < ApplicationController
       render :action => 'new', :layout => false
     end
   end
-  
+
   def edit
     @article = Article.find(params[:id])
     render :action => 'new', :layout => false
   end
-  
+
   # Updates one Article and highlights the line if succeded
   def update
     @article = Article.find(params[:id])
@@ -66,8 +66,8 @@ class ArticlesController < ApplicationController
     @article = Article.find(params[:id])
     @article.mark_as_deleted unless @order = @article.in_open_order # If article is in an active Order, the Order will be returned
     render :layout => false
-  end   
+  end
-  
+
   # Renders a form for editing all articles from a supplier
   def edit_all
     @articles = @supplier.articles.undeleted
@@ -102,7 +102,7 @@ class ArticlesController < ApplicationController
       redirect_to supplier_articles_path(@supplier), notice: I18n.t('articles.controller.update_all.notice')
     end
   end
-  
+
   # makes different actions on selected articles
   def update_selected
     raise I18n.t('articles.controller.error_nosel') if params[:selected_articles].nil?
@@ -129,96 +129,44 @@ class ArticlesController < ApplicationController
     redirect_to supplier_articles_url(@supplier, :per_page => params[:per_page]),
                 :alert => I18n.t('errors.general_msg', :msg => error)
   end
- 
+
   # lets start with parsing articles from uploaded file, yeah
   # Renders the upload form
   def upload
   end
- 
+
-  # parses the articles from a csv and creates a form-table with the parsed data.
+  # Update articles from a spreadsheet
-  # the csv must have the following format:
-  # status | number | name | note | manufacturer | origin | unit | clear price | unit_quantity | tax | deposit | scale quantity | scale price | category
-  # the first line will be ignored. 
-  # field-seperator: ";"
-  # text-seperator: ""
   def parse_upload
-    begin
+    uploaded_file = params[:articles]['file']
-      @articles = Array.new
+    options = {filename: uploaded_file.original_filename}
-      articles, outlisted_articles = FoodsoftFile::parse(params[:articles]["file"])
+    @updated_article_pairs, @outlisted_articles, @new_articles = [], [], []
-      no_category = ArticleCategory.new
+    FoodsoftFile::parse uploaded_file.tempfile, options do |status, new_attrs, line|
-      articles.each do |row|
+      article = @supplier.articles.where(order_number: new_attrs[:order_number]).first
-        # fallback to Others category
+      new_attrs[:article_category] = ArticleCategory.find_match(new_attrs[:article_category])
-        category = (ArticleCategory.find_match(row[:category]) || no_category)
+      new_attrs[:tax] ||= FoodsoftConfig[:tax_default]
-        # creates a new article and price
+      new_article = @supplier.articles.build(new_attrs)
-        article = @supplier.articles.build(:name => row[:name], 
-                               :note => row[:note],
-                               :manufacturer => row[:manufacturer],
-                               :origin => row[:origin],
-                               :unit => row[:unit],
-                               :article_category => category,
-                               :price => row[:price],
-                               :unit_quantity => row[:unit_quantity],
-                               :order_number => row[:number],
-                               :deposit => row[:deposit],
-                               :tax => (row[:tax] || FoodsoftConfig[:tax_default]))
-        # stop parsing, when an article isn't valid
-        unless article.valid?
-          raise I18n.t('articles.controller.error_parse', :msg => article.errors.full_messages.join(", "), :line => (articles.index(row) + 2).to_s)
-        end
-        @articles << article
-      end
-      flash.now[:notice] = I18n.t('articles.controller.parse_upload.notice', :count => @articles.size)
-    rescue => error
-      redirect_to upload_supplier_articles_path(@supplier), :alert => I18n.t('errors.general_msg', :msg => error.message)
-    end
-  end
- 
-  # creates articles from form
-  def create_from_upload
-    begin
-      Article.transaction do
-        invalid_articles = false
-        @articles = []
-        params[:articles].each do |_key, article_attributes|
-          @articles << (article = @supplier.articles.build(article_attributes))
-          invalid_articles = true unless article.save
-        end
 
-        raise I18n.t('articles.controller.error_invalid') if invalid_articles
+      if status.nil? && article.nil?
-      end
+        @new_articles << new_article
-      # Successfully done.
+      elsif status.nil? && article.present?
-      redirect_to supplier_articles_path(@supplier), notice: I18n.t('articles.controller.create_from_upload.notice', :count => @articles.size)
+        unequal_attributes = article.unequal_attributes(new_article)
+        article.attributes = unequal_attributes
+        @updated_article_pairs << [article, unequal_attributes]
+      elsif status == :outlisted && article.present?
+        @outlisted_articles << article
+
+      # stop when there is a parsing error
+      elsif status.is_a? String
+        raise I18n.t('articles.controller.error_parse', :msg => status, :line => line.to_s)
+      end
 
-    rescue => error
-      # An error has occurred, transaction has been rolled back.
-      flash.now[:error] = I18n.t('errors.general_msg', :msg => error.message)
-      render :parse_upload
     end
+    @ignored_article_count = 0
+    render :sync
+  rescue => error
+    redirect_to upload_supplier_articles_path(@supplier), :alert => I18n.t('errors.general_msg', :msg => error.message)
   end
-  
+
-  # renders a view to import articles in local database
-  #   
-  def shared
-    # build array of keywords, required for ransack _all suffix
-    params[:q][:name_cont_all] = params[:q][:name_cont_all].split(' ') if params[:q]
-    # Build search with meta search plugin
-    @search = @supplier.shared_supplier.shared_articles.search(params[:q])
-    @articles = @search.result.page(params[:page]).per(10)
-    render :layout => false
-  end
-  
-  # fills a form whith values of the selected shared_article
-  # when the direct parameter is set and the article is valid, it is imported directly
-  def import
-    @article = SharedArticle.find(params[:shared_article_id]).build_new_article(@supplier)
-    @article.article_category_id = params[:article_category_id] unless params[:article_category_id].blank?
-    if params[:direct] && !params[:article_category_id].blank? && @article.valid? && @article.save
-      render :action => 'create', :layout => false
-    else
-      render :action => 'new', :layout => false
-    end
-  end
-  
   # sync all articles with the external database
   # renders a form with articles, which should be updated
   def sync
@@ -227,16 +175,14 @@ class ArticlesController < ApplicationController
       redirect_to supplier_articles_url(@supplier), :alert => I18n.t('articles.controller.sync.shared_alert', :supplier => @supplier.name)
     end
     # sync articles against external database
-    @updated_articles, @outlisted_articles, @new_articles = @supplier.sync_all
+    @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_all
-    # convert to db-compatible-string
+    if @updated_article_pairs.empty? && @outlisted_articles.empty? && @new_articles.empty?
-    @updated_articles.each {|a, b| a.shared_updated_on = a.shared_updated_on.to_formatted_s(:db)}
-    if @updated_articles.empty? && @outlisted_articles.empty? && @new_articles.empty?
       redirect_to supplier_articles_path(@supplier), :notice => I18n.t('articles.controller.sync.notice')
     end
     @ignored_article_count = @supplier.articles.where(order_number: [nil, '']).count
   end
 
-  # Updates, deletes articles when sync form is submitted
+  # Updates, deletes articles when upload or sync form is submitted
   def update_synchronized
     begin
       Article.transaction do
@@ -277,4 +223,27 @@ class ArticlesController < ApplicationController
                   alert: I18n.t('errors.general_msg', :msg => error.message)
     end
   end
+
+  # renders a view to import articles in local database
+  #
+  def shared
+    # build array of keywords, required for ransack _all suffix
+    params[:q][:name_cont_all] = params[:q][:name_cont_all].split(' ') if params[:q]
+    # Build search with meta search plugin
+    @search = @supplier.shared_supplier.shared_articles.search(params[:q])
+    @articles = @search.result.page(params[:page]).per(10)
+    render :layout => false
+  end
+
+  # fills a form whith values of the selected shared_article
+  # when the direct parameter is set and the article is valid, it is imported directly
+  def import
+    @article = SharedArticle.find(params[:shared_article_id]).build_new_article(@supplier)
+    @article.article_category_id = params[:article_category_id] unless params[:article_category_id].blank?
+    if params[:direct] && !params[:article_category_id].blank? && @article.valid? && @article.save
+      render :action => 'create', :layout => false
+    else
+      render :action => 'new', :layout => false
+    end
+  end
 end

app/helpers/articles_helper.rb

@@ -3,7 +3,7 @@ module ArticlesHelper
   # useful for highlighting attributes, when synchronizing articles
   def highlight_new(unequal_attributes, attribute)
     return unless unequal_attributes
-    unequal_attributes.detect {|a| a == attribute} ? "background-color: yellow" : ""
+    unequal_attributes.has_key?(attribute) ? "background-color: yellow" : ""
   end
 
   def row_classes(article)

app/models/article.rb

@@ -62,11 +62,11 @@ class Article < ActiveRecord::Base
   #validates_uniqueness_of :name, :scope => [:supplier_id, :deleted_at, :type], if: Proc.new {|a| a.supplier.shared_sync_method.blank? or a.supplier.shared_sync_method == 'import' }
   #validates_uniqueness_of :name, :scope => [:supplier_id, :deleted_at, :type, :unit, :unit_quantity]
   validate :uniqueness_of_name
-  
+
   # Callbacks
   before_save :update_price_history
   before_destroy :check_article_in_use
-  
+
   # The financial gross, net plus tax and deposti
   def gross_price
     ((price + deposit) * (tax / 100 + 1)).round(2)
@@ -76,12 +76,12 @@ class Article < ActiveRecord::Base
   def fc_price
     (gross_price  * (FoodsoftConfig[:price_markup] / 100 + 1)).round(2)
   end
-  
+
   # Returns true if article has been updated at least 2 days ago
   def recently_updated
     updated_at > 2.days.ago
   end
-  
+
   # If the article is used in an open Order, the Order will be returned.
   def in_open_order
     @in_open_order ||= begin
@@ -90,92 +90,101 @@ class Article < ActiveRecord::Base
       order_article ? order_article.order : nil
     end
   end
-  
+
   # Returns true if the article has been ordered in the given order at least once
   def ordered_in_order?(order)
     order.order_articles.where(article_id: id).where('quantity > 0').one?
   end
-  
+
   # this method checks, if the shared_article has been changed
   # unequal attributes will returned in array
-  # if only the timestamps differ and the attributes are equal, 
+  # if only the timestamps differ and the attributes are equal,
   # false will returned and self.shared_updated_on will be updated
   def shared_article_changed?(supplier = self.supplier)
     # skip early if the timestamp hasn't changed
     shared_article = self.shared_article(supplier)
     unless shared_article.nil? || self.shared_updated_on == shared_article.updated_on
-      
+      attrs = unequal_attributes(shared_article)
-      # try to convert units
+      if attrs.empty?
-      # convert supplier's price and unit_quantity into fc-size
-      new_price, new_unit_quantity = self.convert_units
-      new_unit = self.unit
-      unless new_price && new_unit_quantity
-        # if convertion isn't possible, take shared_article-price/unit_quantity
-        new_price, new_unit_quantity, new_unit = shared_article.price, shared_article.unit_quantity, shared_article.unit
-      end
-      
-      # check if all attributes differ
-      unequal_attributes = Article.compare_attributes(
-        {
-          :name => [self.name, shared_article.name],
-          :manufacturer => [self.manufacturer, shared_article.manufacturer.to_s],
-          :origin => [self.origin, shared_article.origin],
-          :unit => [self.unit, new_unit],
-          :price => [self.price.to_f.round(2), new_price.to_f.round(2)],
-          :tax => [self.tax, shared_article.tax],
-          :deposit => [self.deposit.to_f.round(2), shared_article.deposit.to_f.round(2)],
-          # take care of different num-objects.
-          :unit_quantity => [self.unit_quantity.to_s.to_f, new_unit_quantity.to_s.to_f],
-          :note => [self.note.to_s, shared_article.note.to_s]
-        }
-      )
-      if unequal_attributes.empty?            
         # when attributes not changed, update timestamp of article
         self.update_attribute(:shared_updated_on, shared_article.updated_on)
         false
       else
-        unequal_attributes
+        attrs
       end
     end
   end
-  
+
-  # compare attributes from different articles. used for auto-synchronization
+  # Return article attributes that were changed (incl. unit conversion)
-  # returns array of symbolized unequal attributes
+  # @param [Article] New article to update self
+  # @return [Hash<Symbol, Object>] Attributes with new values
+  def unequal_attributes(new_article)
+    # try to convert different units
+    new_price, new_unit_quantity = convert_units(new_article)
+    if new_price && new_unit_quantity
+      new_unit = self.unit
+    else
+      new_price = new_article.price
+      new_unit_quantity = new_article.unit_quantity
+      new_unit = new_article.unit
+    end
+
+    return Article.compare_attributes(
+      {
+        :name => [self.name, new_article.name],
+        :manufacturer => [self.manufacturer, new_article.manufacturer.to_s],
+        :origin => [self.origin, new_article.origin],
+        :unit => [self.unit, new_unit],
+        :price => [self.price.to_f.round(2), new_price.to_f.round(2)],
+        :tax => [self.tax, new_article.tax],
+        :deposit => [self.deposit.to_f.round(2), new_article.deposit.to_f.round(2)],
+        # take care of different num-objects.
+        :unit_quantity => [self.unit_quantity.to_s.to_f, new_unit_quantity.to_s.to_f],
+        :note => [self.note.to_s, new_article.note.to_s]
+      }
+    )
+  end
+
+  # Compare attributes from two different articles.
+  #
+  # This is used for auto-synchronization
+  # @param attributes [Hash<Symbol, Array>] Attributes with old and new values
+  # @return [Hash<Symbol, Object>] Changed attributes with new values
   def self.compare_attributes(attributes)
     unequal_attributes = attributes.select { |name, values| values[0] != values[1] && !(values[0].blank? && values[1].blank?) }
-    unequal_attributes.collect { |pair| pair[0] }
+    Hash[unequal_attributes.to_a.map {|a| [a[0], a[1].last]}]
   end
-  
+
   # to get the correspondent shared article
   def shared_article(supplier = self.supplier)
     self.order_number.blank? and return nil
     @shared_article ||= supplier.shared_supplier.shared_articles.find_by_number(self.order_number) rescue nil
   end
-  
+
   # convert units in foodcoop-size
   # uses unit factors in app_config.yml to calc the price/unit_quantity
   # returns new price and unit_quantity in array, when calc is possible => [price, unit_quanity]
   # returns false if units aren't foodsoft-compatible
   # returns nil if units are eqal
-  def convert_units
+  def convert_units(new_article = shared_article)
-    if unit != shared_article.unit
+    if unit != new_article.unit
       # legacy, used by foodcoops in Germany
-      if shared_article.unit == "KI" && unit == "ST" # 'KI' means a box, with a different amount of items in it
+      if new_article.unit == "KI" && unit == "ST" # 'KI' means a box, with a different amount of items in it
         # try to match the size out of its name, e.g. "banana 10-12 St" => 10
-        new_unit_quantity = /[0-9\-\s]+(St)/.match(shared_article.name).to_s.to_i
+        new_unit_quantity = /[0-9\-\s]+(St)/.match(new_article.name).to_s.to_i
         if new_unit_quantity && new_unit_quantity > 0
-          new_price = (shared_article.price/new_unit_quantity.to_f).round(2)
+          new_price = (new_article.price/new_unit_quantity.to_f).round(2)
           [new_price, new_unit_quantity]
         else
           false
         end
       else # use ruby-units to convert
         fc_unit = (::Unit.new(unit) rescue nil)
-        supplier_unit = (::Unit.new(shared_article.unit) rescue nil)
+        supplier_unit = (::Unit.new(new_article.unit) rescue nil)
         if fc_unit && supplier_unit && fc_unit =~ supplier_unit
           conversion_factor = (supplier_unit / fc_unit).to_base.to_r
-          new_price = shared_article.price / conversion_factor
+          new_price = new_article.price / conversion_factor
-          new_unit_quantity = shared_article.unit_quantity * conversion_factor
+          new_unit_quantity = new_article.unit_quantity * conversion_factor
           [new_price, new_unit_quantity]
         else
           false
@@ -196,7 +205,7 @@ class Article < ActiveRecord::Base
   end
 
   protected
-  
+
   # Checks if the article is in use before it will deleted
   def check_article_in_use
     raise I18n.t('articles.model.error_in_use', :article => self.name.to_s) if self.in_open_order

app/models/supplier.rb

@@ -34,30 +34,9 @@ class Supplier < ActiveRecord::Base
       shared_article = article.shared_article(self)
 
       if shared_article # article will be updated
-        
         unequal_attributes = article.shared_article_changed?(self)
         unless unequal_attributes.blank? # skip if shared_article has not been changed
-          
+          article.attributes = unequal_attributes
-          # try to convert different units
-          new_price, new_unit_quantity = article.convert_units
-          if new_price && new_unit_quantity
-            article.price = new_price
-            article.unit_quantity = new_unit_quantity
-          else
-            article.price = shared_article.price
-            article.unit_quantity = shared_article.unit_quantity
-            article.unit = shared_article.unit
-          end
-          # update other attributes
-          article.attributes = {
-            :name => shared_article.name,
-            :manufacturer => shared_article.manufacturer,
-            :origin => shared_article.origin,
-            :shared_updated_on => shared_article.updated_on,
-            :tax => shared_article.tax,
-            :deposit => shared_article.deposit,
-            :note => shared_article.note
-          }
           updated_articles << [article, unequal_attributes]
         end
       # Articles with no order number can be used to put non-shared articles

app/views/articles/sync.html.haml

@@ -14,13 +14,13 @@
               .alert= t '.outlist.alert_used', article: article.name
     %hr/
 
-  - if @updated_articles.any?
+  - if @updated_article_pairs.any?
     %h2= t '.update.title'
     %p
       %i
-        = t '.update.update_msg', count: @updated_articles.size
+        = t '.update.update_msg', count: @updated_article_pairs.size
         = t '.update.body'
-    = render 'sync_table', articles: @updated_articles, field: 'articles', hidden_fields: %w(shared_updated_on)
+    = render 'sync_table', articles: @updated_article_pairs, field: 'articles', hidden_fields: %w(shared_updated_on)
     %hr/
 
   - if @new_articles.any?

lib/foodsoft_file.rb

@@ -1,20 +1,25 @@
-# Module for Foodsoft-File import
+require 'roo'
-# The Foodsoft-File is a cvs-file, with columns separated by semicolons
 
-require 'csv'
+# Foodsoft-file import
+class FoodsoftFile
 
-module FoodsoftFile
-  
   # parses a string from a foodsoft-file
   # returns two arrays with articles and outlisted_articles
   # the parsed article is a simple hash
-  def self.parse(file)
+  def self.parse(file, options = {})
-    articles, outlisted_articles = Array.new, Array.new
+    filepath = file.is_a?(String) ? file : file.to_path
-    row_index = 2
+    filename = options[:filename] || filepath
-    ::CSV.parse(file.read.force_encoding('utf-8'), {:col_sep => ";", :headers => true}) do |row|
+    fileext = ::File.extname(filename)
-      # check if the line is empty
+    options = {col_sep: ';', encoding: 'utf-8'}.merge(options)
-      unless row[2] == "" || row[2].nil?        
+    s = Roo::Spreadsheet.open filepath, extension: fileext, csv_options: options
-        article = {:number => row[1],
+
+    row_index = 1
+    s.each do |row|
+      if row_index == 1
+        # @todo try to detect headers; for now using the index is ok
+
+      elsif !row[2].blank?
+        article = {:order_number => row[1],
                    :name => row[2],
                    :note => row[3],
                    :manufacturer => row[4],
@@ -24,20 +29,13 @@ module FoodsoftFile
                    :tax => row[8],
                    :deposit => (row[9].nil? ? "0" : row[9]),
                    :unit_quantity => row[10],
-                   :scale_quantity => row[11],
+                   :article_category => row[13]}
-                   :scale_price => row[12],
+        status = row[0] && row[0].strip.downcase == 'x' ? :outlisted : nil
-                   :category => row[13]}
+        yield status, article, row_index
-        case row[0]
-        when "x"
-          # check if the article is outlisted
-          outlisted_articles << article
-        else
-          articles << article
-        end
       end
       row_index += 1
     end
-    return [articles, outlisted_articles]
+    row_index
   end
-    
+
 end