From 07ba6f05355b547180ea409069e8f01e462cdd7d Mon Sep 17 00:00:00 2001 From: wvengen Date: Sun, 18 Jan 2015 02:04:57 +0100 Subject: [PATCH] Import multiple spreadsheet formats. Make upload work like sync. --- Gemfile | 2 + Gemfile.lock | 9 ++ app/controllers/articles_controller.rb | 153 ++++++++++--------------- app/helpers/articles_helper.rb | 2 +- app/models/article.rb | 107 +++++++++-------- app/models/supplier.rb | 23 +--- app/views/articles/sync.html.haml | 6 +- lib/foodsoft_file.rb | 46 ++++---- 8 files changed, 157 insertions(+), 191 deletions(-) diff --git a/Gemfile b/Gemfile index 6c389157..a8213477 100644 --- a/Gemfile +++ b/Gemfile @@ -42,6 +42,8 @@ gem 'ruby-units' gem 'attribute_normalizer' gem 'ice_cube', github: 'wvengen/ice_cube', branch: 'issues/50-from_ical-rebased' # fork until merged gem 'recurring_select' +gem 'roo', '~> 1.13.2' +gem 'spreadsheet' # we use the git version of acts_as_versioned, and need to include it in this Gemfile gem 'acts_as_versioned', github: 'technoweenie/acts_as_versioned' diff --git a/Gemfile.lock b/Gemfile.lock index 422a66d1..1d9c60f3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -333,6 +333,10 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 3.0) netrc (~> 0.7) + roo (1.13.2) + nokogiri + rubyzip + spreadsheet (> 0.6.4) rspec (2.99.0) rspec-core (~> 2.99.0) rspec-expectations (~> 2.99.0) @@ -354,6 +358,7 @@ GEM rspec-mocks (~> 2.99.0) rspec-rerun (0.3.0) rspec + ruby-ole (1.2.11.8) ruby-prof (0.15.6) ruby-units (1.4.5) ruby_parser (3.6.5) @@ -395,6 +400,8 @@ GEM eventmachine (~> 1.0.0) thin (~> 1.5.0) slop (3.6.0) + spreadsheet (1.0.0) + ruby-ole (>= 1.0) sprockets (2.12.3) hike (~> 1.2) multi_json (~> 1.0) @@ -510,6 +517,7 @@ DEPENDENCIES ransack recurring_select resque + roo (~> 1.13.2) rspec-core (~> 2.99) rspec-rails rspec-rerun @@ -522,6 +530,7 @@ DEPENDENCIES simple-navigation-bootstrap simple_form simplecov + spreadsheet sqlite3 therubyracer thin diff --git a/app/controllers/articles_controller.rb b/app/controllers/articles_controller.rb index a96a19d5..cc14cac1 100644 --- a/app/controllers/articles_controller.rb +++ b/app/controllers/articles_controller.rb @@ -35,7 +35,7 @@ class ArticlesController < ApplicationController @article = @supplier.articles.build(:tax => FoodsoftConfig[:tax_default]) render :layout => false end - + def create @article = Article.new(params[:article]) if @article.valid? && @article.save @@ -44,12 +44,12 @@ class ArticlesController < ApplicationController render :action => 'new', :layout => false end end - + def edit @article = Article.find(params[:id]) render :action => 'new', :layout => false end - + # Updates one Article and highlights the line if succeded def update @article = Article.find(params[:id]) @@ -66,8 +66,8 @@ class ArticlesController < ApplicationController @article = Article.find(params[:id]) @article.mark_as_deleted unless @order = @article.in_open_order # If article is in an active Order, the Order will be returned render :layout => false - end - + end + # Renders a form for editing all articles from a supplier def edit_all @articles = @supplier.articles.undeleted @@ -102,7 +102,7 @@ class ArticlesController < ApplicationController redirect_to supplier_articles_path(@supplier), notice: I18n.t('articles.controller.update_all.notice') end end - + # makes different actions on selected articles def update_selected raise I18n.t('articles.controller.error_nosel') if params[:selected_articles].nil? @@ -129,96 +129,44 @@ class ArticlesController < ApplicationController redirect_to supplier_articles_url(@supplier, :per_page => params[:per_page]), :alert => I18n.t('errors.general_msg', :msg => error) end - + # lets start with parsing articles from uploaded file, yeah # Renders the upload form def upload end - - # parses the articles from a csv and creates a form-table with the parsed data. - # the csv must have the following format: - # status | number | name | note | manufacturer | origin | unit | clear price | unit_quantity | tax | deposit | scale quantity | scale price | category - # the first line will be ignored. - # field-seperator: ";" - # text-seperator: "" + + # Update articles from a spreadsheet def parse_upload - begin - @articles = Array.new - articles, outlisted_articles = FoodsoftFile::parse(params[:articles]["file"]) - no_category = ArticleCategory.new - articles.each do |row| - # fallback to Others category - category = (ArticleCategory.find_match(row[:category]) || no_category) - # creates a new article and price - article = @supplier.articles.build(:name => row[:name], - :note => row[:note], - :manufacturer => row[:manufacturer], - :origin => row[:origin], - :unit => row[:unit], - :article_category => category, - :price => row[:price], - :unit_quantity => row[:unit_quantity], - :order_number => row[:number], - :deposit => row[:deposit], - :tax => (row[:tax] || FoodsoftConfig[:tax_default])) - # stop parsing, when an article isn't valid - unless article.valid? - raise I18n.t('articles.controller.error_parse', :msg => article.errors.full_messages.join(", "), :line => (articles.index(row) + 2).to_s) - end - @articles << article - end - flash.now[:notice] = I18n.t('articles.controller.parse_upload.notice', :count => @articles.size) - rescue => error - redirect_to upload_supplier_articles_path(@supplier), :alert => I18n.t('errors.general_msg', :msg => error.message) - end - end - - # creates articles from form - def create_from_upload - begin - Article.transaction do - invalid_articles = false - @articles = [] - params[:articles].each do |_key, article_attributes| - @articles << (article = @supplier.articles.build(article_attributes)) - invalid_articles = true unless article.save - end + uploaded_file = params[:articles]['file'] + options = {filename: uploaded_file.original_filename} + @updated_article_pairs, @outlisted_articles, @new_articles = [], [], [] + FoodsoftFile::parse uploaded_file.tempfile, options do |status, new_attrs, line| + article = @supplier.articles.where(order_number: new_attrs[:order_number]).first + new_attrs[:article_category] = ArticleCategory.find_match(new_attrs[:article_category]) + new_attrs[:tax] ||= FoodsoftConfig[:tax_default] + new_article = @supplier.articles.build(new_attrs) - raise I18n.t('articles.controller.error_invalid') if invalid_articles - end - # Successfully done. - redirect_to supplier_articles_path(@supplier), notice: I18n.t('articles.controller.create_from_upload.notice', :count => @articles.size) + if status.nil? && article.nil? + @new_articles << new_article + elsif status.nil? && article.present? + unequal_attributes = article.unequal_attributes(new_article) + article.attributes = unequal_attributes + @updated_article_pairs << [article, unequal_attributes] + elsif status == :outlisted && article.present? + @outlisted_articles << article + + # stop when there is a parsing error + elsif status.is_a? String + raise I18n.t('articles.controller.error_parse', :msg => status, :line => line.to_s) + end - rescue => error - # An error has occurred, transaction has been rolled back. - flash.now[:error] = I18n.t('errors.general_msg', :msg => error.message) - render :parse_upload end + @ignored_article_count = 0 + render :sync + rescue => error + redirect_to upload_supplier_articles_path(@supplier), :alert => I18n.t('errors.general_msg', :msg => error.message) end - - # renders a view to import articles in local database - # - def shared - # build array of keywords, required for ransack _all suffix - params[:q][:name_cont_all] = params[:q][:name_cont_all].split(' ') if params[:q] - # Build search with meta search plugin - @search = @supplier.shared_supplier.shared_articles.search(params[:q]) - @articles = @search.result.page(params[:page]).per(10) - render :layout => false - end - - # fills a form whith values of the selected shared_article - # when the direct parameter is set and the article is valid, it is imported directly - def import - @article = SharedArticle.find(params[:shared_article_id]).build_new_article(@supplier) - @article.article_category_id = params[:article_category_id] unless params[:article_category_id].blank? - if params[:direct] && !params[:article_category_id].blank? && @article.valid? && @article.save - render :action => 'create', :layout => false - else - render :action => 'new', :layout => false - end - end - + # sync all articles with the external database # renders a form with articles, which should be updated def sync @@ -227,16 +175,14 @@ class ArticlesController < ApplicationController redirect_to supplier_articles_url(@supplier), :alert => I18n.t('articles.controller.sync.shared_alert', :supplier => @supplier.name) end # sync articles against external database - @updated_articles, @outlisted_articles, @new_articles = @supplier.sync_all - # convert to db-compatible-string - @updated_articles.each {|a, b| a.shared_updated_on = a.shared_updated_on.to_formatted_s(:db)} - if @updated_articles.empty? && @outlisted_articles.empty? && @new_articles.empty? + @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_all + if @updated_article_pairs.empty? && @outlisted_articles.empty? && @new_articles.empty? redirect_to supplier_articles_path(@supplier), :notice => I18n.t('articles.controller.sync.notice') end @ignored_article_count = @supplier.articles.where(order_number: [nil, '']).count end - # Updates, deletes articles when sync form is submitted + # Updates, deletes articles when upload or sync form is submitted def update_synchronized begin Article.transaction do @@ -277,4 +223,27 @@ class ArticlesController < ApplicationController alert: I18n.t('errors.general_msg', :msg => error.message) end end + + # renders a view to import articles in local database + # + def shared + # build array of keywords, required for ransack _all suffix + params[:q][:name_cont_all] = params[:q][:name_cont_all].split(' ') if params[:q] + # Build search with meta search plugin + @search = @supplier.shared_supplier.shared_articles.search(params[:q]) + @articles = @search.result.page(params[:page]).per(10) + render :layout => false + end + + # fills a form whith values of the selected shared_article + # when the direct parameter is set and the article is valid, it is imported directly + def import + @article = SharedArticle.find(params[:shared_article_id]).build_new_article(@supplier) + @article.article_category_id = params[:article_category_id] unless params[:article_category_id].blank? + if params[:direct] && !params[:article_category_id].blank? && @article.valid? && @article.save + render :action => 'create', :layout => false + else + render :action => 'new', :layout => false + end + end end diff --git a/app/helpers/articles_helper.rb b/app/helpers/articles_helper.rb index ecd45ec3..90dbdc43 100644 --- a/app/helpers/articles_helper.rb +++ b/app/helpers/articles_helper.rb @@ -3,7 +3,7 @@ module ArticlesHelper # useful for highlighting attributes, when synchronizing articles def highlight_new(unequal_attributes, attribute) return unless unequal_attributes - unequal_attributes.detect {|a| a == attribute} ? "background-color: yellow" : "" + unequal_attributes.has_key?(attribute) ? "background-color: yellow" : "" end def row_classes(article) diff --git a/app/models/article.rb b/app/models/article.rb index e1011b1f..af14b31d 100644 --- a/app/models/article.rb +++ b/app/models/article.rb @@ -62,11 +62,11 @@ class Article < ActiveRecord::Base #validates_uniqueness_of :name, :scope => [:supplier_id, :deleted_at, :type], if: Proc.new {|a| a.supplier.shared_sync_method.blank? or a.supplier.shared_sync_method == 'import' } #validates_uniqueness_of :name, :scope => [:supplier_id, :deleted_at, :type, :unit, :unit_quantity] validate :uniqueness_of_name - + # Callbacks before_save :update_price_history before_destroy :check_article_in_use - + # The financial gross, net plus tax and deposti def gross_price ((price + deposit) * (tax / 100 + 1)).round(2) @@ -76,12 +76,12 @@ class Article < ActiveRecord::Base def fc_price (gross_price * (FoodsoftConfig[:price_markup] / 100 + 1)).round(2) end - + # Returns true if article has been updated at least 2 days ago def recently_updated updated_at > 2.days.ago end - + # If the article is used in an open Order, the Order will be returned. def in_open_order @in_open_order ||= begin @@ -90,92 +90,101 @@ class Article < ActiveRecord::Base order_article ? order_article.order : nil end end - + # Returns true if the article has been ordered in the given order at least once def ordered_in_order?(order) order.order_articles.where(article_id: id).where('quantity > 0').one? end - + # this method checks, if the shared_article has been changed # unequal attributes will returned in array - # if only the timestamps differ and the attributes are equal, + # if only the timestamps differ and the attributes are equal, # false will returned and self.shared_updated_on will be updated def shared_article_changed?(supplier = self.supplier) # skip early if the timestamp hasn't changed shared_article = self.shared_article(supplier) unless shared_article.nil? || self.shared_updated_on == shared_article.updated_on - - # try to convert units - # convert supplier's price and unit_quantity into fc-size - new_price, new_unit_quantity = self.convert_units - new_unit = self.unit - unless new_price && new_unit_quantity - # if convertion isn't possible, take shared_article-price/unit_quantity - new_price, new_unit_quantity, new_unit = shared_article.price, shared_article.unit_quantity, shared_article.unit - end - - # check if all attributes differ - unequal_attributes = Article.compare_attributes( - { - :name => [self.name, shared_article.name], - :manufacturer => [self.manufacturer, shared_article.manufacturer.to_s], - :origin => [self.origin, shared_article.origin], - :unit => [self.unit, new_unit], - :price => [self.price.to_f.round(2), new_price.to_f.round(2)], - :tax => [self.tax, shared_article.tax], - :deposit => [self.deposit.to_f.round(2), shared_article.deposit.to_f.round(2)], - # take care of different num-objects. - :unit_quantity => [self.unit_quantity.to_s.to_f, new_unit_quantity.to_s.to_f], - :note => [self.note.to_s, shared_article.note.to_s] - } - ) - if unequal_attributes.empty? + attrs = unequal_attributes(shared_article) + if attrs.empty? # when attributes not changed, update timestamp of article self.update_attribute(:shared_updated_on, shared_article.updated_on) false else - unequal_attributes + attrs end end end - - # compare attributes from different articles. used for auto-synchronization - # returns array of symbolized unequal attributes + + # Return article attributes that were changed (incl. unit conversion) + # @param [Article] New article to update self + # @return [Hash] Attributes with new values + def unequal_attributes(new_article) + # try to convert different units + new_price, new_unit_quantity = convert_units(new_article) + if new_price && new_unit_quantity + new_unit = self.unit + else + new_price = new_article.price + new_unit_quantity = new_article.unit_quantity + new_unit = new_article.unit + end + + return Article.compare_attributes( + { + :name => [self.name, new_article.name], + :manufacturer => [self.manufacturer, new_article.manufacturer.to_s], + :origin => [self.origin, new_article.origin], + :unit => [self.unit, new_unit], + :price => [self.price.to_f.round(2), new_price.to_f.round(2)], + :tax => [self.tax, new_article.tax], + :deposit => [self.deposit.to_f.round(2), new_article.deposit.to_f.round(2)], + # take care of different num-objects. + :unit_quantity => [self.unit_quantity.to_s.to_f, new_unit_quantity.to_s.to_f], + :note => [self.note.to_s, new_article.note.to_s] + } + ) + end + + # Compare attributes from two different articles. + # + # This is used for auto-synchronization + # @param attributes [Hash] Attributes with old and new values + # @return [Hash] Changed attributes with new values def self.compare_attributes(attributes) unequal_attributes = attributes.select { |name, values| values[0] != values[1] && !(values[0].blank? && values[1].blank?) } - unequal_attributes.collect { |pair| pair[0] } + Hash[unequal_attributes.to_a.map {|a| [a[0], a[1].last]}] end - + # to get the correspondent shared article def shared_article(supplier = self.supplier) self.order_number.blank? and return nil @shared_article ||= supplier.shared_supplier.shared_articles.find_by_number(self.order_number) rescue nil end - + # convert units in foodcoop-size # uses unit factors in app_config.yml to calc the price/unit_quantity # returns new price and unit_quantity in array, when calc is possible => [price, unit_quanity] # returns false if units aren't foodsoft-compatible # returns nil if units are eqal - def convert_units - if unit != shared_article.unit + def convert_units(new_article = shared_article) + if unit != new_article.unit # legacy, used by foodcoops in Germany - if shared_article.unit == "KI" && unit == "ST" # 'KI' means a box, with a different amount of items in it + if new_article.unit == "KI" && unit == "ST" # 'KI' means a box, with a different amount of items in it # try to match the size out of its name, e.g. "banana 10-12 St" => 10 - new_unit_quantity = /[0-9\-\s]+(St)/.match(shared_article.name).to_s.to_i + new_unit_quantity = /[0-9\-\s]+(St)/.match(new_article.name).to_s.to_i if new_unit_quantity && new_unit_quantity > 0 - new_price = (shared_article.price/new_unit_quantity.to_f).round(2) + new_price = (new_article.price/new_unit_quantity.to_f).round(2) [new_price, new_unit_quantity] else false end else # use ruby-units to convert fc_unit = (::Unit.new(unit) rescue nil) - supplier_unit = (::Unit.new(shared_article.unit) rescue nil) + supplier_unit = (::Unit.new(new_article.unit) rescue nil) if fc_unit && supplier_unit && fc_unit =~ supplier_unit conversion_factor = (supplier_unit / fc_unit).to_base.to_r - new_price = shared_article.price / conversion_factor - new_unit_quantity = shared_article.unit_quantity * conversion_factor + new_price = new_article.price / conversion_factor + new_unit_quantity = new_article.unit_quantity * conversion_factor [new_price, new_unit_quantity] else false @@ -196,7 +205,7 @@ class Article < ActiveRecord::Base end protected - + # Checks if the article is in use before it will deleted def check_article_in_use raise I18n.t('articles.model.error_in_use', :article => self.name.to_s) if self.in_open_order diff --git a/app/models/supplier.rb b/app/models/supplier.rb index a8f77781..e792835b 100644 --- a/app/models/supplier.rb +++ b/app/models/supplier.rb @@ -34,30 +34,9 @@ class Supplier < ActiveRecord::Base shared_article = article.shared_article(self) if shared_article # article will be updated - unequal_attributes = article.shared_article_changed?(self) unless unequal_attributes.blank? # skip if shared_article has not been changed - - # try to convert different units - new_price, new_unit_quantity = article.convert_units - if new_price && new_unit_quantity - article.price = new_price - article.unit_quantity = new_unit_quantity - else - article.price = shared_article.price - article.unit_quantity = shared_article.unit_quantity - article.unit = shared_article.unit - end - # update other attributes - article.attributes = { - :name => shared_article.name, - :manufacturer => shared_article.manufacturer, - :origin => shared_article.origin, - :shared_updated_on => shared_article.updated_on, - :tax => shared_article.tax, - :deposit => shared_article.deposit, - :note => shared_article.note - } + article.attributes = unequal_attributes updated_articles << [article, unequal_attributes] end # Articles with no order number can be used to put non-shared articles diff --git a/app/views/articles/sync.html.haml b/app/views/articles/sync.html.haml index 65b4d97c..e200f595 100644 --- a/app/views/articles/sync.html.haml +++ b/app/views/articles/sync.html.haml @@ -14,13 +14,13 @@ .alert= t '.outlist.alert_used', article: article.name %hr/ - - if @updated_articles.any? + - if @updated_article_pairs.any? %h2= t '.update.title' %p %i - = t '.update.update_msg', count: @updated_articles.size + = t '.update.update_msg', count: @updated_article_pairs.size = t '.update.body' - = render 'sync_table', articles: @updated_articles, field: 'articles', hidden_fields: %w(shared_updated_on) + = render 'sync_table', articles: @updated_article_pairs, field: 'articles', hidden_fields: %w(shared_updated_on) %hr/ - if @new_articles.any? diff --git a/lib/foodsoft_file.rb b/lib/foodsoft_file.rb index fb04e1a3..43a784c2 100644 --- a/lib/foodsoft_file.rb +++ b/lib/foodsoft_file.rb @@ -1,20 +1,25 @@ -# Module for Foodsoft-File import -# The Foodsoft-File is a cvs-file, with columns separated by semicolons +require 'roo' -require 'csv' +# Foodsoft-file import +class FoodsoftFile -module FoodsoftFile - # parses a string from a foodsoft-file # returns two arrays with articles and outlisted_articles # the parsed article is a simple hash - def self.parse(file) - articles, outlisted_articles = Array.new, Array.new - row_index = 2 - ::CSV.parse(file.read.force_encoding('utf-8'), {:col_sep => ";", :headers => true}) do |row| - # check if the line is empty - unless row[2] == "" || row[2].nil? - article = {:number => row[1], + def self.parse(file, options = {}) + filepath = file.is_a?(String) ? file : file.to_path + filename = options[:filename] || filepath + fileext = ::File.extname(filename) + options = {col_sep: ';', encoding: 'utf-8'}.merge(options) + s = Roo::Spreadsheet.open filepath, extension: fileext, csv_options: options + + row_index = 1 + s.each do |row| + if row_index == 1 + # @todo try to detect headers; for now using the index is ok + + elsif !row[2].blank? + article = {:order_number => row[1], :name => row[2], :note => row[3], :manufacturer => row[4], @@ -24,20 +29,13 @@ module FoodsoftFile :tax => row[8], :deposit => (row[9].nil? ? "0" : row[9]), :unit_quantity => row[10], - :scale_quantity => row[11], - :scale_price => row[12], - :category => row[13]} - case row[0] - when "x" - # check if the article is outlisted - outlisted_articles << article - else - articles << article - end + :article_category => row[13]} + status = row[0] && row[0].strip.downcase == 'x' ? :outlisted : nil + yield status, article, row_index end row_index += 1 end - return [articles, outlisted_articles] + row_index end - + end