diff --git a/Gemfile b/Gemfile index 01c2cfd7..75a515cb 100644 --- a/Gemfile +++ b/Gemfile @@ -49,6 +49,7 @@ gem 'attribute_normalizer' gem 'ice_cube' # At time of development 01-06-2022 mmddyyyy necessary fix for config_helper.rb form builder was not in rubygems so we pull from github, see: https://github.com/gregschmit/recurring_select/pull/152 gem 'recurring_select', git: 'https://github.com/gregschmit/recurring_select' +gem 'foodsoft_article_import', git: 'https://git.local-it.org/Foodsoft/foodsoft_article_import', tag: 'v0.1' gem 'roo' gem 'roo-xls' gem 'spreadsheet' diff --git a/Gemfile.lock b/Gemfile.lock index 5b1a9fe7..ad8f2c0f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,12 @@ +GIT + remote: https://git.local-it.org/Foodsoft/foodsoft_article_import + revision: c7e432c247ca6aa327ae889d22f78227efed2479 + tag: v0.1 + specs: + foodsoft_article_import (1.0.0) + roo (~> 2.9.0) + simplecov + GIT remote: https://github.com/gregschmit/recurring_select revision: 29febc4c4abdd6c30636c33a7d2daecb09973ecf @@ -618,6 +627,7 @@ DEPENDENCIES exception_notification factory_bot_rails faker + foodsoft_article_import! foodsoft_discourse! foodsoft_documents! foodsoft_links! diff --git a/app/controllers/articles_controller.rb b/app/controllers/articles_controller.rb index 4161e66a..a33d378a 100644 --- a/app/controllers/articles_controller.rb +++ b/app/controllers/articles_controller.rb @@ -148,10 +148,11 @@ class ArticlesController < ApplicationController # Update articles from a spreadsheet def parse_upload uploaded_file = params[:articles]['file'] or raise I18n.t('articles.controller.parse_upload.no_file') + type = params[:articles]['type'] options = { filename: uploaded_file.original_filename } options[:outlist_absent] = (params[:articles]['outlist_absent'] == '1') options[:convert_units] = (params[:articles]['convert_units'] == '1') - @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_from_file uploaded_file.tempfile, options + @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_from_file uploaded_file.tempfile, type, options if @updated_article_pairs.empty? && @outlisted_articles.empty? && @new_articles.empty? redirect_to supplier_articles_path(@supplier), :notice => I18n.t('articles.controller.parse_upload.notice') end diff --git a/app/lib/foodsoft_file.rb b/app/lib/foodsoft_file.rb deleted file mode 100644 index 95d06c60..00000000 --- a/app/lib/foodsoft_file.rb +++ /dev/null @@ -1,25 +0,0 @@ -# Foodsoft-file import -class FoodsoftFile - # parses a string from a foodsoft-file - # returns two arrays with articles and outlisted_articles - # the parsed article is a simple hash - def self.parse(file, options = {}) - SpreadsheetFile.parse file, options do |row, row_index| - next if row[2].blank? - - article = { :order_number => row[1], - :name => row[2], - :note => row[3], - :manufacturer => row[4], - :origin => row[5], - :unit => row[6], - :price => row[7], - :tax => row[8], - :deposit => (row[9].nil? ? "0" : row[9]), - :unit_quantity => row[10], - :article_category => row[13] } - status = row[0] && row[0].strip.downcase == 'x' ? :outlisted : nil - yield status, article, row_index - end - end -end diff --git a/app/lib/spreadsheet_file.rb b/app/lib/spreadsheet_file.rb deleted file mode 100644 index dbca9c90..00000000 --- a/app/lib/spreadsheet_file.rb +++ /dev/null @@ -1,22 +0,0 @@ -require 'roo' - -class SpreadsheetFile - def self.parse(file, options = {}) - filepath = file.is_a?(String) ? file : file.to_path - filename = options.delete(:filename) || filepath - fileext = File.extname(filename) - options[:csv_options] = { col_sep: ';', encoding: 'utf-8' }.merge(options[:csv_options] || {}) - s = Roo::Spreadsheet.open(filepath, options.merge({ extension: fileext })) - - row_index = 1 - s.each do |row| - if row_index == 1 - # @todo try to detect headers; for now using the index is ok - else - yield row, row_index - end - row_index += 1 - end - row_index - end -end diff --git a/app/models/concerns/localize_input.rb b/app/models/concerns/localize_input.rb index cfb44a44..b6330fcc 100644 --- a/app/models/concerns/localize_input.rb +++ b/app/models/concerns/localize_input.rb @@ -8,7 +8,7 @@ module LocalizeInput separator = I18n.t("separator", scope: "number.format") delimiter = I18n.t("delimiter", scope: "number.format") input.gsub!(delimiter, "") if input.match(/\d+#{Regexp.escape(delimiter)}+\d+#{Regexp.escape(separator)}+\d+/) # Remove delimiter - input.gsub!(separator, ".") # Replace separator with db compatible character + input.gsub!(separator, ".") or input.gsub!(",", ".") # Replace separator with db compatible character input rescue Rails.logger.warn "Can't localize input: #{input}" diff --git a/app/models/supplier.rb b/app/models/supplier.rb index 862f5c24..9d6cd8a5 100644 --- a/app/models/supplier.rb +++ b/app/models/supplier.rb @@ -1,3 +1,4 @@ +require 'foodsoft_article_import' class Supplier < ApplicationRecord include MarkAsDeletedWithName include CustomFields @@ -73,13 +74,16 @@ class Supplier < ApplicationRecord # Synchronise articles with spreadsheet. # # @param file [File] Spreadsheet file to parse - # @param options [Hash] Options passed to {FoodsoftFile#parse} except when listed here. + # @param options [Hash] Options passed to {FoodsoftArticleImport#parse} except when listed here. # @option options [Boolean] :outlist_absent Set to +true+ to remove articles not in spreadsheet. # @option options [Boolean] :convert_units Omit or set to +true+ to keep current units, recomputing unit quantity and price. - def sync_from_file(file, options = {}) + def sync_from_file(file, type, options = {}) all_order_numbers = [] updated_article_pairs, outlisted_articles, new_articles = [], [], [] - FoodsoftFile::parse file, options do |status, new_attrs, line| + custom_codes_path = File.join(Rails.root, "config", "custom_codes.yml") + opts = options.except(:convert_units, :outlist_absent) + custom_codes_file_path = custom_codes_path if File.exist?(custom_codes_path) + FoodsoftArticleImport.parse(file, custom_file_path: custom_codes_file_path, type: type, **opts) do |new_attrs, status, line| article = articles.undeleted.where(order_number: new_attrs[:order_number]).first new_attrs[:article_category] = ArticleCategory.find_match(new_attrs[:article_category]) new_attrs[:tax] ||= FoodsoftConfig[:tax_default] diff --git a/app/views/articles/upload.html.haml b/app/views/articles/upload.html.haml index 8f91d790..1b67ca1d 100644 --- a/app/views/articles/upload.html.haml +++ b/app/views/articles/upload.html.haml @@ -71,9 +71,14 @@ = form_for :articles, :url => parse_upload_supplier_articles_path(@supplier), :html => { multipart: true, class: "form-horizontal" } do |f| + .control-group - %label(for="articles_file")= t '.file_label' + %label(for="articles_file") + %strong= t '.file_label' = f.file_field "file" + %label(for="articles_file") + %strong="select the file type you are about to upload" + =f.collection_select :type, ["bnn","foodsoft","odin"], :to_s, :to_s .control-group %label(for="articles_outlist_absent") diff --git a/spec/controllers/articles_controller_spec.rb b/spec/controllers/articles_controller_spec.rb index b8772054..e2940f48 100644 --- a/spec/controllers/articles_controller_spec.rb +++ b/spec/controllers/articles_controller_spec.rb @@ -187,8 +187,8 @@ describe ArticlesController, type: :controller do describe '#parse_upload' do let(:file) { Rack::Test::UploadedFile.new(Rails.root.join('spec/fixtures/files/upload_test.csv'), original_filename: 'upload_test.csv') } - it 'updates particles from spreadsheet' do - post_with_supplier :parse_upload, params: { articles: { file: file, outlist_absent: '1', convert_units: '1' } } + it 'updates articles from spreadsheet' do + post_with_supplier :parse_upload, params: { articles: { file: file, outlist_absent: '1', convert_units: '1', type: 'foodsoft' } } expect(response).to have_http_status(:success) end diff --git a/spec/fixtures/bnn_file_01.bnn b/spec/fixtures/bnn_file_01.bnn new file mode 100644 index 00000000..177da7be --- /dev/null +++ b/spec/fixtures/bnn_file_01.bnn @@ -0,0 +1,5 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +29932;;;;4280001958081;4280001958203;Walnoten (ongeroosterd);bio;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;;1;1 kg;1;N;930190;99260;;1,41;;;;1;;;4,49;2,34;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +28391;;;;4280001958081;4280001958203;Pijnboompitten;dem;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;;1;100 g;10;N;930190;99260;;1,41;;;;1;;;5,56;2.89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +1829;;;;4280001958081;4280001958203;Appelsap (verpakt);;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;4x250 ml;10;4x250 ml;10;N;930190;99260;;3,21;;;;1;;;4,49;2.89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;ml;28,571;; +177813;;;;4280001958081;4280001958203;Tomaten;bio;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;;1;500 g;20;N;930190;99260;;1,20;;;;1;;;4,49;2.89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;g;28,571;; diff --git a/spec/fixtures/bnn_file_02.bnn b/spec/fixtures/bnn_file_02.bnn new file mode 100644 index 00000000..e3dba5bb --- /dev/null +++ b/spec/fixtures/bnn_file_02.bnn @@ -0,0 +1,2 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +1;;;;4280001958081;4280001958203;Tomatoes;organic;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;;20;500 g;1;N;930190;99260;;1,41;;;;1;;;4,49;1,20;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;g;28,571;; \ No newline at end of file diff --git a/spec/fixtures/odin_file_01.xml b/spec/fixtures/odin_file_01.xml new file mode 100644 index 00000000..3b60e83e --- /dev/null +++ b/spec/fixtures/odin_file_01.xml @@ -0,0 +1,273 @@ + + + +1039 +1.08 +Estafette Associatie C.V. +Geldermalsen + + +8719325207668 +Walnoten (ongeroosterd) +Nucli rose + +0 +0 +0 +1 +kg +Stuk +0 +Het warme woud +bio + + +NL + +6 +1017515 +29932 +10 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +2.34 +7.95 + + + +8719325207668 +Pijnboompitten +Nucli rose + +0 +0 +0 +100 +g +Stuk +0 +NELEMAN +dem + + +TR + +6 +1017515 +28391 +10 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +5.56 +7.95 + + + +8719325207668 +Appelsap (verpakt) +Nucli rose + +0 +0 +0 +4x250 +ml +Stuk +0.4 +Appelgaarde + + + +DE + +6 +1017515 +1829 +10 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +3.21 +7.95 + + + +8719325207668 +Tomaten +Nucli rose + +0 +0 +0 +500 +g +Stuk +0 +De röde hof +bio + + +DE + +6 +1017515 +177813 +20 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +1.2 +7.95 + + + \ No newline at end of file diff --git a/spec/fixtures/odin_file_02.xml b/spec/fixtures/odin_file_02.xml new file mode 100644 index 00000000..c732b4d5 --- /dev/null +++ b/spec/fixtures/odin_file_02.xml @@ -0,0 +1,75 @@ + + + +1039 +1.08 +Estafette Associatie C.V. +Geldermalsen + + +8719325207668 +Tomatoes +Nucli rose + +0 +0 +0 +500 +g +Stuk +0 +De röde hof +organic + + +Somewhere, UK + +6 +1017515 +1 +20 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +1.2 +7.95 + + + \ No newline at end of file diff --git a/spec/integration/articles_spec.rb b/spec/integration/articles_spec.rb index bbd5e375..15608311 100644 --- a/spec/integration/articles_spec.rb +++ b/spec/integration/articles_spec.rb @@ -1,9 +1,9 @@ require_relative '../spec_helper' feature ArticlesController do - let(:user) { create :user, groups: [create(:workgroup, role_article_meta: true)] } - let(:supplier) { create :supplier } - let!(:article_category) { create :article_category } + let(:user) { create(:user, groups: [create(:workgroup, role_article_meta: true)]) } + let(:supplier) { create(:supplier) } + let!(:article_category) { create(:article_category) } before { login user } @@ -18,7 +18,7 @@ feature ArticlesController do it 'can create a new article' do click_on I18n.t('articles.index.new') expect(page).to have_selector('form#new_article') - article = build :article, supplier: supplier, article_category: article_category + article = build(:article, supplier: supplier, article_category: article_category) within('#new_article') do fill_in 'article_name', :with => article.name fill_in 'article_unit', :with => article.unit @@ -49,6 +49,7 @@ feature ArticlesController do let(:file) { Rails.root.join(test_file) } it do + find("#articles_type option[value='foodsoft']").select_option find('input[type="submit"]').click expect(find("tr:nth-child(1) #new_articles__note").value).to eq "bio ◎" expect(find("tr:nth-child(2) #new_articles__name").value).to eq "Pijnboompitten" @@ -64,56 +65,99 @@ feature ArticlesController do end end - describe "can update existing article" do - let!(:article) { create :article, supplier: supplier, name: 'Foobar', order_number: 1, unit: '250 g' } + Dir.glob('spec/fixtures/bnn_file_01.*') do |test_file| + describe "can import articles from #{test_file}" do + let(:file) { Rails.root.join(test_file) } - it do - find('input[type="submit"]').click - expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes' - find('input[type="submit"]').click - article.reload - expect(article.name).to eq 'Tomatoes' - expect([article.unit, article.unit_quantity, article.price]).to eq ['500 g', 20, 1.2] + it do + find("#articles_type option[value='bnn']").select_option + find('input[type="submit"]').click + expect(find("tr:nth-child(1) #new_articles__note").value).to eq "bio" + expect(find("tr:nth-child(1) #new_articles__name").value).to eq "Walnoten (ongeroosterd)" + # set article category + 4.times do |i| + all("tr:nth-child(#{i + 1}) select > option")[1].select_option + end + find('input[type="submit"]').click + + expect(page).to have_content("Pijnboompitten") + + expect(supplier.articles.count).to eq 4 + end end end + end - describe "handles missing data" do - it do - find('input[type="submit"]').click # to overview - find('input[type="submit"]').click # missing category, re-show form - expect(find('tr.alert')).to be_present - expect(supplier.articles.count).to eq 0 + describe "updates" do + file_paths = ['spec/fixtures/foodsoft_file_02.csv', 'spec/fixtures/bnn_file_02.bnn', 'spec/fixtures/odin_file_02.xml'] + let(:filename) { 'foodsoft_file_02.csv' } + let(:file) { Rails.root.join("spec/fixtures/#{filename}") } + let(:val) { 'foodsoft' } + let(:type) { %w[foodsoft bnn odin] } - all("tr select > option")[1].select_option - find('input[type="submit"]').click # now it should succeed - expect(supplier.articles.count).to eq 1 - end + before do + visit upload_supplier_articles_path(supplier_id: supplier.id) + attach_file 'articles_file', file + find("#articles_type option[value='#{val}']").select_option end - describe "can remove an existing article" do - let!(:article) { create :article, supplier: supplier, name: 'Foobar', order_number: 99999 } + file_paths.each_with_index do |test_file, index| + describe "updates article for #{test_file}" do + let(:article) { create(:article, supplier: supplier, name: 'Foobar', order_number: 1, unit: '250 g') } + let(:file) { Rails.root.join(test_file) } + let(:val) { type[index] } - it do - check('articles_outlist_absent') - find('input[type="submit"]').click - expect(find("#outlisted_articles_#{article.id}", visible: :all)).to be_present + it do + article.reload + find('input[type="submit"]').click + expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes' + find('input[type="submit"]').click + article.reload + expect(article.name).to eq 'Tomatoes' + if type[index] == "odin" + expect([article.unit, article.unit_quantity, article.price]).to eq ['500gr', 20, 1.20] + else + expect([article.unit, article.unit_quantity, article.price]).to eq ['500 g', 20, 1.20] + end + end - all("tr select > option")[1].select_option - find('input[type="submit"]').click - expect(article.reload.deleted?).to be true + it "handles missing data" do + find('input[type="submit"]').click # to overview + find('input[type="submit"]').click # missing category, re-show form + expect(find('tr.alert')).to be_present + expect(supplier.articles.count).to eq 0 + + all("tr select > option")[1].select_option + find('input[type="submit"]').click # now it should succeed + expect(supplier.articles.count).to eq 1 + end end - end - describe "can convert units when updating" do - let!(:article) { create :article, supplier: supplier, order_number: 1, unit: '250 g' } + describe "can remove an existing article" do + let!(:article) { create(:article, supplier: supplier, name: 'Foobar', order_number: 99999) } - it do - check('articles_convert_units') - find('input[type="submit"]').click - expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes' - find('input[type="submit"]').click - article.reload - expect([article.unit, article.unit_quantity, article.price]).to eq ['250 g', 40, 0.6] + it do + check('articles_outlist_absent') + find('input[type="submit"]').click + expect(find("#outlisted_articles_#{article.id}", visible: :all)).to be_present + + all("tr select > option")[1].select_option + find('input[type="submit"]').click + expect(article.reload.deleted?).to be true + end + end + + describe "can convert units when updating" do + let!(:article) { create(:article, supplier: supplier, order_number: 1, unit: '250 g') } + + it do + check('articles_convert_units') + find('input[type="submit"]').click + expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes' + find('input[type="submit"]').click + article.reload + expect([article.unit, article.unit_quantity, article.price]).to eq ['250 g', 40, 0.6] + end end end end diff --git a/spec/models/supplier_spec.rb b/spec/models/supplier_spec.rb index 6bcc6e7b..42b4a304 100644 --- a/spec/models/supplier_spec.rb +++ b/spec/models/supplier_spec.rb @@ -11,7 +11,7 @@ describe Supplier do options = { filename: 'foodsoft_file_01.csv' } options[:outlist_absent] = true options[:convert_units] = true - updated_article_pairs, outlisted_articles, new_articles = supplier.sync_from_file(Rails.root.join('spec/fixtures/foodsoft_file_01.csv'), options) + updated_article_pairs, outlisted_articles, new_articles = supplier.sync_from_file(Rails.root.join('spec/fixtures/foodsoft_file_01.csv'), "foodsoft", options) expect(new_articles.length).to be > 0 expect(updated_article_pairs.first[1][:name]).to eq 'Tomaten' expect(outlisted_articles.first).to eq article2