From 4dd45b7b2fa58f73c31e54badefe3e6f4d268d45 Mon Sep 17 00:00:00 2001 From: viehlieb Date: Tue, 31 Jan 2023 15:25:36 +0100 Subject: [PATCH] include foodsoft-article-import use filetypes for manual uploading bnn, odin, foodsoft file use opts in .parse adapt specs to include file format add specs for odin, bnn, foodsoft files adapt localize input to remove ',' separator and replace with '.' remove depr foodsoftfile.rb and spreadsheet.rb remove todo --- Gemfile | 1 + Gemfile.lock | 10 + app/controllers/articles_controller.rb | 3 +- app/lib/foodsoft_file.rb | 25 -- app/lib/spreadsheet_file.rb | 22 -- app/models/concerns/localize_input.rb | 2 +- app/models/supplier.rb | 10 +- app/views/articles/upload.html.haml | 7 +- spec/controllers/articles_controller_spec.rb | 4 +- spec/fixtures/bnn_file_01.bnn | 5 + spec/fixtures/bnn_file_02.bnn | 2 + spec/fixtures/odin_file_01.xml | 273 +++++++++++++++++++ spec/fixtures/odin_file_02.xml | 75 +++++ spec/integration/articles_spec.rb | 128 ++++++--- spec/models/supplier_spec.rb | 2 +- 15 files changed, 471 insertions(+), 98 deletions(-) delete mode 100644 app/lib/foodsoft_file.rb delete mode 100644 app/lib/spreadsheet_file.rb create mode 100644 spec/fixtures/bnn_file_01.bnn create mode 100644 spec/fixtures/bnn_file_02.bnn create mode 100644 spec/fixtures/odin_file_01.xml create mode 100644 spec/fixtures/odin_file_02.xml diff --git a/Gemfile b/Gemfile index 01c2cfd7..75a515cb 100644 --- a/Gemfile +++ b/Gemfile @@ -49,6 +49,7 @@ gem 'attribute_normalizer' gem 'ice_cube' # At time of development 01-06-2022 mmddyyyy necessary fix for config_helper.rb form builder was not in rubygems so we pull from github, see: https://github.com/gregschmit/recurring_select/pull/152 gem 'recurring_select', git: 'https://github.com/gregschmit/recurring_select' +gem 'foodsoft_article_import', git: 'https://git.local-it.org/Foodsoft/foodsoft_article_import', tag: 'v0.1' gem 'roo' gem 'roo-xls' gem 'spreadsheet' diff --git a/Gemfile.lock b/Gemfile.lock index 5b1a9fe7..ad8f2c0f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,12 @@ +GIT + remote: https://git.local-it.org/Foodsoft/foodsoft_article_import + revision: c7e432c247ca6aa327ae889d22f78227efed2479 + tag: v0.1 + specs: + foodsoft_article_import (1.0.0) + roo (~> 2.9.0) + simplecov + GIT remote: https://github.com/gregschmit/recurring_select revision: 29febc4c4abdd6c30636c33a7d2daecb09973ecf @@ -618,6 +627,7 @@ DEPENDENCIES exception_notification factory_bot_rails faker + foodsoft_article_import! foodsoft_discourse! foodsoft_documents! foodsoft_links! diff --git a/app/controllers/articles_controller.rb b/app/controllers/articles_controller.rb index 4161e66a..a33d378a 100644 --- a/app/controllers/articles_controller.rb +++ b/app/controllers/articles_controller.rb @@ -148,10 +148,11 @@ class ArticlesController < ApplicationController # Update articles from a spreadsheet def parse_upload uploaded_file = params[:articles]['file'] or raise I18n.t('articles.controller.parse_upload.no_file') + type = params[:articles]['type'] options = { filename: uploaded_file.original_filename } options[:outlist_absent] = (params[:articles]['outlist_absent'] == '1') options[:convert_units] = (params[:articles]['convert_units'] == '1') - @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_from_file uploaded_file.tempfile, options + @updated_article_pairs, @outlisted_articles, @new_articles = @supplier.sync_from_file uploaded_file.tempfile, type, options if @updated_article_pairs.empty? && @outlisted_articles.empty? && @new_articles.empty? redirect_to supplier_articles_path(@supplier), :notice => I18n.t('articles.controller.parse_upload.notice') end diff --git a/app/lib/foodsoft_file.rb b/app/lib/foodsoft_file.rb deleted file mode 100644 index 95d06c60..00000000 --- a/app/lib/foodsoft_file.rb +++ /dev/null @@ -1,25 +0,0 @@ -# Foodsoft-file import -class FoodsoftFile - # parses a string from a foodsoft-file - # returns two arrays with articles and outlisted_articles - # the parsed article is a simple hash - def self.parse(file, options = {}) - SpreadsheetFile.parse file, options do |row, row_index| - next if row[2].blank? - - article = { :order_number => row[1], - :name => row[2], - :note => row[3], - :manufacturer => row[4], - :origin => row[5], - :unit => row[6], - :price => row[7], - :tax => row[8], - :deposit => (row[9].nil? ? "0" : row[9]), - :unit_quantity => row[10], - :article_category => row[13] } - status = row[0] && row[0].strip.downcase == 'x' ? :outlisted : nil - yield status, article, row_index - end - end -end diff --git a/app/lib/spreadsheet_file.rb b/app/lib/spreadsheet_file.rb deleted file mode 100644 index dbca9c90..00000000 --- a/app/lib/spreadsheet_file.rb +++ /dev/null @@ -1,22 +0,0 @@ -require 'roo' - -class SpreadsheetFile - def self.parse(file, options = {}) - filepath = file.is_a?(String) ? file : file.to_path - filename = options.delete(:filename) || filepath - fileext = File.extname(filename) - options[:csv_options] = { col_sep: ';', encoding: 'utf-8' }.merge(options[:csv_options] || {}) - s = Roo::Spreadsheet.open(filepath, options.merge({ extension: fileext })) - - row_index = 1 - s.each do |row| - if row_index == 1 - # @todo try to detect headers; for now using the index is ok - else - yield row, row_index - end - row_index += 1 - end - row_index - end -end diff --git a/app/models/concerns/localize_input.rb b/app/models/concerns/localize_input.rb index cfb44a44..b6330fcc 100644 --- a/app/models/concerns/localize_input.rb +++ b/app/models/concerns/localize_input.rb @@ -8,7 +8,7 @@ module LocalizeInput separator = I18n.t("separator", scope: "number.format") delimiter = I18n.t("delimiter", scope: "number.format") input.gsub!(delimiter, "") if input.match(/\d+#{Regexp.escape(delimiter)}+\d+#{Regexp.escape(separator)}+\d+/) # Remove delimiter - input.gsub!(separator, ".") # Replace separator with db compatible character + input.gsub!(separator, ".") or input.gsub!(",", ".") # Replace separator with db compatible character input rescue Rails.logger.warn "Can't localize input: #{input}" diff --git a/app/models/supplier.rb b/app/models/supplier.rb index 862f5c24..9d6cd8a5 100644 --- a/app/models/supplier.rb +++ b/app/models/supplier.rb @@ -1,3 +1,4 @@ +require 'foodsoft_article_import' class Supplier < ApplicationRecord include MarkAsDeletedWithName include CustomFields @@ -73,13 +74,16 @@ class Supplier < ApplicationRecord # Synchronise articles with spreadsheet. # # @param file [File] Spreadsheet file to parse - # @param options [Hash] Options passed to {FoodsoftFile#parse} except when listed here. + # @param options [Hash] Options passed to {FoodsoftArticleImport#parse} except when listed here. # @option options [Boolean] :outlist_absent Set to +true+ to remove articles not in spreadsheet. # @option options [Boolean] :convert_units Omit or set to +true+ to keep current units, recomputing unit quantity and price. - def sync_from_file(file, options = {}) + def sync_from_file(file, type, options = {}) all_order_numbers = [] updated_article_pairs, outlisted_articles, new_articles = [], [], [] - FoodsoftFile::parse file, options do |status, new_attrs, line| + custom_codes_path = File.join(Rails.root, "config", "custom_codes.yml") + opts = options.except(:convert_units, :outlist_absent) + custom_codes_file_path = custom_codes_path if File.exist?(custom_codes_path) + FoodsoftArticleImport.parse(file, custom_file_path: custom_codes_file_path, type: type, **opts) do |new_attrs, status, line| article = articles.undeleted.where(order_number: new_attrs[:order_number]).first new_attrs[:article_category] = ArticleCategory.find_match(new_attrs[:article_category]) new_attrs[:tax] ||= FoodsoftConfig[:tax_default] diff --git a/app/views/articles/upload.html.haml b/app/views/articles/upload.html.haml index 8f91d790..1b67ca1d 100644 --- a/app/views/articles/upload.html.haml +++ b/app/views/articles/upload.html.haml @@ -71,9 +71,14 @@ = form_for :articles, :url => parse_upload_supplier_articles_path(@supplier), :html => { multipart: true, class: "form-horizontal" } do |f| + .control-group - %label(for="articles_file")= t '.file_label' + %label(for="articles_file") + %strong= t '.file_label' = f.file_field "file" + %label(for="articles_file") + %strong="select the file type you are about to upload" + =f.collection_select :type, ["bnn","foodsoft","odin"], :to_s, :to_s .control-group %label(for="articles_outlist_absent") diff --git a/spec/controllers/articles_controller_spec.rb b/spec/controllers/articles_controller_spec.rb index b8772054..e2940f48 100644 --- a/spec/controllers/articles_controller_spec.rb +++ b/spec/controllers/articles_controller_spec.rb @@ -187,8 +187,8 @@ describe ArticlesController, type: :controller do describe '#parse_upload' do let(:file) { Rack::Test::UploadedFile.new(Rails.root.join('spec/fixtures/files/upload_test.csv'), original_filename: 'upload_test.csv') } - it 'updates particles from spreadsheet' do - post_with_supplier :parse_upload, params: { articles: { file: file, outlist_absent: '1', convert_units: '1' } } + it 'updates articles from spreadsheet' do + post_with_supplier :parse_upload, params: { articles: { file: file, outlist_absent: '1', convert_units: '1', type: 'foodsoft' } } expect(response).to have_http_status(:success) end diff --git a/spec/fixtures/bnn_file_01.bnn b/spec/fixtures/bnn_file_01.bnn new file mode 100644 index 00000000..177da7be --- /dev/null +++ b/spec/fixtures/bnn_file_01.bnn @@ -0,0 +1,5 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +29932;;;;4280001958081;4280001958203;Walnoten (ongeroosterd);bio;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;;1;1 kg;1;N;930190;99260;;1,41;;;;1;;;4,49;2,34;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +28391;;;;4280001958081;4280001958203;Pijnboompitten;dem;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;;1;100 g;10;N;930190;99260;;1,41;;;;1;;;5,56;2.89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +1829;;;;4280001958081;4280001958203;Appelsap (verpakt);;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;4x250 ml;10;4x250 ml;10;N;930190;99260;;3,21;;;;1;;;4,49;2.89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;ml;28,571;; +177813;;;;4280001958081;4280001958203;Tomaten;bio;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;;1;500 g;20;N;930190;99260;;1,20;;;;1;;;4,49;2.89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;g;28,571;; diff --git a/spec/fixtures/bnn_file_02.bnn b/spec/fixtures/bnn_file_02.bnn new file mode 100644 index 00000000..e3dba5bb --- /dev/null +++ b/spec/fixtures/bnn_file_02.bnn @@ -0,0 +1,2 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +1;;;;4280001958081;4280001958203;Tomatoes;organic;;;med;;GR;C%;DE-?KO-001;120;1302;10;55;;1;;20;500 g;1;N;930190;99260;;1,41;;;;1;;;4,49;1,20;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;g;28,571;; \ No newline at end of file diff --git a/spec/fixtures/odin_file_01.xml b/spec/fixtures/odin_file_01.xml new file mode 100644 index 00000000..3b60e83e --- /dev/null +++ b/spec/fixtures/odin_file_01.xml @@ -0,0 +1,273 @@ + + + +1039 +1.08 +Estafette Associatie C.V. +Geldermalsen + + +8719325207668 +Walnoten (ongeroosterd) +Nucli rose + +0 +0 +0 +1 +kg +Stuk +0 +Het warme woud +bio + + +NL + +6 +1017515 +29932 +10 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +2.34 +7.95 + + + +8719325207668 +Pijnboompitten +Nucli rose + +0 +0 +0 +100 +g +Stuk +0 +NELEMAN +dem + + +TR + +6 +1017515 +28391 +10 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +5.56 +7.95 + + + +8719325207668 +Appelsap (verpakt) +Nucli rose + +0 +0 +0 +4x250 +ml +Stuk +0.4 +Appelgaarde + + + +DE + +6 +1017515 +1829 +10 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +3.21 +7.95 + + + +8719325207668 +Tomaten +Nucli rose + +0 +0 +0 +500 +g +Stuk +0 +De röde hof +bio + + +DE + +6 +1017515 +177813 +20 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +1.2 +7.95 + + + \ No newline at end of file diff --git a/spec/fixtures/odin_file_02.xml b/spec/fixtures/odin_file_02.xml new file mode 100644 index 00000000..c732b4d5 --- /dev/null +++ b/spec/fixtures/odin_file_02.xml @@ -0,0 +1,75 @@ + + + +1039 +1.08 +Estafette Associatie C.V. +Geldermalsen + + +8719325207668 +Tomatoes +Nucli rose + +0 +0 +0 +500 +g +Stuk +0 +De röde hof +organic + + +Somewhere, UK + +6 +1017515 +1 +20 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +1.2 +7.95 + + + \ No newline at end of file diff --git a/spec/integration/articles_spec.rb b/spec/integration/articles_spec.rb index bbd5e375..15608311 100644 --- a/spec/integration/articles_spec.rb +++ b/spec/integration/articles_spec.rb @@ -1,9 +1,9 @@ require_relative '../spec_helper' feature ArticlesController do - let(:user) { create :user, groups: [create(:workgroup, role_article_meta: true)] } - let(:supplier) { create :supplier } - let!(:article_category) { create :article_category } + let(:user) { create(:user, groups: [create(:workgroup, role_article_meta: true)]) } + let(:supplier) { create(:supplier) } + let!(:article_category) { create(:article_category) } before { login user } @@ -18,7 +18,7 @@ feature ArticlesController do it 'can create a new article' do click_on I18n.t('articles.index.new') expect(page).to have_selector('form#new_article') - article = build :article, supplier: supplier, article_category: article_category + article = build(:article, supplier: supplier, article_category: article_category) within('#new_article') do fill_in 'article_name', :with => article.name fill_in 'article_unit', :with => article.unit @@ -49,6 +49,7 @@ feature ArticlesController do let(:file) { Rails.root.join(test_file) } it do + find("#articles_type option[value='foodsoft']").select_option find('input[type="submit"]').click expect(find("tr:nth-child(1) #new_articles__note").value).to eq "bio ◎" expect(find("tr:nth-child(2) #new_articles__name").value).to eq "Pijnboompitten" @@ -64,56 +65,99 @@ feature ArticlesController do end end - describe "can update existing article" do - let!(:article) { create :article, supplier: supplier, name: 'Foobar', order_number: 1, unit: '250 g' } + Dir.glob('spec/fixtures/bnn_file_01.*') do |test_file| + describe "can import articles from #{test_file}" do + let(:file) { Rails.root.join(test_file) } - it do - find('input[type="submit"]').click - expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes' - find('input[type="submit"]').click - article.reload - expect(article.name).to eq 'Tomatoes' - expect([article.unit, article.unit_quantity, article.price]).to eq ['500 g', 20, 1.2] + it do + find("#articles_type option[value='bnn']").select_option + find('input[type="submit"]').click + expect(find("tr:nth-child(1) #new_articles__note").value).to eq "bio" + expect(find("tr:nth-child(1) #new_articles__name").value).to eq "Walnoten (ongeroosterd)" + # set article category + 4.times do |i| + all("tr:nth-child(#{i + 1}) select > option")[1].select_option + end + find('input[type="submit"]').click + + expect(page).to have_content("Pijnboompitten") + + expect(supplier.articles.count).to eq 4 + end end end + end - describe "handles missing data" do - it do - find('input[type="submit"]').click # to overview - find('input[type="submit"]').click # missing category, re-show form - expect(find('tr.alert')).to be_present - expect(supplier.articles.count).to eq 0 + describe "updates" do + file_paths = ['spec/fixtures/foodsoft_file_02.csv', 'spec/fixtures/bnn_file_02.bnn', 'spec/fixtures/odin_file_02.xml'] + let(:filename) { 'foodsoft_file_02.csv' } + let(:file) { Rails.root.join("spec/fixtures/#{filename}") } + let(:val) { 'foodsoft' } + let(:type) { %w[foodsoft bnn odin] } - all("tr select > option")[1].select_option - find('input[type="submit"]').click # now it should succeed - expect(supplier.articles.count).to eq 1 - end + before do + visit upload_supplier_articles_path(supplier_id: supplier.id) + attach_file 'articles_file', file + find("#articles_type option[value='#{val}']").select_option end - describe "can remove an existing article" do - let!(:article) { create :article, supplier: supplier, name: 'Foobar', order_number: 99999 } + file_paths.each_with_index do |test_file, index| + describe "updates article for #{test_file}" do + let(:article) { create(:article, supplier: supplier, name: 'Foobar', order_number: 1, unit: '250 g') } + let(:file) { Rails.root.join(test_file) } + let(:val) { type[index] } - it do - check('articles_outlist_absent') - find('input[type="submit"]').click - expect(find("#outlisted_articles_#{article.id}", visible: :all)).to be_present + it do + article.reload + find('input[type="submit"]').click + expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes' + find('input[type="submit"]').click + article.reload + expect(article.name).to eq 'Tomatoes' + if type[index] == "odin" + expect([article.unit, article.unit_quantity, article.price]).to eq ['500gr', 20, 1.20] + else + expect([article.unit, article.unit_quantity, article.price]).to eq ['500 g', 20, 1.20] + end + end - all("tr select > option")[1].select_option - find('input[type="submit"]').click - expect(article.reload.deleted?).to be true + it "handles missing data" do + find('input[type="submit"]').click # to overview + find('input[type="submit"]').click # missing category, re-show form + expect(find('tr.alert')).to be_present + expect(supplier.articles.count).to eq 0 + + all("tr select > option")[1].select_option + find('input[type="submit"]').click # now it should succeed + expect(supplier.articles.count).to eq 1 + end end - end - describe "can convert units when updating" do - let!(:article) { create :article, supplier: supplier, order_number: 1, unit: '250 g' } + describe "can remove an existing article" do + let!(:article) { create(:article, supplier: supplier, name: 'Foobar', order_number: 99999) } - it do - check('articles_convert_units') - find('input[type="submit"]').click - expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes' - find('input[type="submit"]').click - article.reload - expect([article.unit, article.unit_quantity, article.price]).to eq ['250 g', 40, 0.6] + it do + check('articles_outlist_absent') + find('input[type="submit"]').click + expect(find("#outlisted_articles_#{article.id}", visible: :all)).to be_present + + all("tr select > option")[1].select_option + find('input[type="submit"]').click + expect(article.reload.deleted?).to be true + end + end + + describe "can convert units when updating" do + let!(:article) { create(:article, supplier: supplier, order_number: 1, unit: '250 g') } + + it do + check('articles_convert_units') + find('input[type="submit"]').click + expect(find("#articles_#{article.id}_name").value).to eq 'Tomatoes' + find('input[type="submit"]').click + article.reload + expect([article.unit, article.unit_quantity, article.price]).to eq ['250 g', 40, 0.6] + end end end end diff --git a/spec/models/supplier_spec.rb b/spec/models/supplier_spec.rb index 6bcc6e7b..42b4a304 100644 --- a/spec/models/supplier_spec.rb +++ b/spec/models/supplier_spec.rb @@ -11,7 +11,7 @@ describe Supplier do options = { filename: 'foodsoft_file_01.csv' } options[:outlist_absent] = true options[:convert_units] = true - updated_article_pairs, outlisted_articles, new_articles = supplier.sync_from_file(Rails.root.join('spec/fixtures/foodsoft_file_01.csv'), options) + updated_article_pairs, outlisted_articles, new_articles = supplier.sync_from_file(Rails.root.join('spec/fixtures/foodsoft_file_01.csv'), "foodsoft", options) expect(new_articles.length).to be > 0 expect(updated_article_pairs.first[1][:name]).to eq 'Tomaten' expect(outlisted_articles.first).to eq article2