From edcbeaea2b5ce5ef32d195af4b7328ff0c645a81 Mon Sep 17 00:00:00 2001 From: viehlieb Date: Wed, 25 Jan 2023 18:27:33 +0100 Subject: [PATCH 1/3] change to optional named parameter --- lib/foodsoft_article_import.rb | 15 +++++++++++---- lib/foodsoft_article_import/bioromeo.rb | 3 ++- lib/foodsoft_article_import/bnn.rb | 7 ++++--- lib/foodsoft_article_import/borkenstein.rb | 3 ++- lib/foodsoft_article_import/dnb_xml.rb | 18 +++++++++++------- lib/foodsoft_article_import/foodsoft.rb | 4 +++- lib/foodsoft_article_import/utf8_encoder.rb | 9 +++++++++ 7 files changed, 42 insertions(+), 17 deletions(-) create mode 100644 lib/foodsoft_article_import/utf8_encoder.rb diff --git a/lib/foodsoft_article_import.rb b/lib/foodsoft_article_import.rb index e70748b..4ea4d55 100644 --- a/lib/foodsoft_article_import.rb +++ b/lib/foodsoft_article_import.rb @@ -5,6 +5,7 @@ require 'yaml' require 'active_support/core_ext/hash/keys' require_relative 'foodsoft_article_import/bioromeo' require_relative 'foodsoft_article_import/bnn' +require_relative 'foodsoft_article_import/utf8_encoder' require_relative 'foodsoft_article_import/borkenstein' require_relative 'foodsoft_article_import/dnb_xml' require_relative 'foodsoft_article_import/foodsoft' @@ -34,11 +35,13 @@ module FoodsoftArticleImport # @param file [File, Tempfile] # @option opts [String] type file format (required) (see {.file_formats}) # @return [File, Roo::Spreadsheet] file with encoding set if needed - def self.parse(file, custom_file_path=nil, type='bnn', **opts, &blk) + def self.parse(file, custom_file_path: nil, type: nil, **opts, &blk) # @todo handle wrong or undetected type - type = opts[:type] || 'bnn' - puts type + custom_file_path ||= nil + type ||= 'bnn' + parser = file_formats[type] + puts parser if block_given? parser.parse(file, custom_file_path, **opts, &blk) else @@ -71,6 +74,10 @@ module FoodsoftArticleImport opts[:csv_options][:encoding] = encoding if encoding opts[:csv_options][:col_sep] = col_sep if col_sep opts[:extension] = File.extname(filename) if filename - Roo::Spreadsheet.open(file, **opts) + begin + Roo::Spreadsheet.open(file, **opts) + rescue => e + raise "Failed to parse foodsoft file. make sure file format is correct: #{e.message}" + end end end \ No newline at end of file diff --git a/lib/foodsoft_article_import/bioromeo.rb b/lib/foodsoft_article_import/bioromeo.rb index e32f34c..a7f73ac 100644 --- a/lib/foodsoft_article_import/bioromeo.rb +++ b/lib/foodsoft_article_import/bioromeo.rb @@ -27,7 +27,8 @@ module FoodsoftArticleImport RES_PARSE_UNIT_LIST.map {|r| /#{r}\s*$/} + RES_PARSE_UNIT_LIST.map {|r| /-#{r}/} - def self.parse(file, custom_file_path=nil, **opts) + def self.parse(file, custom_file_path: nil, **opts) + custom_file_path ||= nil opts = OPTIONS.merge(opts) ss = FoodsoftArticleImport.open_spreadsheet(file, **opts) diff --git a/lib/foodsoft_article_import/bnn.rb b/lib/foodsoft_article_import/bnn.rb index fbf2ed4..da221d1 100644 --- a/lib/foodsoft_article_import/bnn.rb +++ b/lib/foodsoft_article_import/bnn.rb @@ -52,7 +52,8 @@ module FoodsoftArticleImport }.freeze # parses a bnn-file - def self.parse(file, custom_file_path=nil, **opts) + def self.parse(file, custom_file_path: nil, **opts) + custom_file_path ||= nil encoding = opts[:encoding] || OPTIONS[:encoding] col_sep = opts[:col_sep] || OPTIONS[:col_sep] self.load_codes(custom_file_path) @@ -60,9 +61,9 @@ module FoodsoftArticleImport # check if the line is empty unless row[0] == "" || row[0].nil? article = { - :name => row[6], + :name => UTF8Encoder.clean(row[6]), :order_number => row[0], - :note => row[7], + :note => UTF8Encoder.clean(row[7]), :manufacturer => self.translate(:manufacturer, row[10]), :origin => row[12], :article_category => self.translate(:category, row[16]), diff --git a/lib/foodsoft_article_import/borkenstein.rb b/lib/foodsoft_article_import/borkenstein.rb index 72be944..820aee3 100644 --- a/lib/foodsoft_article_import/borkenstein.rb +++ b/lib/foodsoft_article_import/borkenstein.rb @@ -19,7 +19,8 @@ module FoodsoftArticleImport encoding: "UTF-8" # @todo check this }.freeze - def self.parse(file, custom_file_path=nil, **opts) + def self.parse(file, custom_file_path: nil, **opts) + custom_file_path ||= nil global_manufacturer = nil file.set_encoding(opts[:encoding] || OPTIONS[:encoding]) diff --git a/lib/foodsoft_article_import/dnb_xml.rb b/lib/foodsoft_article_import/dnb_xml.rb index 4f7ac3d..377881f 100644 --- a/lib/foodsoft_article_import/dnb_xml.rb +++ b/lib/foodsoft_article_import/dnb_xml.rb @@ -14,26 +14,31 @@ module FoodsoftArticleImport OPTIONS = {}.freeze # parses a string or file - def self.parse(file, custom_file_path=nil, opts={}) - doc = Nokogiri.XML(file, nil, nil, + def self.parse(file, custom_file_path: nil, **opts) + custom_file_path ||= nil + xml = File.open(file) + doc = Nokogiri.XML(xml, nil, nil, Nokogiri::XML::ParseOptions::RECOVER + Nokogiri::XML::ParseOptions::NONET + Nokogiri::XML::ParseOptions::COMPACT # do not modify doc! ) + doc.search('product').each.with_index(1) do |row, i| # create a new article unit = row.search('eenheid').text - unit = case(unit) - when blank? then 'st' + unit = case(unit.strip) + when '' then 'st' when 'stuk' then 'st' when 'g' then 'gr' # need at least 2 chars when 'l' then 'ltr' else unit end + return if i==3 + puts unit, i inhoud = row.search('inhoud').text - inhoud.blank? or (inhoud.to_f-1).abs > 1e-3 and unit = inhoud.gsub(/\.0+\s*$/,'') + unit + inhoud.to_s.strip.empty? or (inhoud.to_f-1).abs > 1e-3 and unit = inhoud.gsub(/\.0+\s*$/,'') + unit deposit = row.search('statiegeld').text - deposit.blank? and deposit = 0 + deposit.to_s.strip.empty? and deposit = 0 category = [ @@codes[:indeling][row.search('indeling').text.to_i], @@codes[:indeling][row.search('subindeling').text.to_i] @@ -69,7 +74,6 @@ module FoodsoftArticleImport raise "Failed to load dnb_codes: #{dir}/dnb_codes.yml: #{e.message}" end end - end FoodsoftArticleImport::DnbXml.load_codes diff --git a/lib/foodsoft_article_import/foodsoft.rb b/lib/foodsoft_article_import/foodsoft.rb index 6916a16..ada6de1 100644 --- a/lib/foodsoft_article_import/foodsoft.rb +++ b/lib/foodsoft_article_import/foodsoft.rb @@ -16,8 +16,10 @@ module FoodsoftArticleImport::Foodsoft # Parses Foodsoft file # the yielded article is a simple hash - def self.parse(file, custom_file_path=nil, **opts) + def self.parse(file, custom_file_path: nil, **opts) + custom_file_path ||= nil opts = OPTIONS.merge(opts) + ss = FoodsoftArticleImport.open_spreadsheet(file, **opts) header_row = true diff --git a/lib/foodsoft_article_import/utf8_encoder.rb b/lib/foodsoft_article_import/utf8_encoder.rb new file mode 100644 index 0000000..0715a7e --- /dev/null +++ b/lib/foodsoft_article_import/utf8_encoder.rb @@ -0,0 +1,9 @@ +module UTF8Encoder + def self.clean(string) + if string.nil? + string + else + string.encode('UTF-8') + end + end +end From 4ed1764b753478d10af300c39ec0a97dc1b36960 Mon Sep 17 00:00:00 2001 From: viehlieb Date: Tue, 31 Jan 2023 12:02:46 +0100 Subject: [PATCH 2/3] add specs fr foodsoft, odin and bnn... bioromeo incomplete --- spec/files/bioromeo/bioromeo_flawless.csv | 12 +++ spec/files/bnn/bnn_bad_encoding.BNN | 3 + spec/files/bnn/bnn_flawless.BNN | 3 + spec/files/bnn/bnn_flawless_category.BNN | 3 + spec/files/bnn/bnn_flawless_special.BNN | 3 + spec/files/bnn/bnn_missing_entries.BNN | 3 + spec/files/bnn/bnn_missing_order_number.BNN | 3 + spec/files/bnn/fusion.rb | 66 ++++++++++++ spec/files/custom_codes.yml | 8 ++ spec/files/foodsoft/foodsoft_flawless.csv | 3 + .../foodsoft_generate_order_number.csv | 3 + .../foodsoft/foodsoft_missing_entries.csv | 2 + spec/files/odin/odin_flawless.xml | 75 +++++++++++++ .../odin/odin_flawless_custom_category.xml | 77 ++++++++++++++ spec/files/odin/odin_missing_entries.xml | 75 +++++++++++++ spec/files/odin/odin_missing_order_number.xml | 75 +++++++++++++ .../foodsoft_article_import_bioromeo_spec.rb | 24 +++++ .../bnn/foodsoft_article_import_bnn_spec.rb | 71 +++++++++++++ .../foodsoft_article_import_foodsoft_spec.rb | 57 ++++++++++ .../odin/foodsoft_article_import_odin_spec.rb | 67 ++++++++++++ spec/spec_helper.rb | 100 ++++++++++++++++++ 21 files changed, 733 insertions(+) create mode 100644 spec/files/bioromeo/bioromeo_flawless.csv create mode 100644 spec/files/bnn/bnn_bad_encoding.BNN create mode 100644 spec/files/bnn/bnn_flawless.BNN create mode 100644 spec/files/bnn/bnn_flawless_category.BNN create mode 100644 spec/files/bnn/bnn_flawless_special.BNN create mode 100644 spec/files/bnn/bnn_missing_entries.BNN create mode 100644 spec/files/bnn/bnn_missing_order_number.BNN create mode 100644 spec/files/bnn/fusion.rb create mode 100644 spec/files/custom_codes.yml create mode 100644 spec/files/foodsoft/foodsoft_flawless.csv create mode 100644 spec/files/foodsoft/foodsoft_generate_order_number.csv create mode 100644 spec/files/foodsoft/foodsoft_missing_entries.csv create mode 100644 spec/files/odin/odin_flawless.xml create mode 100644 spec/files/odin/odin_flawless_custom_category.xml create mode 100644 spec/files/odin/odin_missing_entries.xml create mode 100644 spec/files/odin/odin_missing_order_number.xml create mode 100644 spec/lib/bioromeo/foodsoft_article_import_bioromeo_spec.rb create mode 100644 spec/lib/bnn/foodsoft_article_import_bnn_spec.rb create mode 100644 spec/lib/foodsoft/foodsoft_article_import_foodsoft_spec.rb create mode 100644 spec/lib/odin/foodsoft_article_import_odin_spec.rb create mode 100644 spec/spec_helper.rb diff --git a/spec/files/bioromeo/bioromeo_flawless.csv b/spec/files/bioromeo/bioromeo_flawless.csv new file mode 100644 index 0000000..950d3dd --- /dev/null +++ b/spec/files/bioromeo/bioromeo_flawless.csv @@ -0,0 +1,12 @@ +,,,,,, +,Bestelling plaatsen via info@bioromeo.nl,,,,, +,Op maandag bestellen.,,,,, +,Uitleveren. ,,,,, +Zwijnsweg 5,Prijzen blablabla,,,,, +8307 PP Ens,"En nog meer wat €0,- kosten",,,,, +skal 012345,bla in overleg,,,,, +Vragen: bel iemand 01-23456789,,,,,, +,,,,,, +Artnr.,Product,Skal,Demeter, Prijs per eenh. , Prijs per colli ,Bestelling,opm. +,"Aardappels ""nieuwe oogst""",,,,,, +1,Wilde aardappels - 5kg,1234,123456, € 1.00 , € 5.00 ,,Kopervrij diff --git a/spec/files/bnn/bnn_bad_encoding.BNN b/spec/files/bnn/bnn_bad_encoding.BNN new file mode 100644 index 0000000..0b7cb14 --- /dev/null +++ b/spec/files/bnn/bnn_bad_encoding.BNN @@ -0,0 +1,3 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +64721;A;;;4280001958081;4280001958203;Greek Dressing - Kräuter Mix;Oregano, Basilikum und Minze;;;med;;GR;C%;DE-ÖKO-001;120;1302;10;55;;1;6 x35g;6;35g;1;N;930190;99260;;1,41;;;;1;;;4,49;2,89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +;;99 \ No newline at end of file diff --git a/spec/files/bnn/bnn_flawless.BNN b/spec/files/bnn/bnn_flawless.BNN new file mode 100644 index 0000000..1c5ac61 --- /dev/null +++ b/spec/files/bnn/bnn_flawless.BNN @@ -0,0 +1,3 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +64721;X;;;4280001958081;4280001958203;Greek Dressing - Kr„uter Mix;Oregano, Basilikum und Minze;;;med;;GR;C%;DE-™KO-001;120;1302;10;55;;1;6 x35g;6;35g;1;N;930190;99260;;1,41;;;;1;;;4,49;2,89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +;;99 \ No newline at end of file diff --git a/spec/files/bnn/bnn_flawless_category.BNN b/spec/files/bnn/bnn_flawless_category.BNN new file mode 100644 index 0000000..6f09dbf --- /dev/null +++ b/spec/files/bnn/bnn_flawless_category.BNN @@ -0,0 +1,3 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +64721;A;;;4280001958081;4280001958203;Greek Dressing - Kr„uter Mix;Oregano, Basilikum und Minze;;;med;;GR;C%;DE-™KO-001;120;4000;10;55;;1;6 x35g;6;35g;1;N;930190;99260;;1,41;;;;1;;;4,49;2,89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +;;99 \ No newline at end of file diff --git a/spec/files/bnn/bnn_flawless_special.BNN b/spec/files/bnn/bnn_flawless_special.BNN new file mode 100644 index 0000000..0f285f6 --- /dev/null +++ b/spec/files/bnn/bnn_flawless_special.BNN @@ -0,0 +1,3 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +64721;A;;;4280001958081;4280001958203;Greek Dressing - Kr„uter Mix;Oregano, Basilikum und Minze;;;med;;GR;C%;DE-™KO-001;120;1302;10;55;;1;6 x35g;6;35g;1;N;930190;99260;;1,41;;;;1;;;4,49;2,89;J;;2;3;;;;;;;;;;;;;;;;;;;A;;20230101;20230201;;Kg;28,571;; +;;99 diff --git a/spec/files/bnn/bnn_missing_entries.BNN b/spec/files/bnn/bnn_missing_entries.BNN new file mode 100644 index 0000000..6c8dafe --- /dev/null +++ b/spec/files/bnn/bnn_missing_entries.BNN @@ -0,0 +1,3 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +64721;A;;;4280001958081;4280001958203;Greek Dressing - Kr„uter Mix;Oregano, Basilikum und Minze;;;HDE;;GR;C%;DE-™KO-001;120;1100;10;55;;1;6 x35g;6;35g;1;N;;99260;;1,41;;;;1;;;4,49;2,89;J;;;;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +;;99 diff --git a/spec/files/bnn/bnn_missing_order_number.BNN b/spec/files/bnn/bnn_missing_order_number.BNN new file mode 100644 index 0000000..aadcb9b --- /dev/null +++ b/spec/files/bnn/bnn_missing_order_number.BNN @@ -0,0 +1,3 @@ +BNN;3;0;Naturkost Nord, Hamburg;T;Angebot Nr. 0922;EUR;20220905;20221001;20220825;837;1 +;A;;;4280001958081;4280001958203;Greek Dressing - Kr„uter Mix;Oregano, Basilikum und Minze;;;HDE;;GR;C%;DE-™KO-001;120;1100;10;55;;1;6 x35g;6;35g;1;N;;99260;;1,41;;;;1;;;4,49;2,89;J;;;;;;;;;;;;;;;;;;;;;;A;;;;;Kg;28,571;; +;;99 diff --git a/spec/files/bnn/fusion.rb b/spec/files/bnn/fusion.rb new file mode 100644 index 0000000..2f0d579 --- /dev/null +++ b/spec/files/bnn/fusion.rb @@ -0,0 +1,66 @@ +event_ids = [25, 6, 10, 9, 18, 12, 20, 21, 23, 30, 11, 29, 28, 19] +# events = Event.where("starting_at >= ?", Date.parse("2016-01-01")) +events = Event.where(id: event_ids).chronological +puts events.pluck(:name) + +# Fusion 2016 +# at.tension#7 +# Fusion 2018 +# Fusion 2019 +# at.tension#8 +# Fusion 2020 +# Fusion 2021 RED +# Fusion 2021 BLACK +# Plan:et C - alpha +# Plan:et C - beta +# be.tween #1 +# Plan:et C - gamma +# Fusion 2022 +# at.tension#9 + +bookings = Booking.where(event: events) +ap bookings.count + +sums = {} +sums[:quellensteuer] = {} +sums[:doitsch] = {} +sums[:quellensteuer][:guest_tickets] = 0 +sums[:quellensteuer][:artist_tickets] = 0 +sums[:doitsch][:guest_tickets] = 0 +sums[:doitsch][:artist_tickets] = 0 + +events.each do |event| + bookings = Booking.where(event: event) + sums = {} + sums[:quellensteuer] = {} + sums[:doitsch] = {} + sums[:quellensteuer][:guest_tickets] = 0 + sums[:quellensteuer][:artist_tickets] = 0 + sums[:doitsch][:guest_tickets] = 0 + sums[:doitsch][:artist_tickets] = 0 + bookings.each do |booking| + ti = booking.task_instance("Quellensteuer") + #ap "=============================================" + #ap "Booking #{booking.id} - #{booking.artist.name}" + has_quellensteuer = (ti.present? && (ti.is_state?("teilweise gemeldet") || ti.is_state?("komplett gemeldet"))) + cat = has_quellensteuer ? :quellensteuer : :doitsch + booking.admissions.each do |adm| + # ap "Admission ##{adm.id}" + if adm.canceled? + # ap "cancelled" + next + end + # Nur die eingecheckten (Gäste) + # Alle angelegten, nicht stornierten (Artists) + if adm.guest_admission? + if adm.checked_in? # Guest & Checked in + sums[cat][:guest_tickets] += 1 + end + else # Artist + sums[cat][:artist_tickets] += 1 + end + end + end + ap event.name + ap sums +end diff --git a/spec/files/custom_codes.yml b/spec/files/custom_codes.yml new file mode 100644 index 0000000..5e9020f --- /dev/null +++ b/spec/files/custom_codes.yml @@ -0,0 +1,8 @@ +# BNN Codes +category: + "4000": "Schuhe" +additional: + "additional": "value" +indeling: + 11: Test Indeling + 111: Test Subindeling \ No newline at end of file diff --git a/spec/files/foodsoft/foodsoft_flawless.csv b/spec/files/foodsoft/foodsoft_flawless.csv new file mode 100644 index 0000000..a9a94c2 --- /dev/null +++ b/spec/files/foodsoft/foodsoft_flawless.csv @@ -0,0 +1,3 @@ +status;number;name;note;manufacturer;origin;unit ;clear price;tax;deposit;unit quantity;scale quantity;scale price;category +;1;product;bio;someone;eu;1 kg;1.23;6;0;10;;;coolstuff +;12;other product;bio;someone;eu;2 kg;3.45;6;0;10;;;coolstuff \ No newline at end of file diff --git a/spec/files/foodsoft/foodsoft_generate_order_number.csv b/spec/files/foodsoft/foodsoft_generate_order_number.csv new file mode 100644 index 0000000..a50dde3 --- /dev/null +++ b/spec/files/foodsoft/foodsoft_generate_order_number.csv @@ -0,0 +1,3 @@ +status;number;name;note;manufacturer;origin;unit ;clear price;tax;deposit;unit quantity;scale quantity;scale price;category +;;product;bio;someone;eu;1 kg;1.23;6;0;10;;;coolstuff +;;other product;bio;someone;eu;2 kg;3.45;6;0;10;;;coolstuff \ No newline at end of file diff --git a/spec/files/foodsoft/foodsoft_missing_entries.csv b/spec/files/foodsoft/foodsoft_missing_entries.csv new file mode 100644 index 0000000..560c11a --- /dev/null +++ b/spec/files/foodsoft/foodsoft_missing_entries.csv @@ -0,0 +1,2 @@ +status;number;name;note;manufacturer;origin;unit ;clear price;tax;deposit;unit quantity;scale quantity;scale price;category +;12;product;bio;;eu;1 kg;1.23;;0;10;;;coolstuff \ No newline at end of file diff --git a/spec/files/odin/odin_flawless.xml b/spec/files/odin/odin_flawless.xml new file mode 100644 index 0000000..5b5a28f --- /dev/null +++ b/spec/files/odin/odin_flawless.xml @@ -0,0 +1,75 @@ + + + +1039 +1.08 +Estafette Associatie C.V. +Geldermalsen + + +8719325207668 +nucli rose +Nucli rose + +0 +0 +0 +750 +g +Stuk +0 +NELEMAN +Biologisch + + +ES + +21 +1017515 +0109 +6 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +4.52 +7.95 + + + \ No newline at end of file diff --git a/spec/files/odin/odin_flawless_custom_category.xml b/spec/files/odin/odin_flawless_custom_category.xml new file mode 100644 index 0000000..460da24 --- /dev/null +++ b/spec/files/odin/odin_flawless_custom_category.xml @@ -0,0 +1,77 @@ + + + +1039 +1.08 +Estafette Associatie C.V. +Geldermalsen + + +8719325207668 +nucli rose +Nucli rose + +0 +0 +0 +750 +g +Stuk +0 +NELEMAN +Biologisch + + +ES + +21 +1017515 +0109 +11 +111 +6 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +4.52 +7.95 + + + \ No newline at end of file diff --git a/spec/files/odin/odin_missing_entries.xml b/spec/files/odin/odin_missing_entries.xml new file mode 100644 index 0000000..5089b91 --- /dev/null +++ b/spec/files/odin/odin_missing_entries.xml @@ -0,0 +1,75 @@ + + + +1039 +1.08 +Estafette Associatie C.V. +Geldermalsen + + +8719325207668 +nucli rose +Nucli rose + +0 +0 +0 +750 + +Stuk +0 + +Biologisch + + +ES + +21 +1017515 +0109 +6 +Non Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +4.52 +7.95 + + + \ No newline at end of file diff --git a/spec/files/odin/odin_missing_order_number.xml b/spec/files/odin/odin_missing_order_number.xml new file mode 100644 index 0000000..d43a943 --- /dev/null +++ b/spec/files/odin/odin_missing_order_number.xml @@ -0,0 +1,75 @@ + + + +1039 +1.08 +Estafette Associatie C.V. +Geldermalsen + + +8719325207668 +nucli rose +Nucli rose + +0 +0 +0 +750 +g +Stuk +0 +NELEMAN +Biologisch + + +ES + +21 +1017515 + +6 +Actief +druiven* +0 +0 +2 +2 +0 +0 +0 +2 +2 +0 +2 +0 +2 +0 +2 +2 +2 +2 +1 +0 +2 +0 +2 +2 + + + +0 +0 +0 +0 +1 + +2 +0 + +adviesprijs +2022-08-18 +4.52 +7.95 + + + \ No newline at end of file diff --git a/spec/lib/bioromeo/foodsoft_article_import_bioromeo_spec.rb b/spec/lib/bioromeo/foodsoft_article_import_bioromeo_spec.rb new file mode 100644 index 0000000..63bd522 --- /dev/null +++ b/spec/lib/bioromeo/foodsoft_article_import_bioromeo_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' +require_relative '../../../lib/foodsoft_article_import' + +describe FoodsoftArticleImport do + + files_path = File.expand_path '../../files', __dir__ + bioromeo_files_path = File.join(files_path, 'bioromeo') + + dummy_article = {:order_number=>"1", :name => "Wilde aardappels",:article_category => "Aardappels \"nieuwe oogst\"", :deposit => 0, :manufacturer => nil, :origin => "Noordoostpolder, NL", :price => 5.0, :tax => 6, :unit => "5kg", :unit_quantity => 1, :note => "Skal 1234; 123456; Demeter 123456; (Kopervrij)"} + + + empty = {} + + context "bioromeo" do + it 'parses file correctly with type parameter bioromeo' do + FoodsoftArticleImport.parse(File.open(File.join(bioromeo_files_path, 'bioromeo_flawless.csv')), type: 'bioromeo') do |new_attrs, status, line| + if new_attrs==nil + next + end + expect(new_attrs).to eq dummy_article + end + end + end +end diff --git a/spec/lib/bnn/foodsoft_article_import_bnn_spec.rb b/spec/lib/bnn/foodsoft_article_import_bnn_spec.rb new file mode 100644 index 0000000..54e666d --- /dev/null +++ b/spec/lib/bnn/foodsoft_article_import_bnn_spec.rb @@ -0,0 +1,71 @@ +require 'spec_helper' +require_relative '../../../lib/foodsoft_article_import' + +describe FoodsoftArticleImport do + + files_path = File.expand_path '../../files', __dir__ + bnn_files_path = File.join(files_path, 'bnn') + + dummy_article = { name: 'Greek Dressing - Kräuter Mix', order_number: '64721', note: 'Oregano, Basilikum und Minze', + manufacturer: 'Medousa, Griechenland Importe', origin: 'GR', article_category: 'Kräutermischungen', unit: '35g', price: '2,89', tax: 7.0, unit_quantity: '6'} + + article = dummy_article.merge({scale_price: "3", scale_quantity: "2", deposit: 0.08}) + article_special = article.merge(note: 'Sonderpreis: 2,89 von 20230101 bis 20230201') + + article_2 = dummy_article.merge({manufacturer: nil, article_category: nil}) + + article_custom_code = article.merge(article_category: "Schuhe") + + empty = {} + + context 'bnn' do + it 'parses bnn file correctly without type parameter' do + FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_flawless.BNN'))) do |new_attrs, status, line| + expect(new_attrs).to eq article + expect(status).to eq :outlisted + end + end + it 'parses file correctly with type parameter' do + FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_flawless.BNN')), type: 'bnn') do |new_attrs, status, line| + expect(new_attrs).to eq article + expect(status).to eq :outlisted + end + end + it 'raises error wenn wrong type (except dnb_xml) specified' do + expect{FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_flawless.BNN')),type: 'foodsoft')}.to raise_error(RuntimeError) + expect{FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_flawless.BNN')),type: 'bioromeo')}.to raise_error(RuntimeError) + + expect(FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_flawless.BNN')),type: 'dnb_xml')).to eq [] + end + it 'parses article with special correctly' do + FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_flawless_special.BNN')), type: 'bnn') do |new_attrs, status, line| + expect(new_attrs).to eq article_special + expect(status).to eq :special + end + end + it 'parses missing entries correctly' do + FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_missing_entries.BNN')), type: 'bnn') do |new_attrs, status, line| + expect(new_attrs).to eq article_2 + expect(status).to eq nil + end + end + it 'skips rows without order_number' do + FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_missing_order_number.BNN')), type: 'bnn') do |new_attrs, status, line| + expect(new_attrs).to eq empty + end + end + it 'joins custom_codes file' do + custom_file_path = File.join(files_path, 'custom_codes.yml').to_s + FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_flawless_category.BNN')), custom_file_path: custom_file_path, type: 'bnn') do |new_attrs, status, line| + expect(new_attrs).to eq article_custom_code + end + end + it 'parses file with different encoding' do + #the bnn file is loaded with encoding ibm850. If file is not ibm850 encoded, some characters might look weird + FoodsoftArticleImport.parse(File.open(File.join(bnn_files_path, 'bnn_bad_encoding.BNN')), type: 'bnn') do |new_attrs, status, line| + expect(new_attrs[:order_number]).to eq("64721") + expect(new_attrs[:name]).to eq("Greek Dressing - Kräuter Mix") + end + end + end +end diff --git a/spec/lib/foodsoft/foodsoft_article_import_foodsoft_spec.rb b/spec/lib/foodsoft/foodsoft_article_import_foodsoft_spec.rb new file mode 100644 index 0000000..24d1c27 --- /dev/null +++ b/spec/lib/foodsoft/foodsoft_article_import_foodsoft_spec.rb @@ -0,0 +1,57 @@ +require 'spec_helper' +require_relative '../../../lib/foodsoft_article_import' + +describe FoodsoftArticleImport do + + files_path = File.expand_path '../../files', __dir__ + foodsoft_files_path = File.join(files_path, 'foodsoft') + + dummy_article = {:order_number=>"1", :name=>"product", :note=>"bio", :manufacturer=>"someone", :origin=>"eu", :unit=>"1 kg", :price=>"1.23", :tax=>"6", :unit_quantity=>"10", :scale_quantity=>nil, :scale_price=>nil, :article_category=>"coolstuff", :deposit=>"0"} + + + dummy_article_2 = {:order_number=>"12", :name=>"other product", :note=>"bio", :manufacturer=>"someone", :origin=>"eu", :unit=>"2 kg", :price=>"3.45", :tax=>"6", :unit_quantity=>"10", :scale_quantity=>nil, :scale_price=>nil, :article_category=>"coolstuff", :deposit=>"0"} + + articles=[dummy_article, dummy_article_2] + + dummy_article_3 = dummy_article.merge({ order_number: ":d8df298"}) + dummy_article_4 = dummy_article_2.merge({ order_number: ":1f37e39"}) + articles_number_generated= [dummy_article_3, dummy_article_4] + empty = {} + + context "foodsoft" do + it 'parses file correctly with type parameter foodsoft' do + count =0 + FoodsoftArticleImport.parse(File.open(File.join(foodsoft_files_path, 'foodsoft_flawless.csv')), type: 'foodsoft') do |new_attrs, status, line| + expect(new_attrs).to eq articles[count] + expect(status).to eq nil + count+=1 + end + end + it 'raises error wenn wrong type specified' do + expect{FoodsoftArticleImport.parse(File.open(File.join(foodsoft_files_path, 'foodsoft_flawless.csv')),type: 'bioromeo')}.to raise_error(Roo::HeaderRowNotFoundError) + expect(FoodsoftArticleImport.parse(File.open(File.join(foodsoft_files_path, 'foodsoft_flawless.csv')),type: 'odin')).to eq [] + + expect(FoodsoftArticleImport.parse(File.open(File.join(foodsoft_files_path, 'foodsoft_flawless.csv')),type: 'bnn')).to eq [] + end + it 'parses missing entries correctly' do + FoodsoftArticleImport.parse(File.open(File.join(foodsoft_files_path, 'foodsoft_missing_entries.csv')), type: 'foodsoft') do |new_attrs, status, line| + expect(status).to eq 'Error: unit, price and tax must be entered' + expect(new_attrs[:unit]).to eq "1 kg" + expect(new_attrs[:manufacturer]).to eq nil + end + end + it 'generates order numbers for articles without order number' do + count=0 + FoodsoftArticleImport.parse(File.open(File.join(foodsoft_files_path, 'foodsoft_generate_order_number.csv')), type: 'foodsoft') do |new_attrs, status, line| + expect(new_attrs).to eq articles_number_generated[count] + count+=1 + end + end + xit 'joins custom_codes file' do + custom_file_path = File.join(files_path, 'custom_codes.yml').to_s + FoodsoftArticleImport.parse(File.open(File.join(foodsoft_files_path, 'foodsoft_flawless_custom_category.csv')), custom_file_path: custom_file_path, type: 'foodsoft') do |new_attrs, status, line| + expect(new_attrs[:article_category]).to eq "Test Indeling - Test Subindeling" + end + end + end +end diff --git a/spec/lib/odin/foodsoft_article_import_odin_spec.rb b/spec/lib/odin/foodsoft_article_import_odin_spec.rb new file mode 100644 index 0000000..bdac2ae --- /dev/null +++ b/spec/lib/odin/foodsoft_article_import_odin_spec.rb @@ -0,0 +1,67 @@ +require 'spec_helper' +require_relative '../../../lib/foodsoft_article_import' + +describe FoodsoftArticleImport do + + files_path = File.expand_path '../../files', __dir__ + odin_files_path = File.join(files_path, 'odin') + + dummy_article = {:order_number=>"0109", :name=>"nucli rose", :note=>"Biologisch", :manufacturer=>"NELEMAN", :origin=>"ES", :unit=>"750gr", :price=>"4.52", :unit_quantity=>"6", :tax=>"21", :deposit=>"0", :article_category=>""} + + + empty = {} + + context "odin/dnb_xml" do + it 'parses file correctly with type parameter dnb_xml' do + FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'odin_flawless.xml')), type: 'dnb_xml') do |new_attrs, status, line| + expect(new_attrs).to eq dummy_article + expect(status).to eq nil + end + end + it 'parses file correctly with type parameter odin' do + FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'odin_flawless.xml')), type: 'odin') do |new_attrs, status, line| + expect(new_attrs).to eq dummy_article + expect(status).to eq nil + end + end + it 'raises error wenn wrong type specified' do + expect{FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'odin_flawless.xml')),type: 'foodsoft')}.to raise_error(RuntimeError) + expect{FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'odin_flawless.xml')),type: 'bioromeo')}.to raise_error(RuntimeError) + + expect{FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'odin_flawless.xml')),type: 'bnn')}.to raise_error(CSV::MalformedCSVError) + end + it 'parses missing entries correctly' do + FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'odin_missing_entries.xml')), type: 'odin') do |new_attrs, status, line| + expect(status).to eq :outlisted + expect(new_attrs[:unit]).to eq "750st" + expect(new_attrs[:manufacturer]).to eq "" + end + end + it 'skips rows without order_number' do + FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'odin_missing_order_number.xml')), type: 'odin') do |new_attrs, status, line| + expect(new_attrs).to eq empty + end + end + it 'joins custom_codes file' do + custom_file_path = File.join(files_path, 'custom_codes.yml').to_s + FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'odin_flawless_custom_category.xml')), custom_file_path: custom_file_path, type: 'odin') do |new_attrs, status, line| + expect(new_attrs[:article_category]).to eq "Test Indeling - Test Subindeling" + end + end + + xit 'parses dummy_article with special correctly' do + #TODO: find out whether there are special prices for odin files + FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'bnn_flawless_special.BNN')), type: 'bnn') do |new_attrs, status, line| + expect(new_attrs.manufacturer).to eq nil + expect(new_attrs.unit).to eq "750st" + end + end + xit 'parses file with different encoding' do + #the bnn file is loaded with encoding ibm850. If file is not ibm850 encoded, some characters might look weird + FoodsoftArticleImport.parse(File.open(File.join(odin_files_path, 'bnn_bad_encoding.BNN')), type: 'bnn') do |new_attrs, status, line| + expect(new_attrs[:order_number]).to eq("64721") + expect(new_attrs[:name]).to eq("Greek Dressing - Kräuter Mix") + end + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..03990c4 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,100 @@ +require 'simplecov' +SimpleCov.start +# This file was generated by the `rspec --init` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + +# The settings below are suggested to provide a good initial experience +# with RSpec, but feel free to customize to your heart's content. +=begin + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # https://relishapp.com/rspec/rspec-core/docs/configuration/zero-monkey-patching-mode + config.disable_monkey_patching! + + # This setting enables warnings. It's recommended, but in some cases may + # be too noisy due to issues in dependencies. + config.warnings = true + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 10 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +=end +end From 77474c0811911d045a9665567032d6ed18b851cf Mon Sep 17 00:00:00 2001 From: viehlieb Date: Tue, 31 Jan 2023 12:08:01 +0100 Subject: [PATCH 3/3] update gemspec, finetune parser classes --- Gemfile.lock | 8 ++ LICENSE | 6 ++ README.md | 4 + foodsoft_article_import.gemspec | 1 + lib/foodsoft_article_import.rb | 12 ++- lib/foodsoft_article_import/bioromeo.rb | 35 ++++---- lib/foodsoft_article_import/borkenstein.rb | 97 ---------------------- lib/foodsoft_article_import/dnb_xml.rb | 49 ++++++----- lib/foodsoft_article_import/foodsoft.rb | 6 +- 9 files changed, 74 insertions(+), 144 deletions(-) delete mode 100644 lib/foodsoft_article_import/borkenstein.rb diff --git a/Gemfile.lock b/Gemfile.lock index f9d61d9..4c62d8f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,10 +3,12 @@ PATH specs: foodsoft_article_import (1.0.0) roo (~> 2.9.0) + simplecov GEM remote: http://rubygems.org/ specs: + docile (1.4.0) nokogiri (1.14.0-x86_64-linux) racc (~> 1.4) racc (1.6.2) @@ -14,6 +16,12 @@ GEM nokogiri (~> 1) rubyzip (>= 1.3.0, < 3.0.0) rubyzip (2.3.2) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) PLATFORMS x86_64-linux diff --git a/LICENSE b/LICENSE index 591ff22..e7023b1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,11 @@ Copyright (C) 2022 Viehlieb + +Comment: Most of the source code was originally written for https://github.com/foodcoops/sharedlists under the GNU License and therefore special credit is attributed to the contributors of the sharedlists application: + +Authors: Kidhab: https://github.com/kidhab, +Benjamin Meichsner: https://github.com/benni-as, Wvengen: https://github.com/wvengen, Robwa: https://github.com/robwa, 1resu: https://github.com/1resu, JuliusR: https://github.com/JuliusR + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or diff --git a/README.md b/README.md index 398cabf..2e68041 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # FoodsoftArticleImport This gem provides FoodsoftArticleImport integration for Ruby on Rails and allows to parse avariety of files containing article information. These article information are standardized or customly declared. Possible File Ending are: .bnn, .BNN, .csv, .CSV . It relies on [roo](https://github.com/roo-rb/roo) to read and parse the data + ## Getting started TODO: add bnn codes, explain how to add bnn codes TODO: review GNU License + +For correct import of bnn files, please ensure the correct language setting. Your foodsoft account has to be settings set to german if you want to import articles from file. +To set this is imcrucial for guaranteeing the correct format of article prices. ### Requirements This gem requires Ruby 2.7 \ No newline at end of file diff --git a/foodsoft_article_import.gemspec b/foodsoft_article_import.gemspec index 6e57555..e8e54cb 100644 --- a/foodsoft_article_import.gemspec +++ b/foodsoft_article_import.gemspec @@ -25,4 +25,5 @@ Gem::Specification.new do |spec| spec.extra_rdoc_files = ['README.md'] spec.add_dependency 'roo', '~> 2.9.0' + spec.add_dependency 'simplecov' end diff --git a/lib/foodsoft_article_import.rb b/lib/foodsoft_article_import.rb index 4ea4d55..c08cf2e 100644 --- a/lib/foodsoft_article_import.rb +++ b/lib/foodsoft_article_import.rb @@ -6,7 +6,6 @@ require 'active_support/core_ext/hash/keys' require_relative 'foodsoft_article_import/bioromeo' require_relative 'foodsoft_article_import/bnn' require_relative 'foodsoft_article_import/utf8_encoder' -require_relative 'foodsoft_article_import/borkenstein' require_relative 'foodsoft_article_import/dnb_xml' require_relative 'foodsoft_article_import/foodsoft' module FoodsoftArticleImport @@ -23,9 +22,9 @@ module FoodsoftArticleImport def self.file_formats @@file_formats ||= { 'bnn' => FoodsoftArticleImport::Bnn, - 'borkenstein' => FoodsoftArticleImport::Borkenstein, 'foodsoft' => FoodsoftArticleImport::Foodsoft, 'dnb_xml' => FoodsoftArticleImport::DnbXml, + 'odin' => FoodsoftArticleImport::DnbXml, 'bioromeo' => FoodsoftArticleImport::Bioromeo, }.freeze end @@ -36,17 +35,15 @@ module FoodsoftArticleImport # @option opts [String] type file format (required) (see {.file_formats}) # @return [File, Roo::Spreadsheet] file with encoding set if needed def self.parse(file, custom_file_path: nil, type: nil, **opts, &blk) - # @todo handle wrong or undetected type custom_file_path ||= nil type ||= 'bnn' parser = file_formats[type] - puts parser if block_given? - parser.parse(file, custom_file_path, **opts, &blk) + parser.parse(file, custom_file_path: custom_file_path, **opts, &blk) else data = [] - parser.parse(file, custom_file_path, **opts) { |a| data << a } + parser.parse(file, custom_file_path: custom_file_path, **opts) { |a| data << a } data end end @@ -69,10 +66,11 @@ module FoodsoftArticleImport # @param encoding [String, NilClass] optional CSV encoding # @param col_sep [String, NilClass] optional column separator # @return [Roo::Spreadsheet] - def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil) + def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil, liberal_parsing: nil) opts = {csv_options: {}} opts[:csv_options][:encoding] = encoding if encoding opts[:csv_options][:col_sep] = col_sep if col_sep + opts[:csv_options][:liberal_parsing] = true if liberal_parsing opts[:extension] = File.extname(filename) if filename begin Roo::Spreadsheet.open(file, **opts) diff --git a/lib/foodsoft_article_import/bioromeo.rb b/lib/foodsoft_article_import/bioromeo.rb index a7f73ac..be076b4 100644 --- a/lib/foodsoft_article_import/bioromeo.rb +++ b/lib/foodsoft_article_import/bioromeo.rb @@ -30,42 +30,42 @@ module FoodsoftArticleImport def self.parse(file, custom_file_path: nil, **opts) custom_file_path ||= nil opts = OPTIONS.merge(opts) + opts[:liberal_parsing]=true + opts[:col_sep]="," ss = FoodsoftArticleImport.open_spreadsheet(file, **opts) - header_row = true sheet = ss.sheet(0).parse(clean: true, - number: /^artnr/i, - name: /^product/i, - skal: /^skal$/i, - demeter: /^demeter$/i, + order_number: /Artnr./, + name: /Product/, + skal: /Skal$/, + demeter: /Demeter$/, unit_price: /prijs\b.*\beenh/i, pack_price: /prijs\b.*\bcolli/i, - comment: /^opm(erking)?/i, + comment: /opm(erking)?/i, ) linenum = 0 category = nil sheet.each do |row| - puts("[ROW] #{row.inspect}") linenum += 1 - row[:name].blank? and next + row[:name].to_s.strip.empty? and next # (sub)categories are in first two content cells - assume if there's a price it's a product - if row[:order_number].blank? && row[:unit_price].blank? + if row[:order_number].to_s.strip.empty? && row[:unit_price].to_s.strip.empty? category = row[:name] yield nil, nil, linenum next end # skip products without a number - if row[:order_number].blank? + if row[:order_number].to_s.strip.empty? yield nil, nil, linenum next end # extract name and unit errors = [] notes = [] - unit_price = row[:unit_price] - pack_price = row[:pack_price] + unit_price = row[:unit_price].gsub("€","").to_s.strip.to_f + pack_price = row[:pack_price].gsub("€","").to_s.strip.to_f number = row[:order_number] name = row[:name] unit = nil @@ -75,6 +75,7 @@ module FoodsoftArticleImport m=name.match(re) unless m yield nil, nil, linenum + next end unit = self.normalize_unit(m[3]) name = name.sub(re, '').sub(/\(\s*\)\s*$/,'').sub(/\s+/, ' ').sub(/\.\s*$/, '').strip @@ -120,10 +121,10 @@ module FoodsoftArticleImport end end # note from various fields - notes.append("Skal #{row[:skal]}") if row[:skal].present? - notes.append(row[:demeter]) if row[:demeter].present? && row[:demeter].is_a?(String) - notes.append("Demeter #{row[:demeter]}") if row[:demeter].present? && row[:demeter].is_a?(Fixnum) - notes.append "(#{row[:comment]})" unless row[:comment].blank? + notes.append("Skal #{row[:skal]}") unless row[:skal].to_s.strip.empty? + notes.append(row[:demeter]) unless row[:skal].to_s.strip.empty? + notes.append("Demeter #{row[:demeter]}") unless row[:skal].to_s.strip.empty? && row[:demeter].is_a?(Fixnum) + notes.append "(#{row[:comment]})" unless row[:comment].to_s.strip.empty? name.sub!(/(,\.?\s*)?\bDemeter\b/i, '') and notes.prepend("Demeter") name.sub!(/(,\.?\s*)?\bBIO\b/i, '') and notes.prepend "BIO" # unit check @@ -172,7 +173,7 @@ module FoodsoftArticleImport elsif what =~ /^gr/ pack_price.to_f / amount.to_f * 1000 end - if kgprice.present? && (kgprice - unit_price.to_f).abs < 1e-2 + unless kgprice.to_s.strip.empty? && (kgprice - unit_price.to_f).abs < 1e-2 return end diff --git a/lib/foodsoft_article_import/borkenstein.rb b/lib/foodsoft_article_import/borkenstein.rb deleted file mode 100644 index 820aee3..0000000 --- a/lib/foodsoft_article_import/borkenstein.rb +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- -# Module for Borkenstein csv import - -require 'csv' - -module FoodsoftArticleImport - class Borkenstein - - REGEX = { - :main => /^(.+)\s+\[([^\[\]]+)\]\s+(\d+\.\d+)\((\d+\.\d+)\)$/, - :manufacturer => /^(.+)\s{4}\[\]\s{4}\(\)$/, - :origin => /(.+)\s+(\w+)\/\w+[\/[\w\-]+]?/ - }.freeze - - NAME = "Borkenstein (CSV)" - OUTLIST = false - OPTIONS = { - col_sep: ",", - encoding: "UTF-8" # @todo check this - }.freeze - - def self.parse(file, custom_file_path: nil, **opts) - custom_file_path ||= nil - global_manufacturer = nil - - file.set_encoding(opts[:encoding] || OPTIONS[:encoding]) - col_sep = opts[:col_sep] || OPTIONS[:col_sep] - CSV.new(file, {col_sep: col_sep, :headers => false}).each.with_index(1) do |row, i| - - # Set manufacturer - if row[1] == "-" - match = row[2].match(REGEX[:manufacturer]) - global_manufacturer = match.captures.first unless match.nil? - end - - # check if the line is empty - unless row[1].blank? || row[1] == "-" - - # Split string and remove beginning " - matched = row[2].gsub(/^\"/, "").gsub(/\"$/, "").match(REGEX[:main]) - - if matched.nil? - puts "No regular article data for #{row[1]}: #{row[2]}" - yield nil, nil, nil - else - name, units, price_high, price_low = matched.captures - - # Try to get origin - matched_name = name.match(REGEX[:origin]) - if matched_name - name, origin = matched_name.captures - else - name, origin = name.gsub(/\s{2,}/, ""), nil - end - - # Manufacturer - if name.match(/^[A-Za-z]{2,3}\s{1}/) - name.gsub!(/^[A-Za-z]{2,3}\s{1}/, "") - manufacturer = global_manufacturer - end - - - # Get unit quantities - units = units.split("x") - if units.size == 2 - unit_quantity = units.first - unit = units.last - else - unit_quantity = 1 - unit = units.first - end - - article = { - :order_number => row[1], - :name => name, - :origin => origin, - :manufacturer => manufacturer, - :unit_quantity => unit_quantity, - :unit => unit, - :price => price_low, # Inklusive Rabattstufe von 10% - :tax => 0.0 # Tax is included - } - - # test, if neccecary attributes exists - if article[:unit].nil? || article[:price].nil? || article[:unit_quantity].nil? - raise "Fehler: Einheit, Preis und MwSt. müssen gegeben sein: #{article.inspect}" - end - - yield article, nil, i - end - end - yield nil, nil, i - end - end - - end -end \ No newline at end of file diff --git a/lib/foodsoft_article_import/dnb_xml.rb b/lib/foodsoft_article_import/dnb_xml.rb index 377881f..c5e2f11 100644 --- a/lib/foodsoft_article_import/dnb_xml.rb +++ b/lib/foodsoft_article_import/dnb_xml.rb @@ -22,7 +22,7 @@ module FoodsoftArticleImport Nokogiri::XML::ParseOptions::NONET + Nokogiri::XML::ParseOptions::COMPACT # do not modify doc! ) - + self.load_codes(custom_file_path) doc.search('product').each.with_index(1) do |row, i| # create a new article unit = row.search('eenheid').text @@ -33,8 +33,6 @@ module FoodsoftArticleImport when 'l' then 'ltr' else unit end - return if i==3 - puts unit, i inhoud = row.search('inhoud').text inhoud.to_s.strip.empty? or (inhoud.to_f-1).abs > 1e-3 and unit = inhoud.gsub(/\.0+\s*$/,'') + unit deposit = row.search('statiegeld').text @@ -44,20 +42,23 @@ module FoodsoftArticleImport @@codes[:indeling][row.search('subindeling').text.to_i] ].compact.join(' - ') - article = {:order_number => row.search('bestelnummer').text, - #:ean => row.search('eancode').text, - :name => row.search('omschrijving').text, - :note => row.search('kwaliteit').text, - :manufacturer => row.search('merk').text, - :origin => row.search('herkomst').text, - :unit => unit, - :price => row.search('prijs inkoopprijs').text, - :unit_quantity => row.search('sve').text, - :tax => row.search('btw').text, - :deposit => deposit, - :article_category => category} - - yield article, (row.search('status') == 'Actief' ? :outlisted : nil), i + status = row.search('status').text == "Actief" ? nil : :outlisted + article = {} + unless row.search('bestelnummer').text == "" + article = {:order_number => row.search('bestelnummer').text, + #:ean => row.search('eancode').text, + :name => row.search('omschrijving').text, + :note => row.search('kwaliteit').text, + :manufacturer => row.search('merk').text, + :origin => row.search('herkomst').text, + :unit => unit, + :price => row.search('prijs inkoopprijs').text, + :unit_quantity => row.search('sve').text, + :tax => row.search('btw').text, + :deposit => deposit, + :article_category => category} + end + yield article, status, i end end @@ -65,16 +66,24 @@ module FoodsoftArticleImport @@codes = Hash.new - def self.load_codes + def self.load_codes(custom_file_path=nil) @gem_lib = File.expand_path "../../", __FILE__ dir = File.join @gem_lib, 'foodsoft_article_import' begin @@codes = YAML::load(File.open(File.join(dir, "dnb_codes.yml"))).symbolize_keys + if(custom_file_path) + custom_codes = YAML::load(File.open(custom_file_path)).symbolize_keys + custom_codes.keys.each do |key| + if @@codes.keys.include?(key) + custom_codes[key] =custom_codes[key].merge @@codes[key] + end + @@codes = @@codes.merge custom_codes + end + end + @@codes rescue => e raise "Failed to load dnb_codes: #{dir}/dnb_codes.yml: #{e.message}" end end end - - FoodsoftArticleImport::DnbXml.load_codes end \ No newline at end of file diff --git a/lib/foodsoft_article_import/foodsoft.rb b/lib/foodsoft_article_import/foodsoft.rb index ada6de1..5ced872 100644 --- a/lib/foodsoft_article_import/foodsoft.rb +++ b/lib/foodsoft_article_import/foodsoft.rb @@ -27,11 +27,11 @@ module FoodsoftArticleImport::Foodsoft # skip first header row if header_row header_row = false - yield nil, nil, i next end # skip empty lines - if row[2].blank? + if row[2].to_s.strip.empty? + # raise no order number given yield nil, nil, i next end @@ -49,7 +49,7 @@ module FoodsoftArticleImport::Foodsoft :scale_price => row[12], :article_category => row[13]} article.merge!(:deposit => row[9]) unless row[9].nil? - article[:order_number].blank? and ArticleImport.generate_number(article) + FoodsoftArticleImport.generate_number(article) if article[:order_number].to_s.strip.empty? if row[6].nil? || row[7].nil? or row[8].nil? yield article, "Error: unit, price and tax must be entered", i else