diff --git a/Gemfile.lock b/Gemfile.lock index f9d61d9..4c62d8f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,10 +3,12 @@ PATH specs: foodsoft_article_import (1.0.0) roo (~> 2.9.0) + simplecov GEM remote: http://rubygems.org/ specs: + docile (1.4.0) nokogiri (1.14.0-x86_64-linux) racc (~> 1.4) racc (1.6.2) @@ -14,6 +16,12 @@ GEM nokogiri (~> 1) rubyzip (>= 1.3.0, < 3.0.0) rubyzip (2.3.2) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) PLATFORMS x86_64-linux diff --git a/LICENSE b/LICENSE index 591ff22..e7023b1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,11 @@ Copyright (C) 2022 Viehlieb + +Comment: Most of the source code was originally written for https://github.com/foodcoops/sharedlists under the GNU License and therefore special credit is attributed to the contributors of the sharedlists application: + +Authors: Kidhab: https://github.com/kidhab, +Benjamin Meichsner: https://github.com/benni-as, Wvengen: https://github.com/wvengen, Robwa: https://github.com/robwa, 1resu: https://github.com/1resu, JuliusR: https://github.com/JuliusR + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or diff --git a/README.md b/README.md index 398cabf..2e68041 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # FoodsoftArticleImport This gem provides FoodsoftArticleImport integration for Ruby on Rails and allows to parse avariety of files containing article information. These article information are standardized or customly declared. Possible File Ending are: .bnn, .BNN, .csv, .CSV . It relies on [roo](https://github.com/roo-rb/roo) to read and parse the data + ## Getting started TODO: add bnn codes, explain how to add bnn codes TODO: review GNU License + +For correct import of bnn files, please ensure the correct language setting. Your foodsoft account has to be settings set to german if you want to import articles from file. +To set this is imcrucial for guaranteeing the correct format of article prices. ### Requirements This gem requires Ruby 2.7 \ No newline at end of file diff --git a/foodsoft_article_import.gemspec b/foodsoft_article_import.gemspec index 6e57555..e8e54cb 100644 --- a/foodsoft_article_import.gemspec +++ b/foodsoft_article_import.gemspec @@ -25,4 +25,5 @@ Gem::Specification.new do |spec| spec.extra_rdoc_files = ['README.md'] spec.add_dependency 'roo', '~> 2.9.0' + spec.add_dependency 'simplecov' end diff --git a/lib/foodsoft_article_import.rb b/lib/foodsoft_article_import.rb index 4ea4d55..c08cf2e 100644 --- a/lib/foodsoft_article_import.rb +++ b/lib/foodsoft_article_import.rb @@ -6,7 +6,6 @@ require 'active_support/core_ext/hash/keys' require_relative 'foodsoft_article_import/bioromeo' require_relative 'foodsoft_article_import/bnn' require_relative 'foodsoft_article_import/utf8_encoder' -require_relative 'foodsoft_article_import/borkenstein' require_relative 'foodsoft_article_import/dnb_xml' require_relative 'foodsoft_article_import/foodsoft' module FoodsoftArticleImport @@ -23,9 +22,9 @@ module FoodsoftArticleImport def self.file_formats @@file_formats ||= { 'bnn' => FoodsoftArticleImport::Bnn, - 'borkenstein' => FoodsoftArticleImport::Borkenstein, 'foodsoft' => FoodsoftArticleImport::Foodsoft, 'dnb_xml' => FoodsoftArticleImport::DnbXml, + 'odin' => FoodsoftArticleImport::DnbXml, 'bioromeo' => FoodsoftArticleImport::Bioromeo, }.freeze end @@ -36,17 +35,15 @@ module FoodsoftArticleImport # @option opts [String] type file format (required) (see {.file_formats}) # @return [File, Roo::Spreadsheet] file with encoding set if needed def self.parse(file, custom_file_path: nil, type: nil, **opts, &blk) - # @todo handle wrong or undetected type custom_file_path ||= nil type ||= 'bnn' parser = file_formats[type] - puts parser if block_given? - parser.parse(file, custom_file_path, **opts, &blk) + parser.parse(file, custom_file_path: custom_file_path, **opts, &blk) else data = [] - parser.parse(file, custom_file_path, **opts) { |a| data << a } + parser.parse(file, custom_file_path: custom_file_path, **opts) { |a| data << a } data end end @@ -69,10 +66,11 @@ module FoodsoftArticleImport # @param encoding [String, NilClass] optional CSV encoding # @param col_sep [String, NilClass] optional column separator # @return [Roo::Spreadsheet] - def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil) + def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil, liberal_parsing: nil) opts = {csv_options: {}} opts[:csv_options][:encoding] = encoding if encoding opts[:csv_options][:col_sep] = col_sep if col_sep + opts[:csv_options][:liberal_parsing] = true if liberal_parsing opts[:extension] = File.extname(filename) if filename begin Roo::Spreadsheet.open(file, **opts) diff --git a/lib/foodsoft_article_import/bioromeo.rb b/lib/foodsoft_article_import/bioromeo.rb index a7f73ac..be076b4 100644 --- a/lib/foodsoft_article_import/bioromeo.rb +++ b/lib/foodsoft_article_import/bioromeo.rb @@ -30,42 +30,42 @@ module FoodsoftArticleImport def self.parse(file, custom_file_path: nil, **opts) custom_file_path ||= nil opts = OPTIONS.merge(opts) + opts[:liberal_parsing]=true + opts[:col_sep]="," ss = FoodsoftArticleImport.open_spreadsheet(file, **opts) - header_row = true sheet = ss.sheet(0).parse(clean: true, - number: /^artnr/i, - name: /^product/i, - skal: /^skal$/i, - demeter: /^demeter$/i, + order_number: /Artnr./, + name: /Product/, + skal: /Skal$/, + demeter: /Demeter$/, unit_price: /prijs\b.*\beenh/i, pack_price: /prijs\b.*\bcolli/i, - comment: /^opm(erking)?/i, + comment: /opm(erking)?/i, ) linenum = 0 category = nil sheet.each do |row| - puts("[ROW] #{row.inspect}") linenum += 1 - row[:name].blank? and next + row[:name].to_s.strip.empty? and next # (sub)categories are in first two content cells - assume if there's a price it's a product - if row[:order_number].blank? && row[:unit_price].blank? + if row[:order_number].to_s.strip.empty? && row[:unit_price].to_s.strip.empty? category = row[:name] yield nil, nil, linenum next end # skip products without a number - if row[:order_number].blank? + if row[:order_number].to_s.strip.empty? yield nil, nil, linenum next end # extract name and unit errors = [] notes = [] - unit_price = row[:unit_price] - pack_price = row[:pack_price] + unit_price = row[:unit_price].gsub("€","").to_s.strip.to_f + pack_price = row[:pack_price].gsub("€","").to_s.strip.to_f number = row[:order_number] name = row[:name] unit = nil @@ -75,6 +75,7 @@ module FoodsoftArticleImport m=name.match(re) unless m yield nil, nil, linenum + next end unit = self.normalize_unit(m[3]) name = name.sub(re, '').sub(/\(\s*\)\s*$/,'').sub(/\s+/, ' ').sub(/\.\s*$/, '').strip @@ -120,10 +121,10 @@ module FoodsoftArticleImport end end # note from various fields - notes.append("Skal #{row[:skal]}") if row[:skal].present? - notes.append(row[:demeter]) if row[:demeter].present? && row[:demeter].is_a?(String) - notes.append("Demeter #{row[:demeter]}") if row[:demeter].present? && row[:demeter].is_a?(Fixnum) - notes.append "(#{row[:comment]})" unless row[:comment].blank? + notes.append("Skal #{row[:skal]}") unless row[:skal].to_s.strip.empty? + notes.append(row[:demeter]) unless row[:skal].to_s.strip.empty? + notes.append("Demeter #{row[:demeter]}") unless row[:skal].to_s.strip.empty? && row[:demeter].is_a?(Fixnum) + notes.append "(#{row[:comment]})" unless row[:comment].to_s.strip.empty? name.sub!(/(,\.?\s*)?\bDemeter\b/i, '') and notes.prepend("Demeter") name.sub!(/(,\.?\s*)?\bBIO\b/i, '') and notes.prepend "BIO" # unit check @@ -172,7 +173,7 @@ module FoodsoftArticleImport elsif what =~ /^gr/ pack_price.to_f / amount.to_f * 1000 end - if kgprice.present? && (kgprice - unit_price.to_f).abs < 1e-2 + unless kgprice.to_s.strip.empty? && (kgprice - unit_price.to_f).abs < 1e-2 return end diff --git a/lib/foodsoft_article_import/borkenstein.rb b/lib/foodsoft_article_import/borkenstein.rb deleted file mode 100644 index 820aee3..0000000 --- a/lib/foodsoft_article_import/borkenstein.rb +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- -# Module for Borkenstein csv import - -require 'csv' - -module FoodsoftArticleImport - class Borkenstein - - REGEX = { - :main => /^(.+)\s+\[([^\[\]]+)\]\s+(\d+\.\d+)\((\d+\.\d+)\)$/, - :manufacturer => /^(.+)\s{4}\[\]\s{4}\(\)$/, - :origin => /(.+)\s+(\w+)\/\w+[\/[\w\-]+]?/ - }.freeze - - NAME = "Borkenstein (CSV)" - OUTLIST = false - OPTIONS = { - col_sep: ",", - encoding: "UTF-8" # @todo check this - }.freeze - - def self.parse(file, custom_file_path: nil, **opts) - custom_file_path ||= nil - global_manufacturer = nil - - file.set_encoding(opts[:encoding] || OPTIONS[:encoding]) - col_sep = opts[:col_sep] || OPTIONS[:col_sep] - CSV.new(file, {col_sep: col_sep, :headers => false}).each.with_index(1) do |row, i| - - # Set manufacturer - if row[1] == "-" - match = row[2].match(REGEX[:manufacturer]) - global_manufacturer = match.captures.first unless match.nil? - end - - # check if the line is empty - unless row[1].blank? || row[1] == "-" - - # Split string and remove beginning " - matched = row[2].gsub(/^\"/, "").gsub(/\"$/, "").match(REGEX[:main]) - - if matched.nil? - puts "No regular article data for #{row[1]}: #{row[2]}" - yield nil, nil, nil - else - name, units, price_high, price_low = matched.captures - - # Try to get origin - matched_name = name.match(REGEX[:origin]) - if matched_name - name, origin = matched_name.captures - else - name, origin = name.gsub(/\s{2,}/, ""), nil - end - - # Manufacturer - if name.match(/^[A-Za-z]{2,3}\s{1}/) - name.gsub!(/^[A-Za-z]{2,3}\s{1}/, "") - manufacturer = global_manufacturer - end - - - # Get unit quantities - units = units.split("x") - if units.size == 2 - unit_quantity = units.first - unit = units.last - else - unit_quantity = 1 - unit = units.first - end - - article = { - :order_number => row[1], - :name => name, - :origin => origin, - :manufacturer => manufacturer, - :unit_quantity => unit_quantity, - :unit => unit, - :price => price_low, # Inklusive Rabattstufe von 10% - :tax => 0.0 # Tax is included - } - - # test, if neccecary attributes exists - if article[:unit].nil? || article[:price].nil? || article[:unit_quantity].nil? - raise "Fehler: Einheit, Preis und MwSt. müssen gegeben sein: #{article.inspect}" - end - - yield article, nil, i - end - end - yield nil, nil, i - end - end - - end -end \ No newline at end of file diff --git a/lib/foodsoft_article_import/dnb_xml.rb b/lib/foodsoft_article_import/dnb_xml.rb index 377881f..c5e2f11 100644 --- a/lib/foodsoft_article_import/dnb_xml.rb +++ b/lib/foodsoft_article_import/dnb_xml.rb @@ -22,7 +22,7 @@ module FoodsoftArticleImport Nokogiri::XML::ParseOptions::NONET + Nokogiri::XML::ParseOptions::COMPACT # do not modify doc! ) - + self.load_codes(custom_file_path) doc.search('product').each.with_index(1) do |row, i| # create a new article unit = row.search('eenheid').text @@ -33,8 +33,6 @@ module FoodsoftArticleImport when 'l' then 'ltr' else unit end - return if i==3 - puts unit, i inhoud = row.search('inhoud').text inhoud.to_s.strip.empty? or (inhoud.to_f-1).abs > 1e-3 and unit = inhoud.gsub(/\.0+\s*$/,'') + unit deposit = row.search('statiegeld').text @@ -44,20 +42,23 @@ module FoodsoftArticleImport @@codes[:indeling][row.search('subindeling').text.to_i] ].compact.join(' - ') - article = {:order_number => row.search('bestelnummer').text, - #:ean => row.search('eancode').text, - :name => row.search('omschrijving').text, - :note => row.search('kwaliteit').text, - :manufacturer => row.search('merk').text, - :origin => row.search('herkomst').text, - :unit => unit, - :price => row.search('prijs inkoopprijs').text, - :unit_quantity => row.search('sve').text, - :tax => row.search('btw').text, - :deposit => deposit, - :article_category => category} - - yield article, (row.search('status') == 'Actief' ? :outlisted : nil), i + status = row.search('status').text == "Actief" ? nil : :outlisted + article = {} + unless row.search('bestelnummer').text == "" + article = {:order_number => row.search('bestelnummer').text, + #:ean => row.search('eancode').text, + :name => row.search('omschrijving').text, + :note => row.search('kwaliteit').text, + :manufacturer => row.search('merk').text, + :origin => row.search('herkomst').text, + :unit => unit, + :price => row.search('prijs inkoopprijs').text, + :unit_quantity => row.search('sve').text, + :tax => row.search('btw').text, + :deposit => deposit, + :article_category => category} + end + yield article, status, i end end @@ -65,16 +66,24 @@ module FoodsoftArticleImport @@codes = Hash.new - def self.load_codes + def self.load_codes(custom_file_path=nil) @gem_lib = File.expand_path "../../", __FILE__ dir = File.join @gem_lib, 'foodsoft_article_import' begin @@codes = YAML::load(File.open(File.join(dir, "dnb_codes.yml"))).symbolize_keys + if(custom_file_path) + custom_codes = YAML::load(File.open(custom_file_path)).symbolize_keys + custom_codes.keys.each do |key| + if @@codes.keys.include?(key) + custom_codes[key] =custom_codes[key].merge @@codes[key] + end + @@codes = @@codes.merge custom_codes + end + end + @@codes rescue => e raise "Failed to load dnb_codes: #{dir}/dnb_codes.yml: #{e.message}" end end end - - FoodsoftArticleImport::DnbXml.load_codes end \ No newline at end of file diff --git a/lib/foodsoft_article_import/foodsoft.rb b/lib/foodsoft_article_import/foodsoft.rb index ada6de1..5ced872 100644 --- a/lib/foodsoft_article_import/foodsoft.rb +++ b/lib/foodsoft_article_import/foodsoft.rb @@ -27,11 +27,11 @@ module FoodsoftArticleImport::Foodsoft # skip first header row if header_row header_row = false - yield nil, nil, i next end # skip empty lines - if row[2].blank? + if row[2].to_s.strip.empty? + # raise no order number given yield nil, nil, i next end @@ -49,7 +49,7 @@ module FoodsoftArticleImport::Foodsoft :scale_price => row[12], :article_category => row[13]} article.merge!(:deposit => row[9]) unless row[9].nil? - article[:order_number].blank? and ArticleImport.generate_number(article) + FoodsoftArticleImport.generate_number(article) if article[:order_number].to_s.strip.empty? if row[6].nil? || row[7].nil? or row[8].nil? yield article, "Error: unit, price and tax must be entered", i else