update gemspec, finetune parser classes

Gemfile.lock

@@ -3,10 +3,12 @@ PATH
   specs:
     foodsoft_article_import (1.0.0)
       roo (~> 2.9.0)
+      simplecov
 
 GEM
   remote: http://rubygems.org/
   specs:
+    docile (1.4.0)
     nokogiri (1.14.0-x86_64-linux)
       racc (~> 1.4)
     racc (1.6.2)
@@ -14,6 +16,12 @@ GEM
       nokogiri (~> 1)
       rubyzip (>= 1.3.0, < 3.0.0)
     rubyzip (2.3.2)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
 
 PLATFORMS
   x86_64-linux

LICENSE

@@ -1,5 +1,11 @@
 Copyright (C) 2022 Viehlieb
 
+
+Comment: Most of the source code was originally written for https://github.com/foodcoops/sharedlists under the GNU License and therefore special credit is attributed to the contributors of the sharedlists application:
+
+Authors: Kidhab: https://github.com/kidhab, 
+Benjamin Meichsner: https://github.com/benni-as, Wvengen: https://github.com/wvengen, Robwa: https://github.com/robwa, 1resu: https://github.com/1resu, JuliusR: https://github.com/JuliusR
+
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or

README.md

@@ -1,8 +1,12 @@
 # FoodsoftArticleImport
 This gem provides FoodsoftArticleImport integration for Ruby on Rails and allows to parse avariety of files containing article information. These article information are standardized or customly declared. Possible File Ending are: .bnn, .BNN, .csv, .CSV . It relies on [roo](https://github.com/roo-rb/roo) to read and parse the data
 
+
 ## Getting started
 TODO: add bnn codes, explain how to add bnn codes
 TODO: review GNU License
+
+For correct import of bnn files, please ensure the correct language setting. Your foodsoft account has to be settings set to german if you want to import articles from file.
+To set this is imcrucial for guaranteeing the correct format of article prices.
 ### Requirements
 This gem requires Ruby 2.7

foodsoft_article_import.gemspec

@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
   spec.extra_rdoc_files = ['README.md']
   
   spec.add_dependency 'roo', '~> 2.9.0'
+  spec.add_dependency 'simplecov'
 end

lib/foodsoft_article_import.rb

@@ -6,7 +6,6 @@ require 'active_support/core_ext/hash/keys'
 require_relative 'foodsoft_article_import/bioromeo'
 require_relative 'foodsoft_article_import/bnn'
 require_relative 'foodsoft_article_import/utf8_encoder'
-require_relative 'foodsoft_article_import/borkenstein'
 require_relative 'foodsoft_article_import/dnb_xml'
 require_relative 'foodsoft_article_import/foodsoft'
 module FoodsoftArticleImport
@@ -23,9 +22,9 @@ module FoodsoftArticleImport
   def self.file_formats
     @@file_formats ||= {
       'bnn' => FoodsoftArticleImport::Bnn,
-      'borkenstein' => FoodsoftArticleImport::Borkenstein,
       'foodsoft' => FoodsoftArticleImport::Foodsoft,
       'dnb_xml' => FoodsoftArticleImport::DnbXml,
+      'odin' => FoodsoftArticleImport::DnbXml,
       'bioromeo' => FoodsoftArticleImport::Bioromeo,
     }.freeze
   end
@@ -36,17 +35,15 @@ module FoodsoftArticleImport
   # @option opts [String] type file format (required) (see {.file_formats})
   # @return [File, Roo::Spreadsheet] file with encoding set if needed
   def self.parse(file, custom_file_path: nil, type: nil, **opts, &blk)
-    # @todo handle wrong or undetected type
     custom_file_path ||= nil
     type ||= 'bnn'
 
     parser = file_formats[type]
-    puts parser
     if block_given?
-      parser.parse(file, custom_file_path, **opts, &blk)
+      parser.parse(file, custom_file_path: custom_file_path, **opts, &blk)
     else
       data = []
-      parser.parse(file, custom_file_path, **opts) { |a| data << a }
+      parser.parse(file, custom_file_path: custom_file_path, **opts) { |a| data << a }
       data
     end
   end
@@ -69,10 +66,11 @@ module FoodsoftArticleImport
   # @param encoding [String, NilClass] optional CSV encoding
   # @param col_sep [String, NilClass] optional column separator
   # @return [Roo::Spreadsheet]
-  def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil)
+  def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil, liberal_parsing: nil)
     opts = {csv_options: {}}
     opts[:csv_options][:encoding] = encoding if encoding
     opts[:csv_options][:col_sep] = col_sep if col_sep
+    opts[:csv_options][:liberal_parsing] = true if liberal_parsing
     opts[:extension] = File.extname(filename) if filename
     begin
       Roo::Spreadsheet.open(file, **opts)

lib/foodsoft_article_import/bioromeo.rb

@@ -30,42 +30,42 @@ module FoodsoftArticleImport
     def self.parse(file, custom_file_path: nil, **opts)
       custom_file_path ||= nil
       opts = OPTIONS.merge(opts)
+      opts[:liberal_parsing]=true
+      opts[:col_sep]=","
       ss = FoodsoftArticleImport.open_spreadsheet(file, **opts)
-
       header_row = true
       sheet = ss.sheet(0).parse(clean: true,
-        number:     /^artnr/i,
+        order_number:     /Artnr./,
-        name:       /^product/i,
+        name:       /Product/,
-        skal:       /^skal$/i,
+        skal:       /Skal$/,
-        demeter:    /^demeter$/i,
+        demeter:    /Demeter$/,
         unit_price: /prijs\b.*\beenh/i,
         pack_price: /prijs\b.*\bcolli/i,
-        comment:    /^opm(erking)?/i,
+        comment:    /opm(erking)?/i,
       )
 
       linenum = 0
       category = nil
 
       sheet.each do |row|
-        puts("[ROW] #{row.inspect}")
         linenum += 1
-        row[:name].blank? and next
+        row[:name].to_s.strip.empty? and next
         # (sub)categories are in first two content cells - assume if there's a price it's a product
-        if row[:order_number].blank? && row[:unit_price].blank?
+        if row[:order_number].to_s.strip.empty? && row[:unit_price].to_s.strip.empty?
           category = row[:name]
           yield nil, nil, linenum
           next
         end
         # skip products without a number
-        if row[:order_number].blank?
+        if row[:order_number].to_s.strip.empty?
           yield nil, nil, linenum
           next
         end
         # extract name and unit
         errors = []
         notes = []
-        unit_price = row[:unit_price]
+        unit_price = row[:unit_price].gsub("€","").to_s.strip.to_f
-        pack_price = row[:pack_price]
+        pack_price = row[:pack_price].gsub("€","").to_s.strip.to_f
         number = row[:order_number]
         name = row[:name]
         unit = nil
@@ -75,6 +75,7 @@ module FoodsoftArticleImport
           m=name.match(re)
           unless m
             yield nil, nil, linenum
+            next
           end
           unit = self.normalize_unit(m[3])
           name = name.sub(re, '').sub(/\(\s*\)\s*$/,'').sub(/\s+/, ' ').sub(/\.\s*$/, '').strip
@@ -120,10 +121,10 @@ module FoodsoftArticleImport
           end
         end
         # note from various fields
-        notes.append("Skal #{row[:skal]}") if row[:skal].present?
+        notes.append("Skal #{row[:skal]}") unless row[:skal].to_s.strip.empty?
-        notes.append(row[:demeter]) if row[:demeter].present? && row[:demeter].is_a?(String)
+        notes.append(row[:demeter]) unless row[:skal].to_s.strip.empty?
-        notes.append("Demeter #{row[:demeter]}") if row[:demeter].present? && row[:demeter].is_a?(Fixnum)
+        notes.append("Demeter #{row[:demeter]}") unless row[:skal].to_s.strip.empty? && row[:demeter].is_a?(Fixnum)
-        notes.append "(#{row[:comment]})" unless row[:comment].blank?
+        notes.append "(#{row[:comment]})" unless row[:comment].to_s.strip.empty?
         name.sub!(/(,\.?\s*)?\bDemeter\b/i, '') and notes.prepend("Demeter")
         name.sub!(/(,\.?\s*)?\bBIO\b/i, '') and notes.prepend "BIO"
         # unit check
@@ -172,7 +173,7 @@ module FoodsoftArticleImport
                 elsif what =~ /^gr/
                   pack_price.to_f / amount.to_f * 1000
                 end
-      if kgprice.present? && (kgprice - unit_price.to_f).abs < 1e-2
+      unless kgprice.to_s.strip.empty? && (kgprice - unit_price.to_f).abs < 1e-2
         return
       end
 

lib/foodsoft_article_import/borkenstein.rb

@@ -1,97 +0,0 @@
-# -*- coding: utf-8 -*-
-# Module for Borkenstein csv import
-
-require 'csv'
-
-module FoodsoftArticleImport
-  class Borkenstein
-
-    REGEX = {
-      :main => /^(.+)\s+\[([^\[\]]+)\]\s+(\d+\.\d+)\((\d+\.\d+)\)$/,
-      :manufacturer => /^(.+)\s{4}\[\]\s{4}\(\)$/,
-      :origin => /(.+)\s+(\w+)\/\w+[\/[\w\-]+]?/
-    }.freeze
-    
-    NAME = "Borkenstein (CSV)"
-    OUTLIST = false
-    OPTIONS = {
-      col_sep: ",",
-      encoding: "UTF-8" # @todo check this
-    }.freeze
-
-    def self.parse(file, custom_file_path: nil, **opts)
-      custom_file_path ||= nil
-      global_manufacturer = nil
-
-      file.set_encoding(opts[:encoding] || OPTIONS[:encoding])
-      col_sep = opts[:col_sep] || OPTIONS[:col_sep]
-      CSV.new(file, {col_sep: col_sep, :headers => false}).each.with_index(1) do |row, i|
-
-        # Set manufacturer
-        if row[1] == "-"
-          match = row[2].match(REGEX[:manufacturer])
-          global_manufacturer = match.captures.first unless match.nil?
-        end
-
-        # check if the line is empty
-        unless row[1].blank? || row[1] == "-"
-
-          # Split string and remove beginning "
-          matched = row[2].gsub(/^\"/, "").gsub(/\"$/, "").match(REGEX[:main])
-
-          if matched.nil?
-            puts "No regular article data for #{row[1]}: #{row[2]}"
-            yield nil, nil, nil
-          else
-            name, units, price_high, price_low = matched.captures
-
-            # Try to get origin
-            matched_name = name.match(REGEX[:origin])
-            if matched_name
-              name, origin = matched_name.captures
-            else
-              name, origin = name.gsub(/\s{2,}/, ""), nil
-            end
-
-            # Manufacturer
-            if name.match(/^[A-Za-z]{2,3}\s{1}/)
-              name.gsub!(/^[A-Za-z]{2,3}\s{1}/, "")
-              manufacturer = global_manufacturer
-            end
-
-
-            # Get unit quantities
-            units = units.split("x")
-            if units.size == 2
-              unit_quantity = units.first
-              unit = units.last
-            else
-              unit_quantity = 1
-              unit = units.first
-            end
-
-            article = {
-              :order_number => row[1],
-              :name => name,
-              :origin => origin,
-              :manufacturer => manufacturer,
-              :unit_quantity => unit_quantity,
-              :unit => unit,
-              :price => price_low, # Inklusive Rabattstufe von 10%
-              :tax => 0.0 # Tax is included
-            }
-
-            # test, if neccecary attributes exists
-            if article[:unit].nil? || article[:price].nil? || article[:unit_quantity].nil?
-              raise "Fehler: Einheit, Preis und MwSt. müssen gegeben sein: #{article.inspect}"
-            end
-
-            yield article, nil, i
-          end
-        end
-        yield nil, nil, i
-      end
-    end
-
-  end
-end

lib/foodsoft_article_import/dnb_xml.rb

@@ -22,7 +22,7 @@ module FoodsoftArticleImport
         Nokogiri::XML::ParseOptions::NONET +
         Nokogiri::XML::ParseOptions::COMPACT # do not modify doc!
       )
-
+      self.load_codes(custom_file_path)
       doc.search('product').each.with_index(1) do |row, i|
         # create a new article
         unit = row.search('eenheid').text
@@ -33,8 +33,6 @@ module FoodsoftArticleImport
           when 'l'    then 'ltr'
           else             unit
         end
-        return if i==3
-        puts unit, i
         inhoud = row.search('inhoud').text
         inhoud.to_s.strip.empty? or (inhoud.to_f-1).abs > 1e-3 and unit = inhoud.gsub(/\.0+\s*$/,'') + unit
         deposit = row.search('statiegeld').text
@@ -44,20 +42,23 @@ module FoodsoftArticleImport
           @@codes[:indeling][row.search('subindeling').text.to_i]
         ].compact.join(' - ')
 
-        article = {:order_number => row.search('bestelnummer').text,
+        status = row.search('status').text == "Actief" ? nil : :outlisted
-                  #:ean => row.search('eancode').text,
+        article = {}
-                  :name => row.search('omschrijving').text,
+        unless row.search('bestelnummer').text == ""
-                  :note => row.search('kwaliteit').text,
+          article = {:order_number => row.search('bestelnummer').text,
-                  :manufacturer => row.search('merk').text,
+                    #:ean => row.search('eancode').text,
-                  :origin => row.search('herkomst').text,
+                    :name => row.search('omschrijving').text,
-                  :unit => unit,
+                    :note => row.search('kwaliteit').text,
-                  :price => row.search('prijs inkoopprijs').text,
+                    :manufacturer => row.search('merk').text,
-                  :unit_quantity => row.search('sve').text,
+                    :origin => row.search('herkomst').text,
-                  :tax => row.search('btw').text,
+                    :unit => unit,
-                  :deposit => deposit,
+                    :price => row.search('prijs inkoopprijs').text,
-                  :article_category => category}
+                    :unit_quantity => row.search('sve').text,
-
+                    :tax => row.search('btw').text,
-        yield article, (row.search('status') == 'Actief' ? :outlisted : nil), i
+                    :deposit => deposit,
+                    :article_category => category}
+        end
+        yield article, status, i
       end
     end
 
@@ -65,16 +66,24 @@ module FoodsoftArticleImport
 
     @@codes = Hash.new
 
-    def self.load_codes
+    def self.load_codes(custom_file_path=nil)
       @gem_lib = File.expand_path "../../", __FILE__
       dir = File.join @gem_lib, 'foodsoft_article_import'
       begin
         @@codes = YAML::load(File.open(File.join(dir, "dnb_codes.yml"))).symbolize_keys
+        if(custom_file_path)
+          custom_codes = YAML::load(File.open(custom_file_path)).symbolize_keys
+          custom_codes.keys.each do |key|
+            if @@codes.keys.include?(key)
+              custom_codes[key] =custom_codes[key].merge @@codes[key]
+            end
+          @@codes = @@codes.merge custom_codes
+          end
+        end
+        @@codes
       rescue => e
         raise "Failed to load dnb_codes: #{dir}/dnb_codes.yml: #{e.message}"
       end
     end
   end
-
-  FoodsoftArticleImport::DnbXml.load_codes
 end

lib/foodsoft_article_import/foodsoft.rb

@@ -27,11 +27,11 @@ module FoodsoftArticleImport::Foodsoft
       # skip first header row
       if header_row
         header_row = false
-        yield nil, nil, i
         next
       end
       # skip empty lines
-      if row[2].blank?
+      if row[2].to_s.strip.empty?
+        # raise no order number given
         yield nil, nil, i
         next
       end
@@ -49,7 +49,7 @@ module FoodsoftArticleImport::Foodsoft
                  :scale_price => row[12],
                  :article_category => row[13]}
       article.merge!(:deposit => row[9]) unless row[9].nil?
-      article[:order_number].blank? and ArticleImport.generate_number(article)
+      FoodsoftArticleImport.generate_number(article) if article[:order_number].to_s.strip.empty?
       if row[6].nil? || row[7].nil? or row[8].nil?
         yield article, "Error: unit, price and tax must be entered", i
       else