update gemspec, finetune parser classes
This commit is contained in:
parent
4ed1764b75
commit
77474c0811
9 changed files with 74 additions and 144 deletions
|
|
@ -3,10 +3,12 @@ PATH
|
|||
specs:
|
||||
foodsoft_article_import (1.0.0)
|
||||
roo (~> 2.9.0)
|
||||
simplecov
|
||||
|
||||
GEM
|
||||
remote: http://rubygems.org/
|
||||
specs:
|
||||
docile (1.4.0)
|
||||
nokogiri (1.14.0-x86_64-linux)
|
||||
racc (~> 1.4)
|
||||
racc (1.6.2)
|
||||
|
|
@ -14,6 +16,12 @@ GEM
|
|||
nokogiri (~> 1)
|
||||
rubyzip (>= 1.3.0, < 3.0.0)
|
||||
rubyzip (2.3.2)
|
||||
simplecov (0.22.0)
|
||||
docile (~> 1.1)
|
||||
simplecov-html (~> 0.11)
|
||||
simplecov_json_formatter (~> 0.1)
|
||||
simplecov-html (0.12.3)
|
||||
simplecov_json_formatter (0.1.4)
|
||||
|
||||
PLATFORMS
|
||||
x86_64-linux
|
||||
|
|
|
|||
6
LICENSE
6
LICENSE
|
|
@ -1,5 +1,11 @@
|
|||
Copyright (C) 2022 Viehlieb
|
||||
|
||||
|
||||
Comment: Most of the source code was originally written for https://github.com/foodcoops/sharedlists under the GNU License and therefore special credit is attributed to the contributors of the sharedlists application:
|
||||
|
||||
Authors: Kidhab: https://github.com/kidhab,
|
||||
Benjamin Meichsner: https://github.com/benni-as, Wvengen: https://github.com/wvengen, Robwa: https://github.com/robwa, 1resu: https://github.com/1resu, JuliusR: https://github.com/JuliusR
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
# FoodsoftArticleImport
|
||||
This gem provides FoodsoftArticleImport integration for Ruby on Rails and allows to parse avariety of files containing article information. These article information are standardized or customly declared. Possible File Ending are: .bnn, .BNN, .csv, .CSV . It relies on [roo](https://github.com/roo-rb/roo) to read and parse the data
|
||||
|
||||
|
||||
## Getting started
|
||||
TODO: add bnn codes, explain how to add bnn codes
|
||||
TODO: review GNU License
|
||||
|
||||
For correct import of bnn files, please ensure the correct language setting. Your foodsoft account has to be settings set to german if you want to import articles from file.
|
||||
To set this is imcrucial for guaranteeing the correct format of article prices.
|
||||
### Requirements
|
||||
This gem requires Ruby 2.7
|
||||
|
|
@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
|
|||
spec.extra_rdoc_files = ['README.md']
|
||||
|
||||
spec.add_dependency 'roo', '~> 2.9.0'
|
||||
spec.add_dependency 'simplecov'
|
||||
end
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ require 'active_support/core_ext/hash/keys'
|
|||
require_relative 'foodsoft_article_import/bioromeo'
|
||||
require_relative 'foodsoft_article_import/bnn'
|
||||
require_relative 'foodsoft_article_import/utf8_encoder'
|
||||
require_relative 'foodsoft_article_import/borkenstein'
|
||||
require_relative 'foodsoft_article_import/dnb_xml'
|
||||
require_relative 'foodsoft_article_import/foodsoft'
|
||||
module FoodsoftArticleImport
|
||||
|
|
@ -23,9 +22,9 @@ module FoodsoftArticleImport
|
|||
def self.file_formats
|
||||
@@file_formats ||= {
|
||||
'bnn' => FoodsoftArticleImport::Bnn,
|
||||
'borkenstein' => FoodsoftArticleImport::Borkenstein,
|
||||
'foodsoft' => FoodsoftArticleImport::Foodsoft,
|
||||
'dnb_xml' => FoodsoftArticleImport::DnbXml,
|
||||
'odin' => FoodsoftArticleImport::DnbXml,
|
||||
'bioromeo' => FoodsoftArticleImport::Bioromeo,
|
||||
}.freeze
|
||||
end
|
||||
|
|
@ -36,17 +35,15 @@ module FoodsoftArticleImport
|
|||
# @option opts [String] type file format (required) (see {.file_formats})
|
||||
# @return [File, Roo::Spreadsheet] file with encoding set if needed
|
||||
def self.parse(file, custom_file_path: nil, type: nil, **opts, &blk)
|
||||
# @todo handle wrong or undetected type
|
||||
custom_file_path ||= nil
|
||||
type ||= 'bnn'
|
||||
|
||||
parser = file_formats[type]
|
||||
puts parser
|
||||
if block_given?
|
||||
parser.parse(file, custom_file_path, **opts, &blk)
|
||||
parser.parse(file, custom_file_path: custom_file_path, **opts, &blk)
|
||||
else
|
||||
data = []
|
||||
parser.parse(file, custom_file_path, **opts) { |a| data << a }
|
||||
parser.parse(file, custom_file_path: custom_file_path, **opts) { |a| data << a }
|
||||
data
|
||||
end
|
||||
end
|
||||
|
|
@ -69,10 +66,11 @@ module FoodsoftArticleImport
|
|||
# @param encoding [String, NilClass] optional CSV encoding
|
||||
# @param col_sep [String, NilClass] optional column separator
|
||||
# @return [Roo::Spreadsheet]
|
||||
def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil)
|
||||
def self.open_spreadsheet(file, filename: nil, encoding: nil, col_sep: nil, liberal_parsing: nil)
|
||||
opts = {csv_options: {}}
|
||||
opts[:csv_options][:encoding] = encoding if encoding
|
||||
opts[:csv_options][:col_sep] = col_sep if col_sep
|
||||
opts[:csv_options][:liberal_parsing] = true if liberal_parsing
|
||||
opts[:extension] = File.extname(filename) if filename
|
||||
begin
|
||||
Roo::Spreadsheet.open(file, **opts)
|
||||
|
|
|
|||
|
|
@ -30,42 +30,42 @@ module FoodsoftArticleImport
|
|||
def self.parse(file, custom_file_path: nil, **opts)
|
||||
custom_file_path ||= nil
|
||||
opts = OPTIONS.merge(opts)
|
||||
opts[:liberal_parsing]=true
|
||||
opts[:col_sep]=","
|
||||
ss = FoodsoftArticleImport.open_spreadsheet(file, **opts)
|
||||
|
||||
header_row = true
|
||||
sheet = ss.sheet(0).parse(clean: true,
|
||||
number: /^artnr/i,
|
||||
name: /^product/i,
|
||||
skal: /^skal$/i,
|
||||
demeter: /^demeter$/i,
|
||||
order_number: /Artnr./,
|
||||
name: /Product/,
|
||||
skal: /Skal$/,
|
||||
demeter: /Demeter$/,
|
||||
unit_price: /prijs\b.*\beenh/i,
|
||||
pack_price: /prijs\b.*\bcolli/i,
|
||||
comment: /^opm(erking)?/i,
|
||||
comment: /opm(erking)?/i,
|
||||
)
|
||||
|
||||
linenum = 0
|
||||
category = nil
|
||||
|
||||
sheet.each do |row|
|
||||
puts("[ROW] #{row.inspect}")
|
||||
linenum += 1
|
||||
row[:name].blank? and next
|
||||
row[:name].to_s.strip.empty? and next
|
||||
# (sub)categories are in first two content cells - assume if there's a price it's a product
|
||||
if row[:order_number].blank? && row[:unit_price].blank?
|
||||
if row[:order_number].to_s.strip.empty? && row[:unit_price].to_s.strip.empty?
|
||||
category = row[:name]
|
||||
yield nil, nil, linenum
|
||||
next
|
||||
end
|
||||
# skip products without a number
|
||||
if row[:order_number].blank?
|
||||
if row[:order_number].to_s.strip.empty?
|
||||
yield nil, nil, linenum
|
||||
next
|
||||
end
|
||||
# extract name and unit
|
||||
errors = []
|
||||
notes = []
|
||||
unit_price = row[:unit_price]
|
||||
pack_price = row[:pack_price]
|
||||
unit_price = row[:unit_price].gsub("€","").to_s.strip.to_f
|
||||
pack_price = row[:pack_price].gsub("€","").to_s.strip.to_f
|
||||
number = row[:order_number]
|
||||
name = row[:name]
|
||||
unit = nil
|
||||
|
|
@ -75,6 +75,7 @@ module FoodsoftArticleImport
|
|||
m=name.match(re)
|
||||
unless m
|
||||
yield nil, nil, linenum
|
||||
next
|
||||
end
|
||||
unit = self.normalize_unit(m[3])
|
||||
name = name.sub(re, '').sub(/\(\s*\)\s*$/,'').sub(/\s+/, ' ').sub(/\.\s*$/, '').strip
|
||||
|
|
@ -120,10 +121,10 @@ module FoodsoftArticleImport
|
|||
end
|
||||
end
|
||||
# note from various fields
|
||||
notes.append("Skal #{row[:skal]}") if row[:skal].present?
|
||||
notes.append(row[:demeter]) if row[:demeter].present? && row[:demeter].is_a?(String)
|
||||
notes.append("Demeter #{row[:demeter]}") if row[:demeter].present? && row[:demeter].is_a?(Fixnum)
|
||||
notes.append "(#{row[:comment]})" unless row[:comment].blank?
|
||||
notes.append("Skal #{row[:skal]}") unless row[:skal].to_s.strip.empty?
|
||||
notes.append(row[:demeter]) unless row[:skal].to_s.strip.empty?
|
||||
notes.append("Demeter #{row[:demeter]}") unless row[:skal].to_s.strip.empty? && row[:demeter].is_a?(Fixnum)
|
||||
notes.append "(#{row[:comment]})" unless row[:comment].to_s.strip.empty?
|
||||
name.sub!(/(,\.?\s*)?\bDemeter\b/i, '') and notes.prepend("Demeter")
|
||||
name.sub!(/(,\.?\s*)?\bBIO\b/i, '') and notes.prepend "BIO"
|
||||
# unit check
|
||||
|
|
@ -172,7 +173,7 @@ module FoodsoftArticleImport
|
|||
elsif what =~ /^gr/
|
||||
pack_price.to_f / amount.to_f * 1000
|
||||
end
|
||||
if kgprice.present? && (kgprice - unit_price.to_f).abs < 1e-2
|
||||
unless kgprice.to_s.strip.empty? && (kgprice - unit_price.to_f).abs < 1e-2
|
||||
return
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -1,97 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Module for Borkenstein csv import
|
||||
|
||||
require 'csv'
|
||||
|
||||
module FoodsoftArticleImport
|
||||
class Borkenstein
|
||||
|
||||
REGEX = {
|
||||
:main => /^(.+)\s+\[([^\[\]]+)\]\s+(\d+\.\d+)\((\d+\.\d+)\)$/,
|
||||
:manufacturer => /^(.+)\s{4}\[\]\s{4}\(\)$/,
|
||||
:origin => /(.+)\s+(\w+)\/\w+[\/[\w\-]+]?/
|
||||
}.freeze
|
||||
|
||||
NAME = "Borkenstein (CSV)"
|
||||
OUTLIST = false
|
||||
OPTIONS = {
|
||||
col_sep: ",",
|
||||
encoding: "UTF-8" # @todo check this
|
||||
}.freeze
|
||||
|
||||
def self.parse(file, custom_file_path: nil, **opts)
|
||||
custom_file_path ||= nil
|
||||
global_manufacturer = nil
|
||||
|
||||
file.set_encoding(opts[:encoding] || OPTIONS[:encoding])
|
||||
col_sep = opts[:col_sep] || OPTIONS[:col_sep]
|
||||
CSV.new(file, {col_sep: col_sep, :headers => false}).each.with_index(1) do |row, i|
|
||||
|
||||
# Set manufacturer
|
||||
if row[1] == "-"
|
||||
match = row[2].match(REGEX[:manufacturer])
|
||||
global_manufacturer = match.captures.first unless match.nil?
|
||||
end
|
||||
|
||||
# check if the line is empty
|
||||
unless row[1].blank? || row[1] == "-"
|
||||
|
||||
# Split string and remove beginning "
|
||||
matched = row[2].gsub(/^\"/, "").gsub(/\"$/, "").match(REGEX[:main])
|
||||
|
||||
if matched.nil?
|
||||
puts "No regular article data for #{row[1]}: #{row[2]}"
|
||||
yield nil, nil, nil
|
||||
else
|
||||
name, units, price_high, price_low = matched.captures
|
||||
|
||||
# Try to get origin
|
||||
matched_name = name.match(REGEX[:origin])
|
||||
if matched_name
|
||||
name, origin = matched_name.captures
|
||||
else
|
||||
name, origin = name.gsub(/\s{2,}/, ""), nil
|
||||
end
|
||||
|
||||
# Manufacturer
|
||||
if name.match(/^[A-Za-z]{2,3}\s{1}/)
|
||||
name.gsub!(/^[A-Za-z]{2,3}\s{1}/, "")
|
||||
manufacturer = global_manufacturer
|
||||
end
|
||||
|
||||
|
||||
# Get unit quantities
|
||||
units = units.split("x")
|
||||
if units.size == 2
|
||||
unit_quantity = units.first
|
||||
unit = units.last
|
||||
else
|
||||
unit_quantity = 1
|
||||
unit = units.first
|
||||
end
|
||||
|
||||
article = {
|
||||
:order_number => row[1],
|
||||
:name => name,
|
||||
:origin => origin,
|
||||
:manufacturer => manufacturer,
|
||||
:unit_quantity => unit_quantity,
|
||||
:unit => unit,
|
||||
:price => price_low, # Inklusive Rabattstufe von 10%
|
||||
:tax => 0.0 # Tax is included
|
||||
}
|
||||
|
||||
# test, if neccecary attributes exists
|
||||
if article[:unit].nil? || article[:price].nil? || article[:unit_quantity].nil?
|
||||
raise "Fehler: Einheit, Preis und MwSt. müssen gegeben sein: #{article.inspect}"
|
||||
end
|
||||
|
||||
yield article, nil, i
|
||||
end
|
||||
end
|
||||
yield nil, nil, i
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
|
@ -22,7 +22,7 @@ module FoodsoftArticleImport
|
|||
Nokogiri::XML::ParseOptions::NONET +
|
||||
Nokogiri::XML::ParseOptions::COMPACT # do not modify doc!
|
||||
)
|
||||
|
||||
self.load_codes(custom_file_path)
|
||||
doc.search('product').each.with_index(1) do |row, i|
|
||||
# create a new article
|
||||
unit = row.search('eenheid').text
|
||||
|
|
@ -33,8 +33,6 @@ module FoodsoftArticleImport
|
|||
when 'l' then 'ltr'
|
||||
else unit
|
||||
end
|
||||
return if i==3
|
||||
puts unit, i
|
||||
inhoud = row.search('inhoud').text
|
||||
inhoud.to_s.strip.empty? or (inhoud.to_f-1).abs > 1e-3 and unit = inhoud.gsub(/\.0+\s*$/,'') + unit
|
||||
deposit = row.search('statiegeld').text
|
||||
|
|
@ -44,20 +42,23 @@ module FoodsoftArticleImport
|
|||
@@codes[:indeling][row.search('subindeling').text.to_i]
|
||||
].compact.join(' - ')
|
||||
|
||||
article = {:order_number => row.search('bestelnummer').text,
|
||||
#:ean => row.search('eancode').text,
|
||||
:name => row.search('omschrijving').text,
|
||||
:note => row.search('kwaliteit').text,
|
||||
:manufacturer => row.search('merk').text,
|
||||
:origin => row.search('herkomst').text,
|
||||
:unit => unit,
|
||||
:price => row.search('prijs inkoopprijs').text,
|
||||
:unit_quantity => row.search('sve').text,
|
||||
:tax => row.search('btw').text,
|
||||
:deposit => deposit,
|
||||
:article_category => category}
|
||||
|
||||
yield article, (row.search('status') == 'Actief' ? :outlisted : nil), i
|
||||
status = row.search('status').text == "Actief" ? nil : :outlisted
|
||||
article = {}
|
||||
unless row.search('bestelnummer').text == ""
|
||||
article = {:order_number => row.search('bestelnummer').text,
|
||||
#:ean => row.search('eancode').text,
|
||||
:name => row.search('omschrijving').text,
|
||||
:note => row.search('kwaliteit').text,
|
||||
:manufacturer => row.search('merk').text,
|
||||
:origin => row.search('herkomst').text,
|
||||
:unit => unit,
|
||||
:price => row.search('prijs inkoopprijs').text,
|
||||
:unit_quantity => row.search('sve').text,
|
||||
:tax => row.search('btw').text,
|
||||
:deposit => deposit,
|
||||
:article_category => category}
|
||||
end
|
||||
yield article, status, i
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -65,16 +66,24 @@ module FoodsoftArticleImport
|
|||
|
||||
@@codes = Hash.new
|
||||
|
||||
def self.load_codes
|
||||
def self.load_codes(custom_file_path=nil)
|
||||
@gem_lib = File.expand_path "../../", __FILE__
|
||||
dir = File.join @gem_lib, 'foodsoft_article_import'
|
||||
begin
|
||||
@@codes = YAML::load(File.open(File.join(dir, "dnb_codes.yml"))).symbolize_keys
|
||||
if(custom_file_path)
|
||||
custom_codes = YAML::load(File.open(custom_file_path)).symbolize_keys
|
||||
custom_codes.keys.each do |key|
|
||||
if @@codes.keys.include?(key)
|
||||
custom_codes[key] =custom_codes[key].merge @@codes[key]
|
||||
end
|
||||
@@codes = @@codes.merge custom_codes
|
||||
end
|
||||
end
|
||||
@@codes
|
||||
rescue => e
|
||||
raise "Failed to load dnb_codes: #{dir}/dnb_codes.yml: #{e.message}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
FoodsoftArticleImport::DnbXml.load_codes
|
||||
end
|
||||
|
|
@ -27,11 +27,11 @@ module FoodsoftArticleImport::Foodsoft
|
|||
# skip first header row
|
||||
if header_row
|
||||
header_row = false
|
||||
yield nil, nil, i
|
||||
next
|
||||
end
|
||||
# skip empty lines
|
||||
if row[2].blank?
|
||||
if row[2].to_s.strip.empty?
|
||||
# raise no order number given
|
||||
yield nil, nil, i
|
||||
next
|
||||
end
|
||||
|
|
@ -49,7 +49,7 @@ module FoodsoftArticleImport::Foodsoft
|
|||
:scale_price => row[12],
|
||||
:article_category => row[13]}
|
||||
article.merge!(:deposit => row[9]) unless row[9].nil?
|
||||
article[:order_number].blank? and ArticleImport.generate_number(article)
|
||||
FoodsoftArticleImport.generate_number(article) if article[:order_number].to_s.strip.empty?
|
||||
if row[6].nil? || row[7].nil? or row[8].nil?
|
||||
yield article, "Error: unit, price and tax must be entered", i
|
||||
else
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue