0d8b81b5-8e6f-41b6-8c19-b7ea0d37bb04

Committed 03 Jan 2025 08:42PM UTC coverage: 91.952% (-0.07%) from 92.02%

Build # 0d8b81b5-8e6f-41b6-8c19-b7ea0d37bb04

Build Type

Pull #2585

circleci

Committed by

maxkadel

Commit Message

Put CSV Validation into its own child job, run download & process in callback

Pull Request Pull Request #2585: Put CSV Validation into its own child job

Run Details

45 of 45 new or added lines in 5 files covered. (100.0%)

4 existing lines in 1 file now uncovered.

3382 of 3678 relevant lines covered (91.95%)

377.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.3

/app/models/scsb/partner_updates/update.rb

require 'marc_cleanup'

module Scsb
  class PartnerUpdates
    class Update
      attr_accessor :dump

      def initialize(dump:, dump_file_type:, timestamp:)
        @last_dump = timestamp
        @dump = dump
        @dump_file_type = dump_file_type
        @s3_bucket = Scsb::S3Bucket.partner_transfer_client
        @scsb_file_dir = ENV['SCSB_FILE_DIR']
        @update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
      end

      def attach_dump_file(filepath, dump_file_type: nil)
        dump_file_type ||= @dump_file_type
        df = DumpFile.create(dump_file_type:, path: filepath)
        df.zip
        df.save
        @dump.dump_files << df
        @dump.save
      end

      def process_partner_updates(files:, file_prefix: 'scsb_update_')
        xml_files = []
        files.each do |file|
          filename = File.basename(file, '.zip')
          filename.gsub!(/^[^_]+_([0-9]+)_([0-9]+).*$/, '\1_\2')
          file_increment = 1
          Zip::File.open(file) do |zip_file|
            zip_file.each do |entry|
              target = "#{@update_directory}/#{filename}_#{file_increment}.xml"
              xml_files << target
              entry.extract(target)
              file_increment += 1
            end
          end
          File.unlink(file)
        end
        xml_files.each do |file|
          filename = File.basename(file)
          reader = MARC::XMLReader.new(file.to_s, external_encoding: 'UTF-8')
          filepath = "#{@scsb_file_dir}/#{file_prefix}#{filename}"
          writer = MARC::XMLWriter.new(filepath)
          reader.each { |record| writer.write(process_record(record)) }
          writer.close
          File.unlink(file)
          attach_dump_file(filepath)
        end
      end

      def process_record(record)
        record = field_delete(['856', '959'], record)
        record.leader[5] = 'c' if record.leader[5].eql?('d')
        record = bad_utf8_fix(record) if bad_utf8?(record)
        record = invalid_xml_fix(record) if invalid_xml_chars?(record)
        record = tab_newline_fix(record) if tab_newline_char?(record)
        record = leaderfix(record) if leader_errors?(record)
        record = composed_chars_normalize(record) if composed_chars_errors?(record)
        record = extra_space_fix(record)
        empty_subfield_fix(record)
      end

      def add_error(message:)
        error = Array.wrap(@dump.event.error)
        error << message
        @dump.event.error = error.join('; ')
        @dump.event.save
      end

      def date_strings
        @dump.dump_files.map do |df|
          if df.dump_file_type == 'recap_records_full_metadata'
            File.basename(df.path).split('_')[3]
          else
            File.basename(df.path).split('_')[2]
          end
        end
      end

      def prepare_directory
        FileUtils.mkdir_p(@update_directory)
      end
    end
  end
end

1	require 'marc_cleanup'	1✔
2
3	module Scsb	1✔
4	class PartnerUpdates	1✔
5	class Update	1✔
6	attr_accessor :dump	1✔
7
8	def initialize(dump:, dump_file_type:, timestamp:)	1✔
9	@last_dump = timestamp	17✔
10	@dump = dump	17✔
11	@dump_file_type = dump_file_type	17✔
12	@s3_bucket = Scsb::S3Bucket.partner_transfer_client	17✔
13	@scsb_file_dir = ENV['SCSB_FILE_DIR']	17✔
14	@update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	17✔
15	end
16
17	def attach_dump_file(filepath, dump_file_type: nil)	1✔
18	dump_file_type \|\|= @dump_file_type	29✔
19	df = DumpFile.create(dump_file_type:, path: filepath)	29✔
20	df.zip	29✔
21	df.save	29✔
22	@dump.dump_files << df	29✔
23	@dump.save	29✔
24	end
25
26	def process_partner_updates(files:, file_prefix: 'scsb_update_')	1✔
27	xml_files = []	14✔
28	files.each do \|file\|	14✔
29	filename = File.basename(file, '.zip')	14✔
30	filename.gsub!(/^[^_]+_([0-9]+)_([0-9]+).*$/, '\1_\2')	14✔
31	file_increment = 1	14✔
32	Zip::File.open(file) do \|zip_file\|	14✔
33	zip_file.each do \|entry\|	14✔
34	target = "#{@update_directory}/#{filename}_#{file_increment}.xml"	28✔
35	xml_files << target	28✔
36	entry.extract(target)	28✔
37	file_increment += 1	28✔
38	end
39	end
40	File.unlink(file)	14✔
41	end
42	xml_files.each do \|file\|	14✔
43	filename = File.basename(file)	28✔
44	reader = MARC::XMLReader.new(file.to_s, external_encoding: 'UTF-8')	28✔
45	filepath = "#{@scsb_file_dir}/#{file_prefix}#{filename}"	28✔
46	writer = MARC::XMLWriter.new(filepath)	28✔
47	reader.each { \|record\| writer.write(process_record(record)) }	84✔
48	writer.close	28✔
49	File.unlink(file)	28✔
50	attach_dump_file(filepath)	28✔
51	end
52	end
53
54	def process_record(record)	1✔
55	record = field_delete(['856', '959'], record)	57✔
56	record.leader[5] = 'c' if record.leader[5].eql?('d')	57✔
57	record = bad_utf8_fix(record) if bad_utf8?(record)	57✔
58	record = invalid_xml_fix(record) if invalid_xml_chars?(record)	57✔
59	record = tab_newline_fix(record) if tab_newline_char?(record)	57✔
60	record = leaderfix(record) if leader_errors?(record)	57✔
61	record = composed_chars_normalize(record) if composed_chars_errors?(record)	57✔
62	record = extra_space_fix(record)	57✔
63	empty_subfield_fix(record)	57✔
64	end
65
66	def add_error(message:)	1✔
UNCOV 67	error = Array.wrap(@dump.event.error)	×
UNCOV 68	error << message	×
UNCOV 69	@dump.event.error = error.join('; ')	×
UNCOV 70	@dump.event.save	×
71	end
72
73	def date_strings	1✔
74	@dump.dump_files.map do \|df\|	×
75	if df.dump_file_type == 'recap_records_full_metadata'	×
76	File.basename(df.path).split('_')[3]	×
77	else
78	File.basename(df.path).split('_')[2]	×
79	end
80	end
81	end
82
83	def prepare_directory	1✔
84	FileUtils.mkdir_p(@update_directory)	3✔
85	end
86	end
87	end
88	end

pulibrary / bibdata / 0d8b81b5-8e6f-41b6-8c19-b7ea0d37bb04

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous