6eaae75e-8582-4162-bf7f-32b0a072fade

Committed 31 Dec 2024 06:09PM UTC coverage: 90.366% (-1.5%) from 91.859%

Build # 6eaae75e-8582-4162-bf7f-32b0a072fade

Build Type

push

circleci

Committed by

sandbergja

Commit Message

Bundle update rails

Add mutex_m
Don't require marc cleanup

Co-authored-by: Jane Sandberg <sandbergja@users.noreply.github.com>

Run Details

3330 of 3685 relevant lines covered (90.37%)

376.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.09

/app/models/scsb/partner_updates/update.rb

module Scsb
  class PartnerUpdates
    class Update
      attr_accessor :dump

      def initialize(dump:, dump_file_type:, timestamp:)
        @last_dump = timestamp
        @dump = dump
        @dump_file_type = dump_file_type
        @s3_bucket = Scsb::S3Bucket.partner_transfer_client
        @scsb_file_dir = ENV['SCSB_FILE_DIR']
        @update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
        @inv_xml = []
        @tab_newline = []
        @leader = []
        @composed_chars = []
        @bad_utf8 = []
      end

      def attach_dump_file(filepath, dump_file_type: nil)
        dump_file_type ||= @dump_file_type
        df = DumpFile.create(dump_file_type:, path: filepath)
        df.zip
        df.save
        @dump.dump_files << df
        @dump.save
      end

      def process_partner_updates(files:, file_prefix: 'scsb_update_')
        xml_files = []
        files.each do |file|
          filename = File.basename(file, '.zip')
          filename.gsub!(/^[^_]+_([0-9]+)_([0-9]+).*$/, '\1_\2')
          file_increment = 1
          Zip::File.open(file) do |zip_file|
            zip_file.each do |entry|
              target = "#{@update_directory}/#{filename}_#{file_increment}.xml"
              xml_files << target
              entry.extract(target)
              file_increment += 1
            end
          end
          File.unlink(file)
        end
        xml_files.each do |file|
          filename = File.basename(file)
          reader = MARC::XMLReader.new(file.to_s, external_encoding: 'UTF-8')
          filepath = "#{@scsb_file_dir}/#{file_prefix}#{filename}"
          writer = MARC::XMLWriter.new(filepath)
          reader.each { |record| writer.write(process_record(record)) }
          writer.close
          File.unlink(file)
          attach_dump_file(filepath)
        end
      end

      def process_record(record)
        record = field_delete(['856', '959'], record)
        record.leader[5] = 'c' if record.leader[5].eql?('d')
        if bad_utf8?(record)
          @bad_utf8 << record['001']
          record = bad_utf8_fix(record)
        end
        if invalid_xml_chars?(record)
          @inv_xml << record['001']
          record = invalid_xml_fix(record)
        end
        if tab_newline_char?(record)
          @tab_newline << record['001']
          record = tab_newline_fix(record)
        end
        if leader_errors?(record)
          @leader << record['001']
          record = leaderfix(record)
        end
        if composed_chars_errors?(record)
          @composed_chars << record['001']
          record = composed_chars_normalize(record)
        end
        record = extra_space_fix(record)
        empty_subfield_fix(record)
      end

      def add_error(message:)
        error = Array.wrap(@dump.event.error)
        error << message
        @dump.event.error = error.join('; ')
        @dump.event.save
      end

      def log_record_fixes
        log_file = {
          inv_xml: @inv_xml,
          tab_newline: @tab_newline,
          leader: @leader,
          composed_chars: @composed_chars,
          bad_utf8: @bad_utf8
        }
        filepath = log_file_name
        File.write(filepath, log_file.to_json.to_s)
        attach_dump_file(filepath, dump_file_type: :log_file)
      end

      def log_file_name
        "#{@scsb_file_dir}/fixes_#{@last_dump.to_time.strftime('%Y_%m_%d')}.json"
      end

      def set_generated_date
        @dump.generated_date = date_strings.map { |str| DateTime.parse(str) }.sort.first
      end

      def date_strings
        @dump.dump_files.map do |df|
          if df.dump_file_type == 'recap_records_full_metadata'
            File.basename(df.path).split('_')[3]
          else
            File.basename(df.path).split('_')[2]
          end
        end
      end

      def prepare_directory
        FileUtils.mkdir_p(@update_directory)
      end
    end
  end
end

1	module Scsb	1✔
2	class PartnerUpdates	1✔
3	class Update	1✔
4	attr_accessor :dump	1✔
5
6	def initialize(dump:, dump_file_type:, timestamp:)	1✔
7	@last_dump = timestamp	16✔
8	@dump = dump	16✔
9	@dump_file_type = dump_file_type	16✔
10	@s3_bucket = Scsb::S3Bucket.partner_transfer_client	16✔
11	@scsb_file_dir = ENV['SCSB_FILE_DIR']	16✔
12	@update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	16✔
13	@inv_xml = []	16✔
14	@tab_newline = []	16✔
15	@leader = []	16✔
16	@composed_chars = []	16✔
17	@bad_utf8 = []	16✔
18	end
19
20	def attach_dump_file(filepath, dump_file_type: nil)	1✔
21	dump_file_type \|\|= @dump_file_type	8✔
22	df = DumpFile.create(dump_file_type:, path: filepath)	8✔
23	df.zip	8✔
24	df.save	8✔
25	@dump.dump_files << df	8✔
26	@dump.save	8✔
27	end
28
29	def process_partner_updates(files:, file_prefix: 'scsb_update_')	1✔
30	xml_files = []	7✔
31	files.each do \|file\|	7✔
32	filename = File.basename(file, '.zip')	7✔
33	filename.gsub!(/^[^_]+_([0-9]+)_([0-9]+).*$/, '\1_\2')	7✔
34	file_increment = 1	7✔
35	Zip::File.open(file) do \|zip_file\|	7✔
36	zip_file.each do \|entry\|	7✔
37	target = "#{@update_directory}/#{filename}_#{file_increment}.xml"	14✔
38	xml_files << target	14✔
39	entry.extract(target)	14✔
40	file_increment += 1	14✔
41	end
42	end
43	File.unlink(file)	7✔
44	end
45	xml_files.each do \|file\|	7✔
46	filename = File.basename(file)	7✔
47	reader = MARC::XMLReader.new(file.to_s, external_encoding: 'UTF-8')	7✔
48	filepath = "#{@scsb_file_dir}/#{file_prefix}#{filename}"	7✔
49	writer = MARC::XMLWriter.new(filepath)	7✔
50	reader.each { \|record\| writer.write(process_record(record)) }	14✔
51	writer.close	×
52	File.unlink(file)	×
53	attach_dump_file(filepath)	×
54	end
55	end
56
57	def process_record(record)	1✔
58	record = field_delete(['856', '959'], record)	8✔
59	record.leader[5] = 'c' if record.leader[5].eql?('d')	×
60	if bad_utf8?(record)	×
61	@bad_utf8 << record['001']	×
62	record = bad_utf8_fix(record)	×
63	end
64	if invalid_xml_chars?(record)	×
65	@inv_xml << record['001']	×
66	record = invalid_xml_fix(record)	×
67	end
68	if tab_newline_char?(record)	×
69	@tab_newline << record['001']	×
70	record = tab_newline_fix(record)	×
71	end
72	if leader_errors?(record)	×
73	@leader << record['001']	×
74	record = leaderfix(record)	×
75	end
76	if composed_chars_errors?(record)	×
77	@composed_chars << record['001']	×
78	record = composed_chars_normalize(record)	×
79	end
80	record = extra_space_fix(record)	×
81	empty_subfield_fix(record)	×
82	end
83
84	def add_error(message:)	1✔
85	error = Array.wrap(@dump.event.error)	4✔
86	error << message	4✔
87	@dump.event.error = error.join('; ')	4✔
88	@dump.event.save	4✔
89	end
90
91	def log_record_fixes	1✔
92	log_file = {
93	inv_xml: @inv_xml,	1✔
94	tab_newline: @tab_newline,
95	leader: @leader,
96	composed_chars: @composed_chars,
97	bad_utf8: @bad_utf8
98	}
99	filepath = log_file_name	1✔
100	File.write(filepath, log_file.to_json.to_s)	1✔
101	attach_dump_file(filepath, dump_file_type: :log_file)	1✔
102	end
103
104	def log_file_name	1✔
105	"#{@scsb_file_dir}/fixes_#{@last_dump.to_time.strftime('%Y_%m_%d')}.json"	1✔
106	end
107
108	def set_generated_date	1✔
109	@dump.generated_date = date_strings.map { \|str\| DateTime.parse(str) }.sort.first	1✔
110	end
111
112	def date_strings	1✔
113	@dump.dump_files.map do \|df\|	1✔
114	if df.dump_file_type == 'recap_records_full_metadata'	×
115	File.basename(df.path).split('_')[3]	×
116	else
117	File.basename(df.path).split('_')[2]	×
118	end
119	end
120	end
121
122	def prepare_directory	1✔
123	FileUtils.mkdir_p(@update_directory)	10✔
124	end
125	end
126	end
127	end

pulibrary / bibdata / 6eaae75e-8582-4162-bf7f-32b0a072fade

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous