166d57e5-7cd2-4a93-abb1-e0b236109bef

Committed 31 Dec 2024 08:55PM UTC coverage: 90.88% (-1.1%) from 91.93%

Build # 166d57e5-7cd2-4a93-abb1-e0b236109bef

Build Type

push

circleci

Committed by

maxkadel

Commit Message

Break on purpose to check retries

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

40 existing lines in 3 files now uncovered.

3458 of 3805 relevant lines covered (90.88%)

365.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.32

/app/models/scsb/partner_updates/update.rb

module Scsb
  class PartnerUpdates
    class Update
      attr_accessor :dump

      def initialize(dump:, dump_file_type:, timestamp:)
        @last_dump = timestamp
        @dump = dump
        @dump_file_type = dump_file_type
        @s3_bucket = Scsb::S3Bucket.partner_transfer_client
        @scsb_file_dir = ENV['SCSB_FILE_DIR']
        @update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
      end

      def process_partner_updates(files:, file_prefix: 'scsb_update_')
        xml_files = []
        files.each do |file|
          xml_files << Scsb::PartnerUpdates::Update.extract_files(file:, update_directory: @update_directory)
        end
        xml_files.flatten.each do |file|
          Scsb::PartnerUpdates::Update.attach_cleaned_dump_file(file:, dump_id: @dump.id, file_prefix:, dump_file_type: @dump_file_type)
        end
      end

      def self.extract_files(file:, update_directory:)
        extracted_files = []
        filename = File.basename(file, '.zip')
        filename.gsub!(/^[^_]+_([0-9]+)_([0-9]+).*$/, '\1_\2')
        file_increment = 1
        Zip::File.open(file) do |zip_file|
          zip_file.each do |entry|
            target = "#{update_directory}/#{filename}_#{file_increment}.xml"
            extracted_files << target
            entry.extract(target)
            file_increment += 1
          end
        end
        File.unlink(file)
        extracted_files
      end

      def self.attach_cleaned_dump_file(file:, dump_id:, file_prefix:, dump_file_type: :recap_records_full)
        scsb_file_dir = ENV['SCSB_FILE_DIR']
        @inv_xml = []
        @tab_newline = []
        @leader = []
        @composed_chars = []
        @bad_utf8 = []
        original_filename = File.basename(file)
        reader = MARC::XMLReader.new(file.to_s, external_encoding: 'UTF-8')
        new_filepath = "#{scsb_file_dir}/#{file_prefix}#{original_filename}"
        writer = MARC::XMLWriter.new(new_filepath)
        reader.each { |record| writer.write(process_record(record)) }
        writer.close
        File.unlink(file) if dump_file_type == :recap_records
        Dump.attach_dump_file(dump_id:, filepath: new_filepath, dump_file_type:)
      end

      def self.process_record(record)
        # DON'T LEAVE THIS IN! SO BROKEN!
        raise if Time.now.utc.sec % 4 == 0
        record = field_delete(['856', '959'], record)
        record.leader[5] = 'c' if record.leader[5].eql?('d')
        if bad_utf8?(record)
          @bad_utf8 << record['001']
          record = bad_utf8_fix(record)
        end
        if invalid_xml_chars?(record)
          @inv_xml << record['001']
          record = invalid_xml_fix(record)
        end
        if tab_newline_char?(record)
          @tab_newline << record['001']
          record = tab_newline_fix(record)
        end
        if leader_errors?(record)
          @leader << record['001']
          record = leaderfix(record)
        end
        if composed_chars_errors?(record)
          @composed_chars << record['001']
          record = composed_chars_normalize(record)
        end
        record = extra_space_fix(record)
        empty_subfield_fix(record)
      end

      def add_error(message:)
        error = Array.wrap(@dump.event.error)
        error << message
        @dump.event.error = error.join('; ')
        @dump.event.save
      end

      def log_record_fixes
        log_file = {
          inv_xml: @inv_xml,
          tab_newline: @tab_newline,
          leader: @leader,
          composed_chars: @composed_chars,
          bad_utf8: @bad_utf8
        }
        filepath = log_file_name
        File.write(filepath, log_file.to_json.to_s)
        Dump.attach_dump_file(dump_id: @dump.id, filepath:, dump_file_type: :log_file)
      end

      def log_file_name
        "#{@scsb_file_dir}/fixes_#{@last_dump.to_time.strftime('%Y_%m_%d')}.json"
      end

      def self.generated_date(dump_id:)
        dump = Dump.find(dump_id)
        dump.generated_date = date_strings(dump:).map { |str| DateTime.parse(str) }.sort.first
      end

      def self.date_strings(dump:)
        dump.dump_files.map do |df|
          if df.dump_file_type == 'recap_records_full_metadata'
            File.basename(df.path).split('_')[3]
          else
            File.basename(df.path).split('_')[2]
          end
        end
      end

      def set_generated_date
        @dump.generated_date = date_strings.map { |str| DateTime.parse(str) }.sort.first
      end

      def date_strings
        @dump.dump_files.map do |df|
          if df.dump_file_type == 'recap_records_full_metadata'
            File.basename(df.path).split('_')[3]
          else
            File.basename(df.path).split('_')[2]
          end
        end
      end

      def prepare_directory
        FileUtils.mkdir_p(@update_directory)
      end
    end
  end
end

1	module Scsb	1✔
2	class PartnerUpdates	1✔
3	class Update	1✔
4	attr_accessor :dump	1✔
5
6	def initialize(dump:, dump_file_type:, timestamp:)	1✔
7	@last_dump = timestamp	11✔
8	@dump = dump	11✔
9	@dump_file_type = dump_file_type	11✔
10	@s3_bucket = Scsb::S3Bucket.partner_transfer_client	11✔
11	@scsb_file_dir = ENV['SCSB_FILE_DIR']	11✔
12	@update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	11✔
13	end
14
15	def process_partner_updates(files:, file_prefix: 'scsb_update_')	1✔
16	xml_files = []	3✔
17	files.each do \|file\|	3✔
18	xml_files << Scsb::PartnerUpdates::Update.extract_files(file:, update_directory: @update_directory)	3✔
19	end
20	xml_files.flatten.each do \|file\|	3✔
21	Scsb::PartnerUpdates::Update.attach_cleaned_dump_file(file:, dump_id: @dump.id, file_prefix:, dump_file_type: @dump_file_type)	3✔
22	end
23	end
24
25	def self.extract_files(file:, update_directory:)	1✔
26	extracted_files = []	15✔
27	filename = File.basename(file, '.zip')	15✔
28	filename.gsub!(/^[^_]+_([0-9]+)_([0-9]+).*$/, '\1_\2')	15✔
29	file_increment = 1	15✔
30	Zip::File.open(file) do \|zip_file\|	15✔
31	zip_file.each do \|entry\|	12✔
32	target = "#{update_directory}/#{filename}_#{file_increment}.xml"	24✔
33	extracted_files << target	24✔
34	entry.extract(target)	24✔
35	file_increment += 1	24✔
36	end
37	end
38	File.unlink(file)	14✔
39	extracted_files	14✔
40	end
41
42	def self.attach_cleaned_dump_file(file:, dump_id:, file_prefix:, dump_file_type: :recap_records_full)	1✔
43	scsb_file_dir = ENV['SCSB_FILE_DIR']	15✔
44	@inv_xml = []	15✔
45	@tab_newline = []	15✔
46	@leader = []	15✔
47	@composed_chars = []	15✔
48	@bad_utf8 = []	15✔
49	original_filename = File.basename(file)	15✔
50	reader = MARC::XMLReader.new(file.to_s, external_encoding: 'UTF-8')	15✔
51	new_filepath = "#{scsb_file_dir}/#{file_prefix}#{original_filename}"	15✔
52	writer = MARC::XMLWriter.new(new_filepath)	15✔
53	reader.each { \|record\| writer.write(process_record(record)) }	36✔
54	writer.close	6✔
55	File.unlink(file) if dump_file_type == :recap_records	6✔
56	Dump.attach_dump_file(dump_id:, filepath: new_filepath, dump_file_type:)	6✔
57	end
58
59	def self.process_record(record)	1✔
60	# DON'T LEAVE THIS IN! SO BROKEN!
61	raise if Time.now.utc.sec % 4 == 0	22✔
62	record = field_delete(['856', '959'], record)	12✔
63	record.leader[5] = 'c' if record.leader[5].eql?('d')	12✔
64	if bad_utf8?(record)	12✔
65	@bad_utf8 << record['001']	×
66	record = bad_utf8_fix(record)	×
67	end
68	if invalid_xml_chars?(record)	12✔
69	@inv_xml << record['001']	×
70	record = invalid_xml_fix(record)	×
71	end
72	if tab_newline_char?(record)	12✔
UNCOV 73	@tab_newline << record['001']	×
UNCOV 74	record = tab_newline_fix(record)	×
75	end
76	if leader_errors?(record)	12✔
77	@leader << record['001']	1✔
78	record = leaderfix(record)	1✔
79	end
80	if composed_chars_errors?(record)	12✔
81	@composed_chars << record['001']	4✔
82	record = composed_chars_normalize(record)	4✔
83	end
84	record = extra_space_fix(record)	12✔
85	empty_subfield_fix(record)	12✔
86	end
87
88	def add_error(message:)	1✔
89	error = Array.wrap(@dump.event.error)	×
90	error << message	×
91	@dump.event.error = error.join('; ')	×
92	@dump.event.save	×
93	end
94
95	def log_record_fixes	1✔
96	log_file = {
97	inv_xml: @inv_xml,	1✔
98	tab_newline: @tab_newline,
99	leader: @leader,
100	composed_chars: @composed_chars,
101	bad_utf8: @bad_utf8
102	}
103	filepath = log_file_name	1✔
104	File.write(filepath, log_file.to_json.to_s)	1✔
105	Dump.attach_dump_file(dump_id: @dump.id, filepath:, dump_file_type: :log_file)	1✔
106	end
107
108	def log_file_name	1✔
109	"#{@scsb_file_dir}/fixes_#{@last_dump.to_time.strftime('%Y_%m_%d')}.json"	1✔
110	end
111
112	def self.generated_date(dump_id:)	1✔
113	dump = Dump.find(dump_id)	1✔
114	dump.generated_date = date_strings(dump:).map { \|str\| DateTime.parse(str) }.sort.first	10✔
115	end
116
117	def self.date_strings(dump:)	1✔
118	dump.dump_files.map do \|df\|	1✔
119	if df.dump_file_type == 'recap_records_full_metadata'	9✔
120	File.basename(df.path).split('_')[3]	3✔
121	else
122	File.basename(df.path).split('_')[2]	6✔
123	end
124	end
125	end
126
127	def set_generated_date	1✔
128	@dump.generated_date = date_strings.map { \|str\| DateTime.parse(str) }.sort.first	1✔
129	end
130
131	def date_strings	1✔
132	@dump.dump_files.map do \|df\|	1✔
UNCOV 133	if df.dump_file_type == 'recap_records_full_metadata'	×
UNCOV 134	File.basename(df.path).split('_')[3]	×
135	else
UNCOV 136	File.basename(df.path).split('_')[2]	×
137	end
138	end
139	end
140
141	def prepare_directory	1✔
142	FileUtils.mkdir_p(@update_directory)	8✔
143	end
144	end
145	end
146	end

pulibrary / bibdata / 166d57e5-7cd2-4a93-abb1-e0b236109bef

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous