5cb1e831-5540-4e5a-a704-17755a4eb060

Committed 06 Jun 2024 06:08PM UTC coverage: 91.214% (-0.05%) from 91.26%

Build # 5cb1e831-5540-4e5a-a704-17755a4eb060

Build Type

Pull #2388

circleci

Committed by

maxkadel

Commit Message

Checkpoint - green

- All but one "full only" method moved into Full class

Pull Request Pull Request #2388: Do not process full dumps with private records

Run Details

127 of 139 new or added lines in 4 files covered. (91.37%)

9 existing lines in 1 file now uncovered.

3509 of 3847 relevant lines covered (91.21%)

342.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.94

/app/models/scsb/partner_updates/update.rb

module Scsb
  class PartnerUpdates
    class Update
      attr_accessor :dump
      def initialize(dump:, dump_file_type:, timestamp:)
        @dump = dump
        @dump_file_type = dump_file_type
        @last_dump = timestamp
        @scsb_file_dir = ENV['SCSB_FILE_DIR']
        @update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] || '/tmp/updates'
        @inv_xml = []
        @tab_newline = []
        @leader = []
        @composed_chars = []
        @bad_utf8 = []
      end

      def attach_dump_file(filepath, dump_file_type: nil)
        dump_file_type ||= @dump_file_type
        df = DumpFile.create(dump_file_type:, path: filepath)
        df.zip
        df.save
        @dump.dump_files << df
        @dump.save
      end

      def process_partner_updates(files:, file_prefix: 'scsb_update_')
        xml_files = []
        files.each do |file|
          filename = File.basename(file, '.zip')
          filename.gsub!(/^[^_]+_([0-9]+)_([0-9]+).*$/, '\1_\2')
          file_increment = 1
          Zip::File.open(file) do |zip_file|
            zip_file.each do |entry|
              target = "#{@update_directory}/#{filename}_#{file_increment}.xml"
              xml_files << target
              entry.extract(target)
              file_increment += 1
            end
          end
          File.unlink(file)
        end
        xml_files.each do |file|
          filename = File.basename(file)
          reader = MARC::XMLReader.new(file.to_s, external_encoding: 'UTF-8')
          filepath = "#{@scsb_file_dir}/#{file_prefix}#{filename}"
          writer = MARC::XMLWriter.new(filepath)
          reader.each { |record| writer.write(process_record(record)) }
          writer.close
          File.unlink(file)
          attach_dump_file(filepath)
        end
      end

      def process_record(record)
        record = field_delete(['856', '959'], record)
        record.leader[5] = 'c' if record.leader[5].eql?('d')
        if bad_utf8?(record)
          @bad_utf8 << record['001']
          record = bad_utf8_fix(record)
        end
        if invalid_xml_chars?(record)
          @inv_xml << record['001']
          record = invalid_xml_fix(record)
        end
        if tab_newline_char?(record)
          @tab_newline << record['001']
          record = tab_newline_fix(record)
        end
        if leader_errors?(record)
          @leader << record['001']
          record = leaderfix(record)
        end
        if composed_chars_errors?(record)
          @composed_chars << record['001']
          record = composed_chars_normalize(record)
        end
        record = extra_space_fix(record)
        empty_subfield_fix(record)
      end

      def add_error(message:)
        error = Array.wrap(@dump.event.error)
        error << message
        @dump.event.error = error.join("; ")
        @dump.event.save
      end

      def log_record_fixes
        log_file = {
          inv_xml: @inv_xml,
          tab_newline: @tab_newline,
          leader: @leader,
          composed_chars: @composed_chars,
          bad_utf8: @bad_utf8
        }
        filepath = log_file_name
        File.write(filepath, log_file.to_json.to_s)
        attach_dump_file(filepath, dump_file_type: :log_file)
      end

      def log_file_name
        "#{@scsb_file_dir}/fixes_#{@last_dump.strftime('%Y_%m_%d')}.json"
      end

      def set_generated_date
        @dump.generated_date = date_strings.map { |str| DateTime.parse(str) }.sort.first
      end

      def date_strings
        @dump.dump_files.map do |df|
          if df.dump_file_type == "recap_records_full_metadata"
            File.basename(df.path).split("_")[3]
          else
            File.basename(df.path).split("_")[2]
          end
        end
      end

      def prepare_directory
        FileUtils.mkdir_p(@update_directory)
      end
    end
  end
end

1	module Scsb	1✔
2	class PartnerUpdates	1✔
3	class Update	1✔
4	attr_accessor :dump	1✔
5	def initialize(dump:, dump_file_type:, timestamp:)	1✔
6	@dump = dump	8✔
7	@dump_file_type = dump_file_type	8✔
8	@last_dump = timestamp	8✔
9	@scsb_file_dir = ENV['SCSB_FILE_DIR']	8✔
10	@update_directory = ENV['SCSB_PARTNER_UPDATE_DIRECTORY'] \|\| '/tmp/updates'	8✔
11	@inv_xml = []	8✔
12	@tab_newline = []	8✔
13	@leader = []	8✔
14	@composed_chars = []	8✔
15	@bad_utf8 = []	8✔
16	end
17
18	def attach_dump_file(filepath, dump_file_type: nil)	1✔
19	dump_file_type \|\|= @dump_file_type	32✔
20	df = DumpFile.create(dump_file_type:, path: filepath)	32✔
21	df.zip	32✔
22	df.save	31✔
23	@dump.dump_files << df	31✔
24	@dump.save	31✔
25	end
26
27	def process_partner_updates(files:, file_prefix: 'scsb_update_')	1✔
28	xml_files = []	8✔
29	files.each do \|file\|	8✔
30	filename = File.basename(file, '.zip')	8✔
31	filename.gsub!(/^[^_]+_([0-9]+)_([0-9]+).*$/, '\1_\2')	8✔
32	file_increment = 1	8✔
33	Zip::File.open(file) do \|zip_file\|	8✔
34	zip_file.each do \|entry\|	8✔
35	target = "#{@update_directory}/#{filename}_#{file_increment}.xml"	16✔
36	xml_files << target	16✔
37	entry.extract(target)	16✔
38	file_increment += 1	16✔
39	end
40	end
41	File.unlink(file)	8✔
42	end
43	xml_files.each do \|file\|	8✔
44	filename = File.basename(file)	16✔
45	reader = MARC::XMLReader.new(file.to_s, external_encoding: 'UTF-8')	16✔
46	filepath = "#{@scsb_file_dir}/#{file_prefix}#{filename}"	16✔
47	writer = MARC::XMLWriter.new(filepath)	16✔
48	reader.each { \|record\| writer.write(process_record(record)) }	48✔
49	writer.close	16✔
50	File.unlink(file)	16✔
51	attach_dump_file(filepath)	16✔
52	end
53	end
54
55	def process_record(record)	1✔
56	record = field_delete(['856', '959'], record)	32✔
57	record.leader[5] = 'c' if record.leader[5].eql?('d')	32✔
58	if bad_utf8?(record)	32✔
NEW 59	@bad_utf8 << record['001']	×
NEW 60	record = bad_utf8_fix(record)	×
61	end
62	if invalid_xml_chars?(record)	32✔
NEW 63	@inv_xml << record['001']	×
NEW 64	record = invalid_xml_fix(record)	×
65	end
66	if tab_newline_char?(record)	32✔
NEW 67	@tab_newline << record['001']	×
NEW 68	record = tab_newline_fix(record)	×
69	end
70	if leader_errors?(record)	32✔
71	@leader << record['001']	3✔
72	record = leaderfix(record)	3✔
73	end
74	if composed_chars_errors?(record)	32✔
75	@composed_chars << record['001']	12✔
76	record = composed_chars_normalize(record)	12✔
77	end
78	record = extra_space_fix(record)	32✔
79	empty_subfield_fix(record)	32✔
80	end
81
82	def add_error(message:)	1✔
83	error = Array.wrap(@dump.event.error)	7✔
84	error << message	7✔
85	@dump.event.error = error.join("; ")	7✔
86	@dump.event.save	7✔
87	end
88
89	def log_record_fixes	1✔
90	log_file = {
91	inv_xml: @inv_xml,	5✔
92	tab_newline: @tab_newline,
93	leader: @leader,
94	composed_chars: @composed_chars,
95	bad_utf8: @bad_utf8
96	}
97	filepath = log_file_name	5✔
98	File.write(filepath, log_file.to_json.to_s)	5✔
99	attach_dump_file(filepath, dump_file_type: :log_file)	5✔
100	end
101
102	def log_file_name	1✔
103	"#{@scsb_file_dir}/fixes_#{@last_dump.strftime('%Y_%m_%d')}.json"	5✔
104	end
105
106	def set_generated_date	1✔
107	@dump.generated_date = date_strings.map { \|str\| DateTime.parse(str) }.sort.first	31✔
108	end
109
110	def date_strings	1✔
111	@dump.dump_files.map do \|df\|	5✔
112	if df.dump_file_type == "recap_records_full_metadata"	26✔
113	File.basename(df.path).split("_")[3]	10✔
114	else
115	File.basename(df.path).split("_")[2]	16✔
116	end
117	end
118	end
119
120	def prepare_directory	1✔
121	FileUtils.mkdir_p(@update_directory)	6✔
122	end
123	end
124	end
125	end

pulibrary / bibdata / 5cb1e831-5540-4e5a-a704-17755a4eb060

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous