cdc23dd4-7656-4660-bfe1-6a9fb1bebcca

module Scsb
  class PartnerUpdates
    class Full < Update
      def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)
        super
      end

      def process_full_files
        prepare_directory
        download_and_process_full(inst: "NYPL", prefix: 'scsbfull_nypl_')
        download_and_process_full(inst: "CUL", prefix: 'scsbfull_cul_')
        download_and_process_full(inst: "HL", prefix: 'scsbfull_hl_')
        set_generated_date
        log_record_fixes
      end

      def download_and_process_full(inst:, prefix:)
        return false unless validate_csv(inst:)

        matcher = /#{inst}.*\.zip/
        file = download_full_file(matcher)
        if file
          process_partner_updates(files: [file], file_prefix: prefix)
        else
          add_error(message: "No full dump files found matching #{inst}")
        end
      end

      def validate_csv(inst:)
        matcher = /#{inst}.*\.csv/
        file = download_full_file(matcher)
        includes_private = ''
        if file
          csv = CSV.read(file, headers: true)
          includes_private = csv["Collection Group Id(s)"].first.include?('3')
          add_error(message: "Metadata file indicates that dump for #{inst} includes private records, not processing.") if includes_private
          filename = File.basename(file)
          destination_filepath = "#{@scsb_file_dir}/#{filename}"
          FileUtils.move(file, destination_filepath)
          attach_dump_file(destination_filepath, dump_file_type: :recap_records_full_metadata)
          File.unlink(destination_filepath) if File.exist?(destination_filepath)
        else
          add_error(message: "No metadata files found matching #{inst}")
        end
        !includes_private
      end

      def download_full_file(file_filter)
        prefix = ENV['SCSB_S3_PARTNER_FULLS'] || 'data-exports/PUL/MARCXml/Full'
        @s3_bucket.download_recent(prefix:, output_directory: @update_directory, file_filter:)
      end
    end
  end
end

1	module Scsb	1✔
2	class PartnerUpdates	1✔
3	class Full < Update	1✔
4	def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)	1✔
5	super	9✔
6	end
7
8	def process_full_files	1✔
9	prepare_directory	6✔
10	download_and_process_full(inst: "NYPL", prefix: 'scsbfull_nypl_')	6✔
11	download_and_process_full(inst: "CUL", prefix: 'scsbfull_cul_')	5✔
12	download_and_process_full(inst: "HL", prefix: 'scsbfull_hl_')	5✔
13	set_generated_date	5✔
14	log_record_fixes	5✔
15	end
16
17	def download_and_process_full(inst:, prefix:)	1✔
18	return false unless validate_csv(inst:)	16✔
19
20	matcher = /#{inst}.*\.zip/	8✔
21	file = download_full_file(matcher)	8✔
22	if file	8✔
23	process_partner_updates(files: [file], file_prefix: prefix)	8✔
24	else
NEW 25	add_error(message: "No full dump files found matching #{inst}")	×
26	end
27	end
28
29	def validate_csv(inst:)	1✔
30	matcher = /#{inst}.*\.csv/	18✔
31	file = download_full_file(matcher)	18✔
32	includes_private = ''	17✔
33	if file	17✔
34	csv = CSV.read(file, headers: true)	12✔
35	includes_private = csv["Collection Group Id(s)"].first.include?('3')	12✔
36	add_error(message: "Metadata file indicates that dump for #{inst} includes private records, not processing.") if includes_private	12✔
37	filename = File.basename(file)	12✔
38	destination_filepath = "#{@scsb_file_dir}/#{filename}"	12✔
39	FileUtils.move(file, destination_filepath)	12✔
40	attach_dump_file(destination_filepath, dump_file_type: :recap_records_full_metadata)	12✔
41	File.unlink(destination_filepath) if File.exist?(destination_filepath)	12✔
42	else
43	add_error(message: "No metadata files found matching #{inst}")	5✔
44	end
45	!includes_private	17✔
46	end
47
48	def download_full_file(file_filter)	1✔
49	prefix = ENV['SCSB_S3_PARTNER_FULLS'] \|\| 'data-exports/PUL/MARCXml/Full'	26✔
50	@s3_bucket.download_recent(prefix:, output_directory: @update_directory, file_filter:)	26✔
51	end
52	end
53	end
54	end

pulibrary / bibdata / cdc23dd4-7656-4660-bfe1-6a9fb1bebcca

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous