5cb1e831-5540-4e5a-a704-17755a4eb060

Committed 06 Jun 2024 06:08PM UTC coverage: 91.214% (-0.05%) from 91.26%

Build # 5cb1e831-5540-4e5a-a704-17755a4eb060

Build Type

Pull #2388

circleci

Committed by

maxkadel

Commit Message

Checkpoint - green

- All but one "full only" method moved into Full class

Pull Request Pull Request #2388: Do not process full dumps with private records

Run Details

127 of 139 new or added lines in 4 files covered. (91.37%)

9 existing lines in 1 file now uncovered.

3509 of 3847 relevant lines covered (91.21%)

342.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.5

/app/models/scsb/partner_updates/full.rb

module Scsb
  class PartnerUpdates
    class Full < Update
      attr_accessor :s3_bucket

      def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)
        super
        @s3_bucket = Scsb::S3Bucket.partner_transfer_client
        @scsb_file_dir = ENV['SCSB_FILE_DIR']
      end

      def process_full_files
        prepare_directory
        # full_download = Scsb::PartnerUpdates::Full.new(s3_bucket: @s3_bucket, dump: @dump, dump_file_type: @dump_file_type)
        download_and_process_full(inst: "NYPL", prefix: 'scsbfull_nypl_')
        download_and_process_full(inst: "CUL", prefix: 'scsbfull_cul_')
        download_and_process_full(inst: "HL", prefix: 'scsbfull_hl_')
        set_generated_date
        log_record_fixes
      end

      def validate_csv(inst:)
        matcher = /#{inst}.*\.csv/
        file = download_full_file(matcher)
        includes_private = ''
        if file
          csv = CSV.read(file, headers: true)
          includes_private = csv["Collection Group Id(s)"].first.include?('3')
          add_error(message: "Metadata file indicates that dump for #{inst} includes private records, not processing.") if includes_private
          filename = File.basename(file)
          destination_filepath = "#{@scsb_file_dir}/#{filename}"
          FileUtils.move(file, destination_filepath)
          attach_dump_file(destination_filepath, dump_file_type: :recap_records_full_metadata)
          File.unlink(destination_filepath) if File.exist?(destination_filepath)
        else
          add_error(message: "No metadata files found matching #{inst}")
        end
        !includes_private
      end

      def download_full_file(file_filter)
        prefix = ENV['SCSB_S3_PARTNER_FULLS'] || 'data-exports/PUL/MARCXml/Full'
        @s3_bucket.download_recent(prefix:, output_directory: @update_directory, file_filter:)
      end

      def download_and_process_full(inst:, prefix:)
        # full_download = Scsb::PartnerUpdates::Full.new(s3_bucket: @s3_bucket, dump: @dump)
        return false unless validate_csv(inst:)

        matcher = /#{inst}.*\.zip/
        file = download_full_file(matcher)
        if file
          process_partner_updates(files: [file], file_prefix: prefix)
        else
          add_error(message: "No full dump files found matching #{inst}")
        end
      end
    end
  end
end

1	module Scsb	1✔
2	class PartnerUpdates	1✔
3	class Full < Update	1✔
4	attr_accessor :s3_bucket	1✔
5
6	def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)	1✔
7	super	7✔
8	@s3_bucket = Scsb::S3Bucket.partner_transfer_client	7✔
9	@scsb_file_dir = ENV['SCSB_FILE_DIR']	7✔
10	end
11
12	def process_full_files	1✔
13	prepare_directory	6✔
14	# full_download = Scsb::PartnerUpdates::Full.new(s3_bucket: @s3_bucket, dump: @dump, dump_file_type: @dump_file_type)
15	download_and_process_full(inst: "NYPL", prefix: 'scsbfull_nypl_')	6✔
16	download_and_process_full(inst: "CUL", prefix: 'scsbfull_cul_')	5✔
17	download_and_process_full(inst: "HL", prefix: 'scsbfull_hl_')	5✔
18	set_generated_date	5✔
19	log_record_fixes	5✔
20	end
21
22	def validate_csv(inst:)	1✔
23	matcher = /#{inst}.*\.csv/	16✔
24	file = download_full_file(matcher)	16✔
25	includes_private = ''	15✔
26	if file	15✔
27	csv = CSV.read(file, headers: true)	10✔
28	includes_private = csv["Collection Group Id(s)"].first.include?('3')	10✔
29	add_error(message: "Metadata file indicates that dump for #{inst} includes private records, not processing.") if includes_private	10✔
30	filename = File.basename(file)	10✔
31	destination_filepath = "#{@scsb_file_dir}/#{filename}"	10✔
32	FileUtils.move(file, destination_filepath)	10✔
33	attach_dump_file(destination_filepath, dump_file_type: :recap_records_full_metadata)	10✔
34	File.unlink(destination_filepath) if File.exist?(destination_filepath)	10✔
35	else
36	add_error(message: "No metadata files found matching #{inst}")	5✔
37	end
38	!includes_private	15✔
39	end
40
41	def download_full_file(file_filter)	1✔
42	prefix = ENV['SCSB_S3_PARTNER_FULLS'] \|\| 'data-exports/PUL/MARCXml/Full'	24✔
43	@s3_bucket.download_recent(prefix:, output_directory: @update_directory, file_filter:)	24✔
44	end
45
46	def download_and_process_full(inst:, prefix:)	1✔
47	# full_download = Scsb::PartnerUpdates::Full.new(s3_bucket: @s3_bucket, dump: @dump)
48	return false unless validate_csv(inst:)	16✔
49
50	matcher = /#{inst}.*\.zip/	8✔
51	file = download_full_file(matcher)	8✔
52	if file	8✔
53	process_partner_updates(files: [file], file_prefix: prefix)	8✔
54	else
NEW 55	add_error(message: "No full dump files found matching #{inst}")	×
56	end
57	end
58	end
59	end
60	end

pulibrary / bibdata / 5cb1e831-5540-4e5a-a704-17755a4eb060

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous