pulibrary / bibdata / 6a4c4995-c2f1-4e47-a066-5fb67d9777ae

coverage: 91.317% (+0.06%) from 91.26%
6a4c4995-c2f1-4e47-a066-5fb67d9777ae

Pull #2388

circleci

maxkadel 
Make sure CSV for full dump matches group ids exactly
Pull Request #2388: Do not process full dumps with private records

165 of 170 new or added lines in 5 files covered. (97.06%)

3439 of 3766 relevant lines covered (91.32%)

349.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.37
/app/models/scsb/partner_updates/full.rb
1 
module Scsb
1✔
2 
  class PartnerUpdates
1✔
3 
    class Full < Update
1✔
4 
      def initialize(dump:, dump_file_type:, timestamp: DateTime.now.to_time)
1✔
5 
        super
9✔
6 
      end
7 










8



      def process_full_files




1✔






9



        prepare_directory




6✔






10



        download_and_process_full(inst: "NYPL", prefix: 'scsbfull_nypl_')




6✔






11



        download_and_process_full(inst: "CUL", prefix: 'scsbfull_cul_')




5✔






12



        download_and_process_full(inst: "HL", prefix: 'scsbfull_hl_')




5✔






13



        set_generated_date




5✔






14



        log_record_fixes




5✔





15



      end










16













17



      def download_and_process_full(inst:, prefix:)




1✔






18



        return false unless validate_csv(inst:)




16✔





19













20



        matcher = /#{inst}.*\.zip/




8✔






21



        file = download_full_file(matcher)




8✔






22



        if file




8✔






23



          process_partner_updates(files: [file], file_prefix: prefix)




8✔





24



        else











NEW


25



          add_error(message: "No full dump files found matching #{inst}")




×







26



        end










27



      end










28













29



      def validate_csv(inst:)




1✔






30



        matcher = /#{inst}.*\.csv/




18✔






31



        file = download_full_file(matcher)




18✔






32



        matches_expected_collections = false




17✔






33



        if file




17✔






34



          csv = CSV.read(file, headers: true)




12✔






35



          matches_expected_collections = csv["Collection Group Id(s)"].first == '1*2*5*6'




12✔






36



          unless matches_expected_collections




12✔






37



            add_error(message: "Metadata file indicates that dump for #{inst} does not include the correct Group IDs, not processing. Group ids: #{csv['Collection Group Id(s)'].first}")




3✔





38



          end











39



          filename = File.basename(file)




12✔






40



          destination_filepath = "#{@scsb_file_dir}/#{filename}"




12✔






41



          FileUtils.move(file, destination_filepath)




12✔






42



          attach_dump_file(destination_filepath, dump_file_type: :recap_records_full_metadata)




12✔






43



          File.unlink(destination_filepath) if File.exist?(destination_filepath)




12✔





44



        else











45



          add_error(message: "No metadata files found matching #{inst}")




5✔





46



        end











47



        matches_expected_collections




17✔





48



      end










49













50



      def download_full_file(file_filter)




1✔






51



        prefix = ENV['SCSB_S3_PARTNER_FULLS'] || 'data-exports/PUL/MARCXml/Full'




26✔






52



        @s3_bucket.download_recent(prefix:, output_directory: @update_directory, file_filter:)




26✔





53



      end










54



    end










55



  end










56



end
























    

  • 

    •