ebd2715e-56b5-40ce-841f-3d6e105293d4

Committed 13 Nov 2024 02:51PM UTC coverage: 80.641% (+0.9%) from 79.778%

Build # ebd2715e-56b5-40ce-841f-3d6e105293d4

Build Type

Pull #132

circleci

Committed by

mzelesky

Commit Message

finish testing coverage of empty_subfields? method

Pull Request Pull Request #132: [WIP] test extra space methods

Run Details

4 of 29 new or added lines in 1 file covered. (13.79%)

62 existing lines in 1 file now uncovered.

1812 of 2247 relevant lines covered (80.64%)

3.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.21

/lib/marc_cleanup/variable_fields.rb

# frozen_string_literal: true

module MarcCleanup
  ### Remove non-numerical strings and append a new 020$q with the string
  def new_020_q(record)
    record.fields('020').each do |f020|
      f020.subfields.each do |subfield|
        next unless subfield.code == 'a'
        isbn_parts = /^\s*([\d\-]+)\s*(\(.*?\))\s*$/.match(subfield.value)
        next if isbn_parts.nil?
        subfield.value = isbn_parts[1]
        f020.append(MARC::Subfield.new('q', isbn_parts[2]))
      end
    end
    record
  end

  ### Convert ISBN-10 to ISBN-13
  def isbn10_to_13(isbn)
    stem = isbn[0..8]
    return nil if stem =~ /\D/

    existing_check = isbn[9]
    return nil if existing_check && existing_check != checkdigit_10(stem)

    main = ISBN13PREFIX + stem
    checkdigit = checkdigit_13(main)
    main + checkdigit
  end

  ### Calculate check digit for ISBN-10
  def checkdigit_10(stem)
    int_index = 0
    int_sum = 0
    stem.each_char do |digit|
      int_sum += digit.to_i * (10 - int_index)
      int_index += 1
    end
    mod = (11 - (int_sum % 11)) % 11
    mod == 10 ? 'X' : mod.to_s
  end

  ### Calculate check digit for ISBN-13
  def checkdigit_13(stem)
    int_index = 0
    int_sum = 0
    stem.each_char do |digit|
      int_sum += int_index.even? ? digit.to_i : digit.to_i * 3
      int_index += 1
    end
    ((10 - (int_sum % 10)) % 10).to_s
  end

  ### Normalize ISBN-13
  def isbn13_normalize(raw_isbn)
    int_sum = 0
    stem = raw_isbn[0..11]
    return nil if stem =~ /\D/

    int_index = 0
    stem.each_char do |digit|
      int_sum += int_index.even? ? digit.to_i : digit.to_i * 3
      int_index += 1
    end
    checkdigit = checkdigit_13(stem)
    return nil if raw_isbn[12] && raw_isbn[12] != checkdigit

    stem + checkdigit
  end

  ### Normalize any given string that is supposed to include an ISBN
  def isbn_normalize(isbn)
    return nil unless isbn

    raw_isbn = isbn.dup
    raw_isbn.delete!('-')
    raw_isbn.delete!('\\')
    raw_isbn.gsub!(/\([^\)]*\)/, '')
    raw_isbn.gsub!(/^(.*)\$c.*$/, '\1')
    raw_isbn.gsub!(/^(.*)\$q.*$/, '\1')
    raw_isbn.gsub!(/^\D+([0-9].*)$/, '\1')
    if raw_isbn =~ /^978/
      raw_isbn.gsub!(/^(978[0-9 ]+).*$/, '\1')
      raw_isbn.delete!(' ')
    else
      raw_isbn.gsub!(/([0-9])\s*([0-9]{4})\s*([0-9]{4})\s*([0-9xX]).*$/, '\1\2\3\4')
    end
    raw_isbn.gsub!(/^([0-9]{9,13}[xX]?)[^0-9xX].*$/, '\1')
    raw_isbn.gsub!(/^([0-9]+?)\D.*$/, '\1')
    if raw_isbn.length > 6 && raw_isbn.length < 9 && raw_isbn =~ /^[0-9]+$/
      raw_isbn = raw_isbn.ljust(9, '0')
    end
    valid_lengths = [9, 10, 12, 13] # ISBN10 and ISBN13 with/out check digits
    return nil unless valid_lengths.include? raw_isbn.length

    if raw_isbn.length < 12
      isbn10_to_13(raw_isbn)
    else
      isbn13_normalize(raw_isbn)
    end
  end

  ### If the ISBN is invalid, change the subfield code to z
  ### Otherwise, replace ISBN with normalized ISBN
  def move_invalid_isbn(record)
    record.fields('020').each do |f020|
      f020.subfields.each do |subfield|
        next unless subfield.code == 'a'
        isbn = subfield.value
        normalized_isbn = isbn_normalize(isbn)
        if normalized_isbn
          subfield.value = normalized_isbn
        else
          subfield.code = 'z'
        end
      end
    end
    record
  end

  # check the 041 field for errors
  # 041 is a language code
  def f041_errors?(record)
    f041 = record.fields('041')
    return false if f041.empty?

    f041.each do |field|
      field.subfields.each do |subfield|
        val = subfield.value
        return true if (val.size > 3) && (val.size % 3).zero?
      end
    end
    false
  end

  # http://www.loc.gov/standards/valuelist/marcauthen.html
  def auth_codes_f042
    %w[
      anuc croatica dc dhca dlr
      gamma gils gnd1 gnd2 gnd3 gnd4 gnd5 gnd6 gnd7 gndz isds/c issnuk
      lacderived lc lcac lccopycat lccopycat-nm lcd lcderive
      lchlas lcllh lcnccp lcnitrate lcnuc lcode
      msc natgaz nbr nlc nlmcopyc norbibl nsdp nst ntccf nznb
      pcc premarc reveal sanb scipio toknb
      ukblcatcopy ukblderived ukblproject ukblsr ukscp
      xisds/c xissnuk xlc xnlc xnsdp
    ]
  end

  def auth_code_error?(record)
    return false unless record['042']
    return true if record.fields('042').size > 1

    record['042'].subfields.each do |subfield|
      next if subfield.code != 'a'
      return true unless auth_codes_f042.include?(subfield.value)
    end
    false
  end

  def invalid_indicators?(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)
      return true unless field.indicator1.match?(/^[0-9 ]$/)
      return true unless field.indicator2.match?(/^[0-9 ]$/)
    end
    false
  end

  def invalid_subfield_code?(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      field.subfields.each do |subfield|
        return true unless subfield.code =~ /^[0-9a-z]$/
      end
    end
    false
  end

  def empty_subfields?(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      field.subfields.each do |subfield|
        return true if subfield.value =~ /^[[:blank:]]*$/
      end
    end
    false
  end

  def extra_spaces?(record)
    blank_regex = /^.*[[:blank:]]{2,}.*$|^.*[[:blank:]]+$|^[[:blank:]]+(.*)$/
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag != '010'

      case field.tag
      when /[1-469]..|0[2-9].|01[1-9]|7[0-5].|5[0-24-9].|53[0-24-9]/
        field.subfields.each do |subfield|
          return true if subfield.value =~ blank_regex
        end
      when '533'
        field.subfields.each do |subfield|
          next if subfield.code == '7'

          return true if subfield.value =~ blank_regex
        end
      when /7[6-8]./
        field.subfields.each do |subfield|
          next unless subfield.code =~ /[a-v3-8]/

          return true if subfield.value =~ blank_regex
        end
      when /8../
        field.subfields.each do |subfield|
          next unless subfield.code =~ /[^w7]/

          return true if subfield.value =~ blank_regex
        end
      end
    end
    false
  end

  def extra_space_gsub(string)
    string.gsub!(/([[:blank:]]){2,}/, '\1')
    string.gsub!(/^(.*)[[:blank:]]+$/, '\1')
    string.gsub(/^[[:blank:]]+(.*)$/, '\1')
  end

  ### Remove extra spaces from all fields that are not positionally defined
  def extra_space_fix(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField && field.tag != '010'

      field_index = record.fields.index(field)
      curr_subfield = -1
      case field.tag
      when /^[1-469]..|0[2-9].|01[1-9]|7[0-5].|5[0-24-9].|53[0-24-9]/
        field.subfields.each do |subfield|
          curr_subfield += 1
          next if subfield.value.nil?

          record.fields[field_index].subfields[curr_subfield].value = extra_space_gsub(subfield.value)
        end
      when '533'
        field.subfields.each do |subfield|
          curr_subfield += 1
          next if subfield.code == '7' || subfield.value.nil?

          record.fields[field_index].subfields[curr_subfield].value = extra_space_gsub(subfield.value)
        end
      when /^7[6-8]./
        field.subfields.each do |subfield|
          curr_subfield += 1
          next if subfield.code =~ /[^a-v3-8]/ || subfield.value.nil?

          record.fields[field_index].subfields[curr_subfield].value = extra_space_gsub(subfield.value)
        end
      when /^8../
        field.subfields.each do |subfield|
          curr_subfield += 1
          next if %w[w 7].include?(subfield.code) || subfield.value.nil?

          record.fields[field_index].subfields[curr_subfield].value = extra_space_gsub(subfield.value)
        end
      end
    end
    record
  end

  def multiple_no_040?(record)
    record.fields('040').size != 1
  end

  def multiple_no_040b?(record)
    f040 = record.fields('040')
    return true if f040.size != 1

    f040 = f040.first
    b040 = f040.subfields.select { |subfield| subfield.code == 'b' }
    return true if b040.size != 1

    b040.first.value.match?(/^\s*$/)
  end

  def f046_errors?(record)
    subf_codes = %w[b c d e]
    subf_a_values = %w[r s p t x q n i k r m t x n]
    f046 = record.fields('046')
    return false if f046.empty?

    f046.each do |field|
      codes = field.subfields.map(&:code)
      return true if field['a'] && !subf_a_values.include?(field['a'])
      return true if field['a'].nil? && (subf_codes & codes).size.positive?
    end
    false
  end

  def multiple_no_245?(record)
    record.fields('245').size != 1
  end

  def f245_subfield_errors?(record)
    fields = record.fields('245')
    return true if fields.empty?

    fields.each do |field|
      subfields = field.subfields.map(&:code)
      return true if subfields.count('a') != 1
      return true if subfields.count('b') > 1
      return true if subfields.count('c') > 1
    end
    false
  end

  def missing_040c?(record)
    return true unless record['040'] && record['040']['c']

    false
  end

  def pair_880_errors?(record)
    pair_880s = []
    linked_fields = []
    return false unless record['880']

    record.fields.each do |field|
      return true if field.tag == '880' && field['6'].nil?

      next unless field.tag =~ /^[0-8]..$/ && field.class == MARC::DataField && field['6']

      if field.tag == '880'
        pair_880s << field['6'].gsub(/^([0-9]{3}-[0-9]{2}).*$/, '\1')
      else
        linked_fields << "#{field.tag}-#{field['6'].gsub(/^880-([0-9]{2}).*$/, '\1').chomp}"
      end
    end
    pair_880s.delete_if { |x| x =~ /^.*-00/ }
    return true if pair_880s.uniq != pair_880s || pair_880s.uniq.sort != linked_fields.uniq.sort

    false
  end

  def has_130_240?(record)
    (%w[130 240] - record.tags).empty?
  end

  def multiple_1xx?(record)
    record.fields('100'..'199').size > 1
  end

  def relator_chars?(record)
    record.fields.each do |field|
      if field.tag =~ /[17][01]0/
        field.subfields.each do |subfield|
          next unless subfield.code == 'e' && subfield.value =~ /.*[^a-z, \.].*/

          return true
        end
      elsif field.tag =~ /[17]11/
        field.subfields.each do |subfield|
          next unless subfield.code == 'j' && subfield.value =~ /.*[^a-z, \.].*/

          return true
        end
      end
    end
    false
  end

  def x00_subfq?(record)
    record.fields(%w[100 600 700 800]).each do |field|
      field.subfields.each do |subfield|
        next unless subfield.code == 'q' && subfield.value =~ /^(?!\([^\)]*\))$/

        return true
      end
    end
    false
  end

  def no_comma_x00?(record)
    record.fields(%w[100 600 700 800]).each do |field|
      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      subfx_d_index = code_array.index(/.d/)
      return true if subfx_d_index && field.subfields[subfx_d_index].value =~ /^.*[^,]$/
    end
    false
  end

  def relator_comma?(record)
    comma_regex = /^.*[^,]$/
    record.fields.each do |field|
      next unless field.tag =~ /[17][01][01]/

      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      if field.tag =~ /[17][01]0/
        subfx_e_index = code_array.index(/.e/)
        return true if subfx_e_index && field.subfields[subfx_e_index].value =~ comma_regex
      elsif field.tag =~ /[17]11/
        subfx_j_index = code_array.index(/.j/)
        return true if subfx_j_index && field.subfields[subfx_j_index].value =~ comma_regex
      end
    end
    false
  end

  def heading_end_punct?(record)
    punct_regex = /.*[^"\).\!\?\-]$/
    record.fields.each do |field|
      next unless field.tag =~ /^[167][0-5].$/ && field.indicator2 =~ /[^47]/

      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      last_heading_subfield_index = code_array.index(/[a-vx-z8][^a-vx-z8]*$/)
      return true if last_heading_subfield_index && field.subfields[last_heading_subfield_index].value =~ punct_regex
    end
    false
  end

  def lowercase_headings?(record)
    record.fields.each do |field|
      next unless field.tag =~ /[1678]../

      return true if field['a'] =~ /^[a-z]{3,}/
    end
    false
  end

  def subf_0_uri?(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag =~ /^[^9]/ && field['0']

      field.subfields.each do |subfield|
        return true if subfield.code == '0' && subfield.value =~ /^\(uri\)/
      end
    end
    false
  end

  def bad_uri?(record)
    target_fields = record.fields('856')
    return false if target_fields.empty?

    target_fields.each do |field|
      next unless field['u']

      field.subfields.each do |subfield|
        next unless subfield.code == 'u'

        string = subfield.value
        return true unless URI.escape(URI.unescape(string).scrub) == string
      end
    end
    false
  end

  ### Normalize to the NFC (combined) form of diacritics for characters with
  #     Arabic diacritics; normalize to NFD for characters below U+622 and
  #     between U+1E00 and U+2A28
  def composed_chars_normalize(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField

      field_index = record.fields.index(field)
      curr_subfield = 0
      field.subfields.each do |subfield|
        fixed_subfield = ''
        prevalue = subfield.value
        if prevalue =~ /^.*[\u0653\u0654\u0655].*$/
          prevalue = prevalue.unicode_normalize(:nfc)
        end
        prevalue.each_codepoint do |c|
          char = c.chr(Encoding::UTF_8)
          char.unicode_normalize!(:nfd) if c < 1570 || (7_680..10_792).cover?(c)
          fixed_subfield << char
        end
        record.fields[field_index].subfields[curr_subfield].value = fixed_subfield
        curr_subfield += 1
      end
    end
    record
  end

  ### Replace empty indicators with a space;
  ###   scrub indicators with bad UTF-8;
  ###   The ruby-marc gem converts nil subfields to spaces
  def empty_indicator_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      ind1_value = field.indicator1.dup
      ind1_value.scrub!('')
      field.indicator1 = ' ' if ind1_value.empty?
      ind2_value = field.indicator2.dup
      ind2_value.scrub!('')
      field.indicator2 = ' ' if ind2_value.empty?
    end
    record
  end

  ### Remove empty subfields from DataFields
  def empty_subfield_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      field.subfields.delete_if { |subfield| subfield.value.nil? || subfield.value.empty? }
    end
    record.fields.delete_if { |field| field.instance_of?(MARC::DataField) && field.subfields.empty? }
    record
  end

  ### Remove the (uri) prefix from subfield 0s
  def subf_0_uri_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag[0] != '9' && field['0']

      field.subfields.each do |subfield|
        next unless subfield.code == '0' && subfield.value =~ /^\(uri\)/

        subfield.value = subfield.value.dup.delete_prefix('(uri)')
      end
    end
    record
  end

  ### Escape URIs
  def uri_escape(record)
    target_fields = record.fields('856')
    return record if target_fields.empty?

    fixed_record = record
    target_fields.each do |field|
      next unless field['u']

      field_index = fixed_record.fields.index(field)
      field.subfields.each do |subfield|
        next unless subfield.code == 'u'

        subfield_index = field.subfields.index(subfield)
        string = subfield.value
        fixed_string = URI.escape(URI.unescape(string).scrub)
        fixed_record.fields[field_index].subfields[subfield_index].value = fixed_string
      end
    end
    fixed_record
  end

  ### Make the 040 $b 'eng' if it doesn't have a value
  def fix_040b(record)
    return record unless record.fields('040').size == 1

    f040 = record['040']
    field_index = record.fields.index(f040)
    b040 = f040.subfields.select { |subfield| subfield.code == 'b' }
    return record unless b040.empty?

    subf_codes = f040.subfields.map(&:code)
    subf_index = if f040['a']
                   (subf_codes.index { |i| i == 'a' }) + 1
                 else
                   0
                 end
    subf_b = MARC::Subfield.new('b', 'eng')
    record.fields[field_index].subfields.insert(subf_index, subf_b)
    record
  end

  ### Split up subfields that contain multiple 3-letter language codes
  def fix_f041(record)
    f041 = record.fields('041')
    return record if f041.empty?

    f041.each do |field|
      f_index = record.fields.index(field)
      new_field = MARC::DataField.new('041', field.indicator1, field.indicator2)
      field.subfields.each do |subfield|
        code = subfield.code
        val = subfield.value
        if (val.size % 3).zero?
          langs = val.scan(/.../)
          langs.each do |lang|
            new_field.append(MARC::Subfield.new(code, lang))
          end
        else
          new_field.append(MARC::Subfield.new(code, val))
        end
      end
      record.fields[f_index] = new_field
    end
    record
  end

  ### Removes text from the beginning of a subfield
  ### An array of hashes of the format { field:, subfields: } will be passed
  ###   in the targets: symbol
  ###   subfield: is an array of subfield codes
  def remove_prefix_from_subfield(record:, targets:, string:)
    targets.each do |target|
      record.fields(target[:field]).each do |field|
        field.subfields.each do |subfield|
          next unless target[:subfields].include?(subfield.code)

          subfield.value = subfield.value.dup.delete_prefix(string)
        end
      end
    end
    record
  end

  ### Adds text to the beginning of a subfield
  ### An array of hashes of the format { field:, subfields: } will be passed
  ###   in the targets: symbol
  ###   subfield: is an array of subfield codes
  def add_prefix_to_subfield(record:, targets:, string:)
    targets.each do |target|
      record.fields(target[:field]).each do |field|
        field.subfields.each do |subfield|
          next unless target[:subfields].include?(subfield.code)

          subfield.value = subfield.value.dup.prepend(string)
        end
      end
    end
    record
  end

  ### Sort subfields for target fields with an arbitrary order
  def subfield_sort(record, target_tags, order_array = nil)
    target_fields = record.fields.select { |f| target_tags.include?(f.tag) }
    return record if target_fields.empty?

    target_fields.each do |field|
      next unless field.class == MARC::DataField

      orig_codes = field.subfields.map { |subfield| subfield.code }.uniq.sort
      order_array = orig_codes if order_array.nil?
      new_subfields = []
      order_array.each do |code|
        next unless orig_codes.include?(code)

        target_subf = field.subfields.select { |subfield| subfield.code == code }
        target_subf.each { |subfield| new_subfields << subfield }
      end
      rem_subfields = field.subfields.select { |subf| !order_array.include?(subf.code) }
      rem_subfields.each do |subfield|
        new_subfields << subfield
      end
      field.subfields = new_subfields
    end
    record
  end
end

1	# frozen_string_literal: true
2
3	module MarcCleanup	1✔
4	### Remove non-numerical strings and append a new 020$q with the string
5	def new_020_q(record)	1✔
6	record.fields('020').each do \|f020\|	1✔
7	f020.subfields.each do \|subfield\|	1✔
8	next unless subfield.code == 'a'	2✔
9	isbn_parts = /^\s([\d\-]+)\s(\(.?\))\s$/.match(subfield.value)	1✔
10	next if isbn_parts.nil?	1✔
11	subfield.value = isbn_parts[1]	1✔
12	f020.append(MARC::Subfield.new('q', isbn_parts[2]))	1✔
13	end
14	end
15	record	1✔
16	end
17
18	### Convert ISBN-10 to ISBN-13
19	def isbn10_to_13(isbn)	1✔
20	stem = isbn[0..8]	2✔
21	return nil if stem =~ /\D/	2✔
22
23	existing_check = isbn[9]	2✔
24	return nil if existing_check && existing_check != checkdigit_10(stem)	2✔
25
26	main = ISBN13PREFIX + stem	2✔
27	checkdigit = checkdigit_13(main)	2✔
28	main + checkdigit	2✔
29	end
30
31	### Calculate check digit for ISBN-10
32	def checkdigit_10(stem)	1✔
33	int_index = 0	1✔
34	int_sum = 0	1✔
35	stem.each_char do \|digit\|	1✔
36	int_sum += digit.to_i * (10 - int_index)	9✔
37	int_index += 1	9✔
38	end
39	mod = (11 - (int_sum % 11)) % 11	1✔
40	mod == 10 ? 'X' : mod.to_s	1✔
41	end
42
43	### Calculate check digit for ISBN-13
44	def checkdigit_13(stem)	1✔
45	int_index = 0	4✔
46	int_sum = 0	4✔
47	stem.each_char do \|digit\|	4✔
48	int_sum += int_index.even? ? digit.to_i : digit.to_i * 3	48✔
49	int_index += 1	48✔
50	end
51	((10 - (int_sum % 10)) % 10).to_s	4✔
52	end
53
54	### Normalize ISBN-13
55	def isbn13_normalize(raw_isbn)	1✔
56	int_sum = 0	2✔
57	stem = raw_isbn[0..11]	2✔
58	return nil if stem =~ /\D/	2✔
59
60	int_index = 0	2✔
61	stem.each_char do \|digit\|	2✔
62	int_sum += int_index.even? ? digit.to_i : digit.to_i * 3	24✔
63	int_index += 1	24✔
64	end
65	checkdigit = checkdigit_13(stem)	2✔
66	return nil if raw_isbn[12] && raw_isbn[12] != checkdigit	2✔
67
68	stem + checkdigit	1✔
69	end
70
71	### Normalize any given string that is supposed to include an ISBN
72	def isbn_normalize(isbn)	1✔
73	return nil unless isbn	4✔
74
75	raw_isbn = isbn.dup	4✔
76	raw_isbn.delete!('-')	4✔
77	raw_isbn.delete!('\\')	4✔
78	raw_isbn.gsub!(/\([^\)]*\)/, '')	4✔
79	raw_isbn.gsub!(/^(.)\$c.$/, '\1')	4✔
80	raw_isbn.gsub!(/^(.)\$q.$/, '\1')	4✔
81	raw_isbn.gsub!(/^\D+([0-9].*)$/, '\1')	4✔
82	if raw_isbn =~ /^978/	4✔
83	raw_isbn.gsub!(/^(978[0-9 ]+).*$/, '\1')	2✔
84	raw_isbn.delete!(' ')	2✔
85	else
86	raw_isbn.gsub!(/([0-9])\s([0-9]{4})\s([0-9]{4})\s([0-9xX]).$/, '\1\2\3\4')	2✔
87	end
88	raw_isbn.gsub!(/^([0-9]{9,13}[xX]?)[^0-9xX].*$/, '\1')	4✔
89	raw_isbn.gsub!(/^([0-9]+?)\D.*$/, '\1')	4✔
90	if raw_isbn.length > 6 && raw_isbn.length < 9 && raw_isbn =~ /^[0-9]+$/	4✔
91	raw_isbn = raw_isbn.ljust(9, '0')	1✔
92	end
93	valid_lengths = [9, 10, 12, 13] # ISBN10 and ISBN13 with/out check digits	4✔
94	return nil unless valid_lengths.include? raw_isbn.length	4✔
95
96	if raw_isbn.length < 12	4✔
97	isbn10_to_13(raw_isbn)	2✔
98	else
99	isbn13_normalize(raw_isbn)	2✔
100	end
101	end
102
103	### If the ISBN is invalid, change the subfield code to z
104	### Otherwise, replace ISBN with normalized ISBN
105	def move_invalid_isbn(record)	1✔
106	record.fields('020').each do \|f020\|	4✔
107	f020.subfields.each do \|subfield\|	4✔
108	next unless subfield.code == 'a'	4✔
109	isbn = subfield.value	4✔
110	normalized_isbn = isbn_normalize(isbn)	4✔
111	if normalized_isbn	4✔
112	subfield.value = normalized_isbn	3✔
113	else
114	subfield.code = 'z'	1✔
115	end
116	end
117	end
118	record	4✔
119	end
120
121	# check the 041 field for errors
122	# 041 is a language code
123	def f041_errors?(record)	1✔
124	f041 = record.fields('041')	1✔
125	return false if f041.empty?	1✔
126
127	f041.each do \|field\|	1✔
128	field.subfields.each do \|subfield\|	1✔
129	val = subfield.value	1✔
130	return true if (val.size > 3) && (val.size % 3).zero?	1✔
131	end
132	end
133	false	1✔
134	end
135
136	# http://www.loc.gov/standards/valuelist/marcauthen.html
137	def auth_codes_f042	1✔
138	%w[	3✔
139	anuc croatica dc dhca dlr
140	gamma gils gnd1 gnd2 gnd3 gnd4 gnd5 gnd6 gnd7 gndz isds/c issnuk
141	lacderived lc lcac lccopycat lccopycat-nm lcd lcderive
142	lchlas lcllh lcnccp lcnitrate lcnuc lcode
143	msc natgaz nbr nlc nlmcopyc norbibl nsdp nst ntccf nznb
144	pcc premarc reveal sanb scipio toknb
145	ukblcatcopy ukblderived ukblproject ukblsr ukscp
146	xisds/c xissnuk xlc xnlc xnsdp
147	]
148	end
149
150	def auth_code_error?(record)	1✔
151	return false unless record['042']	4✔
152	return true if record.fields('042').size > 1	3✔
153
154	record['042'].subfields.each do \|subfield\|	2✔
155	next if subfield.code != 'a'	3✔
156	return true unless auth_codes_f042.include?(subfield.value)	3✔
157	end
158	false	1✔
159	end
160
161	def invalid_indicators?(record)	1✔
162	record.fields.each do \|field\|	3✔
163	next unless field.instance_of?(MARC::DataField)	3✔
164	return true unless field.indicator1.match?(/^[0-9 ]$/)	3✔
165	return true unless field.indicator2.match?(/^[0-9 ]$/)	2✔
166	end
167	false	1✔
168	end
169
170	def invalid_subfield_code?(record)	1✔
171	record.fields.each do \|field\|	2✔
172	next unless field.instance_of?(MARC::DataField)	2✔
173
174	field.subfields.each do \|subfield\|	2✔
175	return true unless subfield.code =~ /^[0-9a-z]$/	2✔
176	end
177	end
178	false	1✔
179	end
180
181	def empty_subfields?(record)	1✔
182	record.fields.each do \|field\|	2✔
183	next unless field.instance_of?(MARC::DataField)	2✔
184
185	field.subfields.each do \|subfield\|	2✔
186	return true if subfield.value =~ /^[[:blank:]]*$/	3✔
187	end
188	end
189	false	1✔
190	end
191
192	def extra_spaces?(record)	1✔
193	blank_regex = /^.[[:blank:]]{2,}.$\|^.[[:blank:]]+$\|^[[:blank:]]+(.)$/	4✔
194	record.fields.each do \|field\|	4✔
195	next unless field.instance_of?(MARC::DataField) && field.tag != '010'	4✔
196
197	case field.tag	4✔
198	when /[1-469]..\|0[2-9].\|01[1-9]\|7[0-5].\|5[0-24-9].\|53[0-24-9]/
199	field.subfields.each do \|subfield\|	1✔
200	return true if subfield.value =~ blank_regex	1✔
201	end
202	when '533'
203	field.subfields.each do \|subfield\|	1✔
204	next if subfield.code == '7'	2✔
205
206	return true if subfield.value =~ blank_regex	1✔
207	end
208	when /7[6-8]./
209	field.subfields.each do \|subfield\|	1✔
210	next unless subfield.code =~ /[a-v3-8]/	1✔
211
212	return true if subfield.value =~ blank_regex	1✔
213	end
214	when /8../
215	field.subfields.each do \|subfield\|	1✔
216	next unless subfield.code =~ /[^w7]/	1✔
217
218	return true if subfield.value =~ blank_regex	1✔
219	end
220	end
221	end
222	false	2✔
223	end
224
225	def extra_space_gsub(string)	1✔
NEW 226	string.gsub!(/([[:blank:]]){2,}/, '\1')	×
NEW 227	string.gsub!(/^(.*)[[:blank:]]+$/, '\1')	×
NEW 228	string.gsub(/^[[:blank:]]+(.*)$/, '\1')	×
229	end
230
231	### Remove extra spaces from all fields that are not positionally defined
232	def extra_space_fix(record)	1✔
NEW 233	record.fields.each do \|field\|	×
NEW 234	next unless field.class == MARC::DataField && field.tag != '010'	×
235
NEW 236	field_index = record.fields.index(field)	×
NEW 237	curr_subfield = -1	×
NEW 238	case field.tag	×
239	when /^[1-469]..\|0[2-9].\|01[1-9]\|7[0-5].\|5[0-24-9].\|53[0-24-9]/
NEW 240	field.subfields.each do \|subfield\|	×
NEW 241	curr_subfield += 1	×
NEW 242	next if subfield.value.nil?	×
243
NEW 244	record.fields[field_index].subfields[curr_subfield].value = extra_space_gsub(subfield.value)	×
245	end
246	when '533'
NEW 247	field.subfields.each do \|subfield\|	×
NEW 248	curr_subfield += 1	×
NEW 249	next if subfield.code == '7' \|\| subfield.value.nil?	×
250
NEW 251	record.fields[field_index].subfields[curr_subfield].value = extra_space_gsub(subfield.value)	×
252	end
253	when /^7[6-8]./
NEW 254	field.subfields.each do \|subfield\|	×
NEW 255	curr_subfield += 1	×
NEW 256	next if subfield.code =~ /[^a-v3-8]/ \|\| subfield.value.nil?	×
257
NEW 258	record.fields[field_index].subfields[curr_subfield].value = extra_space_gsub(subfield.value)	×
259	end
260	when /^8../
NEW 261	field.subfields.each do \|subfield\|	×
NEW 262	curr_subfield += 1	×
NEW 263	next if %w[w 7].include?(subfield.code) \|\| subfield.value.nil?	×
264
NEW 265	record.fields[field_index].subfields[curr_subfield].value = extra_space_gsub(subfield.value)	×
266	end
267	end
268	end
NEW UNCOV 269	record	×
270	end
271
272	def multiple_no_040?(record)	1✔
273	record.fields('040').size != 1	1✔
274	end
275
276	def multiple_no_040b?(record)	1✔
277	f040 = record.fields('040')	3✔
278	return true if f040.size != 1	3✔
279
280	f040 = f040.first	3✔
281	b040 = f040.subfields.select { \|subfield\| subfield.code == 'b' }	7✔
282	return true if b040.size != 1	3✔
283
284	b040.first.value.match?(/^\s*$/)	2✔
285	end
286
287	def f046_errors?(record)	1✔
288	subf_codes = %w[b c d e]	4✔
289	subf_a_values = %w[r s p t x q n i k r m t x n]	4✔
290	f046 = record.fields('046')	4✔
291	return false if f046.empty?	4✔
292
293	f046.each do \|field\|	3✔
294	codes = field.subfields.map(&:code)	3✔
295	return true if field['a'] && !subf_a_values.include?(field['a'])	3✔
296	return true if field['a'].nil? && (subf_codes & codes).size.positive?	2✔
297	end
298	false	1✔
299	end
300
301	def multiple_no_245?(record)	1✔
302	record.fields('245').size != 1	23✔
303	end
304
305	def f245_subfield_errors?(record)	1✔
306	fields = record.fields('245')	×
307	return true if fields.empty?	×
308
309	fields.each do \|field\|	×
UNCOV 310	subfields = field.subfields.map(&:code)	×
UNCOV 311	return true if subfields.count('a') != 1	×
UNCOV 312	return true if subfields.count('b') > 1	×
UNCOV 313	return true if subfields.count('c') > 1	×
314	end
UNCOV 315	false	×
316	end
317
318	def missing_040c?(record)	1✔
319	return true unless record['040'] && record['040']['c']	2✔
320
321	false	1✔
322	end
323
324	def pair_880_errors?(record)	1✔
UNCOV 325	pair_880s = []	×
326	linked_fields = []	×
UNCOV 327	return false unless record['880']	×
328
329	record.fields.each do \|field\|	×
UNCOV 330	return true if field.tag == '880' && field['6'].nil?	×
331
UNCOV 332	next unless field.tag =~ /^[0-8]..$/ && field.class == MARC::DataField && field['6']	×
333
334	if field.tag == '880'	×
335	pair_880s << field['6'].gsub(/^([0-9]{3}-[0-9]{2}).*$/, '\1')	×
336	else
337	linked_fields << "#{field.tag}-#{field['6'].gsub(/^880-([0-9]{2}).*$/, '\1').chomp}"	×
338	end
339	end
UNCOV 340	pair_880s.delete_if { \|x\| x =~ /^.*-00/ }	×
UNCOV 341	return true if pair_880s.uniq != pair_880s \|\| pair_880s.uniq.sort != linked_fields.uniq.sort	×
342
UNCOV 343	false	×
344	end
345
346	def has_130_240?(record)	1✔
347	(%w[130 240] - record.tags).empty?	23✔
348	end
349
350	def multiple_1xx?(record)	1✔
351	record.fields('100'..'199').size > 1	23✔
352	end
353
354	def relator_chars?(record)	1✔
UNCOV 355	record.fields.each do \|field\|	×
356	if field.tag =~ /[17][01]0/	×
357	field.subfields.each do \|subfield\|	×
358	next unless subfield.code == 'e' && subfield.value =~ /.[^a-z, \.]./	×
359
360	return true	×
361	end
UNCOV 362	elsif field.tag =~ /[17]11/	×
UNCOV 363	field.subfields.each do \|subfield\|	×
364	next unless subfield.code == 'j' && subfield.value =~ /.[^a-z, \.]./	×
365
UNCOV 366	return true	×
367	end
368	end
369	end
370	false	×
371	end
372
373	def x00_subfq?(record)	1✔
UNCOV 374	record.fields(%w[100 600 700 800]).each do \|field\|	×
375	field.subfields.each do \|subfield\|	×
UNCOV 376	next unless subfield.code == 'q' && subfield.value =~ /^(?!\([^\)]*\))$/	×
377
UNCOV 378	return true	×
379	end
380	end
381	false	×
382	end
383
384	def no_comma_x00?(record)	1✔
385	record.fields(%w[100 600 700 800]).each do \|field\|	×
UNCOV 386	code_array = ''	×
387	field.subfields.each do \|subfield\|	×
UNCOV 388	code_array << subfield.code	×
389	end
UNCOV 390	subfx_d_index = code_array.index(/.d/)	×
391	return true if subfx_d_index && field.subfields[subfx_d_index].value =~ /^.*[^,]$/	×
392	end
393	false	×
394	end
395
396	def relator_comma?(record)	1✔
397	comma_regex = /^.*[^,]$/	×
UNCOV 398	record.fields.each do \|field\|	×
399	next unless field.tag =~ /[17][01][01]/	×
400
401	code_array = ''	×
402	field.subfields.each do \|subfield\|	×
403	code_array << subfield.code	×
404	end
UNCOV 405	if field.tag =~ /[17][01]0/	×
UNCOV 406	subfx_e_index = code_array.index(/.e/)	×
407	return true if subfx_e_index && field.subfields[subfx_e_index].value =~ comma_regex	×
UNCOV 408	elsif field.tag =~ /[17]11/	×
UNCOV 409	subfx_j_index = code_array.index(/.j/)	×
UNCOV 410	return true if subfx_j_index && field.subfields[subfx_j_index].value =~ comma_regex	×
411	end
412	end
413	false	×
414	end
415
416	def heading_end_punct?(record)	1✔
417	punct_regex = /.*[^"\).\!\?\-]$/	×
UNCOV 418	record.fields.each do \|field\|	×
419	next unless field.tag =~ /^[167][0-5].$/ && field.indicator2 =~ /[^47]/	×
420
UNCOV 421	code_array = ''	×
422	field.subfields.each do \|subfield\|	×
UNCOV 423	code_array << subfield.code	×
424	end
UNCOV 425	last_heading_subfield_index = code_array.index(/[a-vx-z8][^a-vx-z8]*$/)	×
426	return true if last_heading_subfield_index && field.subfields[last_heading_subfield_index].value =~ punct_regex	×
427	end
UNCOV 428	false	×
429	end
430
431	def lowercase_headings?(record)	1✔
UNCOV 432	record.fields.each do \|field\|	×
UNCOV 433	next unless field.tag =~ /[1678]../	×
434
UNCOV 435	return true if field['a'] =~ /^[a-z]{3,}/	×
436	end
UNCOV 437	false	×
438	end
439
440	def subf_0_uri?(record)	1✔
441	record.fields.each do \|field\|	3✔
442	next unless field.instance_of?(MARC::DataField) && field.tag =~ /^[^9]/ && field['0']	3✔
443
444	field.subfields.each do \|subfield\|	2✔
445	return true if subfield.code == '0' && subfield.value =~ /^\(uri\)/	4✔
446	end
447	end
448	false	2✔
449	end
450
451	def bad_uri?(record)	1✔
452	target_fields = record.fields('856')	×
453	return false if target_fields.empty?	×
454
455	target_fields.each do \|field\|	×
456	next unless field['u']	×
457
UNCOV 458	field.subfields.each do \|subfield\|	×
459	next unless subfield.code == 'u'	×
460
UNCOV 461	string = subfield.value	×
UNCOV 462	return true unless URI.escape(URI.unescape(string).scrub) == string	×
463	end
464	end
UNCOV 465	false	×
466	end
467
468	### Normalize to the NFC (combined) form of diacritics for characters with
469	# Arabic diacritics; normalize to NFD for characters below U+622 and
470	# between U+1E00 and U+2A28
471	def composed_chars_normalize(record)	1✔
472	record.fields.each do \|field\|	×
473	next unless field.class == MARC::DataField	×
474
475	field_index = record.fields.index(field)	×
UNCOV 476	curr_subfield = 0	×
477	field.subfields.each do \|subfield\|	×
478	fixed_subfield = ''	×
479	prevalue = subfield.value	×
480	if prevalue =~ /^.[\u0653\u0654\u0655].$/	×
UNCOV 481	prevalue = prevalue.unicode_normalize(:nfc)	×
482	end
483	prevalue.each_codepoint do \|c\|	×
UNCOV 484	char = c.chr(Encoding::UTF_8)	×
UNCOV 485	char.unicode_normalize!(:nfd) if c < 1570 \|\| (7_680..10_792).cover?(c)	×
486	fixed_subfield << char	×
487	end
UNCOV 488	record.fields[field_index].subfields[curr_subfield].value = fixed_subfield	×
UNCOV 489	curr_subfield += 1	×
490	end
491	end
UNCOV 492	record	×
493	end
494
495	### Replace empty indicators with a space;
496	### scrub indicators with bad UTF-8;
497	### The ruby-marc gem converts nil subfields to spaces
498	def empty_indicator_fix(record)	1✔
499	record.fields.each do \|field\|	2✔
500	next unless field.instance_of?(MARC::DataField)	2✔
501
502	ind1_value = field.indicator1.dup	2✔
503	ind1_value.scrub!('')	2✔
504	field.indicator1 = ' ' if ind1_value.empty?	2✔
505	ind2_value = field.indicator2.dup	2✔
506	ind2_value.scrub!('')	2✔
507	field.indicator2 = ' ' if ind2_value.empty?	2✔
508	end
509	record	2✔
510	end
511
512	### Remove empty subfields from DataFields
513	def empty_subfield_fix(record)	1✔
514	record.fields.each do \|field\|	1✔
515	next unless field.instance_of?(MARC::DataField)	2✔
516
517	field.subfields.delete_if { \|subfield\| subfield.value.nil? \|\| subfield.value.empty? }	11✔
518	end
519	record.fields.delete_if { \|field\| field.instance_of?(MARC::DataField) && field.subfields.empty? }	3✔
520	record	1✔
521	end
522
523	### Remove the (uri) prefix from subfield 0s
524	def subf_0_uri_fix(record)	1✔
525	record.fields.each do \|field\|	3✔
526	next unless field.instance_of?(MARC::DataField) && field.tag[0] != '9' && field['0']	3✔
527
528	field.subfields.each do \|subfield\|	2✔
529	next unless subfield.code == '0' && subfield.value =~ /^\(uri\)/	4✔
530
531	subfield.value = subfield.value.dup.delete_prefix('(uri)')	1✔
532	end
533	end
534	record	3✔
535	end
536
537	### Escape URIs
538	def uri_escape(record)	1✔
UNCOV 539	target_fields = record.fields('856')	×
540	return record if target_fields.empty?	×
541
542	fixed_record = record	×
UNCOV 543	target_fields.each do \|field\|	×
544	next unless field['u']	×
545
546	field_index = fixed_record.fields.index(field)	×
547	field.subfields.each do \|subfield\|	×
UNCOV 548	next unless subfield.code == 'u'	×
549
550	subfield_index = field.subfields.index(subfield)	×
UNCOV 551	string = subfield.value	×
UNCOV 552	fixed_string = URI.escape(URI.unescape(string).scrub)	×
UNCOV 553	fixed_record.fields[field_index].subfields[subfield_index].value = fixed_string	×
554	end
555	end
UNCOV 556	fixed_record	×
557	end
558
559	### Make the 040 $b 'eng' if it doesn't have a value
560	def fix_040b(record)	1✔
561	return record unless record.fields('040').size == 1	2✔
562
563	f040 = record['040']	2✔
564	field_index = record.fields.index(f040)	2✔
565	b040 = f040.subfields.select { \|subfield\| subfield.code == 'b' }	4✔
566	return record unless b040.empty?	2✔
567
568	subf_codes = f040.subfields.map(&:code)	2✔
569	subf_index = if f040['a']	2✔
570	(subf_codes.index { \|i\| i == 'a' }) + 1	2✔
571	else
572	0	1✔
573	end
574	subf_b = MARC::Subfield.new('b', 'eng')	2✔
575	record.fields[field_index].subfields.insert(subf_index, subf_b)	2✔
576	record	2✔
577	end
578
579	### Split up subfields that contain multiple 3-letter language codes
580	def fix_f041(record)	1✔
581	f041 = record.fields('041')	2✔
582	return record if f041.empty?	2✔
583
584	f041.each do \|field\|	2✔
585	f_index = record.fields.index(field)	2✔
586	new_field = MARC::DataField.new('041', field.indicator1, field.indicator2)	2✔
587	field.subfields.each do \|subfield\|	2✔
588	code = subfield.code	2✔
589	val = subfield.value	2✔
590	if (val.size % 3).zero?	2✔
591	langs = val.scan(/.../)	1✔
592	langs.each do \|lang\|	1✔
593	new_field.append(MARC::Subfield.new(code, lang))	3✔
594	end
595	else
596	new_field.append(MARC::Subfield.new(code, val))	1✔
597	end
598	end
599	record.fields[f_index] = new_field	2✔
600	end
601	record	2✔
602	end
603
604	### Removes text from the beginning of a subfield
605	### An array of hashes of the format { field:, subfields: } will be passed
606	### in the targets: symbol
607	### subfield: is an array of subfield codes
608	def remove_prefix_from_subfield(record:, targets:, string:)	1✔
609	targets.each do \|target\|	1✔
610	record.fields(target[:field]).each do \|field\|	1✔
611	field.subfields.each do \|subfield\|	1✔
612	next unless target[:subfields].include?(subfield.code)	2✔
613
614	subfield.value = subfield.value.dup.delete_prefix(string)	1✔
615	end
616	end
617	end
618	record	1✔
619	end
620
621	### Adds text to the beginning of a subfield
622	### An array of hashes of the format { field:, subfields: } will be passed
623	### in the targets: symbol
624	### subfield: is an array of subfield codes
625	def add_prefix_to_subfield(record:, targets:, string:)	1✔
626	targets.each do \|target\|	1✔
627	record.fields(target[:field]).each do \|field\|	1✔
628	field.subfields.each do \|subfield\|	1✔
629	next unless target[:subfields].include?(subfield.code)	2✔
630
631	subfield.value = subfield.value.dup.prepend(string)	1✔
632	end
633	end
634	end
635	record	1✔
636	end
637
638	### Sort subfields for target fields with an arbitrary order
639	def subfield_sort(record, target_tags, order_array = nil)	1✔
640	target_fields = record.fields.select { \|f\| target_tags.include?(f.tag) }	×
641	return record if target_fields.empty?	×
642
643	target_fields.each do \|field\|	×
644	next unless field.class == MARC::DataField	×
645
646	orig_codes = field.subfields.map { \|subfield\| subfield.code }.uniq.sort	×
647	order_array = orig_codes if order_array.nil?	×
UNCOV 648	new_subfields = []	×
649	order_array.each do \|code\|	×
650	next unless orig_codes.include?(code)	×
651
UNCOV 652	target_subf = field.subfields.select { \|subfield\| subfield.code == code }	×
653	target_subf.each { \|subfield\| new_subfields << subfield }	×
654	end
655	rem_subfields = field.subfields.select { \|subf\| !order_array.include?(subf.code) }	×
UNCOV 656	rem_subfields.each do \|subfield\|	×
UNCOV 657	new_subfields << subfield	×
658	end
UNCOV 659	field.subfields = new_subfields	×
660	end
UNCOV 661	record	×
662	end
663	end

pulibrary / marc_cleanup / ebd2715e-56b5-40ce-841f-3d6e105293d4

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous