7a49e047-f3ae-41b9-a3d7-9924d268f02a

Committed 07 Mar 2025 05:42PM UTC coverage: 90.693% (+1.0%) from 89.644%

Build # 7a49e047-f3ae-41b9-a3d7-9924d268f02a

Build Type

Pull #159

circleci

Committed by

mzelesky

Commit Message

refactor relator_comma? and heading_end_punct?

Pull Request Pull Request #159: refactor relator_comma? and heading_end_punct?

Run Details

27 of 27 new or added lines in 1 file covered. (100.0%)

9 existing lines in 1 file now uncovered.

1910 of 2106 relevant lines covered (90.69%)

4.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.79

/lib/marc_cleanup/variable_fields.rb

# frozen_string_literal: true

module MarcCleanup
  ### Remove non-numerical strings and append a new 020$q with the string
  def new_020_q(record)
    record.fields('020').each do |f020|
      f020.subfields.each do |subfield|
        next unless subfield.code == 'a'
        isbn_parts = /^\s*([\d\-]+)\s*(\(.*?\))\s*$/.match(subfield.value)
        next if isbn_parts.nil?
        subfield.value = isbn_parts[1]
        f020.append(MARC::Subfield.new('q', isbn_parts[2]))
      end
    end
    record
  end

  ### Convert ISBN-10 to ISBN-13
  def isbn10_to_13(isbn)
    stem = isbn[0..8]
    return nil if stem =~ /\D/

    existing_check = isbn[9]
    return nil if existing_check && existing_check != checkdigit_10(stem)

    main = ISBN13PREFIX + stem
    checkdigit = checkdigit_13(main)
    main + checkdigit
  end

  ### Calculate check digit for ISBN-10
  def checkdigit_10(stem)
    int_index = 0
    int_sum = 0
    stem.each_char do |digit|
      int_sum += digit.to_i * (10 - int_index)
      int_index += 1
    end
    mod = (11 - (int_sum % 11)) % 11
    mod == 10 ? 'X' : mod.to_s
  end

  ### Calculate check digit for ISBN-13
  def checkdigit_13(stem)
    int_index = 0
    int_sum = 0
    stem.each_char do |digit|
      int_sum += int_index.even? ? digit.to_i : digit.to_i * 3
      int_index += 1
    end
    ((10 - (int_sum % 10)) % 10).to_s
  end

  ### Normalize ISBN-13
  def isbn13_normalize(raw_isbn)
    int_sum = 0
    stem = raw_isbn[0..11]
    return nil if stem =~ /\D/

    int_index = 0
    stem.each_char do |digit|
      int_sum += int_index.even? ? digit.to_i : digit.to_i * 3
      int_index += 1
    end
    checkdigit = checkdigit_13(stem)
    return nil if raw_isbn[12] && raw_isbn[12] != checkdigit

    stem + checkdigit
  end

  ### Normalize any given string that is supposed to include an ISBN
  def isbn_normalize(isbn)
    return nil unless isbn

    raw_isbn = isbn.dup
    raw_isbn.delete!('-')
    raw_isbn.delete!('\\')
    raw_isbn.gsub!(/\([^\)]*\)/, '')
    raw_isbn.gsub!(/^(.*)\$c.*$/, '\1')
    raw_isbn.gsub!(/^(.*)\$q.*$/, '\1')
    raw_isbn.gsub!(/^\D+([0-9].*)$/, '\1')
    if raw_isbn =~ /^978/
      raw_isbn.gsub!(/^(978[0-9 ]+).*$/, '\1')
      raw_isbn.delete!(' ')
    else
      raw_isbn.gsub!(/([0-9])\s*([0-9]{4})\s*([0-9]{4})\s*([0-9xX]).*$/, '\1\2\3\4')
    end
    raw_isbn.gsub!(/^([0-9]{9,13}[xX]?)[^0-9xX].*$/, '\1')
    raw_isbn.gsub!(/^([0-9]+?)\D.*$/, '\1')
    if raw_isbn.length > 6 && raw_isbn.length < 9 && raw_isbn =~ /^[0-9]+$/
      raw_isbn = raw_isbn.ljust(9, '0')
    end
    valid_lengths = [9, 10, 12, 13] # ISBN10 and ISBN13 with/out check digits
    return nil unless valid_lengths.include? raw_isbn.length

    if raw_isbn.length < 12
      isbn10_to_13(raw_isbn)
    else
      isbn13_normalize(raw_isbn)
    end
  end

  ### If the ISBN is invalid, change the subfield code to z
  ### Otherwise, replace ISBN with normalized ISBN
  def move_invalid_isbn(record)
    record.fields('020').each do |f020|
      f020.subfields.each do |subfield|
        next unless subfield.code == 'a'
        isbn = subfield.value
        normalized_isbn = isbn_normalize(isbn)
        if normalized_isbn
          subfield.value = normalized_isbn
        else
          subfield.code = 'z'
        end
      end
    end
    record
  end

  # check the 041 field for errors
  # 041 is a language code
  def f041_errors?(record)
    f041 = record.fields('041')
    return false if f041.empty?

    f041.each do |field|
      field.subfields.each do |subfield|
        val = subfield.value
        return true if (val.size > 3) && (val.size % 3).zero?
      end
    end
    false
  end

  # http://www.loc.gov/standards/valuelist/marcauthen.html
  def auth_codes_f042
    %w[
      anuc croatica dc dhca dlr
      gamma gils gnd1 gnd2 gnd3 gnd4 gnd5 gnd6 gnd7 gndz isds/c issnuk
      lacderived lc lcac lccopycat lccopycat-nm lcd lcderive
      lchlas lcllh lcnccp lcnitrate lcnuc lcode
      msc natgaz nbr nlc nlmcopyc norbibl nsdp nst ntccf nznb
      pcc premarc reveal sanb scipio toknb
      ukblcatcopy ukblderived ukblproject ukblsr ukscp
      xisds/c xissnuk xlc xnlc xnsdp
    ]
  end

  def auth_code_error?(record)
    return false unless record['042']
    return true if record.fields('042').size > 1

    record['042'].subfields.each do |subfield|
      next if subfield.code != 'a'
      return true unless auth_codes_f042.include?(subfield.value)
    end
    false
  end

  def empty_subfields?(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      field.subfields.each do |subfield|
        return true if subfield.value =~ /^[[:blank:]]*$/
      end
    end
    false
  end

  def extra_spaces?(record)
    blank_regex = /^.*[[:blank:]]{2,}.*$|^.*[[:blank:]]+$|^[[:blank:]]+(.*)$/
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag != '010'

      case field.tag
      when /[1-469]..|0[2-9].|01[1-9]|7[0-5].|5[0-24-9].|53[0-24-9]/
        field.subfields.each do |subfield|
          return true if subfield.value =~ blank_regex
        end
      when '533'
        field.subfields.each do |subfield|
          next if subfield.code == '7'

          return true if subfield.value =~ blank_regex
        end
      when /7[6-8]./
        field.subfields.each do |subfield|
          next unless subfield.code =~ /[a-v3-8]/

          return true if subfield.value =~ blank_regex
        end
      when /8../
        field.subfields.each do |subfield|
          next unless subfield.code =~ /[^w7]/

          return true if subfield.value =~ blank_regex
        end
      end
    end
    false
  end

  def extra_space_gsub(string)
    string.gsub!(/([[:blank:]]){2,}/, '\1')
    string.gsub!(/^(.*)[[:blank:]]+$/, '\1')
    string.gsub(/^[[:blank:]]+(.*)$/, '\1')
  end

  ### Remove extra spaces from all fields that are not positionally defined
  def extra_space_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag != '010'

      case field.tag
      when /^[1-469]..|0[2-9].|01[1-9]|7[0-5].|5[0-24-9].|53[0-24-9]/
        field.subfields.each do |subfield|
          next if subfield.value.nil?

          subfield.value = extra_space_gsub(subfield.value.dup)
        end
      when '533'
        field.subfields.each do |subfield|
          next if subfield.code == '7' || subfield.value.nil?

          subfield.value = extra_space_gsub(subfield.value.dup)
        end
      when /^7[6-8]./
        field.subfields.each do |subfield|
          next if subfield.code =~ /[^a-v3-8]/ || subfield.value.nil?

          subfield.value = extra_space_gsub(subfield.value.dup)
        end
      when /^8../
        field.subfields.each do |subfield|
          next if %w[w 7].include?(subfield.code) || subfield.value.nil?

          subfield.value = extra_space_gsub(subfield.value.dup)
        end
      end
    end
    record
  end

  def multiple_no_040?(record)
    record.fields('040').size != 1
  end

  def multiple_no_040b?(record)
    f040 = record.fields('040')
    return true if f040.size != 1

    f040 = f040.first
    b040 = f040.subfields.select { |subfield| subfield.code == 'b' }
    return true if b040.size != 1

    b040.first.value.match?(/^\s*$/)
  end

  def f046_errors?(record)
    subf_codes = %w[b c d e]
    subf_a_values = %w[r s p t x q n i k r m t x n]
    f046 = record.fields('046')
    return false if f046.empty?

    f046.each do |field|
      codes = field.subfields.map(&:code)
      return true if field['a'] && !subf_a_values.include?(field['a'])
      return true if field['a'].nil? && (subf_codes & codes).size.positive?
    end
    false
  end

  def multiple_no_245?(record)
    record.fields('245').size != 1
  end

  def missing_040c?(record)
    return true unless record['040'] && record['040']['c']

    false
  end

  def pair_880_errors?(record)
    return true if record.fields('880').select { |field| field['6'].nil? }.size.positive?

    pair_880s = f880_pairings(record)
    linked_fields = linked_field_pairings(record)
    pair_880s.uniq != pair_880s || pair_880s.uniq.sort != linked_fields.uniq.sort
  end

  def f880_pairings(record)
    target_fields = record.fields('880').select do |field|
      field['6'] =~ /^[0-9]{3}-[0-9][1-9]/
    end
    target_fields.map do |field|
      field['6'].gsub(/^([0-9]{3}-[0-9]{2}).*$/, '\1')
    end
  end

  def linked_field_pairings(record)
    target_fields = record.fields('010'..'899').select do |field|
      field.tag != '880' && field['6']
    end
    target_fields.map do |field|
      "#{field.tag}-#{field['6'].gsub(/^880-([0-9]{2}).*$/, '\1')}"
    end
  end

  def has_130_240?(record)
    (%w[130 240] - record.tags).empty?
  end

  def multiple_1xx?(record)
    record.fields('100'..'199').size > 1
  end

  def relator_chars?(record)
    record.fields(%w[100 110 111 700 710 711]).each do |field|
      target_subfields = relator_chars_target_subfields(field)
      return true if target_subfields.select do |subfield|
        subfield.value =~ /[^a-z\-, .]/
      end.size.positive?
    end
    false
  end

  def relator_chars_target_subfields(field)
    case field.tag
    when '111', '711'
      field.subfields.select { |subf| subf.code == 'j' }
    else
      field.subfields.select { |subf| subf.code == 'e' }
    end
  end

  def x00_subfq?(record)
    record.fields(%w[100 600 700 800]).each do |field|
      field.subfields.each do |subfield|
        return true if subfield.code == 'q' && subfield.value =~ /^[^(].*[^)]$/
      end
    end
    false
  end

  def x00_subfd_no_comma?(record)
    record.fields(%w[100 600 700 800]).each do |field|
      subf_d_index = field.subfields.index { |subfield| subfield.code == 'd' }
      next unless subf_d_index
      return true if field.subfields[subf_d_index - 1].value =~ /[^,]$/
    end
    false
  end

  def relator_comma?(record)
    record.fields(%w[100 110 111 700 710 711]).each do |field|
      relator_index = relator_subfield_index(field)
      next unless relator_index
      return true if field.subfields[relator_index - 1].value =~ /[^,]$/
    end
    false
  end

  def relator_subfield_index(field)
    case field.tag
    when '111', '711'
      field.subfields.index { |subfield| subfield.code == 'j' }
    else
      field.subfields.index { |subfield| subfield.code == 'e' }
    end
  end

  def heading_end_punct?(record)
    punct_regex = /[^").!?-]$/
    record.fields(punctuated_heading_fields).each do |field|
      last_heading_subfield = last_heading_subfield(field)
      next unless last_heading_subfield
      return true if last_heading_subfield.value =~ punct_regex
    end
    false
  end

  def punctuated_heading_fields
    %w[
      100 110 111 130
      600 610 611 630 650 651 654 655 656 657 658 662
      700 710 711 730 740 752 754
      800 810 811 830
    ]
  end

  def last_heading_subfield(field)
    regex = /[^02345]/
    heading_subfields = field.subfields.select do |subfield|
      subfield.code =~ regex
    end
    if heading_subfields.empty?
      nil
    else
      heading_subfields[-1]
    end
  end

  def subf_0_uri?(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag =~ /^[^9]/ && field['0']

      field.subfields.each do |subfield|
        return true if subfield.code == '0' && subfield.value =~ /^\(uri\)/
      end
    end
    false
  end

  def bad_uri?(record)
    target_fields = record.fields('856')
    return false if target_fields.empty?

    target_fields.each do |field|
      next unless field['u']

      field.subfields.each do |subfield|
        next unless subfield.code == 'u'

        string = subfield.value
        return true unless URI.escape(URI.unescape(string).scrub) == string
      end
    end
    false
  end

  ### Normalize to the NFC (combined) form of diacritics for characters with
  #     Arabic diacritics; normalize to NFD for characters below U+622 and
  #     between U+1E00 and U+2A28
  def composed_chars_normalize(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField

      field_index = record.fields.index(field)
      curr_subfield = 0
      field.subfields.each do |subfield|
        prevalue = subfield.value
        if prevalue =~ /^.*[\u0653\u0654\u0655].*$/
          prevalue = prevalue.unicode_normalize(:nfc)
        end
        fixed_subfield = prevalue.codepoints.map do |codepoint|
          char = codepoint.chr(Encoding::UTF_8)
          char.unicode_normalize!(:nfd) if codepoint < 1570 || (7_680..10_792).cover?(codepoint)
          char
        end.join
        record.fields[field_index].subfields[curr_subfield].value = fixed_subfield
        curr_subfield += 1
      end
    end
    record
  end

  ### Replace empty indicators with a space;
  ###   scrub indicators with bad UTF-8;
  ###   The ruby-marc gem converts nil subfields to spaces
  def empty_indicator_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      ind1_value = field.indicator1.dup
      ind1_value.scrub!('')
      field.indicator1 = ' ' if ind1_value.empty?
      ind2_value = field.indicator2.dup
      ind2_value.scrub!('')
      field.indicator2 = ' ' if ind2_value.empty?
    end
    record
  end

  ### Remove empty subfields from DataFields
  def empty_subfield_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField)

      field.subfields.delete_if { |subfield| subfield.value.nil? || subfield.value.empty? }
    end
    record.fields.delete_if { |field| field.instance_of?(MARC::DataField) && field.subfields.empty? }
    record
  end

  ### Remove the (uri) prefix from subfield 0s
  def subf_0_uri_fix(record)
    record.fields.each do |field|
      next unless field.instance_of?(MARC::DataField) && field.tag[0] != '9' && field['0']

      field.subfields.each do |subfield|
        next unless subfield.code == '0' && subfield.value =~ /^\(uri\)/

        subfield.value = subfield.value.dup.delete_prefix('(uri)')
      end
    end
    record
  end

  ### Escape URIs
  def uri_escape(record)
    target_fields = record.fields('856')
    return record if target_fields.empty?

    fixed_record = record
    target_fields.each do |field|
      next unless field['u']

      field_index = fixed_record.fields.index(field)
      field.subfields.each do |subfield|
        next unless subfield.code == 'u'

        subfield_index = field.subfields.index(subfield)
        string = subfield.value
        fixed_string = URI.escape(URI.unescape(string).scrub)
        fixed_record.fields[field_index].subfields[subfield_index].value = fixed_string
      end
    end
    fixed_record
  end

  ### Make the 040 $b 'eng' if it doesn't have a value
  def fix_040b(record)
    return record unless record.fields('040').size == 1

    f040 = record['040']
    field_index = record.fields.index(f040)
    b040 = f040.subfields.select { |subfield| subfield.code == 'b' }
    return record unless b040.empty?

    subf_codes = f040.subfields.map(&:code)
    subf_index = if f040['a']
                   (subf_codes.index { |i| i == 'a' }) + 1
                 else
                   0
                 end
    subf_b = MARC::Subfield.new('b', 'eng')
    record.fields[field_index].subfields.insert(subf_index, subf_b)
    record
  end

  ### Split up subfields that contain multiple 3-letter language codes
  def fix_f041(record)
    f041 = record.fields('041')
    return record if f041.empty?

    f041.each do |field|
      f_index = record.fields.index(field)
      new_field = MARC::DataField.new('041', field.indicator1, field.indicator2)
      field.subfields.each do |subfield|
        code = subfield.code
        val = subfield.value
        if (val.size % 3).zero?
          langs = val.scan(/.../)
          langs.each do |lang|
            new_field.append(MARC::Subfield.new(code, lang))
          end
        else
          new_field.append(MARC::Subfield.new(code, val))
        end
      end
      record.fields[f_index] = new_field
    end
    record
  end

  ### Removes text from the beginning of a subfield
  ### An array of hashes of the format { field:, subfields: } will be passed
  ###   in the targets: symbol
  ###   subfield: is an array of subfield codes
  def remove_prefix_from_subfield(record:, targets:, string:)
    targets.each do |target|
      record.fields(target[:field]).each do |field|
        field.subfields.each do |subfield|
          next unless target[:subfields].include?(subfield.code)

          subfield.value = subfield.value.dup.delete_prefix(string)
        end
      end
    end
    record
  end

  ### Adds text to the beginning of a subfield
  ### An array of hashes of the format { field:, subfields: } will be passed
  ###   in the targets: symbol
  ###   subfield: is an array of subfield codes
  def add_prefix_to_subfield(record:, targets:, string:)
    targets.each do |target|
      record.fields(target[:field]).each do |field|
        field.subfields.each do |subfield|
          next unless target[:subfields].include?(subfield.code)

          subfield.value = subfield.value.dup.prepend(string)
        end
      end
    end
    record
  end

  ### Sort subfields for target fields with an arbitrary order
  ### Example order_array: ['a', 'b', 'c']
  def subfield_sort(record:, target_tags:, order_array: nil)
    record.fields(target_tags).each do |field|
      next if field.instance_of?(MARC::ControlField)

      order_array ||= field.subfields.map(&:code).uniq.sort
      new_subfields = sort_listed_subfields(field: field, order_array: order_array)
      new_subfields += find_unlisted_subfields(field: field, order_array: order_array)
      field.subfields = new_subfields
    end
    record
  end

  def find_unlisted_subfields(field:, order_array:)
    field.subfields.reject do |subfield|
      order_array.include?(subfield.code)
    end
  end

  def sort_listed_subfields(field:, order_array:)
    listed_subfields = field.subfields.select do |subfield|
      order_array.include?(subfield.code)
    end
    listed_subfields.sort_by! do |subfield|
      order_array.index { |code| code == subfield.code }
    end
  end
end

1	# frozen_string_literal: true
2
3	module MarcCleanup	1✔
4	### Remove non-numerical strings and append a new 020$q with the string
5	def new_020_q(record)	1✔
6	record.fields('020').each do \|f020\|	1✔
7	f020.subfields.each do \|subfield\|	1✔
8	next unless subfield.code == 'a'	2✔
9	isbn_parts = /^\s([\d\-]+)\s(\(.?\))\s$/.match(subfield.value)	1✔
10	next if isbn_parts.nil?	1✔
11	subfield.value = isbn_parts[1]	1✔
12	f020.append(MARC::Subfield.new('q', isbn_parts[2]))	1✔
13	end
14	end
15	record	1✔
16	end
17
18	### Convert ISBN-10 to ISBN-13
19	def isbn10_to_13(isbn)	1✔
20	stem = isbn[0..8]	2✔
21	return nil if stem =~ /\D/	2✔
22
23	existing_check = isbn[9]	2✔
24	return nil if existing_check && existing_check != checkdigit_10(stem)	2✔
25
26	main = ISBN13PREFIX + stem	2✔
27	checkdigit = checkdigit_13(main)	2✔
28	main + checkdigit	2✔
29	end
30
31	### Calculate check digit for ISBN-10
32	def checkdigit_10(stem)	1✔
33	int_index = 0	1✔
34	int_sum = 0	1✔
35	stem.each_char do \|digit\|	1✔
36	int_sum += digit.to_i * (10 - int_index)	9✔
37	int_index += 1	9✔
38	end
39	mod = (11 - (int_sum % 11)) % 11	1✔
40	mod == 10 ? 'X' : mod.to_s	1✔
41	end
42
43	### Calculate check digit for ISBN-13
44	def checkdigit_13(stem)	1✔
45	int_index = 0	4✔
46	int_sum = 0	4✔
47	stem.each_char do \|digit\|	4✔
48	int_sum += int_index.even? ? digit.to_i : digit.to_i * 3	48✔
49	int_index += 1	48✔
50	end
51	((10 - (int_sum % 10)) % 10).to_s	4✔
52	end
53
54	### Normalize ISBN-13
55	def isbn13_normalize(raw_isbn)	1✔
56	int_sum = 0	2✔
57	stem = raw_isbn[0..11]	2✔
58	return nil if stem =~ /\D/	2✔
59
60	int_index = 0	2✔
61	stem.each_char do \|digit\|	2✔
62	int_sum += int_index.even? ? digit.to_i : digit.to_i * 3	24✔
63	int_index += 1	24✔
64	end
65	checkdigit = checkdigit_13(stem)	2✔
66	return nil if raw_isbn[12] && raw_isbn[12] != checkdigit	2✔
67
68	stem + checkdigit	1✔
69	end
70
71	### Normalize any given string that is supposed to include an ISBN
72	def isbn_normalize(isbn)	1✔
73	return nil unless isbn	4✔
74
75	raw_isbn = isbn.dup	4✔
76	raw_isbn.delete!('-')	4✔
77	raw_isbn.delete!('\\')	4✔
78	raw_isbn.gsub!(/\([^\)]*\)/, '')	4✔
79	raw_isbn.gsub!(/^(.)\$c.$/, '\1')	4✔
80	raw_isbn.gsub!(/^(.)\$q.$/, '\1')	4✔
81	raw_isbn.gsub!(/^\D+([0-9].*)$/, '\1')	4✔
82	if raw_isbn =~ /^978/	4✔
83	raw_isbn.gsub!(/^(978[0-9 ]+).*$/, '\1')	2✔
84	raw_isbn.delete!(' ')	2✔
85	else
86	raw_isbn.gsub!(/([0-9])\s([0-9]{4})\s([0-9]{4})\s([0-9xX]).$/, '\1\2\3\4')	2✔
87	end
88	raw_isbn.gsub!(/^([0-9]{9,13}[xX]?)[^0-9xX].*$/, '\1')	4✔
89	raw_isbn.gsub!(/^([0-9]+?)\D.*$/, '\1')	4✔
90	if raw_isbn.length > 6 && raw_isbn.length < 9 && raw_isbn =~ /^[0-9]+$/	4✔
91	raw_isbn = raw_isbn.ljust(9, '0')	1✔
92	end
93	valid_lengths = [9, 10, 12, 13] # ISBN10 and ISBN13 with/out check digits	4✔
94	return nil unless valid_lengths.include? raw_isbn.length	4✔
95
96	if raw_isbn.length < 12	4✔
97	isbn10_to_13(raw_isbn)	2✔
98	else
99	isbn13_normalize(raw_isbn)	2✔
100	end
101	end
102
103	### If the ISBN is invalid, change the subfield code to z
104	### Otherwise, replace ISBN with normalized ISBN
105	def move_invalid_isbn(record)	1✔
106	record.fields('020').each do \|f020\|	4✔
107	f020.subfields.each do \|subfield\|	4✔
108	next unless subfield.code == 'a'	4✔
109	isbn = subfield.value	4✔
110	normalized_isbn = isbn_normalize(isbn)	4✔
111	if normalized_isbn	4✔
112	subfield.value = normalized_isbn	3✔
113	else
114	subfield.code = 'z'	1✔
115	end
116	end
117	end
118	record	4✔
119	end
120
121	# check the 041 field for errors
122	# 041 is a language code
123	def f041_errors?(record)	1✔
124	f041 = record.fields('041')	1✔
125	return false if f041.empty?	1✔
126
127	f041.each do \|field\|	1✔
128	field.subfields.each do \|subfield\|	1✔
129	val = subfield.value	1✔
130	return true if (val.size > 3) && (val.size % 3).zero?	1✔
131	end
132	end
133	false	1✔
134	end
135
136	# http://www.loc.gov/standards/valuelist/marcauthen.html
137	def auth_codes_f042	1✔
138	%w[	3✔
139	anuc croatica dc dhca dlr
140	gamma gils gnd1 gnd2 gnd3 gnd4 gnd5 gnd6 gnd7 gndz isds/c issnuk
141	lacderived lc lcac lccopycat lccopycat-nm lcd lcderive
142	lchlas lcllh lcnccp lcnitrate lcnuc lcode
143	msc natgaz nbr nlc nlmcopyc norbibl nsdp nst ntccf nznb
144	pcc premarc reveal sanb scipio toknb
145	ukblcatcopy ukblderived ukblproject ukblsr ukscp
146	xisds/c xissnuk xlc xnlc xnsdp
147	]
148	end
149
150	def auth_code_error?(record)	1✔
151	return false unless record['042']	4✔
152	return true if record.fields('042').size > 1	3✔
153
154	record['042'].subfields.each do \|subfield\|	2✔
155	next if subfield.code != 'a'	3✔
156	return true unless auth_codes_f042.include?(subfield.value)	3✔
157	end
158	false	1✔
159	end
160
161	def empty_subfields?(record)	1✔
162	record.fields.each do \|field\|	2✔
163	next unless field.instance_of?(MARC::DataField)	2✔
164
165	field.subfields.each do \|subfield\|	2✔
166	return true if subfield.value =~ /^[[:blank:]]*$/	3✔
167	end
168	end
169	false	1✔
170	end
171
172	def extra_spaces?(record)	1✔
173	blank_regex = /^.[[:blank:]]{2,}.$\|^.[[:blank:]]+$\|^[[:blank:]]+(.)$/	4✔
174	record.fields.each do \|field\|	4✔
175	next unless field.instance_of?(MARC::DataField) && field.tag != '010'	4✔
176
177	case field.tag	4✔
178	when /[1-469]..\|0[2-9].\|01[1-9]\|7[0-5].\|5[0-24-9].\|53[0-24-9]/
179	field.subfields.each do \|subfield\|	1✔
180	return true if subfield.value =~ blank_regex	1✔
181	end
182	when '533'
183	field.subfields.each do \|subfield\|	1✔
184	next if subfield.code == '7'	2✔
185
186	return true if subfield.value =~ blank_regex	1✔
187	end
188	when /7[6-8]./
189	field.subfields.each do \|subfield\|	1✔
190	next unless subfield.code =~ /[a-v3-8]/	1✔
191
192	return true if subfield.value =~ blank_regex	1✔
193	end
194	when /8../
195	field.subfields.each do \|subfield\|	1✔
196	next unless subfield.code =~ /[^w7]/	1✔
197
198	return true if subfield.value =~ blank_regex	1✔
199	end
200	end
201	end
202	false	2✔
203	end
204
205	def extra_space_gsub(string)	1✔
206	string.gsub!(/([[:blank:]]){2,}/, '\1')	4✔
207	string.gsub!(/^(.*)[[:blank:]]+$/, '\1')	4✔
208	string.gsub(/^[[:blank:]]+(.*)$/, '\1')	4✔
209	end
210
211	### Remove extra spaces from all fields that are not positionally defined
212	def extra_space_fix(record)	1✔
213	record.fields.each do \|field\|	4✔
214	next unless field.instance_of?(MARC::DataField) && field.tag != '010'	4✔
215
216	case field.tag	4✔
217	when /^[1-469]..\|0[2-9].\|01[1-9]\|7[0-5].\|5[0-24-9].\|53[0-24-9]/
218	field.subfields.each do \|subfield\|	1✔
219	next if subfield.value.nil?	1✔
220
221	subfield.value = extra_space_gsub(subfield.value.dup)	1✔
222	end
223	when '533'
224	field.subfields.each do \|subfield\|	1✔
225	next if subfield.code == '7' \|\| subfield.value.nil?	2✔
226
227	subfield.value = extra_space_gsub(subfield.value.dup)	1✔
228	end
229	when /^7[6-8]./
230	field.subfields.each do \|subfield\|	1✔
231	next if subfield.code =~ /[^a-v3-8]/ \|\| subfield.value.nil?	1✔
232
233	subfield.value = extra_space_gsub(subfield.value.dup)	1✔
234	end
235	when /^8../
236	field.subfields.each do \|subfield\|	1✔
237	next if %w[w 7].include?(subfield.code) \|\| subfield.value.nil?	1✔
238
239	subfield.value = extra_space_gsub(subfield.value.dup)	1✔
240	end
241	end
242	end
243	record	4✔
244	end
245
246	def multiple_no_040?(record)	1✔
247	record.fields('040').size != 1	1✔
248	end
249
250	def multiple_no_040b?(record)	1✔
251	f040 = record.fields('040')	3✔
252	return true if f040.size != 1	3✔
253
254	f040 = f040.first	3✔
255	b040 = f040.subfields.select { \|subfield\| subfield.code == 'b' }	7✔
256	return true if b040.size != 1	3✔
257
258	b040.first.value.match?(/^\s*$/)	2✔
259	end
260
261	def f046_errors?(record)	1✔
262	subf_codes = %w[b c d e]	4✔
263	subf_a_values = %w[r s p t x q n i k r m t x n]	4✔
264	f046 = record.fields('046')	4✔
265	return false if f046.empty?	4✔
266
267	f046.each do \|field\|	3✔
268	codes = field.subfields.map(&:code)	3✔
269	return true if field['a'] && !subf_a_values.include?(field['a'])	3✔
270	return true if field['a'].nil? && (subf_codes & codes).size.positive?	2✔
271	end
272	false	1✔
273	end
274
275	def multiple_no_245?(record)	1✔
276	record.fields('245').size != 1	24✔
277	end
278
279	def missing_040c?(record)	1✔
280	return true unless record['040'] && record['040']['c']	2✔
281
282	false	1✔
283	end
284
285	def pair_880_errors?(record)	1✔
286	return true if record.fields('880').select { \|field\| field['6'].nil? }.size.positive?	12✔
287
288	pair_880s = f880_pairings(record)	5✔
289	linked_fields = linked_field_pairings(record)	5✔
290	pair_880s.uniq != pair_880s \|\| pair_880s.uniq.sort != linked_fields.uniq.sort	5✔
291	end
292
293	def f880_pairings(record)	1✔
294	target_fields = record.fields('880').select do \|field\|	5✔
295	field['6'] =~ /^[0-9]{3}-[0-9][1-9]/	5✔
296	end
297	target_fields.map do \|field\|	5✔
298	field['6'].gsub(/^([0-9]{3}-[0-9]{2}).*$/, '\1')	4✔
299	end
300	end
301
302	def linked_field_pairings(record)	1✔
303	target_fields = record.fields('010'..'899').select do \|field\|	5✔
304	field.tag != '880' && field['6']	10✔
305	end
306	target_fields.map do \|field\|	5✔
307	"#{field.tag}-#{field['6'].gsub(/^880-([0-9]{2}).*$/, '\1')}"	3✔
308	end
309	end
310
311	def has_130_240?(record)	1✔
312	(%w[130 240] - record.tags).empty?	24✔
313	end
314
315	def multiple_1xx?(record)	1✔
316	record.fields('100'..'199').size > 1	24✔
317	end
318
319	def relator_chars?(record)	1✔
320	record.fields(%w[100 110 111 700 710 711]).each do \|field\|	4✔
321	target_subfields = relator_chars_target_subfields(field)	4✔
322	return true if target_subfields.select do \|subfield\|	4✔
323	subfield.value =~ /[^a-z\-, .]/	12✔
324	end.size.positive?
325	end
326	false	2✔
327	end
328
329	def relator_chars_target_subfields(field)	1✔
330	case field.tag	4✔
331	when '111', '711'
332	field.subfields.select { \|subf\| subf.code == 'j' }	10✔
333	else
334	field.subfields.select { \|subf\| subf.code == 'e' }	10✔
335	end
336	end
337
338	def x00_subfq?(record)	1✔
339	record.fields(%w[100 600 700 800]).each do \|field\|	2✔
340	field.subfields.each do \|subfield\|	2✔
341	return true if subfield.code == 'q' && subfield.value =~ /^[^(].*[^)]$/	4✔
342	end
343	end
344	false	1✔
345	end
346
347	def x00_subfd_no_comma?(record)	1✔
348	record.fields(%w[100 600 700 800]).each do \|field\|	2✔
349	subf_d_index = field.subfields.index { \|subfield\| subfield.code == 'd' }	7✔
350	next unless subf_d_index	2✔
351	return true if field.subfields[subf_d_index - 1].value =~ /[^,]$/	2✔
352	end
353	false	1✔
354	end
355
356	def relator_comma?(record)	1✔
357	record.fields(%w[100 110 111 700 710 711]).each do \|field\|	4✔
358	relator_index = relator_subfield_index(field)	4✔
359	next unless relator_index	4✔
360	return true if field.subfields[relator_index - 1].value =~ /[^,]$/	4✔
361	end
362	false	2✔
363	end
364
365	def relator_subfield_index(field)	1✔
366	case field.tag	4✔
367	when '111', '711'
368	field.subfields.index { \|subfield\| subfield.code == 'j' }	6✔
369	else
370	field.subfields.index { \|subfield\| subfield.code == 'e' }	6✔
371	end
372	end
373
374	def heading_end_punct?(record)	1✔
375	punct_regex = /[^").!?-]$/	2✔
376	record.fields(punctuated_heading_fields).each do \|field\|	2✔
377	last_heading_subfield = last_heading_subfield(field)	2✔
378	next unless last_heading_subfield	2✔
379	return true if last_heading_subfield.value =~ punct_regex	2✔
380	end
381	false	1✔
382	end
383
384	def punctuated_heading_fields	1✔
385	%w[	2✔
386	100 110 111 130
387	600 610 611 630 650 651 654 655 656 657 658 662
388	700 710 711 730 740 752 754
389	800 810 811 830
390	]
391	end
392
393	def last_heading_subfield(field)	1✔
394	regex = /[^02345]/	2✔
395	heading_subfields = field.subfields.select do \|subfield\|	2✔
396	subfield.code =~ regex	6✔
397	end
398	if heading_subfields.empty?	2✔
399	nil
400	else
401	heading_subfields[-1]	2✔
402	end
403	end
404
405	def subf_0_uri?(record)	1✔
406	record.fields.each do \|field\|	3✔
407	next unless field.instance_of?(MARC::DataField) && field.tag =~ /^[^9]/ && field['0']	3✔
408
409	field.subfields.each do \|subfield\|	2✔
410	return true if subfield.code == '0' && subfield.value =~ /^\(uri\)/	4✔
411	end
412	end
413	false	2✔
414	end
415
416	def bad_uri?(record)	1✔
417	target_fields = record.fields('856')	×
UNCOV 418	return false if target_fields.empty?	×
419
420	target_fields.each do \|field\|	×
UNCOV 421	next unless field['u']	×
422
423	field.subfields.each do \|subfield\|	×
UNCOV 424	next unless subfield.code == 'u'	×
425
426	string = subfield.value	×
UNCOV 427	return true unless URI.escape(URI.unescape(string).scrub) == string	×
428	end
429	end
UNCOV 430	false	×
431	end
432
433	### Normalize to the NFC (combined) form of diacritics for characters with
434	# Arabic diacritics; normalize to NFD for characters below U+622 and
435	# between U+1E00 and U+2A28
436	def composed_chars_normalize(record)	1✔
437	record.fields.each do \|field\|	2✔
438	next unless field.class == MARC::DataField	2✔
439
440	field_index = record.fields.index(field)	2✔
441	curr_subfield = 0	2✔
442	field.subfields.each do \|subfield\|	2✔
443	prevalue = subfield.value	4✔
444	if prevalue =~ /^.[\u0653\u0654\u0655].$/	4✔
445	prevalue = prevalue.unicode_normalize(:nfc)	1✔
446	end
447	fixed_subfield = prevalue.codepoints.map do \|codepoint\|	4✔
448	char = codepoint.chr(Encoding::UTF_8)	84✔
449	char.unicode_normalize!(:nfd) if codepoint < 1570 \|\| (7_680..10_792).cover?(codepoint)	84✔
450	char	84✔
451	end.join
452	record.fields[field_index].subfields[curr_subfield].value = fixed_subfield	4✔
453	curr_subfield += 1	4✔
454	end
455	end
456	record	2✔
457	end
458
459	### Replace empty indicators with a space;
460	### scrub indicators with bad UTF-8;
461	### The ruby-marc gem converts nil subfields to spaces
462	def empty_indicator_fix(record)	1✔
463	record.fields.each do \|field\|	2✔
464	next unless field.instance_of?(MARC::DataField)	2✔
465
466	ind1_value = field.indicator1.dup	2✔
467	ind1_value.scrub!('')	2✔
468	field.indicator1 = ' ' if ind1_value.empty?	2✔
469	ind2_value = field.indicator2.dup	2✔
470	ind2_value.scrub!('')	2✔
471	field.indicator2 = ' ' if ind2_value.empty?	2✔
472	end
473	record	2✔
474	end
475
476	### Remove empty subfields from DataFields
477	def empty_subfield_fix(record)	1✔
478	record.fields.each do \|field\|	1✔
479	next unless field.instance_of?(MARC::DataField)	2✔
480
481	field.subfields.delete_if { \|subfield\| subfield.value.nil? \|\| subfield.value.empty? }	11✔
482	end
483	record.fields.delete_if { \|field\| field.instance_of?(MARC::DataField) && field.subfields.empty? }	3✔
484	record	1✔
485	end
486
487	### Remove the (uri) prefix from subfield 0s
488	def subf_0_uri_fix(record)	1✔
489	record.fields.each do \|field\|	3✔
490	next unless field.instance_of?(MARC::DataField) && field.tag[0] != '9' && field['0']	3✔
491
492	field.subfields.each do \|subfield\|	2✔
493	next unless subfield.code == '0' && subfield.value =~ /^\(uri\)/	4✔
494
495	subfield.value = subfield.value.dup.delete_prefix('(uri)')	1✔
496	end
497	end
498	record	3✔
499	end
500
501	### Escape URIs
502	def uri_escape(record)	1✔
503	target_fields = record.fields('856')	×
504	return record if target_fields.empty?	×
505
506	fixed_record = record	×
507	target_fields.each do \|field\|	×
508	next unless field['u']	×
509
510	field_index = fixed_record.fields.index(field)	×
511	field.subfields.each do \|subfield\|	×
512	next unless subfield.code == 'u'	×
513
UNCOV 514	subfield_index = field.subfields.index(subfield)	×
UNCOV 515	string = subfield.value	×
516	fixed_string = URI.escape(URI.unescape(string).scrub)	×
UNCOV 517	fixed_record.fields[field_index].subfields[subfield_index].value = fixed_string	×
518	end
519	end
UNCOV 520	fixed_record	×
521	end
522
523	### Make the 040 $b 'eng' if it doesn't have a value
524	def fix_040b(record)	1✔
525	return record unless record.fields('040').size == 1	2✔
526
527	f040 = record['040']	2✔
528	field_index = record.fields.index(f040)	2✔
529	b040 = f040.subfields.select { \|subfield\| subfield.code == 'b' }	4✔
530	return record unless b040.empty?	2✔
531
532	subf_codes = f040.subfields.map(&:code)	2✔
533	subf_index = if f040['a']	2✔
534	(subf_codes.index { \|i\| i == 'a' }) + 1	2✔
535	else
536	0	1✔
537	end
538	subf_b = MARC::Subfield.new('b', 'eng')	2✔
539	record.fields[field_index].subfields.insert(subf_index, subf_b)	2✔
540	record	2✔
541	end
542
543	### Split up subfields that contain multiple 3-letter language codes
544	def fix_f041(record)	1✔
545	f041 = record.fields('041')	2✔
546	return record if f041.empty?	2✔
547
548	f041.each do \|field\|	2✔
549	f_index = record.fields.index(field)	2✔
550	new_field = MARC::DataField.new('041', field.indicator1, field.indicator2)	2✔
551	field.subfields.each do \|subfield\|	2✔
552	code = subfield.code	2✔
553	val = subfield.value	2✔
554	if (val.size % 3).zero?	2✔
555	langs = val.scan(/.../)	1✔
556	langs.each do \|lang\|	1✔
557	new_field.append(MARC::Subfield.new(code, lang))	3✔
558	end
559	else
560	new_field.append(MARC::Subfield.new(code, val))	1✔
561	end
562	end
563	record.fields[f_index] = new_field	2✔
564	end
565	record	2✔
566	end
567
568	### Removes text from the beginning of a subfield
569	### An array of hashes of the format { field:, subfields: } will be passed
570	### in the targets: symbol
571	### subfield: is an array of subfield codes
572	def remove_prefix_from_subfield(record:, targets:, string:)	1✔
573	targets.each do \|target\|	1✔
574	record.fields(target[:field]).each do \|field\|	1✔
575	field.subfields.each do \|subfield\|	1✔
576	next unless target[:subfields].include?(subfield.code)	2✔
577
578	subfield.value = subfield.value.dup.delete_prefix(string)	1✔
579	end
580	end
581	end
582	record	1✔
583	end
584
585	### Adds text to the beginning of a subfield
586	### An array of hashes of the format { field:, subfields: } will be passed
587	### in the targets: symbol
588	### subfield: is an array of subfield codes
589	def add_prefix_to_subfield(record:, targets:, string:)	1✔
590	targets.each do \|target\|	1✔
591	record.fields(target[:field]).each do \|field\|	1✔
592	field.subfields.each do \|subfield\|	1✔
593	next unless target[:subfields].include?(subfield.code)	2✔
594
595	subfield.value = subfield.value.dup.prepend(string)	1✔
596	end
597	end
598	end
599	record	1✔
600	end
601
602	### Sort subfields for target fields with an arbitrary order
603	### Example order_array: ['a', 'b', 'c']
604	def subfield_sort(record:, target_tags:, order_array: nil)	1✔
605	record.fields(target_tags).each do \|field\|	3✔
606	next if field.instance_of?(MARC::ControlField)	3✔
607
608	order_array \|\|= field.subfields.map(&:code).uniq.sort	2✔
609	new_subfields = sort_listed_subfields(field: field, order_array: order_array)	2✔
610	new_subfields += find_unlisted_subfields(field: field, order_array: order_array)	2✔
611	field.subfields = new_subfields	2✔
612	end
613	record	3✔
614	end
615
616	def find_unlisted_subfields(field:, order_array:)	1✔
617	field.subfields.reject do \|subfield\|	2✔
618	order_array.include?(subfield.code)	7✔
619	end
620	end
621
622	def sort_listed_subfields(field:, order_array:)	1✔
623	listed_subfields = field.subfields.select do \|subfield\|	2✔
624	order_array.include?(subfield.code)	7✔
625	end
626	listed_subfields.sort_by! do \|subfield\|	2✔
627	order_array.index { \|code\| code == subfield.code }	20✔
628	end
629	end
630	end

pulibrary / marc_cleanup / 7a49e047-f3ae-41b9-a3d7-9924d268f02a

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous