f2723d8b-bf8c-4ddb-ba2c-9aece62d6d17

Committed 07 Nov 2024 04:29PM UTC coverage: 77.101% (+0.1%) from 76.957%

Build # f2723d8b-bf8c-4ddb-ba2c-9aece62d6d17

Build Type

push

circleci

Committed by

web-flow

Commit Message

add subfield prefix methods (#127)

Run Details

90 of 90 new or added lines in 1 file covered. (100.0%)

1 existing line in 1 file now uncovered.

1734 of 2249 relevant lines covered (77.1%)

3.51 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.51

/lib/marc_cleanup/variable_fields.rb

# frozen_string_literal: true

module MarcCleanup
  ### Remove non-numerical strings and append a new 020$q with the string
  def new_020_q(record)
    record.fields('020').each do |f020|
      f020.subfields.each do |subfield|
        next unless subfield.code == 'a'
        isbn_parts = /^\s*([\d\-]+)\s*(\(.*?\))\s*$/.match(subfield.value)
        next if isbn_parts.nil?
        subfield.value = isbn_parts[1]
        f020.append(MARC::Subfield.new('q', isbn_parts[2]))
      end
    end
    record
  end

  ### Convert ISBN-10 to ISBN-13
  def isbn10_to_13(isbn)
    stem = isbn[0..8]
    return nil if stem =~ /\D/

    existing_check = isbn[9]
    return nil if existing_check && existing_check != checkdigit_10(stem)

    main = ISBN13PREFIX + stem
    checkdigit = checkdigit_13(main)
    main + checkdigit
  end

  ### Calculate check digit for ISBN-10
  def checkdigit_10(stem)
    int_index = 0
    int_sum = 0
    stem.each_char do |digit|
      int_sum += digit.to_i * (10 - int_index)
      int_index += 1
    end
    mod = (11 - (int_sum % 11)) % 11
    mod == 10 ? 'X' : mod.to_s
  end

  ### Calculate check digit for ISBN-13
  def checkdigit_13(stem)
    int_index = 0
    int_sum = 0
    stem.each_char do |digit|
      int_sum += int_index.even? ? digit.to_i : digit.to_i * 3
      int_index += 1
    end
    ((10 - (int_sum % 10)) % 10).to_s
  end

  ### Normalize ISBN-13
  def isbn13_normalize(raw_isbn)
    int_sum = 0
    stem = raw_isbn[0..11]
    return nil if stem =~ /\D/

    int_index = 0
    stem.each_char do |digit|
      int_sum += int_index.even? ? digit.to_i : digit.to_i * 3
      int_index += 1
    end
    checkdigit = checkdigit_13(stem)
    return nil if raw_isbn[12] && raw_isbn[12] != checkdigit

    stem + checkdigit
  end

  ### Normalize any given string that is supposed to include an ISBN
  def isbn_normalize(isbn)
    return nil unless isbn

    raw_isbn = isbn.dup
    raw_isbn.delete!('-')
    raw_isbn.delete!('\\')
    raw_isbn.gsub!(/\([^\)]*\)/, '')
    raw_isbn.gsub!(/^(.*)\$c.*$/, '\1')
    raw_isbn.gsub!(/^(.*)\$q.*$/, '\1')
    raw_isbn.gsub!(/^\D+([0-9].*)$/, '\1')
    if raw_isbn =~ /^978/
      raw_isbn.gsub!(/^(978[0-9 ]+).*$/, '\1')
      raw_isbn.delete!(' ')
    else
      raw_isbn.gsub!(/([0-9])\s*([0-9]{4})\s*([0-9]{4})\s*([0-9xX]).*$/, '\1\2\3\4')
    end
    raw_isbn.gsub!(/^([0-9]{9,13}[xX]?)[^0-9xX].*$/, '\1')
    raw_isbn.gsub!(/^([0-9]+?)\D.*$/, '\1')
    if raw_isbn.length > 6 && raw_isbn.length < 9 && raw_isbn =~ /^[0-9]+$/
      raw_isbn = raw_isbn.ljust(9, '0')
    end
    valid_lengths = [9, 10, 12, 13] # ISBN10 and ISBN13 with/out check digits
    return nil unless valid_lengths.include? raw_isbn.length

    if raw_isbn.length < 12
      isbn10_to_13(raw_isbn)
    else
      isbn13_normalize(raw_isbn)
    end
  end

  ### If the ISBN is invalid, change the subfield code to z
  ### Otherwise, replace ISBN with normalized ISBN
  def move_invalid_isbn(record)
    record.fields('020').each do |f020|
      f020.subfields.each do |subfield|
        next unless subfield.code == 'a'
        isbn = subfield.value
        normalized_isbn = isbn_normalize(isbn)
        if normalized_isbn
          subfield.value = normalized_isbn
        else
          subfield.code = 'z'
        end
      end
    end
    record
  end

  # check the 041 field for errors
  # 041 is a language code
  def f041_errors?(record)
    f041 = record.fields('041')
    return false if f041.empty?

    f041.each do |field|
      field.subfields.each do |subfield|
        val = subfield.value
        return true if (val.size > 3) && (val.size % 3 == 0)
      end
    end
    false
  end

  # http://www.loc.gov/standards/valuelist/marcauthen.html
  def auth_codes_042
    %w[
      anuc
      croatica
      dc
      dhca
      dlr
      gamma
      gils
      gnd1
      gnd2
      gnd3
      gnd4
      gnd5
      gnd6
      gnd7
      gndz
      isds/c
      issnuk
      lacderived
      lc
      lcac
      lccopycat
      lccopycat-nm
      lcd
      lcderive
      lchlas
      lcllh
      lcnccp
      lcnitrate
      lcnuc
      lcode
      msc
      natgaz
      nbr
      nlc
      nlmcopyc
      norbibl
      nsdp
      nst
      ntccf
      nznb
      pcc
      premarc
      reveal
      sanb
      scipio
      toknb
      ukblcatcopy
      ukblderived
      ukblproject
      ukblsr
      ukscp
      xisds/c
      xissnuk
      xlc
      xnlc
      xnsdp
    ]
  end

  def auth_code_error?(record)
    return false unless record['042']

    auth_codes_042.include?(record['042']['a']) ? false : true
  end

  def invalid_indicators?(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField

      return true unless field.indicator1 =~ /^[0-9 ]$/ && field.indicator2 =~ /^[0-9 ]$/
    end
    false
  end

  def invalid_subfield_code?(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField

      field.subfields.each do |subfield|
        return true unless subfield.code =~ /^[0-9a-z]$/
      end
    end
    false
  end

  def empty_subfields?(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField

      field.subfields.each do |subfield|
        return true if subfield.value =~ /^[[:blank:]]*$/
      end
    end
    false
  end

  def extra_spaces?(record)
    blank_regex = /^.*[[:blank:]]{2,}.*$|^.*[[:blank:]]+$|^[[:blank:]]+(.*)$/
    record.fields.each do |field|
      next unless field.class == MARC::DataField && field.tag != '010'

      if field.tag =~ /[1-469]..|0[2-9].|01[1-9]|7[0-5].|5[0-24-9].|53[0-24-9]/
        field.subfields.each do |subfield|
          return true if subfield.value =~ blank_regex
        end
      elsif field.tag == '533'
        field.subfields.each do |subfield|
          next if subfield.code == '7'

          return true if subfield.value =~ blank_regex
        end
      elsif field.tag =~ /7[6-8]./
        field.subfields.each do |subfield|
          next unless subfield.code =~ /[a-v3-8]/

          return true if subfield.value =~ blank_regex
        end
      elsif field.tag =~ /8../
        field.subfields.each do |subfield|
          next unless subfield.code =~ /[^w7]/

          return true if subfield.value =~ blank_regex
        end
      end
    end
    false
  end

  def multiple_no_040?(record)
    f040 = record.fields('040')
    f040.size != 1
  end

  def multiple_no_040b?(record)
    f040 = record.fields('040')
    return true if f040.size != 1

    f040 = f040.first
    b040 = f040.subfields.select { |subfield| subfield.code == 'b' }
    return true if b040.size != 1

    b040 = b040.first.value
    b040.gsub!(/[ ]/, '')
    b040 == ''
  end

  def f046_subfield_errors?(record)
    f046 = record.fields('046')
    return false if f046.empty?

    f046.each do |field|
      subf_codes = field.subfields.map { |subfield| subfield.code }
      return true if field['a'].nil? && (subf_codes & %w[b c d e]).size > 0
    end
    false
  end

  def multiple_no_245?(record)
    record.fields('245').size != 1
  end

  def f245_subfield_errors?(record)
    fields = record.fields('245')
    return true if fields.empty?

    fields.each do |field|
      subfields = field.subfields.map(&:code)
      return true if subfields.count('a') != 1
      return true if subfields.count('b') > 1
      return true if subfields.count('c') > 1
    end
    false
  end

  def missing_040c?(record)
    return true unless record['040'] && record['040']['c']

    false
  end

  def pair_880_errors?(record)
    pair_880s = []
    linked_fields = []
    return false unless record['880']

    record.fields.each do |field|
      return true if field.tag == '880' && field['6'].nil?

      next unless field.tag =~ /^[0-8]..$/ && field.class == MARC::DataField && field['6']

      if field.tag == '880'
        pair_880s << field['6'].gsub(/^([0-9]{3}-[0-9]{2}).*$/, '\1')
      else
        linked_fields << "#{field.tag}-#{field['6'].gsub(/^880-([0-9]{2}).*$/, '\1').chomp}"
      end
    end
    pair_880s.delete_if { |x| x =~ /^.*-00/ }
    return true if pair_880s.uniq != pair_880s || pair_880s.uniq.sort != linked_fields.uniq.sort

    false
  end

  def has_130_240?(record)
    (%w[130 240] - record.tags).empty?
  end

  def multiple_1xx?(record)
    record.fields('100'..'199').size > 1
  end

  def relator_chars?(record)
    record.fields.each do |field|
      if field.tag =~ /[17][01]0/
        field.subfields.each do |subfield|
          next unless subfield.code == 'e' && subfield.value =~ /.*[^a-z, \.].*/

          return true
        end
      elsif field.tag =~ /[17]11/
        field.subfields.each do |subfield|
          next unless subfield.code == 'j' && subfield.value =~ /.*[^a-z, \.].*/

          return true
        end
      end
    end
    false
  end

  def x00_subfq?(record)
    record.fields(%w[100 600 700 800]).each do |field|
      field.subfields.each do |subfield|
        next unless subfield.code == 'q' && subfield.value =~ /^(?!\([^\)]*\))$/

        return true
      end
    end
    false
  end

  def no_comma_x00?(record)
    record.fields(%w[100 600 700 800]).each do |field|
      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      subfx_d_index = code_array.index(/.d/)
      return true if subfx_d_index && field.subfields[subfx_d_index].value =~ /^.*[^,]$/
    end
    false
  end

  def relator_comma?(record)
    comma_regex = /^.*[^,]$/
    record.fields.each do |field|
      next unless field.tag =~ /[17][01][01]/

      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      if field.tag =~ /[17][01]0/
        subfx_e_index = code_array.index(/.e/)
        return true if subfx_e_index && field.subfields[subfx_e_index].value =~ comma_regex
      elsif field.tag =~ /[17]11/
        subfx_j_index = code_array.index(/.j/)
        return true if subfx_j_index && field.subfields[subfx_j_index].value =~ comma_regex
      end
    end
    false
  end

  def heading_end_punct?(record)
    punct_regex = /.*[^"\).\!\?\-]$/
    record.fields.each do |field|
      next unless field.tag =~ /^[167][0-5].$/ && field.indicator2 =~ /[^47]/

      code_array = ''
      field.subfields.each do |subfield|
        code_array << subfield.code
      end
      last_heading_subfield_index = code_array.index(/[a-vx-z8][^a-vx-z8]*$/)
      return true if last_heading_subfield_index && field.subfields[last_heading_subfield_index].value =~ punct_regex
    end
    false
  end

  def lowercase_headings?(record)
    record.fields.each do |field|
      next unless field.tag =~ /[1678]../

      return true if field['a'] =~ /^[a-z]{3,}/
    end
    false
  end

  def subf_0_uri?(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField && field.tag =~ /^[^9]/ && field['0']

      field.subfields.each do |subfield|
        return true if subfield.code == '0' && subfield.value =~ /^\(uri\)/
      end
    end
    false
  end

  def bad_uri?(record)
    target_fields = record.fields('856')
    return false if target_fields.empty?

    target_fields.each do |field|
      next unless field['u']

      field.subfields.each do |subfield|
        next unless subfield.code == 'u'

        string = subfield.value
        return true unless URI.escape(URI.unescape(string).scrub) == string
      end
    end
    false
  end

  ### Normalize to the NFC (combined) form of diacritics for characters with
  #     Arabic diacritics; normalize to NFD for characters below U+622 and
  #     between U+1E00 and U+2A28
  def composed_chars_normalize(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField

      field_index = record.fields.index(field)
      curr_subfield = 0
      field.subfields.each do |subfield|
        fixed_subfield = ''
        prevalue = subfield.value
        if prevalue =~ /^.*[\u0653\u0654\u0655].*$/
          prevalue = prevalue.unicode_normalize(:nfc)
        end
        prevalue.each_codepoint do |c|
          char = c.chr(Encoding::UTF_8)
          char.unicode_normalize!(:nfd) if c < 1570 || (7_680..10_792).cover?(c)
          fixed_subfield << char
        end
        record.fields[field_index].subfields[curr_subfield].value = fixed_subfield
        curr_subfield += 1
      end
    end
    record
  end

  ### Replace empty indicators with a space;
  #     scrub indicators with bad UTF-8
  def empty_indicator_fix(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField

      if field.indicator1.nil?
        field.indicator1 = ' '
      else
        field.indicator1.scrub!('')
        field.indicator1 = ' ' if field.indicator1.size < 1
      end
      if field.indicator2.nil?
        field.indicator2 = ' '
      else
        field.indicator2.scrub!('')
        field.indicator2 = ' ' if field.indicator2.size < 1
      end
    end
    record
  end

  ### Remove empty subfields from DataFields
  def empty_subfield_fix(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField
      field.subfields.delete_if { |subfield| subfield.value.nil? || subfield.value.empty? }
    end
    record.fields.delete_if { |field| field.class == MARC::DataField && field.subfields.empty? }
    record
  end

  ### Remove the (uri) prefix from subfield 0s
  def subf_0_fix(record)
    record.fields.each do |field|
      next unless field.class == MARC::DataField && field.tag =~ /^[^9]/ && field['0']

      field_index = record.fields.index(field)
      field.subfields.each do |subfield|
        next unless subfield.code == '0' && subfield.value =~ /^\(uri\)/

        subfield_index = field.subfields.index(subfield)
        record.fields[field_index].subfields[subfield_index].value = subfield.value.gsub(/^\(uri\)(.*)$/, '\1')
      end
    end
    record
  end

  ### Escape URIs
  def uri_escape(record)
    target_fields = record.fields('856')
    return record if target_fields.empty?

    fixed_record = record
    target_fields.each do |field|
      next unless field['u']

      field_index = fixed_record.fields.index(field)
      field.subfields.each do |subfield|
        next unless subfield.code == 'u'

        subfield_index = field.subfields.index(subfield)
        string = subfield.value
        fixed_string = URI.escape(URI.unescape(string).scrub)
        fixed_record.fields[field_index].subfields[subfield_index].value = fixed_string
      end
    end
    fixed_record
  end

  ### Make the 040 $b 'eng' if it doesn't have a value
  def fix_040b(record)
    f040 = record.fields('040')
    return record unless f040.size == 1

    f040 = f040.first
    field_index = record.fields.index(f040)
    b040 = f040.subfields.select { |subfield| subfield.code == 'b' }
    return record unless b040.empty?

    subf_codes = f040.subfields.map { |subfield| subfield.code }
    subf_index = if f040['a']
                   (subf_codes.index { |i| i == 'a' }) + 1
                 else
                   0
                 end
    subf_b = MARC::Subfield.new('b', 'eng')
    record.fields[field_index].subfields.insert(subf_index, subf_b)
    record
  end

  ### Split up subfields that contain multiple 3-letter language codes
  def fix_041(record)
    f041 = record.fields('041')
    return record if f041.empty?

    f041.each do |field|
      f_index = record.fields.index(field)
      new_field = MARC::DataField.new('041', field.indicator1, field.indicator2)
      field.subfields.each do |subfield|
        code = subfield.code
        val = subfield.value
        next unless val.size % 3 == 0

        langs = val.scan(/.../)
        langs.each do |lang|
          new_field.append(MARC::Subfield.new(code, lang))
        end
      end
      record.fields[f_index] = new_field
    end
    record
  end

  ### Removes text from the beginning of a subfield
  ### An array of hashes of the format { field:, subfields: } will be passed
  ###   in the targets: symbol
  ###   subfield: is an array of subfield codes
  def remove_prefix_from_subfield(record:, targets:, string:)
    targets.each do |target|
      record.fields(target[:field]).each do |field|
        field.subfields.each do |subfield|
          next unless target[:subfields].include?(subfield.code)

          subfield.value = subfield.value.dup.delete_prefix(string)
        end
      end
    end
    record
  end

  ### Adds text to the beginning of a subfield
  ### An array of hashes of the format { field:, subfields: } will be passed
  ###   in the targets: symbol
  ###   subfield: is an array of subfield codes
  def add_prefix_to_subfield(record:, targets:, string:)
    targets.each do |target|
      record.fields(target[:field]).each do |field|
        field.subfields.each do |subfield|
          next unless target[:subfields].include?(subfield.code)

          subfield.value = subfield.value.dup.prepend(string)
        end
      end
    end
    record
  end

  ### Sort subfields for target fields with an arbitrary order
  def subfield_sort(record, target_tags, order_array = nil)
    target_fields = record.fields.select { |f| target_tags.include?(f.tag) }
    return record if target_fields.empty?

    target_fields.each do |field|
      next unless field.class == MARC::DataField

      orig_codes = field.subfields.map { |subfield| subfield.code }.uniq.sort
      order_array = orig_codes if order_array.nil?
      new_subfields = []
      order_array.each do |code|
        next unless orig_codes.include?(code)

        target_subf = field.subfields.select { |subfield| subfield.code == code }
        target_subf.each { |subfield| new_subfields << subfield }
      end
      rem_subfields = field.subfields.select { |subf| !order_array.include?(subf.code) }
      rem_subfields.each do |subfield|
        new_subfields << subfield
      end
      field.subfields = new_subfields
    end
    record
  end
end

1	# frozen_string_literal: true
2
3	module MarcCleanup	1✔
4	### Remove non-numerical strings and append a new 020$q with the string
5	def new_020_q(record)	1✔
6	record.fields('020').each do \|f020\|	1✔
7	f020.subfields.each do \|subfield\|	1✔
8	next unless subfield.code == 'a'	2✔
9	isbn_parts = /^\s([\d\-]+)\s(\(.?\))\s$/.match(subfield.value)	1✔
10	next if isbn_parts.nil?	1✔
11	subfield.value = isbn_parts[1]	1✔
12	f020.append(MARC::Subfield.new('q', isbn_parts[2]))	1✔
13	end
14	end
15	record	1✔
16	end
17
18	### Convert ISBN-10 to ISBN-13
19	def isbn10_to_13(isbn)	1✔
20	stem = isbn[0..8]	2✔
21	return nil if stem =~ /\D/	2✔
22
23	existing_check = isbn[9]	2✔
24	return nil if existing_check && existing_check != checkdigit_10(stem)	2✔
25
26	main = ISBN13PREFIX + stem	2✔
27	checkdigit = checkdigit_13(main)	2✔
28	main + checkdigit	2✔
29	end
30
31	### Calculate check digit for ISBN-10
32	def checkdigit_10(stem)	1✔
33	int_index = 0	1✔
34	int_sum = 0	1✔
35	stem.each_char do \|digit\|	1✔
36	int_sum += digit.to_i * (10 - int_index)	9✔
37	int_index += 1	9✔
38	end
39	mod = (11 - (int_sum % 11)) % 11	1✔
40	mod == 10 ? 'X' : mod.to_s	1✔
41	end
42
43	### Calculate check digit for ISBN-13
44	def checkdigit_13(stem)	1✔
45	int_index = 0	4✔
46	int_sum = 0	4✔
47	stem.each_char do \|digit\|	4✔
48	int_sum += int_index.even? ? digit.to_i : digit.to_i * 3	48✔
49	int_index += 1	48✔
50	end
51	((10 - (int_sum % 10)) % 10).to_s	4✔
52	end
53
54	### Normalize ISBN-13
55	def isbn13_normalize(raw_isbn)	1✔
56	int_sum = 0	2✔
57	stem = raw_isbn[0..11]	2✔
58	return nil if stem =~ /\D/	2✔
59
60	int_index = 0	2✔
61	stem.each_char do \|digit\|	2✔
62	int_sum += int_index.even? ? digit.to_i : digit.to_i * 3	24✔
63	int_index += 1	24✔
64	end
65	checkdigit = checkdigit_13(stem)	2✔
66	return nil if raw_isbn[12] && raw_isbn[12] != checkdigit	2✔
67
68	stem + checkdigit	1✔
69	end
70
71	### Normalize any given string that is supposed to include an ISBN
72	def isbn_normalize(isbn)	1✔
73	return nil unless isbn	4✔
74
75	raw_isbn = isbn.dup	4✔
76	raw_isbn.delete!('-')	4✔
77	raw_isbn.delete!('\\')	4✔
78	raw_isbn.gsub!(/\([^\)]*\)/, '')	4✔
79	raw_isbn.gsub!(/^(.)\$c.$/, '\1')	4✔
80	raw_isbn.gsub!(/^(.)\$q.$/, '\1')	4✔
81	raw_isbn.gsub!(/^\D+([0-9].*)$/, '\1')	4✔
82	if raw_isbn =~ /^978/	4✔
83	raw_isbn.gsub!(/^(978[0-9 ]+).*$/, '\1')	2✔
84	raw_isbn.delete!(' ')	2✔
85	else
86	raw_isbn.gsub!(/([0-9])\s([0-9]{4})\s([0-9]{4})\s([0-9xX]).$/, '\1\2\3\4')	2✔
87	end
88	raw_isbn.gsub!(/^([0-9]{9,13}[xX]?)[^0-9xX].*$/, '\1')	4✔
89	raw_isbn.gsub!(/^([0-9]+?)\D.*$/, '\1')	4✔
90	if raw_isbn.length > 6 && raw_isbn.length < 9 && raw_isbn =~ /^[0-9]+$/	4✔
91	raw_isbn = raw_isbn.ljust(9, '0')	1✔
92	end
93	valid_lengths = [9, 10, 12, 13] # ISBN10 and ISBN13 with/out check digits	4✔
94	return nil unless valid_lengths.include? raw_isbn.length	4✔
95
96	if raw_isbn.length < 12	4✔
97	isbn10_to_13(raw_isbn)	2✔
98	else
99	isbn13_normalize(raw_isbn)	2✔
100	end
101	end
102
103	### If the ISBN is invalid, change the subfield code to z
104	### Otherwise, replace ISBN with normalized ISBN
105	def move_invalid_isbn(record)	1✔
106	record.fields('020').each do \|f020\|	4✔
107	f020.subfields.each do \|subfield\|	4✔
108	next unless subfield.code == 'a'	4✔
109	isbn = subfield.value	4✔
110	normalized_isbn = isbn_normalize(isbn)	4✔
111	if normalized_isbn	4✔
112	subfield.value = normalized_isbn	3✔
113	else
114	subfield.code = 'z'	1✔
115	end
116	end
117	end
118	record	4✔
119	end
120
121	# check the 041 field for errors
122	# 041 is a language code
123	def f041_errors?(record)	1✔
124	f041 = record.fields('041')	1✔
125	return false if f041.empty?	1✔
126
127	f041.each do \|field\|	1✔
128	field.subfields.each do \|subfield\|	1✔
129	val = subfield.value	1✔
130	return true if (val.size > 3) && (val.size % 3 == 0)	1✔
131	end
132	end
133	false	1✔
134	end
135
136	# http://www.loc.gov/standards/valuelist/marcauthen.html
137	def auth_codes_042	1✔
UNCOV 138	%w[	×
139	anuc
140	croatica
141	dc
142	dhca
143	dlr
144	gamma
145	gils
146	gnd1
147	gnd2
148	gnd3
149	gnd4
150	gnd5
151	gnd6
152	gnd7
153	gndz
154	isds/c
155	issnuk
156	lacderived
157	lc
158	lcac
159	lccopycat
160	lccopycat-nm
161	lcd
162	lcderive
163	lchlas
164	lcllh
165	lcnccp
166	lcnitrate
167	lcnuc
168	lcode
169	msc
170	natgaz
171	nbr
172	nlc
173	nlmcopyc
174	norbibl
175	nsdp
176	nst
177	ntccf
178	nznb
179	pcc
180	premarc
181	reveal
182	sanb
183	scipio
184	toknb
185	ukblcatcopy
186	ukblderived
187	ukblproject
188	ukblsr
189	ukscp
190	xisds/c
191	xissnuk
192	xlc
193	xnlc
194	xnsdp
195	]
196	end
197
198	def auth_code_error?(record)	1✔
199	return false unless record['042']	×
200
201	auth_codes_042.include?(record['042']['a']) ? false : true	×
202	end
203
204	def invalid_indicators?(record)	1✔
205	record.fields.each do \|field\|	×
206	next unless field.class == MARC::DataField	×
207
208	return true unless field.indicator1 =~ /^[0-9 ]$/ && field.indicator2 =~ /^[0-9 ]$/	×
209	end
210	false	×
211	end
212
213	def invalid_subfield_code?(record)	1✔
214	record.fields.each do \|field\|	×
215	next unless field.class == MARC::DataField	×
216
217	field.subfields.each do \|subfield\|	×
218	return true unless subfield.code =~ /^[0-9a-z]$/	×
219	end
220	end
221	false	×
222	end
223
224	def empty_subfields?(record)	1✔
225	record.fields.each do \|field\|	×
226	next unless field.class == MARC::DataField	×
227
228	field.subfields.each do \|subfield\|	×
229	return true if subfield.value =~ /^[[:blank:]]*$/	×
230	end
231	end
232	false	×
233	end
234
235	def extra_spaces?(record)	1✔
236	blank_regex = /^.[[:blank:]]{2,}.$\|^.[[:blank:]]+$\|^[[:blank:]]+(.)$/	×
237	record.fields.each do \|field\|	×
238	next unless field.class == MARC::DataField && field.tag != '010'	×
239
240	if field.tag =~ /[1-469]..\|0[2-9].\|01[1-9]\|7[0-5].\|5[0-24-9].\|53[0-24-9]/	×
241	field.subfields.each do \|subfield\|	×
242	return true if subfield.value =~ blank_regex	×
243	end
244	elsif field.tag == '533'	×
245	field.subfields.each do \|subfield\|	×
246	next if subfield.code == '7'	×
247
248	return true if subfield.value =~ blank_regex	×
249	end
250	elsif field.tag =~ /7[6-8]./	×
251	field.subfields.each do \|subfield\|	×
252	next unless subfield.code =~ /[a-v3-8]/	×
253
254	return true if subfield.value =~ blank_regex	×
255	end
256	elsif field.tag =~ /8../	×
257	field.subfields.each do \|subfield\|	×
258	next unless subfield.code =~ /[^w7]/	×
259
260	return true if subfield.value =~ blank_regex	×
261	end
262	end
263	end
264	false	×
265	end
266
267	def multiple_no_040?(record)	1✔
268	f040 = record.fields('040')	1✔
269	f040.size != 1	1✔
270	end
271
272	def multiple_no_040b?(record)	1✔
273	f040 = record.fields('040')	1✔
274	return true if f040.size != 1	1✔
275
276	f040 = f040.first	1✔
277	b040 = f040.subfields.select { \|subfield\| subfield.code == 'b' }	2✔
278	return true if b040.size != 1	1✔
279
280	b040 = b040.first.value	1✔
281	b040.gsub!(/[ ]/, '')	1✔
282	b040 == ''	1✔
283	end
284
285	def f046_subfield_errors?(record)	1✔
286	f046 = record.fields('046')	×
287	return false if f046.empty?	×
288
289	f046.each do \|field\|	×
290	subf_codes = field.subfields.map { \|subfield\| subfield.code }	×
291	return true if field['a'].nil? && (subf_codes & %w[b c d e]).size > 0	×
292	end
293	false	×
294	end
295
296	def multiple_no_245?(record)	1✔
297	record.fields('245').size != 1	23✔
298	end
299
300	def f245_subfield_errors?(record)	1✔
301	fields = record.fields('245')	×
302	return true if fields.empty?	×
303
304	fields.each do \|field\|	×
305	subfields = field.subfields.map(&:code)	×
306	return true if subfields.count('a') != 1	×
307	return true if subfields.count('b') > 1	×
308	return true if subfields.count('c') > 1	×
309	end
310	false	×
311	end
312
313	def missing_040c?(record)	1✔
314	return true unless record['040'] && record['040']['c']	2✔
315
316	false	1✔
317	end
318
319	def pair_880_errors?(record)	1✔
320	pair_880s = []	×
321	linked_fields = []	×
322	return false unless record['880']	×
323
324	record.fields.each do \|field\|	×
325	return true if field.tag == '880' && field['6'].nil?	×
326
327	next unless field.tag =~ /^[0-8]..$/ && field.class == MARC::DataField && field['6']	×
328
329	if field.tag == '880'	×
330	pair_880s << field['6'].gsub(/^([0-9]{3}-[0-9]{2}).*$/, '\1')	×
331	else
332	linked_fields << "#{field.tag}-#{field['6'].gsub(/^880-([0-9]{2}).*$/, '\1').chomp}"	×
333	end
334	end
335	pair_880s.delete_if { \|x\| x =~ /^.*-00/ }	×
336	return true if pair_880s.uniq != pair_880s \|\| pair_880s.uniq.sort != linked_fields.uniq.sort	×
337
338	false	×
339	end
340
341	def has_130_240?(record)	1✔
342	(%w[130 240] - record.tags).empty?	23✔
343	end
344
345	def multiple_1xx?(record)	1✔
346	record.fields('100'..'199').size > 1	23✔
347	end
348
349	def relator_chars?(record)	1✔
350	record.fields.each do \|field\|	×
351	if field.tag =~ /[17][01]0/	×
352	field.subfields.each do \|subfield\|	×
353	next unless subfield.code == 'e' && subfield.value =~ /.[^a-z, \.]./	×
354
355	return true	×
356	end
357	elsif field.tag =~ /[17]11/	×
358	field.subfields.each do \|subfield\|	×
359	next unless subfield.code == 'j' && subfield.value =~ /.[^a-z, \.]./	×
360
361	return true	×
362	end
363	end
364	end
365	false	×
366	end
367
368	def x00_subfq?(record)	1✔
369	record.fields(%w[100 600 700 800]).each do \|field\|	×
370	field.subfields.each do \|subfield\|	×
371	next unless subfield.code == 'q' && subfield.value =~ /^(?!\([^\)]*\))$/	×
372
373	return true	×
374	end
375	end
376	false	×
377	end
378
379	def no_comma_x00?(record)	1✔
380	record.fields(%w[100 600 700 800]).each do \|field\|	×
381	code_array = ''	×
382	field.subfields.each do \|subfield\|	×
383	code_array << subfield.code	×
384	end
385	subfx_d_index = code_array.index(/.d/)	×
386	return true if subfx_d_index && field.subfields[subfx_d_index].value =~ /^.*[^,]$/	×
387	end
388	false	×
389	end
390
391	def relator_comma?(record)	1✔
392	comma_regex = /^.*[^,]$/	×
393	record.fields.each do \|field\|	×
394	next unless field.tag =~ /[17][01][01]/	×
395
396	code_array = ''	×
397	field.subfields.each do \|subfield\|	×
398	code_array << subfield.code	×
399	end
400	if field.tag =~ /[17][01]0/	×
401	subfx_e_index = code_array.index(/.e/)	×
402	return true if subfx_e_index && field.subfields[subfx_e_index].value =~ comma_regex	×
403	elsif field.tag =~ /[17]11/	×
404	subfx_j_index = code_array.index(/.j/)	×
405	return true if subfx_j_index && field.subfields[subfx_j_index].value =~ comma_regex	×
406	end
407	end
408	false	×
409	end
410
411	def heading_end_punct?(record)	1✔
412	punct_regex = /.*[^"\).\!\?\-]$/	×
413	record.fields.each do \|field\|	×
414	next unless field.tag =~ /^[167][0-5].$/ && field.indicator2 =~ /[^47]/	×
415
416	code_array = ''	×
417	field.subfields.each do \|subfield\|	×
418	code_array << subfield.code	×
419	end
420	last_heading_subfield_index = code_array.index(/[a-vx-z8][^a-vx-z8]*$/)	×
421	return true if last_heading_subfield_index && field.subfields[last_heading_subfield_index].value =~ punct_regex	×
422	end
423	false	×
424	end
425
426	def lowercase_headings?(record)	1✔
427	record.fields.each do \|field\|	×
428	next unless field.tag =~ /[1678]../	×
429
430	return true if field['a'] =~ /^[a-z]{3,}/	×
431	end
432	false	×
433	end
434
435	def subf_0_uri?(record)	1✔
436	record.fields.each do \|field\|	×
437	next unless field.class == MARC::DataField && field.tag =~ /^[^9]/ && field['0']	×
438
439	field.subfields.each do \|subfield\|	×
440	return true if subfield.code == '0' && subfield.value =~ /^\(uri\)/	×
441	end
442	end
443	false	×
444	end
445
446	def bad_uri?(record)	1✔
447	target_fields = record.fields('856')	×
448	return false if target_fields.empty?	×
449
450	target_fields.each do \|field\|	×
451	next unless field['u']	×
452
453	field.subfields.each do \|subfield\|	×
454	next unless subfield.code == 'u'	×
455
456	string = subfield.value	×
457	return true unless URI.escape(URI.unescape(string).scrub) == string	×
458	end
459	end
460	false	×
461	end
462
463	### Normalize to the NFC (combined) form of diacritics for characters with
464	# Arabic diacritics; normalize to NFD for characters below U+622 and
465	# between U+1E00 and U+2A28
466	def composed_chars_normalize(record)	1✔
467	record.fields.each do \|field\|	×
468	next unless field.class == MARC::DataField	×
469
470	field_index = record.fields.index(field)	×
471	curr_subfield = 0	×
472	field.subfields.each do \|subfield\|	×
473	fixed_subfield = ''	×
474	prevalue = subfield.value	×
475	if prevalue =~ /^.[\u0653\u0654\u0655].$/	×
476	prevalue = prevalue.unicode_normalize(:nfc)	×
477	end
478	prevalue.each_codepoint do \|c\|	×
479	char = c.chr(Encoding::UTF_8)	×
480	char.unicode_normalize!(:nfd) if c < 1570 \|\| (7_680..10_792).cover?(c)	×
481	fixed_subfield << char	×
482	end
483	record.fields[field_index].subfields[curr_subfield].value = fixed_subfield	×
484	curr_subfield += 1	×
485	end
486	end
487	record	×
488	end
489
490	### Replace empty indicators with a space;
491	# scrub indicators with bad UTF-8
492	def empty_indicator_fix(record)	1✔
493	record.fields.each do \|field\|	×
494	next unless field.class == MARC::DataField	×
495
496	if field.indicator1.nil?	×
497	field.indicator1 = ' '	×
498	else
499	field.indicator1.scrub!('')	×
500	field.indicator1 = ' ' if field.indicator1.size < 1	×
501	end
502	if field.indicator2.nil?	×
503	field.indicator2 = ' '	×
504	else
505	field.indicator2.scrub!('')	×
506	field.indicator2 = ' ' if field.indicator2.size < 1	×
507	end
508	end
509	record	×
510	end
511
512	### Remove empty subfields from DataFields
513	def empty_subfield_fix(record)	1✔
514	record.fields.each do \|field\|	1✔
515	next unless field.class == MARC::DataField	2✔
516	field.subfields.delete_if { \|subfield\| subfield.value.nil? \|\| subfield.value.empty? }	11✔
517	end
518	record.fields.delete_if { \|field\| field.class == MARC::DataField && field.subfields.empty? }	3✔
519	record	1✔
520	end
521
522	### Remove the (uri) prefix from subfield 0s
523	def subf_0_fix(record)	1✔
524	record.fields.each do \|field\|	×
525	next unless field.class == MARC::DataField && field.tag =~ /^[^9]/ && field['0']	×
526
527	field_index = record.fields.index(field)	×
528	field.subfields.each do \|subfield\|	×
529	next unless subfield.code == '0' && subfield.value =~ /^\(uri\)/	×
530
531	subfield_index = field.subfields.index(subfield)	×
532	record.fields[field_index].subfields[subfield_index].value = subfield.value.gsub(/^\(uri\)(.*)$/, '\1')	×
533	end
534	end
535	record	×
536	end
537
538	### Escape URIs
539	def uri_escape(record)	1✔
540	target_fields = record.fields('856')	×
541	return record if target_fields.empty?	×
542
543	fixed_record = record	×
544	target_fields.each do \|field\|	×
545	next unless field['u']	×
546
547	field_index = fixed_record.fields.index(field)	×
548	field.subfields.each do \|subfield\|	×
549	next unless subfield.code == 'u'	×
550
551	subfield_index = field.subfields.index(subfield)	×
552	string = subfield.value	×
553	fixed_string = URI.escape(URI.unescape(string).scrub)	×
554	fixed_record.fields[field_index].subfields[subfield_index].value = fixed_string	×
555	end
556	end
557	fixed_record	×
558	end
559
560	### Make the 040 $b 'eng' if it doesn't have a value
561	def fix_040b(record)	1✔
562	f040 = record.fields('040')	2✔
563	return record unless f040.size == 1	2✔
564
565	f040 = f040.first	2✔
566	field_index = record.fields.index(f040)	2✔
567	b040 = f040.subfields.select { \|subfield\| subfield.code == 'b' }	4✔
568	return record unless b040.empty?	2✔
569
570	subf_codes = f040.subfields.map { \|subfield\| subfield.code }	4✔
571	subf_index = if f040['a']	2✔
572	(subf_codes.index { \|i\| i == 'a' }) + 1	2✔
573	else
574	0	1✔
575	end
576	subf_b = MARC::Subfield.new('b', 'eng')	2✔
577	record.fields[field_index].subfields.insert(subf_index, subf_b)	2✔
578	record	2✔
579	end
580
581	### Split up subfields that contain multiple 3-letter language codes
582	def fix_041(record)	1✔
583	f041 = record.fields('041')	×
584	return record if f041.empty?	×
585
586	f041.each do \|field\|	×
587	f_index = record.fields.index(field)	×
588	new_field = MARC::DataField.new('041', field.indicator1, field.indicator2)	×
589	field.subfields.each do \|subfield\|	×
590	code = subfield.code	×
591	val = subfield.value	×
592	next unless val.size % 3 == 0	×
593
594	langs = val.scan(/.../)	×
595	langs.each do \|lang\|	×
596	new_field.append(MARC::Subfield.new(code, lang))	×
597	end
598	end
599	record.fields[f_index] = new_field	×
600	end
601	record	×
602	end
603
604	### Removes text from the beginning of a subfield
605	### An array of hashes of the format { field:, subfields: } will be passed
606	### in the targets: symbol
607	### subfield: is an array of subfield codes
608	def remove_prefix_from_subfield(record:, targets:, string:)	1✔
609	targets.each do \|target\|	1✔
610	record.fields(target[:field]).each do \|field\|	1✔
611	field.subfields.each do \|subfield\|	1✔
612	next unless target[:subfields].include?(subfield.code)	2✔
613
614	subfield.value = subfield.value.dup.delete_prefix(string)	1✔
615	end
616	end
617	end
618	record	1✔
619	end
620
621	### Adds text to the beginning of a subfield
622	### An array of hashes of the format { field:, subfields: } will be passed
623	### in the targets: symbol
624	### subfield: is an array of subfield codes
625	def add_prefix_to_subfield(record:, targets:, string:)	1✔
626	targets.each do \|target\|	1✔
627	record.fields(target[:field]).each do \|field\|	1✔
628	field.subfields.each do \|subfield\|	1✔
629	next unless target[:subfields].include?(subfield.code)	2✔
630
631	subfield.value = subfield.value.dup.prepend(string)	1✔
632	end
633	end
634	end
635	record	1✔
636	end
637
638	### Sort subfields for target fields with an arbitrary order
639	def subfield_sort(record, target_tags, order_array = nil)	1✔
640	target_fields = record.fields.select { \|f\| target_tags.include?(f.tag) }	×
641	return record if target_fields.empty?	×
642
643	target_fields.each do \|field\|	×
644	next unless field.class == MARC::DataField	×
645
646	orig_codes = field.subfields.map { \|subfield\| subfield.code }.uniq.sort	×
647	order_array = orig_codes if order_array.nil?	×
648	new_subfields = []	×
649	order_array.each do \|code\|	×
650	next unless orig_codes.include?(code)	×
651
652	target_subf = field.subfields.select { \|subfield\| subfield.code == code }	×
653	target_subf.each { \|subfield\| new_subfields << subfield }	×
654	end
655	rem_subfields = field.subfields.select { \|subf\| !order_array.include?(subf.code) }	×
656	rem_subfields.each do \|subfield\|	×
657	new_subfields << subfield	×
658	end
659	field.subfields = new_subfields	×
660	end
661	record	×
662	end
663	end

pulibrary / marc_cleanup / f2723d8b-bf8c-4ddb-ba2c-9aece62d6d17

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous