11c952aa-d8e8-4073-941a-ba08eee9b0bb

Committed 24 Sep 2024 03:41PM UTC coverage: 70.281% (-28.5%) from 98.795%

Build # 11c952aa-d8e8-4073-941a-ba08eee9b0bb

Build Type

push

circleci

Committed by

jrgriffiniii

Commit Message

wip

Run Details

525 of 747 relevant lines covered (70.28%)

9.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.0

/app/models/employee.rb

# frozen_string_literal: true

require "csv"

class Employee < ApplicationRecord
  # USE THIS when searching for all employees / all employees in a department
  # this number should be big greater than both
  # * the number of employees with a given common name
  # * the number of employees in a given department
  @@MAX_EMPLOYEE_MATCH = 200

  self.per_page = 20

  before_validation :clean_strings
  before_validation :derive_attr_values
  before_validation :format_unique_id

  validates_presence_of :first_name, :last_name, :unique_id, :netid, :department
  validates_presence_of :preferred_name, allow_blank: false
  validates_uniqueness_of :netid, case_sensitive: false

  validates_uniqueness_of :email, case_sensitive: false
  validates_format_of :email, with: URI::MailTo::EMAIL_REGEXP

  validates_uniqueness_of :unique_id, message: "id has already been taken"
  validates_format_of :unique_id,
                      with: /\b\d{9}\z/,
                      message: "id must be 9 digits"

  # derive preferred_name from first and last_name if it is nil
  def derive_attr_values
    self.preferred_name = "#{last_name}, #{first_name}" if preferred_name.nil?
  end

  # pad unique+id with 0 if it has less than 9 characters
  def format_unique_id
    uid = self[:unique_id]
    begin
      raise(ArgumentError, "Invalid unique ID: #{uid} (#{uid.class})") unless uid.class == String

      len = uid.length
      uid = "000000000".slice(0, 9 - uid.length) + uid if len < 9

      self.unique_id = uid
    rescue StandardError => error
      Rails.logger.warn(error.message)
    end

    self
  end

  def full_name
    "#{last_name}, #{first_name}"
  end

  # ---------------------
  # search/solr
  # ---------------------
  def all_names
    [first_name, last_name, preferred_name]
  end

  def order_by_name
    "#{last_name}, #{first_name} #{id}"
  end

  searchable do
    text :department, stored: true
    string :order_by_name, multiple: false, stored: true
    text :all_names, stored: true
  end

  # Reindex using Sunspot
  # This is ported from https://github.com/sunspot/sunspot/blob/v2.5.0/sunspot_rails/lib/sunspot/rails/searchable.rb#L257
  # Please see https://github.com/sunspot/sunspot/issues/1008
  def self.solr_index(opts = {})
    options = {
      batch_size: Sunspot.config.indexing.default_batch_size,
      batch_commit: true,
      include: sunspot_options[:include],
      start: opts.delete(:first_id)
    }.merge(opts)

    if options[:batch_size].to_i.positive?
      batch_counter = 0
      batch_options = options.slice(:batch_size, :start)
      batch_size = batch_options[:batch_size]
      include_option = options[:include]

      includes(include_option).find_in_batches(batch_size: batch_size) do |records|
        solr_benchmark(options[:batch_size], batch_counter += 1) do
          Sunspot.index(records.select(&:indexable?))
          Sunspot.commit if options[:batch_commit]
        end
      end
    else
      Sunspot.index! includes(options[:include]).select(&:indexable?)
    end

    Sunspot.commit unless options[:batch_commit]
  end

  # get all employees with a matching first, last or preferred_name
  # return Sunspot::Search object
  def self.all_by_name(word)
    search_by_field(:all_names, word, nil, @@MAX_EMPLOYEE_MATCH)
  end

  # get all employees with a matching first, last or preferred_name
  # paginate results according to given page and per_page parameters
  # return Sunspot::Search object
  def self.search_by_name(word, page = nil, per_page = nil)
    search_by_field(:all_names, word, page, per_page)
  end

  # search for employees from the given department
  # paginate results according to given page and per_page parameters
  # return Sunspot::Search object
  def self.all_by_department(word)
    search_by_field(:department, word, nil, @@MAX_EMPLOYEE_MATCH)
  end

  def self.search_by_field(field, word, page, per_page)
    per_page ||= self.per_page
    Employee.search do
      fulltext word, fields: [field]
      order_by :order_by_name
      paginate page: page, per_page: per_page
    end
  end

  def self.convert_str(str)
    str.chars.reject { |c| c.chr == "\u0000" }.join
  end

  # deletes all existing employees and loads new employees from the given file (.csv format)
  # returns { :loaded => number of employees :failed => hash of line numbers to errors
  def self.loadCsv(filename, logger)
    cognosCrossWalk = {
      "[Firstname]" => :first_name,
      "[Lastname]" => :last_name,
      "[KnownAs]" => :preferred_name,
      "[Department]" => :department,
      "[Proprietary_ID]" => :unique_id,
      "[Email]" => :email,
      "[Username]" => :netid
    }

    logger.debug "Start loading '#{filename}'"
    loaded = 0
    skipped = 0
    failed = {}

    ActiveRecord::Base.transaction(requires_new: true) do
      Employee.destroy_all

      headers = nil
      i = 0
      CSV.foreach(filename, col_sep: "\t", encoding: "IBM437") do |row|
        i += 1
        logger.debug "#{i} #{row}"

        if headers.nil?
          headers = []
          row.each do |h|
            headers << convert_str(h)
          end
          logger.info "#{headers.length} Column headers: #{headers.join(',')}"
          next
        end

        attrs = {}
        (0..row.length - 1).each do |c|
          val = convert_str(row[c])
          # puts "#{i} #{c}: #{val} #{headers[c]}"
          attrs[cognosCrossWalk[headers[c]]] = val if cognosCrossWalk[headers[c]] && val && !val.strip.empty?
        end
        logger.debug attrs.inspect
        if attrs.empty?
          skipped += 1
          next
        end
        employee = Employee.create(attrs)
        if employee.save
          loaded += 1
          logger.info "Employee.loadCsv #{filename} line #{i}: loaded #{employee}"
        else
          failed[i] = [employee.errors.messages]
          logger.error "Employee.loadCsv #{filename} line: #{i}: #{failed[i]}"
        end
      end

      logger.info " Loaded #{loaded} employee records; Failed to read #{failed.length} lines, Skipped #{skipped} lines"

      unless failed.empty?
        logger.error "Rolling back all changes"
        raise ActiveRecord::Rollback
      end
    end
    logger.debug "Stop loading '#{filename}'"

    Employee.reindex unless failed.empty?

    { loaded: loaded, failed: failed, skipped: skipped }
  end

  private

  def to_s
    "#<Employee::#{preferred_name} #{netid} #{unique_id}>"
  end

  # ---------------------
  # utils
  # ---------------------
  def clean_strings
    if changed? || id.nil?
      attributes.each do |n, _v|
        self[n] = nil if self[n] == ""
        self[n] = self[n].gsub(/\s+/, " ").strip if !self[n].nil? && (self[n].class == String)
      end
    end
    self
  end
end

1	# frozen_string_literal: true
2
3	require "csv"	1✔
4
5	class Employee < ApplicationRecord	1✔
6	# USE THIS when searching for all employees / all employees in a department
7	# this number should be big greater than both
8	# * the number of employees with a given common name
9	# * the number of employees in a given department
10	@@MAX_EMPLOYEE_MATCH = 200	1✔
11
12	self.per_page = 20	1✔
13
14	before_validation :clean_strings	1✔
15	before_validation :derive_attr_values	1✔
16	before_validation :format_unique_id	1✔
17
18	validates_presence_of :first_name, :last_name, :unique_id, :netid, :department	1✔
19	validates_presence_of :preferred_name, allow_blank: false	1✔
20	validates_uniqueness_of :netid, case_sensitive: false	1✔
21
22	validates_uniqueness_of :email, case_sensitive: false	1✔
23	validates_format_of :email, with: URI::MailTo::EMAIL_REGEXP	1✔
24
25	validates_uniqueness_of :unique_id, message: "id has already been taken"	1✔
26	validates_format_of :unique_id,	1✔
27	with: /\b\d{9}\z/,
28	message: "id must be 9 digits"
29
30	# derive preferred_name from first and last_name if it is nil
31	def derive_attr_values	1✔
32	self.preferred_name = "#{last_name}, #{first_name}" if preferred_name.nil?	60✔
33	end
34
35	# pad unique+id with 0 if it has less than 9 characters
36	def format_unique_id	1✔
37	uid = self[:unique_id]	63✔
38	begin
39	raise(ArgumentError, "Invalid unique ID: #{uid} (#{uid.class})") unless uid.class == String	63✔
40
41	len = uid.length	59✔
42	uid = "000000000".slice(0, 9 - uid.length) + uid if len < 9	59✔
43
44	self.unique_id = uid	59✔
45	rescue StandardError => error
46	Rails.logger.warn(error.message)	4✔
47	end
48
49	self	63✔
50	end
51
52	def full_name	1✔
53	"#{last_name}, #{first_name}"	×
54	end
55
56	# ---------------------
57	# search/solr
58	# ---------------------
59	def all_names	1✔
60	[first_name, last_name, preferred_name]	33✔
61	end
62
63	def order_by_name	1✔
64	"#{last_name}, #{first_name} #{id}"	33✔
65	end
66
67	searchable do	1✔
68	text :department, stored: true	1✔
69	string :order_by_name, multiple: false, stored: true	1✔
70	text :all_names, stored: true	1✔
71	end
72
73	# Reindex using Sunspot
74	# This is ported from https://github.com/sunspot/sunspot/blob/v2.5.0/sunspot_rails/lib/sunspot/rails/searchable.rb#L257
75	# Please see https://github.com/sunspot/sunspot/issues/1008
76	def self.solr_index(opts = {})	1✔
77	options = {
78	batch_size: Sunspot.config.indexing.default_batch_size,	×
79	batch_commit: true,
80	include: sunspot_options[:include],
81	start: opts.delete(:first_id)
82	}.merge(opts)
83
84	if options[:batch_size].to_i.positive?	×
85	batch_counter = 0	×
86	batch_options = options.slice(:batch_size, :start)	×
87	batch_size = batch_options[:batch_size]	×
88	include_option = options[:include]	×
89
90	includes(include_option).find_in_batches(batch_size: batch_size) do \|records\|	×
91	solr_benchmark(options[:batch_size], batch_counter += 1) do	×
92	Sunspot.index(records.select(&:indexable?))	×
93	Sunspot.commit if options[:batch_commit]	×
94	end
95	end
96	else
97	Sunspot.index! includes(options[:include]).select(&:indexable?)	×
98	end
99
100	Sunspot.commit unless options[:batch_commit]	×
101	end
102
103	# get all employees with a matching first, last or preferred_name
104	# return Sunspot::Search object
105	def self.all_by_name(word)	1✔
106	search_by_field(:all_names, word, nil, @@MAX_EMPLOYEE_MATCH)	×
107	end
108
109	# get all employees with a matching first, last or preferred_name
110	# paginate results according to given page and per_page parameters
111	# return Sunspot::Search object
112	def self.search_by_name(word, page = nil, per_page = nil)	1✔
113	search_by_field(:all_names, word, page, per_page)	×
114	end
115
116	# search for employees from the given department
117	# paginate results according to given page and per_page parameters
118	# return Sunspot::Search object
119	def self.all_by_department(word)	1✔
120	search_by_field(:department, word, nil, @@MAX_EMPLOYEE_MATCH)	×
121	end
122
123	def self.search_by_field(field, word, page, per_page)	1✔
124	per_page \|\|= self.per_page	×
125	Employee.search do	×
126	fulltext word, fields: [field]	×
127	order_by :order_by_name	×
128	paginate page: page, per_page: per_page	×
129	end
130	end
131
132	def self.convert_str(str)	1✔
133	str.chars.reject { \|c\| c.chr == "\u0000" }.join	3,016✔
134	end
135
136	# deletes all existing employees and loads new employees from the given file (.csv format)
137	# returns { :loaded => number of employees :failed => hash of line numbers to errors
138	def self.loadCsv(filename, logger)	1✔
139	cognosCrossWalk = {	2✔
140	"[Firstname]" => :first_name,
141	"[Lastname]" => :last_name,
142	"[KnownAs]" => :preferred_name,
143	"[Department]" => :department,
144	"[Proprietary_ID]" => :unique_id,
145	"[Email]" => :email,
146	"[Username]" => :netid
147	}
148
149	logger.debug "Start loading '#{filename}'"	2✔
150	loaded = 0	2✔
151	skipped = 0	2✔
152	failed = {}	2✔
153
154	ActiveRecord::Base.transaction(requires_new: true) do	2✔
155	Employee.destroy_all	2✔
156
157	headers = nil	2✔
158	i = 0	2✔
159	CSV.foreach(filename, col_sep: "\t", encoding: "IBM437") do \|row\|	2✔
160	i += 1	10✔
161	logger.debug "#{i} #{row}"	10✔
162
163	if headers.nil?	10✔
164	headers = []	2✔
165	row.each do \|h\|	2✔
166	headers << convert_str(h)	36✔
167	end
168	logger.info "#{headers.length} Column headers: #{headers.join(',')}"	2✔
169	next	2✔
170	end
171
172	attrs = {}	8✔
173	(0..row.length - 1).each do \|c\|	8✔
174	val = convert_str(row[c])	143✔
175	# puts "#{i} #{c}: #{val} #{headers[c]}"
176	attrs[cognosCrossWalk[headers[c]]] = val if cognosCrossWalk[headers[c]] && val && !val.strip.empty?	143✔
177	end
178	logger.debug attrs.inspect	8✔
179	if attrs.empty?	8✔
180	skipped += 1	×
181	next	×
182	end
183	employee = Employee.create(attrs)	8✔
184	if employee.save	6✔
185	loaded += 1	×
186	logger.info "Employee.loadCsv #{filename} line #{i}: loaded #{employee}"	×
187	else
188	failed[i] = [employee.errors.messages]	6✔
189	logger.error "Employee.loadCsv #{filename} line: #{i}: #{failed[i]}"	6✔
190	end
191	end
192
193	logger.info " Loaded #{loaded} employee records; Failed to read #{failed.length} lines, Skipped #{skipped} lines"	×
194
195	unless failed.empty?	×
196	logger.error "Rolling back all changes"	×
197	raise ActiveRecord::Rollback	×
198	end
199	end
200	logger.debug "Stop loading '#{filename}'"	×
201
202	Employee.reindex unless failed.empty?	×
203
204	{ loaded: loaded, failed: failed, skipped: skipped }	×
205	end
206
207	private	1✔
208
209	def to_s	1✔
210	"#<Employee::#{preferred_name} #{netid} #{unique_id}>"	×
211	end
212
213	# ---------------------
214	# utils
215	# ---------------------
216	def clean_strings	1✔
217	if changed? \|\| id.nil?	60✔
218	attributes.each do \|n, _v\|	60✔
219	self[n] = nil if self[n] == ""	600✔
220	self[n] = self[n].gsub(/\s+/, " ").strip if !self[n].nil? && (self[n].class == String)	600✔
221	end
222	end
223	self	60✔
224	end
225	end

pulibrary / oawaiver / 11c952aa-d8e8-4073-941a-ba08eee9b0bb

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous