2467ba7c-9779-4cbb-8f0b-bfce51097878

Committed 11 Oct 2023 05:26PM UTC coverage: 98.667% (+0.04%) from 98.63%

Build # 2467ba7c-9779-4cbb-8f0b-bfce51097878

Build Type

Pull #189

circleci

Committed by

hackartisan

Commit Message

Reduce resources used on 2nd async with a semaphore

Pull Request Pull Request #189: Faster image load

Run Details

39 of 39 new or added lines in 1 file covered. (100.0%)

148 of 150 relevant lines covered (98.67%)

26.12 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.35

/app/services/card_image_loading_service.rb

# frozen_string_literal: true

require 'ruby-progressbar'
require 'ruby-progressbar/outputs/null'
require 'async/semaphore'
require 'async/barrier'

# Class for card image loading service
class CardImageLoadingService
  attr_reader :logger, :suppress_progress

  def initialize(logger: nil, suppress_progress: false)
    @logger = logger || Logger.new($stdout)
    @suppress_progress = suppress_progress
  end

  def import
    barrier = Async::Barrier.new
    Sync do
      semaphore = Async::Semaphore.new(8, parent: barrier)

      (1..22).map do |disk|
        semaphore.async do
          import_disk(disk)
        end
      end.map(&:wait)
    ensure
      barrier.stop
    end
  end

  def import_disk(disk)
    logger.info("Fetching disk #{disk} file list")
    filenames = disk_array(disk)
    progress_bar.total += filenames.count
    Sync do
      semaphore = Async::Semaphore.new(10)
      filenames.map do |file_name|
        semaphore.async do
          progress_bar.increment
          find_or_create_card_image(file_name)
        end
      end.map(&:wait)
    end
  end

  private

  # returns something like
  # ["imagecat-disk9-0091-A3037-1358.0110.tif", "imagecat-disk9-0091-A3037-1358.0111.tif"]
  def disk_array(disk)
    s3_disk_list(disk).split("\n").map(&:split).map(&:last)
  end

  # returns something like
  # "2023-07-19 14:39:38       3422 imagecat-disk9-0091-A3037-1358.0110.tif\n2023-07-19 14:39:38       7010 imagecat-disk9-0091-A3037-1358.0111.tif\n"
  def s3_disk_list(disk)
    `aws s3 ls s3://puliiif-production/imagecat-disk#{disk}-`
  end

  def find_or_create_card_image(file_name)
    path = file_name.gsub('imagecat-disk', '').split('-')[0..-2].join('/')
    ci = CardImage.find_by(path: path, image_name: file_name)
    return if ci

    CardImage.create(path: path, image_name: file_name)
  end

  def progress_bar
    @progress_bar ||= ProgressBar.create(format: '%a %e %P% Loading: %c from %C', output: progress_output, total: 0, title: 'Image import')
  end

  def progress_bar_old(total)
    ProgressBar.create(format: '%a %e %P% Loading: %c from %C', total: total, output: progress_output)
  end

  def progress_output
    ProgressBar::Outputs::Null if suppress_progress
  end
end

1	# frozen_string_literal: true
2
3	require 'ruby-progressbar'	1✔
4	require 'ruby-progressbar/outputs/null'	1✔
5	require 'async/semaphore'	1✔
6	require 'async/barrier'	1✔
7
8	# Class for card image loading service
9	class CardImageLoadingService	1✔
10	attr_reader :logger, :suppress_progress	1✔
11
12	def initialize(logger: nil, suppress_progress: false)	1✔
13	@logger = logger \|\| Logger.new($stdout)	1✔
14	@suppress_progress = suppress_progress	1✔
15	end
16
17	def import	1✔
18	barrier = Async::Barrier.new	2✔
19	Sync do	2✔
20	semaphore = Async::Semaphore.new(8, parent: barrier)	2✔
21
22	(1..22).map do \|disk\|	2✔
23	semaphore.async do	44✔
24	import_disk(disk)	44✔
25	end
26	end.map(&:wait)
27	ensure
28	barrier.stop	2✔
29	end
30	end
31
32	def import_disk(disk)	1✔
33	logger.info("Fetching disk #{disk} file list")	44✔
34	filenames = disk_array(disk)	44✔
35	progress_bar.total += filenames.count	44✔
36	Sync do	44✔
37	semaphore = Async::Semaphore.new(10)	44✔
38	filenames.map do \|file_name\|	44✔
39	semaphore.async do	18✔
40	progress_bar.increment	18✔
41	find_or_create_card_image(file_name)	18✔
42	end
43	end.map(&:wait)
44	end
45	end
46
47	private	1✔
48
49	# returns something like
50	# ["imagecat-disk9-0091-A3037-1358.0110.tif", "imagecat-disk9-0091-A3037-1358.0111.tif"]
51	def disk_array(disk)	1✔
52	s3_disk_list(disk).split("\n").map(&:split).map(&:last)	44✔
53	end
54
55	# returns something like
56	# "2023-07-19 14:39:38 3422 imagecat-disk9-0091-A3037-1358.0110.tif\n2023-07-19 14:39:38 7010 imagecat-disk9-0091-A3037-1358.0111.tif\n"
57	def s3_disk_list(disk)	1✔
58	`aws s3 ls s3://puliiif-production/imagecat-disk#{disk}-`	×
59	end
60
61	def find_or_create_card_image(file_name)	1✔
62	path = file_name.gsub('imagecat-disk', '').split('-')[0..-2].join('/')	18✔
63	ci = CardImage.find_by(path: path, image_name: file_name)	18✔
64	return if ci	18✔
65
66	CardImage.create(path: path, image_name: file_name)	9✔
67	end
68
69	def progress_bar	1✔
70	@progress_bar \|\|= ProgressBar.create(format: '%a %e %P% Loading: %c from %C', output: progress_output, total: 0, title: 'Image import')	62✔
71	end
72
73	def progress_bar_old(total)	1✔
74	ProgressBar.create(format: '%a %e %P% Loading: %c from %C', total: total, output: progress_output)	×
75	end
76
77	def progress_output	1✔
78	ProgressBar::Outputs::Null if suppress_progress	1✔
79	end
80	end

pulibrary / imagecat-rails / 2467ba7c-9779-4cbb-8f0b-bfce51097878

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous