1dcebae2-3318-4e77-bc53-82276e293354

Committed 02 May 2025 04:45PM UTC coverage: 28.256% (-63.9%) from 92.189%

Build # 1dcebae2-3318-4e77-bc53-82276e293354

Build Type

push

circleci

Committed by

sandbergja

Commit Message

Add basic infrastructure for compiling rust code

* Add a rake compile task to compile
* Run the rake task in CI
* Run the rake task before rspec tests with the rust tag, to provide quick feedback on rust changes in TDD cycles

Run Details

2 of 7 new or added lines in 2 files covered. (28.57%)

2467 existing lines in 97 files now uncovered.

1089 of 3854 relevant lines covered (28.26%)

0.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

28.26

/app/models/scsb/s3_bucket.rb

module Scsb
  class S3Bucket
    def self.partner_transfer_client
      new(
        s3_client:
          Aws::S3::Client.new(
            region: 'us-east-2',
            credentials: Aws::Credentials.new(
              ENV['SCSB_S3_PARTNER_ACCESS_KEY'],
              ENV['SCSB_S3_PARTNER_SECRET_ACCESS_KEY']
            )
          ),
        s3_bucket_name: ENV['SCSB_S3_PARTNER_BUCKET_NAME']
      )
    end

    def self.recap_transfer_client
      new(
        s3_client:
        Aws::S3::Client.new(
          region: 'us-east-2',
          credentials: Aws::Credentials.new(
            ENV['SCSB_S3_ACCESS_KEY'],
            ENV['SCSB_S3_SECRET_ACCESS_KEY']
          )
        ),
        s3_bucket_name: ENV['SCSB_S3_BUCKET_NAME']
      )
    end

    attr_reader :s3_client, :s3_bucket_name

    def initialize(s3_client:, s3_bucket_name:)
      @s3_client = s3_client
      @s3_bucket_name = s3_bucket_name
    end

    def list_files(prefix:)
      file_list = []
      response = s3_client.list_objects_v2(bucket: s3_bucket_name, prefix:, delimiter: '')
      file_list << response.contents.entries
      while response.next_continuation_token
        response = s3_client.list_objects_v2(bucket: s3_bucket_name, prefix:, delimiter: '', continuation_token: response.next_continuation_token)
        file_list << response.contents.entries
      end
      file_list.flatten
    end

    def download_file(key:)
      object = s3_client.get_object(bucket: s3_bucket_name, key:)
      object.body
    end

    def upload_file(key:, file_path:, prefix: ENV['SCSB_S3_UPDATES'] || 'data-feed/submitcollections/PUL/cgd_protection')
      status = true
      File.open(file_path, 'rb') do |file|
        status && s3_client.put_object(bucket: s3_bucket_name, body: file, key: "#{prefix}/scsb_#{key}")
      end
      status
    rescue StandardError => e
      Rails.logger.warn("Error uploading object: #{e.message}")
      false
    end

    # @param prefix [String] the bucket path to download from
    # @param output_directory [String] the destination directory
    # @param file_filter [Regexp] a matcher to filter files, e.g. /CUL.*\.zip/
    # @return [String] path to the downloaded files, or nil if none were found
    def download_recent(prefix:, output_directory:, file_filter:)
      matching_files = list_files(prefix:).select { |obj| obj.key.match?(file_filter) }
      recent_file = matching_files.sort_by(&:last_modified).last
      return nil unless recent_file

      fetch_files([recent_file], output_directory).first
    end

    # @return [Array<String>] paths to the downloaded files
    def download_files(files:, timestamp_filter:, output_directory:, file_filter: /CUL-NYPL-HL.*\.zip/)
      files_by_extension = files.select { |obj| obj.key.match?(file_filter) }
      files_to_download = files_by_extension.select { |obj| obj.last_modified > timestamp_filter }
      fetch_files(files_to_download, output_directory)
    end

    private

      # @return [Array<String>] paths to the downloaded files
      def fetch_files(files_to_download, output_directory)
        files_to_download.map do |obj|
          filename = File.basename(obj[:key])
          data = download_file(key: obj.key)
          dest = File.join(output_directory, filename)
          File.open(dest, 'wb') do |output|
            output.write(data.read)
          end
          dest
        end
      end
  end
end

1	module Scsb	1✔
2	class S3Bucket	1✔
3	def self.partner_transfer_client	1✔
UNCOV 4	new(	×
5	s3_client:
6	Aws::S3::Client.new(
7	region: 'us-east-2',
8	credentials: Aws::Credentials.new(
9	ENV['SCSB_S3_PARTNER_ACCESS_KEY'],
10	ENV['SCSB_S3_PARTNER_SECRET_ACCESS_KEY']
11	)
12	),
13	s3_bucket_name: ENV['SCSB_S3_PARTNER_BUCKET_NAME']
14	)
15	end
16
17	def self.recap_transfer_client	1✔
UNCOV 18	new(	×
19	s3_client:
20	Aws::S3::Client.new(
21	region: 'us-east-2',
22	credentials: Aws::Credentials.new(
23	ENV['SCSB_S3_ACCESS_KEY'],
24	ENV['SCSB_S3_SECRET_ACCESS_KEY']
25	)
26	),
27	s3_bucket_name: ENV['SCSB_S3_BUCKET_NAME']
28	)
29	end
30
31	attr_reader :s3_client, :s3_bucket_name	1✔
32
33	def initialize(s3_client:, s3_bucket_name:)	1✔
UNCOV 34	@s3_client = s3_client	×
UNCOV 35	@s3_bucket_name = s3_bucket_name	×
36	end
37
38	def list_files(prefix:)	1✔
UNCOV 39	file_list = []	×
UNCOV 40	response = s3_client.list_objects_v2(bucket: s3_bucket_name, prefix:, delimiter: '')	×
UNCOV 41	file_list << response.contents.entries	×
UNCOV 42	while response.next_continuation_token	×
UNCOV 43	response = s3_client.list_objects_v2(bucket: s3_bucket_name, prefix:, delimiter: '', continuation_token: response.next_continuation_token)	×
UNCOV 44	file_list << response.contents.entries	×
45	end
UNCOV 46	file_list.flatten	×
47	end
48
49	def download_file(key:)	1✔
UNCOV 50	object = s3_client.get_object(bucket: s3_bucket_name, key:)	×
UNCOV 51	object.body	×
52	end
53
54	def upload_file(key:, file_path:, prefix: ENV['SCSB_S3_UPDATES'] \|\| 'data-feed/submitcollections/PUL/cgd_protection')	1✔
UNCOV 55	status = true	×
UNCOV 56	File.open(file_path, 'rb') do \|file\|	×
UNCOV 57	status && s3_client.put_object(bucket: s3_bucket_name, body: file, key: "#{prefix}/scsb_#{key}")	×
58	end
UNCOV 59	status	×
60	rescue StandardError => e
UNCOV 61	Rails.logger.warn("Error uploading object: #{e.message}")	×
UNCOV 62	false	×
63	end
64
65	# @param prefix [String] the bucket path to download from
66	# @param output_directory [String] the destination directory
67	# @param file_filter [Regexp] a matcher to filter files, e.g. /CUL.*\.zip/
68	# @return [String] path to the downloaded files, or nil if none were found
69	def download_recent(prefix:, output_directory:, file_filter:)	1✔
UNCOV 70	matching_files = list_files(prefix:).select { \|obj\| obj.key.match?(file_filter) }	×
UNCOV 71	recent_file = matching_files.sort_by(&:last_modified).last	×
UNCOV 72	return nil unless recent_file	×
73
UNCOV 74	fetch_files([recent_file], output_directory).first	×
75	end
76
77	# @return [Array<String>] paths to the downloaded files
78	def download_files(files:, timestamp_filter:, output_directory:, file_filter: /CUL-NYPL-HL.*\.zip/)	1✔
UNCOV 79	files_by_extension = files.select { \|obj\| obj.key.match?(file_filter) }	×
UNCOV 80	files_to_download = files_by_extension.select { \|obj\| obj.last_modified > timestamp_filter }	×
UNCOV 81	fetch_files(files_to_download, output_directory)	×
82	end
83
84	private	1✔
85
86	# @return [Array<String>] paths to the downloaded files
87	def fetch_files(files_to_download, output_directory)	1✔
UNCOV 88	files_to_download.map do \|obj\|	×
UNCOV 89	filename = File.basename(obj[:key])	×
UNCOV 90	data = download_file(key: obj.key)	×
UNCOV 91	dest = File.join(output_directory, filename)	×
UNCOV 92	File.open(dest, 'wb') do \|output\|	×
UNCOV 93	output.write(data.read)	×
94	end
UNCOV 95	dest	×
96	end
97	end
98	end
99	end

pulibrary / bibdata / 1dcebae2-3318-4e77-bc53-82276e293354

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous