31a62f9e-3a5b-4033-a3e3-effd2d651737

Build # 31a62f9e-3a5b-4033-a3e3-effd2d651737

Build Type

circleci

Committed by Hector Correa

Commit Message

Fixes tests for S3 query service (and a couple of code duplications from a merge?)

Pull Request Pull Request #962: Fetch pre-curation files from directly from AWS

Run Details

35 of 35 new or added lines in 4 files covered. (100.0%)

1821 of 1864 relevant lines covered (97.69%)

149.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.23

/app/services/s3_query_service.rb

# frozen_string_literal: true

require "aws-sdk-s3"

# A service to query an S3 bucket for information about a given data set
# rubocop:disable Metrics/ClassLength
class S3QueryService
  attr_reader :model

  def self.configuration
    Rails.configuration.s3
  end

  def self.pre_curation_config
    configuration.pre_curation
  end

  def self.post_curation_config
    configuration.post_curation
  end

  ##
  # @param [Work] model
  # @param [Boolean] pre_curation
  # @example S3QueryService.new(Work.find(1), true)
  def initialize(model, pre_curation = true)
    @model = model
    @doi = model.doi
    @pre_curation = pre_curation
  end

  def config
    return self.class.post_curation_config if post_curation?

    self.class.pre_curation_config
  end

  def pre_curation?
    @pre_curation
  end

  def post_curation?
    !pre_curation?
  end

  ##
  # The name of the bucket this class is configured to use.
  # See config/s3.yml for configuration file.
  def bucket_name
    config.fetch(:bucket, nil)
  end

  def region
    config.fetch(:region, nil)
  end

  ##
  # The S3 prefix for this object, i.e., the address within the S3 bucket,
  # which is based on the DOI
  def prefix
    "#{@doi}/#{model.id}/"
  end

  ##
  # Construct an S3 address for this data set
  def s3_address
    "s3://#{bucket_name}/#{prefix}"
  end

  ##
  # Public signed URL to fetch this file from the S3 (valid for a limited time)
  def file_url(key)
    signer = Aws::S3::Presigner.new(client: client)
    signer.presigned_url(:get_object, bucket: bucket_name, key: key)
  end

  def delete_file(key)
    client.delete_object({ bucket: bucket_name, key: key })
  end

  # There is probably a better way to fetch the current ActiveStorage configuration but we have
  # not found it.
  def active_storage_configuration
    Rails.configuration.active_storage.service_configurations[Rails.configuration.active_storage.service.to_s]
  end

  def access_key_id
    active_storage_configuration["access_key_id"]
  end

  def secret_access_key
    active_storage_configuration["secret_access_key"]
  end

  def credentials
    @credentials ||= Aws::Credentials.new(access_key_id, secret_access_key)
  end

  def client
    @client ||= Aws::S3::Client.new(region: region, credentials: credentials)
  end

  # Retrieve the S3 resources attached to the Work model
  # @return [Array<S3File>]
  def model_s3_files
    objects = []
    return objects if model.nil?

    model_uploads.each do |attachment|
      s3_file = S3File.new(query_service: self,
                           filename: attachment.key,
                           last_modified: attachment.created_at,
                           size: attachment.byte_size,
                           checksum: attachment.checksum)
      objects << s3_file
    end

    objects
  end

  def get_s3_object(key:)
    response = client.get_object({
                                   bucket: bucket_name,
                                   key: key
                                 })
    object = response.to_h
    return if object.empty?

    object
  end

  def find_s3_file(filename:)
    s3_object_key = "#{prefix}#{filename}"

    object = get_s3_object(key: s3_object_key)
    return if object.nil?

    S3File.new(query_service: self, filename: s3_object_key, last_modified: object[:last_modified], size: object[:content_length], checksum: object[:etag])
  end

  # Retrieve the S3 resources uploaded to the S3 Bucket
  # @return [Array<S3File>]
  def client_s3_files(reload: false)
    @client_s3_files = nil if reload # force a reload
    @client_s3_files ||= begin
      Rails.logger.debug("Bucket: #{bucket_name}")
      Rails.logger.debug("Prefix: #{prefix}")
      resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix })
      resp_hash = resp.to_h
      objects = parse_objects(resp_hash)
      objects += parse_continuation(resp_hash)
      objects
    end
  end

  def file_count
    client_s3_files.count
  end

  # TODO: delete this (or client_s3_files)
  def s3_files
    client_s3_files
  end

  ##
  # Query the S3 bucket for what we know about the doi
  # For docs see:
  # * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_objects_v2-instance_method
  # * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#get_object_attributes-instance_method
  # @return Hash with two properties {objects: [<S3File>], ok: Bool}
  #   objects is an Array of S3File objects
  #   ok is false if there is an error connecting to S3. Otherwise true.
  def data_profile
    { objects: s3_files, ok: true }
  rescue => ex
    Rails.logger.error("Error querying S3. Bucket: #{bucket_name}. DOI: #{@doi}. Exception: #{ex.message}")

    { objects: [], ok: false }
  end

  ##
  # Copies the existing files from the pre-curation bucket to the post-curation bucket.
  # Notice that the copy process happens at AWS (i.e. the files are not downloaded and re-uploaded).
  # Returns an array with the files that were copied.
  def publish_files
    files = []
    source_bucket = S3QueryService.pre_curation_config[:bucket]
    target_bucket = S3QueryService.post_curation_config[:bucket]
    model.pre_curation_uploads.each do |file|
      params = {
        copy_source: "/#{source_bucket}/#{file.key}",
        bucket: target_bucket,
        key: file.key
      }
      Rails.logger.info("Copying #{params[:copy_source]} to #{params[:bucket]}/#{params[:key]}")
      client.copy_object(params)
      files << file
    end
    files
  end

  def delete_s3_object(s3_file_key)
    resp = client.delete_object({ bucket: bucket_name,  key: s3_file_key})
    resp.to_h
  end

  private

    def model_uploads
      if pre_curation?
        client_s3_files
      else
        []
      end
    end

    def parse_objects(resp)
      objects = []
      resp_hash = resp.to_h
      response_objects = resp_hash[:contents]
      Rails.logger.debug("Objects: #{response_objects}")
      response_objects&.each do |object|
        next if object[:size] == 0 # ignore directories whose size is zero
        s3_file = S3File.new(query_service: self, filename: object[:key], last_modified: object[:last_modified], size: object[:size], checksum: object[:etag])
        objects << s3_file
      end
      objects
    end

    def parse_continuation(resp_hash)
      objects = []
      while resp_hash[:is_truncated]
        token = resp_hash[:next_continuation_token]
        resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix, continuation_token: token })
        resp_hash = resp.to_h
        objects += parse_objects(resp_hash)
      end
      objects
    end
end
# rubocop:enable Metrics/ClassLength

1	# frozen_string_literal: true
2
3	require "aws-sdk-s3"	1✔
4
5	# A service to query an S3 bucket for information about a given data set
6	# rubocop:disable Metrics/ClassLength
7	class S3QueryService	1✔
8	attr_reader :model	1✔
9
10	def self.configuration	1✔
11	Rails.configuration.s3	338✔
12	end
13
14	def self.pre_curation_config	1✔
15	configuration.pre_curation	232✔
16	end
17
18	def self.post_curation_config	1✔
19	configuration.post_curation	106✔
20	end
21
22	##
23	# @param [Work] model
24	# @param [Boolean] pre_curation
25	# @example S3QueryService.new(Work.find(1), true)
26	def initialize(model, pre_curation = true)	1✔
27	@model = model	98✔
28	@doi = model.doi	98✔
29	@pre_curation = pre_curation	98✔
30	end
31
32	def config	1✔
33	return self.class.post_curation_config if post_curation?	288✔
34
35	self.class.pre_curation_config	205✔
36	end
37
38	def pre_curation?	1✔
39	@pre_curation	288✔
40	end
41
42	def post_curation?	1✔
43	!pre_curation?	288✔
44	end
45
46	##
47	# The name of the bucket this class is configured to use.
48	# See config/s3.yml for configuration file.
49	def bucket_name	1✔
50	config.fetch(:bucket, nil)	207✔
51	end
52
53	def region	1✔
54	config.fetch(:region, nil)	81✔
55	end
56
57	##
58	# The S3 prefix for this object, i.e., the address within the S3 bucket,
59	# which is based on the DOI
60	def prefix	1✔
61	"#{@doi}/#{model.id}/"	131✔
62	end
63
64	##
65	# Construct an S3 address for this data set
66	def s3_address	1✔
67	"s3://#{bucket_name}/#{prefix}"	1✔
68	end
69
70	##
71	# Public signed URL to fetch this file from the S3 (valid for a limited time)
72	def file_url(key)	1✔
73	signer = Aws::S3::Presigner.new(client: client)	2✔
74	signer.presigned_url(:get_object, bucket: bucket_name, key: key)	2✔
75	end
76
77	def delete_file(key)	1✔
78	client.delete_object({ bucket: bucket_name, key: key })	×
79	end
80
81	# There is probably a better way to fetch the current ActiveStorage configuration but we have
82	# not found it.
83	def active_storage_configuration	1✔
84	Rails.configuration.active_storage.service_configurations[Rails.configuration.active_storage.service.to_s]	162✔
85	end
86
87	def access_key_id	1✔
88	active_storage_configuration["access_key_id"]	81✔
89	end
90
91	def secret_access_key	1✔
92	active_storage_configuration["secret_access_key"]	81✔
93	end
94
95	def credentials	1✔
96	@credentials \|\|= Aws::Credentials.new(access_key_id, secret_access_key)	81✔
97	end
98
99	def client	1✔
100	@client \|\|= Aws::S3::Client.new(region: region, credentials: credentials)	150✔
101	end
102
103	# Retrieve the S3 resources attached to the Work model
104	# @return [Array<S3File>]
105	def model_s3_files	1✔
106	objects = []	×
107	return objects if model.nil?	×
108
109	model_uploads.each do \|attachment\|	×
110	s3_file = S3File.new(query_service: self,	×
111	filename: attachment.key,
112	last_modified: attachment.created_at,
113	size: attachment.byte_size,
114	checksum: attachment.checksum)
115	objects << s3_file	×
116	end
117
118	objects	×
119	end
120
121	def get_s3_object(key:)	1✔
122	response = client.get_object({	2✔
123	bucket: bucket_name,
124	key: key
125	})
126	object = response.to_h	2✔
127	return if object.empty?	2✔
128
129	object	2✔
130	end
131
132	def find_s3_file(filename:)	1✔
133	s3_object_key = "#{prefix}#{filename}"	1✔
134
135	object = get_s3_object(key: s3_object_key)	1✔
136	return if object.nil?	1✔
137
138	S3File.new(query_service: self, filename: s3_object_key, last_modified: object[:last_modified], size: object[:content_length], checksum: object[:etag])	1✔
139	end
140
141	# Retrieve the S3 resources uploaded to the S3 Bucket
142	# @return [Array<S3File>]
143	def client_s3_files(reload: false)	1✔
144	@client_s3_files = nil if reload # force a reload	83✔
145	@client_s3_files \|\|= begin	83✔
146	Rails.logger.debug("Bucket: #{bucket_name}")	64✔
147	Rails.logger.debug("Prefix: #{prefix}")	64✔
148	resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix })	64✔
149	resp_hash = resp.to_h	39✔
150	objects = parse_objects(resp_hash)	39✔
151	objects += parse_continuation(resp_hash)	39✔
152	objects	39✔
153	end
154	end
155
156	def file_count	1✔
157	client_s3_files.count	5✔
158	end
159
160	# TODO: delete this (or client_s3_files)
161	def s3_files	1✔
162	client_s3_files	33✔
163	end
164
165	##
166	# Query the S3 bucket for what we know about the doi
167	# For docs see:
168	# * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_objects_v2-instance_method
169	# * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#get_object_attributes-instance_method
170	# @return Hash with two properties {objects: [<S3File>], ok: Bool}
171	# objects is an Array of S3File objects
172	# ok is false if there is an error connecting to S3. Otherwise true.
173	def data_profile	1✔
174	{ objects: s3_files, ok: true }	33✔
175	rescue => ex
176	Rails.logger.error("Error querying S3. Bucket: #{bucket_name}. DOI: #{@doi}. Exception: #{ex.message}")	2✔
177
178	{ objects: [], ok: false }	2✔
179	end
180
181	##
182	# Copies the existing files from the pre-curation bucket to the post-curation bucket.
183	# Notice that the copy process happens at AWS (i.e. the files are not downloaded and re-uploaded).
184	# Returns an array with the files that were copied.
185	def publish_files	1✔
186	files = []	23✔
187	source_bucket = S3QueryService.pre_curation_config[:bucket]	23✔
188	target_bucket = S3QueryService.post_curation_config[:bucket]	23✔
189	model.pre_curation_uploads.each do \|file\|	23✔
190	params = {
191	copy_source: "/#{source_bucket}/#{file.key}",	24✔
192	bucket: target_bucket,
193	key: file.key
194	}
195	Rails.logger.info("Copying #{params[:copy_source]} to #{params[:bucket]}/#{params[:key]}")	24✔
196	client.copy_object(params)	24✔
197	files << file	24✔
198	end
199	files	23✔
200	end
201
202	def delete_s3_object(s3_file_key)	1✔
203	resp = client.delete_object({ bucket: bucket_name, key: s3_file_key})	4✔
204	resp.to_h	4✔
205	end
206
207	private	1✔
208
209	def model_uploads	1✔
210	if pre_curation?	×
211	client_s3_files	×
212	else
213	[]	×
214	end
215	end
216
217	def parse_objects(resp)	1✔
218	objects = []	40✔
219	resp_hash = resp.to_h	40✔
220	response_objects = resp_hash[:contents]	40✔
221	Rails.logger.debug("Objects: #{response_objects}")	40✔
222	response_objects&.each do \|object\|	40✔
223	next if object[:size] == 0 # ignore directories whose size is zero	22✔
224	s3_file = S3File.new(query_service: self, filename: object[:key], last_modified: object[:last_modified], size: object[:size], checksum: object[:etag])	16✔
225	objects << s3_file	16✔
226	end
227	objects	40✔
228	end
229
230	def parse_continuation(resp_hash)	1✔
231	objects = []	39✔
232	while resp_hash[:is_truncated]	39✔
233	token = resp_hash[:next_continuation_token]	1✔
234	resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix, continuation_token: token })	1✔
235	resp_hash = resp.to_h	1✔
236	objects += parse_objects(resp_hash)	1✔
237	end
238	objects	39✔
239	end
240	end
241	# rubocop:enable Metrics/ClassLength

pulibrary / pdc_describe / 31a62f9e-3a5b-4033-a3e3-effd2d651737

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous