bcfc06e8-a5d8-4369-a72f-de9ecf1ada29

Build # bcfc06e8-a5d8-4369-a72f-de9ecf1ada29

Build Type

circleci

Committed by mccalluc

Commit Message

cursor css

Pull Request Pull Request #963: Add row deletion and reordering to irb tables

Run Details

1622 of 1848 relevant lines covered (87.77%)

66.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.02

/app/services/s3_query_service.rb

# frozen_string_literal: true

require "aws-sdk-s3"

# A service to query an S3 bucket for information about a given data set
class S3QueryService
  attr_reader :model

  def self.configuration
    Rails.configuration.s3
  end

  def self.pre_curation_config
    configuration.pre_curation
  end

  def self.post_curation_config
    configuration.post_curation
  end

  ##
  # @param [Work] model
  # @param [Boolean] pre_curation
  # @example S3QueryService.new(Work.find(1), true)
  def initialize(model, pre_curation = true)
    @model = model
    @doi = model.doi
    @pre_curation = pre_curation
  end

  def config
    return self.class.post_curation_config if post_curation?

    self.class.pre_curation_config
  end

  def pre_curation?
    @pre_curation
  end

  def post_curation?
    !pre_curation?
  end

  ##
  # The name of the bucket this class is configured to use.
  # See config/s3.yml for configuration file.
  def bucket_name
    config.fetch(:bucket, nil)
  end

  def region
    config.fetch(:region, nil)
  end

  ##
  # The S3 prefix for this object, i.e., the address within the S3 bucket,
  # which is based on the DOI
  def prefix
    "#{@doi}/#{model.id}/"
  end

  ##
  # Construct an S3 address for this data set
  def s3_address
    "s3://#{bucket_name}/#{prefix}"
  end

  # There is probably a better way to fetch the current ActiveStorage configuration but we have
  # not found it.
  def active_storage_configuration
    Rails.configuration.active_storage.service_configurations[Rails.configuration.active_storage.service.to_s]
  end

  def access_key_id
    active_storage_configuration["access_key_id"]
  end

  def secret_access_key
    active_storage_configuration["secret_access_key"]
  end

  def credentials
    @credentials ||= Aws::Credentials.new(access_key_id, secret_access_key)
  end

  def client
    @client ||= Aws::S3::Client.new(region: region, credentials: credentials)
  end

  # Retrieve the S3 resources attached to the Work model
  # @return [Array<S3File>]
  def model_s3_files
    objects = []
    return objects if model.nil?

    model_uploads.each do |attachment|
      s3_file = S3File.new(query_service: self,
                           filename: attachment.key,
                           last_modified: attachment.created_at,
                           size: attachment.byte_size,
                           checksum: attachment.checksum)
      objects << s3_file
    end

    objects
  end

  def get_s3_object(key:)
    response = client.get_object({
                                   bucket: bucket_name,
                                   key: key
                                 })
    object = response.to_h
    return if object.empty?

    object
  end

  def find_s3_file(filename:)
    s3_object_key = "#{prefix}#{filename}"

    object = get_s3_object(key: s3_object_key)
    return if object.nil?

    S3File.new(query_service: self, filename: s3_object_key, last_modified: object[:last_modified], size: object[:content_length], checksum: object[:etag])
  end

  # Retrieve the S3 resources uploaded to the S3 Bucket
  # @return [Array<S3File>]
  def client_s3_files(reload: false)
    @client_s3_files = nil if reload # force a reload
    @client_s3_files ||= begin
      Rails.logger.debug("Bucket: #{bucket_name}")
      Rails.logger.debug("Prefix: #{prefix}")
      resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix })
      resp_hash = resp.to_h
      objects = parse_objects(resp_hash)
      parse_continuation(resp_hash, objects)
    end
  end

  def file_count
    client_s3_files.count
  end

  # Retrieve the S3 resources from the S3 Bucket without those attached to the Work model
  # @return [Array<S3File>]
  def s3_files
    model_s3_file_keys = model_s3_files.map(&:filename)
    client_s3_files.reject { |client_s3_file| model_s3_file_keys.include?(client_s3_file.filename) }
  end

  ##
  # Query the S3 bucket for what we know about the doi
  # For docs see:
  # * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_objects_v2-instance_method
  # * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#get_object_attributes-instance_method
  # @return Hash with two properties {objects: [<S3File>], ok: Bool}
  #   objects is an Array of S3File objects
  #   ok is false if there is an error connecting to S3. Otherwise true.
  def data_profile
    { objects: s3_files, ok: true }
  rescue => ex
    Rails.logger.error("Error querying S3. Bucket: #{bucket_name}. DOI: #{@doi}. Exception: #{ex.message}")

    { objects: [], ok: false }
  end

  ##
  # Copies the existing files from the pre-curation bucket to the post-curation bucket.
  # Notice that the copy process happens at AWS (i.e. the files are not downloaded and re-uploaded).
  # Returns an array with the files that were copied.
  def publish_files
    files = []
    source_bucket = S3QueryService.pre_curation_config[:bucket]
    target_bucket = S3QueryService.post_curation_config[:bucket]
    model.pre_curation_uploads.each do |file|
      params = {
        copy_source: "/#{source_bucket}/#{file.key}",
        bucket: target_bucket,
        key: file.key
      }
      Rails.logger.info("Copying #{params[:copy_source]} to #{params[:bucket]}/#{params[:key]}")
      client.copy_object(params)
      files << file
    end
    files
  end

  private

    def model_uploads
      if pre_curation?
        model.pre_curation_uploads
      else
        []
      end
    end

    def parse_objects(resp)
      objects = []
      resp_hash = resp.to_h
      response_objects = resp_hash[:contents]
      Rails.logger.debug("Objects: #{response_objects}")
      response_objects&.each do |object|
        next if object[:size] == 0 # ignore directories whose size is zero
        s3_file = S3File.new(query_service: self, filename: object[:key], last_modified: object[:last_modified], size: object[:size], checksum: object[:etag])
        objects << s3_file
      end
      objects
    end

    def parse_continuation(resp_hash, objects)
      while resp_hash[:is_truncated]
        token = resp_hash[:next_continuation_token]
        resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix, continuation_token: token })
        resp_hash = resp.to_h
        more_objects = parse_objects(resp_hash)
        objects += more_objects
      end
      objects
    end
end

1	# frozen_string_literal: true
2
3	require "aws-sdk-s3"	1✔
4
5	# A service to query an S3 bucket for information about a given data set
6	class S3QueryService	1✔
7	attr_reader :model	1✔
8
9	def self.configuration	1✔
10	Rails.configuration.s3	59✔
11	end
12
13	def self.pre_curation_config	1✔
14	configuration.pre_curation	40✔
15	end
16
17	def self.post_curation_config	1✔
18	configuration.post_curation	19✔
19	end
20
21	##
22	# @param [Work] model
23	# @param [Boolean] pre_curation
24	# @example S3QueryService.new(Work.find(1), true)
25	def initialize(model, pre_curation = true)	1✔
26	@model = model	13✔
27	@doi = model.doi	13✔
28	@pre_curation = pre_curation	13✔
29	end
30
31	def config	1✔
32	return self.class.post_curation_config if post_curation?	47✔
33
34	self.class.pre_curation_config	32✔
35	end
36
37	def pre_curation?	1✔
38	@pre_curation	56✔
39	end
40
41	def post_curation?	1✔
42	!pre_curation?	47✔
43	end
44
45	##
46	# The name of the bucket this class is configured to use.
47	# See config/s3.yml for configuration file.
48	def bucket_name	1✔
49	config.fetch(:bucket, nil)	34✔
50	end
51
52	def region	1✔
53	config.fetch(:region, nil)	13✔
54	end
55
56	##
57	# The S3 prefix for this object, i.e., the address within the S3 bucket,
58	# which is based on the DOI
59	def prefix	1✔
60	"#{@doi}/#{model.id}/"	18✔
61	end
62
63	##
64	# Construct an S3 address for this data set
65	def s3_address	1✔
66	"s3://#{bucket_name}/#{prefix}"	×
67	end
68
69	# There is probably a better way to fetch the current ActiveStorage configuration but we have
70	# not found it.
71	def active_storage_configuration	1✔
72	Rails.configuration.active_storage.service_configurations[Rails.configuration.active_storage.service.to_s]	26✔
73	end
74
75	def access_key_id	1✔
76	active_storage_configuration["access_key_id"]	13✔
77	end
78
79	def secret_access_key	1✔
80	active_storage_configuration["secret_access_key"]	13✔
81	end
82
83	def credentials	1✔
84	@credentials \|\|= Aws::Credentials.new(access_key_id, secret_access_key)	13✔
85	end
86
87	def client	1✔
88	@client \|\|= Aws::S3::Client.new(region: region, credentials: credentials)	25✔
89	end
90
91	# Retrieve the S3 resources attached to the Work model
92	# @return [Array<S3File>]
93	def model_s3_files	1✔
94	objects = []	9✔
95	return objects if model.nil?	9✔
96
97	model_uploads.each do \|attachment\|	9✔
98	s3_file = S3File.new(query_service: self,	4✔
99	filename: attachment.key,
100	last_modified: attachment.created_at,
101	size: attachment.byte_size,
102	checksum: attachment.checksum)
103	objects << s3_file	4✔
104	end
105
106	objects	9✔
107	end
108
109	def get_s3_object(key:)	1✔
110	response = client.get_object({	×
111	bucket: bucket_name,
112	key: key
113	})
114	object = response.to_h	×
115	return if object.empty?	×
116
117	object	×
118	end
119
120	def find_s3_file(filename:)	1✔
121	s3_object_key = "#{prefix}#{filename}"	×
122
123	object = get_s3_object(key: s3_object_key)	×
124	return if object.nil?	×
125
126	S3File.new(query_service: self, filename: s3_object_key, last_modified: object[:last_modified], size: object[:content_length], checksum: object[:etag])	×
127	end
128
129	# Retrieve the S3 resources uploaded to the S3 Bucket
130	# @return [Array<S3File>]
131	def client_s3_files(reload: false)	1✔
132	@client_s3_files = nil if reload # force a reload	9✔
133	@client_s3_files \|\|= begin	9✔
134	Rails.logger.debug("Bucket: #{bucket_name}")	9✔
135	Rails.logger.debug("Prefix: #{prefix}")	9✔
136	resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix })	9✔
137	resp_hash = resp.to_h	5✔
138	objects = parse_objects(resp_hash)	5✔
139	parse_continuation(resp_hash, objects)	5✔
140	end
141	end
142
143	def file_count	1✔
144	client_s3_files.count	×
145	end
146
147	# Retrieve the S3 resources from the S3 Bucket without those attached to the Work model
148	# @return [Array<S3File>]
149	def s3_files	1✔
150	model_s3_file_keys = model_s3_files.map(&:filename)	9✔
151	client_s3_files.reject { \|client_s3_file\| model_s3_file_keys.include?(client_s3_file.filename) }	9✔
152	end
153
154	##
155	# Query the S3 bucket for what we know about the doi
156	# For docs see:
157	# * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_objects_v2-instance_method
158	# * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#get_object_attributes-instance_method
159	# @return Hash with two properties {objects: [<S3File>], ok: Bool}
160	# objects is an Array of S3File objects
161	# ok is false if there is an error connecting to S3. Otherwise true.
162	def data_profile	1✔
163	{ objects: s3_files, ok: true }	9✔
164	rescue => ex
165	Rails.logger.error("Error querying S3. Bucket: #{bucket_name}. DOI: #{@doi}. Exception: #{ex.message}")	4✔
166
167	{ objects: [], ok: false }	4✔
168	end
169
170	##
171	# Copies the existing files from the pre-curation bucket to the post-curation bucket.
172	# Notice that the copy process happens at AWS (i.e. the files are not downloaded and re-uploaded).
173	# Returns an array with the files that were copied.
174	def publish_files	1✔
175	files = []	4✔
176	source_bucket = S3QueryService.pre_curation_config[:bucket]	4✔
177	target_bucket = S3QueryService.post_curation_config[:bucket]	4✔
178	model.pre_curation_uploads.each do \|file\|	4✔
179	params = {
180	copy_source: "/#{source_bucket}/#{file.key}",	4✔
181	bucket: target_bucket,
182	key: file.key
183	}
184	Rails.logger.info("Copying #{params[:copy_source]} to #{params[:bucket]}/#{params[:key]}")	4✔
185	client.copy_object(params)	4✔
186	files << file	4✔
187	end
188	files	4✔
189	end
190
191	private	1✔
192
193	def model_uploads	1✔
194	if pre_curation?	9✔
195	model.pre_curation_uploads	8✔
196	else
197	[]	1✔
198	end
199	end
200
201	def parse_objects(resp)	1✔
202	objects = []	5✔
203	resp_hash = resp.to_h	5✔
204	response_objects = resp_hash[:contents]	5✔
205	Rails.logger.debug("Objects: #{response_objects}")	5✔
206	response_objects&.each do \|object\|	5✔
207	next if object[:size] == 0 # ignore directories whose size is zero	×
208	s3_file = S3File.new(query_service: self, filename: object[:key], last_modified: object[:last_modified], size: object[:size], checksum: object[:etag])	×
209	objects << s3_file	×
210	end
211	objects	5✔
212	end
213
214	def parse_continuation(resp_hash, objects)	1✔
215	while resp_hash[:is_truncated]	5✔
216	token = resp_hash[:next_continuation_token]	×
217	resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix, continuation_token: token })	×
218	resp_hash = resp.to_h	×
219	more_objects = parse_objects(resp_hash)	×
220	objects += more_objects	×
221	end
222	objects	5✔
223	end
224	end

pulibrary / pdc_describe / bcfc06e8-a5d8-4369-a72f-de9ecf1ada29

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous