7031ff5b-e2d7-4285-af8b-e7b55adfc48a

Build # 7031ff5b-e2d7-4285-af8b-e7b55adfc48a

Build Type

circleci

Committed by mccalluc

Commit Message

Finish removal of explicit if-then in js

Pull Request Pull Request #899: Move messages up to sidebar on the right which can be toggled

Run Details

1623 of 1788 relevant lines covered (90.77%)

105.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.25

/app/services/s3_query_service.rb

# frozen_string_literal: true

require "aws-sdk-s3"

# A service to query an S3 bucket for information about a given data set
class S3QueryService
  attr_reader :model

  def self.configuration
    Rails.configuration.s3
  end

  def self.pre_curation_config
    configuration.pre_curation
  end

  def self.post_curation_config
    configuration.post_curation
  end

  def self.url_protocol
    "https"
  end

  def self.s3_host
    "s3.amazonaws.com"
  end

  ##
  # @param [Work] model
  # @param [Boolean] pre_curation
  # @example S3QueryService.new(Work.find(1), true)
  def initialize(model, pre_curation = true)
    @model = model
    @doi = model.doi
    @pre_curation = pre_curation
  end

  def config
    return self.class.post_curation_config if post_curation?

    self.class.pre_curation_config
  end

  def pre_curation?
    @pre_curation
  end

  def post_curation?
    !pre_curation?
  end

  ##
  # The name of the bucket this class is configured to use.
  # See config/s3.yml for configuration file.
  def bucket_name
    config.fetch(:bucket, nil)
  end

  def region
    config.fetch(:region, nil)
  end

  ##
  # The S3 prefix for this object, i.e., the address within the S3 bucket,
  # which is based on the DOI
  def prefix
    "#{@doi}/#{model.id}/"
  end

  ##
  # Construct an S3 address for this data set
  def s3_address
    "s3://#{bucket_name}/#{prefix}"
  end

  # There is probably a better way to fetch the current ActiveStorage configuration but we have
  # not found it.
  def active_storage_configuration
    Rails.configuration.active_storage.service_configurations[Rails.configuration.active_storage.service.to_s]
  end

  def access_key_id
    active_storage_configuration["access_key_id"]
  end

  def secret_access_key
    active_storage_configuration["secret_access_key"]
  end

  def credentials
    @credentials ||= Aws::Credentials.new(access_key_id, secret_access_key)
  end

  def client
    @client ||= Aws::S3::Client.new(region: region, credentials: credentials)
  end

  # Retrieve the S3 resources attached to the Work model
  # @return [Array<S3File>]
  def model_s3_files
    objects = []
    return objects if model.nil?

    model_uploads.each do |attachment|
      s3_file = S3File.new(query_service: self,
                           filename: attachment.key,
                           last_modified: attachment.created_at,
                           size: attachment.byte_size,
                           checksum: attachment.checksum)
      objects << s3_file
    end

    objects
  end

  def get_s3_object(key:)
    response = client.get_object({
                                   bucket: bucket_name,
                                   key: key
                                 })
    object = response.to_h
    return if object.empty?

    object
  end

  def find_s3_file(filename:)
    s3_object_key = "#{prefix}#{filename}"

    object = get_s3_object(key: s3_object_key)
    return if object.nil?

    S3File.new(query_service: self, filename: s3_object_key, last_modified: object[:last_modified], size: object[:content_length], checksum: object[:etag])
  end

  # Retrieve the S3 resources uploaded to the S3 Bucket
  # @return [Array<S3File>]
  def client_s3_files
    objects = []
    Rails.logger.debug("Bucket: #{bucket_name}")
    Rails.logger.debug("Prefix: #{prefix}")
    resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix })
    response_objects = resp.to_h[:contents]
    Rails.logger.debug("Objects: #{response_objects}")
    response_objects&.each do |object|
      next if object[:size] == 0 # ignore directories whose size is zero
      s3_file = S3File.new(query_service: self, filename: object[:key], last_modified: object[:last_modified], size: object[:size], checksum: object[:etag])
      objects << s3_file
    end

    objects
  end

  # Retrieve the S3 resources from the S3 Bucket without those attached to the Work model
  # @return [Array<S3File>]
  def s3_files
    model_s3_file_keys = model_s3_files.map(&:filename)
    client_s3_files.reject { |client_s3_file| model_s3_file_keys.include?(client_s3_file.filename) }
  end

  ##
  # Query the S3 bucket for what we know about the doi
  # For docs see:
  # * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_objects_v2-instance_method
  # * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#get_object_attributes-instance_method
  # @return Hash with two properties {objects: [<S3File>], ok: Bool}
  #   objects is an Array of S3File objects
  #   ok is false if there is an error connecting to S3. Otherwise true.
  def data_profile
    { objects: s3_files, ok: true }
  rescue => ex
    Rails.logger.error("Error querying S3. Bucket: #{bucket_name}. DOI: #{@doi}. Exception: #{ex.message}")

    { objects: [], ok: false }
  end

  ##
  # Copies the existing files from the pre-curation bucket to the post-curation bucket.
  # Notice that the copy process happens at AWS (i.e. the files are not downloaded and re-uploaded).
  # Returns an array with the files that were copied.
  def publish_files
    files = []
    source_bucket = S3QueryService.pre_curation_config[:bucket]
    target_bucket = S3QueryService.post_curation_config[:bucket]
    model.pre_curation_uploads.each do |file|
      params = {
        copy_source: "/#{source_bucket}/#{file.key}",
        bucket: target_bucket,
        key: file.key
      }
      Rails.logger.info("Copying #{params[:copy_source]} to #{params[:bucket]}/#{params[:key]}")
      client.copy_object(params)
      files << file
    end
    files
  end

  private

    def model_uploads
      if pre_curation?
        model.pre_curation_uploads
      else
        []
      end
    end
end

1	# frozen_string_literal: true
2
3	require "aws-sdk-s3"	1✔
4
5	# A service to query an S3 bucket for information about a given data set
6	class S3QueryService	1✔
7	attr_reader :model	1✔
8
9	def self.configuration	1✔
10	Rails.configuration.s3	297✔
11	end
12
13	def self.pre_curation_config	1✔
14	configuration.pre_curation	169✔
15	end
16
17	def self.post_curation_config	1✔
18	configuration.post_curation	128✔
19	end
20
21	def self.url_protocol	1✔
22	"https"	×
23	end
24
25	def self.s3_host	1✔
26	"s3.amazonaws.com"	×
27	end
28
29	##
30	# @param [Work] model
31	# @param [Boolean] pre_curation
32	# @example S3QueryService.new(Work.find(1), true)
33	def initialize(model, pre_curation = true)	1✔
34	@model = model	129✔
35	@doi = model.doi	129✔
36	@pre_curation = pre_curation	129✔
37	end
38
39	def config	1✔
40	return self.class.post_curation_config if post_curation?	259✔
41
42	self.class.pre_curation_config	148✔
43	end
44
45	def pre_curation?	1✔
46	@pre_curation	303✔
47	end
48
49	def post_curation?	1✔
50	!pre_curation?	259✔
51	end
52
53	##
54	# The name of the bucket this class is configured to use.
55	# See config/s3.yml for configuration file.
56	def bucket_name	1✔
57	config.fetch(:bucket, nil)	155✔
58	end
59
60	def region	1✔
61	config.fetch(:region, nil)	104✔
62	end
63
64	##
65	# The S3 prefix for this object, i.e., the address within the S3 bucket,
66	# which is based on the DOI
67	def prefix	1✔
68	"#{@doi}/#{model.id}/"	89✔
69	end
70
71	##
72	# Construct an S3 address for this data set
73	def s3_address	1✔
74	"s3://#{bucket_name}/#{prefix}"	1✔
75	end
76
77	# There is probably a better way to fetch the current ActiveStorage configuration but we have
78	# not found it.
79	def active_storage_configuration	1✔
80	Rails.configuration.active_storage.service_configurations[Rails.configuration.active_storage.service.to_s]	208✔
81	end
82
83	def access_key_id	1✔
84	active_storage_configuration["access_key_id"]	104✔
85	end
86
87	def secret_access_key	1✔
88	active_storage_configuration["secret_access_key"]	104✔
89	end
90
91	def credentials	1✔
92	@credentials \|\|= Aws::Credentials.new(access_key_id, secret_access_key)	104✔
93	end
94
95	def client	1✔
96	@client \|\|= Aws::S3::Client.new(region: region, credentials: credentials)	104✔
97	end
98
99	# Retrieve the S3 resources attached to the Work model
100	# @return [Array<S3File>]
101	def model_s3_files	1✔
102	objects = []	44✔
103	return objects if model.nil?	44✔
104
105	model_uploads.each do \|attachment\|	44✔
106	s3_file = S3File.new(query_service: self,	17✔
107	filename: attachment.key,
108	last_modified: attachment.created_at,
109	size: attachment.byte_size,
110	checksum: attachment.checksum)
111	objects << s3_file	17✔
112	end
113
114	objects	44✔
115	end
116
117	def get_s3_object(key:)	1✔
118	response = client.get_object({	×
119	bucket: bucket_name,
120	key: key
121	})
122	object = response.to_h	×
123	return if object.empty?	×
124
125	object	×
126	end
127
128	def find_s3_file(filename:)	1✔
129	s3_object_key = "#{prefix}#{filename}"	×
130
131	object = get_s3_object(key: s3_object_key)	×
132	return if object.nil?	×
133
134	S3File.new(query_service: self, filename: s3_object_key, last_modified: object[:last_modified], size: object[:content_length], checksum: object[:etag])	×
135	end
136
137	# Retrieve the S3 resources uploaded to the S3 Bucket
138	# @return [Array<S3File>]
139	def client_s3_files	1✔
140	objects = []	44✔
141	Rails.logger.debug("Bucket: #{bucket_name}")	44✔
142	Rails.logger.debug("Prefix: #{prefix}")	44✔
143	resp = client.list_objects_v2({ bucket: bucket_name, max_keys: 1000, prefix: prefix })	44✔
144	response_objects = resp.to_h[:contents]	30✔
145	Rails.logger.debug("Objects: #{response_objects}")	30✔
146	response_objects&.each do \|object\|	30✔
147	next if object[:size] == 0 # ignore directories whose size is zero	12✔
148	s3_file = S3File.new(query_service: self, filename: object[:key], last_modified: object[:last_modified], size: object[:size], checksum: object[:etag])	10✔
149	objects << s3_file	10✔
150	end
151
152	objects	30✔
153	end
154
155	# Retrieve the S3 resources from the S3 Bucket without those attached to the Work model
156	# @return [Array<S3File>]
157	def s3_files	1✔
158	model_s3_file_keys = model_s3_files.map(&:filename)	44✔
159	client_s3_files.reject { \|client_s3_file\| model_s3_file_keys.include?(client_s3_file.filename) }	54✔
160	end
161
162	##
163	# Query the S3 bucket for what we know about the doi
164	# For docs see:
165	# * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_objects_v2-instance_method
166	# * https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#get_object_attributes-instance_method
167	# @return Hash with two properties {objects: [<S3File>], ok: Bool}
168	# objects is an Array of S3File objects
169	# ok is false if there is an error connecting to S3. Otherwise true.
170	def data_profile	1✔
171	{ objects: s3_files, ok: true }	44✔
172	rescue => ex
173	Rails.logger.error("Error querying S3. Bucket: #{bucket_name}. DOI: #{@doi}. Exception: #{ex.message}")	14✔
174
175	{ objects: [], ok: false }	14✔
176	end
177
178	##
179	# Copies the existing files from the pre-curation bucket to the post-curation bucket.
180	# Notice that the copy process happens at AWS (i.e. the files are not downloaded and re-uploaded).
181	# Returns an array with the files that were copied.
182	def publish_files	1✔
183	files = []	17✔
184	source_bucket = S3QueryService.pre_curation_config[:bucket]	17✔
185	target_bucket = S3QueryService.post_curation_config[:bucket]	17✔
186	model.pre_curation_uploads.each do \|file\|	17✔
187	params = {
188	copy_source: "/#{source_bucket}/#{file.key}",	17✔
189	bucket: target_bucket,
190	key: file.key
191	}
192	Rails.logger.info("Copying #{params[:copy_source]} to #{params[:bucket]}/#{params[:key]}")	17✔
193	client.copy_object(params)	17✔
194	files << file	17✔
195	end
196	files	17✔
197	end
198
199	private	1✔
200
201	def model_uploads	1✔
202	if pre_curation?	44✔
203	model.pre_curation_uploads	24✔
204	else
205	[]	20✔
206	end
207	end
208	end

pulibrary / pdc_describe / 7031ff5b-e2d7-4285-af8b-e7b55adfc48a

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous