5205f704-d89d-4c43-8cfa-9e6783edc33a

Committed 29 Feb 2024 07:04PM UTC coverage: 60.206% (-29.9%) from 90.092%

Build # 5205f704-d89d-4c43-8cfa-9e6783edc33a

Build Type

push

circleci

Committed by

jrgriffiniii

Commit Message

Ensuring that the storage usage and storage capacity is retrieved for
Mediaflux Projects and rendered on the "contents" Project View

Run Details

11 of 34 new or added lines in 4 files covered. (32.35%)

451 existing lines in 34 files now uncovered.

935 of 1553 relevant lines covered (60.21%)

9.47 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.94

/app/models/project.rb

# frozen_string_literal: true
class Project < ApplicationRecord
  validates_with ProjectValidator
  has_many :provenance_events, dependent: :destroy

  # TODO: What are the valid statuses?
  PENDING_STATUS = "pending"
  APPROVE_STATUS = "active"
  delegate :to_json, to: :metadata_json

  def metadata
    (metadata_json || {}).with_indifferent_access
  end

  def metadata=(metadata)
    self.metadata_json = metadata
  end

  # TODO: Presumably we should display other statuses as well?
  def title
    trailer = if in_mediaflux?
                ""
              else
                " (#{::Project::PENDING_STATUS})"
              end
    metadata[:title] + trailer
  end

  def departments
    unsorted = metadata[:departments] || []
    unsorted.sort
  end

  def directory
    metadata[:directory]
  end

  def status
    metadata[:status]
  end

  def in_mediaflux?
    mediaflux_id.present?
  end

  def self.sponsored_projects(sponsor)
    Project.where("metadata_json->>'data_sponsor' = ?", sponsor)
  end

  def self.managed_projects(manager)
    Project.where("metadata_json->>'data_manager' = ?", manager)
  end

  def self.pending_projects
    Project.where("mediaflux_id IS NULL")
  end

  def self.approved_projects
    Project.where("mediaflux_id IS NOT NULL")
  end

  def self.data_user_projects(user)
    # See https://scalegrid.io/blog/using-jsonb-in-postgresql-how-to-effectively-store-index-json-data-in-postgresql/
    # for information on the @> operator
    query_ro = '{"data_user_read_only":["' + user + '"]}'
    query_rw = '{"data_user_read_write":["' + user + '"]}'
    Project.where("(metadata_json @> ? :: jsonb) OR (metadata_json @> ? :: jsonb)", query_ro, query_rw)
  end

  def save_in_mediaflux(session_id:)
    if mediaflux_id.nil?
      self.mediaflux_id = ProjectMediaflux.create!(project: self, session_id: session_id)
      save!
      Rails.logger.debug "Project #{id} has been created in MediaFlux (asset id #{mediaflux_id})"
    else
      ProjectMediaflux.update(project: self, session_id: session_id)
      Rails.logger.debug "Project #{id} has been updated in MediaFlux (asset id #{mediaflux_id})"
    end
    mediaflux_id
  end

  def created_by_user
    User.find_by(uid: metadata[:created_by])
  end

  def to_xml
    ProjectMediaflux.xml_payload(project: self)
  end

  def mediaflux_document
    ProjectMediaflux.document(project: self)
  end

  def mediaflux_metadata(session_id:)
    accum_req = Mediaflux::Http::GetMetadataRequest.new(session_token: session_id, id: mediaflux_id)
    v = accum_req.metadata
    puts v
    v
  end

  def asset_count(session_id:)
    values = mediaflux_metadata(session_id:)
    values.fetch(:total_file_count, 0)
  end

  def self.default_storage_usage
    "0 KB"
  end

  def storage_usage(session_id:)
    return unless in_mediaflux?

    values = mediaflux_metadata(session_id:)
    value = values[:size_human]

    return self.class.default_storage_usage if value.blank?
    value
  end

  def self.default_storage_capacity
    "0 GB"
  end

  def storage_capacity_xml
    return unless in_mediaflux?

    mediaflux_document.at_xpath("/request/service/args/meta/tigerdata:project/StorageCapacity/text()", tigerdata: "http://tigerdata.princeton.edu")
  end

  def storage_capacity(session_id:)
    value = storage_capacity_xml

    return self.class.default_storage_capacity if value.blank?
    value
  end

  # Fetches the first n files
  def file_list(session_id:, size: 10)
    return { files: [] } if mediaflux_id.nil?

    query_req = Mediaflux::Http::QueryRequest.new(session_token: session_id, collection: mediaflux_id, deep_search: true)
    iterator_id = query_req.result

    iterator_req = Mediaflux::Http::IteratorRequest.new(session_token: session_id, iterator: iterator_id, size: size)
    results = iterator_req.result

    # Destroy _after_ fetching the first set of results from iterator_req.
    # This call is required since it possible that we have read less assets than
    # what the collection has but we are done with the iterator.
    Mediaflux::Http::IteratorDestroyRequest.new(session_token: session_id, iterator: iterator_id).resolve

    results
  end

  # Fetches the entire file list to a file
  def file_list_to_file(session_id:, filename:)
    return { files: [] } if mediaflux_id.nil?

    query_req = Mediaflux::Http::QueryRequest.new(session_token: session_id, collection: mediaflux_id, deep_search: true)
    iterator_id = query_req.result

    File.open(filename, "w") do |file|
      # file header
      file.write("ID, PATH, NAME, COLLECTION?, LAST_MODIFIED, SIZE\r\n")
      loop do
        iterator_req = Mediaflux::Http::IteratorRequest.new(session_token: session_id, iterator: iterator_id, size: 1000)
        iterator_resp = iterator_req.result
        lines = files_from_iterator(iterator_resp)
        file.write(lines.join("\r\n") + "\r\n")
        break if iterator_resp[:complete]
      end
    end

    # Destroy _after_ fetching the results from iterator_req
    # This call is technically not necessary since Mediaflux automatically deletes the iterator
    # once we have ran through it and by now we have. But it does not hurt either.
    Mediaflux::Http::IteratorDestroyRequest.new(session_token: session_id, iterator: iterator_id).resolve
  end

  private

    def files_from_iterator(iterator_resp)
      lines = []
      iterator_resp[:files].each do |asset|
        lines << "#{asset.id}, #{asset.path_only}, #{asset.name}, #{asset.collection}, #{asset.last_modified}, #{asset.size}"
      end
      lines
    end
end

1	# frozen_string_literal: true
2	class Project < ApplicationRecord	1✔
3	validates_with ProjectValidator	1✔
4	has_many :provenance_events, dependent: :destroy	1✔
5
6	# TODO: What are the valid statuses?
7	PENDING_STATUS = "pending"	1✔
8	APPROVE_STATUS = "active"	1✔
9	delegate :to_json, to: :metadata_json	1✔
10
11	def metadata	1✔
12	(metadata_json \|\| {}).with_indifferent_access	1,564✔
13	end
14
15	def metadata=(metadata)	1✔
16	self.metadata_json = metadata	123✔
17	end
18
19	# TODO: Presumably we should display other statuses as well?
20	def title	1✔
21	trailer = if in_mediaflux?	57✔
22	""	9✔
23	else
24	" (#{::Project::PENDING_STATUS})"	48✔
25	end
26	metadata[:title] + trailer	57✔
27	end
28
29	def departments	1✔
30	unsorted = metadata[:departments] \|\| []	108✔
31	unsorted.sort	108✔
32	end
33
34	def directory	1✔
35	metadata[:directory]	16✔
36	end
37
38	def status	1✔
39	metadata[:status]	1✔
40	end
41
42	def in_mediaflux?	1✔
43	mediaflux_id.present?	115✔
44	end
45
46	def self.sponsored_projects(sponsor)	1✔
47	Project.where("metadata_json->>'data_sponsor' = ?", sponsor)	27✔
48	end
49
50	def self.managed_projects(manager)	1✔
51	Project.where("metadata_json->>'data_manager' = ?", manager)	27✔
52	end
53
54	def self.pending_projects	1✔
55	Project.where("mediaflux_id IS NULL")	27✔
56	end
57
58	def self.approved_projects	1✔
59	Project.where("mediaflux_id IS NOT NULL")	27✔
60	end
61
62	def self.data_user_projects(user)	1✔
63	# See https://scalegrid.io/blog/using-jsonb-in-postgresql-how-to-effectively-store-index-json-data-in-postgresql/
64	# for information on the @> operator
65	query_ro = '{"data_user_read_only":["' + user + '"]}'	27✔
66	query_rw = '{"data_user_read_write":["' + user + '"]}'	27✔
67	Project.where("(metadata_json @> ? :: jsonb) OR (metadata_json @> ? :: jsonb)", query_ro, query_rw)	27✔
68	end
69
70	def save_in_mediaflux(session_id:)	1✔
71	if mediaflux_id.nil?	×
72	self.mediaflux_id = ProjectMediaflux.create!(project: self, session_id: session_id)	×
73	save!	×
74	Rails.logger.debug "Project #{id} has been created in MediaFlux (asset id #{mediaflux_id})"	×
75	else
76	ProjectMediaflux.update(project: self, session_id: session_id)	×
77	Rails.logger.debug "Project #{id} has been updated in MediaFlux (asset id #{mediaflux_id})"	×
78	end
79	mediaflux_id	×
80	end
81
82	def created_by_user	1✔
UNCOV 83	User.find_by(uid: metadata[:created_by])	×
84	end
85
86	def to_xml	1✔
87	ProjectMediaflux.xml_payload(project: self)	2✔
88	end
89
90	def mediaflux_document	1✔
NEW 91	ProjectMediaflux.document(project: self)	×
92	end
93
94	def mediaflux_metadata(session_id:)	1✔
UNCOV 95	accum_req = Mediaflux::Http::GetMetadataRequest.new(session_token: session_id, id: mediaflux_id)	×
NEW 96	v = accum_req.metadata	×
NEW 97	puts v	×
NEW 98	v	×
99	end
100
101	def asset_count(session_id:)	1✔
NEW 102	values = mediaflux_metadata(session_id:)	×
NEW 103	values.fetch(:total_file_count, 0)	×
104	end
105
106	def self.default_storage_usage	1✔
NEW 107	"0 KB"	×
108	end
109
110	def storage_usage(session_id:)	1✔
NEW 111	return unless in_mediaflux?	×
112
NEW 113	values = mediaflux_metadata(session_id:)	×
NEW 114	value = values[:size_human]	×
115
NEW 116	return self.class.default_storage_usage if value.blank?	×
NEW 117	value	×
118	end
119
120	def self.default_storage_capacity	1✔
NEW 121	"0 GB"	×
122	end
123
124	def storage_capacity_xml	1✔
NEW 125	return unless in_mediaflux?	×
126
NEW 127	mediaflux_document.at_xpath("/request/service/args/meta/tigerdata:project/StorageCapacity/text()", tigerdata: "http://tigerdata.princeton.edu")	×
128	end
129
130	def storage_capacity(session_id:)	1✔
NEW 131	value = storage_capacity_xml	×
132
NEW 133	return self.class.default_storage_capacity if value.blank?	×
NEW 134	value	×
135	end
136
137	# Fetches the first n files
138	def file_list(session_id:, size: 10)	1✔
139	return { files: [] } if mediaflux_id.nil?	1✔
140
141	query_req = Mediaflux::Http::QueryRequest.new(session_token: session_id, collection: mediaflux_id, deep_search: true)	1✔
UNCOV 142	iterator_id = query_req.result	×
143
UNCOV 144	iterator_req = Mediaflux::Http::IteratorRequest.new(session_token: session_id, iterator: iterator_id, size: size)	×
UNCOV 145	results = iterator_req.result	×
146
147	# Destroy _after_ fetching the first set of results from iterator_req.
148	# This call is required since it possible that we have read less assets than
149	# what the collection has but we are done with the iterator.
UNCOV 150	Mediaflux::Http::IteratorDestroyRequest.new(session_token: session_id, iterator: iterator_id).resolve	×
151
UNCOV 152	results	×
153	end
154
155	# Fetches the entire file list to a file
156	def file_list_to_file(session_id:, filename:)	1✔
UNCOV 157	return { files: [] } if mediaflux_id.nil?	×
158
UNCOV 159	query_req = Mediaflux::Http::QueryRequest.new(session_token: session_id, collection: mediaflux_id, deep_search: true)	×
UNCOV 160	iterator_id = query_req.result	×
161
UNCOV 162	File.open(filename, "w") do \|file\|	×
163	# file header
UNCOV 164	file.write("ID, PATH, NAME, COLLECTION?, LAST_MODIFIED, SIZE\r\n")	×
UNCOV 165	loop do	×
UNCOV 166	iterator_req = Mediaflux::Http::IteratorRequest.new(session_token: session_id, iterator: iterator_id, size: 1000)	×
UNCOV 167	iterator_resp = iterator_req.result	×
UNCOV 168	lines = files_from_iterator(iterator_resp)	×
UNCOV 169	file.write(lines.join("\r\n") + "\r\n")	×
UNCOV 170	break if iterator_resp[:complete]	×
171	end
172	end
173
174	# Destroy _after_ fetching the results from iterator_req
175	# This call is technically not necessary since Mediaflux automatically deletes the iterator
176	# once we have ran through it and by now we have. But it does not hurt either.
UNCOV 177	Mediaflux::Http::IteratorDestroyRequest.new(session_token: session_id, iterator: iterator_id).resolve	×
178	end
179
180	private	1✔
181
182	def files_from_iterator(iterator_resp)	1✔
UNCOV 183	lines = []	×
UNCOV 184	iterator_resp[:files].each do \|asset\|	×
UNCOV 185	lines << "#{asset.id}, #{asset.path_only}, #{asset.name}, #{asset.collection}, #{asset.last_modified}, #{asset.size}"	×
186	end
UNCOV 187	lines	×
188	end
189	end

pulibrary / tigerdata-app / 5205f704-d89d-4c43-8cfa-9e6783edc33a

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous