5af13607-7741-41ac-b110-c94c8a3ee0f0

pending completion

Build # 5af13607-7741-41ac-b110-c94c8a3ee0f0

Build Type

Pull #441

circleci

Committed by

hectorcorrea

Commit Message

Better error handling

Pull Request Pull Request #441: Indexing to a new collection

Run Details

99 of 99 new or added lines in 3 files covered. (100.0%)

1816 of 2220 relevant lines covered (81.8%)

95.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.7

/lib/traject/solr_cloud_helper.rb

# frozen_string_literal: true

# rubocop:disable Metrics/ClassLength
class SolrCloudHelper
  def self.alias_uri
    Blacklight.default_index.connection.uri
  end

  def self.alias_url
    Blacklight.default_index.connection.uri.to_s
  end

  def self.config_set
    Blacklight.default_index.connection.options[:solr_config_set]
  end

  def self.collection_writer_commit!
    commit_url = "#{collection_writer_url}/update?commit=true"
    HTTParty.get(commit_url)
  end

  # Returns the Solr collection that we should use for quering data
  def self.collection_reader_url
    collection = current_collection_for_alias(alias_uri)
    build_solr_url_for_collection(alias_uri, collection)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exist)
  def self.collection_writer_url
    collection_writer_for_alias(alias_uri, false)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exists, deletes & recreates it if already exists)
  def self.create_collection_writer
    collection_writer_for_alias(alias_uri, true)
  end

  # For a given Solr alias, returns a URL that is suitable for writing new data without affecting
  # the existing Solr index. It does this by creating a new alternate Solr collection to write data,
  # instead of overwriting the data on the collection used by the provided solr_alias_uri.
  #
  # The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
  #
  # For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
  # and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
  # the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
  #
  # If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
  # then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
  def self.collection_writer_for_alias(solr_alias_uri, recreate)
    if config_set.nil?
      # We are not running in a Solr cloud environment - nothing to do.
      return solr_alias_uri.to_s
    end

    alternate_collection = alternate_collection_for_alias(solr_alias_uri)
    if collection_exist?(solr_alias_uri, alternate_collection)
      if recreate
        # Re-create it
        delete_collection!(solr_alias_uri, alternate_collection)
        create_collection(solr_alias_uri, alternate_collection)
      end
    else
      # Create it
      create_collection(solr_alias_uri, alternate_collection)
    end

    build_solr_url_for_collection(solr_alias_uri, alternate_collection)
  end

  # Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
  # the indicated `collection` instead of pointing to the alias.
  def self.build_solr_url_for_collection(solr_alias_uri, collection)
    URI::HTTP.build(schema: solr_alias_uri.scheme, host: solr_alias_uri.host, port: solr_alias_uri.port, path: "/solr/#{collection}").to_s
  end

  def self.current_collection_for_alias(solr_alias_uri)
    alias_name = solr_alias_uri.path.split("/").last
    alias_list_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=LISTALIASES"
    )
    response = HTTParty.get(alias_list_query.to_s)
    # The response will indicate the actual collection the alias points to.
    collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200
    collection_name || alias_name
  end

  def self.alternate_collection_for_alias(solr_alias_uri)
    solr_collection = current_collection_for_alias(solr_alias_uri)
    if solr_collection.end_with?("-1")
      solr_collection.gsub("-1", "-2")
    elsif solr_collection.end_with?("-2")
      solr_collection.gsub("-2", "-1")
    else
      solr_collection
    end
  end

  def self.collection_exist?(solr_alias_uri, collection_name)
    collection_list_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=LIST"
    )
    response = HTTParty.get(collection_list_query.to_s)
    collections = if response.code == 200
                    response.parsed_response.dig("collections") || []
                  else
                    []
                  end
    collections.any?(collection_name)
  end

  def self.create_collection(solr_alias_uri, collection_name)
    create_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
    )
    response = HTTParty.get(create_query.to_s)
    return unless response.code == 200 && response.parsed_response.key?("success")
    collection_name
  end

  def self.delete_collection!(solr_alias_uri, collection_name)
    create_query = URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=DELETE&name=#{collection_name}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end

  # Set the solr_alias to point to the current writer collection
  def self.update_solr_alias!
    writer_collection = collection_writer_url.split("/").last

    alias_name = alias_uri.path.split("/").last
    if alias_name == writer_collection
      # Nothing to do
      # (we are probably using standalone Solr in development)
      return true
    end

    create_query = URI::HTTP.build(
      schema: alias_uri.scheme,
      host: alias_uri.host,
      port: alias_uri.port,
      path: "/solr/admin/collections",
      query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end
end
# rubocop:enable Metrics/ClassLength

1	# frozen_string_literal: true
2
3	# rubocop:disable Metrics/ClassLength
4	class SolrCloudHelper	1✔
5	def self.alias_uri	1✔
6	Blacklight.default_index.connection.uri	207✔
7	end
8
9	def self.alias_url	1✔
10	Blacklight.default_index.connection.uri.to_s	×
11	end
12
13	def self.config_set	1✔
14	Blacklight.default_index.connection.options[:solr_config_set]	415✔
15	end
16
17	def self.collection_writer_commit!	1✔
18	commit_url = "#{collection_writer_url}/update?commit=true"	×
19	HTTParty.get(commit_url)	×
20	end
21
22	# Returns the Solr collection that we should use for quering data
23	def self.collection_reader_url	1✔
24	collection = current_collection_for_alias(alias_uri)	×
25	build_solr_url_for_collection(alias_uri, collection)	×
26	end
27
28	# Returns the Solr collection that we should write to
29	# (creates it if does not exist)
30	def self.collection_writer_url	1✔
31	collection_writer_for_alias(alias_uri, false)	207✔
32	end
33
34	# Returns the Solr collection that we should write to
35	# (creates it if does not exists, deletes & recreates it if already exists)
36	def self.create_collection_writer	1✔
37	collection_writer_for_alias(alias_uri, true)	×
38	end
39
40	# For a given Solr alias, returns a URL that is suitable for writing new data without affecting
41	# the existing Solr index. It does this by creating a new alternate Solr collection to write data,
42	# instead of overwriting the data on the collection used by the provided solr_alias_uri.
43	#
44	# The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
45	#
46	# For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
47	# and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
48	# the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
49	#
50	# If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
51	# then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
52	def self.collection_writer_for_alias(solr_alias_uri, recreate)	1✔
53	if config_set.nil?	208✔
54	# We are not running in a Solr cloud environment - nothing to do.
55	return solr_alias_uri.to_s	×
56	end
57
58	alternate_collection = alternate_collection_for_alias(solr_alias_uri)	208✔
59	if collection_exist?(solr_alias_uri, alternate_collection)	208✔
60	if recreate	1✔
61	# Re-create it
62	delete_collection!(solr_alias_uri, alternate_collection)	×
63	create_collection(solr_alias_uri, alternate_collection)	×
64	end
65	else
66	# Create it
67	create_collection(solr_alias_uri, alternate_collection)	207✔
68	end
69
70	build_solr_url_for_collection(solr_alias_uri, alternate_collection)	208✔
71	end
72
73	# Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
74	# the indicated `collection` instead of pointing to the alias.
75	def self.build_solr_url_for_collection(solr_alias_uri, collection)	1✔
76	URI::HTTP.build(schema: solr_alias_uri.scheme, host: solr_alias_uri.host, port: solr_alias_uri.port, path: "/solr/#{collection}").to_s	208✔
77	end
78
79	def self.current_collection_for_alias(solr_alias_uri)	1✔
80	alias_name = solr_alias_uri.path.split("/").last	212✔
81	alias_list_query = URI::HTTP.build(	212✔
82	schema: solr_alias_uri.scheme,
83	host: solr_alias_uri.host,
84	port: solr_alias_uri.port,
85	path: "/solr/admin/collections",
86	query: "action=LISTALIASES"
87	)
88	response = HTTParty.get(alias_list_query.to_s)	212✔
89	# The response will indicate the actual collection the alias points to.
90	collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200	212✔
91	collection_name \|\| alias_name	212✔
92	end
93
94	def self.alternate_collection_for_alias(solr_alias_uri)	1✔
95	solr_collection = current_collection_for_alias(solr_alias_uri)	211✔
96	if solr_collection.end_with?("-1")	211✔
97	solr_collection.gsub("-1", "-2")	2✔
98	elsif solr_collection.end_with?("-2")	209✔
99	solr_collection.gsub("-2", "-1")	1✔
100	else
101	solr_collection	208✔
102	end
103	end
104
105	def self.collection_exist?(solr_alias_uri, collection_name)	1✔
106	collection_list_query = URI::HTTP.build(	210✔
107	schema: solr_alias_uri.scheme,
108	host: solr_alias_uri.host,
109	port: solr_alias_uri.port,
110	path: "/solr/admin/collections",
111	query: "action=LIST"
112	)
113	response = HTTParty.get(collection_list_query.to_s)	210✔
114	collections = if response.code == 200	210✔
115	response.parsed_response.dig("collections") \|\| []	3✔
116	else
117	[]	207✔
118	end
119	collections.any?(collection_name)	210✔
120	end
121
122	def self.create_collection(solr_alias_uri, collection_name)	1✔
123	create_query = URI::HTTP.build(	207✔
124	schema: solr_alias_uri.scheme,
125	host: solr_alias_uri.host,
126	port: solr_alias_uri.port,
127	path: "/solr/admin/collections",
128	query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
129	)
130	response = HTTParty.get(create_query.to_s)	207✔
131	return unless response.code == 200 && response.parsed_response.key?("success")	207✔
132	collection_name	×
133	end
134
135	def self.delete_collection!(solr_alias_uri, collection_name)	1✔
136	create_query = URI::HTTP.build(	×
137	schema: solr_alias_uri.scheme,
138	host: solr_alias_uri.host,
139	port: solr_alias_uri.port,
140	path: "/solr/admin/collections",
141	query: "action=DELETE&name=#{collection_name}"
142	)
143	response = HTTParty.get(create_query.to_s)	×
144	response.code == 200	×
145	end
146
147	# Set the solr_alias to point to the current writer collection
148	def self.update_solr_alias!	1✔
149	writer_collection = collection_writer_url.split("/").last	×
150
151	alias_name = alias_uri.path.split("/").last	×
152	if alias_name == writer_collection	×
153	# Nothing to do
154	# (we are probably using standalone Solr in development)
155	return true	×
156	end
157
158	create_query = URI::HTTP.build(	×
159	schema: alias_uri.scheme,
160	host: alias_uri.host,
161	port: alias_uri.port,
162	path: "/solr/admin/collections",
163	query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
164	)
165	response = HTTParty.get(create_query.to_s)	×
166	response.code == 200	×
167	end
168	end
169	# rubocop:enable Metrics/ClassLength

pulibrary / pdc_discovery / 5af13607-7741-41ac-b110-c94c8a3ee0f0

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous