803026b9-f736-4b16-b861-c3d4bff47a42

pending completion

Build # 803026b9-f736-4b16-b861-c3d4bff47a42

Build Type

Pull #441

circleci

Committed by

hectorcorrea

Commit Message

Account for userinfo in Solr URL (which we only use in CI)

Pull Request Pull Request #441: Indexing to a new collection

Run Details

101 of 101 new or added lines in 3 files covered. (100.0%)

1768 of 2222 relevant lines covered (79.57%)

6.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

39.71

/lib/traject/solr_cloud_helper.rb

# frozen_string_literal: true

# rubocop:disable Metrics/ClassLength
class SolrCloudHelper
  def self.alias_uri
    Blacklight.default_index.connection.uri
  end

  def self.alias_url
    Blacklight.default_index.connection.uri.to_s
  end

  def self.config_set
    Blacklight.default_index.connection.options[:solr_config_set]
  end

  def self.collection_writer_commit!
    commit_url = "#{collection_writer_url}/update?commit=true"
    HTTParty.get(commit_url)
  end

  # Returns the Solr collection that we should use for quering data
  def self.collection_reader_url
    collection = current_collection_for_alias(alias_uri)
    build_solr_url_for_collection(alias_uri, collection)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exist)
  def self.collection_writer_url
    collection_writer_for_alias(alias_uri, false)
  end

  # Returns the Solr collection that we should write to
  # (creates it if does not exists, deletes & recreates it if already exists)
  def self.create_collection_writer
    collection_writer_for_alias(alias_uri, true)
  end

  # For a given Solr alias, returns a URL that is suitable for writing new data without affecting
  # the existing Solr index. It does this by creating a new alternate Solr collection to write data,
  # instead of overwriting the data on the collection used by the provided solr_alias_uri.
  #
  # The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
  #
  # For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
  # and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
  # the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
  #
  # If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
  # then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
  def self.collection_writer_for_alias(solr_alias_uri, recreate)
    if config_set.nil?
      # We are not running in a Solr cloud environment - nothing to do.
      return solr_alias_uri.to_s
    end

    alternate_collection = alternate_collection_for_alias(solr_alias_uri)
    if collection_exist?(solr_alias_uri, alternate_collection)
      if recreate
        # Re-create it
        delete_collection!(solr_alias_uri, alternate_collection)
        create_collection(solr_alias_uri, alternate_collection)
      end
    else
      # Create it
      create_collection(solr_alias_uri, alternate_collection)
    end

    build_solr_url_for_collection(solr_alias_uri, alternate_collection)
  end

  # Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
  # the indicated `collection` instead of pointing to the alias.
  def self.build_solr_url_for_collection(solr_alias_uri, collection)
    URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      userinfo: solr_alias_uri.userinfo,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: "/solr/#{collection}").to_s
  end

  def self.current_collection_for_alias(solr_alias_uri)
    alias_name = solr_alias_uri.path.split("/").last
    alias_list_query = uri_from_solr_alias(
      path: "/solr/admin/collections",
      query: "action=LISTALIASES"
    )
    response = HTTParty.get(alias_list_query.to_s)
    # The response will indicate the actual collection the alias points to.
    collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200
    collection_name || alias_name
  end

  def self.alternate_collection_for_alias(solr_alias_uri)
    solr_collection = current_collection_for_alias(solr_alias_uri)
    if solr_collection.end_with?("-1")
      solr_collection.gsub("-1", "-2")
    elsif solr_collection.end_with?("-2")
      solr_collection.gsub("-2", "-1")
    else
      solr_collection
    end
  end

  def self.collection_exist?(solr_alias_uri, collection_name)
    collection_list_query = uri_from_solr_alias(
      path: "/solr/admin/collections",
      query: "action=LIST"
    )
    response = HTTParty.get(collection_list_query.to_s)
    collections = if response.code == 200
                    response.parsed_response.dig("collections") || []
                  else
                    []
                  end
    collections.any?(collection_name)
  end

  def self.create_collection(solr_alias_uri, collection_name)
    create_query = uri_from_solr_alias(
      path: "/solr/admin/collections",
      query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
    )
    response = HTTParty.get(create_query.to_s)
    return unless response.code == 200 && response.parsed_response.key?("success")
    collection_name
  end

  def self.delete_collection!(solr_alias_uri, collection_name)
    create_query = uri_from_solr_alias(
      path: "/solr/admin/collections",
      query: "action=DELETE&name=#{collection_name}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end

  # Set the solr_alias to point to the current writer collection
  def self.update_solr_alias!
    writer_collection = collection_writer_url.split("/").last

    alias_name = alias_uri.path.split("/").last
    if alias_name == writer_collection
      # Nothing to do
      # (we are probably using standalone Solr in development)
      return true
    end

    create_query = uri_from_solr_alias(
      path: "/solr/admin/collections",
      query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
    )
    response = HTTParty.get(create_query.to_s)
    response.code == 200
  end

  def self.uri_from_solr_alias(path:, query:)
    URI::HTTP.build(
      schema: solr_alias_uri.scheme,
      userinfo: solr_alias.userinfo,
      host: solr_alias_uri.host,
      port: solr_alias_uri.port,
      path: path,
      query: query
    )
  end
end
# rubocop:enable Metrics/ClassLength

1	# frozen_string_literal: true
2
3	# rubocop:disable Metrics/ClassLength
4	class SolrCloudHelper	1✔
5	def self.alias_uri	1✔
6	Blacklight.default_index.connection.uri	207✔
7	end
8
9	def self.alias_url	1✔
10	Blacklight.default_index.connection.uri.to_s	×
11	end
12
13	def self.config_set	1✔
14	Blacklight.default_index.connection.options[:solr_config_set]	208✔
15	end
16
17	def self.collection_writer_commit!	1✔
18	commit_url = "#{collection_writer_url}/update?commit=true"	×
19	HTTParty.get(commit_url)	×
20	end
21
22	# Returns the Solr collection that we should use for quering data
23	def self.collection_reader_url	1✔
24	collection = current_collection_for_alias(alias_uri)	×
25	build_solr_url_for_collection(alias_uri, collection)	×
26	end
27
28	# Returns the Solr collection that we should write to
29	# (creates it if does not exist)
30	def self.collection_writer_url	1✔
31	collection_writer_for_alias(alias_uri, false)	207✔
32	end
33
34	# Returns the Solr collection that we should write to
35	# (creates it if does not exists, deletes & recreates it if already exists)
36	def self.create_collection_writer	1✔
37	collection_writer_for_alias(alias_uri, true)	×
38	end
39
40	# For a given Solr alias, returns a URL that is suitable for writing new data without affecting
41	# the existing Solr index. It does this by creating a new alternate Solr collection to write data,
42	# instead of overwriting the data on the collection used by the provided solr_alias_uri.
43	#
44	# The code is hardcoded to toggle between collections "xxx-1 and "xxx-2".
45	#
46	# For example, if the provided solr_alias_uri is "http://server/solr/pdc-discovery-staging"
47	# and this alias is configured in Solr to point to collection "http://server/solr/pdc-discovery-staging-1"
48	# the returned writer URL will be "http://server/solr/pdc-discovery-staging-2".
49	#
50	# If the alias was configured to point to collection "http://server/solr/pdc-discovery-staging-2"
51	# then the returned writer URL will be "http://server/solr/pdc-discovery-staging-1".
52	def self.collection_writer_for_alias(solr_alias_uri, recreate)	1✔
53	if config_set.nil?	208✔
54	# We are not running in a Solr cloud environment - nothing to do.
55	return solr_alias_uri.to_s	×
56	end
57
58	alternate_collection = alternate_collection_for_alias(solr_alias_uri)	208✔
59	if collection_exist?(solr_alias_uri, alternate_collection)	×
60	if recreate	×
61	# Re-create it
62	delete_collection!(solr_alias_uri, alternate_collection)	×
63	create_collection(solr_alias_uri, alternate_collection)	×
64	end
65	else
66	# Create it
67	create_collection(solr_alias_uri, alternate_collection)	×
68	end
69
70	build_solr_url_for_collection(solr_alias_uri, alternate_collection)	×
71	end
72
73	# Returns the Solr URL based on the provided `solr_alias_uri`` but updated to use
74	# the indicated `collection` instead of pointing to the alias.
75	def self.build_solr_url_for_collection(solr_alias_uri, collection)	1✔
76	URI::HTTP.build(	×
77	schema: solr_alias_uri.scheme,
78	userinfo: solr_alias_uri.userinfo,
79	host: solr_alias_uri.host,
80	port: solr_alias_uri.port,
81	path: "/solr/#{collection}").to_s
82	end
83
84	def self.current_collection_for_alias(solr_alias_uri)	1✔
85	alias_name = solr_alias_uri.path.split("/").last	210✔
86	alias_list_query = uri_from_solr_alias(	210✔
87	path: "/solr/admin/collections",
88	query: "action=LISTALIASES"
89	)
90	response = HTTParty.get(alias_list_query.to_s)	×
91	# The response will indicate the actual collection the alias points to.
92	collection_name = response.parsed_response.dig("aliases", alias_name) if response.code == 200	×
93	collection_name \|\| alias_name	×
94	end
95
96	def self.alternate_collection_for_alias(solr_alias_uri)	1✔
97	solr_collection = current_collection_for_alias(solr_alias_uri)	209✔
98	if solr_collection.end_with?("-1")	×
99	solr_collection.gsub("-1", "-2")	×
100	elsif solr_collection.end_with?("-2")	×
101	solr_collection.gsub("-2", "-1")	×
102	else
103	solr_collection	×
104	end
105	end
106
107	def self.collection_exist?(solr_alias_uri, collection_name)	1✔
108	collection_list_query = uri_from_solr_alias(	1✔
109	path: "/solr/admin/collections",
110	query: "action=LIST"
111	)
112	response = HTTParty.get(collection_list_query.to_s)	×
113	collections = if response.code == 200	×
114	response.parsed_response.dig("collections") \|\| []	×
115	else
116	[]	×
117	end
118	collections.any?(collection_name)	×
119	end
120
121	def self.create_collection(solr_alias_uri, collection_name)	1✔
122	create_query = uri_from_solr_alias(	×
123	path: "/solr/admin/collections",
124	query: "action=CREATE&name=#{collection_name}&collection.configName=#{config_set}&numShards=1&replicationFactor=2"
125	)
126	response = HTTParty.get(create_query.to_s)	×
127	return unless response.code == 200 && response.parsed_response.key?("success")	×
128	collection_name	×
129	end
130
131	def self.delete_collection!(solr_alias_uri, collection_name)	1✔
132	create_query = uri_from_solr_alias(	×
133	path: "/solr/admin/collections",
134	query: "action=DELETE&name=#{collection_name}"
135	)
136	response = HTTParty.get(create_query.to_s)	×
137	response.code == 200	×
138	end
139
140	# Set the solr_alias to point to the current writer collection
141	def self.update_solr_alias!	1✔
142	writer_collection = collection_writer_url.split("/").last	×
143
144	alias_name = alias_uri.path.split("/").last	×
145	if alias_name == writer_collection	×
146	# Nothing to do
147	# (we are probably using standalone Solr in development)
148	return true	×
149	end
150
151	create_query = uri_from_solr_alias(	×
152	path: "/solr/admin/collections",
153	query: "action=CREATEALIAS&name=#{alias_name}&collections=#{writer_collection}"
154	)
155	response = HTTParty.get(create_query.to_s)	×
156	response.code == 200	×
157	end
158
159	def self.uri_from_solr_alias(path:, query:)	1✔
160	URI::HTTP.build(	211✔
161	schema: solr_alias_uri.scheme,
162	userinfo: solr_alias.userinfo,
163	host: solr_alias_uri.host,
164	port: solr_alias_uri.port,
165	path: path,
166	query: query
167	)
168	end
169	end
170	# rubocop:enable Metrics/ClassLength

pulibrary / pdc_discovery / 803026b9-f736-4b16-b861-c3d4bff47a42

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous