1024599080

Committed 03 Oct 2023 06:47PM UTC coverage: 80.244% (+0.1%) from 80.141%

Build # 1024599080

Build Type

push

gitlab-ci

Committed by

lluiscampos

Commit Message

feat: Implement state script retry-later functionality

Ticket: MEN-6677
Changelog: None

Signed-off-by: Lluis Campos <lluis.campos@northern.tech>

Run Details

33 of 33 new or added lines in 2 files covered. (100.0%)

6499 of 8099 relevant lines covered (80.24%)

10689.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.29

/artifact/v3/scripts/executor.cpp

// Copyright 2023 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

#include <artifact/v3/scripts/executor.hpp>

#include <algorithm>
#include <chrono>
#include <regex>
#include <string>

#include <common/common.hpp>
#include <common/expected.hpp>
#include <common/path.hpp>


namespace mender {
namespace artifact {
namespace scripts {
namespace executor {

namespace expected = mender::common::expected;


using expected::ExpectedBool;

namespace common = mender::common;
namespace error = mender::common::error;
namespace path = mender::common::path;


const string expected_state_script_version {"3"};
const int state_script_retry_exit_code {21};

unordered_map<const State, string> state_map {
        {State::Idle, "Idle"},
        {State::Sync, "Sync"},
        {State::Download, "Download"},
        {State::ArtifactInstall, "ArtifactInstall"},
        {State::ArtifactReboot, "ArtifactReboot"},
        {State::ArtifactCommit, "ArtifactCommit"},
        {State::ArtifactRollback, "ArtifactRollback"},
        {State::ArtifactRollbackReboot, "ArtifactRollbackReboot"},
        {State::ArtifactFailure, "ArtifactFailure"},
};

unordered_map<const Action, string> action_map {
        {Action::Enter, "Enter"},
        {Action::Leave, "Leave"},
        {Action::Error, "Error"},
};

error::Error CorrectVersionFile(const string &path) {
        // Missing file is OK
        // This is because previous versions of the client wrote no
        // version file, so no-file=v3
        if (!path::FileExists(path)) {
                return error::NoError;
        }

        ifstream vf {path};

        if (!vf) {
                auto errnum {errno};
                return error::Error(
                        generic_category().default_error_condition(errnum), "Failed to open the version file");
        }

        string version;
        vf >> version;
        if (!vf) {
                auto errnum {errno};
                return error::Error(
                        generic_category().default_error_condition(errnum),
                        "Error reading the version number from the version file");
        }

        if (version != expected_state_script_version) {
                return executor::MakeError(
                        executor::VersionFileError, "Unexpected Artifact script version found: " + version);
        }
        return error::NoError;
}

bool isValidStateScript(const string &file, State state, Action action) {
        string expression {
                "(" + state_map.at(state) + ")" + "_(" + action_map.at(action) + ")_[0-9][0-9](_\\S+)?"};
        log::Trace(
                "verifying the State script format of the file: " + file
                + " using the regular expression: " + expression);
        const regex artifact_script_regexp {expression, std::regex_constants::ECMAScript};
        return regex_match(path::BaseName(file), artifact_script_regexp);
}

function<bool(const string &)> Matcher(State state, Action action) {
        return [state, action](const string &file) {
                const bool is_valid {isValidStateScript(file, state, action)};
                if (!is_valid) {
                        log::Trace(file + " is not a valid State Script for the state: " + Name(state, action));
                        return false;
                }
                auto exp_executable = path::IsExecutable(file, true);
                if (!exp_executable) {
                        log::Debug("Issue figuring the executable bits of: " + exp_executable.error().String());
                        return false;
                }
                return is_valid and exp_executable.value();
        };
}

bool IsArtifactScript(State state) {
        switch (state) {
        case State::Idle:
        case State::Sync:
        case State::Download:
                return false;
        case State::ArtifactInstall:
        case State::ArtifactReboot:
        case State::ArtifactCommit:
        case State::ArtifactRollback:
        case State::ArtifactRollbackReboot:
        case State::ArtifactFailure:
                return true;
        }
        assert(false);
        return false;
}

string ScriptRunner::ScriptPath(State state) {
        if (IsArtifactScript(state)) {
                return this->artifact_script_path_;
        }
        return this->rootfs_script_path_;
}

string Name(const State state, const Action action) {
        return state_map.at(state) + action_map.at(action);
}

ScriptRunner::ScriptRunner(
        events::EventLoop &loop,
        chrono::milliseconds script_timeout,
        chrono::milliseconds retry_interval,
        chrono::milliseconds retry_timeout,
        const string &artifact_script_path,
        const string &rootfs_script_path,
        processes::OutputCallback stdout_callback,
        processes::OutputCallback stderr_callback) :
        loop_ {loop},
        script_timeout_ {script_timeout},
        retry_interval_ {retry_interval},
        retry_timeout_ {retry_timeout},
        artifact_script_path_ {artifact_script_path},
        rootfs_script_path_ {rootfs_script_path},
        stdout_callback_ {stdout_callback},
        stderr_callback_ {stderr_callback},
        error_script_error_ {error::NoError} {};

void ScriptRunner::LogErrAndExecuteNext(
        Error err,
        vector<string>::iterator current_script,
        vector<string>::iterator end,
        bool ignore_error,
        HandlerFunction handler) {
        // Collect the error and carry on
        if (err.code
                == common::processes::MakeError(common::processes::NonZeroExitStatusError, "").code) {
                this->error_script_error_ = this->error_script_error_.FollowedBy(executor::MakeError(
                        executor::NonZeroExitStatusError,
                        "Got non zero exit code from script: " + *current_script));
        } else {
                this->error_script_error_ = this->error_script_error_.FollowedBy(err);
        }

        // Schedule the next script execution
        auto local_err = Execute(std::next(current_script), end, ignore_error, handler);
        if (local_err != error::NoError) {
                return handler(local_err);
        }
}

void ScriptRunner::HandleScriptError(Error err, HandlerFunction handler) {
        if (err.code
                == common::processes::MakeError(common::processes::NonZeroExitStatusError, "").code) {
                return handler(executor::MakeError(
                        executor::NonZeroExitStatusError,
                        "Received error code: " + to_string(this->script_.get()->GetExitStatus())));
        }
        return handler(err);
}

Error ScriptRunner::Execute(
        vector<string>::iterator current_script,
        vector<string>::iterator end,
        bool ignore_error,
        HandlerFunction handler) {
        // No more scripts to execute
        if (current_script == end) {
                if (this->retry_timeout_timer_) {
                        this->retry_timeout_timer_->Cancel();
                }
                handler(this->error_script_error_); // Success
                return error::NoError;
        }

        log::Info("Running State Script: " + *current_script);

        this->script_.reset(new processes::Process({*current_script}));
        auto err {this->script_->Start(stdout_callback_, stderr_callback_)};
        if (err != error::NoError) {
                return err;
        }

        if (!this->retry_timeout_timer_) {
                // First run on this script
                this->retry_timeout_timer_.reset(new events::Timer(this->loop_));
                this->retry_timeout_timer_->AsyncWait(this->retry_timeout_, [this](error::Error err) {
                        if (err.code == make_error_condition(errc::operation_canceled)) {
                                // The timer did not fire up. Do nothing
                        } else {
                                // If we are waiting between retries, cancel
                                if (this->retry_interval_timer_) {
                                        this->retry_interval_timer_->Cancel();
                                }
                                // Cancel the current script execution
                                this->script_->Cancel();
                        }
                });
        }

        return this->script_.get()->AsyncWait(
                this->loop_,
                [this, current_script, end, ignore_error, handler](Error err) {
                        if (err != error::NoError) {
                                if ((err.code
                                         == common::processes::MakeError(common::processes::NonZeroExitStatusError, "")
                                                        .code)
                                        && this->script_->GetExitStatus() == state_script_retry_exit_code) {
                                        log::Info(
                                                "Script returned Retry Later exit code, re-retrying in "
                                                + to_string(this->retry_interval_.count() / 1000) + "s");
                                        this->retry_interval_timer_.reset(new events::Timer(this->loop_));
                                        this->retry_interval_timer_->AsyncWait(
                                                this->retry_interval_,
                                                [this, current_script, end, ignore_error, handler](error::Error err) {
                                                        if (err != error::NoError) {
                                                                handler(err);
                                                        }

                                                        auto local_err = Execute(current_script, end, ignore_error, handler);
                                                        if (local_err != error::NoError) {
                                                                handler(local_err);
                                                        }
                                                });
                                        return;
                                } else {
                                        if (ignore_error) {
                                                return LogErrAndExecuteNext(
                                                        err, current_script, end, ignore_error, handler);
                                        }
                                        return HandleScriptError(err, handler);
                                }
                        }
                        // Stop retry timer and start the next script execution
                        if (this->retry_timeout_timer_) {
                                this->retry_timeout_timer_->Cancel();
                        }
                        auto local_err = Execute(std::next(current_script), end, ignore_error, handler);
                        if (local_err != error::NoError) {
                                return handler(local_err);
                        }
                },
                this->script_timeout_);
}

Error ScriptRunner::AsyncRunScripts(
        State state, Action action, HandlerFunction handler, RunError on_error) {
        if (IsArtifactScript(state)) {
                // Verify the version in the version file (OK if no version file present)
                auto version_file_error {
                        CorrectVersionFile(path::Join(this->artifact_script_path_, "version"))};
                if (version_file_error != error::NoError) {
                        return version_file_error;
                }
        }

        // Collect
        const auto script_path {ScriptPath(state)};
        auto exp_scripts {path::ListFiles(script_path, Matcher(state, action))};
        if (!exp_scripts) {
                // Missing directory is OK
                if (exp_scripts.error().IsErrno(ENOENT)) {
                        log::Warning("Found no state script directory (" + script_path + "). Continuing on");
                        handler(error::NoError);
                        return error::NoError;
                }
                return executor::MakeError(
                        executor::Code::CollectionError,
                        "Failed to get the scripts, error: " + exp_scripts.error().String());
        }

        // Sort
        {
                auto &unsorted_scripts {exp_scripts.value()};

                vector<string> sorted_scripts(unsorted_scripts.begin(), unsorted_scripts.end());

                sort(sorted_scripts.begin(), sorted_scripts.end());
                this->collected_scripts_ = std::move(sorted_scripts);
        }

        bool ignore_error = on_error == RunError::Ignore || action == Action::Error;

        // Execute
        auto scripts_iterator {this->collected_scripts_.begin()};
        auto scripts_iterator_end {this->collected_scripts_.end()};
        return Execute(scripts_iterator, scripts_iterator_end, ignore_error, handler);
}


Error ScriptRunner::RunScripts(State state, Action action, RunError on_error) {
        auto run_err {error::NoError};
        auto err = AsyncRunScripts(
                state,
                action,
                [this, &run_err](Error error) {
                        run_err = error;
                        this->loop_.Stop();
                },
                on_error);
        if (err != error::NoError) {
                return err;
        }
        this->loop_.Run();
        return run_err;
}

} // namespace executor
} // namespace scripts
} // namespace artifact
} // namespace mender

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	#include <artifact/v3/scripts/executor.hpp>
16
17	#include <algorithm>
18	#include <chrono>
19	#include <regex>
20	#include <string>
21
22	#include <common/common.hpp>
23	#include <common/expected.hpp>
24	#include <common/path.hpp>
25
26
27	namespace mender {
28	namespace artifact {
29	namespace scripts {
30	namespace executor {
31
32	namespace expected = mender::common::expected;
33
34
35	using expected::ExpectedBool;
36
37	namespace common = mender::common;
38	namespace error = mender::common::error;
39	namespace path = mender::common::path;
40
41
42	const string expected_state_script_version {"3"};
43	const int state_script_retry_exit_code {21};
44
45	unordered_map<const State, string> state_map {
46	{State::Idle, "Idle"},
47	{State::Sync, "Sync"},
48	{State::Download, "Download"},
49	{State::ArtifactInstall, "ArtifactInstall"},
50	{State::ArtifactReboot, "ArtifactReboot"},
51	{State::ArtifactCommit, "ArtifactCommit"},
52	{State::ArtifactRollback, "ArtifactRollback"},
53	{State::ArtifactRollbackReboot, "ArtifactRollbackReboot"},
54	{State::ArtifactFailure, "ArtifactFailure"},
55	};
56
57	unordered_map<const Action, string> action_map {
58	{Action::Enter, "Enter"},
59	{Action::Leave, "Leave"},
60	{Action::Error, "Error"},
61	};
62
63	error::Error CorrectVersionFile(const string &path) {	624✔
64	// Missing file is OK
65	// This is because previous versions of the client wrote no
66	// version file, so no-file=v3
67	if (!path::FileExists(path)) {	624✔
68	return error::NoError;	167✔
69	}
70
71	ifstream vf {path};	914✔
72
73	if (!vf) {	457✔
74	auto errnum {errno};	×
75	return error::Error(
76	generic_category().default_error_condition(errnum), "Failed to open the version file");	×
77	}
78
79	string version;
80	vf >> version;	457✔
81	if (!vf) {	457✔
82	auto errnum {errno};	×
83	return error::Error(
84	generic_category().default_error_condition(errnum),	×
85	"Error reading the version number from the version file");	×
86	}
87
88	if (version != expected_state_script_version) {	457✔
89	return executor::MakeError(
90	executor::VersionFileError, "Unexpected Artifact script version found: " + version);	2✔
91	}
92	return error::NoError;	456✔
93	}
94
95	bool isValidStateScript(const string &file, State state, Action action) {	27,263✔
96	string expression {
97	"(" + state_map.at(state) + ")" + "_(" + action_map.at(action) + ")_[0-9][0-9](_\\S+)?"};	54,526✔
98	log::Trace(	27,263✔
99	"verifying the State script format of the file: " + file	27,263✔
100	+ " using the regular expression: " + expression);	54,526✔
101	const regex artifact_script_regexp {expression, std::regex_constants::ECMAScript};	27,263✔
102	return regex_match(path::BaseName(file), artifact_script_regexp);	81,789✔
103	}
104
105	function<bool(const string &)> Matcher(State state, Action action) {	×
106	return [state, action](const string &file) {	53,835✔
107	const bool is_valid {isValidStateScript(file, state, action)};	27,263✔
108	if (!is_valid) {	27,263✔
109	log::Trace(file + " is not a valid State Script for the state: " + Name(state, action));	53,144✔
110	return false;	26,572✔
111	}
112	auto exp_executable = path::IsExecutable(file, true);	691✔
113	if (!exp_executable) {	691✔
114	log::Debug("Issue figuring the executable bits of: " + exp_executable.error().String());	×
115	return false;	×
116	}
117	return is_valid and exp_executable.value();	691✔
118	};	×
119	}
120
121	bool IsArtifactScript(State state) {	×
122	switch (state) {	×
123	case State::Idle:
124	case State::Sync:
125	case State::Download:
126	return false;
127	case State::ArtifactInstall:	×
128	case State::ArtifactReboot:
129	case State::ArtifactCommit:
130	case State::ArtifactRollback:
131	case State::ArtifactRollbackReboot:
132	case State::ArtifactFailure:
133	return true;	×
134	}
135	assert(false);
136	return false;
137	}
138
139	string ScriptRunner::ScriptPath(State state) {	1,280✔
140	if (IsArtifactScript(state)) {
141	return this->artifact_script_path_;	623✔
142	}
143	return this->rootfs_script_path_;	657✔
144	}
145
146	string Name(const State state, const Action action) {	27,598✔
147	return state_map.at(state) + action_map.at(action);	27,598✔
148	}
149
150	ScriptRunner::ScriptRunner(	3,011✔
151	events::EventLoop &loop,
152	chrono::milliseconds script_timeout,
153	chrono::milliseconds retry_interval,
154	chrono::milliseconds retry_timeout,
155	const string &artifact_script_path,
156	const string &rootfs_script_path,
157	processes::OutputCallback stdout_callback,
158	processes::OutputCallback stderr_callback) :	3,011✔
159	loop_ {loop},
160	script_timeout_ {script_timeout},
161	retry_interval_ {retry_interval},
162	retry_timeout_ {retry_timeout},
163	artifact_script_path_ {artifact_script_path},
164	rootfs_script_path_ {rootfs_script_path},
165	stdout_callback_ {stdout_callback},
166	stderr_callback_ {stderr_callback},
167	error_script_error_ {error::NoError} {};	3,011✔
168
169	void ScriptRunner::LogErrAndExecuteNext(	10✔
170	Error err,
171	vector<string>::iterator current_script,
172	vector<string>::iterator end,
173	bool ignore_error,
174	HandlerFunction handler) {
175	// Collect the error and carry on
176	if (err.code	10✔
177	== common::processes::MakeError(common::processes::NonZeroExitStatusError, "").code) {	10✔
178	this->error_script_error_ = this->error_script_error_.FollowedBy(executor::MakeError(	10✔
179	executor::NonZeroExitStatusError,
180	"Got non zero exit code from script: " + *current_script));	20✔
181	} else {
182	this->error_script_error_ = this->error_script_error_.FollowedBy(err);	×
183	}
184
185	// Schedule the next script execution
186	auto local_err = Execute(std::next(current_script), end, ignore_error, handler);	10✔
187	if (local_err != error::NoError) {	10✔
188	return handler(local_err);	×
189	}
190	}
191
192	void ScriptRunner::HandleScriptError(Error err, HandlerFunction handler) {	35✔
193	if (err.code	35✔
194	== common::processes::MakeError(common::processes::NonZeroExitStatusError, "").code) {	35✔
195	return handler(executor::MakeError(	66✔
196	executor::NonZeroExitStatusError,
197	"Received error code: " + to_string(this->script_.get()->GetExitStatus())));	66✔
198	}
199	return handler(err);	4✔
200	}
201
202	Error ScriptRunner::Execute(	1,901✔
203	vector<string>::iterator current_script,
204	vector<string>::iterator end,
205	bool ignore_error,
206	HandlerFunction handler) {
207	// No more scripts to execute
208	if (current_script == end) {	1,901✔
209	if (this->retry_timeout_timer_) {	1,192✔
210	this->retry_timeout_timer_->Cancel();	633✔
211	}
212	handler(this->error_script_error_); // Success	1,192✔
213	return error::NoError;	1,192✔
214	}
215
216	log::Info("Running State Script: " + *current_script);	709✔
217
218	this->script_.reset(new processes::Process({*current_script}));	2,127✔
219	auto err {this->script_->Start(stdout_callback_, stderr_callback_)};	1,418✔
220	if (err != error::NoError) {	709✔
221	return err;	×
222	}
223
224	if (!this->retry_timeout_timer_) {	709✔
225	// First run on this script
226	this->retry_timeout_timer_.reset(new events::Timer(this->loop_));	624✔
227	this->retry_timeout_timer_->AsyncWait(this->retry_timeout_, [this](error::Error err) {	628✔
228	if (err.code == make_error_condition(errc::operation_canceled)) {	598✔
229	// The timer did not fire up. Do nothing
230	} else {
231	// If we are waiting between retries, cancel
232	if (this->retry_interval_timer_) {	2✔
233	this->retry_interval_timer_->Cancel();	2✔
234	}
235	// Cancel the current script execution
236	this->script_->Cancel();	2✔
237	}
238	});	624✔
239	}
240
241	return this->script_.get()->AsyncWait(
242	this->loop_,
243	[this, current_script, end, ignore_error, handler](Error err) {	4,997✔
244	if (err != error::NoError) {	708✔
245	if ((err.code	64✔
246	== common::processes::MakeError(common::processes::NonZeroExitStatusError, "")	128✔
247	.code)
248	&& this->script_->GetExitStatus() == state_script_retry_exit_code) {	64✔
249	log::Info(	19✔
250	"Script returned Retry Later exit code, re-retrying in "
251	+ to_string(this->retry_interval_.count() / 1000) + "s");	38✔
252	this->retry_interval_timer_.reset(new events::Timer(this->loop_));	19✔
253	this->retry_interval_timer_->AsyncWait(	19✔
254	this->retry_interval_,
255	[this, current_script, end, ignore_error, handler](error::Error err) {	38✔
256	if (err != error::NoError) {	19✔
257	handler(err);	2✔
258	}
259
260	auto local_err = Execute(current_script, end, ignore_error, handler);	19✔
261	if (local_err != error::NoError) {	19✔
262	handler(local_err);	×
263	}
264	});	57✔
265	return;	64✔
266	} else {
267	if (ignore_error) {	45✔
268	return LogErrAndExecuteNext(	10✔
269	err, current_script, end, ignore_error, handler);	20✔
270	}
271	return HandleScriptError(err, handler);	70✔
272	}
273	}
274	// Stop retry timer and start the next script execution
275	if (this->retry_timeout_timer_) {	644✔
276	this->retry_timeout_timer_->Cancel();	644✔
277	}
278	auto local_err = Execute(std::next(current_script), end, ignore_error, handler);	644✔
279	if (local_err != error::NoError) {	644✔
280	return handler(local_err);	×
281	}
282	},
283	this->script_timeout_);	1,418✔
284	}
285
286	Error ScriptRunner::AsyncRunScripts(	1,281✔
287	State state, Action action, HandlerFunction handler, RunError on_error) {
288	if (IsArtifactScript(state)) {
289	// Verify the version in the version file (OK if no version file present)
290	auto version_file_error {
291	CorrectVersionFile(path::Join(this->artifact_script_path_, "version"))};	624✔
292	if (version_file_error != error::NoError) {	624✔
293	return version_file_error;	1✔
294	}
295	}
296
297	// Collect
298	const auto script_path {ScriptPath(state)};	1,280✔
299	auto exp_scripts {path::ListFiles(script_path, Matcher(state, action))};	2,560✔
300	if (!exp_scripts) {	1,280✔
301	// Missing directory is OK
302	if (exp_scripts.error().IsErrno(ENOENT)) {	52✔
303	log::Warning("Found no state script directory (" + script_path + "). Continuing on");	104✔
304	handler(error::NoError);	52✔
305	return error::NoError;	52✔
306	}
307	return executor::MakeError(
308	executor::Code::CollectionError,
309	"Failed to get the scripts, error: " + exp_scripts.error().String());	×
310	}
311
312	// Sort
313	{
314	auto &unsorted_scripts {exp_scripts.value()};	1,228✔
315
316	vector<string> sorted_scripts(unsorted_scripts.begin(), unsorted_scripts.end());	2,456✔
317
318	sort(sorted_scripts.begin(), sorted_scripts.end());	1,228✔
319	this->collected_scripts_ = std::move(sorted_scripts);	1,228✔
320	}
321
322	bool ignore_error = on_error == RunError::Ignore \|\| action == Action::Error;	1,228✔
323
324	// Execute
325	auto scripts_iterator {this->collected_scripts_.begin()};
326	auto scripts_iterator_end {this->collected_scripts_.end()};
327	return Execute(scripts_iterator, scripts_iterator_end, ignore_error, handler);	2,456✔
328	}
329
330
331	Error ScriptRunner::RunScripts(State state, Action action, RunError on_error) {	247✔
332	auto run_err {error::NoError};	247✔
333	auto err = AsyncRunScripts(
334	state,
335	action,
336	[this, &run_err](Error error) {	494✔
337	run_err = error;	247✔
338	this->loop_.Stop();	247✔
339	},	247✔
340	on_error);	247✔
341	if (err != error::NoError) {	247✔
342	return err;	×
343	}
344	this->loop_.Run();	247✔
345	return run_err;	247✔
346	}
347
348	} // namespace executor
349	} // namespace scripts
350	} // namespace artifact
351	} // namespace mender

mendersoftware / mender / 1024599080

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous