• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / mender / 1027657693

06 Oct 2023 06:46AM UTC coverage: 80.173% (+0.1%) from 80.075%
1027657693

push

gitlab-ci

lluiscampos
feat: Implement state script retry-later functionality

Ticket: MEN-6677
Changelog: None

Signed-off-by: Lluis Campos <lluis.campos@northern.tech>

45 of 45 new or added lines in 2 files covered. (100.0%)

6478 of 8080 relevant lines covered (80.17%)

10732.92 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.52
/artifact/v3/scripts/executor.cpp
1
// Copyright 2023 Northern.tech AS
2
//
3
//    Licensed under the Apache License, Version 2.0 (the "License");
4
//    you may not use this file except in compliance with the License.
5
//    You may obtain a copy of the License at
6
//
7
//        http://www.apache.org/licenses/LICENSE-2.0
8
//
9
//    Unless required by applicable law or agreed to in writing, software
10
//    distributed under the License is distributed on an "AS IS" BASIS,
11
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
//    See the License for the specific language governing permissions and
13
//    limitations under the License.
14

15
#include <artifact/v3/scripts/executor.hpp>
16

17
#include <algorithm>
18
#include <chrono>
19
#include <regex>
20
#include <string>
21

22
#include <common/common.hpp>
23
#include <common/expected.hpp>
24
#include <common/path.hpp>
25

26

27
namespace mender {
28
namespace artifact {
29
namespace scripts {
30
namespace executor {
31

32
namespace expected = mender::common::expected;
33

34

35
using expected::ExpectedBool;
36

37
namespace processes = mender::common::processes;
38
namespace error = mender::common::error;
39
namespace path = mender::common::path;
40

41

42
const string expected_state_script_version {"3"};
43
const int state_script_retry_exit_code {21};
44

45
unordered_map<const State, string> state_map {
46
        {State::Idle, "Idle"},
47
        {State::Sync, "Sync"},
48
        {State::Download, "Download"},
49
        {State::ArtifactInstall, "ArtifactInstall"},
50
        {State::ArtifactReboot, "ArtifactReboot"},
51
        {State::ArtifactCommit, "ArtifactCommit"},
52
        {State::ArtifactRollback, "ArtifactRollback"},
53
        {State::ArtifactRollbackReboot, "ArtifactRollbackReboot"},
54
        {State::ArtifactFailure, "ArtifactFailure"},
55
};
56

57
unordered_map<const Action, string> action_map {
58
        {Action::Enter, "Enter"},
59
        {Action::Leave, "Leave"},
60
        {Action::Error, "Error"},
61
};
62

63
error::Error CorrectVersionFile(const string &path) {
638✔
64
        // Missing file is OK
65
        // This is because previous versions of the client wrote no
66
        // version file, so no-file=v3
67
        if (!path::FileExists(path)) {
638✔
68
                return error::NoError;
171✔
69
        }
70

71
        ifstream vf {path};
934✔
72

73
        if (!vf) {
467✔
74
                auto errnum {errno};
×
75
                return error::Error(
76
                        generic_category().default_error_condition(errnum), "Failed to open the version file");
×
77
        }
78

79
        string version;
80
        vf >> version;
467✔
81
        if (!vf) {
467✔
82
                auto errnum {errno};
×
83
                return error::Error(
84
                        generic_category().default_error_condition(errnum),
×
85
                        "Error reading the version number from the version file");
×
86
        }
87

88
        if (version != expected_state_script_version) {
467✔
89
                return executor::MakeError(
90
                        executor::VersionFileError, "Unexpected Artifact script version found: " + version);
2✔
91
        }
92
        return error::NoError;
466✔
93
}
94

95
bool isValidStateScript(const string &file, State state, Action action) {
27,521✔
96
        string expression {
97
                "(" + state_map.at(state) + ")" + "_(" + action_map.at(action) + ")_[0-9][0-9](_\\S+)?"};
55,042✔
98
        log::Trace(
27,521✔
99
                "verifying the State script format of the file: " + file
27,521✔
100
                + " using the regular expression: " + expression);
55,042✔
101
        const regex artifact_script_regexp {expression, std::regex_constants::ECMAScript};
27,521✔
102
        return regex_match(path::BaseName(file), artifact_script_regexp);
82,563✔
103
}
104

105
function<bool(const string &)> Matcher(State state, Action action) {
×
106
        return [state, action](const string &file) {
54,333✔
107
                const bool is_valid {isValidStateScript(file, state, action)};
27,521✔
108
                if (!is_valid) {
27,521✔
109
                        log::Trace(file + " is not a valid State Script for the state: " + Name(state, action));
53,624✔
110
                        return false;
26,812✔
111
                }
112
                auto exp_executable = path::IsExecutable(file, true);
709✔
113
                if (!exp_executable) {
709✔
114
                        log::Debug("Issue figuring the executable bits of: " + exp_executable.error().String());
×
115
                        return false;
×
116
                }
117
                return is_valid and exp_executable.value();
709✔
118
        };
×
119
}
120

121
bool IsArtifactScript(State state) {
×
122
        switch (state) {
×
123
        case State::Idle:
124
        case State::Sync:
125
        case State::Download:
126
                return false;
127
        case State::ArtifactInstall:
×
128
        case State::ArtifactReboot:
129
        case State::ArtifactCommit:
130
        case State::ArtifactRollback:
131
        case State::ArtifactRollbackReboot:
132
        case State::ArtifactFailure:
133
                return true;
×
134
        }
135
        assert(false);
136
        return false;
137
}
138

139
string ScriptRunner::ScriptPath(State state) {
1,294✔
140
        if (IsArtifactScript(state)) {
141
                return this->artifact_script_path_;
637✔
142
        }
143
        return this->rootfs_script_path_;
657✔
144
}
145

146
string Name(const State state, const Action action) {
27,848✔
147
        return state_map.at(state) + action_map.at(action);
27,848✔
148
}
149

150
ScriptRunner::ScriptRunner(
3,111✔
151
        events::EventLoop &loop,
152
        chrono::milliseconds script_timeout,
153
        chrono::milliseconds retry_interval,
154
        chrono::milliseconds retry_timeout,
155
        const string &artifact_script_path,
156
        const string &rootfs_script_path,
157
        processes::OutputCallback stdout_callback,
158
        processes::OutputCallback stderr_callback) :
3,111✔
159
        loop_ {loop},
160
        script_timeout_ {script_timeout},
161
        retry_interval_ {retry_interval},
162
        retry_timeout_ {retry_timeout},
163
        artifact_script_path_ {artifact_script_path},
164
        rootfs_script_path_ {rootfs_script_path},
165
        stdout_callback_ {stdout_callback},
166
        stderr_callback_ {stderr_callback},
167
        error_script_error_ {error::NoError} {};
3,111✔
168

169
void ScriptRunner::LogErrAndExecuteNext(
12✔
170
        Error err,
171
        vector<string>::iterator current_script,
172
        vector<string>::iterator end,
173
        bool ignore_error,
174
        HandlerFunction handler) {
175
        // Collect the error and carry on
176
        if (err.code == processes::MakeError(processes::NonZeroExitStatusError, "").code) {
12✔
177
                this->error_script_error_ = this->error_script_error_.FollowedBy(executor::MakeError(
12✔
178
                        executor::NonZeroExitStatusError,
179
                        "Got non zero exit code from script: " + *current_script));
24✔
180
        } else {
181
                this->error_script_error_ = this->error_script_error_.FollowedBy(err);
×
182
        }
183

184
        HandleScriptNext(current_script, end, ignore_error, handler);
12✔
185
}
12✔
186

187
void ScriptRunner::HandleScriptNext(
668✔
188
        vector<string>::iterator current_script,
189
        vector<string>::iterator end,
190
        bool ignore_error,
191
        HandlerFunction handler) {
192
        // Stop retry timer and start the next script execution
193
        if (this->retry_timeout_timer_) {
668✔
194
                this->retry_timeout_timer_->Cancel();
2✔
195
                this->retry_timeout_timer_.reset();
196
        }
197

198
        auto local_err = Execute(std::next(current_script), end, ignore_error, handler);
668✔
199
        if (local_err != error::NoError) {
668✔
200
                return handler(local_err);
×
201
        }
202
}
203

204
void ScriptRunner::HandleScriptError(Error err, HandlerFunction handler) {
1,236✔
205
        // Stop retry timer
206
        if (this->retry_timeout_timer_) {
1,236✔
207
                this->retry_timeout_timer_->Cancel();
1✔
208
                this->retry_timeout_timer_.reset();
209
        }
210
        if (err.code == processes::MakeError(processes::NonZeroExitStatusError, "").code) {
1,236✔
211
                return handler(executor::MakeError(
66✔
212
                        executor::NonZeroExitStatusError,
213
                        "Received error code: " + to_string(this->script_.get()->GetExitStatus())));
66✔
214
        }
215
        return handler(err);
2,406✔
216
}
217

218
void ScriptRunner::HandleScriptRetry(
30✔
219
        vector<string>::iterator current_script,
220
        vector<string>::iterator end,
221
        bool ignore_error,
222
        HandlerFunction handler) {
223
        log::Info(
30✔
224
                "Script returned Retry Later exit code, re-retrying in "
225
                + to_string(this->retry_interval_.count() / 1000) + "s");
60✔
226

227
        this->retry_interval_timer_.reset(new events::Timer(this->loop_));
30✔
228
        this->retry_interval_timer_->AsyncWait(
30✔
229
                this->retry_interval_,
230
                [this, current_script, end, ignore_error, handler](error::Error err) {
60✔
231
                        if (err != error::NoError) {
30✔
232
                                return handler(this->error_script_error_.FollowedBy(err));
12✔
233
                        }
234

235
                        auto local_err = Execute(current_script, end, ignore_error, handler);
24✔
236
                        if (local_err != error::NoError) {
24✔
237
                                handler(local_err);
×
238
                        }
239
                });
60✔
240
}
30✔
241

242
void ScriptRunner::MaybeSetupRetryTimeoutTimer() {
30✔
243
        if (!this->retry_timeout_timer_) {
30✔
244
                log::Debug("Setting retry timer for " + to_string(this->retry_timeout_.count()) + "ms");
18✔
245
                // First run on this script
246
                this->retry_timeout_timer_.reset(new events::Timer(this->loop_));
9✔
247
                this->retry_timeout_timer_->AsyncWait(this->retry_timeout_, [this](error::Error err) {
15✔
248
                        if (err.code == make_error_condition(errc::operation_canceled)) {
6✔
249
                                // The timer did not fire up. Do nothing
250
                        } else {
251
                                log::Error("Script Retry Later timeout out, cancelling and returning");
12✔
252
                                this->retry_interval_timer_->Cancel();
6✔
253
                                this->script_->Cancel();
6✔
254
                        }
255
                });
15✔
256
        }
257
}
30✔
258

259
Error ScriptRunner::Execute(
1,934✔
260
        vector<string>::iterator current_script,
261
        vector<string>::iterator end,
262
        bool ignore_error,
263
        HandlerFunction handler) {
264
        // No more scripts to execute
265
        if (current_script == end) {
1,934✔
266
                HandleScriptError(this->error_script_error_, handler); // Success
2,404✔
267
                return error::NoError;
1,202✔
268
        }
269

270
        log::Info("Running State Script: " + *current_script);
732✔
271

272
        this->script_.reset(new processes::Process({*current_script}));
2,196✔
273
        auto err {this->script_->Start(stdout_callback_, stderr_callback_)};
1,464✔
274
        if (err != error::NoError) {
732✔
275
                return err;
×
276
        }
277

278
        return this->script_.get()->AsyncWait(
279
                this->loop_,
280
                [this, current_script, end, ignore_error, handler](Error err) {
4,466✔
281
                        if (err != error::NoError) {
732✔
282
                                const bool is_script_retry_error =
283
                                        err.code == processes::MakeError(processes::NonZeroExitStatusError, "").code
152✔
284
                                        && this->script_->GetExitStatus() == state_script_retry_exit_code;
76✔
285
                                if (is_script_retry_error) {
76✔
286
                                        MaybeSetupRetryTimeoutTimer();
30✔
287
                                        return HandleScriptRetry(current_script, end, ignore_error, handler);
60✔
288
                                } else {
289
                                        if (ignore_error) {
46✔
290
                                                return LogErrAndExecuteNext(
12✔
291
                                                        err, current_script, end, ignore_error, handler);
24✔
292
                                        }
293
                                        return HandleScriptError(err, handler);
68✔
294
                                }
295
                        }
296
                        return HandleScriptNext(current_script, end, ignore_error, handler);
1,312✔
297
                },
298
                this->script_timeout_);
1,464✔
299
}
300

301
Error ScriptRunner::AsyncRunScripts(
1,295✔
302
        State state, Action action, HandlerFunction handler, RunError on_error) {
303
        if (IsArtifactScript(state)) {
304
                // Verify the version in the version file (OK if no version file present)
305
                auto version_file_error {
306
                        CorrectVersionFile(path::Join(this->artifact_script_path_, "version"))};
638✔
307
                if (version_file_error != error::NoError) {
638✔
308
                        return version_file_error;
1✔
309
                }
310
        }
311

312
        // Collect
313
        const auto script_path {ScriptPath(state)};
1,294✔
314
        auto exp_scripts {path::ListFiles(script_path, Matcher(state, action))};
2,588✔
315
        if (!exp_scripts) {
1,294✔
316
                // Missing directory is OK
317
                if (exp_scripts.error().IsErrno(ENOENT)) {
52✔
318
                        log::Debug("Found no state script directory (" + script_path + "). Continuing on");
104✔
319
                        handler(error::NoError);
52✔
320
                        return error::NoError;
52✔
321
                }
322
                return executor::MakeError(
323
                        executor::Code::CollectionError,
324
                        "Failed to get the scripts, error: " + exp_scripts.error().String());
×
325
        }
326

327
        // Sort
328
        {
329
                auto &unsorted_scripts {exp_scripts.value()};
1,242✔
330

331
                vector<string> sorted_scripts(unsorted_scripts.begin(), unsorted_scripts.end());
2,484✔
332

333
                sort(sorted_scripts.begin(), sorted_scripts.end());
1,242✔
334
                this->collected_scripts_ = std::move(sorted_scripts);
1,242✔
335
        }
336

337
        bool ignore_error = on_error == RunError::Ignore || action == Action::Error;
1,242✔
338

339
        // Execute
340
        auto scripts_iterator {this->collected_scripts_.begin()};
341
        auto scripts_iterator_end {this->collected_scripts_.end()};
342
        return Execute(scripts_iterator, scripts_iterator_end, ignore_error, handler);
2,484✔
343
}
344

345

346
Error ScriptRunner::RunScripts(State state, Action action, RunError on_error) {
251✔
347
        auto run_err {error::NoError};
251✔
348
        auto err = AsyncRunScripts(
349
                state,
350
                action,
351
                [this, &run_err](Error error) {
502✔
352
                        run_err = error;
251✔
353
                        this->loop_.Stop();
251✔
354
                },
251✔
355
                on_error);
251✔
356
        if (err != error::NoError) {
251✔
357
                return err;
×
358
        }
359
        this->loop_.Run();
251✔
360
        return run_err;
251✔
361
}
362

363
} // namespace executor
364
} // namespace scripts
365
} // namespace artifact
366
} // namespace mender
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc