• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / mender / 1025630817

04 Oct 2023 02:21PM UTC coverage: 80.272% (+0.2%) from 80.085%
1025630817

push

gitlab-ci

lluiscampos
feat: Implement state script retry-later functionality

Ticket: MEN-6677
Changelog: None

Signed-off-by: Lluis Campos <lluis.campos@northern.tech>

44 of 44 new or added lines in 2 files covered. (100.0%)

6482 of 8075 relevant lines covered (80.27%)

10723.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.43
/artifact/v3/scripts/executor.cpp
1
// Copyright 2023 Northern.tech AS
2
//
3
//    Licensed under the Apache License, Version 2.0 (the "License");
4
//    you may not use this file except in compliance with the License.
5
//    You may obtain a copy of the License at
6
//
7
//        http://www.apache.org/licenses/LICENSE-2.0
8
//
9
//    Unless required by applicable law or agreed to in writing, software
10
//    distributed under the License is distributed on an "AS IS" BASIS,
11
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
//    See the License for the specific language governing permissions and
13
//    limitations under the License.
14

15
#include <artifact/v3/scripts/executor.hpp>
16

17
#include <algorithm>
18
#include <chrono>
19
#include <regex>
20
#include <string>
21

22
#include <common/common.hpp>
23
#include <common/expected.hpp>
24
#include <common/path.hpp>
25

26

27
namespace mender {
28
namespace artifact {
29
namespace scripts {
30
namespace executor {
31

32
namespace expected = mender::common::expected;
33

34

35
using expected::ExpectedBool;
36

37
namespace processes = mender::common::processes;
38
namespace error = mender::common::error;
39
namespace path = mender::common::path;
40

41

42
const string expected_state_script_version {"3"};
43
const int state_script_retry_exit_code {21};
44

45
unordered_map<const State, string> state_map {
46
        {State::Idle, "Idle"},
47
        {State::Sync, "Sync"},
48
        {State::Download, "Download"},
49
        {State::ArtifactInstall, "ArtifactInstall"},
50
        {State::ArtifactReboot, "ArtifactReboot"},
51
        {State::ArtifactCommit, "ArtifactCommit"},
52
        {State::ArtifactRollback, "ArtifactRollback"},
53
        {State::ArtifactRollbackReboot, "ArtifactRollbackReboot"},
54
        {State::ArtifactFailure, "ArtifactFailure"},
55
};
56

57
unordered_map<const Action, string> action_map {
58
        {Action::Enter, "Enter"},
59
        {Action::Leave, "Leave"},
60
        {Action::Error, "Error"},
61
};
62

63
error::Error CorrectVersionFile(const string &path) {
628✔
64
        // Missing file is OK
65
        // This is because previous versions of the client wrote no
66
        // version file, so no-file=v3
67
        if (!path::FileExists(path)) {
628✔
68
                return error::NoError;
171✔
69
        }
70

71
        ifstream vf {path};
914✔
72

73
        if (!vf) {
457✔
74
                auto errnum {errno};
×
75
                return error::Error(
76
                        generic_category().default_error_condition(errnum), "Failed to open the version file");
×
77
        }
78

79
        string version;
80
        vf >> version;
457✔
81
        if (!vf) {
457✔
82
                auto errnum {errno};
×
83
                return error::Error(
84
                        generic_category().default_error_condition(errnum),
×
85
                        "Error reading the version number from the version file");
×
86
        }
87

88
        if (version != expected_state_script_version) {
457✔
89
                return executor::MakeError(
90
                        executor::VersionFileError, "Unexpected Artifact script version found: " + version);
2✔
91
        }
92
        return error::NoError;
456✔
93
}
94

95
bool isValidStateScript(const string &file, State state, Action action) {
27,271✔
96
        string expression {
97
                "(" + state_map.at(state) + ")" + "_(" + action_map.at(action) + ")_[0-9][0-9](_\\S+)?"};
54,542✔
98
        log::Trace(
27,271✔
99
                "verifying the State script format of the file: " + file
27,271✔
100
                + " using the regular expression: " + expression);
54,542✔
101
        const regex artifact_script_regexp {expression, std::regex_constants::ECMAScript};
27,271✔
102
        return regex_match(path::BaseName(file), artifact_script_regexp);
81,813✔
103
}
104

105
function<bool(const string &)> Matcher(State state, Action action) {
×
106
        return [state, action](const string &file) {
53,843✔
107
                const bool is_valid {isValidStateScript(file, state, action)};
27,271✔
108
                if (!is_valid) {
27,271✔
109
                        log::Trace(file + " is not a valid State Script for the state: " + Name(state, action));
53,144✔
110
                        return false;
26,572✔
111
                }
112
                auto exp_executable = path::IsExecutable(file, true);
699✔
113
                if (!exp_executable) {
699✔
114
                        log::Debug("Issue figuring the executable bits of: " + exp_executable.error().String());
×
115
                        return false;
×
116
                }
117
                return is_valid and exp_executable.value();
699✔
118
        };
×
119
}
120

121
bool IsArtifactScript(State state) {
×
122
        switch (state) {
×
123
        case State::Idle:
124
        case State::Sync:
125
        case State::Download:
126
                return false;
127
        case State::ArtifactInstall:
×
128
        case State::ArtifactReboot:
129
        case State::ArtifactCommit:
130
        case State::ArtifactRollback:
131
        case State::ArtifactRollbackReboot:
132
        case State::ArtifactFailure:
133
                return true;
×
134
        }
135
        assert(false);
136
        return false;
137
}
138

139
string ScriptRunner::ScriptPath(State state) {
1,284✔
140
        if (IsArtifactScript(state)) {
141
                return this->artifact_script_path_;
627✔
142
        }
143
        return this->rootfs_script_path_;
657✔
144
}
145

146
string Name(const State state, const Action action) {
27,598✔
147
        return state_map.at(state) + action_map.at(action);
27,598✔
148
}
149

150
ScriptRunner::ScriptRunner(
3,015✔
151
        events::EventLoop &loop,
152
        chrono::milliseconds script_timeout,
153
        chrono::milliseconds retry_interval,
154
        chrono::milliseconds retry_timeout,
155
        const string &artifact_script_path,
156
        const string &rootfs_script_path,
157
        processes::OutputCallback stdout_callback,
158
        processes::OutputCallback stderr_callback) :
3,015✔
159
        loop_ {loop},
160
        script_timeout_ {script_timeout},
161
        retry_interval_ {retry_interval},
162
        retry_timeout_ {retry_timeout},
163
        artifact_script_path_ {artifact_script_path},
164
        rootfs_script_path_ {rootfs_script_path},
165
        stdout_callback_ {stdout_callback},
166
        stderr_callback_ {stderr_callback},
167
        error_script_error_ {error::NoError} {};
3,015✔
168

169
void ScriptRunner::LogErrAndExecuteNext(
12✔
170
        Error err,
171
        vector<string>::iterator current_script,
172
        vector<string>::iterator end,
173
        bool ignore_error,
174
        HandlerFunction handler) {
175
        // Collect the error and carry on
176
        if (err.code == processes::MakeError(processes::NonZeroExitStatusError, "").code) {
12✔
177
                this->error_script_error_ = this->error_script_error_.FollowedBy(executor::MakeError(
12✔
178
                        executor::NonZeroExitStatusError,
179
                        "Got non zero exit code from script: " + *current_script));
24✔
180
        } else {
181
                this->error_script_error_ = this->error_script_error_.FollowedBy(err);
×
182
        }
183

184
        HandleScriptNext(current_script, end, ignore_error, handler);
12✔
185
}
12✔
186

187
void ScriptRunner::HandleScriptNext(
658✔
188
        vector<string>::iterator current_script,
189
        vector<string>::iterator end,
190
        bool ignore_error,
191
        HandlerFunction handler) {
192
        // Stop retry timer and start the next script execution
193
        if (this->retry_timeout_timer_) {
658✔
194
                this->retry_timeout_timer_->Cancel();
2✔
195
                this->retry_timeout_timer_.reset();
196
        }
197

198
        auto local_err = Execute(std::next(current_script), end, ignore_error, handler);
658✔
199
        if (local_err != error::NoError) {
658✔
200
                return handler(local_err);
×
201
        }
202
}
203

204
void ScriptRunner::HandleScriptError(Error err, HandlerFunction handler) {
1,226✔
205
        // Stop retry timer
206
        if (this->retry_timeout_timer_) {
1,226✔
207
                this->retry_timeout_timer_->Cancel();
1✔
208
                this->retry_timeout_timer_.reset();
209
        }
210
        if (err.code == processes::MakeError(processes::NonZeroExitStatusError, "").code) {
1,226✔
211
                return handler(executor::MakeError(
66✔
212
                        executor::NonZeroExitStatusError,
213
                        "Received error code: " + to_string(this->script_.get()->GetExitStatus())));
66✔
214
        }
215
        return handler(err);
2,386✔
216
}
217

218
void ScriptRunner::HandleScriptRetry(
30✔
219
        vector<string>::iterator current_script,
220
        vector<string>::iterator end,
221
        bool ignore_error,
222
        HandlerFunction handler) {
223
        log::Info(
30✔
224
                "Script returned Retry Later exit code, re-retrying in "
225
                + to_string(this->retry_interval_.count() / 1000) + "s");
60✔
226

227
        this->retry_interval_timer_.reset(new events::Timer(this->loop_));
30✔
228
        this->retry_interval_timer_->AsyncWait(
30✔
229
                this->retry_interval_,
230
                [this, current_script, end, ignore_error, handler](error::Error err) {
60✔
231
                        if (err != error::NoError) {
30✔
232
                                return handler(this->error_script_error_.FollowedBy(err));
12✔
233
                        }
234

235
                        auto local_err = Execute(current_script, end, ignore_error, handler);
24✔
236
                        if (local_err != error::NoError) {
24✔
237
                                handler(local_err);
×
238
                        }
239
                });
60✔
240
}
30✔
241

242
void ScriptRunner::MaybeSetupRetryTimeoutTimer() {
30✔
243
        if (!this->retry_timeout_timer_) {
30✔
244
                log::Debug("Setting retry timer for " + to_string(this->retry_timeout_.count()) + "ms");
18✔
245
                // First run on this script
246
                this->retry_timeout_timer_.reset(new events::Timer(this->loop_));
9✔
247
                this->retry_timeout_timer_->AsyncWait(this->retry_timeout_, [this](error::Error err) {
15✔
248
                        if (err.code == make_error_condition(errc::operation_canceled)) {
6✔
249
                                // The timer did not fire up. Do nothing
250
                        } else {
251
                                this->retry_interval_timer_->Cancel();
6✔
252
                                this->script_->Cancel();
6✔
253
                        }
254
                });
9✔
255
        }
256
}
30✔
257

258
Error ScriptRunner::Execute(
1,914✔
259
        vector<string>::iterator current_script,
260
        vector<string>::iterator end,
261
        bool ignore_error,
262
        HandlerFunction handler) {
263
        // No more scripts to execute
264
        if (current_script == end) {
1,914✔
265
                HandleScriptError(this->error_script_error_, handler); // Success
2,384✔
266
                return error::NoError;
1,192✔
267
        }
268

269
        log::Info("Running State Script: " + *current_script);
722✔
270

271
        this->script_.reset(new processes::Process({*current_script}));
2,166✔
272
        auto err {this->script_->Start(stdout_callback_, stderr_callback_)};
1,444✔
273
        if (err != error::NoError) {
722✔
274
                return err;
×
275
        }
276

277
        return this->script_.get()->AsyncWait(
278
                this->loop_,
279
                [this, current_script, end, ignore_error, handler](Error err) {
4,406✔
280
                        if (err != error::NoError) {
722✔
281
                                const bool is_script_retry_error =
282
                                        err.code == processes::MakeError(processes::NonZeroExitStatusError, "").code
152✔
283
                                        && this->script_->GetExitStatus() == state_script_retry_exit_code;
76✔
284
                                if (is_script_retry_error) {
76✔
285
                                        MaybeSetupRetryTimeoutTimer();
30✔
286
                                        return HandleScriptRetry(current_script, end, ignore_error, handler);
60✔
287
                                } else {
288
                                        if (ignore_error) {
46✔
289
                                                return LogErrAndExecuteNext(
12✔
290
                                                        err, current_script, end, ignore_error, handler);
24✔
291
                                        }
292
                                        return HandleScriptError(err, handler);
68✔
293
                                }
294
                        }
295
                        return HandleScriptNext(current_script, end, ignore_error, handler);
1,292✔
296
                },
297
                this->script_timeout_);
1,444✔
298
}
299

300
Error ScriptRunner::AsyncRunScripts(
1,285✔
301
        State state, Action action, HandlerFunction handler, RunError on_error) {
302
        if (IsArtifactScript(state)) {
303
                // Verify the version in the version file (OK if no version file present)
304
                auto version_file_error {
305
                        CorrectVersionFile(path::Join(this->artifact_script_path_, "version"))};
628✔
306
                if (version_file_error != error::NoError) {
628✔
307
                        return version_file_error;
1✔
308
                }
309
        }
310

311
        // Collect
312
        const auto script_path {ScriptPath(state)};
1,284✔
313
        auto exp_scripts {path::ListFiles(script_path, Matcher(state, action))};
2,568✔
314
        if (!exp_scripts) {
1,284✔
315
                // Missing directory is OK
316
                if (exp_scripts.error().IsErrno(ENOENT)) {
52✔
317
                        log::Debug("Found no state script directory (" + script_path + "). Continuing on");
104✔
318
                        handler(error::NoError);
52✔
319
                        return error::NoError;
52✔
320
                }
321
                return executor::MakeError(
322
                        executor::Code::CollectionError,
323
                        "Failed to get the scripts, error: " + exp_scripts.error().String());
×
324
        }
325

326
        // Sort
327
        {
328
                auto &unsorted_scripts {exp_scripts.value()};
1,232✔
329

330
                vector<string> sorted_scripts(unsorted_scripts.begin(), unsorted_scripts.end());
2,464✔
331

332
                sort(sorted_scripts.begin(), sorted_scripts.end());
1,232✔
333
                this->collected_scripts_ = std::move(sorted_scripts);
1,232✔
334
        }
335

336
        bool ignore_error = on_error == RunError::Ignore || action == Action::Error;
1,232✔
337

338
        // Execute
339
        auto scripts_iterator {this->collected_scripts_.begin()};
340
        auto scripts_iterator_end {this->collected_scripts_.end()};
341
        return Execute(scripts_iterator, scripts_iterator_end, ignore_error, handler);
2,464✔
342
}
343

344

345
Error ScriptRunner::RunScripts(State state, Action action, RunError on_error) {
251✔
346
        auto run_err {error::NoError};
251✔
347
        auto err = AsyncRunScripts(
348
                state,
349
                action,
350
                [this, &run_err](Error error) {
502✔
351
                        run_err = error;
251✔
352
                        this->loop_.Stop();
251✔
353
                },
251✔
354
                on_error);
251✔
355
        if (err != error::NoError) {
251✔
356
                return err;
×
357
        }
358
        this->loop_.Run();
251✔
359
        return run_err;
251✔
360
}
361

362
} // namespace executor
363
} // namespace scripts
364
} // namespace artifact
365
} // namespace mender
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc