• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / mender / 1014406969

23 Sep 2023 12:41PM UTC coverage: 77.735% (+0.2%) from 77.579%
1014406969

push

gitlab-ci

lluiscampos
feat: Implement `mender::update::http_resumer`

Implement class to download the Artifact, which will react to server
disconnections or other sorts of short read by scheduling new HTTP
requests with `Range` header.

See https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests for
an introduction to the feature, and read the specification for more
details.

The user calls _once_ `AsyncCall` with the header and body handlers, and
`DownloadResumerClient` will call back these handlers _once_ (each). At
the user's header handler, the Reader returned by `MakeBodyAsyncReader`
is a wrapper of the actual Body reader that is taking care of the resume
of the download, calling the `AsyncRead` finalized handler only after
the download is fully completed (with potential resumes) and or/the
resuming times out.

The validation of the `Content-Range` header and the cases for the unit
tests are heavily inspired by the legacy client. See:
* https://github.com/mendersoftware/mender/blob/<a class=hub.com/mendersoftware/mender/commit/d9010526d35d3ac861ea1e4210d36c2fef748ef8">d9010526d/client/update_resumer.go#L113
* https://github.com/mendersoftware/mender/blob/d9010526d35d3ac861ea1e4210d36c2fef748ef8/client/update_resumer_test.go#L197

Ticket: MEN-6498
Changelog: None

Signed-off-by: Lluis Campos <lluis.campos@northern.tech>

267 of 267 new or added lines in 5 files covered. (100.0%)

6686 of 8601 relevant lines covered (77.74%)

10755.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.89
/common/http/http.cpp
1
// Copyright 2023 Northern.tech AS
2
//
3
//    Licensed under the Apache License, Version 2.0 (the "License");
4
//    you may not use this file except in compliance with the License.
5
//    You may obtain a copy of the License at
6
//
7
//        http://www.apache.org/licenses/LICENSE-2.0
8
//
9
//    Unless required by applicable law or agreed to in writing, software
10
//    distributed under the License is distributed on an "AS IS" BASIS,
11
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
//    See the License for the specific language governing permissions and
13
//    limitations under the License.
14

15
#include <common/http.hpp>
16

17
#include <algorithm>
18
#include <cctype>
19
#include <cstdlib>
20
#include <iomanip>
21
#include <string>
22

23
#include <common/common.hpp>
24

25
namespace mender {
26
namespace http {
27

28
namespace common = mender::common;
29

30
const HttpErrorCategoryClass HttpErrorCategory;
31

32
const char *HttpErrorCategoryClass::name() const noexcept {
×
33
        return "HttpErrorCategory";
×
34
}
35

36
string HttpErrorCategoryClass::message(int code) const {
19✔
37
        switch (code) {
19✔
38
        case NoError:
×
39
                return "Success";
×
40
        case NoSuchHeaderError:
12✔
41
                return "No such header";
12✔
42
        case InvalidUrlError:
×
43
                return "Malformed URL";
×
44
        case BodyMissingError:
×
45
                return "Body is missing";
×
46
        case BodyIgnoredError:
×
47
                return "HTTP stream contains a body, but a reader has not been created for it";
×
48
        case UnsupportedMethodError:
×
49
                return "Unsupported HTTP method";
×
50
        case StreamCancelledError:
×
51
                return "Stream has been cancelled/destroyed";
×
52
        case UnsupportedBodyType:
×
53
                return "HTTP stream has a body type we don't understand";
×
54
        case MaxRetryError:
2✔
55
                return "Tried maximum number of times";
2✔
56
        case DownloadResumerError:
5✔
57
                return "Resume download error";
5✔
58
        }
59
        // Don't use "default" case. This should generate a warning if we ever add any enums. But
60
        // still assert here for safety.
61
        assert(false);
×
62
        return "Unknown";
63
}
64

65
error::Error MakeError(ErrorCode code, const string &msg) {
329✔
66
        return error::Error(error_condition(code, HttpErrorCategory), msg);
658✔
67
}
68

69
string MethodToString(Method method) {
163✔
70
        switch (method) {
163✔
71
        case Method::Invalid:
×
72
                return "Invalid";
×
73
        case Method::GET:
147✔
74
                return "GET";
147✔
75
        case Method::HEAD:
×
76
                return "HEAD";
×
77
        case Method::POST:
×
78
                return "POST";
×
79
        case Method::PUT:
16✔
80
                return "PUT";
16✔
81
        case Method::PATCH:
×
82
                return "PATCH";
×
83
        case Method::CONNECT:
×
84
                return "CONNECT";
×
85
        }
86
        // Don't use "default" case. This should generate a warning if we ever add any methods. But
87
        // still assert here for safety.
88
        assert(false);
×
89
        return "INVALID_METHOD";
90
}
91

92
error::Error BreakDownUrl(const string &url, BrokenDownUrl &address) {
369✔
93
        const string url_split {"://"};
738✔
94

95
        auto split_index = url.find(url_split);
369✔
96
        if (split_index == string::npos) {
369✔
97
                return MakeError(InvalidUrlError, url + " is not a valid URL.");
2✔
98
        }
99
        if (split_index == 0) {
368✔
100
                return MakeError(InvalidUrlError, url + ": missing hostname");
×
101
        }
102

103
        address.protocol = url.substr(0, split_index);
368✔
104

105
        auto tmp = url.substr(split_index + url_split.size());
736✔
106
        split_index = tmp.find("/");
368✔
107
        if (split_index == string::npos) {
368✔
108
                address.host = tmp;
228✔
109
                address.path = "/";
228✔
110
        } else {
111
                address.host = tmp.substr(0, split_index);
140✔
112
                address.path = tmp.substr(split_index);
140✔
113
        }
114

115
        split_index = address.host.find(":");
368✔
116
        if (split_index != string::npos) {
368✔
117
                tmp = std::move(address.host);
365✔
118
                address.host = tmp.substr(0, split_index);
365✔
119

120
                tmp = tmp.substr(split_index + 1);
365✔
121
                auto port = common::StringToLongLong(tmp);
365✔
122
                if (!port) {
365✔
123
                        return error::Error(port.error().code, url + " contains invalid port number");
×
124
                }
125
                address.port = port.value();
365✔
126
        } else {
127
                if (address.protocol == "http") {
3✔
128
                        address.port = 80;
1✔
129
                } else if (address.protocol == "https") {
2✔
130
                        address.port = 443;
1✔
131
                } else {
132
                        return error::Error(
133
                                make_error_condition(errc::protocol_not_supported),
1✔
134
                                "Cannot deduce port number from protocol " + address.protocol);
3✔
135
                }
136
        }
137

138
        log::Trace(
367✔
139
                "URL broken down into (protocol: " + address.protocol + "), (host: " + address.host
734✔
140
                + "), (port: " + to_string(address.port) + "), (path: " + address.path + ")");
1,468✔
141

142
        return error::NoError;
367✔
143
}
144

145
string URLEncode(const string &value) {
15✔
146
        stringstream escaped;
30✔
147
        escaped << hex;
15✔
148

149
        for (auto c : value) {
288✔
150
                // Keep alphanumeric and other accepted characters intact
151
                if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
273✔
152
                        escaped << c;
251✔
153
                } else {
154
                        // Any other characters are percent-encoded
155
                        escaped << uppercase;
22✔
156
                        escaped << '%' << setw(2) << int((unsigned char) c);
22✔
157
                        escaped << nouppercase;
22✔
158
                }
159
        }
160

161
        return escaped.str();
30✔
162
}
163

164
string JoinOneUrl(const string &prefix, const string &suffix) {
151✔
165
        auto prefix_end = prefix.cend();
151✔
166
        while (prefix_end != prefix.cbegin() && prefix_end[-1] == '/') {
158✔
167
                prefix_end--;
7✔
168
        }
169

170
        auto suffix_start = suffix.cbegin();
151✔
171
        while (suffix_start != suffix.cend() && *suffix_start == '/') {
207✔
172
                suffix_start++;
56✔
173
        }
174

175
        return string(prefix.cbegin(), prefix_end) + "/" + string(suffix_start, suffix.cend());
453✔
176
}
177

178
size_t CaseInsensitiveHasher::operator()(const string &str) const {
2,534✔
179
        string lower_str(str.length(), ' ');
2,534✔
180
        transform(
181
                str.begin(), str.end(), lower_str.begin(), [](unsigned char c) { return std::tolower(c); });
31,135✔
182
        return hash<string>()(lower_str);
5,065✔
183
}
184

185
bool CaseInsensitiveComparator::operator()(const string &str1, const string &str2) const {
1,018✔
186
        return strcasecmp(str1.c_str(), str2.c_str()) == 0;
1,018✔
187
}
188

189
expected::ExpectedString Transaction::GetHeader(const string &name) const {
691✔
190
        if (headers_.find(name) == headers_.end()) {
691✔
191
                return expected::unexpected(MakeError(NoSuchHeaderError, "No such header: " + name));
434✔
192
        }
193
        return headers_.at(name);
474✔
194
}
195

196
Method Request::GetMethod() const {
82✔
197
        return method_;
82✔
198
}
199

200
string Request::GetPath() const {
153✔
201
        return address_.path;
153✔
202
}
203

204
unsigned Response::GetStatusCode() const {
493✔
205
        return status_code_;
493✔
206
}
207

208
string Response::GetStatusMessage() const {
306✔
209
        return status_message_;
306✔
210
}
211

212
void OutgoingRequest::SetMethod(Method method) {
177✔
213
        method_ = method;
177✔
214
}
177✔
215

216
void OutgoingRequest::SetHeader(const string &name, const string &value) {
432✔
217
        headers_[name] = value;
432✔
218
}
432✔
219

220
error::Error OutgoingRequest::SetAddress(const string &address) {
177✔
221
        orig_address_ = address;
177✔
222

223
        return BreakDownUrl(address, address_);
177✔
224
}
225

226
void OutgoingRequest::SetBodyGenerator(BodyGenerator body_gen) {
40✔
227
        async_body_gen_ = nullptr;
40✔
228
        async_body_reader_ = nullptr;
40✔
229
        body_gen_ = body_gen;
40✔
230
}
40✔
231

232
void OutgoingRequest::SetAsyncBodyGenerator(AsyncBodyGenerator body_gen) {
6✔
233
        body_gen_ = nullptr;
6✔
234
        body_reader_ = nullptr;
6✔
235
        async_body_gen_ = body_gen;
6✔
236
}
6✔
237

238
IncomingRequest::~IncomingRequest() {
×
239
        if (!*cancelled_) {
×
240
                stream_.server_.RemoveStream(stream_.shared_from_this());
×
241
        }
242
}
×
243

244
void IncomingRequest::Cancel() {
2✔
245
        if (!*cancelled_) {
2✔
246
                stream_.Cancel();
2✔
247
                stream_.server_.RemoveStream(stream_.shared_from_this());
2✔
248
        }
249
}
2✔
250

251
io::ExpectedAsyncReaderPtr IncomingRequest::MakeBodyAsyncReader() {
52✔
252
        if (*cancelled_) {
52✔
253
                return expected::unexpected(MakeError(
×
254
                        StreamCancelledError, "Cannot make reader for a request that doesn't exist anymore"));
×
255
        }
256
        return stream_.server_.MakeBodyAsyncReader(shared_from_this());
104✔
257
}
258

259
void IncomingRequest::SetBodyWriter(io::WriterPtr writer, BodyWriterErrorMode mode) {
45✔
260
        auto exp_reader = MakeBodyAsyncReader();
45✔
261
        if (!exp_reader) {
45✔
262
                if (exp_reader.error().code != MakeError(BodyMissingError, "").code) {
8✔
263
                        log::Error(exp_reader.error().String());
×
264
                }
265
                return;
8✔
266
        }
267
        auto &reader = exp_reader.value();
37✔
268

269
        io::AsyncCopy(writer, reader, [reader, mode](error::Error err) {
37✔
270
                if (err != error::NoError) {
37✔
271
                        log::Error("Could not copy HTTP stream: " + err.String());
3✔
272
                        if (mode == BodyWriterErrorMode::Cancel) {
3✔
273
                                reader->Cancel();
2✔
274
                        }
275
                }
276
        });
111✔
277
}
278

279
ExpectedOutgoingResponsePtr IncomingRequest::MakeResponse() {
195✔
280
        if (*cancelled_) {
195✔
281
                return expected::unexpected(MakeError(
×
282
                        StreamCancelledError, "Cannot make response for a request that doesn't exist anymore"));
×
283
        }
284
        return stream_.server_.MakeResponse(shared_from_this());
390✔
285
}
286

287
IncomingResponse::IncomingResponse(ClientInterface &client, shared_ptr<bool> cancelled) :
×
288
        client_ {client},
289
        cancelled_ {cancelled} {
×
290
}
×
291

292
void IncomingResponse::Cancel() {
×
293
        if (!*cancelled_) {
×
294
                client_.Cancel();
×
295
        }
296
}
×
297

298
io::ExpectedAsyncReaderPtr IncomingResponse::MakeBodyAsyncReader() {
126✔
299
        if (*cancelled_) {
126✔
300
                return expected::unexpected(MakeError(
×
301
                        StreamCancelledError, "Cannot make reader for a response that doesn't exist anymore"));
×
302
        }
303
        return client_.MakeBodyAsyncReader(shared_from_this());
252✔
304
}
305

306
void IncomingResponse::SetBodyWriter(io::WriterPtr writer, BodyWriterErrorMode mode) {
70✔
307
        auto exp_reader = MakeBodyAsyncReader();
70✔
308
        if (!exp_reader) {
70✔
309
                if (exp_reader.error().code != MakeError(BodyMissingError, "").code) {
11✔
310
                        log::Error(exp_reader.error().String());
×
311
                }
312
                return;
11✔
313
        }
314
        auto &reader = exp_reader.value();
59✔
315

316
        io::AsyncCopy(writer, reader, [reader, mode](error::Error err) {
59✔
317
                if (err != error::NoError) {
42✔
318
                        log::Error("Could not copy HTTP stream: " + err.String());
5✔
319
                        if (mode == BodyWriterErrorMode::Cancel) {
5✔
320
                                reader->Cancel();
4✔
321
                        }
322
                }
323
        });
160✔
324
}
325

326
OutgoingResponse::~OutgoingResponse() {
×
327
        if (!*cancelled_) {
×
328
                stream_.server_.RemoveStream(stream_.shared_from_this());
×
329
        }
330
}
×
331

332
void OutgoingResponse::Cancel() {
×
333
        if (!*cancelled_) {
×
334
                stream_.Cancel();
×
335
                stream_.server_.RemoveStream(stream_.shared_from_this());
×
336
        }
337
}
×
338

339
void OutgoingResponse::SetStatusCodeAndMessage(unsigned code, const string &message) {
192✔
340
        status_code_ = code;
192✔
341
        status_message_ = message;
192✔
342
}
192✔
343

344
void OutgoingResponse::SetHeader(const string &name, const string &value) {
220✔
345
        headers_[name] = value;
220✔
346
}
220✔
347

348
void OutgoingResponse::SetBodyReader(io::ReaderPtr body_reader) {
167✔
349
        async_body_reader_ = nullptr;
167✔
350
        body_reader_ = body_reader;
167✔
351
}
167✔
352

353
void OutgoingResponse::SetAsyncBodyReader(io::AsyncReaderPtr body_reader) {
4✔
354
        body_reader_ = nullptr;
4✔
355
        async_body_reader_ = body_reader;
4✔
356
}
4✔
357

358
error::Error OutgoingResponse::AsyncReply(ReplyFinishedHandler reply_finished_handler) {
193✔
359
        if (*cancelled_) {
193✔
360
                return MakeError(StreamCancelledError, "Cannot reply when response doesn't exist anymore");
1✔
361
        }
362
        return stream_.server_.AsyncReply(shared_from_this(), reply_finished_handler);
384✔
363
}
364

365
ExponentialBackoff::ExpectedInterval ExponentialBackoff::NextInterval() {
117✔
366
        iteration_++;
117✔
367

368
        if (try_count_ > 0 && iteration_ > try_count_) {
117✔
369
                return expected::unexpected(MakeError(MaxRetryError, "Exponential backoff"));
10✔
370
        }
371

372
        chrono::milliseconds current_interval = smallest_interval_;
112✔
373
        // Backoff algorithm: Each interval is returned three times, then it's doubled, and then
374
        // that is returned three times, and so on. But if interval is ever higher than the max
375
        // interval, then return the max interval instead, and once that is returned three times,
376
        // produce MaxRetryError. If try_count_ is set, then that controls the total number of
377
        // retries, but the rest is the same, so then it simply "gets stuck" at max interval for
378
        // many iterations.
379
        for (int count = 3; count < iteration_; count += 3) {
200✔
380
                auto new_interval = current_interval * 2;
93✔
381
                if (new_interval > max_interval_) {
93✔
382
                        new_interval = max_interval_;
17✔
383
                }
384
                if (try_count_ <= 0 && new_interval == current_interval) {
93✔
385
                        return expected::unexpected(MakeError(MaxRetryError, "Exponential backoff"));
10✔
386
                }
387
                current_interval = new_interval;
88✔
388
        }
389

390
        return current_interval;
107✔
391
}
392

393
} // namespace http
394
} // namespace mender
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc