950545443

pending completion

Build # 950545443

Build Type

push

gitlab-ci

Committed by

vpodzime

Commit Message

feat: Introduce deployments::DeploymentLog

A simple class that can scope logging of a specific deployment.

Ticket: MEN-6578
Changelog: none
Signed-off-by: Vratislav Podzimek <v.podzimek@mykolab.com>

Run Details

93 of 93 new or added lines in 1 file covered. (100.0%)

4283 of 5387 relevant lines covered (79.51%)

159.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.95

/mender-update/deployments/platform/boost_log/deployments.cpp

// Copyright 2023 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

#include <mender-update/deployments.hpp>

#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/log/attributes.hpp>
#include <boost/log/common.hpp>
#include <boost/log/sinks.hpp>
#include <boost/smart_ptr/shared_ptr.hpp>

#include <algorithm>
#include <cctype>
#include <filesystem>
#include <string>

#include <common/error.hpp>
#include <common/io.hpp>
#include <common/json.hpp>
#include <common/log.hpp>
#include <common/path.hpp>

namespace mender {
namespace update {
namespace deployments {

using namespace std;

namespace logging = boost::log;
namespace expr = boost::log::expressions;
namespace sinks = boost::log::sinks;

namespace fs = std::filesystem;

namespace error = mender::common::error;
namespace io = mender::common::io;
namespace json = mender::common::json;
namespace mlog = mender::common::log;
namespace path = mender::common::path;

static void JsonLogFormatter(logging::record_view const &rec, logging::formatting_ostream &strm) {
        strm << "{";

        auto val = logging::extract<boost::posix_time::ptime>("TimeStamp", rec);
        if (val) {
                strm << R"("timestamp":")"
                         << json::EscapeString(boost::posix_time::to_iso_extended_string(val.get())) << "\",";
        }

        auto level = logging::extract<mlog::LogLevel>("Severity", rec);
        if (level) {
                string lvl = mlog::ToStringLogLevel(level.get());
                strm << R"("level":")" << json::EscapeString(lvl) << "\",";
        }

        strm << R"("message":")" << json::EscapeString(*rec[expr::smessage]) << "\"}";
}

static const size_t kMaxExistingLogs = 5;
static const uintmax_t kLogsFreeSpaceRequired = 100 * 1024; // 100 KiB

error::Error DeploymentLog::PrepareLogDirectory() {
        fs::path dir_path(data_store_dir_);
        bool created = fs::create_directories(dir_path);
        if (created) {
                // should basically never happen, but if we happened to create the
                // directory, it's empty and thus well-prepared
                return error::NoError;
        }

        vector<string> old_logs;
        for (auto const &entry : fs::directory_iterator {dir_path}) {
                fs::path file_path = entry.path();
                if (!fs::is_regular_file(file_path)) {
                        continue;
                }

                string file_name = file_path.filename().string();

                if (file_name == "deployments.0000." + id_ + ".log") {
                        // this log file will be (re)used, leave it alone
                        continue;
                }

                if ((file_name.find("deployments.") != 0)
                        || (file_name.substr(file_name.size() - 4) != ".log")) {
                        continue;
                }

                // expected file name: deployments.NNNN.ID.log
                // "deployments.".size() == 12
                auto second_dot_pos = file_name.find('.', 12);
                auto last_dot_pos = file_name.find_last_of('.');
                if ((second_dot_pos == string::npos) || (last_dot_pos == string::npos)
                        || (second_dot_pos == last_dot_pos) || (second_dot_pos != 16)
                        || any_of(file_name.cbegin() + 12, file_name.cbegin() + second_dot_pos, [](char c) {
                                   return !isdigit(c);
                           })) {
                        mlog::Warning("Old deployment log with a malformed file name found: " + file_name);
                        continue;
                }

                old_logs.push_back(file_name);
        }
        std::sort(old_logs.begin(), old_logs.end());

        error_code ec;
        fs::space_info space_info = fs::space(dir_path, ec);
        if (ec) {
                return error::Error(
                        ec.default_error_condition(), "Failed to check free space for log files");
        }

        while ((old_logs.size() > 0)
                   && ((space_info.available < kLogsFreeSpaceRequired)
                           || (old_logs.size() > (kMaxExistingLogs - 1)))) {
                auto last_log_file = old_logs[old_logs.size() - 1];
                old_logs.pop_back();
                if (!fs::remove(dir_path / last_log_file, ec) && ec) {
                        return error::Error(
                                ec.default_error_condition(),
                                "Failed to remove old log file '" + last_log_file + "'");
                }
                if (space_info.available < kLogsFreeSpaceRequired) {
                        space_info = fs::space(dir_path, ec);
                        if (ec) {
                                return error::Error(
                                        ec.default_error_condition(), "Failed to check free space for log files");
                        }
                }
        }

        // now let's make sure old logs have an increasing index starting with 0001
        for (ssize_t i = old_logs.size() - 1; i >= 0; i--) {
                // "deployments.".size() == 12
                auto second_dot_pos = old_logs[i].find('.', 12);
                auto last_dot_pos = old_logs[i].find_last_of('.');

                // should never happen due the filter above when populating old_logs and
                // due to how these files are named
                assert(second_dot_pos != string::npos);
                assert(last_dot_pos != string::npos);
                assert(second_dot_pos != last_dot_pos);

                string deployment_id;
                if ((second_dot_pos == string::npos) || (last_dot_pos == string::npos)) {
                        deployment_id = "unknown_deployment";
                } else {
                        deployment_id =
                                old_logs[i].substr(second_dot_pos + 1, (last_dot_pos - second_dot_pos - 1));
                }
                stringstream ss;
                ss << "deployments.";
                ss << setfill('0') << setw(4) << to_string(i + 1);
                ss << "." + deployment_id;
                ss << ".log";

                string new_name = ss.str();
                fs::rename(dir_path / old_logs[i], dir_path / new_name, ec);
                if (ec) {
                        return error::Error(
                                ec.default_error_condition(),
                                "Failed to rename old log file '" + old_logs[i] + "'");
                }
        }

        return error::NoError;
}

error::Error DeploymentLog::BeginLogging() {
        auto err = PrepareLogDirectory();
        if (err != error::NoError) {
                return err;
        }

        auto log_file_path = path::Join(data_store_dir_, "deployments.0000." + id_ + ".log");
        auto ex_ofstr = io::OpenOfstream(log_file_path, true);
        if (!ex_ofstr) {
                return ex_ofstr.error();
        }

        auto log_stream = boost::make_shared<std::ofstream>(std::move(ex_ofstr.value()));
        sink_ = boost::make_shared<text_sink>();
        sink_->set_formatter(&JsonLogFormatter);
        sink_->locked_backend()->add_stream(log_stream);
        sink_->locked_backend()->auto_flush(true);

        logging::core::get()->add_sink(sink_);

        return error::NoError;
}

error::Error DeploymentLog::FinishLogging() {
        logging::core::get()->remove_sink(sink_);
        sink_.reset();
        return error::NoError;
}

} // namespace deployments
} // namespace update
} // namespace mender

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	#include <mender-update/deployments.hpp>
16
17	#include <boost/date_time/posix_time/posix_time.hpp>
18	#include <boost/log/attributes.hpp>
19	#include <boost/log/common.hpp>
20	#include <boost/log/sinks.hpp>
21	#include <boost/smart_ptr/shared_ptr.hpp>
22
23	#include <algorithm>
24	#include <cctype>
25	#include <filesystem>
26	#include <string>
27
28	#include <common/error.hpp>
29	#include <common/io.hpp>
30	#include <common/json.hpp>
31	#include <common/log.hpp>
32	#include <common/path.hpp>
33
34	namespace mender {
35	namespace update {
36	namespace deployments {
37
38	using namespace std;
39
40	namespace logging = boost::log;
41	namespace expr = boost::log::expressions;
42	namespace sinks = boost::log::sinks;
43
44	namespace fs = std::filesystem;
45
46	namespace error = mender::common::error;
47	namespace io = mender::common::io;
48	namespace json = mender::common::json;
49	namespace mlog = mender::common::log;
50	namespace path = mender::common::path;
51
52	static void JsonLogFormatter(logging::record_view const &rec, logging::formatting_ostream &strm) {	8✔
53	strm << "{";	8✔
54
55	auto val = logging::extract<boost::posix_time::ptime>("TimeStamp", rec);	8✔
56	if (val) {	8✔
57	strm << R"("timestamp":")"
58	<< json::EscapeString(boost::posix_time::to_iso_extended_string(val.get())) << "\",";	8✔
59	}
60
61	auto level = logging::extract<mlog::LogLevel>("Severity", rec);	8✔
62	if (level) {	8✔
63	string lvl = mlog::ToStringLogLevel(level.get());	8✔
64	strm << R"("level":")" << json::EscapeString(lvl) << "\",";	8✔
65	}
66
67	strm << R"("message":")" << json::EscapeString(*rec[expr::smessage]) << "\"}";	8✔
68	}	8✔
69
70	static const size_t kMaxExistingLogs = 5;
71	static const uintmax_t kLogsFreeSpaceRequired = 100 * 1024; // 100 KiB
72
73	error::Error DeploymentLog::PrepareLogDirectory() {	5✔
74	fs::path dir_path(data_store_dir_);	10✔
75	bool created = fs::create_directories(dir_path);	5✔
76	if (created) {	5✔
77	// should basically never happen, but if we happened to create the
78	// directory, it's empty and thus well-prepared
79	return error::NoError;	×
80	}
81
82	vector<string> old_logs;	10✔
83	for (auto const &entry : fs::directory_iterator {dir_path}) {	25✔
84	fs::path file_path = entry.path();	10✔
85	if (!fs::is_regular_file(file_path)) {	10✔
86	continue;	×
87	}
88
89	string file_name = file_path.filename().string();	10✔
90
91	if (file_name == "deployments.0000." + id_ + ".log") {	10✔
92	// this log file will be (re)used, leave it alone
93	continue;	1✔
94	}
95
96	if ((file_name.find("deployments.") != 0)	18✔
97	\|\| (file_name.substr(file_name.size() - 4) != ".log")) {	9✔
98	continue;	×
99	}
100
101	// expected file name: deployments.NNNN.ID.log
102	// "deployments.".size() == 12
103	auto second_dot_pos = file_name.find('.', 12);	9✔
104	auto last_dot_pos = file_name.find_last_of('.');	9✔
105	if ((second_dot_pos == string::npos) \|\| (last_dot_pos == string::npos)	8✔
106	\|\| (second_dot_pos == last_dot_pos) \|\| (second_dot_pos != 16)	8✔
107	\|\| any_of(file_name.cbegin() + 12, file_name.cbegin() + second_dot_pos, [](char c) {	17✔
108	return !isdigit(c);	20✔
109	})) {
110	mlog::Warning("Old deployment log with a malformed file name found: " + file_name);	4✔
111	continue;	4✔
112	}
113
114	old_logs.push_back(file_name);	5✔
115	}
116	std::sort(old_logs.begin(), old_logs.end());	5✔
117
118	error_code ec;	5✔
119	fs::space_info space_info = fs::space(dir_path, ec);	5✔
120	if (ec) {	5✔
121	return error::Error(
122	ec.default_error_condition(), "Failed to check free space for log files");	×
123	}
124
125	while ((old_logs.size() > 0)	6✔
126	&& ((space_info.available < kLogsFreeSpaceRequired)	8✔
127	\|\| (old_logs.size() > (kMaxExistingLogs - 1)))) {	2✔
128	auto last_log_file = old_logs[old_logs.size() - 1];	1✔
129	old_logs.pop_back();	1✔
130	if (!fs::remove(dir_path / last_log_file, ec) && ec) {	1✔
131	return error::Error(
132	ec.default_error_condition(),	×
133	"Failed to remove old log file '" + last_log_file + "'");	×
134	}
135	if (space_info.available < kLogsFreeSpaceRequired) {	1✔
136	space_info = fs::space(dir_path, ec);	×
137	if (ec) {	×
138	return error::Error(
139	ec.default_error_condition(), "Failed to check free space for log files");	×
140	}
141	}
142	}
143
144	// now let's make sure old logs have an increasing index starting with 0001
145	for (ssize_t i = old_logs.size() - 1; i >= 0; i--) {	9✔
146	// "deployments.".size() == 12
147	auto second_dot_pos = old_logs[i].find('.', 12);	4✔
148	auto last_dot_pos = old_logs[i].find_last_of('.');	4✔
149
150	// should never happen due the filter above when populating old_logs and
151	// due to how these files are named
152	assert(second_dot_pos != string::npos);	4✔
153	assert(last_dot_pos != string::npos);	4✔
154	assert(second_dot_pos != last_dot_pos);	4✔
155
156	string deployment_id;	4✔
157	if ((second_dot_pos == string::npos) \|\| (last_dot_pos == string::npos)) {	4✔
158	deployment_id = "unknown_deployment";	×
159	} else {
160	deployment_id =
161	old_logs[i].substr(second_dot_pos + 1, (last_dot_pos - second_dot_pos - 1));	4✔
162	}
163	stringstream ss;	4✔
164	ss << "deployments.";	4✔
165	ss << setfill('0') << setw(4) << to_string(i + 1);	4✔
166	ss << "." + deployment_id;	4✔
167	ss << ".log";	4✔
168
169	string new_name = ss.str();	4✔
170	fs::rename(dir_path / old_logs[i], dir_path / new_name, ec);	4✔
171	if (ec) {	4✔
172	return error::Error(
173	ec.default_error_condition(),	×
174	"Failed to rename old log file '" + old_logs[i] + "'");	×
175	}
176	}
177
178	return error::NoError;	5✔
179	}
180
181	error::Error DeploymentLog::BeginLogging() {	5✔
182	auto err = PrepareLogDirectory();	10✔
183	if (err != error::NoError) {	5✔
184	return err;	×
185	}
186
187	auto log_file_path = path::Join(data_store_dir_, "deployments.0000." + id_ + ".log");	15✔
188	auto ex_ofstr = io::OpenOfstream(log_file_path, true);	10✔
189	if (!ex_ofstr) {	5✔
190	return ex_ofstr.error();	×
191	}
192
193	auto log_stream = boost::make_shared<std::ofstream>(std::move(ex_ofstr.value()));	10✔
194	sink_ = boost::make_shared<text_sink>();	5✔
195	sink_->set_formatter(&JsonLogFormatter);	5✔
196	sink_->locked_backend()->add_stream(log_stream);	5✔
197	sink_->locked_backend()->auto_flush(true);	5✔
198
199	logging::core::get()->add_sink(sink_);	5✔
200
201	return error::NoError;	5✔
202	}
203
204	error::Error DeploymentLog::FinishLogging() {	5✔
205	logging::core::get()->remove_sink(sink_);	5✔
206	sink_.reset();	5✔
207	return error::NoError;	5✔
208	}
209
210	} // namespace deployments
211	} // namespace update
212	} // namespace mender

mendersoftware / mender / 950545443

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous