1600238645

Committed 23 Dec 2024 10:30AM UTC coverage: 76.615% (+3.8%) from 72.818%

Build # 1600238645

Build Type

Pull #274

gitlab-ci

Committed by

bahaa-ghazal

Commit Message

fix: implement signal handler for `server` commands

Changelog: Title
Ticket: QA-782
Signed-off-by: Bahaa Aldeen Ghazal <bahaa.ghazal@northern.tech>

Pull Request Pull Request #274: fix: implement signal handler for `server` commands

Run Details

4254 of 6166 branches covered (68.99%)

Branch coverage included in aggregate %.

63 of 91 new or added lines in 8 files covered. (69.23%)

46 existing lines in 5 files now uncovered.

45257 of 58457 relevant lines covered (77.42%)

21.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.11

/backend/services/deployments/server.go

// Copyright 2023 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package main

import (
        "context"
        "encoding/base64"
        "net/http"
        "net/url"
        "os"
        "os/signal"
        "strings"
        "time"

        "github.com/pkg/errors"
        "golang.org/x/sys/unix"

        "github.com/mendersoftware/mender-server/pkg/config"
        "github.com/mendersoftware/mender-server/pkg/log"

        api "github.com/mendersoftware/mender-server/services/deployments/api/http"
        "github.com/mendersoftware/mender-server/services/deployments/app"
        "github.com/mendersoftware/mender-server/services/deployments/client/reporting"
        dconfig "github.com/mendersoftware/mender-server/services/deployments/config"
        "github.com/mendersoftware/mender-server/services/deployments/storage"
        "github.com/mendersoftware/mender-server/services/deployments/storage/azblob"
        "github.com/mendersoftware/mender-server/services/deployments/storage/manager"
        "github.com/mendersoftware/mender-server/services/deployments/storage/s3"
        mstore "github.com/mendersoftware/mender-server/services/deployments/store/mongo"
)

func SetupS3(ctx context.Context, defaultOptions *s3.Options) (storage.ObjectStorage, error) {
        c := config.Config

        bucket := c.GetString(dconfig.SettingStorageBucket)

        // Copy / merge defaultOptions
        options := s3.NewOptions(defaultOptions).
                SetBucketName(bucket).
                SetForcePathStyle(c.GetBool(dconfig.SettingAwsS3ForcePathStyle))

        // The following parameters falls back on AWS_* environment if not set
        if c.IsSet(dconfig.SettingAwsS3Region) {
                options.SetRegion(c.GetString(dconfig.SettingAwsS3Region))
        }
        if c.IsSet(dconfig.SettingsAwsAuth) ||
                (c.IsSet(dconfig.SettingAwsAuthKeyId) &&
                        c.IsSet(dconfig.SettingAwsAuthSecret)) {
                options.SetStaticCredentials(
                        c.GetString(dconfig.SettingAwsAuthKeyId),
                        c.GetString(dconfig.SettingAwsAuthSecret),
                        c.GetString(dconfig.SettingAwsAuthToken),
                )
        }
        useAccelerate := c.GetBool(dconfig.SettingAwsS3UseAccelerate)
        if c.IsSet(dconfig.SettingAwsURI) {
                options.SetURI(c.GetString(dconfig.SettingAwsURI))
                if useAccelerate {
                        log.FromContext(ctx).
                                Warn(`cannot use s3 transfer acceleration with custom "uri": ` +
                                        "disabling transfer acceleration")
                }
        } else {
                options.SetUseAccelerate(c.GetBool(dconfig.SettingAwsS3UseAccelerate))
        }
        if c.IsSet(dconfig.SettingAwsExternalURI) {
                options.SetExternalURI(c.GetString(dconfig.SettingAwsExternalURI))
        }
        if c.IsSet(dconfig.SettingStorageProxyURI) {
                rawURL := c.GetString(dconfig.SettingStorageProxyURI)
                proxyURL, err := url.Parse(rawURL)
                if err != nil {
                        return nil, errors.WithMessage(err, "invalid setting `storage.proxy_uri`")
                }
                options.SetProxyURI(proxyURL)
        }
        if c.IsSet(dconfig.SettingAwsUnsignedHeaders) {
                options.SetUnsignedHeaders(c.GetStringSlice(dconfig.SettingAwsUnsignedHeaders))
        }

        storage, err := s3.New(ctx, options)
        return storage, err
}

func SetupBlobStorage(
        ctx context.Context,
        defaultOptions *azblob.Options,
) (storage.ObjectStorage, error) {
        c := config.Config

        // Copy / merge options
        options := azblob.NewOptions(defaultOptions)

        if c.IsSet(dconfig.SettingAzureConnectionString) {
                options.SetConnectionString(c.GetString(dconfig.SettingAzureConnectionString))
        } else if c.IsSet(dconfig.SettingAzureSharedKeyAccount) &&
                c.IsSet(dconfig.SettingAzureSharedKeyAccountKey) {
                creds := azblob.SharedKeyCredentials{
                        AccountName: c.GetString(dconfig.SettingAzureSharedKeyAccount),
                        AccountKey:  c.GetString(dconfig.SettingAzureSharedKeyAccountKey),
                }
                if c.IsSet(dconfig.SettingAzureSharedKeyURI) {
                        uri := c.GetString(dconfig.SettingAzureSharedKeyURI)
                        creds.URI = &uri
                }
                options.SetSharedKey(creds)
        }
        if c.IsSet(dconfig.SettingStorageProxyURI) {
                rawURL := c.GetString(dconfig.SettingStorageProxyURI)
                proxyURL, err := url.Parse(rawURL)
                if err != nil {
                        return nil, errors.WithMessage(err, `invalid setting "storage.proxy_uri"`)
                }
                if !strings.HasPrefix(strings.ToLower(proxyURL.Scheme), "https") {
                        log.FromContext(ctx).
                                Warnf(`setting "storage.proxy_uri" (%s) is not using https`, rawURL)
                }
                options.SetProxyURI(proxyURL)
        }
        return azblob.New(ctx, c.GetString(dconfig.SettingStorageBucket), options)
}

func SetupObjectStorage(ctx context.Context) (objManager storage.ObjectStorage, err error) {
        c := config.Config

        // Calculate s3 multipart buffer size: the minimum buffer size that
        // covers the maximum image size aligned to multiple of 5MiB.
        maxImageSize := c.GetInt64(dconfig.SettingStorageMaxImageSize)
        bufferSize := (((maxImageSize - 1) /
                (s3.MultipartMaxParts * s3.MultipartMinSize)) + 1) *
                s3.MultipartMinSize
        var (
                s3Options = s3.NewOptions().
                                SetContentType(app.ArtifactContentType).
                                SetBufferSize(int(bufferSize))
                azOptions = azblob.NewOptions().
                                SetContentType(app.ArtifactContentType)
        )
        var defaultStorage storage.ObjectStorage
        switch defType := c.GetString(dconfig.SettingDefaultStorage); defType {
        case dconfig.StorageTypeAWS:
                defaultStorage, err = SetupS3(ctx, s3Options)
        case dconfig.StorageTypeAzure:
                defaultStorage, err = SetupBlobStorage(ctx, azOptions)
        default:
                err = errors.Errorf(
                        `storage type must be one of %q or %q, received value %q`,
                        dconfig.StorageTypeAWS, dconfig.StorageTypeAzure, defType,
                )
        }
        if err != nil {
                return nil, err
        }
        return manager.New(ctx, defaultStorage, s3Options, azOptions)
}

func RunServer(ctx context.Context) error {
        c := config.Config
        l := log.New(log.Ctx{})
        dbClient, err := mstore.NewMongoClient(ctx, c)
        if err != nil {
                return err
        }
        defer func() {
                _ = dbClient.Disconnect(context.Background())
        }()

        ds := mstore.NewDataStoreMongoWithClient(dbClient)

        // Storage Layer
        objStore, err := SetupObjectStorage(ctx)
        if err != nil {
                return errors.WithMessage(err, "main: failed to setup storage client")
        }

        app := app.NewDeployments(ds, objStore, 0, false)
        if addr := c.GetString(dconfig.SettingReportingAddr); addr != "" {
                c := reporting.NewClient(addr)
                app = app.WithReporting(c)
        }

        // Setup API Router configuration
        base64Repl := strings.NewReplacer("-", "+", "_", "/", "=", "")
        expireSec := c.GetDuration(dconfig.SettingPresignExpireSeconds)
        apiConf := api.NewConfig().
                SetPresignExpire(time.Second * expireSec).
                SetPresignHostname(c.GetString(dconfig.SettingPresignHost)).
                SetPresignScheme(c.GetString(dconfig.SettingPresignScheme)).
                SetMaxImageSize(c.GetInt64(dconfig.SettingStorageMaxImageSize)).
                SetMaxGenerateDataSize(c.GetInt64(dconfig.SettingStorageMaxGenerateSize)).
                SetEnableDirectUpload(c.GetBool(dconfig.SettingStorageEnableDirectUpload)).
                SetEnableDirectUploadSkipVerify(c.GetBool(dconfig.SettingStorageDirectUploadSkipVerify)).
                SetDisableNewReleasesFeature(c.GetBool(dconfig.SettingDisableNewReleasesFeature))
        if key, err := base64.RawStdEncoding.DecodeString(
                base64Repl.Replace(
                        c.GetString(dconfig.SettingPresignSecret),
                ),
        ); err == nil {
                apiConf.SetPresignSecret(key)
        }
        handler, err := api.NewHandler(ctx, app, ds, apiConf)
        if err != nil {
                return err
        }

        listen := c.GetString(dconfig.SettingListen)

        if c.IsSet(dconfig.SettingHttps) {

                cert := c.GetString(dconfig.SettingHttpsCertificate)
                key := c.GetString(dconfig.SettingHttpsKey)

                return http.ListenAndServeTLS(listen, cert, key, handler)
        }

        srv := &http.Server{
                Addr:    listen,
                Handler: handler,
        }

        errChan := make(chan error, 1)
        go func() {
                if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
                        errChan <- err
                }
        }()
        quit := make(chan os.Signal, 1)
        signal.Notify(quit, unix.SIGINT, unix.SIGTERM)
        select {
        case sig := <-quit:
                l.Infof("received signal %s: terminating", sig)
        case err = <-errChan:
                l.Errorf("server terminated unexpectedly: %s", err.Error())
                return err
        }
        return nil
}

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package main
16
17	import (
18	"context"
19	"encoding/base64"
20	"net/http"
21	"net/url"
22	"os"
23	"os/signal"
24	"strings"
25	"time"
26
27	"github.com/pkg/errors"
28	"golang.org/x/sys/unix"
29
30	"github.com/mendersoftware/mender-server/pkg/config"
31	"github.com/mendersoftware/mender-server/pkg/log"
32
33	api "github.com/mendersoftware/mender-server/services/deployments/api/http"
34	"github.com/mendersoftware/mender-server/services/deployments/app"
35	"github.com/mendersoftware/mender-server/services/deployments/client/reporting"
36	dconfig "github.com/mendersoftware/mender-server/services/deployments/config"
37	"github.com/mendersoftware/mender-server/services/deployments/storage"
38	"github.com/mendersoftware/mender-server/services/deployments/storage/azblob"
39	"github.com/mendersoftware/mender-server/services/deployments/storage/manager"
40	"github.com/mendersoftware/mender-server/services/deployments/storage/s3"
41	mstore "github.com/mendersoftware/mender-server/services/deployments/store/mongo"
42	)
43
44	func SetupS3(ctx context.Context, defaultOptions *s3.Options) (storage.ObjectStorage, error) {	2✔
45	c := config.Config	2✔
46		2✔
47	bucket := c.GetString(dconfig.SettingStorageBucket)	2✔
48		2✔
49	// Copy / merge defaultOptions	2✔
50	options := s3.NewOptions(defaultOptions).	2✔
51	SetBucketName(bucket).	2✔
52	SetForcePathStyle(c.GetBool(dconfig.SettingAwsS3ForcePathStyle))	2✔
53		2✔
54	// The following parameters falls back on AWS_* environment if not set	2✔
55	if c.IsSet(dconfig.SettingAwsS3Region) {	4✔
56	options.SetRegion(c.GetString(dconfig.SettingAwsS3Region))	2✔
57	}	2✔
58	if c.IsSet(dconfig.SettingsAwsAuth) \|\|	2✔
59	(c.IsSet(dconfig.SettingAwsAuthKeyId) &&	2✔
60	c.IsSet(dconfig.SettingAwsAuthSecret)) {	4✔
61	options.SetStaticCredentials(	2✔
62	c.GetString(dconfig.SettingAwsAuthKeyId),	2✔
63	c.GetString(dconfig.SettingAwsAuthSecret),	2✔
64	c.GetString(dconfig.SettingAwsAuthToken),	2✔
65	)	2✔
66	}	2✔
67	useAccelerate := c.GetBool(dconfig.SettingAwsS3UseAccelerate)	2✔
68	if c.IsSet(dconfig.SettingAwsURI) {	4✔
69	options.SetURI(c.GetString(dconfig.SettingAwsURI))	2✔
70	if useAccelerate {	2✔
71	log.FromContext(ctx).	×
72	Warn(`cannot use s3 transfer acceleration with custom "uri": ` +	×
73	"disabling transfer acceleration")	×
74	}	×
75	} else {	×
76	options.SetUseAccelerate(c.GetBool(dconfig.SettingAwsS3UseAccelerate))	×
77	}	×
78	if c.IsSet(dconfig.SettingAwsExternalURI) {	4✔
79	options.SetExternalURI(c.GetString(dconfig.SettingAwsExternalURI))	2✔
80	}	2✔
81	if c.IsSet(dconfig.SettingStorageProxyURI) {	2✔
82	rawURL := c.GetString(dconfig.SettingStorageProxyURI)	×
83	proxyURL, err := url.Parse(rawURL)	×
84	if err != nil {	×
85	return nil, errors.WithMessage(err, "invalid setting `storage.proxy_uri`")	×
86	}	×
87	options.SetProxyURI(proxyURL)	×
88	}
89	if c.IsSet(dconfig.SettingAwsUnsignedHeaders) {	4✔
90	options.SetUnsignedHeaders(c.GetStringSlice(dconfig.SettingAwsUnsignedHeaders))	2✔
91	}	2✔
92
93	storage, err := s3.New(ctx, options)	2✔
94	return storage, err	2✔
95	}
96
97	func SetupBlobStorage(
98	ctx context.Context,
99	defaultOptions *azblob.Options,
100	) (storage.ObjectStorage, error) {	×
101	c := config.Config	×
102		×
103	// Copy / merge options	×
104	options := azblob.NewOptions(defaultOptions)	×
105		×
106	if c.IsSet(dconfig.SettingAzureConnectionString) {	×
107	options.SetConnectionString(c.GetString(dconfig.SettingAzureConnectionString))	×
108	} else if c.IsSet(dconfig.SettingAzureSharedKeyAccount) &&	×
109	c.IsSet(dconfig.SettingAzureSharedKeyAccountKey) {	×
110	creds := azblob.SharedKeyCredentials{	×
111	AccountName: c.GetString(dconfig.SettingAzureSharedKeyAccount),	×
112	AccountKey: c.GetString(dconfig.SettingAzureSharedKeyAccountKey),	×
113	}	×
114	if c.IsSet(dconfig.SettingAzureSharedKeyURI) {	×
115	uri := c.GetString(dconfig.SettingAzureSharedKeyURI)	×
116	creds.URI = &uri	×
117	}	×
118	options.SetSharedKey(creds)	×
119	}
120	if c.IsSet(dconfig.SettingStorageProxyURI) {	×
121	rawURL := c.GetString(dconfig.SettingStorageProxyURI)	×
122	proxyURL, err := url.Parse(rawURL)	×
123	if err != nil {	×
124	return nil, errors.WithMessage(err, `invalid setting "storage.proxy_uri"`)	×
125	}	×
126	if !strings.HasPrefix(strings.ToLower(proxyURL.Scheme), "https") {	×
127	log.FromContext(ctx).	×
128	Warnf(`setting "storage.proxy_uri" (%s) is not using https`, rawURL)	×
129	}	×
130	options.SetProxyURI(proxyURL)	×
131	}
132	return azblob.New(ctx, c.GetString(dconfig.SettingStorageBucket), options)	×
133	}
134
135	func SetupObjectStorage(ctx context.Context) (objManager storage.ObjectStorage, err error) {	2✔
136	c := config.Config	2✔
137		2✔
138	// Calculate s3 multipart buffer size: the minimum buffer size that	2✔
139	// covers the maximum image size aligned to multiple of 5MiB.	2✔
140	maxImageSize := c.GetInt64(dconfig.SettingStorageMaxImageSize)	2✔
141	bufferSize := (((maxImageSize - 1) /	2✔
142	(s3.MultipartMaxParts * s3.MultipartMinSize)) + 1) *	2✔
143	s3.MultipartMinSize	2✔
144	var (	2✔
145	s3Options = s3.NewOptions().	2✔
146	SetContentType(app.ArtifactContentType).	2✔
147	SetBufferSize(int(bufferSize))	2✔
148	azOptions = azblob.NewOptions().	2✔
149	SetContentType(app.ArtifactContentType)	2✔
150	)	2✔
151	var defaultStorage storage.ObjectStorage	2✔
152	switch defType := c.GetString(dconfig.SettingDefaultStorage); defType {	2✔
153	case dconfig.StorageTypeAWS:	2✔
154	defaultStorage, err = SetupS3(ctx, s3Options)	2✔
155	case dconfig.StorageTypeAzure:	×
156	defaultStorage, err = SetupBlobStorage(ctx, azOptions)	×
157	default:	×
158	err = errors.Errorf(	×
159	`storage type must be one of %q or %q, received value %q`,	×
160	dconfig.StorageTypeAWS, dconfig.StorageTypeAzure, defType,	×
161	)	×
162	}
163	if err != nil {	2✔
164	return nil, err	×
165	}	×
166	return manager.New(ctx, defaultStorage, s3Options, azOptions)	2✔
167	}
168
169	func RunServer(ctx context.Context) error {	2✔
170	c := config.Config	2✔
171	l := log.New(log.Ctx{})	2✔
172	dbClient, err := mstore.NewMongoClient(ctx, c)	2✔
173	if err != nil {	2✔
174	return err	×
175	}	×
176	defer func() {	4✔
177	_ = dbClient.Disconnect(context.Background())	2✔
178	}()	2✔
179
180	ds := mstore.NewDataStoreMongoWithClient(dbClient)	2✔
181		2✔
182	// Storage Layer	2✔
183	objStore, err := SetupObjectStorage(ctx)	2✔
184	if err != nil {	2✔
185	return errors.WithMessage(err, "main: failed to setup storage client")	×
186	}	×
187
188	app := app.NewDeployments(ds, objStore, 0, false)	2✔
189	if addr := c.GetString(dconfig.SettingReportingAddr); addr != "" {	2✔
190	c := reporting.NewClient(addr)	×
191	app = app.WithReporting(c)	×
192	}	×
193
194	// Setup API Router configuration
195	base64Repl := strings.NewReplacer("-", "+", "_", "/", "=", "")	2✔
196	expireSec := c.GetDuration(dconfig.SettingPresignExpireSeconds)	2✔
197	apiConf := api.NewConfig().	2✔
198	SetPresignExpire(time.Second * expireSec).	2✔
199	SetPresignHostname(c.GetString(dconfig.SettingPresignHost)).	2✔
200	SetPresignScheme(c.GetString(dconfig.SettingPresignScheme)).	2✔
201	SetMaxImageSize(c.GetInt64(dconfig.SettingStorageMaxImageSize)).	2✔
202	SetMaxGenerateDataSize(c.GetInt64(dconfig.SettingStorageMaxGenerateSize)).	2✔
203	SetEnableDirectUpload(c.GetBool(dconfig.SettingStorageEnableDirectUpload)).	2✔
204	SetEnableDirectUploadSkipVerify(c.GetBool(dconfig.SettingStorageDirectUploadSkipVerify)).	2✔
205	SetDisableNewReleasesFeature(c.GetBool(dconfig.SettingDisableNewReleasesFeature))	2✔
206	if key, err := base64.RawStdEncoding.DecodeString(	2✔
207	base64Repl.Replace(	2✔
208	c.GetString(dconfig.SettingPresignSecret),	2✔
209	),	2✔
210	); err == nil {	4✔
211	apiConf.SetPresignSecret(key)	2✔
212	}	2✔
213	handler, err := api.NewHandler(ctx, app, ds, apiConf)	2✔
214	if err != nil {	2✔
215	return err	×
216	}	×
217
218	listen := c.GetString(dconfig.SettingListen)	2✔
219		2✔
220	if c.IsSet(dconfig.SettingHttps) {	2✔
221		×
222	cert := c.GetString(dconfig.SettingHttpsCertificate)	×
223	key := c.GetString(dconfig.SettingHttpsKey)	×
224		×
225	return http.ListenAndServeTLS(listen, cert, key, handler)	×
226	}	×
227
228	srv := &http.Server{	2✔
229	Addr: listen,	2✔
230	Handler: handler,	2✔
231	}	2✔
232		2✔
233	errChan := make(chan error, 1)	2✔
234	go func() {	4✔
235	if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {	2✔
NEW 236	errChan <- err	×
NEW 237	}	×
238	}()
239	quit := make(chan os.Signal, 1)	2✔
240	signal.Notify(quit, unix.SIGINT, unix.SIGTERM)	2✔
241	select {	2✔
242	case sig := <-quit:	2✔
243	l.Infof("received signal %s: terminating", sig)	2✔
NEW 244	case err = <-errChan:	×
NEW 245	l.Errorf("server terminated unexpectedly: %s", err.Error())	×
NEW 246	return err	×
247	}
248	return nil	2✔
249	}

mendersoftware / mender-server / 1600238645

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous