• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / mender-server / 1622978334

13 Jan 2025 03:51PM UTC coverage: 72.802% (-3.8%) from 76.608%
1622978334

Pull #300

gitlab-ci

alfrunes
fix: Deployment device count should not exceed max devices

Added a condition to skip deployments when the device count reaches max
devices.

Changelog: Title
Ticket: MEN-7847
Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>
Pull Request #300: fix: Deployment device count should not exceed max devices

4251 of 6164 branches covered (68.96%)

Branch coverage included in aggregate %.

0 of 18 new or added lines in 1 file covered. (0.0%)

2544 existing lines in 83 files now uncovered.

42741 of 58384 relevant lines covered (73.21%)

21.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.62
/backend/services/deployments/storage/s3/options.go
1
// Copyright 2023 Northern.tech AS
2
//
3
//    Licensed under the Apache License, Version 2.0 (the "License");
4
//    you may not use this file except in compliance with the License.
5
//    You may obtain a copy of the License at
6
//
7
//        http://www.apache.org/licenses/LICENSE-2.0
8
//
9
//    Unless required by applicable law or agreed to in writing, software
10
//    distributed under the License is distributed on an "AS IS" BASIS,
11
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
//    See the License for the specific language governing permissions and
13
//    limitations under the License.
14

15
package s3
16

17
import (
18
        "context"
19
        "crypto/tls"
20
        "net/http"
21
        "net/textproto"
22
        "net/url"
23
        "time"
24

25
        "github.com/aws/aws-sdk-go-v2/aws"
26
        v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
27
        "github.com/aws/aws-sdk-go-v2/service/s3"
28
        validation "github.com/go-ozzo/ozzo-validation/v4"
29

30
        "github.com/mendersoftware/mender-server/services/deployments/model"
31
        "github.com/mendersoftware/mender-server/services/deployments/storage"
32
)
33

34
const (
35
        kib = 1024
36
        mib = kib * 1024
37

38
        DefaultBufferSize = 10 * mib
39
        DefaultExpire     = 15 * time.Minute
40
)
41

42
var (
43
        validAtLeast5MiB = validation.Min(MultipartMinSize).
44
                Error("must be at least 5MiB")
45
)
46

47
// Subset of model.StorageSettings applicable to s3.
48
type storageSettings struct {
49
        // BucketName defines the bucket name.
50
        BucketName *string
51

52
        // StaticCredentials that overrides AWS config.
53
        StaticCredentials *StaticCredentials `json:"auth"`
54

55
        // Region where the bucket lives
56
        Region *string
57
        // ExternalURI is the URI used for signing requests.
58
        ExternalURI *string
59
        // URI is the URI for the s3 API.
60
        URI *string
61
        // ProxyURI is used for rewriting presigned requests, pointing the
62
        // requests to the proxy URL instead of the direct URL to s3.
63
        ProxyURI *url.URL
64

65
        // ForcePathStyle encodes bucket in the API path.
66
        ForcePathStyle bool
67
        // UseAccelerate enables s3 Accelerate
68
        UseAccelerate bool
69
}
70

UNCOV
71
func newFromParent(defaults *storageSettings, parent *model.StorageSettings) *storageSettings {
×
UNCOV
72
        ret := new(storageSettings)
×
UNCOV
73
        ret.patch(defaults)
×
UNCOV
74
        if parent.Bucket != "" {
×
UNCOV
75
                ret.BucketName = &parent.Bucket
×
UNCOV
76
        }
×
UNCOV
77
        if parent.Region != "" {
×
UNCOV
78
                ret.Region = &parent.Region
×
UNCOV
79
        }
×
UNCOV
80
        if parent.Key != "" && parent.Secret != "" {
×
UNCOV
81
                ret.StaticCredentials = &StaticCredentials{
×
UNCOV
82
                        Key:    parent.Key,
×
UNCOV
83
                        Secret: parent.Secret,
×
UNCOV
84
                        Token:  parent.Token,
×
UNCOV
85
                }
×
UNCOV
86
        }
×
UNCOV
87
        if parent.ExternalUri != "" {
×
UNCOV
88
                ret.ExternalURI = &parent.ExternalUri
×
UNCOV
89
        }
×
UNCOV
90
        if parent.ProxyURI != nil {
×
91
                ret.ProxyURI, _ = url.Parse(*parent.ProxyURI)
×
92
        }
×
UNCOV
93
        if parent.Uri != "" {
×
UNCOV
94
                ret.URI = &parent.Uri
×
UNCOV
95
        }
×
UNCOV
96
        if parent.ForcePathStyle != ret.ForcePathStyle {
×
UNCOV
97
                ret.ForcePathStyle = parent.ForcePathStyle
×
UNCOV
98
        }
×
UNCOV
99
        if parent.UseAccelerate != ret.UseAccelerate {
×
100
                ret.UseAccelerate = parent.UseAccelerate
×
101
        }
×
UNCOV
102
        return ret
×
103
}
104

105
func (s storageSettings) Validate() error {
1✔
106
        return validation.ValidateStruct(&s,
1✔
107
                validation.Field(&s.StaticCredentials),
1✔
108
        )
1✔
109
}
1✔
110

111
func (s storageSettings) options(opts *s3.Options) {
1✔
112
        if s.StaticCredentials != nil {
2✔
113
                opts.Credentials = *s.StaticCredentials
1✔
114
        }
1✔
115
        if s.Region != nil {
2✔
116
                opts.Region = *s.Region
1✔
117
        }
1✔
118
        opts.BaseEndpoint = s.URI
1✔
119
        opts.UsePathStyle = s.ForcePathStyle
1✔
120
        opts.UseAccelerate = s.UseAccelerate
1✔
121
}
122

123
func (s storageSettings) presignOptions(opts *s3.PresignOptions) {
1✔
124
        opts.ClientOptions = append(opts.ClientOptions, s.options)
1✔
125
        if s.ExternalURI != nil {
1✔
UNCOV
126
                opts.ClientOptions = append(opts.ClientOptions, func(clientOpts *s3.Options) {
×
UNCOV
127
                        clientOpts.BaseEndpoint = s.ExternalURI
×
UNCOV
128
                })
×
129
        }
130
}
131

132
func (s *storageSettings) patch(setting *storageSettings) *storageSettings {
1✔
133
        if setting == nil {
1✔
134
                return s
×
135
        }
×
136
        if setting.BucketName != nil {
2✔
137
                s.BucketName = setting.BucketName
1✔
138
        }
1✔
139
        if setting.StaticCredentials != nil {
2✔
140
                s.StaticCredentials = setting.StaticCredentials
1✔
141
        }
1✔
142
        if setting.Region != nil {
2✔
143
                s.Region = setting.Region
1✔
144
        }
1✔
145
        if setting.ExternalURI != nil {
1✔
UNCOV
146
                s.ExternalURI = setting.ExternalURI
×
UNCOV
147
        }
×
148
        if setting.URI != nil {
2✔
149
                s.URI = setting.URI
1✔
150
        }
1✔
151
        if setting.ProxyURI != nil {
1✔
152
                s.ProxyURI = setting.ProxyURI
×
153
        }
×
154
        if setting.ForcePathStyle != s.ForcePathStyle {
1✔
UNCOV
155
                s.ForcePathStyle = setting.ForcePathStyle
×
UNCOV
156
        }
×
157
        if setting.UseAccelerate != s.UseAccelerate {
1✔
158
                s.UseAccelerate = setting.UseAccelerate
×
159
        }
×
160
        return s
1✔
161
}
162

163
type Options struct {
164
        storageSettings
165

166
        // ContentType of the uploaded objects
167
        ContentType *string
168
        // FilenameSuffix adds the suffix to the content-disposition for object downloads
169
        FilenameSuffix *string
170
        // DefaultExpire is the fallback presign expire duration
171
        // (defaults to 15min).
172
        DefaultExpire *time.Duration
173
        // BufferSize sets the buffer size allocated for uploads.
174
        // This implicitly sets the upper limit for upload size:
175
        // BufferSize * 10000 (defaults to: 5MiB).
176
        BufferSize *int
177

178
        // UnsignedHeaders forces the driver to skip the named headers from the
179
        // being signed.
180
        UnsignedHeaders []string
181

182
        // Transport sets an alternative RoundTripper used by the Go HTTP
183
        // client.
184
        Transport http.RoundTripper
185
}
186

187
func NewOptions(opts ...*Options) *Options {
1✔
188
        defaultBufferSize := DefaultBufferSize
1✔
189
        ret := &Options{
1✔
190
                BufferSize: &defaultBufferSize,
1✔
191
        }
1✔
192
        for _, opt := range opts {
2✔
193
                ret.storageSettings.patch(&opt.storageSettings)
1✔
194
                if opt.DefaultExpire != nil {
2✔
195
                        ret.DefaultExpire = opt.DefaultExpire
1✔
196
                }
1✔
197
                if opt.ContentType != nil {
2✔
198
                        ret.ContentType = opt.ContentType
1✔
199
                }
1✔
200
                if opt.BufferSize != nil {
2✔
201
                        ret.BufferSize = opt.BufferSize
1✔
202
                }
1✔
203
                if opt.UnsignedHeaders != nil {
2✔
204
                        ret.UnsignedHeaders = opt.UnsignedHeaders
1✔
205
                }
1✔
206
                if opt.Transport != nil {
2✔
207
                        ret.Transport = opt.Transport
1✔
208
                }
1✔
209
        }
210
        return ret
1✔
211
}
212

213
func (opts Options) Validate() error {
1✔
214
        return validation.ValidateStruct(&opts,
1✔
215
                validation.Field(&opts.storageSettings),
1✔
216
                validation.Field(&opts.BufferSize, validAtLeast5MiB),
1✔
217
        )
1✔
218
}
1✔
219

220
func (opts *Options) SetBucketName(bucketName string) *Options {
1✔
221
        opts.storageSettings.BucketName = &bucketName
1✔
222
        return opts
1✔
223
}
1✔
224

225
func (opts *Options) SetStaticCredentials(key, secret, sessionToken string) *Options {
1✔
226
        opts.StaticCredentials = &StaticCredentials{
1✔
227
                Key:    key,
1✔
228
                Secret: secret,
1✔
229
                Token:  sessionToken,
1✔
230
        }
1✔
231
        return opts
1✔
232
}
1✔
233

234
func (opts *Options) SetRegion(region string) *Options {
1✔
235
        opts.Region = &region
1✔
236
        return opts
1✔
237
}
1✔
238

239
func (opts *Options) SetContentType(contentType string) *Options {
1✔
240
        opts.ContentType = &contentType
1✔
241
        return opts
1✔
242
}
1✔
243

244
func (opts *Options) SetFilenameSuffix(suffix string) *Options {
×
245
        opts.FilenameSuffix = &suffix
×
246
        return opts
×
247
}
×
248

UNCOV
249
func (opts *Options) SetExternalURI(externalURI string) *Options {
×
UNCOV
250
        opts.ExternalURI = &externalURI
×
UNCOV
251
        return opts
×
UNCOV
252
}
×
253

254
func (opts *Options) SetURI(URI string) *Options {
1✔
255
        opts.URI = &URI
1✔
256
        return opts
1✔
257
}
1✔
258

259
func (opts *Options) SetProxyURI(proxyURI *url.URL) *Options {
×
260
        opts.ProxyURI = proxyURI
×
261
        return opts
×
262
}
×
263

264
func (opts *Options) SetForcePathStyle(forcePathStyle bool) *Options {
1✔
265
        opts.ForcePathStyle = forcePathStyle
1✔
266
        return opts
1✔
267
}
1✔
268

269
func (opts *Options) SetUseAccelerate(useAccelerate bool) *Options {
1✔
270
        opts.UseAccelerate = useAccelerate
1✔
271
        return opts
1✔
272
}
1✔
273

274
func (opts *Options) SetDefaultExpire(defaultExpire time.Duration) *Options {
1✔
275
        opts.DefaultExpire = &defaultExpire
1✔
276
        return opts
1✔
277
}
1✔
278

279
func (opts *Options) SetBufferSize(bufferSize int) *Options {
1✔
280
        opts.BufferSize = &bufferSize
1✔
281
        return opts
1✔
282
}
1✔
283

284
func (opts *Options) SetUnsignedHeaders(unsignedHeaders []string) *Options {
1✔
285
        opts.UnsignedHeaders = unsignedHeaders
1✔
286
        return opts
1✔
287
}
1✔
288

289
func (opts *Options) SetTransport(transport http.RoundTripper) *Options {
1✔
290
        opts.Transport = transport
1✔
291
        return opts
1✔
292
}
1✔
293

294
type Signer struct {
295
        v4.Signer
296
        // Google Cloud Storage does not tolerate signing the Accept-Encoding header
297
        UnsignedHeaders []string
298
}
299

300
func NewSigner(
301
        unsignedHeaders []string,
302
        optFns ...func(signer *v4.SignerOptions),
303
) v4.HTTPSigner {
1✔
304
        for i := range unsignedHeaders {
2✔
305
                unsignedHeaders[i] = textproto.CanonicalMIMEHeaderKey(unsignedHeaders[i])
1✔
306
        }
1✔
307
        return &Signer{
1✔
308
                Signer:          *v4.NewSigner(optFns...),
1✔
309
                UnsignedHeaders: unsignedHeaders,
1✔
310
        }
1✔
311
}
312

313
func (s Signer) SignHTTP(
314
        ctx context.Context,
315
        credentials aws.Credentials,
316
        r *http.Request,
317
        payloadHash, service, region string,
318
        signingTime time.Time,
319
        optFns ...func(*v4.SignerOptions)) error {
1✔
320
        unsignedHeaders := make(http.Header)
1✔
321
        for _, hdr := range s.UnsignedHeaders {
2✔
322
                if value, ok := r.Header[hdr]; ok {
2✔
323
                        unsignedHeaders[hdr] = value
1✔
324
                        delete(r.Header, hdr)
1✔
325
                }
1✔
326
        }
327
        err := s.Signer.SignHTTP(
1✔
328
                ctx, credentials,
1✔
329
                r, payloadHash,
1✔
330
                service, region,
1✔
331
                signingTime, optFns...,
1✔
332
        )
1✔
333
        for key, value := range unsignedHeaders {
2✔
334
                r.Header[key] = value
1✔
335
        }
1✔
336
        return err
1✔
337
}
338

339
func (opts *Options) toS3Options() (
340
        clientOpts func(*s3.Options),
341
        presignOpts func(*s3.PresignOptions),
342
) {
1✔
343
        clientOpts = func(s3Opts *s3.Options) {
2✔
344
                opts.options(s3Opts)
1✔
345
                if len(opts.UnsignedHeaders) > 0 {
2✔
346
                        s3Opts.HTTPSignerV4 = NewSigner(opts.UnsignedHeaders)
1✔
347
                }
1✔
348
                roundTripper := opts.Transport
1✔
349
                if roundTripper == nil {
1✔
UNCOV
350
                        roundTripper = &http.Transport{
×
UNCOV
351
                                TLSClientConfig: &tls.Config{
×
UNCOV
352
                                        RootCAs: storage.GetRootCAs(),
×
UNCOV
353
                                },
×
UNCOV
354
                        }
×
UNCOV
355
                }
×
356
                s3Opts.HTTPClient = &http.Client{
1✔
357
                        Transport: roundTripper,
1✔
358
                }
1✔
359
        }
360

361
        expires := DefaultExpire
1✔
362
        if opts.DefaultExpire != nil {
2✔
363
                expires = *opts.DefaultExpire
1✔
364
        }
1✔
365
        presignOpts = func(s3Opts *s3.PresignOptions) {
2✔
366
                opts.presignOptions(s3Opts)
1✔
367
                s3.WithPresignExpires(expires)(s3Opts)
1✔
368
        }
1✔
369
        return clientOpts, presignOpts
1✔
370
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc