• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / deployments / 1062206731

06 Nov 2023 10:31AM UTC coverage: 77.615% (-3.3%) from 80.909%
1062206731

push

gitlab-ci

web-flow
Merge pull request #948 from alfrunes/upgrade-deps

Upgrade go mod dependencies

13 of 25 new or added lines in 4 files covered. (52.0%)

5 existing lines in 2 files now uncovered.

4133 of 5325 relevant lines covered (77.62%)

63.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.6
/storage/s3/options.go
1
// Copyright 2023 Northern.tech AS
2
//
3
//    Licensed under the Apache License, Version 2.0 (the "License");
4
//    you may not use this file except in compliance with the License.
5
//    You may obtain a copy of the License at
6
//
7
//        http://www.apache.org/licenses/LICENSE-2.0
8
//
9
//    Unless required by applicable law or agreed to in writing, software
10
//    distributed under the License is distributed on an "AS IS" BASIS,
11
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
//    See the License for the specific language governing permissions and
13
//    limitations under the License.
14

15
package s3
16

17
import (
18
        "context"
19
        "crypto/tls"
20
        "net/http"
21
        "net/textproto"
22
        "net/url"
23
        "time"
24

25
        v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
26
        "github.com/aws/aws-sdk-go-v2/service/s3"
27
        "github.com/aws/smithy-go/middleware"
28
        smithyhttp "github.com/aws/smithy-go/transport/http"
29
        validation "github.com/go-ozzo/ozzo-validation/v4"
30

31
        "github.com/mendersoftware/deployments/model"
32
        "github.com/mendersoftware/deployments/storage"
33
)
34

35
const (
36
        kib = 1024
37
        mib = kib * 1024
38

39
        DefaultBufferSize = 10 * mib
40
        DefaultExpire     = 15 * time.Minute
41
)
42

43
var (
44
        validAtLeast5MiB = validation.Min(MultipartMinSize).
45
                Error("must be at least 5MiB")
46
)
47

48
// Subset of model.StorageSettings applicable to s3.
49
type storageSettings struct {
50
        // BucketName defines the bucket name.
51
        BucketName *string
52

53
        // StaticCredentials that overrides AWS config.
54
        StaticCredentials *StaticCredentials `json:"auth"`
55

56
        // Region where the bucket lives
57
        Region *string
58
        // ExternalURI is the URI used for signing requests.
59
        ExternalURI *string
60
        // URI is the URI for the s3 API.
61
        URI *string
62
        // ProxyURI is used for rewriting presigned requests, pointing the
63
        // requests to the proxy URL instead of the direct URL to s3.
64
        ProxyURI *url.URL
65

66
        // ForcePathStyle encodes bucket in the API path.
67
        ForcePathStyle bool
68
        // UseAccelerate enables s3 Accelerate
69
        UseAccelerate bool
70
}
71

72
func newFromParent(defaults *storageSettings, parent *model.StorageSettings) *storageSettings {
1✔
73
        ret := new(storageSettings)
1✔
74
        ret.patch(defaults)
1✔
75
        if parent.Bucket != "" {
2✔
76
                ret.BucketName = &parent.Bucket
1✔
77
        }
1✔
78
        if parent.Region != "" {
2✔
79
                ret.Region = &parent.Region
1✔
80
        }
1✔
81
        if parent.Key != "" && parent.Secret != "" {
2✔
82
                ret.StaticCredentials = &StaticCredentials{
1✔
83
                        Key:    parent.Key,
1✔
84
                        Secret: parent.Secret,
1✔
85
                        Token:  parent.Token,
1✔
86
                }
1✔
87
        }
1✔
88
        if parent.ExternalUri != "" {
2✔
89
                ret.ExternalURI = &parent.ExternalUri
1✔
90
        }
1✔
91
        if parent.ProxyURI != nil {
1✔
92
                ret.ProxyURI, _ = url.Parse(*parent.ProxyURI)
×
93
        }
×
94
        if parent.Uri != "" {
2✔
95
                ret.URI = &parent.Uri
1✔
96
        }
1✔
97
        if parent.ForcePathStyle != ret.ForcePathStyle {
2✔
98
                ret.ForcePathStyle = parent.ForcePathStyle
1✔
99
        }
1✔
100
        if parent.UseAccelerate != ret.UseAccelerate {
1✔
UNCOV
101
                ret.UseAccelerate = parent.UseAccelerate
×
UNCOV
102
        }
×
103
        return ret
1✔
104
}
105

106
func (s storageSettings) Validate() error {
5✔
107
        return validation.ValidateStruct(&s,
5✔
108
                validation.Field(&s.StaticCredentials),
5✔
109
        )
5✔
110
}
5✔
111

112
func (s storageSettings) options(opts *s3.Options) {
12✔
113
        if s.StaticCredentials != nil {
23✔
114
                opts.Credentials = *s.StaticCredentials
11✔
115
        }
11✔
116
        if s.Region != nil {
23✔
117
                opts.Region = *s.Region
11✔
118
        }
11✔
119
        opts.BaseEndpoint = s.URI
12✔
120
        opts.UsePathStyle = s.ForcePathStyle
12✔
121
        opts.UseAccelerate = s.UseAccelerate
12✔
122
}
123

124
func (s storageSettings) presignOptions(opts *s3.PresignOptions) {
5✔
125
        opts.ClientOptions = append(opts.ClientOptions, s.options)
5✔
126
        if s.ExternalURI != nil {
6✔
127
                opts.ClientOptions = append(opts.ClientOptions, func(clientOpts *s3.Options) {
2✔
128
                        clientOpts.BaseEndpoint = s.ExternalURI
1✔
129
                })
1✔
130
        }
131
}
132

133
func (s *storageSettings) patch(setting *storageSettings) *storageSettings {
8✔
134
        if setting == nil {
8✔
135
                return s
×
136
        }
×
137
        if setting.BucketName != nil {
16✔
138
                s.BucketName = setting.BucketName
8✔
139
        }
8✔
140
        if setting.StaticCredentials != nil {
16✔
141
                s.StaticCredentials = setting.StaticCredentials
8✔
142
        }
8✔
143
        if setting.Region != nil {
16✔
144
                s.Region = setting.Region
8✔
145
        }
8✔
146
        if setting.ExternalURI != nil {
9✔
147
                s.ExternalURI = setting.ExternalURI
1✔
148
        }
1✔
149
        if setting.URI != nil {
10✔
150
                s.URI = setting.URI
2✔
151
        }
2✔
152
        if setting.ProxyURI != nil {
8✔
153
                s.ProxyURI = setting.ProxyURI
×
154
        }
×
155
        if setting.ForcePathStyle != s.ForcePathStyle {
9✔
156
                s.ForcePathStyle = setting.ForcePathStyle
1✔
157
        }
1✔
158
        if setting.UseAccelerate != s.UseAccelerate {
8✔
UNCOV
159
                s.UseAccelerate = setting.UseAccelerate
×
UNCOV
160
        }
×
161
        return s
8✔
162
}
163

164
type Options struct {
165
        storageSettings
166

167
        // ContentType of the uploaded objects
168
        ContentType *string
169
        // FilenameSuffix adds the suffix to the content-disposition for object downloads
170
        FilenameSuffix *string
171
        // DefaultExpire is the fallback presign expire duration
172
        // (defaults to 15min).
173
        DefaultExpire *time.Duration
174
        // BufferSize sets the buffer size allocated for uploads.
175
        // This implicitly sets the upper limit for upload size:
176
        // BufferSize * 10000 (defaults to: 5MiB).
177
        BufferSize *int
178

179
        // UnsignedHeaders forces the driver to skip the named headers from the
180
        // being signed.
181
        UnsignedHeaders []string
182

183
        // Transport sets an alternative RoundTripper used by the Go HTTP
184
        // client.
185
        Transport http.RoundTripper
186
}
187

188
func NewOptions(opts ...*Options) *Options {
12✔
189
        defaultBufferSize := DefaultBufferSize
12✔
190
        ret := &Options{
12✔
191
                BufferSize: &defaultBufferSize,
12✔
192
        }
12✔
193
        for _, opt := range opts {
20✔
194
                ret.storageSettings.patch(&opt.storageSettings)
8✔
195
                if opt.DefaultExpire != nil {
9✔
196
                        ret.DefaultExpire = opt.DefaultExpire
1✔
197
                }
1✔
198
                if opt.ContentType != nil {
10✔
199
                        ret.ContentType = opt.ContentType
2✔
200
                }
2✔
201
                if opt.BufferSize != nil {
16✔
202
                        ret.BufferSize = opt.BufferSize
8✔
203
                }
8✔
204
                if opt.UnsignedHeaders != nil {
10✔
205
                        ret.UnsignedHeaders = opt.UnsignedHeaders
2✔
206
                }
2✔
207
                if opt.Transport != nil {
12✔
208
                        ret.Transport = opt.Transport
4✔
209
                }
4✔
210
        }
211
        return ret
12✔
212
}
213

214
func (opts Options) Validate() error {
5✔
215
        return validation.ValidateStruct(&opts,
5✔
216
                validation.Field(&opts.storageSettings),
5✔
217
                validation.Field(&opts.BufferSize, validAtLeast5MiB),
5✔
218
        )
5✔
219
}
5✔
220

221
func (opts *Options) SetBucketName(bucketName string) *Options {
5✔
222
        opts.storageSettings.BucketName = &bucketName
5✔
223
        return opts
5✔
224
}
5✔
225

226
func (opts *Options) SetStaticCredentials(key, secret, sessionToken string) *Options {
5✔
227
        opts.StaticCredentials = &StaticCredentials{
5✔
228
                Key:    key,
5✔
229
                Secret: secret,
5✔
230
                Token:  sessionToken,
5✔
231
        }
5✔
232
        return opts
5✔
233
}
5✔
234

235
func (opts *Options) SetRegion(region string) *Options {
5✔
236
        opts.Region = &region
5✔
237
        return opts
5✔
238
}
5✔
239

240
func (opts *Options) SetContentType(contentType string) *Options {
2✔
241
        opts.ContentType = &contentType
2✔
242
        return opts
2✔
243
}
2✔
244

245
func (opts *Options) SetFilenameSuffix(suffix string) *Options {
×
246
        opts.FilenameSuffix = &suffix
×
247
        return opts
×
248
}
×
249

250
func (opts *Options) SetExternalURI(externalURI string) *Options {
1✔
251
        opts.ExternalURI = &externalURI
1✔
252
        return opts
1✔
253
}
1✔
254

255
func (opts *Options) SetURI(URI string) *Options {
2✔
256
        opts.URI = &URI
2✔
257
        return opts
2✔
258
}
2✔
259

260
func (opts *Options) SetProxyURI(proxyURI *url.URL) *Options {
×
261
        opts.ProxyURI = proxyURI
×
262
        return opts
×
263
}
×
264

265
func (opts *Options) SetForcePathStyle(forcePathStyle bool) *Options {
2✔
266
        opts.ForcePathStyle = forcePathStyle
2✔
267
        return opts
2✔
268
}
2✔
269

270
func (opts *Options) SetUseAccelerate(useAccelerate bool) *Options {
1✔
271
        opts.UseAccelerate = useAccelerate
1✔
272
        return opts
1✔
273
}
1✔
274

275
func (opts *Options) SetDefaultExpire(defaultExpire time.Duration) *Options {
1✔
276
        opts.DefaultExpire = &defaultExpire
1✔
277
        return opts
1✔
278
}
1✔
279

280
func (opts *Options) SetBufferSize(bufferSize int) *Options {
2✔
281
        opts.BufferSize = &bufferSize
2✔
282
        return opts
2✔
283
}
2✔
284

285
func (opts *Options) SetUnsignedHeaders(unsignedHeaders []string) *Options {
2✔
286
        opts.UnsignedHeaders = unsignedHeaders
2✔
287
        return opts
2✔
288
}
2✔
289

290
func (opts *Options) SetTransport(transport http.RoundTripper) *Options {
4✔
291
        opts.Transport = transport
4✔
292
        return opts
4✔
293
}
4✔
294

295
type apiOptions func(*middleware.Stack) error
296

297
// Google Cloud Storage does not tolerate signing the Accept-Encoding header
298
func unsignedHeadersMiddleware(headers []string) apiOptions {
2✔
299
        signMiddlewareID := (&v4.SignHTTPRequestMiddleware{}).ID()
2✔
300
        for i := range headers {
4✔
301
                headers[i] = textproto.CanonicalMIMEHeaderKey(headers[i])
2✔
302
        }
2✔
303
        return func(stack *middleware.Stack) error {
6✔
304
                if _, ok := stack.Finalize.Get("Signing"); !ok {
5✔
305
                        // If the operation does not invoke signing, we're done.
1✔
306
                        return nil
1✔
307
                }
1✔
308
                // ... -> RemoveUnsignedHeaders -> Signing -> AddUnsignedHeaders
309
                var unsignedHeaders http.Header = make(http.Header)
4✔
310
                err := stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc(
4✔
311
                        "RemoveUnsignedHeaders", func(
4✔
312
                                ctx context.Context,
4✔
313
                                in middleware.FinalizeInput,
4✔
314
                                next middleware.FinalizeHandler,
4✔
315
                        ) (middleware.FinalizeOutput, middleware.Metadata, error) {
8✔
316
                                if req, ok := in.Request.(*smithyhttp.Request); ok {
8✔
317
                                        for _, hdr := range headers {
8✔
318
                                                if value, ok := req.Header[hdr]; ok {
8✔
319
                                                        unsignedHeaders[hdr] = value
4✔
320
                                                        req.Header.Del(hdr)
4✔
321
                                                }
4✔
322
                                        }
323
                                }
324
                                return next.HandleFinalize(ctx, in)
4✔
325
                        }), signMiddlewareID, middleware.Before)
326
                if err != nil {
4✔
327
                        return err
×
328
                }
×
329
                return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc(
4✔
330
                        "AddUnsignedHeaders", func(
4✔
331
                                ctx context.Context,
4✔
332
                                in middleware.FinalizeInput,
4✔
333
                                next middleware.FinalizeHandler,
4✔
334
                        ) (middleware.FinalizeOutput, middleware.Metadata, error) {
8✔
335
                                if req, ok := in.Request.(*smithyhttp.Request); ok {
8✔
336
                                        for key, value := range unsignedHeaders {
8✔
337
                                                req.Header[key] = value
4✔
338
                                        }
4✔
339
                                }
340
                                return next.HandleFinalize(ctx, in)
4✔
341
                        }), signMiddlewareID, middleware.After)
342
        }
343
}
344

345
func (opts *Options) toS3Options() (
346
        clientOpts func(*s3.Options),
347
        presignOpts func(*s3.PresignOptions),
348
) {
5✔
349
        clientOpts = func(s3Opts *s3.Options) {
10✔
350
                opts.options(s3Opts)
5✔
351
                if len(opts.UnsignedHeaders) > 0 {
7✔
352
                        s3Opts.APIOptions = append(
2✔
353
                                s3Opts.APIOptions,
2✔
354
                                unsignedHeadersMiddleware(opts.UnsignedHeaders),
2✔
355
                        )
2✔
356
                }
2✔
357
                roundTripper := opts.Transport
5✔
358
                if roundTripper == nil {
6✔
359
                        roundTripper = &http.Transport{
1✔
360
                                TLSClientConfig: &tls.Config{
1✔
361
                                        RootCAs: storage.GetRootCAs(),
1✔
362
                                },
1✔
363
                        }
1✔
364
                }
1✔
365
                s3Opts.HTTPClient = &http.Client{
5✔
366
                        Transport: roundTripper,
5✔
367
                }
5✔
368
        }
369

370
        expires := DefaultExpire
5✔
371
        if opts.DefaultExpire != nil {
6✔
372
                expires = *opts.DefaultExpire
1✔
373
        }
1✔
374
        presignOpts = func(s3Opts *s3.PresignOptions) {
10✔
375
                opts.presignOptions(s3Opts)
5✔
376
                s3.WithPresignExpires(expires)(s3Opts)
5✔
377
        }
5✔
378
        return clientOpts, presignOpts
5✔
379
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc