• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / deployments / 914758580

pending completion
914758580

Pull #869

gitlab-ci

alfrunes
ci: Bump golangci-lint to v1.15.3 (go1.20)

Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>
Pull Request #869: feat: New configuration parameter storage.proxy_uri

180 of 261 new or added lines in 6 files covered. (68.97%)

2 existing lines in 2 files now uncovered.

7374 of 9361 relevant lines covered (78.77%)

33.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.17
/storage/s3/options.go
1
// Copyright 2023 Northern.tech AS
2
//
3
//    Licensed under the Apache License, Version 2.0 (the "License");
4
//    you may not use this file except in compliance with the License.
5
//    You may obtain a copy of the License at
6
//
7
//        http://www.apache.org/licenses/LICENSE-2.0
8
//
9
//    Unless required by applicable law or agreed to in writing, software
10
//    distributed under the License is distributed on an "AS IS" BASIS,
11
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
//    See the License for the specific language governing permissions and
13
//    limitations under the License.
14

15
package s3
16

17
import (
18
        "context"
19
        "crypto/tls"
20
        "net/http"
21
        "net/textproto"
22
        "net/url"
23
        "time"
24

25
        "github.com/aws/aws-sdk-go-v2/aws"
26
        v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
27
        "github.com/aws/aws-sdk-go-v2/service/s3"
28
        "github.com/aws/smithy-go/middleware"
29
        smithyhttp "github.com/aws/smithy-go/transport/http"
30
        validation "github.com/go-ozzo/ozzo-validation/v4"
31

32
        "github.com/mendersoftware/deployments/model"
33
        "github.com/mendersoftware/deployments/storage"
34
)
35

36
const (
37
        kib = 1024
38
        mib = kib * 1024
39

40
        DefaultBufferSize = 10 * mib
41
        DefaultExpire     = 15 * time.Minute
42
)
43

44
var (
45
        validAtLeast5MiB = validation.Min(MultipartMinSize).
46
                Error("must be at least 5MiB")
47
)
48

49
// Subset of model.StorageSettings applicable to s3.
50
type storageSettings struct {
51
        // BucketName defines the bucket name.
52
        BucketName *string
53

54
        // StaticCredentials that overrides AWS config.
55
        StaticCredentials *StaticCredentials `json:"auth"`
56

57
        // Region where the bucket lives
58
        Region *string
59
        // ExternalURI is the URI used for signing requests.
60
        ExternalURI *string
61
        // URI is the URI for the s3 API.
62
        URI *string
63
        // ProxyURI is used for rewriting presigned requests, pointing the
64
        // requests to the proxy URL instead of the direct URL to s3.
65
        ProxyURI *url.URL
66

67
        // ForcePathStyle encodes bucket in the API path.
68
        ForcePathStyle bool
69
        // UseAccelerate enables s3 Accelerate
70
        UseAccelerate bool
71
}
72

73
func newFromParent(defaults *storageSettings, parent *model.StorageSettings) *storageSettings {
1✔
74
        ret := new(storageSettings)
1✔
75
        ret.patch(defaults)
1✔
76
        if parent.Bucket != "" {
2✔
77
                ret.BucketName = &parent.Bucket
1✔
78
        }
1✔
79
        if parent.Region != "" {
2✔
80
                ret.Region = &parent.Region
1✔
81
        }
1✔
82
        if parent.Key != "" && parent.Secret != "" {
2✔
83
                ret.StaticCredentials = &StaticCredentials{
1✔
84
                        Key:    parent.Key,
1✔
85
                        Secret: parent.Secret,
1✔
86
                        Token:  parent.Token,
1✔
87
                }
1✔
88
        }
1✔
89
        if parent.ExternalUri != "" {
2✔
90
                ret.ExternalURI = &parent.ExternalUri
1✔
91
        }
1✔
92
        if parent.Uri != "" {
2✔
93
                ret.URI = &parent.Uri
1✔
94
        }
1✔
95
        if parent.ForcePathStyle != ret.ForcePathStyle {
2✔
96
                ret.ForcePathStyle = parent.ForcePathStyle
1✔
97
        }
1✔
98
        if parent.UseAccelerate != ret.UseAccelerate {
2✔
99
                ret.UseAccelerate = parent.UseAccelerate
1✔
100
        }
1✔
101
        return ret
1✔
102
}
103

104
func (s storageSettings) Validate() error {
5✔
105
        return validation.ValidateStruct(&s,
5✔
106
                validation.Field(&s.StaticCredentials),
5✔
107
        )
5✔
108
}
5✔
109

110
func (s storageSettings) options(opts *s3.Options) {
8✔
111
        if s.StaticCredentials != nil {
15✔
112
                opts.Credentials = *s.StaticCredentials
7✔
113
        }
7✔
114
        if s.Region != nil {
15✔
115
                opts.Region = *s.Region
7✔
116
        }
7✔
117
        if s.URI != nil {
10✔
118
                endpointURI := *s.URI
2✔
119
                opts.EndpointResolver = s3.EndpointResolverFromURL(endpointURI,
2✔
120
                        func(ep *aws.Endpoint) {
4✔
121
                                ep.HostnameImmutable = s.ForcePathStyle
2✔
122
                        },
2✔
123
                )
124
        }
125
        opts.UsePathStyle = s.ForcePathStyle
8✔
126
        opts.UseAccelerate = s.UseAccelerate
8✔
127
}
128

129
func (s storageSettings) presignOptions(opts *s3.PresignOptions) {
5✔
130
        if s.ExternalURI != nil {
6✔
131
                presignURL := *s.ExternalURI
1✔
132
                resolver := s3.EndpointResolverFromURL(presignURL,
1✔
133
                        func(ep *aws.Endpoint) {
2✔
134
                                ep.HostnameImmutable = s.ForcePathStyle
1✔
135
                        },
1✔
136
                )
137
                s3.WithPresignClientFromClientOptions(
1✔
138
                        s.options,
1✔
139
                        s3.WithEndpointResolver(resolver),
1✔
140
                )(opts)
1✔
141
        }
142
}
143

144
func (s *storageSettings) patch(setting *storageSettings) *storageSettings {
8✔
145
        if setting == nil {
8✔
NEW
146
                return s
×
NEW
147
        }
×
148
        if setting.BucketName != nil {
16✔
149
                s.BucketName = setting.BucketName
8✔
150
        }
8✔
151
        if setting.StaticCredentials != nil {
16✔
152
                s.StaticCredentials = setting.StaticCredentials
8✔
153
        }
8✔
154
        if setting.Region != nil {
16✔
155
                s.Region = setting.Region
8✔
156
        }
8✔
157
        if setting.ExternalURI != nil {
9✔
158
                s.ExternalURI = setting.ExternalURI
1✔
159
        }
1✔
160
        if setting.URI != nil {
10✔
161
                s.URI = setting.URI
2✔
162
        }
2✔
163
        if setting.ProxyURI != nil {
8✔
NEW
164
                s.ProxyURI = setting.ProxyURI
×
NEW
165
        }
×
166
        if setting.ForcePathStyle != s.ForcePathStyle {
9✔
167
                s.ForcePathStyle = setting.ForcePathStyle
1✔
168
        }
1✔
169
        if setting.UseAccelerate != s.UseAccelerate {
9✔
170
                s.UseAccelerate = setting.UseAccelerate
1✔
171
        }
1✔
172
        return s
8✔
173
}
174

NEW
175
func (s storageSettings) RewriteLink(link *model.Link) (*model.Link, error) {
×
NEW
176
        if s.ProxyURI == nil {
×
NEW
177
                return link, nil
×
NEW
178
        }
×
NEW
179
        uri, err := url.Parse(link.Uri)
×
NEW
180
        if err != nil {
×
NEW
181
                return link, err
×
NEW
182
        }
×
NEW
183
        new := s.ProxyURI.JoinPath(uri.Path)
×
NEW
184
        q := new.Query()
×
NEW
185
        for key, value := range uri.Query() {
×
NEW
186
                q[key] = value
×
NEW
187
        }
×
NEW
188
        new.RawQuery = q.Encode()
×
NEW
189
        new.Fragment = uri.Fragment
×
NEW
190
        link.Uri = new.String()
×
NEW
191
        return link, nil
×
192
}
193

194
type Options struct {
195
        storageSettings
196

197
        // ContentType of the uploaded objects
198
        ContentType *string
199
        // FilenameSuffix adds the suffix to the content-disposition for object downloads
200
        FilenameSuffix *string
201
        // DefaultExpire is the fallback presign expire duration
202
        // (defaults to 15min).
203
        DefaultExpire *time.Duration
204
        // BufferSize sets the buffer size allocated for uploads.
205
        // This implicitly sets the upper limit for upload size:
206
        // BufferSize * 10000 (defaults to: 5MiB).
207
        BufferSize *int
208

209
        // UnsignedHeaders forces the driver to skip the named headers from the
210
        // being signed.
211
        UnsignedHeaders []string
212

213
        // Transport sets an alternative RoundTripper used by the Go HTTP
214
        // client.
215
        Transport http.RoundTripper
216
}
217

218
func NewOptions(opts ...*Options) *Options {
12✔
219
        defaultBufferSize := DefaultBufferSize
12✔
220
        ret := &Options{
12✔
221
                BufferSize: &defaultBufferSize,
12✔
222
        }
12✔
223
        for _, opt := range opts {
20✔
224
                ret.storageSettings.patch(&opt.storageSettings)
8✔
225
                if opt.DefaultExpire != nil {
9✔
226
                        ret.DefaultExpire = opt.DefaultExpire
1✔
227
                }
1✔
228
                if opt.ContentType != nil {
10✔
229
                        ret.ContentType = opt.ContentType
2✔
230
                }
2✔
231
                if opt.BufferSize != nil {
16✔
232
                        ret.BufferSize = opt.BufferSize
8✔
233
                }
8✔
234
                if opt.UnsignedHeaders != nil {
10✔
235
                        ret.UnsignedHeaders = opt.UnsignedHeaders
2✔
236
                }
2✔
237
                if opt.Transport != nil {
12✔
238
                        ret.Transport = opt.Transport
4✔
239
                }
4✔
240
        }
241
        return ret
12✔
242
}
243

244
func (opts Options) Validate() error {
5✔
245
        return validation.ValidateStruct(&opts,
5✔
246
                validation.Field(&opts.storageSettings),
5✔
247
                validation.Field(&opts.BufferSize, validAtLeast5MiB),
5✔
248
        )
5✔
249
}
5✔
250

251
func (opts *Options) SetBucketName(bucketName string) *Options {
5✔
252
        opts.storageSettings.BucketName = &bucketName
5✔
253
        return opts
5✔
254
}
5✔
255

256
func (opts *Options) SetStaticCredentials(key, secret, sessionToken string) *Options {
5✔
257
        opts.StaticCredentials = &StaticCredentials{
5✔
258
                Key:    key,
5✔
259
                Secret: secret,
5✔
260
                Token:  sessionToken,
5✔
261
        }
5✔
262
        return opts
5✔
263
}
5✔
264

265
func (opts *Options) SetRegion(region string) *Options {
5✔
266
        opts.Region = &region
5✔
267
        return opts
5✔
268
}
5✔
269

270
func (opts *Options) SetContentType(contentType string) *Options {
2✔
271
        opts.ContentType = &contentType
2✔
272
        return opts
2✔
273
}
2✔
274

275
func (opts *Options) SetFilenameSuffix(suffix string) *Options {
×
276
        opts.FilenameSuffix = &suffix
×
277
        return opts
×
278
}
×
279

280
func (opts *Options) SetExternalURI(externalURI string) *Options {
1✔
281
        opts.ExternalURI = &externalURI
1✔
282
        return opts
1✔
283
}
1✔
284

285
func (opts *Options) SetURI(URI string) *Options {
2✔
286
        opts.URI = &URI
2✔
287
        return opts
2✔
288
}
2✔
289

NEW
290
func (opts *Options) SetProxyURI(proxyURI *url.URL) *Options {
×
NEW
291
        opts.ProxyURI = proxyURI
×
NEW
292
        return opts
×
NEW
293
}
×
294

295
func (opts *Options) SetForcePathStyle(forcePathStyle bool) *Options {
2✔
296
        opts.ForcePathStyle = forcePathStyle
2✔
297
        return opts
2✔
298
}
2✔
299

300
func (opts *Options) SetUseAccelerate(useAccelerate bool) *Options {
2✔
301
        opts.UseAccelerate = useAccelerate
2✔
302
        return opts
2✔
303
}
2✔
304

305
func (opts *Options) SetDefaultExpire(defaultExpire time.Duration) *Options {
1✔
306
        opts.DefaultExpire = &defaultExpire
1✔
307
        return opts
1✔
308
}
1✔
309

310
func (opts *Options) SetBufferSize(bufferSize int) *Options {
2✔
311
        opts.BufferSize = &bufferSize
2✔
312
        return opts
2✔
313
}
2✔
314

315
func (opts *Options) SetUnsignedHeaders(unsignedHeaders []string) *Options {
2✔
316
        opts.UnsignedHeaders = unsignedHeaders
2✔
317
        return opts
2✔
318
}
2✔
319

320
func (opts *Options) SetTransport(transport http.RoundTripper) *Options {
4✔
321
        opts.Transport = transport
4✔
322
        return opts
4✔
323
}
4✔
324

325
type apiOptions func(*middleware.Stack) error
326

327
// Google Cloud Storage does not tolerate signing the Accept-Encoding header
328
func unsignedHeadersMiddleware(headers []string) apiOptions {
2✔
329
        signMiddlewareID := (&v4.SignHTTPRequestMiddleware{}).ID()
2✔
330
        for i := range headers {
4✔
331
                headers[i] = textproto.CanonicalMIMEHeaderKey(headers[i])
2✔
332
        }
2✔
333
        return func(stack *middleware.Stack) error {
6✔
334
                if _, ok := stack.Finalize.Get("Signing"); !ok {
5✔
335
                        // If the operation does not invoke signing, we're done.
1✔
336
                        return nil
1✔
337
                }
1✔
338
                // ... -> RemoveUnsignedHeaders -> Signing -> AddUnsignedHeaders
339
                var unsignedHeaders http.Header = make(http.Header)
4✔
340
                err := stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc(
4✔
341
                        "RemoveUnsignedHeaders", func(
4✔
342
                                ctx context.Context,
4✔
343
                                in middleware.FinalizeInput,
4✔
344
                                next middleware.FinalizeHandler,
4✔
345
                        ) (middleware.FinalizeOutput, middleware.Metadata, error) {
8✔
346
                                if req, ok := in.Request.(*smithyhttp.Request); ok {
8✔
347
                                        for _, hdr := range headers {
8✔
348
                                                if value, ok := req.Header[hdr]; ok {
8✔
349
                                                        unsignedHeaders[hdr] = value
4✔
350
                                                        req.Header.Del(hdr)
4✔
351
                                                }
4✔
352
                                        }
353
                                }
354
                                return next.HandleFinalize(ctx, in)
4✔
355
                        }), signMiddlewareID, middleware.Before)
356
                if err != nil {
4✔
357
                        return err
×
358
                }
×
359
                return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc(
4✔
360
                        "AddUnsignedHeaders", func(
4✔
361
                                ctx context.Context,
4✔
362
                                in middleware.FinalizeInput,
4✔
363
                                next middleware.FinalizeHandler,
4✔
364
                        ) (middleware.FinalizeOutput, middleware.Metadata, error) {
8✔
365
                                if req, ok := in.Request.(*smithyhttp.Request); ok {
8✔
366
                                        for key, value := range unsignedHeaders {
8✔
367
                                                req.Header[key] = value
4✔
368
                                        }
4✔
369
                                }
370
                                return next.HandleFinalize(ctx, in)
4✔
371
                        }), signMiddlewareID, middleware.After)
372
        }
373
}
374

375
func (opts *Options) toS3Options() (
376
        clientOpts func(*s3.Options),
377
        presignOpts func(*s3.PresignOptions),
378
) {
5✔
379
        clientOpts = func(s3Opts *s3.Options) {
10✔
380
                opts.options(s3Opts)
5✔
381
                if len(opts.UnsignedHeaders) > 0 {
7✔
382
                        s3Opts.APIOptions = append(
2✔
383
                                s3Opts.APIOptions,
2✔
384
                                unsignedHeadersMiddleware(opts.UnsignedHeaders),
2✔
385
                        )
2✔
386
                }
2✔
387
                roundTripper := opts.Transport
5✔
388
                if roundTripper == nil {
6✔
389
                        roundTripper = &http.Transport{
1✔
390
                                TLSClientConfig: &tls.Config{
1✔
391
                                        RootCAs: storage.GetRootCAs(),
1✔
392
                                },
1✔
393
                        }
1✔
394
                }
1✔
395
                s3Opts.HTTPClient = &http.Client{
5✔
396
                        Transport: roundTripper,
5✔
397
                }
5✔
398
        }
399

400
        expires := DefaultExpire
5✔
401
        if opts.DefaultExpire != nil {
6✔
402
                expires = *opts.DefaultExpire
1✔
403
        }
1✔
404
        presignOpts = func(s3Opts *s3.PresignOptions) {
10✔
405
                opts.presignOptions(s3Opts)
5✔
406
                s3.WithPresignExpires(expires)(s3Opts)
5✔
407
        }
5✔
408
        return clientOpts, presignOpts
5✔
409
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc