920043239

pending completion

Build # 920043239

Build Type

Pull #872

gitlab-ci

Committed by

alfrunes

Commit Message

chore: Restrict tag character set

Changelog: None
Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>

Pull Request Pull Request #872: MEN-6348: Add tags to releases

Run Details

220 of 229 new or added lines in 7 files covered. (96.07%)

223 existing lines in 7 files now uncovered.

7560 of 9480 relevant lines covered (79.75%)

34.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.76

/storage/s3/options.go

// Copyright 2023 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package s3

import (
        "context"
        "crypto/tls"
        "net/http"
        "net/textproto"
        "time"

        "github.com/aws/aws-sdk-go-v2/aws"
        v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
        "github.com/aws/aws-sdk-go-v2/service/s3"
        "github.com/aws/smithy-go/middleware"
        smithyhttp "github.com/aws/smithy-go/transport/http"
        validation "github.com/go-ozzo/ozzo-validation/v4"

        "github.com/mendersoftware/deployments/storage"
)

const (
        kib = 1024
        mib = kib * 1024

        DefaultBufferSize = 10 * mib
        DefaultExpire     = 15 * time.Minute
)

var (
        validAtLeast5MiB = validation.Min(MultipartMinSize).
                Error("must be at least 5MiB")
)

type Options struct {
        // StaticCredentials that overrides AWS config.
        StaticCredentials *StaticCredentials `json:"auth"`

        // Region where the bucket lives
        Region *string
        // ContentType of the uploaded objects
        ContentType *string
        // FilenameSuffix adds the suffix to the content-disposition for object downloads>
        FilenameSuffix *string
        // ExternalURI is the URI used for signing requests.
        ExternalURI *string
        // URI is the URI for the s3 API.
        URI *string

        // ForcePathStyle encodes bucket in the API path.
        ForcePathStyle bool
        // UseAccelerate enables s3 Accelerate
        UseAccelerate bool

        // DefaultExpire is the fallback presign expire duration
        // (defaults to 15min).
        DefaultExpire *time.Duration
        // BufferSize sets the buffer size allocated for uploads.
        // This implicitly sets the upper limit for upload size:
        // BufferSize * 10000 (defaults to: 5MiB).
        BufferSize *int

        // UnsignedHeaders forces the driver to skip the named headers from the
        // being signed.
        UnsignedHeaders []string

        // Transport sets an alternative RoundTripper used by the Go HTTP
        // client.
        Transport http.RoundTripper
}

func NewOptions(opts ...*Options) *Options {
        defaultBufferSize := DefaultBufferSize
        ret := &Options{
                BufferSize: &defaultBufferSize,
        }
        for _, opt := range opts {
                if opt.StaticCredentials != nil {
                        ret.StaticCredentials = opt.StaticCredentials
                }
                if opt.Region != nil {
                        ret.Region = opt.Region
                }
                if opt.ContentType != nil {
                        ret.ContentType = opt.ContentType
                }
                if opt.ExternalURI != nil {
                        ret.ExternalURI = opt.ExternalURI
                }
                if opt.URI != nil {
                        ret.URI = opt.URI
                }
                if opt.ForcePathStyle != ret.ForcePathStyle {
                        ret.ForcePathStyle = opt.ForcePathStyle
                }
                if opt.UseAccelerate != ret.UseAccelerate {
                        ret.UseAccelerate = opt.UseAccelerate
                }
                if opt.DefaultExpire != nil {
                        ret.DefaultExpire = opt.DefaultExpire
                }
                if opt.BufferSize != nil {
                        ret.BufferSize = opt.BufferSize
                }
                if opt.UnsignedHeaders != nil {
                        ret.UnsignedHeaders = opt.UnsignedHeaders
                }
                if opt.Transport != nil {
                        ret.Transport = opt.Transport
                }
        }
        return ret
}

func (opts Options) Validate() error {
        return validation.ValidateStruct(&opts,
                validation.Field(&opts.StaticCredentials),
                validation.Field(&opts.BufferSize, validAtLeast5MiB),
        )
}

func (opts *Options) SetStaticCredentials(key, secret, sessionToken string) *Options {
        opts.StaticCredentials = &StaticCredentials{
                Key:    key,
                Secret: secret,
                Token:  sessionToken,
        }
        return opts
}

func (opts *Options) SetRegion(region string) *Options {
        opts.Region = &region
        return opts
}

func (opts *Options) SetContentType(contentType string) *Options {
        opts.ContentType = &contentType
        return opts
}

func (opts *Options) SetFilenameSuffix(suffix string) *Options {
        opts.FilenameSuffix = &suffix
        return opts
}

func (opts *Options) SetExternalURI(externalURI string) *Options {
        opts.ExternalURI = &externalURI
        return opts
}

func (opts *Options) SetURI(URI string) *Options {
        opts.URI = &URI
        return opts
}

func (opts *Options) SetForcePathStyle(forcePathStyle bool) *Options {
        opts.ForcePathStyle = forcePathStyle
        return opts
}

func (opts *Options) SetUseAccelerate(useAccelerate bool) *Options {
        opts.UseAccelerate = useAccelerate
        return opts
}

func (opts *Options) SetDefaultExpire(defaultExpire time.Duration) *Options {
        opts.DefaultExpire = &defaultExpire
        return opts
}

func (opts *Options) SetBufferSize(bufferSize int) *Options {
        opts.BufferSize = &bufferSize
        return opts
}

func (opts *Options) SetUnsignedHeaders(unsignedHeaders []string) *Options {
        opts.UnsignedHeaders = unsignedHeaders
        return opts
}

func (opts *Options) SetTransport(transport http.RoundTripper) *Options {
        opts.Transport = transport
        return opts
}

type apiOptions func(*middleware.Stack) error

// Google Cloud Storage does not tolerate signing the Accept-Encoding header
func unsignedHeadersMiddleware(headers []string) apiOptions {
        signMiddlewareID := (&v4.SignHTTPRequestMiddleware{}).ID()
        for i := range headers {
                headers[i] = textproto.CanonicalMIMEHeaderKey(headers[i])
        }
        return func(stack *middleware.Stack) error {
                if _, ok := stack.Finalize.Get("Signing"); !ok {
                        // If the operation does not invoke signing, we're done.
                        return nil
                }
                // ... -> RemoveUnsignedHeaders -> Signing -> AddUnsignedHeaders
                var unsignedHeaders http.Header = make(http.Header)
                err := stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc(
                        "RemoveUnsignedHeaders", func(
                                ctx context.Context,
                                in middleware.FinalizeInput,
                                next middleware.FinalizeHandler,
                        ) (middleware.FinalizeOutput, middleware.Metadata, error) {
                                if req, ok := in.Request.(*smithyhttp.Request); ok {
                                        for _, hdr := range headers {
                                                if value, ok := req.Header[hdr]; ok {
                                                        unsignedHeaders[hdr] = value
                                                        req.Header.Del(hdr)
                                                }
                                        }
                                }
                                return next.HandleFinalize(ctx, in)
                        }), signMiddlewareID, middleware.Before)
                if err != nil {
                        return err
                }
                return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc(
                        "AddUnsignedHeaders", func(
                                ctx context.Context,
                                in middleware.FinalizeInput,
                                next middleware.FinalizeHandler,
                        ) (middleware.FinalizeOutput, middleware.Metadata, error) {
                                if req, ok := in.Request.(*smithyhttp.Request); ok {
                                        for key, value := range unsignedHeaders {
                                                req.Header[key] = value
                                        }
                                }
                                return next.HandleFinalize(ctx, in)
                        }), signMiddlewareID, middleware.After)
        }
}

func (opts *Options) toS3Options() (
        clientOpts func(*s3.Options),
        presignOpts func(*s3.PresignOptions),
) {
        clientOpts = func(s3Opts *s3.Options) {
                if opts.StaticCredentials != nil {
                        s3Opts.Credentials = *opts.StaticCredentials
                }
                if opts.Region != nil {
                        s3Opts.Region = *opts.Region
                }
                if len(opts.UnsignedHeaders) > 0 {
                        s3Opts.APIOptions = append(
                                s3Opts.APIOptions,
                                unsignedHeadersMiddleware(opts.UnsignedHeaders),
                        )
                }
                if opts.URI != nil {
                        endpointURI := *opts.URI
                        s3Opts.EndpointResolver = s3.EndpointResolverFromURL(endpointURI,
                                func(ep *aws.Endpoint) {
                                        ep.HostnameImmutable = opts.ForcePathStyle
                                },
                        )
                }
                roundTripper := opts.Transport
                if roundTripper == nil {
                        roundTripper = &http.Transport{
                                TLSClientConfig: &tls.Config{
                                        RootCAs: storage.GetRootCAs(),
                                },
                        }
                }
                s3Opts.UsePathStyle = opts.ForcePathStyle
                s3Opts.UseAccelerate = opts.UseAccelerate
                s3Opts.HTTPClient = &http.Client{
                        Transport: roundTripper,
                }
        }

        expires := DefaultExpire
        if opts.DefaultExpire != nil {
                expires = *opts.DefaultExpire
        }
        presignOpts = func(s3Opts *s3.PresignOptions) {
                s3.WithPresignExpires(expires)(s3Opts)
                if opts.ExternalURI != nil {
                        presignURL := *opts.ExternalURI
                        resolver := s3.EndpointResolverFromURL(presignURL,
                                func(ep *aws.Endpoint) {
                                        ep.HostnameImmutable = opts.ForcePathStyle
                                },
                        )
                        s3.WithPresignClientFromClientOptions(
                                s3.WithEndpointResolver(resolver),
                        )(s3Opts)
                }
        }
        return clientOpts, presignOpts
}

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package s3
16
17	import (
18	"context"
19	"crypto/tls"
20	"net/http"
21	"net/textproto"
22	"time"
23
24	"github.com/aws/aws-sdk-go-v2/aws"
25	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
26	"github.com/aws/aws-sdk-go-v2/service/s3"
27	"github.com/aws/smithy-go/middleware"
28	smithyhttp "github.com/aws/smithy-go/transport/http"
29	validation "github.com/go-ozzo/ozzo-validation/v4"
30
31	"github.com/mendersoftware/deployments/storage"
32	)
33
34	const (
35	kib = 1024
36	mib = kib * 1024
37
38	DefaultBufferSize = 10 * mib
39	DefaultExpire = 15 * time.Minute
40	)
41
42	var (
43	validAtLeast5MiB = validation.Min(MultipartMinSize).
44	Error("must be at least 5MiB")
45	)
46
47	type Options struct {
48	// StaticCredentials that overrides AWS config.
49	StaticCredentials *StaticCredentials `json:"auth"`
50
51	// Region where the bucket lives
52	Region *string
53	// ContentType of the uploaded objects
54	ContentType *string
55	// FilenameSuffix adds the suffix to the content-disposition for object downloads>
56	FilenameSuffix *string
57	// ExternalURI is the URI used for signing requests.
58	ExternalURI *string
59	// URI is the URI for the s3 API.
60	URI *string
61
62	// ForcePathStyle encodes bucket in the API path.
63	ForcePathStyle bool
64	// UseAccelerate enables s3 Accelerate
65	UseAccelerate bool
66
67	// DefaultExpire is the fallback presign expire duration
68	// (defaults to 15min).
69	DefaultExpire *time.Duration
70	// BufferSize sets the buffer size allocated for uploads.
71	// This implicitly sets the upper limit for upload size:
72	// BufferSize * 10000 (defaults to: 5MiB).
73	BufferSize *int
74
75	// UnsignedHeaders forces the driver to skip the named headers from the
76	// being signed.
77	UnsignedHeaders []string
78
79	// Transport sets an alternative RoundTripper used by the Go HTTP
80	// client.
81	Transport http.RoundTripper
82	}
83
84	func NewOptions(opts ...Options) Options {	12✔
85	defaultBufferSize := DefaultBufferSize	12✔
86	ret := &Options{	12✔
87	BufferSize: &defaultBufferSize,	12✔
88	}	12✔
89	for _, opt := range opts {	20✔
90	if opt.StaticCredentials != nil {	16✔
91	ret.StaticCredentials = opt.StaticCredentials	8✔
92	}	8✔
93	if opt.Region != nil {	16✔
94	ret.Region = opt.Region	8✔
95	}	8✔
96	if opt.ContentType != nil {	10✔
97	ret.ContentType = opt.ContentType	2✔
98	}	2✔
99	if opt.ExternalURI != nil {	9✔
100	ret.ExternalURI = opt.ExternalURI	1✔
101	}	1✔
102	if opt.URI != nil {	10✔
103	ret.URI = opt.URI	2✔
104	}	2✔
105	if opt.ForcePathStyle != ret.ForcePathStyle {	9✔
106	ret.ForcePathStyle = opt.ForcePathStyle	1✔
107	}	1✔
108	if opt.UseAccelerate != ret.UseAccelerate {	9✔
109	ret.UseAccelerate = opt.UseAccelerate	1✔
110	}	1✔
111	if opt.DefaultExpire != nil {	9✔
112	ret.DefaultExpire = opt.DefaultExpire	1✔
113	}	1✔
114	if opt.BufferSize != nil {	16✔
115	ret.BufferSize = opt.BufferSize	8✔
116	}	8✔
117	if opt.UnsignedHeaders != nil {	10✔
118	ret.UnsignedHeaders = opt.UnsignedHeaders	2✔
119	}	2✔
120	if opt.Transport != nil {	12✔
121	ret.Transport = opt.Transport	4✔
122	}	4✔
123	}
124	return ret	12✔
125	}
126
127	func (opts Options) Validate() error {	5✔
128	return validation.ValidateStruct(&opts,	5✔
129	validation.Field(&opts.StaticCredentials),	5✔
130	validation.Field(&opts.BufferSize, validAtLeast5MiB),	5✔
131	)	5✔
132	}	5✔
133
134	func (opts Options) SetStaticCredentials(key, secret, sessionToken string) Options {	5✔
135	opts.StaticCredentials = &StaticCredentials{	5✔
136	Key: key,	5✔
137	Secret: secret,	5✔
138	Token: sessionToken,	5✔
139	}	5✔
140	return opts	5✔
141	}	5✔
142
143	func (opts Options) SetRegion(region string) Options {	5✔
144	opts.Region = &region	5✔
145	return opts	5✔
146	}	5✔
147
148	func (opts Options) SetContentType(contentType string) Options {	2✔
149	opts.ContentType = &contentType	2✔
150	return opts	2✔
151	}	2✔
152
UNCOV 153	func (opts Options) SetFilenameSuffix(suffix string) Options {	×
UNCOV 154	opts.FilenameSuffix = &suffix	×
UNCOV 155	return opts	×
UNCOV 156	}	×
157
158	func (opts Options) SetExternalURI(externalURI string) Options {	1✔
159	opts.ExternalURI = &externalURI	1✔
160	return opts	1✔
161	}	1✔
162
163	func (opts Options) SetURI(URI string) Options {	2✔
164	opts.URI = &URI	2✔
165	return opts	2✔
166	}	2✔
167
168	func (opts Options) SetForcePathStyle(forcePathStyle bool) Options {	2✔
169	opts.ForcePathStyle = forcePathStyle	2✔
170	return opts	2✔
171	}	2✔
172
173	func (opts Options) SetUseAccelerate(useAccelerate bool) Options {	2✔
174	opts.UseAccelerate = useAccelerate	2✔
175	return opts	2✔
176	}	2✔
177
178	func (opts Options) SetDefaultExpire(defaultExpire time.Duration) Options {	1✔
179	opts.DefaultExpire = &defaultExpire	1✔
180	return opts	1✔
181	}	1✔
182
183	func (opts Options) SetBufferSize(bufferSize int) Options {	2✔
184	opts.BufferSize = &bufferSize	2✔
185	return opts	2✔
186	}	2✔
187
188	func (opts Options) SetUnsignedHeaders(unsignedHeaders []string) Options {	2✔
189	opts.UnsignedHeaders = unsignedHeaders	2✔
190	return opts	2✔
191	}	2✔
192
193	func (opts Options) SetTransport(transport http.RoundTripper) Options {	4✔
194	opts.Transport = transport	4✔
195	return opts	4✔
196	}	4✔
197
198	type apiOptions func(*middleware.Stack) error
199
200	// Google Cloud Storage does not tolerate signing the Accept-Encoding header
201	func unsignedHeadersMiddleware(headers []string) apiOptions {	2✔
202	signMiddlewareID := (&v4.SignHTTPRequestMiddleware{}).ID()	2✔
203	for i := range headers {	4✔
204	headers[i] = textproto.CanonicalMIMEHeaderKey(headers[i])	2✔
205	}	2✔
206	return func(stack *middleware.Stack) error {	6✔
207	if _, ok := stack.Finalize.Get("Signing"); !ok {	5✔
208	// If the operation does not invoke signing, we're done.	1✔
209	return nil	1✔
210	}	1✔
211	// ... -> RemoveUnsignedHeaders -> Signing -> AddUnsignedHeaders
212	var unsignedHeaders http.Header = make(http.Header)	4✔
213	err := stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc(	4✔
214	"RemoveUnsignedHeaders", func(	4✔
215	ctx context.Context,	4✔
216	in middleware.FinalizeInput,	4✔
217	next middleware.FinalizeHandler,	4✔
218	) (middleware.FinalizeOutput, middleware.Metadata, error) {	8✔
219	if req, ok := in.Request.(*smithyhttp.Request); ok {	8✔
220	for _, hdr := range headers {	8✔
221	if value, ok := req.Header[hdr]; ok {	8✔
222	unsignedHeaders[hdr] = value	4✔
223	req.Header.Del(hdr)	4✔
224	}	4✔
225	}
226	}
227	return next.HandleFinalize(ctx, in)	4✔
228	}), signMiddlewareID, middleware.Before)
229	if err != nil {	4✔
UNCOV 230	return err	×
UNCOV 231	}	×
232	return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc(	4✔
233	"AddUnsignedHeaders", func(	4✔
234	ctx context.Context,	4✔
235	in middleware.FinalizeInput,	4✔
236	next middleware.FinalizeHandler,	4✔
237	) (middleware.FinalizeOutput, middleware.Metadata, error) {	8✔
238	if req, ok := in.Request.(*smithyhttp.Request); ok {	8✔
239	for key, value := range unsignedHeaders {	8✔
240	req.Header[key] = value	4✔
241	}	4✔
242	}
243	return next.HandleFinalize(ctx, in)	4✔
244	}), signMiddlewareID, middleware.After)
245	}
246	}
247
248	func (opts *Options) toS3Options() (
249	clientOpts func(*s3.Options),
250	presignOpts func(*s3.PresignOptions),
251	) {	5✔
252	clientOpts = func(s3Opts *s3.Options) {	10✔
253	if opts.StaticCredentials != nil {	10✔
254	s3Opts.Credentials = *opts.StaticCredentials	5✔
255	}	5✔
256	if opts.Region != nil {	10✔
257	s3Opts.Region = *opts.Region	5✔
258	}	5✔
259	if len(opts.UnsignedHeaders) > 0 {	7✔
260	s3Opts.APIOptions = append(	2✔
261	s3Opts.APIOptions,	2✔
262	unsignedHeadersMiddleware(opts.UnsignedHeaders),	2✔
263	)	2✔
264	}	2✔
265	if opts.URI != nil {	7✔
266	endpointURI := *opts.URI	2✔
267	s3Opts.EndpointResolver = s3.EndpointResolverFromURL(endpointURI,	2✔
268	func(ep *aws.Endpoint) {	4✔
269	ep.HostnameImmutable = opts.ForcePathStyle	2✔
270	},	2✔
271	)
272	}
273	roundTripper := opts.Transport	5✔
274	if roundTripper == nil {	6✔
275	roundTripper = &http.Transport{	1✔
276	TLSClientConfig: &tls.Config{	1✔
277	RootCAs: storage.GetRootCAs(),	1✔
278	},	1✔
279	}	1✔
280	}	1✔
281	s3Opts.UsePathStyle = opts.ForcePathStyle	5✔
282	s3Opts.UseAccelerate = opts.UseAccelerate	5✔
283	s3Opts.HTTPClient = &http.Client{	5✔
284	Transport: roundTripper,	5✔
285	}	5✔
286	}
287
288	expires := DefaultExpire	5✔
289	if opts.DefaultExpire != nil {	6✔
290	expires = *opts.DefaultExpire	1✔
291	}	1✔
292	presignOpts = func(s3Opts *s3.PresignOptions) {	10✔
293	s3.WithPresignExpires(expires)(s3Opts)	5✔
294	if opts.ExternalURI != nil {	6✔
295	presignURL := *opts.ExternalURI	1✔
296	resolver := s3.EndpointResolverFromURL(presignURL,	1✔
297	func(ep *aws.Endpoint) {	2✔
298	ep.HostnameImmutable = opts.ForcePathStyle	1✔
299	},	1✔
300	)
301	s3.WithPresignClientFromClientOptions(	1✔
302	s3.WithEndpointResolver(resolver),	1✔
303	)(s3Opts)	1✔
304	}
305	}
306	return clientOpts, presignOpts	5✔
307	}

mendersoftware / deployments / 920043239

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous