• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

mendersoftware / deployments / 756572627

pending completion
756572627

push

gitlab-ci

GitHub
Merge pull request #815 from alfrunes/MEN-6187

53 of 58 new or added lines in 2 files covered. (91.38%)

77 existing lines in 1 file now uncovered.

6250 of 7963 relevant lines covered (78.49%)

76.47 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.32
/storage/azblob/azblob.go
1
// Copyright 2023 Northern.tech AS
2
//
3
//    Licensed under the Apache License, Version 2.0 (the "License");
4
//    you may not use this file except in compliance with the License.
5
//    You may obtain a copy of the License at
6
//
7
//        http://www.apache.org/licenses/LICENSE-2.0
8
//
9
//    Unless required by applicable law or agreed to in writing, software
10
//    distributed under the License is distributed on an "AS IS" BASIS,
11
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
//    See the License for the specific language governing permissions and
13
//    limitations under the License.
14

15
package azblob
16

17
import (
18
        "context"
19
        "crypto/tls"
20
        "fmt"
21
        "io"
22
        "net/http"
23
        "net/url"
24
        "time"
25

26
        "github.com/mendersoftware/deployments/model"
27
        "github.com/mendersoftware/deployments/storage"
28

29
        "github.com/Azure/azure-sdk-for-go/sdk/azcore"
30
        "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
31
        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
32
        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
33
        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
34
        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
35
        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
36
        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
37
)
38

39
const (
40
        headerBlobType = "x-ms-blob-type"
41

42
        blobTypeBlock = "BlockBlob"
43
)
44

45
type client struct {
46
        DefaultClient *container.Client
47
        credentials   *azblob.SharedKeyCredential
48
        contentType   *string
49
        bufferSize    int64
50
}
51

52
func NewEmpty(ctx context.Context, opts ...*Options) (storage.ObjectStorage, error) {
5✔
53
        opt := NewOptions(opts...)
5✔
54
        objStore := &client{
5✔
55
                bufferSize:  opt.BufferSize,
5✔
56
                contentType: opt.ContentType,
5✔
57
        }
5✔
58
        return objStore, nil
5✔
59
}
5✔
60

61
func New(ctx context.Context, bucket string, opts ...*Options) (storage.ObjectStorage, error) {
4✔
62
        var (
4✔
63
                err    error
4✔
64
                cc     *container.Client
4✔
65
                azCred *azblob.SharedKeyCredential
4✔
66
        )
4✔
67
        opt := NewOptions(opts...)
4✔
68
        objectStorage, err := NewEmpty(ctx, opt)
4✔
69
        if err != nil {
4✔
UNCOV
70
                return nil, err
×
UNCOV
71
        }
×
72
        clientOptions := &container.ClientOptions{
4✔
73
                ClientOptions: azcore.ClientOptions{
4✔
74
                        Transport: &http.Client{
4✔
75
                                Transport: &http.Transport{
4✔
76
                                        TLSClientConfig: &tls.Config{
4✔
77
                                                RootCAs: storage.GetRootCAs(),
4✔
78
                                        },
4✔
79
                                },
4✔
80
                        },
4✔
81
                },
4✔
82
        }
4✔
83
        if opt.ConnectionString != nil {
4✔
NEW
84
                cc, err = container.NewClientFromConnectionString(
×
UNCOV
85
                        *opt.ConnectionString, bucket, clientOptions,
×
UNCOV
86
                )
×
UNCOV
87
                if err == nil {
×
UNCOV
88
                        azCred, err = keyFromConnString(*opt.ConnectionString)
×
UNCOV
89
                }
×
90
        } else if sk := opt.SharedKey; sk != nil {
6✔
91
                var containerURL string
2✔
92
                containerURL, azCred, err = sk.azParams(bucket)
2✔
93
                if err == nil {
4✔
94
                        cc, err = container.NewClientWithSharedKeyCredential(
2✔
95
                                containerURL,
2✔
96
                                azCred,
2✔
97
                                clientOptions,
2✔
98
                        )
2✔
99
                }
2✔
100
        }
101
        if err != nil {
4✔
UNCOV
102
                return nil, err
×
UNCOV
103
        }
×
104
        objectStorage.(*client).DefaultClient = cc
4✔
105
        objectStorage.(*client).credentials = azCred
4✔
106
        if err := objectStorage.HealthCheck(ctx); err != nil {
4✔
UNCOV
107
                return nil, err
×
UNCOV
108
        }
×
109
        return objectStorage, nil
4✔
110
}
111

112
func (c *client) clientFromContext(
113
        ctx context.Context,
114
) (client *container.Client, err error) {
48✔
115
        client = c.DefaultClient
48✔
116
        if settings := storage.SettingsFromContext(ctx); settings != nil {
70✔
117
                if err = settings.Validate(); err != nil {
22✔
UNCOV
118
                        return nil, err
×
119
                } else if settings.ConnectionString != nil {
22✔
NEW
120
                        client, err = container.NewClientFromConnectionString(
×
UNCOV
121
                                *settings.ConnectionString,
×
UNCOV
122
                                settings.Bucket,
×
NEW
123
                                &container.ClientOptions{},
×
UNCOV
124
                        )
×
125
                } else {
22✔
126
                        var (
22✔
127
                                containerURL string
22✔
128
                                azCreds      *azblob.SharedKeyCredential
22✔
129
                        )
22✔
130
                        creds := SharedKeyCredentials{
22✔
131
                                AccountName: settings.Key,
22✔
132
                                AccountKey:  settings.Secret,
22✔
133
                        }
22✔
134
                        if settings.Uri != "" {
22✔
UNCOV
135
                                creds.URI = &settings.Uri
×
UNCOV
136
                        }
×
137

138
                        containerURL, azCreds, err = creds.azParams(settings.Bucket)
22✔
139
                        if err == nil {
44✔
140
                                client, err = container.NewClientWithSharedKeyCredential(
22✔
141
                                        containerURL,
22✔
142
                                        azCreds,
22✔
143
                                        &container.ClientOptions{},
22✔
144
                                )
22✔
145
                        }
22✔
146
                }
147
        }
148
        return client, err
48✔
149
}
150

151
func (c *client) HealthCheck(ctx context.Context) error {
4✔
152
        azClient, err := c.clientFromContext(ctx)
4✔
153
        if err != nil {
4✔
UNCOV
154
                return OpError{
×
UNCOV
155
                        Op:     OpHealthCheck,
×
UNCOV
156
                        Reason: err,
×
UNCOV
157
                }
×
158
        } else if azClient == nil {
6✔
159
                return nil
2✔
160
        }
2✔
161
        _, err = azClient.GetProperties(ctx, &container.GetPropertiesOptions{})
2✔
162
        if err != nil {
2✔
163
                return OpError{
×
164
                        Op:     OpHealthCheck,
×
165
                        Reason: err,
×
UNCOV
166
                }
×
UNCOV
167
        }
×
168
        return nil
2✔
169
}
170

171
func (c *client) PutObject(
172
        ctx context.Context,
173
        objectPath string,
174
        src io.Reader,
175
) error {
8✔
176
        azClient, err := c.clientFromContext(ctx)
8✔
177
        if err != nil {
8✔
UNCOV
178
                return OpError{
×
UNCOV
179
                        Op:     OpPutObject,
×
UNCOV
180
                        Reason: err,
×
UNCOV
181
                }
×
182
        } else if azClient == nil {
8✔
UNCOV
183
                return nil
×
UNCOV
184
        }
×
185
        bc := azClient.NewBlockBlobClient(objectPath)
8✔
186
        var blobOpts = &blockblob.UploadStreamOptions{
8✔
187
                HTTPHeaders: &blob.HTTPHeaders{
8✔
188
                        BlobContentType: c.contentType,
8✔
189
                },
8✔
190
        }
8✔
191
        blobOpts.BlockSize = c.bufferSize
8✔
192
        _, err = bc.UploadStream(ctx, src, blobOpts)
8✔
193
        if err != nil {
8✔
UNCOV
194
                return OpError{
×
195
                        Op:      OpPutObject,
×
196
                        Message: "failed to upload object to blob",
×
197
                        Reason:  err,
×
198
                }
×
199
        }
×
200
        return err
8✔
201
}
202

203
func (c *client) DeleteObject(
204
        ctx context.Context,
205
        path string,
206
) error {
8✔
207
        azClient, err := c.clientFromContext(ctx)
8✔
208
        if err != nil {
8✔
UNCOV
209
                return OpError{
×
210
                        Op:     OpDeleteObject,
×
211
                        Reason: err,
×
212
                }
×
213
        } else if azClient == nil {
8✔
UNCOV
214
                return nil
×
215
        }
×
216
        bc := azClient.NewBlockBlobClient(path)
8✔
217
        _, err = bc.Delete(ctx, &blob.DeleteOptions{
8✔
218
                DeleteSnapshots: to.Ptr(azblob.DeleteSnapshotsOptionTypeInclude),
8✔
219
        })
8✔
220
        if bloberror.HasCode(err,
8✔
221
                bloberror.BlobNotFound,
8✔
222
                bloberror.ContainerNotFound,
8✔
223
                bloberror.ResourceNotFound) {
12✔
224
                err = storage.ErrObjectNotFound
4✔
225
        }
4✔
226
        if err != nil {
12✔
227
                return OpError{
4✔
228
                        Op:      OpDeleteObject,
4✔
229
                        Message: "failed to delete object",
4✔
230
                        Reason:  err,
4✔
231
                }
4✔
232
        }
4✔
233
        return nil
4✔
234
}
235

236
func (c *client) StatObject(
237
        ctx context.Context,
238
        path string,
239
) (*storage.ObjectInfo, error) {
12✔
240
        azClient, err := c.clientFromContext(ctx)
12✔
241
        if err != nil {
12✔
242
                return nil, OpError{
×
UNCOV
243
                        Op:     OpStatObject,
×
UNCOV
244
                        Reason: err,
×
245
                }
×
246
        } else if azClient == nil {
12✔
247
                return nil, nil
×
248
        }
×
249
        bc := azClient.NewBlockBlobClient(path)
12✔
250
        if err != nil {
12✔
UNCOV
251
                return nil, OpError{
×
UNCOV
252
                        Op:      OpStatObject,
×
UNCOV
253
                        Message: "failed to initialize blob client",
×
UNCOV
254
                        Reason:  err,
×
UNCOV
255
                }
×
UNCOV
256
        }
×
257
        rsp, err := bc.GetProperties(ctx, &blob.GetPropertiesOptions{})
12✔
258
        if bloberror.HasCode(err,
12✔
259
                bloberror.BlobNotFound,
12✔
260
                bloberror.ContainerNotFound,
12✔
261
                bloberror.ResourceNotFound,
12✔
262
        ) {
16✔
263
                err = storage.ErrObjectNotFound
4✔
264
        }
4✔
265
        if err != nil {
16✔
266
                return nil, OpError{
4✔
267
                        Op:      OpStatObject,
4✔
268
                        Message: "failed to retrieve object properties",
4✔
269
                        Reason:  err,
4✔
270
                }
4✔
271
        }
4✔
272
        return &storage.ObjectInfo{
8✔
273
                Path:         path,
8✔
274
                LastModified: rsp.LastModified,
8✔
275
                Size:         rsp.ContentLength,
8✔
276
        }, nil
8✔
277
}
278

279
func buildSignedURL(
280
        blobURL string,
281
        SASParams sas.QueryParameters,
282
) (string, error) {
4✔
283
        baseURL, err := url.Parse(blobURL)
4✔
284
        if err != nil {
4✔
285
                return "", err
×
UNCOV
286
        }
×
287
        qSAS, err := url.ParseQuery(SASParams.Encode())
4✔
288
        if err != nil {
4✔
UNCOV
289
                return "", err
×
UNCOV
290
        }
×
291
        q := baseURL.Query()
4✔
292
        for key, values := range qSAS {
32✔
293
                for _, value := range values {
56✔
294
                        q.Add(key, value)
28✔
295
                }
28✔
296
        }
297
        baseURL.RawQuery = q.Encode()
4✔
298
        return baseURL.String(), nil
4✔
299
}
300

301
func (c *client) GetRequest(
302
        ctx context.Context,
303
        objectPath string,
304
        filename string,
305
        duration time.Duration,
306
) (*model.Link, error) {
8✔
307
        azClient, err := c.clientFromContext(ctx)
8✔
308
        if err != nil {
8✔
309
                return nil, OpError{
×
310
                        Op:     OpGetRequest,
×
UNCOV
311
                        Reason: err,
×
UNCOV
312
                }
×
313
        } else if azClient == nil {
8✔
314
                return nil, nil
×
315
        }
×
316
        // Check if object exists
317
        bc := azClient.NewBlockBlobClient(objectPath)
8✔
318
        if err != nil {
8✔
319
                return nil, OpError{
×
UNCOV
320
                        Op:      OpGetRequest,
×
UNCOV
321
                        Message: "failed to initialize blob client",
×
UNCOV
322
                        Reason:  err,
×
UNCOV
323
                }
×
UNCOV
324
        }
×
325
        _, err = bc.GetProperties(ctx, &blob.GetPropertiesOptions{})
8✔
326
        if bloberror.HasCode(err,
8✔
327
                bloberror.BlobNotFound,
8✔
328
                bloberror.ContainerNotFound,
8✔
329
                bloberror.ResourceNotFound,
8✔
330
        ) {
12✔
331
                err = storage.ErrObjectNotFound
4✔
332
        }
4✔
333
        if err != nil {
12✔
334
                return nil, OpError{
4✔
335
                        Op:      OpGetRequest,
4✔
336
                        Message: "failed to check preconditions",
4✔
337
                        Reason:  err,
4✔
338
                }
4✔
339
        }
4✔
340
        now := time.Now().UTC()
4✔
341
        exp := now.Add(duration)
4✔
342
        // HACK: We cannot use BlockBlobClient.GetSASToken because the API does
4✔
343
        // not expose the required parameters.
4✔
344
        urlParts, _ := blob.ParseURL(bc.URL())
4✔
345
        sk, err := c.credentialsFromContext(ctx)
4✔
346
        if err != nil {
4✔
347
                return nil, OpError{
×
348
                        Op:      OpGetRequest,
×
UNCOV
349
                        Message: "failed to retrieve credentials",
×
UNCOV
350
                        Reason:  err,
×
UNCOV
351
                }
×
UNCOV
352
        }
×
353
        var contentDisposition string
4✔
354
        if filename != "" {
8✔
355
                contentDisposition = fmt.Sprintf(
4✔
356
                        `attachment; filename="%s"`, filename,
4✔
357
                )
4✔
358
        }
4✔
359
        permissions := &sas.BlobPermissions{
4✔
360
                Read: true,
4✔
361
        }
4✔
362
        qParams, err := sas.BlobSignatureValues{
4✔
363
                ContainerName: urlParts.ContainerName,
4✔
364
                BlobName:      urlParts.BlobName,
4✔
365

4✔
366
                Permissions:        permissions.String(),
4✔
367
                ContentDisposition: contentDisposition,
4✔
368

4✔
369
                StartTime:  now.UTC(),
4✔
370
                ExpiryTime: exp.UTC(),
4✔
371
        }.SignWithSharedKey(sk)
4✔
372
        if err != nil {
4✔
373
                return nil, OpError{
×
374
                        Op:      OpGetRequest,
×
375
                        Message: "failed to build signed URL",
×
UNCOV
376
                        Reason:  err,
×
UNCOV
377
                }
×
378
        }
×
379
        uri, err := buildSignedURL(bc.URL(), qParams)
4✔
380
        if err != nil {
4✔
381
                return nil, OpError{
×
382
                        Op:      OpGetRequest,
×
383
                        Message: "failed to create pre-signed URL",
×
UNCOV
384
                        Reason:  err,
×
UNCOV
385
                }
×
UNCOV
386
        }
×
387
        return &model.Link{
4✔
388
                Uri:    uri,
4✔
389
                Expire: exp,
4✔
390
                Method: http.MethodGet,
4✔
391
        }, nil
4✔
392
}
393

394
func (c *client) DeleteRequest(
395
        ctx context.Context,
396
        path string,
397
        duration time.Duration,
398
) (*model.Link, error) {
4✔
399
        azClient, err := c.clientFromContext(ctx)
4✔
400
        if err != nil {
4✔
401
                return nil, OpError{
×
UNCOV
402
                        Op:     OpGetRequest,
×
403
                        Reason: err,
×
404
                }
×
405
        } else if azClient == nil {
4✔
UNCOV
406
                return nil, nil
×
407
        }
×
408
        bc := azClient.NewBlobClient(path)
4✔
409
        if err != nil {
4✔
410
                return nil, OpError{
×
411
                        Op:      OpDeleteRequest,
×
412
                        Message: "failed to initialize blob client",
×
UNCOV
413
                        Reason:  err,
×
UNCOV
414
                }
×
UNCOV
415
        }
×
416
        now := time.Now().UTC()
4✔
417
        exp := now.Add(duration)
4✔
418
        uri, err := bc.GetSASURL(sas.BlobPermissions{Delete: true}, now, exp)
4✔
419
        if err != nil {
4✔
420
                return nil, OpError{
×
421
                        Op:      OpDeleteRequest,
×
NEW
422
                        Message: "failed to generate signed URL",
×
UNCOV
423
                        Reason:  err,
×
UNCOV
424
                }
×
UNCOV
425
        }
×
426

427
        return &model.Link{
4✔
428
                Uri:    uri,
4✔
429
                Expire: exp,
4✔
430
                Method: http.MethodDelete,
4✔
431
        }, nil
4✔
432
}
433

434
func (c *client) PutRequest(
435
        ctx context.Context,
436
        objectPath string,
437
        duration time.Duration,
438
) (*model.Link, error) {
4✔
439
        azClient, err := c.clientFromContext(ctx)
4✔
440
        if err != nil {
4✔
441
                return nil, OpError{
×
442
                        Op:     OpGetRequest,
×
443
                        Reason: err,
×
444
                }
×
445
        } else if azClient == nil {
4✔
UNCOV
446
                return nil, nil
×
UNCOV
447
        }
×
448
        bc := azClient.NewBlobClient(objectPath)
4✔
449
        if err != nil {
4✔
UNCOV
450
                return nil, OpError{
×
UNCOV
451
                        Op:      OpPutRequest,
×
UNCOV
452
                        Message: "failed to initialize blob client",
×
453
                        Reason:  err,
×
454
                }
×
455
        }
×
456
        now := time.Now().UTC()
4✔
457
        exp := now.Add(duration)
4✔
458
        uri, err := bc.GetSASURL(sas.BlobPermissions{
4✔
459
                Create: true,
4✔
460
                Write:  true,
4✔
461
        }, now, exp)
4✔
462
        if err != nil {
4✔
UNCOV
463
                return nil, OpError{
×
UNCOV
464
                        Op:      OpPutRequest,
×
NEW
465
                        Message: "failed to generate signed URL",
×
UNCOV
466
                        Reason:  err,
×
UNCOV
467
                }
×
UNCOV
468
        }
×
469
        hdrs := map[string]string{
4✔
470
                headerBlobType: blobTypeBlock,
4✔
471
        }
4✔
472
        return &model.Link{
4✔
473
                Uri:    uri,
4✔
474
                Expire: exp,
4✔
475
                Method: http.MethodPut,
4✔
476
                Header: hdrs,
4✔
477
        }, nil
4✔
478
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc