1769701362

Committed 15 Apr 2025 01:54PM UTC coverage: 65.264%. First build

Build # 1769701362

Build Type

Pull #202

gitlab-ci

Committed by

alfrunes

Commit Message

test: Fix deviceauth.cache tests after changing client initialization

Signed-off-by: Alf-Rune Siqveland <alf.rune@northern.tech>

Pull Request Pull Request #202: MEN-7733: Rate limits for devices APIs

Run Details

144 of 389 new or added lines in 8 files covered. (37.02%)

31803 of 48730 relevant lines covered (65.26%)

1.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.2

/backend/services/deviceauth/server.go

// Copyright 2023 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package main

import (
        "context"
        "fmt"
        "net/http"
        "os"
        "os/signal"
        "reflect"
        "strconv"
        "strings"
        "time"

        "github.com/pkg/errors"
        "golang.org/x/sys/unix"

        "github.com/mendersoftware/mender-server/pkg/config"
        "github.com/mendersoftware/mender-server/pkg/log"
        "github.com/mendersoftware/mender-server/pkg/redis"

        api_http "github.com/mendersoftware/mender-server/services/deviceauth/api/http"
        "github.com/mendersoftware/mender-server/services/deviceauth/cache"
        "github.com/mendersoftware/mender-server/services/deviceauth/client/orchestrator"
        "github.com/mendersoftware/mender-server/services/deviceauth/client/tenant"
        dconfig "github.com/mendersoftware/mender-server/services/deviceauth/config"
        "github.com/mendersoftware/mender-server/services/deviceauth/devauth"
        "github.com/mendersoftware/mender-server/services/deviceauth/jwt"
        "github.com/mendersoftware/mender-server/services/deviceauth/store/mongo"
)

func RunServer(c config.Reader) error {
        var tenantadmAddr = c.GetString(dconfig.SettingTenantAdmAddr)

        l := log.New(log.Ctx{})

        db, err := mongo.NewDataStoreMongo(
                mongo.DataStoreMongoConfig{
                        ConnectionString: c.GetString(dconfig.SettingDb),

                        SSL:           c.GetBool(dconfig.SettingDbSSL),
                        SSLSkipVerify: c.GetBool(dconfig.SettingDbSSLSkipVerify),

                        Username: c.GetString(dconfig.SettingDbUsername),
                        Password: c.GetString(dconfig.SettingDbPassword),
                })
        if err != nil {
                return errors.Wrap(err, "database connection failed")
        }

        jwtHandler, err := jwt.NewJWTHandler(
                c.GetString(dconfig.SettingServerPrivKeyPath),
        )
        var jwtFallbackHandler jwt.Handler
        fallback := c.GetString(dconfig.SettingServerFallbackPrivKeyPath)
        if err == nil && fallback != "" {
                jwtFallbackHandler, err = jwt.NewJWTHandler(
                        fallback,
                )
        }
        if err != nil {
                return err
        }

        orchClientConf := orchestrator.Config{
                OrchestratorAddr: c.GetString(dconfig.SettingOrchestratorAddr),
                Timeout:          time.Duration(30) * time.Second,
        }

        devauth := devauth.NewDevAuth(db,
                orchestrator.NewClient(orchClientConf),
                jwtHandler,
                devauth.Config{
                        Issuer:             c.GetString(dconfig.SettingJWTIssuer),
                        ExpirationTime:     int64(c.GetInt(dconfig.SettingJWTExpirationTimeout)),
                        DefaultTenantToken: c.GetString(dconfig.SettingDefaultTenantToken),
                        InventoryAddr:      config.Config.GetString(dconfig.SettingInventoryAddr),

                        EnableReporting: config.Config.GetBool(dconfig.SettingEnableReporting),
                        HaveAddons: config.Config.GetBool(dconfig.SettingHaveAddons) &&
                                tenantadmAddr != "",
                })

        if jwtFallbackHandler != nil {
                devauth = devauth.WithJWTFallbackHandler(jwtFallbackHandler)
        }

        if tenantadmAddr != "" {
                tc := tenant.NewClient(tenant.Config{
                        TenantAdmAddr: tenantadmAddr,
                })
                devauth = devauth.WithTenantVerification(tc)
        }

        cacheConnStr := c.GetString(dconfig.SettingRedisConnectionString)
        if cacheConnStr == "" {
                // for backward compatibility check old redis_addr setting
                cacheConnStr = c.GetString(dconfig.SettingRedisAddr)
        }
        if cacheConnStr != "" {
                l.Infof("setting up redis cache")

                ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
                redisClient, err := redis.ClientFromConnectionString(ctx, cacheConnStr)
                cancel()
                if err != nil {
                        return fmt.Errorf("failed to initialize redis client: %w", err)
                }

                redisKeyPrefix := c.GetString(dconfig.SettingRedisKeyPrefix)
                cache := cache.NewRedisCache(
                        redisClient,
                        redisKeyPrefix,
                        c.GetInt(dconfig.SettingRedisLimitsExpSec),
                )
                devauth = devauth.WithCache(cache)
                err = setupRatelimits(c, devauth, redisKeyPrefix, redisClient)
                if err != nil {
                        return fmt.Errorf("error configuring rate limits: %w", err)
                }
        }

        devauthapi := api_http.NewDevAuthApiHandlers(devauth, db)

        apiHandler, err := devauthapi.Build()
        if err != nil {
                return errors.Wrap(err, "device authentication API handlers setup failed")
        }

        addr := c.GetString(dconfig.SettingListen)
        l.Printf("listening on %s", addr)

        srv := &http.Server{
                Addr:    addr,
                Handler: apiHandler,
        }

        errChan := make(chan error, 1)
        go func() {
                if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
                        errChan <- err
                }
        }()
        quit := make(chan os.Signal, 1)
        signal.Notify(quit, unix.SIGINT, unix.SIGTERM)
        select {
        case sig := <-quit:
                l.Infof("received signal %s: terminating", sig)
        case err = <-errChan:
                l.Errorf("server terminated unexpectedly: %s", err.Error())
                return err
        }

        l.Info("server shutdown")
        ctxWithTimeout, cancel := context.WithTimeout(context.Background(), 5*time.Second)
        defer cancel()
        if err := srv.Shutdown(ctxWithTimeout); err != nil {
                l.Error("error when shutting down the server ", err)
        }
        return nil
}

func setupRatelimits(
        c config.Reader,
        devauth *devauth.DevAuth,
        redisKeyPrefix string,
        redisClient redis.Client,
) error {
        if !c.IsSet(dconfig.SettingRatelimitsQuotas) {
                return nil
        }
        quotas := make(map[string]float64)
        // quotas can be given as either "plan=quota plan2=quota2"
        // or as a map of string -> float64
        // Only the former can be backed by environment variables
        quotaSlice := c.GetStringSlice(dconfig.SettingRatelimitsQuotas)
        if len(quotaSlice) > 0 {
                for i, keyValue := range quotaSlice {
                        key, value, ok := strings.Cut(keyValue, "=")
                        if !ok {
                                return fmt.Errorf(
                                        `invalid config %s: value %v item #%d: missing key/value separator '='`,
                                        dconfig.SettingRatelimitsQuotas, quotaSlice, i+1,
                                )
                        }
                        valueF64, err := strconv.ParseFloat(value, 64)
                        if err != nil {
                                return fmt.Errorf("error parsing quota value: %w", err)
                        }
                        quotas[key] = valueF64
                }
        } else {
                // Check for map in config file
                quotaMap := c.GetStringMap(dconfig.SettingRatelimitsQuotas)
                if len(quotaMap) == 0 {
                        return fmt.Errorf(
                                "invalid config value %s: cannot be empty",
                                dconfig.SettingRatelimitsQuotas)
                }
                for key, valueAny := range quotaMap {
                        rVal := reflect.ValueOf(valueAny)
                        if rVal.CanFloat() {
                                quotas[key] = rVal.Float()
                        } else if rVal.CanInt() {
                                quotas[key] = float64(rVal.Int())
                        } else if rVal.CanUint() {
                                quotas[key] = float64(rVal.Uint())
                        } else {
                                return fmt.Errorf(
                                        "invalid config value %s[%s]: not a numeric value",
                                        dconfig.SettingRatelimitsQuotas, key,
                                )
                        }
                }
        }
        for key := range quotas {
                if quotas[key] < 0.0 {
                        return fmt.Errorf("invalid config value %s[%s]: value must be a positive value",
                                dconfig.SettingRatelimitsQuotas, key)
                }
        }
        log.NewEmpty().Infof("using rate limit quotas: %v", quotas)

        interval := c.GetDuration(dconfig.SettingRatelimitsInterval)
        rateLimiter := redis.NewFixedWindowRateLimiter(redisClient,
                func(ctx context.Context) (*redis.RatelimitParams, error) {
                        limit, eventID, err := devauth.RateLimitsFromContext(ctx)
                        if err != nil {
                                return nil, err
                        } else if limit < 0 {
                                return nil, nil
                        }
                        keyPrefix := redisKeyPrefix + ":" + eventID
                        return &redis.RatelimitParams{
                                Burst:     uint64(limit),
                                Interval:  interval,
                                KeyPrefix: keyPrefix,
                        }, nil
                },
        )
        devauth.WithRatelimits(
                rateLimiter,
                quotas,
                c.GetFloat64(dconfig.SettingRatelimitsQuotaDefault),
        )
        return nil
}

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package main
16
17	import (
18	"context"
19	"fmt"
20	"net/http"
21	"os"
22	"os/signal"
23	"reflect"
24	"strconv"
25	"strings"
26	"time"
27
28	"github.com/pkg/errors"
29	"golang.org/x/sys/unix"
30
31	"github.com/mendersoftware/mender-server/pkg/config"
32	"github.com/mendersoftware/mender-server/pkg/log"
33	"github.com/mendersoftware/mender-server/pkg/redis"
34
35	api_http "github.com/mendersoftware/mender-server/services/deviceauth/api/http"
36	"github.com/mendersoftware/mender-server/services/deviceauth/cache"
37	"github.com/mendersoftware/mender-server/services/deviceauth/client/orchestrator"
38	"github.com/mendersoftware/mender-server/services/deviceauth/client/tenant"
39	dconfig "github.com/mendersoftware/mender-server/services/deviceauth/config"
40	"github.com/mendersoftware/mender-server/services/deviceauth/devauth"
41	"github.com/mendersoftware/mender-server/services/deviceauth/jwt"
42	"github.com/mendersoftware/mender-server/services/deviceauth/store/mongo"
43	)
44
45	func RunServer(c config.Reader) error {	2✔
46	var tenantadmAddr = c.GetString(dconfig.SettingTenantAdmAddr)	2✔
47		2✔
48	l := log.New(log.Ctx{})	2✔
49		2✔
50	db, err := mongo.NewDataStoreMongo(	2✔
51	mongo.DataStoreMongoConfig{	2✔
52	ConnectionString: c.GetString(dconfig.SettingDb),	2✔
53		2✔
54	SSL: c.GetBool(dconfig.SettingDbSSL),	2✔
55	SSLSkipVerify: c.GetBool(dconfig.SettingDbSSLSkipVerify),	2✔
56		2✔
57	Username: c.GetString(dconfig.SettingDbUsername),	2✔
58	Password: c.GetString(dconfig.SettingDbPassword),	2✔
59	})	2✔
60	if err != nil {	2✔
61	return errors.Wrap(err, "database connection failed")	×
62	}	×
63
64	jwtHandler, err := jwt.NewJWTHandler(	2✔
65	c.GetString(dconfig.SettingServerPrivKeyPath),	2✔
66	)	2✔
67	var jwtFallbackHandler jwt.Handler	2✔
68	fallback := c.GetString(dconfig.SettingServerFallbackPrivKeyPath)	2✔
69	if err == nil && fallback != "" {	2✔
70	jwtFallbackHandler, err = jwt.NewJWTHandler(	×
71	fallback,	×
72	)	×
73	}	×
74	if err != nil {	2✔
75	return err	×
76	}	×
77
78	orchClientConf := orchestrator.Config{	2✔
79	OrchestratorAddr: c.GetString(dconfig.SettingOrchestratorAddr),	2✔
80	Timeout: time.Duration(30) * time.Second,	2✔
81	}	2✔
82		2✔
83	devauth := devauth.NewDevAuth(db,	2✔
84	orchestrator.NewClient(orchClientConf),	2✔
85	jwtHandler,	2✔
86	devauth.Config{	2✔
87	Issuer: c.GetString(dconfig.SettingJWTIssuer),	2✔
88	ExpirationTime: int64(c.GetInt(dconfig.SettingJWTExpirationTimeout)),	2✔
89	DefaultTenantToken: c.GetString(dconfig.SettingDefaultTenantToken),	2✔
90	InventoryAddr: config.Config.GetString(dconfig.SettingInventoryAddr),	2✔
91		2✔
92	EnableReporting: config.Config.GetBool(dconfig.SettingEnableReporting),	2✔
93	HaveAddons: config.Config.GetBool(dconfig.SettingHaveAddons) &&	2✔
94	tenantadmAddr != "",	2✔
95	})	2✔
96		2✔
97	if jwtFallbackHandler != nil {	2✔
98	devauth = devauth.WithJWTFallbackHandler(jwtFallbackHandler)	×
99	}	×
100
101	if tenantadmAddr != "" {	2✔
102	tc := tenant.NewClient(tenant.Config{	×
103	TenantAdmAddr: tenantadmAddr,	×
104	})	×
105	devauth = devauth.WithTenantVerification(tc)	×
106	}	×
107
108	cacheConnStr := c.GetString(dconfig.SettingRedisConnectionString)	2✔
109	if cacheConnStr == "" {	4✔
110	// for backward compatibility check old redis_addr setting	2✔
111	cacheConnStr = c.GetString(dconfig.SettingRedisAddr)	2✔
112	}	2✔
113	if cacheConnStr != "" {	2✔
114	l.Infof("setting up redis cache")	×
115		×
NEW 116	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)	×
NEW 117	redisClient, err := redis.ClientFromConnectionString(ctx, cacheConnStr)	×
NEW 118	cancel()	×
119	if err != nil {	×
NEW 120	return fmt.Errorf("failed to initialize redis client: %w", err)	×
121	}	×
122
NEW 123	redisKeyPrefix := c.GetString(dconfig.SettingRedisKeyPrefix)	×
NEW 124	cache := cache.NewRedisCache(	×
NEW 125	redisClient,	×
NEW 126	redisKeyPrefix,	×
NEW 127	c.GetInt(dconfig.SettingRedisLimitsExpSec),	×
NEW 128	)	×
129	devauth = devauth.WithCache(cache)	×
NEW 130	err = setupRatelimits(c, devauth, redisKeyPrefix, redisClient)	×
NEW 131	if err != nil {	×
NEW 132	return fmt.Errorf("error configuring rate limits: %w", err)	×
NEW 133	}	×
134	}
135
136	devauthapi := api_http.NewDevAuthApiHandlers(devauth, db)	2✔
137		2✔
138	apiHandler, err := devauthapi.Build()	2✔
139	if err != nil {	2✔
140	return errors.Wrap(err, "device authentication API handlers setup failed")	×
141	}	×
142
143	addr := c.GetString(dconfig.SettingListen)	2✔
144	l.Printf("listening on %s", addr)	2✔
145		2✔
146	srv := &http.Server{	2✔
147	Addr: addr,	2✔
148	Handler: apiHandler,	2✔
149	}	2✔
150		2✔
151	errChan := make(chan error, 1)	2✔
152	go func() {	4✔
153	if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {	2✔
154	errChan <- err	×
155	}	×
156	}()
157	quit := make(chan os.Signal, 1)	2✔
158	signal.Notify(quit, unix.SIGINT, unix.SIGTERM)	2✔
159	select {	2✔
160	case sig := <-quit:	2✔
161	l.Infof("received signal %s: terminating", sig)	2✔
162	case err = <-errChan:	×
163	l.Errorf("server terminated unexpectedly: %s", err.Error())	×
164	return err	×
165	}
166
167	l.Info("server shutdown")	2✔
168	ctxWithTimeout, cancel := context.WithTimeout(context.Background(), 5*time.Second)	2✔
169	defer cancel()	2✔
170	if err := srv.Shutdown(ctxWithTimeout); err != nil {	2✔
171	l.Error("error when shutting down the server ", err)	×
172	}	×
173	return nil	2✔
174	}
175
176	func setupRatelimits(
177	c config.Reader,
178	devauth *devauth.DevAuth,
179	redisKeyPrefix string,
180	redisClient redis.Client,
181	) error {	1✔
182	if !c.IsSet(dconfig.SettingRatelimitsQuotas) {	2✔
183	return nil	1✔
184	}	1✔
185	quotas := make(map[string]float64)	1✔
186	// quotas can be given as either "plan=quota plan2=quota2"	1✔
187	// or as a map of string -> float64	1✔
188	// Only the former can be backed by environment variables	1✔
189	quotaSlice := c.GetStringSlice(dconfig.SettingRatelimitsQuotas)	1✔
190	if len(quotaSlice) > 0 {	2✔
191	for i, keyValue := range quotaSlice {	2✔
192	key, value, ok := strings.Cut(keyValue, "=")	1✔
193	if !ok {	2✔
194	return fmt.Errorf(	1✔
195	`invalid config %s: value %v item #%d: missing key/value separator '='`,	1✔
196	dconfig.SettingRatelimitsQuotas, quotaSlice, i+1,	1✔
197	)	1✔
198	}	1✔
199	valueF64, err := strconv.ParseFloat(value, 64)	1✔
200	if err != nil {	2✔
201	return fmt.Errorf("error parsing quota value: %w", err)	1✔
202	}	1✔
203	quotas[key] = valueF64	1✔
204	}
205	} else {	1✔
206	// Check for map in config file	1✔
207	quotaMap := c.GetStringMap(dconfig.SettingRatelimitsQuotas)	1✔
208	if len(quotaMap) == 0 {	2✔
209	return fmt.Errorf(	1✔
210	"invalid config value %s: cannot be empty",	1✔
211	dconfig.SettingRatelimitsQuotas)	1✔
212	}	1✔
213	for key, valueAny := range quotaMap {	2✔
214	rVal := reflect.ValueOf(valueAny)	1✔
215	if rVal.CanFloat() {	2✔
216	quotas[key] = rVal.Float()	1✔
217	} else if rVal.CanInt() {	3✔
218	quotas[key] = float64(rVal.Int())	1✔
219	} else if rVal.CanUint() {	3✔
220	quotas[key] = float64(rVal.Uint())	1✔
221	} else {	2✔
222	return fmt.Errorf(	1✔
223	"invalid config value %s[%s]: not a numeric value",	1✔
224	dconfig.SettingRatelimitsQuotas, key,	1✔
225	)	1✔
226	}	1✔
227	}
228	}
229	for key := range quotas {	2✔
230	if quotas[key] < 0.0 {	2✔
231	return fmt.Errorf("invalid config value %s[%s]: value must be a positive value",	1✔
232	dconfig.SettingRatelimitsQuotas, key)	1✔
233	}	1✔
234	}
235	log.NewEmpty().Infof("using rate limit quotas: %v", quotas)	1✔
236		1✔
237	interval := c.GetDuration(dconfig.SettingRatelimitsInterval)	1✔
238	rateLimiter := redis.NewFixedWindowRateLimiter(redisClient,	1✔
239	func(ctx context.Context) (*redis.RatelimitParams, error) {	1✔
NEW 240	limit, eventID, err := devauth.RateLimitsFromContext(ctx)	×
NEW 241	if err != nil {	×
NEW 242	return nil, err	×
NEW 243	} else if limit < 0 {	×
NEW 244	return nil, nil	×
NEW 245	}	×
NEW 246	keyPrefix := redisKeyPrefix + ":" + eventID	×
NEW 247	return &redis.RatelimitParams{	×
NEW 248	Burst: uint64(limit),	×
NEW 249	Interval: interval,	×
NEW 250	KeyPrefix: keyPrefix,	×
NEW 251	}, nil	×
252	},
253	)
254	devauth.WithRatelimits(	1✔
255	rateLimiter,	1✔
256	quotas,	1✔
257	c.GetFloat64(dconfig.SettingRatelimitsQuotaDefault),	1✔
258	)	1✔
259	return nil	1✔
260	}

mendersoftware / mender-server / 1769701362

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous