1401701972

Committed 05 Aug 2024 08:32PM UTC coverage: 91.217%. Remained the same

Build # 1401701972

Build Type

push

gitlab-ci

Committed by

web-flow

Commit Message

Merge pull request #460 from mendersoftware/dependabot/docker/docker-dependencies-03b04ac819

chore: bump golang from 1.22.4-alpine3.19 to 1.22.5-alpine3.19 in the docker-dependencies group

Run Details

3095 of 3393 relevant lines covered (91.22%)

148.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.09

/store/mongo/migration_1_0_0.go

// Copyright 2021 Northern.tech AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package mongo

import (
        "context"
        "time"

        "go.mongodb.org/mongo-driver/bson"
        "go.mongodb.org/mongo-driver/mongo"
        mopts "go.mongodb.org/mongo-driver/mongo/options"

        "github.com/mendersoftware/go-lib-micro/log"
        "github.com/mendersoftware/go-lib-micro/mongo/migrate"
        mstore "github.com/mendersoftware/go-lib-micro/store"

        "github.com/mendersoftware/inventory/model"
)

// batchSize for performing upserts during maintenance.
const batchSize = 1024

type migration_1_0_0 struct {
        ms  *DataStoreMongo
        ctx context.Context
        // maintenance determines whether to run migration in "maintenance mode"
        // NOTE: While maintenance is in progress, the following APIs
        //       MUST be DISABLED:
        //        - DELETE /devices/{id}       (management)
        //        - DELETE /devices/{id}/group (management)
        //        - PUT /devices/{id}/group    (management)
        //        - PATCH /devices/attributes  (devices)
        // Until we have an actual mesh system and a circuit breaker that can handle
        // disabling of the API internally, we'll have to do maintenance manually.
        maintenance bool
}

const (
        createdNameField = DbDevAttributes + "." + model.AttrScopeSystem +
                "-" + model.AttrNameCreated + "." + DbDevAttributesName
        createdValueField = DbDevAttributes + "." + model.AttrScopeSystem +
                "-" + model.AttrNameCreated + "." + DbDevAttributesValue
        createdScopeField = DbDevAttributes + "." + model.AttrScopeSystem +
                "-" + model.AttrNameCreated + "." + DbDevAttributesScope
        updatedNameField = DbDevAttributes + "." + model.AttrScopeSystem +
                "-" + model.AttrNameUpdated + "." + DbDevAttributesName
        updatedValueField = DbDevAttributes + "." + model.AttrScopeSystem +
                "-" + model.AttrNameUpdated + "." + DbDevAttributesValue
        updatedScopeField = DbDevAttributes + "." + model.AttrScopeSystem +
                "-" + model.AttrNameUpdated + "." + DbDevAttributesScope
        groupNameField = DbDevAttributes + "." + model.AttrScopeSystem +
                "-" + model.AttrNameGroup + "." + DbDevAttributesName
        groupScopeField = DbDevAttributes + "." + model.AttrScopeSystem +
                "-" + model.AttrNameGroup + "." + DbDevAttributesScope
)

// doMaintenance applies the migration, but instead of replacing document fields
// the new fields are upserted.
func (m *migration_1_0_0) doMaintenance(from migrate.Version) error {
        var N int
        database := m.ms.client.Database(mstore.DbFromContext(m.ctx, DbName))
        collDevs := database.Collection(DbDevicesColl)
        collMgrInfo := database.Collection("migration_info")
        l := log.FromContext(m.ctx)

        findOpts := mopts.Find()
        findOpts.SetProjection(bson.M{
                "_id":               1,
                model.AttrNameGroup: 1,
                "updated_ts":        1,
                "created_ts":        1,
        })
        findOpts.SetBatchSize(batchSize)
        type partialDevice struct {
                ID        string    `bson:"_id"`
                Group     string    `bson:"group,omitempty"`
                UpdatedTS time.Time `bson:"updated_ts"`
                CreatedTS time.Time `bson:"created_ts"`
        }
        bulkUpdates := make([]mongo.WriteModel, 0, batchSize)
        cur, err := collDevs.Find(m.ctx, bson.M{}, findOpts)
        if err != nil {
                return err
        }
        // Loop until inner loop updates a partial batch
        for i := batchSize; i >= batchSize; {
                for i = 0; cur.Next(m.ctx) && i < batchSize; i++ {
                        var device partialDevice
                        err = cur.Decode(&device)
                        if err != nil {
                                return err
                        }
                        updateModel := mongo.NewUpdateOneModel()
                        updateModel.SetUpsert(true)
                        updateModel.SetFilter(bson.M{"_id": device.ID})
                        update := bson.M{
                                createdNameField:  model.AttrNameCreated,
                                createdScopeField: model.AttrScopeSystem,
                                createdValueField: device.CreatedTS,
                                updatedNameField:  model.AttrNameUpdated,
                                updatedScopeField: model.AttrScopeSystem,
                                updatedValueField: device.UpdatedTS,
                        }
                        if device.Group != "" {
                                update[groupNameField] = model.AttrNameGroup
                                update[groupScopeField] = model.AttrScopeSystem
                                update[DbDevAttributesGroupValue] = device.Group
                        }
                        updateModel.SetUpdate(bson.M{"$set": update})
                        bulkUpdates = append(bulkUpdates, updateModel)
                }
                _, err = collDevs.BulkWrite(m.ctx, bulkUpdates)
                if err != nil {
                        return err
                }
                bulkUpdates = bulkUpdates[:0]
                N += i
        }
        l.Infof("Successfully upserted %d devices", N)
        filter := bson.M{"maintenance": bson.M{"version": m.Version()}}
        doc := bson.M{
                "maintenance": filter["maintenance"],
                "timestamp":   time.Now(),
        }
        // In case maintenance document already exists, replace timestamp
        replOpt := mopts.FindOneAndReplace()
        replOpt.SetUpsert(true)
        collMgrInfo.FindOneAndReplace(m.ctx, filter, doc, replOpt)
        return nil
}

func (m *migration_1_0_0) doMigrate(from migrate.Version) error {
        databaseName := mstore.DbFromContext(m.ctx, DbName)
        collDevs := m.ms.client.Database(databaseName).Collection(DbDevicesColl)

        // Move timestamps to identity scope.
        _, err := collDevs.UpdateMany(m.ctx, bson.M{}, bson.M{
                "$rename": bson.M{
                        model.AttrNameCreated: createdValueField,
                        model.AttrNameUpdated: updatedValueField,
                },
                "$set": bson.M{
                        createdNameField:  model.AttrNameCreated,
                        createdScopeField: model.AttrScopeSystem,
                        updatedNameField:  model.AttrNameUpdated,
                        updatedScopeField: model.AttrScopeSystem,
                },
        })
        if err != nil {
                return nil
        }

        // For devices in a group: move to attribute.
        _, err = collDevs.UpdateMany(
                m.ctx,
                bson.M{DbDevGroup: bson.M{"$exists": true}},
                bson.M{
                        "$rename": bson.M{
                                model.AttrNameGroup: DbDevAttributesGroupValue,
                        },
                        "$set": bson.M{
                                groupNameField:  model.AttrNameGroup,
                                groupScopeField: model.AttrScopeSystem,
                        },
                })
        return err
}

func (m *migration_1_0_0) doCleanup() error {
        databaseName := mstore.DbFromContext(m.ctx, DbName)
        collDevs := m.ms.client.Database(databaseName).Collection(DbDevicesColl)

        update := bson.M{"$unset": bson.M{
                "updated_ts": "",
                "created_ts": "",
                DbDevGroup:   "",
        }}
        _, err := collDevs.UpdateMany(m.ctx, bson.M{}, update)
        return err
}

// Up creates timestamp and group attributes under the identity scope.
// The values reflect the values previously held in the root of the document.
func (m *migration_1_0_0) Up(from migrate.Version) error {
        l := log.FromContext(m.ctx)
        tenantDB := mstore.DbFromContext(m.ctx, DbName)
        if !migrate.VersionIsLess(from, m.Version()) {
                l.Infof("db '%s' already migrated", tenantDB)
                return nil
        }

        if m.maintenance {
                // Perform upserts
                return m.doMaintenance(from)
        }
        database := m.ms.client.Database(tenantDB)
        collMgrInfo := database.Collection("migration_info")
        // Check if maintenance operation has run
        res := collMgrInfo.FindOne(m.ctx, bson.M{"maintenance.version": m.Version()})
        if res.Err() == nil {
                // Perform maintenance cleanup
                err := m.doCleanup()
                if err == nil {
                        _, err = collMgrInfo.DeleteOne(m.ctx, bson.M{"maintenance.version": m.Version()})
                }
                return err
        }
        // Run ordinary migration
        return m.doMigrate(from)
}

func (m *migration_1_0_0) Version() migrate.Version {
        return migrate.MakeVersion(1, 0, 0)
}

1	// Copyright 2021 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package mongo
16
17	import (
18	"context"
19	"time"
20
21	"go.mongodb.org/mongo-driver/bson"
22	"go.mongodb.org/mongo-driver/mongo"
23	mopts "go.mongodb.org/mongo-driver/mongo/options"
24
25	"github.com/mendersoftware/go-lib-micro/log"
26	"github.com/mendersoftware/go-lib-micro/mongo/migrate"
27	mstore "github.com/mendersoftware/go-lib-micro/store"
28
29	"github.com/mendersoftware/inventory/model"
30	)
31
32	// batchSize for performing upserts during maintenance.
33	const batchSize = 1024
34
35	type migration_1_0_0 struct {
36	ms *DataStoreMongo
37	ctx context.Context
38	// maintenance determines whether to run migration in "maintenance mode"
39	// NOTE: While maintenance is in progress, the following APIs
40	// MUST be DISABLED:
41	// - DELETE /devices/{id} (management)
42	// - DELETE /devices/{id}/group (management)
43	// - PUT /devices/{id}/group (management)
44	// - PATCH /devices/attributes (devices)
45	// Until we have an actual mesh system and a circuit breaker that can handle
46	// disabling of the API internally, we'll have to do maintenance manually.
47	maintenance bool
48	}
49
50	const (
51	createdNameField = DbDevAttributes + "." + model.AttrScopeSystem +
52	"-" + model.AttrNameCreated + "." + DbDevAttributesName
53	createdValueField = DbDevAttributes + "." + model.AttrScopeSystem +
54	"-" + model.AttrNameCreated + "." + DbDevAttributesValue
55	createdScopeField = DbDevAttributes + "." + model.AttrScopeSystem +
56	"-" + model.AttrNameCreated + "." + DbDevAttributesScope
57	updatedNameField = DbDevAttributes + "." + model.AttrScopeSystem +
58	"-" + model.AttrNameUpdated + "." + DbDevAttributesName
59	updatedValueField = DbDevAttributes + "." + model.AttrScopeSystem +
60	"-" + model.AttrNameUpdated + "." + DbDevAttributesValue
61	updatedScopeField = DbDevAttributes + "." + model.AttrScopeSystem +
62	"-" + model.AttrNameUpdated + "." + DbDevAttributesScope
63	groupNameField = DbDevAttributes + "." + model.AttrScopeSystem +
64	"-" + model.AttrNameGroup + "." + DbDevAttributesName
65	groupScopeField = DbDevAttributes + "." + model.AttrScopeSystem +
66	"-" + model.AttrNameGroup + "." + DbDevAttributesScope
67	)
68
69	// doMaintenance applies the migration, but instead of replacing document fields
70	// the new fields are upserted.
71	func (m *migration_1_0_0) doMaintenance(from migrate.Version) error {	3✔
72	var N int	3✔
73	database := m.ms.client.Database(mstore.DbFromContext(m.ctx, DbName))	3✔
74	collDevs := database.Collection(DbDevicesColl)	3✔
75	collMgrInfo := database.Collection("migration_info")	3✔
76	l := log.FromContext(m.ctx)	3✔
77		3✔
78	findOpts := mopts.Find()	3✔
79	findOpts.SetProjection(bson.M{	3✔
80	"_id": 1,	3✔
81	model.AttrNameGroup: 1,	3✔
82	"updated_ts": 1,	3✔
83	"created_ts": 1,	3✔
84	})	3✔
85	findOpts.SetBatchSize(batchSize)	3✔
86	type partialDevice struct {	3✔
87	ID string `bson:"_id"`	3✔
88	Group string `bson:"group,omitempty"`	3✔
89	UpdatedTS time.Time `bson:"updated_ts"`	3✔
90	CreatedTS time.Time `bson:"created_ts"`	3✔
91	}	3✔
92	bulkUpdates := make([]mongo.WriteModel, 0, batchSize)	3✔
93	cur, err := collDevs.Find(m.ctx, bson.M{}, findOpts)	3✔
94	if err != nil {	3✔
95	return err	×
96	}	×
97	// Loop until inner loop updates a partial batch
98	for i := batchSize; i >= batchSize; {	6✔
99	for i = 0; cur.Next(m.ctx) && i < batchSize; i++ {	9✔
100	var device partialDevice	6✔
101	err = cur.Decode(&device)	6✔
102	if err != nil {	6✔
103	return err	×
104	}	×
105	updateModel := mongo.NewUpdateOneModel()	6✔
106	updateModel.SetUpsert(true)	6✔
107	updateModel.SetFilter(bson.M{"_id": device.ID})	6✔
108	update := bson.M{	6✔
109	createdNameField: model.AttrNameCreated,	6✔
110	createdScopeField: model.AttrScopeSystem,	6✔
111	createdValueField: device.CreatedTS,	6✔
112	updatedNameField: model.AttrNameUpdated,	6✔
113	updatedScopeField: model.AttrScopeSystem,	6✔
114	updatedValueField: device.UpdatedTS,	6✔
115	}	6✔
116	if device.Group != "" {	9✔
117	update[groupNameField] = model.AttrNameGroup	3✔
118	update[groupScopeField] = model.AttrScopeSystem	3✔
119	update[DbDevAttributesGroupValue] = device.Group	3✔
120	}	3✔
121	updateModel.SetUpdate(bson.M{"$set": update})	6✔
122	bulkUpdates = append(bulkUpdates, updateModel)	6✔
123	}
124	_, err = collDevs.BulkWrite(m.ctx, bulkUpdates)	3✔
125	if err != nil {	3✔
126	return err	×
127	}	×
128	bulkUpdates = bulkUpdates[:0]	3✔
129	N += i	3✔
130	}
131	l.Infof("Successfully upserted %d devices", N)	3✔
132	filter := bson.M{"maintenance": bson.M{"version": m.Version()}}	3✔
133	doc := bson.M{	3✔
134	"maintenance": filter["maintenance"],	3✔
135	"timestamp": time.Now(),	3✔
136	}	3✔
137	// In case maintenance document already exists, replace timestamp	3✔
138	replOpt := mopts.FindOneAndReplace()	3✔
139	replOpt.SetUpsert(true)	3✔
140	collMgrInfo.FindOneAndReplace(m.ctx, filter, doc, replOpt)	3✔
141	return nil	3✔
142	}
143
144	func (m *migration_1_0_0) doMigrate(from migrate.Version) error {	20✔
145	databaseName := mstore.DbFromContext(m.ctx, DbName)	20✔
146	collDevs := m.ms.client.Database(databaseName).Collection(DbDevicesColl)	20✔
147		20✔
148	// Move timestamps to identity scope.	20✔
149	_, err := collDevs.UpdateMany(m.ctx, bson.M{}, bson.M{	20✔
150	"$rename": bson.M{	20✔
151	model.AttrNameCreated: createdValueField,	20✔
152	model.AttrNameUpdated: updatedValueField,	20✔
153	},	20✔
154	"$set": bson.M{	20✔
155	createdNameField: model.AttrNameCreated,	20✔
156	createdScopeField: model.AttrScopeSystem,	20✔
157	updatedNameField: model.AttrNameUpdated,	20✔
158	updatedScopeField: model.AttrScopeSystem,	20✔
159	},	20✔
160	})	20✔
161	if err != nil {	20✔
162	return nil	×
163	}	×
164
165	// For devices in a group: move to attribute.
166	_, err = collDevs.UpdateMany(	20✔
167	m.ctx,	20✔
168	bson.M{DbDevGroup: bson.M{"$exists": true}},	20✔
169	bson.M{	20✔
170	"$rename": bson.M{	20✔
171	model.AttrNameGroup: DbDevAttributesGroupValue,	20✔
172	},	20✔
173	"$set": bson.M{	20✔
174	groupNameField: model.AttrNameGroup,	20✔
175	groupScopeField: model.AttrScopeSystem,	20✔
176	},	20✔
177	})	20✔
178	return err	20✔
179	}
180
181	func (m *migration_1_0_0) doCleanup() error {	3✔
182	databaseName := mstore.DbFromContext(m.ctx, DbName)	3✔
183	collDevs := m.ms.client.Database(databaseName).Collection(DbDevicesColl)	3✔
184		3✔
185	update := bson.M{"$unset": bson.M{	3✔
186	"updated_ts": "",	3✔
187	"created_ts": "",	3✔
188	DbDevGroup: "",	3✔
189	}}	3✔
190	_, err := collDevs.UpdateMany(m.ctx, bson.M{}, update)	3✔
191	return err	3✔
192	}	3✔
193
194	// Up creates timestamp and group attributes under the identity scope.
195	// The values reflect the values previously held in the root of the document.
196	func (m *migration_1_0_0) Up(from migrate.Version) error {	26✔
197	l := log.FromContext(m.ctx)	26✔
198	tenantDB := mstore.DbFromContext(m.ctx, DbName)	26✔
199	if !migrate.VersionIsLess(from, m.Version()) {	26✔
200	l.Infof("db '%s' already migrated", tenantDB)	×
201	return nil	×
202	}	×
203
204	if m.maintenance {	29✔
205	// Perform upserts	3✔
206	return m.doMaintenance(from)	3✔
207	}	3✔
208	database := m.ms.client.Database(tenantDB)	23✔
209	collMgrInfo := database.Collection("migration_info")	23✔
210	// Check if maintenance operation has run	23✔
211	res := collMgrInfo.FindOne(m.ctx, bson.M{"maintenance.version": m.Version()})	23✔
212	if res.Err() == nil {	26✔
213	// Perform maintenance cleanup	3✔
214	err := m.doCleanup()	3✔
215	if err == nil {	6✔
216	_, err = collMgrInfo.DeleteOne(m.ctx, bson.M{"maintenance.version": m.Version()})	3✔
217	}	3✔
218	return err	3✔
219	}
220	// Run ordinary migration
221	return m.doMigrate(from)	20✔
222	}
223
224	func (m *migration_1_0_0) Version() migrate.Version {	126✔
225	return migrate.MakeVersion(1, 0, 0)	126✔
226	}	126✔

mendersoftware / inventory / 1401701972

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous