850112129

Build Type

gitlab-ci

Committed by Peter Grzybowski

Commit Message

chore: migrate dbName upfront in case of migrating a tenant.

Pull Request Pull Request #642: Single db move fixes

Run Details

332 of 405 new or added lines in 5 files covered. (81.98%)

89 existing lines in 3 files now uncovered.

4672 of 5591 relevant lines covered (83.56%)

79.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.6

/store/mongo/migration_2_0_0.go

// Copyright 2023 Northern.tech AS
//
//        Licensed under the Apache License, Version 2.0 (the "License");
//        you may not use this file except in compliance with the License.
//        You may obtain a copy of the License at
//
//            http://www.apache.org/licenses/LICENSE-2.0
//
//        Unless required by applicable law or agreed to in writing, software
//        distributed under the License is distributed on an "AS IS" BASIS,
//        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//        See the License for the specific language governing permissions and
//        limitations under the License.
package mongo

import (
        "context"
        "strings"

        "github.com/pkg/errors"
        "go.mongodb.org/mongo-driver/mongo"
        mopts "go.mongodb.org/mongo-driver/mongo/options"

        "github.com/mendersoftware/go-lib-micro/identity"
        "github.com/mendersoftware/go-lib-micro/mongo/migrate"
        "github.com/mendersoftware/go-lib-micro/mongo/oid"
        mstorev1 "github.com/mendersoftware/go-lib-micro/store"
        mstore "github.com/mendersoftware/go-lib-micro/store/v2"
        "go.mongodb.org/mongo-driver/bson"
)

const (
        findBatchSize = 255
)

type migration_2_0_0 struct {
        ds  *DataStoreMongo
        ctx context.Context
}

var DbDevicesCollectionIndices = []mongo.IndexModel{
        {
                Keys: bson.D{
                        {Key: mstore.FieldTenantID, Value: 1},
                        {Key: dbFieldIDDataSha, Value: 1},
                },
                //nolint:staticcheck // SA1019
                Options: mopts.Index().
                        SetName(strings.Join([]string{
                                mstore.FieldTenantID,
                                dbFieldIDDataSha,
                        }, "_")).
                        SetUnique(true).
                        SetBackground(false),
        },
        {
                Keys: bson.D{
                        {Key: mstore.FieldTenantID, Value: 1},
                        {Key: dbFieldStatus, Value: 1},
                        {Key: dbFieldID, Value: 1},
                },
                //nolint:staticcheck // SA1019
                Options: mopts.Index().
                        SetName(strings.Join([]string{
                                mstore.FieldTenantID,
                                dbFieldStatus,
                                dbFieldID,
                        }, "_")).
                        SetBackground(false),
        },
}

var DbAuthSetsCollectionIndices = []mongo.IndexModel{
        {
                Keys: bson.D{
                        {Key: mstore.FieldTenantID, Value: 1},
                        {Key: dbFieldIDDataSha, Value: 1},
                        {Key: dbFieldPubKey, Value: 1},
                },
                Options: mopts.Index().
                        SetName(strings.Join([]string{
                                mstore.FieldTenantID,
                                dbFieldIDDataSha,
                                dbFieldPubKey,
                        }, "_")).
                        SetUnique(true),
        },
        {
                Keys: bson.D{
                        {Key: mstore.FieldTenantID, Value: 1},
                        {Key: dbFieldDeviceID, Value: 1},
                },
                Options: mopts.Index().
                        SetName(strings.Join([]string{
                                mstore.FieldTenantID,
                                dbFieldDeviceID,
                        }, "_")),
        },
}

var DbLimitsCollectionIndices = []mongo.IndexModel{
        {
                Keys: bson.D{
                        {Key: mstore.FieldTenantID, Value: 1},
                        {Key: dbFieldName, Value: 1},
                },
                Options: mopts.Index().
                        SetName(strings.Join([]string{
                                mstore.FieldTenantID,
                                dbFieldName,
                        },
                                "_",
                        ),
                        ).
                        SetUnique(true),
        },
}

var DbTokensCollectionIndices = []mongo.IndexModel{
        {
                Keys: bson.D{
                        {Key: dbFieldTenantClaim, Value: 1},
                        {Key: dbFieldID, Value: 1},
                },
                Options: mopts.Index().
                        SetName(
                                strings.Join(
                                        []string{
                                                strings.ReplaceAll(dbFieldTenantClaim, ".", "_"),
                                                dbFieldID,
                                        },
                                        "_",
                                ),
                        ),
        },
        {
                Keys: bson.D{
                        {Key: dbFieldTenantClaim, Value: 1},
                        {Key: dbFieldExpTime, Value: 1},
                },
                Options: mopts.Index().
                        SetName(
                                strings.Join(
                                        []string{
                                                strings.ReplaceAll(dbFieldTenantClaim, ".", "_"),
                                                strings.ReplaceAll(dbFieldExpTime, ".", "_"),
                                        },
                                        "_",
                                ),
                        ),
        },
}

// Up creates an index on status and id in the devices collection
// nolint: gocyclo
func (m *migration_2_0_0) Up(from migrate.Version) error {
        ctx := context.Background()
        client := m.ds.client

        collections := map[string]struct {
                Indexes []mongo.IndexModel
        }{
                DbAuthSetColl: {
                        Indexes: DbAuthSetsCollectionIndices,
                },
                DbDevicesColl: {
                        Indexes: DbDevicesCollectionIndices,
                },
                DbLimitsColl: {
                        Indexes: DbLimitsCollectionIndices,
                },
                DbTokensColl: {
                        Indexes: DbTokensCollectionIndices,
                },
        }

        databaseName := mstorev1.DbFromContext(m.ctx, DbName)
        tenantID := mstorev1.TenantFromDbName(databaseName, DbName)
        ctx = identity.WithContext(ctx, &identity.Identity{
                Tenant: tenantID,
        })
        writes := make([]mongo.WriteModel, 0, findBatchSize)

        for collection, idxes := range collections {
                writes = writes[:0]
                findOptions := mopts.Find().
                        SetBatchSize(findBatchSize).
                        SetSort(bson.D{{Key: dbFieldID, Value: 1}})
                collOut := client.Database(DbName).Collection(collection)
                if databaseName == DbName {
                        indices := collOut.Indexes()
                        _, _ = indices.DropAll(ctx)

                        if len(idxes.Indexes) > 0 {
                                _, err := collOut.Indexes().CreateMany(ctx, collections[collection].Indexes)
                                if err != nil {
                                        return err
                                }
                        }
                        if collection == DbTokensColl {
                                continue
                        }
                        _, err := collOut.UpdateMany(ctx, bson.D{
                                {Key: mstore.FieldTenantID, Value: bson.D{
                                        {Key: "$exists", Value: false},
                                }},
                        }, bson.D{{Key: "$set", Value: bson.D{
                                {Key: mstore.FieldTenantID, Value: ""},
                        }}},
                        )
                        if err != nil {
                                return err
                        }
                        continue
                }

                if collection == DbTokensColl {
                        continue
                }
                coll := client.Database(databaseName).Collection(collection)
                // get all the documents in the collection
                cur, err := coll.Find(ctx, bson.D{}, findOptions)
                if err != nil {
                        return err
                }
                defer cur.Close(ctx)

                // migrate the documents
                if collection == DbLimitsColl {
                        for cur.Next(ctx) {
                                id := cur.Current.Lookup(dbFieldID)
                                var currentId string
                                err = id.Unmarshal(&currentId)
                                if err != nil {
                                        return errors.Wrap(err, "the id found is un-parsable")
                                }
                                var item bson.D
                                if err = cur.Decode(&item); err != nil {
                                        return err
                                }
                                item = append(item, bson.E{
                                        Key:   dbFieldName,
                                        Value: currentId,
                                })
                                item = item[1:]
                                item = findAndReplace(item, dbFieldID, oid.NewUUIDv4().String())

                                writes = append(writes, mongo.
                                        NewReplaceOneModel().
                                        SetFilter(bson.D{{Key: dbFieldName, Value: id}}).
                                        SetUpsert(true).
                                        SetReplacement(mstore.WithTenantID(ctx, item)))
                                if len(writes) == findBatchSize {
                                        _, err = collOut.BulkWrite(ctx, writes)
                                        if err != nil {
                                                return err
                                        }
                                        writes = writes[:0]
                                }
                        }
                } else {
                        for cur.Next(ctx) {
                                id := cur.Current.Lookup(dbFieldID)
                                var item bson.D
                                if err = cur.Decode(&item); err != nil {
                                        return err
                                }
                                writes = append(writes, mongo.
                                        NewReplaceOneModel().
                                        SetFilter(bson.D{{Key: dbFieldID, Value: id}}).
                                        SetUpsert(true).
                                        SetReplacement(mstore.WithTenantID(ctx, item)))
                                if len(writes) == findBatchSize {
                                        _, err = collOut.BulkWrite(ctx, writes)
                                        if err != nil {
                                                return err
                                        }
                                        writes = writes[:0]
                                }
                        }
                }
                if len(writes) > 0 {
                        _, err := collOut.BulkWrite(ctx, writes)
                        if err != nil {
                                return err
                        }
                }
        }

        return nil
}

func findAndReplace(item bson.D, key string, value string) bson.D {
        for i, d := range item {
                if d.Key == key {
                        item[i].Value = value
                }
        }
        return item
}

func (m *migration_2_0_0) Version() migrate.Version {
        return migrate.MakeVersion(2, 0, 0)
}

1	// Copyright 2023 Northern.tech AS
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14	package mongo
15
16	import (
17	"context"
18	"strings"
19
20	"github.com/pkg/errors"
21	"go.mongodb.org/mongo-driver/mongo"
22	mopts "go.mongodb.org/mongo-driver/mongo/options"
23
24	"github.com/mendersoftware/go-lib-micro/identity"
25	"github.com/mendersoftware/go-lib-micro/mongo/migrate"
26	"github.com/mendersoftware/go-lib-micro/mongo/oid"
27	mstorev1 "github.com/mendersoftware/go-lib-micro/store"
28	mstore "github.com/mendersoftware/go-lib-micro/store/v2"
29	"go.mongodb.org/mongo-driver/bson"
30	)
31
32	const (
33	findBatchSize = 255
34	)
35
36	type migration_2_0_0 struct {
37	ds *DataStoreMongo
38	ctx context.Context
39	}
40
41	var DbDevicesCollectionIndices = []mongo.IndexModel{
42	{
43	Keys: bson.D{
44	{Key: mstore.FieldTenantID, Value: 1},
45	{Key: dbFieldIDDataSha, Value: 1},
46	},
47	//nolint:staticcheck // SA1019
48	Options: mopts.Index().
49	SetName(strings.Join([]string{
50	mstore.FieldTenantID,
51	dbFieldIDDataSha,
52	}, "_")).
53	SetUnique(true).
54	SetBackground(false),
55	},
56	{
57	Keys: bson.D{
58	{Key: mstore.FieldTenantID, Value: 1},
59	{Key: dbFieldStatus, Value: 1},
60	{Key: dbFieldID, Value: 1},
61	},
62	//nolint:staticcheck // SA1019
63	Options: mopts.Index().
64	SetName(strings.Join([]string{
65	mstore.FieldTenantID,
66	dbFieldStatus,
67	dbFieldID,
68	}, "_")).
69	SetBackground(false),
70	},
71	}
72
73	var DbAuthSetsCollectionIndices = []mongo.IndexModel{
74	{
75	Keys: bson.D{
76	{Key: mstore.FieldTenantID, Value: 1},
77	{Key: dbFieldIDDataSha, Value: 1},
78	{Key: dbFieldPubKey, Value: 1},
79	},
80	Options: mopts.Index().
81	SetName(strings.Join([]string{
82	mstore.FieldTenantID,
83	dbFieldIDDataSha,
84	dbFieldPubKey,
85	}, "_")).
86	SetUnique(true),
87	},
88	{
89	Keys: bson.D{
90	{Key: mstore.FieldTenantID, Value: 1},
91	{Key: dbFieldDeviceID, Value: 1},
92	},
93	Options: mopts.Index().
94	SetName(strings.Join([]string{
95	mstore.FieldTenantID,
96	dbFieldDeviceID,
97	}, "_")),
98	},
99	}
100
101	var DbLimitsCollectionIndices = []mongo.IndexModel{
102	{
103	Keys: bson.D{
104	{Key: mstore.FieldTenantID, Value: 1},
105	{Key: dbFieldName, Value: 1},
106	},
107	Options: mopts.Index().
108	SetName(strings.Join([]string{
109	mstore.FieldTenantID,
110	dbFieldName,
111	},
112	"_",
113	),
114	).
115	SetUnique(true),
116	},
117	}
118
119	var DbTokensCollectionIndices = []mongo.IndexModel{
120	{
121	Keys: bson.D{
122	{Key: dbFieldTenantClaim, Value: 1},
123	{Key: dbFieldID, Value: 1},
124	},
125	Options: mopts.Index().
126	SetName(
127	strings.Join(
128	[]string{
129	strings.ReplaceAll(dbFieldTenantClaim, ".", "_"),
130	dbFieldID,
131	},
132	"_",
133	),
134	),
135	},
136	{
137	Keys: bson.D{
138	{Key: dbFieldTenantClaim, Value: 1},
139	{Key: dbFieldExpTime, Value: 1},
140	},
141	Options: mopts.Index().
142	SetName(
143	strings.Join(
144	[]string{
145	strings.ReplaceAll(dbFieldTenantClaim, ".", "_"),
146	strings.ReplaceAll(dbFieldExpTime, ".", "_"),
147	},
148	"_",
149	),
150	),
151	},
152	}
153
154	// Up creates an index on status and id in the devices collection
155	// nolint: gocyclo
156	func (m *migration_2_0_0) Up(from migrate.Version) error {	217✔
157	ctx := context.Background()	217✔
158	client := m.ds.client	217✔
159		217✔
160	collections := map[string]struct {	217✔
161	Indexes []mongo.IndexModel	217✔
162	}{	217✔
163	DbAuthSetColl: {	217✔
164	Indexes: DbAuthSetsCollectionIndices,	217✔
165	},	217✔
166	DbDevicesColl: {	217✔
167	Indexes: DbDevicesCollectionIndices,	217✔
168	},	217✔
169	DbLimitsColl: {	217✔
170	Indexes: DbLimitsCollectionIndices,	217✔
171	},	217✔
172	DbTokensColl: {	217✔
173	Indexes: DbTokensCollectionIndices,	217✔
174	},	217✔
175	}	217✔
176		217✔
177	databaseName := mstorev1.DbFromContext(m.ctx, DbName)	217✔
178	tenantID := mstorev1.TenantFromDbName(databaseName, DbName)	217✔
179	ctx = identity.WithContext(ctx, &identity.Identity{	217✔
180	Tenant: tenantID,	217✔
181	})	217✔
182	writes := make([]mongo.WriteModel, 0, findBatchSize)	217✔
183		217✔
184	for collection, idxes := range collections {	1,082✔
185	writes = writes[:0]	865✔
186	findOptions := mopts.Find().	865✔
187	SetBatchSize(findBatchSize).	865✔
188	SetSort(bson.D{{Key: dbFieldID, Value: 1}})	865✔
189	collOut := client.Database(DbName).Collection(collection)	865✔
190	if databaseName == DbName {	1,394✔
191	indices := collOut.Indexes()	529✔
192	_, _ = indices.DropAll(ctx)	529✔
193		529✔
194	if len(idxes.Indexes) > 0 {	1,058✔
195	_, err := collOut.Indexes().CreateMany(ctx, collections[collection].Indexes)	529✔
196	if err != nil {	529✔
NEW 197	return err	×
NEW 198	}	×
199	}
200	if collection == DbTokensColl {	662✔
201	continue	133✔
202	}
203	_, err := collOut.UpdateMany(ctx, bson.D{	397✔
204	{Key: mstore.FieldTenantID, Value: bson.D{	397✔
205	{Key: "$exists", Value: false},	397✔
206	}},	397✔
207	}, bson.D{{Key: "$set", Value: bson.D{	397✔
208	{Key: mstore.FieldTenantID, Value: ""},	397✔
209	}}},	397✔
210	)	397✔
211	if err != nil {	397✔
NEW 212	return err	×
NEW 213	}	×
214	continue	397✔
215	}
216
217	if collection == DbTokensColl {	422✔
218	continue	85✔
219	}
220	coll := client.Database(databaseName).Collection(collection)	253✔
221	// get all the documents in the collection	253✔
222	cur, err := coll.Find(ctx, bson.D{}, findOptions)	253✔
223	if err != nil {	253✔
NEW 224	return err	×
NEW 225	}	×
226	defer cur.Close(ctx)	253✔
227		253✔
228	// migrate the documents	253✔
229	if collection == DbLimitsColl {	338✔
230	for cur.Next(ctx) {	85✔
NEW 231	id := cur.Current.Lookup(dbFieldID)	×
NEW 232	var currentId string	×
NEW 233	err = id.Unmarshal(&currentId)	×
NEW 234	if err != nil {	×
NEW 235	return errors.Wrap(err, "the id found is un-parsable")	×
NEW 236	}	×
NEW 237	var item bson.D	×
NEW 238	if err = cur.Decode(&item); err != nil {	×
NEW 239	return err	×
NEW 240	}	×
NEW 241	item = append(item, bson.E{	×
NEW 242	Key: dbFieldName,	×
NEW 243	Value: currentId,	×
NEW 244	})	×
NEW 245	item = item[1:]	×
NEW 246	item = findAndReplace(item, dbFieldID, oid.NewUUIDv4().String())	×
NEW 247		×
NEW 248	writes = append(writes, mongo.	×
NEW 249	NewReplaceOneModel().	×
NEW 250	SetFilter(bson.D{{Key: dbFieldName, Value: id}}).	×
NEW 251	SetUpsert(true).	×
NEW 252	SetReplacement(mstore.WithTenantID(ctx, item)))	×
NEW 253	if len(writes) == findBatchSize {	×
NEW 254	_, err = collOut.BulkWrite(ctx, writes)	×
NEW 255	if err != nil {	×
NEW 256	return err	×
NEW 257	}	×
NEW 258	writes = writes[:0]	×
259	}
260	}
261	} else {	169✔
262	for cur.Next(ctx) {	12,709✔
263	id := cur.Current.Lookup(dbFieldID)	12,540✔
264	var item bson.D	12,540✔
265	if err = cur.Decode(&item); err != nil {	12,540✔
NEW 266	return err	×
NEW 267	}	×
268	writes = append(writes, mongo.	12,540✔
269	NewReplaceOneModel().	12,540✔
270	SetFilter(bson.D{{Key: dbFieldID, Value: id}}).	12,540✔
271	SetUpsert(true).	12,540✔
272	SetReplacement(mstore.WithTenantID(ctx, item)))	12,540✔
273	if len(writes) == findBatchSize {	12,540✔
NEW 274	_, err = collOut.BulkWrite(ctx, writes)	×
NEW 275	if err != nil {	×
NEW 276	return err	×
NEW 277	}	×
NEW 278	writes = writes[:0]	×
279	}
280	}
281	}
282	if len(writes) > 0 {	333✔
283	_, err := collOut.BulkWrite(ctx, writes)	80✔
284	if err != nil {	80✔
NEW 285	return err	×
NEW 286	}	×
287	}
288	}
289
290	return nil	217✔
291	}
292
NEW 293	func findAndReplace(item bson.D, key string, value string) bson.D {	×
NEW 294	for i, d := range item {	×
NEW 295	if d.Key == key {	×
NEW 296	item[i].Value = value	×
NEW 297	}	×
298	}
NEW 299	return item	×
300	}
301
302	func (m *migration_2_0_0) Version() migrate.Version {	281✔
303	return migrate.MakeVersion(2, 0, 0)	281✔
304	}	281✔

mendersoftware / deviceauth / 850112129

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous