• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

lightningnetwork / lnd / 12203658024

06 Dec 2024 05:54PM UTC coverage: 58.965% (+9.2%) from 49.807%
12203658024

Pull #8831

github

bhandras
docs: update release notes for 0.19.0
Pull Request #8831: invoices: migrate KV invoices to native SQL for users of KV SQL backends

506 of 695 new or added lines in 12 files covered. (72.81%)

67 existing lines in 19 files now uncovered.

133874 of 227038 relevant lines covered (58.97%)

19659.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.63
/sqldb/migrations.go
1
package sqldb
2

3
import (
4
        "bytes"
5
        "context"
6
        "database/sql"
7
        "errors"
8
        "fmt"
9
        "io"
10
        "io/fs"
11
        "net/http"
12
        "sort"
13
        "strings"
14
        "time"
15

16
        "github.com/btcsuite/btclog/v2"
17
        "github.com/golang-migrate/migrate/v4"
18
        "github.com/golang-migrate/migrate/v4/database"
19
        "github.com/golang-migrate/migrate/v4/source/httpfs"
20
        "github.com/lightningnetwork/lnd/sqldb/sqlc"
21
)
22

23
var (
24
        // migrationConfig defines a list of migrations to be applied to the
25
        // database. Each migration is assigned a version number, determining
26
        // its execution order.
27
        // The schema version, tracked by golang-migrate, ensures migrations are
28
        // applied to the correct schema. For migrations involving only schema
29
        // changes, the migration function can be left nil. For custom
30
        // migrations an implemented migration function is required.
31
        //
32
        // NOTE: The migration function may have runtime dependencies, which
33
        // must be injected during runtime.
34
        migrationConfig = []MigrationConfig{
35
                {
36
                        Name:          "000001_invoices",
37
                        Version:       1,
38
                        SchemaVersion: 1,
39
                },
40
                {
41
                        Name:          "000002_amp_invoices",
42
                        Version:       2,
43
                        SchemaVersion: 2,
44
                },
45
                {
46
                        Name:          "000003_invoice_events",
47
                        Version:       3,
48
                        SchemaVersion: 3,
49
                },
50
                {
51
                        Name:          "000004_invoice_expiry_fix",
52
                        Version:       4,
53
                        SchemaVersion: 4,
54
                },
55
                {
56
                        Name:          "000005_migration_tracker",
57
                        Version:       5,
58
                        SchemaVersion: 5,
59
                },
60
                {
61
                        Name:          "000006_invoice_migration",
62
                        Version:       6,
63
                        SchemaVersion: 6,
64
                        // A migration function is may be attached to this
65
                        // migration to migrate KV invoices to the native SQL
66
                        // schema. This is optional and can be disabled by the
67
                        // user.
68
                },
69
        }
70
)
71

72
// MigrationConfig is a configuration struct that is used to describe in-code
73
// migration targets. Each such migration is applied at a specific schema
74
// version.
75
type MigrationConfig struct {
76
        // Name is the name of the migration.
77
        Name string
78

79
        // Version represents the "global" database version for this migration.
80
        // Unlike the schema version tracked by golang-migrate, it encompasses
81
        // all migrations, including those managed by golang-migrate as well
82
        // as custom in-code migrations.
83
        Version int
84

85
        // SchemaVersion is the golang-migrate tracked schema version at which
86
        // the migration is applied.
87
        SchemaVersion int
88

89
        // MigrationFn is the migration function that is applied at the given
90
        // version. It can be used to perform custom migrations that are not
91
        // covered by SQL migrations.
92
        MigrationFn func(tx *sqlc.Queries) error
93
}
94

95
// MigrationTarget is a functional option that can be passed to applyMigrations
96
// to specify a target version to migrate to.
97
type MigrationTarget func(mig *migrate.Migrate) error
98

99
// MigrationExecutor is an interface that abstracts the migration functionality.
100
type MigrationExecutor interface {
101
        // CurrentSchemaVersion returns the current schema version of the
102
        // database.
103
        CurrentSchemaVersion() (int, error)
104

105
        // ExecuteMigrations runs migrations for the database, depending on the
106
        // target given, either all migrations or up to a given version.
107
        ExecuteMigrations(target MigrationTarget) error
108
}
109

110
var (
111
        // TargetLatest is a MigrationTarget that migrates to the latest
112
        // version available.
113
        TargetLatest = func(mig *migrate.Migrate) error {
×
114
                return mig.Up()
×
115
        }
×
116

117
        // TargetVersion is a MigrationTarget that migrates to the given
118
        // version.
119
        TargetVersion = func(version uint) MigrationTarget {
3,066✔
120
                return func(mig *migrate.Migrate) error {
6,132✔
121
                        return mig.Migrate(version)
3,066✔
122
                }
3,066✔
123
        }
124
)
125

126
// GetMigrations returns a copy of the migration configuration.
127
func GetMigrations() []MigrationConfig {
506✔
128
        migrations := make([]MigrationConfig, len(migrationConfig))
506✔
129
        copy(migrations, migrationConfig)
506✔
130

506✔
131
        return migrations
506✔
132
}
506✔
133

134
// migrationLogger is a logger that wraps the passed btclog.Logger so it can be
135
// used to log migrations.
136
type migrationLogger struct {
137
        log btclog.Logger
138
}
139

140
// Printf is like fmt.Printf. We map this to the target logger based on the
141
// current log level.
142
func (m *migrationLogger) Printf(format string, v ...interface{}) {
3,060✔
143
        // Trim trailing newlines from the format.
3,060✔
144
        format = strings.TrimRight(format, "\n")
3,060✔
145

3,060✔
146
        switch m.log.Level() {
3,060✔
147
        case btclog.LevelTrace:
×
148
                m.log.Tracef(format, v...)
×
149
        case btclog.LevelDebug:
×
150
                m.log.Debugf(format, v...)
×
151
        case btclog.LevelInfo:
×
152
                m.log.Infof(format, v...)
×
153
        case btclog.LevelWarn:
×
154
                m.log.Warnf(format, v...)
×
155
        case btclog.LevelError:
×
156
                m.log.Errorf(format, v...)
×
157
        case btclog.LevelCritical:
×
158
                m.log.Criticalf(format, v...)
×
159
        case btclog.LevelOff:
3,060✔
160
        }
161
}
162

163
// Verbose should return true when verbose logging output is wanted
164
func (m *migrationLogger) Verbose() bool {
9,180✔
165
        return m.log.Level() <= btclog.LevelDebug
9,180✔
166
}
9,180✔
167

168
// applyMigrations executes all database migration files found in the given file
169
// system under the given path, using the passed database driver and database
170
// name.
171
func applyMigrations(fs fs.FS, driver database.Driver, path,
172
        dbName string, targetVersion MigrationTarget) error {
3,066✔
173

3,066✔
174
        // With the migrate instance open, we'll create a new migration source
3,066✔
175
        // using the embedded file system stored in sqlSchemas. The library
3,066✔
176
        // we're using can't handle a raw file system interface, so we wrap it
3,066✔
177
        // in this intermediate layer.
3,066✔
178
        migrateFileServer, err := httpfs.New(http.FS(fs), path)
3,066✔
179
        if err != nil {
3,066✔
180
                return err
×
181
        }
×
182

183
        // Finally, we'll run the migration with our driver above based on the
184
        // open DB, and also the migration source stored in the file system
185
        // above.
186
        sqlMigrate, err := migrate.NewWithInstance(
3,066✔
187
                "migrations", migrateFileServer, dbName, driver,
3,066✔
188
        )
3,066✔
189
        if err != nil {
3,066✔
190
                return err
×
191
        }
×
192

193
        migrationVersion, _, err := sqlMigrate.Version()
3,066✔
194
        if err != nil && !errors.Is(err, migrate.ErrNilVersion) {
3,066✔
195
                log.Errorf("Unable to determine current migration version: %v",
×
196
                        err)
×
197

×
198
                return err
×
199
        }
×
200

201
        log.Infof("Applying migrations from version=%v", migrationVersion)
3,066✔
202

3,066✔
203
        // Apply our local logger to the migration instance.
3,066✔
204
        sqlMigrate.Log = &migrationLogger{log}
3,066✔
205

3,066✔
206
        // Execute the migration based on the target given.
3,066✔
207
        err = targetVersion(sqlMigrate)
3,066✔
208
        if err != nil && !errors.Is(err, migrate.ErrNoChange) {
3,066✔
209
                return err
×
210
        }
×
211

212
        return nil
3,066✔
213
}
214

215
// replacerFS is an implementation of a fs.FS virtual file system that wraps an
216
// existing file system but does a search-and-replace operation on each file
217
// when it is opened.
218
type replacerFS struct {
219
        parentFS fs.FS
220
        replaces map[string]string
221
}
222

223
// A compile-time assertion to make sure replacerFS implements the fs.FS
224
// interface.
225
var _ fs.FS = (*replacerFS)(nil)
226

227
// newReplacerFS creates a new replacer file system, wrapping the given parent
228
// virtual file system. Each file within the file system is undergoing a
229
// search-and-replace operation when it is opened, using the given map where the
230
// key denotes the search term and the value the term to replace each occurrence
231
// with.
232
func newReplacerFS(parent fs.FS, replaces map[string]string) *replacerFS {
3,066✔
233
        return &replacerFS{
3,066✔
234
                parentFS: parent,
3,066✔
235
                replaces: replaces,
3,066✔
236
        }
3,066✔
237
}
3,066✔
238

239
// Open opens a file in the virtual file system.
240
//
241
// NOTE: This is part of the fs.FS interface.
242
func (t *replacerFS) Open(name string) (fs.File, error) {
11,744✔
243
        f, err := t.parentFS.Open(name)
11,744✔
244
        if err != nil {
11,744✔
245
                return nil, err
×
246
        }
×
247

248
        stat, err := f.Stat()
11,744✔
249
        if err != nil {
11,744✔
250
                return nil, err
×
251
        }
×
252

253
        if stat.IsDir() {
14,810✔
254
                return f, err
3,066✔
255
        }
3,066✔
256

257
        return newReplacerFile(f, t.replaces)
8,678✔
258
}
259

260
type replacerFile struct {
261
        parentFile fs.File
262
        buf        bytes.Buffer
263
}
264

265
// A compile-time assertion to make sure replacerFile implements the fs.File
266
// interface.
267
var _ fs.File = (*replacerFile)(nil)
268

269
func newReplacerFile(parent fs.File, replaces map[string]string) (*replacerFile,
270
        error) {
8,678✔
271

8,678✔
272
        content, err := io.ReadAll(parent)
8,678✔
273
        if err != nil {
8,678✔
274
                return nil, err
×
275
        }
×
276

277
        contentStr := string(content)
8,678✔
278
        for from, to := range replaces {
30,373✔
279
                contentStr = strings.ReplaceAll(contentStr, from, to)
21,695✔
280
        }
21,695✔
281

282
        var buf bytes.Buffer
8,678✔
283
        _, err = buf.WriteString(contentStr)
8,678✔
284
        if err != nil {
8,678✔
285
                return nil, err
×
286
        }
×
287

288
        return &replacerFile{
8,678✔
289
                parentFile: parent,
8,678✔
290
                buf:        buf,
8,678✔
291
        }, nil
8,678✔
292
}
293

294
// Stat returns statistics/info about the file.
295
//
296
// NOTE: This is part of the fs.File interface.
297
func (t *replacerFile) Stat() (fs.FileInfo, error) {
×
298
        return t.parentFile.Stat()
×
299
}
×
300

301
// Read reads as many bytes as possible from the file into the given slice.
302
//
303
// NOTE: This is part of the fs.File interface.
304
func (t *replacerFile) Read(bytes []byte) (int, error) {
9,180✔
305
        return t.buf.Read(bytes)
9,180✔
306
}
9,180✔
307

308
// Close closes the underlying file.
309
//
310
// NOTE: This is part of the fs.File interface.
311
func (t *replacerFile) Close() error {
8,678✔
312
        // We already fully read and then closed the file when creating this
8,678✔
313
        // instance, so there's nothing to do for us here.
8,678✔
314
        return nil
8,678✔
315
}
8,678✔
316

317
// MigratonTxOptions the the implementation of the TxOptions interface for
318
// migration transactions.
319
type MigrationTxOptions struct {
320
}
321

322
// ReadOnly returns false to indicate that migration transactions are not read
323
// only.
324
func (m *MigrationTxOptions) ReadOnly() bool {
3,066✔
325
        return false
3,066✔
326
}
3,066✔
327

328
// ApplyMigrations applies the provided migrations to the database in sequence.
329
// It ensures migrations are executed in the correct order, applying both custom
330
// migration functions and SQL migrations as needed.
331
func ApplyMigrations(ctx context.Context, db *BaseDB,
332
        migrator MigrationExecutor, migrations []MigrationConfig) error {
524✔
333

524✔
334
        // Sort migrations by version to ensure they are applied in order.
524✔
335
        sort.SliceStable(migrations, func(i, j int) bool {
3,102✔
336
                return migrations[i].Version < migrations[j].Version
2,578✔
337
        })
2,578✔
338

339
        // Construct a transaction executor to apply custom migrations.
340
        executor := NewTransactionExecutor(db, func(tx *sql.Tx) *sqlc.Queries {
3,586✔
341
                return db.WithTx(tx)
3,062✔
342
        })
3,062✔
343

344
        currentVersion := 0
524✔
345
        version, err := db.GetDatabaseVersion(ctx)
524✔
346
        if err != nil && err != sql.ErrNoRows {
524✔
NEW
347
                return fmt.Errorf("error getting current database version: %w",
×
NEW
348
                        err)
×
NEW
349
        }
×
350
        if version.Valid {
536✔
351
                currentVersion = int(version.Int32)
12✔
352
        }
12✔
353

354
        for _, migration := range migrations {
3,626✔
355
                if migration.Version <= currentVersion {
3,142✔
356
                        log.Infof("Skipping migration '%s' (version %d) as it "+
40✔
357
                                "has already been applied", migration.Name,
40✔
358
                                migration.Version)
40✔
359

40✔
360
                        continue
40✔
361
                }
362

363
                log.Infof("Migrating SQL schema to version %s",
3,062✔
364
                        migration.SchemaVersion)
3,062✔
365

3,062✔
366
                // Execute SQL schema migrations up to the target version.
3,062✔
367
                err = migrator.ExecuteMigrations(
3,062✔
368
                        TargetVersion(uint(migration.SchemaVersion)),
3,062✔
369
                )
3,062✔
370
                if err != nil {
3,062✔
NEW
371
                        return fmt.Errorf("error executing schema migrations "+
×
NEW
372
                                "to target version %d: %w",
×
NEW
373
                                migration.SchemaVersion, err)
×
NEW
374
                }
×
375

376
                var opts MigrationTxOptions
3,062✔
377

3,062✔
378
                // Run the custom migration as a transaction to ensure
3,062✔
379
                // atomicity. If successful, mark the migration as complete in
3,062✔
380
                // the migration tracker table.
3,062✔
381
                err = executor.ExecTx(ctx, &opts, func(tx *sqlc.Queries) error {
6,124✔
382
                        // Apply the migration function if one is provided.
3,062✔
383
                        if migration.MigrationFn != nil {
3,088✔
384
                                log.Infof("Applying custom migration '%v' "+
26✔
385
                                        "(version %d) to schema version %d",
26✔
386
                                        migration.Name, migration.Version,
26✔
387
                                        migration.SchemaVersion)
26✔
388

26✔
389
                                err = migration.MigrationFn(tx)
26✔
390
                                if err != nil {
32✔
391
                                        return fmt.Errorf("error applying "+
6✔
392
                                                "migration '%v' (version %d) "+
6✔
393
                                                "to schema version %d: %w",
6✔
394
                                                migration.Name,
6✔
395
                                                migration.Version,
6✔
396
                                                migration.SchemaVersion, err)
6✔
397
                                }
6✔
398

399
                                log.Infof("Migration '%v' (version %d) "+
20✔
400
                                        "applied ", migration.Name,
20✔
401
                                        migration.Version)
20✔
402
                        }
403

404
                        // Mark the migration as complete by adding the version
405
                        // to the migration tracker table along with the current
406
                        // timestamp.
407
                        err = tx.SetMigration(ctx, sqlc.SetMigrationParams{
3,056✔
408
                                Version:       SQLInt32(migration.Version),
3,056✔
409
                                MigrationTime: SQLTime(time.Now()),
3,056✔
410
                        })
3,056✔
411
                        if err != nil {
3,056✔
NEW
412
                                return fmt.Errorf("error setting migration "+
×
NEW
413
                                        "version %d: %w", migration.Version,
×
NEW
414
                                        err)
×
NEW
415
                        }
×
416

417
                        return nil
3,056✔
418
                }, func() {})
3,062✔
419
                if err != nil {
3,068✔
420
                        return err
6✔
421
                }
6✔
422
        }
423

424
        return nil
518✔
425
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc