12203658024

Committed 06 Dec 2024 05:54PM UTC coverage: 58.965% (+9.2%) from 49.807%

Build # 12203658024

Build Type

Pull #8831

github

Committed by

bhandras

Commit Message

docs: update release notes for 0.19.0

Pull Request Pull Request #8831: invoices: migrate KV invoices to native SQL for users of KV SQL backends

Run Details

506 of 695 new or added lines in 12 files covered. (72.81%)

67 existing lines in 19 files now uncovered.

133874 of 227038 relevant lines covered (58.97%)

19659.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.44

/sqldb/sqlite.go

//go:build !js && !(windows && (arm || 386)) && !(linux && (ppc64 || mips || mipsle || mips64))

package sqldb

import (
        "context"
        "database/sql"
        "fmt"
        "net/url"
        "path/filepath"
        "testing"

        sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite"
        "github.com/lightningnetwork/lnd/sqldb/sqlc"
        "github.com/stretchr/testify/require"
        _ "modernc.org/sqlite" // Register relevant drivers.
)

const (
        // sqliteOptionPrefix is the string prefix sqlite uses to set various
        // options. This is used in the following format:
        //   * sqliteOptionPrefix || option_name = option_value.
        sqliteOptionPrefix = "_pragma"

        // sqliteTxLockImmediate is a dsn option used to ensure that write
        // transactions are started immediately.
        sqliteTxLockImmediate = "_txlock=immediate"
)

var (
        // sqliteSchemaReplacements is a map of schema strings that need to be
        // replaced for sqlite. This is needed because sqlite doesn't directly
        // support the BIGINT type for primary keys, so we need to replace it
        // with INTEGER.
        sqliteSchemaReplacements = map[string]string{
                "BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY",
        }

        // Make sure SqliteStore implements the MigrationExecutor interface.
        _ MigrationExecutor = (*SqliteStore)(nil)

        // Make sure SqliteStore implements the DB interface.
        _ DB = (*SqliteStore)(nil)
)

// SqliteStore is a database store implementation that uses a sqlite backend.
type SqliteStore struct {
        cfg *SqliteConfig

        *BaseDB
}

// NewSqliteStore attempts to open a new sqlite database based on the passed
// config.
func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) {
        // The set of pragma options are accepted using query options. For now
        // we only want to ensure that foreign key constraints are properly
        // enforced.
        pragmaOptions := []struct {
                name  string
                value string
        }{
                {
                        name:  "foreign_keys",
                        value: "on",
                },
                {
                        name:  "journal_mode",
                        value: "WAL",
                },
                {
                        name:  "busy_timeout",
                        value: "5000",
                },
                {
                        // With the WAL mode, this ensures that we also do an
                        // extra WAL sync after each transaction. The normal
                        // sync mode skips this and gives better performance,
                        // but risks durability.
                        name:  "synchronous",
                        value: "full",
                },
                {
                        // This is used to ensure proper durability for users
                        // running on Mac OS. It uses the correct fsync system
                        // call to ensure items are fully flushed to disk.
                        name:  "fullfsync",
                        value: "true",
                },
        }
        sqliteOptions := make(url.Values)
        for _, option := range pragmaOptions {
                sqliteOptions.Add(
                        sqliteOptionPrefix,
                        fmt.Sprintf("%v=%v", option.name, option.value),
                )
        }

        // Construct the DSN which is just the database file name, appended
        // with the series of pragma options as a query URL string. For more
        // details on the formatting here, see the modernc.org/sqlite docs:
        // https://pkg.go.dev/modernc.org/sqlite#Driver.Open.
        dsn := fmt.Sprintf(
                "%v?%v&%v", dbPath, sqliteOptions.Encode(),
                sqliteTxLockImmediate,
        )
        db, err := sql.Open("sqlite", dsn)
        if err != nil {
                return nil, err
        }

        // Create the migration tracker table before starting migrations to
        // ensure it can be used to track migration progress. Note that a
        // corresponding SQLC migration also creates this table, making this
        // operation a no-op in that context. Its purpose is to ensure
        // compatibility with SQLC query generation.
        migrationTrackerSQL := `
        CREATE TABLE IF NOT EXISTS migration_tracker (
                version INTEGER UNIQUE,
                migration_time TIMESTAMP
        );`

        _, err = db.Exec(migrationTrackerSQL)
        if err != nil {
                return nil, fmt.Errorf("error creating migration tracker: %w",
                        err)
        }

        db.SetMaxOpenConns(defaultMaxConns)
        db.SetMaxIdleConns(defaultMaxConns)
        db.SetConnMaxLifetime(connIdleLifetime)
        queries := sqlc.New(db)

        s := &SqliteStore{
                cfg: cfg,
                BaseDB: &BaseDB{
                        DB:      db,
                        Queries: queries,
                },
        }

        return s, nil
}

// GetBaseDB returns the underlying BaseDB instance for the SQLite store.
// It is a trivial helper method to comply with the sqldb.DB interface.
func (s *SqliteStore) GetBaseDB() *BaseDB {
        return s.BaseDB
}

// ApplyAllMigrations applices both the SQLC and custom in-code migrations to
// the SQLite database.
func (s *SqliteStore) ApplyAllMigrations(migrations []MigrationConfig) error {
        // Execute migrations unless configured to skip them.
        if s.cfg.SkipMigrations {
                return nil
        }

        return ApplyMigrations(
                context.Background(), s.BaseDB, s, migrations,
        )
}

// CurrentSchemaVersion returns the current schema version of the SQLite
// database.
func (s *SqliteStore) CurrentSchemaVersion() (int, error) {
        driver, err := sqlite_migrate.WithInstance(
                s.DB, &sqlite_migrate.Config{},
        )
        if err != nil {
                return 0, fmt.Errorf("error creating SQLite migrator: %w",
                        err)
        }

        version, _, err := driver.Version()
        if err != nil {
                return 0, fmt.Errorf("error getting current version: %w", err)
        }

        return version, nil
}

// ExecuteMigrations runs migrations for the sqlite database, depending on the
// target given, either all migrations or up to a given version.
func (s *SqliteStore) ExecuteMigrations(target MigrationTarget) error {
        driver, err := sqlite_migrate.WithInstance(
                s.DB, &sqlite_migrate.Config{},
        )
        if err != nil {
                return fmt.Errorf("error creating sqlite migration: %w", err)
        }

        // Populate the database with our set of schemas based on our embedded
        // in-memory file system.
        sqliteFS := newReplacerFS(sqlSchemas, sqliteSchemaReplacements)
        return applyMigrations(
                sqliteFS, driver, "sqlc/migrations", "sqlite", target,
        )
}

// NewTestSqliteDB is a helper function that creates an SQLite database for
// testing.
func NewTestSqliteDB(t *testing.T) *SqliteStore {
        t.Helper()

        t.Logf("Creating new SQLite DB for testing")

        // TODO(roasbeef): if we pass :memory: for the file name, then we get
        // an in mem version to speed up tests
        dbFileName := filepath.Join(t.TempDir(), "tmp.db")
        sqlDB, err := NewSqliteStore(&SqliteConfig{
                SkipMigrations: false,
        }, dbFileName)
        require.NoError(t, err)

        require.NoError(t, sqlDB.ApplyAllMigrations(GetMigrations()))

        t.Cleanup(func() {
                require.NoError(t, sqlDB.DB.Close())
        })

        return sqlDB
}

// NewTestSqliteDBWithVersion is a helper function that creates an SQLite
// database for testing and migrates it to the given version.
func NewTestSqliteDBWithVersion(t *testing.T, version uint) *SqliteStore {
        t.Helper()

        t.Logf("Creating new SQLite DB for testing, migrating to version %d",
                version)

        // TODO(roasbeef): if we pass :memory: for the file name, then we get
        // an in mem version to speed up tests
        dbFileName := filepath.Join(t.TempDir(), "tmp.db")
        sqlDB, err := NewSqliteStore(&SqliteConfig{
                SkipMigrations: true,
        }, dbFileName)
        require.NoError(t, err)

        err = sqlDB.ExecuteMigrations(TargetVersion(version))
        require.NoError(t, err)

        t.Cleanup(func() {
                require.NoError(t, sqlDB.DB.Close())
        })

        return sqlDB
}

1	//go:build !js && !(windows && (arm \|\| 386)) && !(linux && (ppc64 \|\| mips \|\| mipsle \|\| mips64))
2
3	package sqldb
4
5	import (
6	"context"
7	"database/sql"
8	"fmt"
9	"net/url"
10	"path/filepath"
11	"testing"
12
13	sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite"
14	"github.com/lightningnetwork/lnd/sqldb/sqlc"
15	"github.com/stretchr/testify/require"
16	_ "modernc.org/sqlite" // Register relevant drivers.
17	)
18
19	const (
20	// sqliteOptionPrefix is the string prefix sqlite uses to set various
21	// options. This is used in the following format:
22	// * sqliteOptionPrefix \|\| option_name = option_value.
23	sqliteOptionPrefix = "_pragma"
24
25	// sqliteTxLockImmediate is a dsn option used to ensure that write
26	// transactions are started immediately.
27	sqliteTxLockImmediate = "_txlock=immediate"
28	)
29
30	var (
31	// sqliteSchemaReplacements is a map of schema strings that need to be
32	// replaced for sqlite. This is needed because sqlite doesn't directly
33	// support the BIGINT type for primary keys, so we need to replace it
34	// with INTEGER.
35	sqliteSchemaReplacements = map[string]string{
36	"BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY",
37	}
38
39	// Make sure SqliteStore implements the MigrationExecutor interface.
40	_ MigrationExecutor = (*SqliteStore)(nil)
41
42	// Make sure SqliteStore implements the DB interface.
43	_ DB = (*SqliteStore)(nil)
44	)
45
46	// SqliteStore is a database store implementation that uses a sqlite backend.
47	type SqliteStore struct {
48	cfg *SqliteConfig
49
50	*BaseDB
51	}
52
53	// NewSqliteStore attempts to open a new sqlite database based on the passed
54	// config.
55	func NewSqliteStore(cfg SqliteConfig, dbPath string) (SqliteStore, error) {	266✔
56	// The set of pragma options are accepted using query options. For now	266✔
57	// we only want to ensure that foreign key constraints are properly	266✔
58	// enforced.	266✔
59	pragmaOptions := []struct {	266✔
60	name string	266✔
61	value string	266✔
62	}{	266✔
63	{	266✔
64	name: "foreign_keys",	266✔
65	value: "on",	266✔
66	},	266✔
67	{	266✔
68	name: "journal_mode",	266✔
69	value: "WAL",	266✔
70	},	266✔
71	{	266✔
72	name: "busy_timeout",	266✔
73	value: "5000",	266✔
74	},	266✔
75	{	266✔
76	// With the WAL mode, this ensures that we also do an	266✔
77	// extra WAL sync after each transaction. The normal	266✔
78	// sync mode skips this and gives better performance,	266✔
79	// but risks durability.	266✔
80	name: "synchronous",	266✔
81	value: "full",	266✔
82	},	266✔
83	{	266✔
84	// This is used to ensure proper durability for users	266✔
85	// running on Mac OS. It uses the correct fsync system	266✔
86	// call to ensure items are fully flushed to disk.	266✔
87	name: "fullfsync",	266✔
88	value: "true",	266✔
89	},	266✔
90	}	266✔
91	sqliteOptions := make(url.Values)	266✔
92	for _, option := range pragmaOptions {	1,596✔
93	sqliteOptions.Add(	1,330✔
94	sqliteOptionPrefix,	1,330✔
95	fmt.Sprintf("%v=%v", option.name, option.value),	1,330✔
96	)	1,330✔
97	}	1,330✔
98
99	// Construct the DSN which is just the database file name, appended
100	// with the series of pragma options as a query URL string. For more
101	// details on the formatting here, see the modernc.org/sqlite docs:
102	// https://pkg.go.dev/modernc.org/sqlite#Driver.Open.
103	dsn := fmt.Sprintf(	266✔
104	"%v?%v&%v", dbPath, sqliteOptions.Encode(),	266✔
105	sqliteTxLockImmediate,	266✔
106	)	266✔
107	db, err := sql.Open("sqlite", dsn)	266✔
108	if err != nil {	266✔
109	return nil, err	×
110	}	×
111
112	// Create the migration tracker table before starting migrations to
113	// ensure it can be used to track migration progress. Note that a
114	// corresponding SQLC migration also creates this table, making this
115	// operation a no-op in that context. Its purpose is to ensure
116	// compatibility with SQLC query generation.
117	migrationTrackerSQL := `	266✔
118	CREATE TABLE IF NOT EXISTS migration_tracker (	266✔
119	version INTEGER UNIQUE,	266✔
120	migration_time TIMESTAMP	266✔
121	);`	266✔
122		266✔
123	_, err = db.Exec(migrationTrackerSQL)	266✔
124	if err != nil {	266✔
NEW 125	return nil, fmt.Errorf("error creating migration tracker: %w",	×
NEW 126	err)	×
NEW 127	}	×
128
129	db.SetMaxOpenConns(defaultMaxConns)	266✔
130	db.SetMaxIdleConns(defaultMaxConns)	266✔
131	db.SetConnMaxLifetime(connIdleLifetime)	266✔
132	queries := sqlc.New(db)	266✔
133		266✔
134	s := &SqliteStore{	266✔
135	cfg: cfg,	266✔
136	BaseDB: &BaseDB{	266✔
137	DB: db,	266✔
138	Queries: queries,	266✔
139	},	266✔
140	}	266✔
141		266✔
142	return s, nil	266✔
143	}
144
145	// GetBaseDB returns the underlying BaseDB instance for the SQLite store.
146	// It is a trivial helper method to comply with the sqldb.DB interface.
NEW 147	func (s SqliteStore) GetBaseDB() BaseDB {	×
NEW 148	return s.BaseDB	×
NEW 149	}	×
150
151	// ApplyAllMigrations applices both the SQLC and custom in-code migrations to
152	// the SQLite database.
153	func (s *SqliteStore) ApplyAllMigrations(migrations []MigrationConfig) error {	262✔
154	// Execute migrations unless configured to skip them.	262✔
155	if s.cfg.SkipMigrations {	262✔
NEW 156	return nil	×
NEW 157	}	×
158
159	return ApplyMigrations(	262✔
160	context.Background(), s.BaseDB, s, migrations,	262✔
161	)	262✔
162	}
163
164	// CurrentSchemaVersion returns the current schema version of the SQLite
165	// database.
NEW 166	func (s *SqliteStore) CurrentSchemaVersion() (int, error) {	×
NEW 167	driver, err := sqlite_migrate.WithInstance(	×
NEW 168	s.DB, &sqlite_migrate.Config{},	×
NEW 169	)	×
NEW 170	if err != nil {	×
NEW 171	return 0, fmt.Errorf("error creating SQLite migrator: %w",	×
NEW 172	err)	×
173	}	×
174
NEW 175	version, _, err := driver.Version()	×
NEW 176	if err != nil {	×
NEW 177	return 0, fmt.Errorf("error getting current version: %w", err)	×
NEW 178	}	×
179
NEW 180	return version, nil	×
181	}
182
183	// ExecuteMigrations runs migrations for the sqlite database, depending on the
184	// target given, either all migrations or up to a given version.
185	func (s *SqliteStore) ExecuteMigrations(target MigrationTarget) error {	1,533✔
186	driver, err := sqlite_migrate.WithInstance(	1,533✔
187	s.DB, &sqlite_migrate.Config{},	1,533✔
188	)	1,533✔
189	if err != nil {	1,533✔
190	return fmt.Errorf("error creating sqlite migration: %w", err)	×
191	}	×
192
193	// Populate the database with our set of schemas based on our embedded
194	// in-memory file system.
195	sqliteFS := newReplacerFS(sqlSchemas, sqliteSchemaReplacements)	1,533✔
196	return applyMigrations(	1,533✔
197	sqliteFS, driver, "sqlc/migrations", "sqlite", target,	1,533✔
198	)	1,533✔
199	}
200
201	// NewTestSqliteDB is a helper function that creates an SQLite database for
202	// testing.
203	func NewTestSqliteDB(t testing.T) SqliteStore {	253✔
204	t.Helper()	253✔
205		253✔
206	t.Logf("Creating new SQLite DB for testing")	253✔
207		253✔
208	// TODO(roasbeef): if we pass :memory: for the file name, then we get	253✔
209	// an in mem version to speed up tests	253✔
210	dbFileName := filepath.Join(t.TempDir(), "tmp.db")	253✔
211	sqlDB, err := NewSqliteStore(&SqliteConfig{	253✔
212	SkipMigrations: false,	253✔
213	}, dbFileName)	253✔
214	require.NoError(t, err)	253✔
215		253✔
216	require.NoError(t, sqlDB.ApplyAllMigrations(GetMigrations()))	253✔
217		253✔
218	t.Cleanup(func() {	506✔
219	require.NoError(t, sqlDB.DB.Close())	253✔
220	})	253✔
221
222	return sqlDB	253✔
223	}
224
225	// NewTestSqliteDBWithVersion is a helper function that creates an SQLite
226	// database for testing and migrates it to the given version.
227	func NewTestSqliteDBWithVersion(t testing.T, version uint) SqliteStore {	1✔
228	t.Helper()	1✔
229		1✔
230	t.Logf("Creating new SQLite DB for testing, migrating to version %d",	1✔
231	version)	1✔
232		1✔
233	// TODO(roasbeef): if we pass :memory: for the file name, then we get	1✔
234	// an in mem version to speed up tests	1✔
235	dbFileName := filepath.Join(t.TempDir(), "tmp.db")	1✔
236	sqlDB, err := NewSqliteStore(&SqliteConfig{	1✔
237	SkipMigrations: true,	1✔
238	}, dbFileName)	1✔
239	require.NoError(t, err)	1✔
240		1✔
241	err = sqlDB.ExecuteMigrations(TargetVersion(version))	1✔
242	require.NoError(t, err)	1✔
243		1✔
244	t.Cleanup(func() {	2✔
245	require.NoError(t, sqlDB.DB.Close())	1✔
246	})	1✔
247
248	return sqlDB	1✔
249	}

lightningnetwork / lnd / 12203658024

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous