12203658024

Committed 06 Dec 2024 05:54PM UTC coverage: 58.965% (+9.2%) from 49.807%

Build # 12203658024

Build Type

Pull #8831

github

Committed by

bhandras

Commit Message

docs: update release notes for 0.19.0

Pull Request Pull Request #8831: invoices: migrate KV invoices to native SQL for users of KV SQL backends

Run Details

506 of 695 new or added lines in 12 files covered. (72.81%)

67 existing lines in 19 files now uncovered.

133874 of 227038 relevant lines covered (58.97%)

19659.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.34

/sqldb/postgres.go

package sqldb

import (
        "context"
        "database/sql"
        "fmt"
        "net/url"
        "path"
        "strings"
        "time"

        pgx_migrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
        _ "github.com/golang-migrate/migrate/v4/source/file" // Read migrations from files. // nolint:ll
        _ "github.com/jackc/pgx/v5"
        "github.com/lightningnetwork/lnd/sqldb/sqlc"
)

var (
        // DefaultPostgresFixtureLifetime is the default maximum time a Postgres
        // test fixture is being kept alive. After that time the docker
        // container will be terminated forcefully, even if the tests aren't
        // fully executed yet. So this time needs to be chosen correctly to be
        // longer than the longest expected individual test run time.
        DefaultPostgresFixtureLifetime = 10 * time.Minute

        // postgresSchemaReplacements is a map of schema strings that need to be
        // replaced for postgres. This is needed because we write the schemas to
        // work with sqlite primarily but in sqlc's own dialect, and postgres
        // has some differences.
        postgresSchemaReplacements = map[string]string{
                "BLOB":                "BYTEA",
                "INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY",
                "BIGINT PRIMARY KEY":  "BIGSERIAL PRIMARY KEY",
                "TIMESTAMP":           "TIMESTAMP WITHOUT TIME ZONE",
        }

        // Make sure PostgresStore implements the MigrationExecutor interface.
        _ MigrationExecutor = (*PostgresStore)(nil)

        // Make sure PostgresStore implements the DB interface.
        _ DB = (*PostgresStore)(nil)
)

// replacePasswordInDSN takes a DSN string and returns it with the password
// replaced by "***".
func replacePasswordInDSN(dsn string) (string, error) {
        // Parse the DSN as a URL
        u, err := url.Parse(dsn)
        if err != nil {
                return "", err
        }

        // Check if the URL has a user info part
        if u.User != nil {
                username := u.User.Username()

                // Reconstruct user info with "***" as password
                userInfo := username + ":***@"

                // Rebuild the DSN with the modified user info
                sanitizeDSN := strings.Replace(
                        dsn, u.User.String()+"@", userInfo, 1,
                )

                return sanitizeDSN, nil
        }

        // Return the original DSN if no user info is present
        return dsn, nil
}

// getDatabaseNameFromDSN extracts the database name from a DSN string.
func getDatabaseNameFromDSN(dsn string) (string, error) {
        // Parse the DSN as a URL
        u, err := url.Parse(dsn)
        if err != nil {
                return "", err
        }

        // The database name is the last segment of the path. Trim leading slash
        // and return the last segment.
        return path.Base(u.Path), nil
}

// PostgresStore is a database store implementation that uses a Postgres
// backend.
type PostgresStore struct {
        cfg *PostgresConfig

        *BaseDB
}

// NewPostgresStore creates a new store that is backed by a Postgres database
// backend.
func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) {
        sanitizedDSN, err := replacePasswordInDSN(cfg.Dsn)
        if err != nil {
                return nil, err
        }
        log.Infof("Using SQL database '%s'", sanitizedDSN)

        db, err := sql.Open("pgx", cfg.Dsn)
        if err != nil {
                return nil, err
        }
        // Create the migration tracker table before starting migrations to
        // ensure it can be used to track migration progress. Note that a
        // corresponding SQLC migration also creates this table, making this
        // operation a no-op in that context. Its purpose is to ensure
        // compatibility with SQLC query generation.
        migrationTrackerSQL := `
        CREATE TABLE IF NOT EXISTS migration_tracker (
                version INTEGER UNIQUE,
                migration_time TIMESTAMP
        );`

        _, err = db.Exec(migrationTrackerSQL)
        if err != nil {
                return nil, fmt.Errorf("error creating migration tracker: %w",
                        err)
        }

        maxConns := defaultMaxConns
        if cfg.MaxConnections > 0 {
                maxConns = cfg.MaxConnections
        }

        db.SetMaxOpenConns(maxConns)
        db.SetMaxIdleConns(maxConns)
        db.SetConnMaxLifetime(connIdleLifetime)

        queries := sqlc.New(db)

        return &PostgresStore{
                cfg: cfg,
                BaseDB: &BaseDB{
                        DB:      db,
                        Queries: queries,
                },
        }, nil
}

// GetBaseDB returns the underlying BaseDB instance for the Postgres store.
// It is a trivial helper method to comply with the sqldb.DB interface.
func (s *PostgresStore) GetBaseDB() *BaseDB {
        return s.BaseDB
}

// ApplyAllMigrations applices both the SQLC and custom in-code migrations to
// the Postgres database.
func (s *PostgresStore) ApplyAllMigrations(migrations []MigrationConfig) error {
        // Execute migrations unless configured to skip them.
        if s.cfg.SkipMigrations {
                return nil
        }

        return ApplyMigrations(
                context.Background(), s.BaseDB, s, migrations,
        )
}

// CurrentSchemaVersion returns the current schema version of the Postgres
// database.
func (s *PostgresStore) CurrentSchemaVersion() (int, error) {
        driver, err := pgx_migrate.WithInstance(s.DB, &pgx_migrate.Config{})
        if err != nil {
                return 0, fmt.Errorf("error creating postgres migrator: %w",
                        err)
        }

        version, _, err := driver.Version()
        if err != nil {
                return 0, fmt.Errorf("error getting current version: %w", err)
        }

        return version, nil
}

// ExecuteMigrations runs migrations for the Postgres database, depending on the
// target given, either all migrations or up to a given version.
func (s *PostgresStore) ExecuteMigrations(target MigrationTarget) error {
        dbName, err := getDatabaseNameFromDSN(s.cfg.Dsn)
        if err != nil {
                return err
        }

        driver, err := pgx_migrate.WithInstance(s.DB, &pgx_migrate.Config{})
        if err != nil {
                return fmt.Errorf("error creating postgres migration: %w", err)
        }

        // Populate the database with our set of schemas based on our embedded
        // in-memory file system.
        postgresFS := newReplacerFS(sqlSchemas, postgresSchemaReplacements)
        return applyMigrations(
                postgresFS, driver, "sqlc/migrations", dbName, target,
        )
}

1	package sqldb
2
3	import (
4	"context"
5	"database/sql"
6	"fmt"
7	"net/url"
8	"path"
9	"strings"
10	"time"
11
12	pgx_migrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
13	_ "github.com/golang-migrate/migrate/v4/source/file" // Read migrations from files. // nolint:ll
14	_ "github.com/jackc/pgx/v5"
15	"github.com/lightningnetwork/lnd/sqldb/sqlc"
16	)
17
18	var (
19	// DefaultPostgresFixtureLifetime is the default maximum time a Postgres
20	// test fixture is being kept alive. After that time the docker
21	// container will be terminated forcefully, even if the tests aren't
22	// fully executed yet. So this time needs to be chosen correctly to be
23	// longer than the longest expected individual test run time.
24	DefaultPostgresFixtureLifetime = 10 * time.Minute
25
26	// postgresSchemaReplacements is a map of schema strings that need to be
27	// replaced for postgres. This is needed because we write the schemas to
28	// work with sqlite primarily but in sqlc's own dialect, and postgres
29	// has some differences.
30	postgresSchemaReplacements = map[string]string{
31	"BLOB": "BYTEA",
32	"INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY",
33	"BIGINT PRIMARY KEY": "BIGSERIAL PRIMARY KEY",
34	"TIMESTAMP": "TIMESTAMP WITHOUT TIME ZONE",
35	}
36
37	// Make sure PostgresStore implements the MigrationExecutor interface.
38	_ MigrationExecutor = (*PostgresStore)(nil)
39
40	// Make sure PostgresStore implements the DB interface.
41	_ DB = (*PostgresStore)(nil)
42	)
43
44	// replacePasswordInDSN takes a DSN string and returns it with the password
45	// replaced by "***".
46	func replacePasswordInDSN(dsn string) (string, error) {	266✔
47	// Parse the DSN as a URL	266✔
48	u, err := url.Parse(dsn)	266✔
49	if err != nil {	266✔
50	return "", err	×
51	}	×
52
53	// Check if the URL has a user info part
54	if u.User != nil {	532✔
55	username := u.User.Username()	266✔
56		266✔
57	// Reconstruct user info with "***" as password	266✔
58	userInfo := username + ":***@"	266✔
59		266✔
60	// Rebuild the DSN with the modified user info	266✔
61	sanitizeDSN := strings.Replace(	266✔
62	dsn, u.User.String()+"@", userInfo, 1,	266✔
63	)	266✔
64		266✔
65	return sanitizeDSN, nil	266✔
66	}	266✔
67
68	// Return the original DSN if no user info is present
69	return dsn, nil	×
70	}
71
72	// getDatabaseNameFromDSN extracts the database name from a DSN string.
73	func getDatabaseNameFromDSN(dsn string) (string, error) {	1,533✔
74	// Parse the DSN as a URL	1,533✔
75	u, err := url.Parse(dsn)	1,533✔
76	if err != nil {	1,533✔
77	return "", err	×
78	}	×
79
80	// The database name is the last segment of the path. Trim leading slash
81	// and return the last segment.
82	return path.Base(u.Path), nil	1,533✔
83	}
84
85	// PostgresStore is a database store implementation that uses a Postgres
86	// backend.
87	type PostgresStore struct {
88	cfg *PostgresConfig
89
90	*BaseDB
91	}
92
93	// NewPostgresStore creates a new store that is backed by a Postgres database
94	// backend.
95	func NewPostgresStore(cfg PostgresConfig) (PostgresStore, error) {	266✔
96	sanitizedDSN, err := replacePasswordInDSN(cfg.Dsn)	266✔
97	if err != nil {	266✔
98	return nil, err	×
99	}	×
100	log.Infof("Using SQL database '%s'", sanitizedDSN)	266✔
101		266✔
102	db, err := sql.Open("pgx", cfg.Dsn)	266✔
103	if err != nil {	266✔
104	return nil, err	×
105	}	×
106	// Create the migration tracker table before starting migrations to
107	// ensure it can be used to track migration progress. Note that a
108	// corresponding SQLC migration also creates this table, making this
109	// operation a no-op in that context. Its purpose is to ensure
110	// compatibility with SQLC query generation.
111	migrationTrackerSQL := `	266✔
112	CREATE TABLE IF NOT EXISTS migration_tracker (	266✔
113	version INTEGER UNIQUE,	266✔
114	migration_time TIMESTAMP	266✔
115	);`	266✔
116		266✔
117	_, err = db.Exec(migrationTrackerSQL)	266✔
118	if err != nil {	266✔
NEW 119	return nil, fmt.Errorf("error creating migration tracker: %w",	×
NEW 120	err)	×
NEW 121	}	×
122
123	maxConns := defaultMaxConns	266✔
124	if cfg.MaxConnections > 0 {	266✔
125	maxConns = cfg.MaxConnections	×
126	}	×
127
128	db.SetMaxOpenConns(maxConns)	266✔
129	db.SetMaxIdleConns(maxConns)	266✔
130	db.SetConnMaxLifetime(connIdleLifetime)	266✔
131		266✔
132	queries := sqlc.New(db)	266✔
133		266✔
134	return &PostgresStore{	266✔
135	cfg: cfg,	266✔
136	BaseDB: &BaseDB{	266✔
137	DB: db,	266✔
138	Queries: queries,	266✔
139	},	266✔
140	}, nil	266✔
141	}
142
143	// GetBaseDB returns the underlying BaseDB instance for the Postgres store.
144	// It is a trivial helper method to comply with the sqldb.DB interface.
NEW 145	func (s PostgresStore) GetBaseDB() BaseDB {	×
NEW 146	return s.BaseDB	×
NEW 147	}	×
148
149	// ApplyAllMigrations applices both the SQLC and custom in-code migrations to
150	// the Postgres database.
151	func (s *PostgresStore) ApplyAllMigrations(migrations []MigrationConfig) error {	262✔
152	// Execute migrations unless configured to skip them.	262✔
153	if s.cfg.SkipMigrations {	262✔
NEW 154	return nil	×
NEW 155	}	×
156
157	return ApplyMigrations(	262✔
158	context.Background(), s.BaseDB, s, migrations,	262✔
159	)	262✔
160	}
161
162	// CurrentSchemaVersion returns the current schema version of the Postgres
163	// database.
NEW 164	func (s *PostgresStore) CurrentSchemaVersion() (int, error) {	×
NEW 165	driver, err := pgx_migrate.WithInstance(s.DB, &pgx_migrate.Config{})	×
NEW 166	if err != nil {	×
NEW 167	return 0, fmt.Errorf("error creating postgres migrator: %w",	×
NEW 168	err)	×
NEW 169	}	×
170
NEW 171	version, _, err := driver.Version()	×
NEW 172	if err != nil {	×
NEW 173	return 0, fmt.Errorf("error getting current version: %w", err)	×
174	}	×
175
NEW 176	return version, nil	×
177	}
178
179	// ExecuteMigrations runs migrations for the Postgres database, depending on the
180	// target given, either all migrations or up to a given version.
181	func (s *PostgresStore) ExecuteMigrations(target MigrationTarget) error {	1,533✔
182	dbName, err := getDatabaseNameFromDSN(s.cfg.Dsn)	1,533✔
183	if err != nil {	1,533✔
184	return err	×
185	}	×
186
187	driver, err := pgx_migrate.WithInstance(s.DB, &pgx_migrate.Config{})	1,533✔
188	if err != nil {	1,533✔
189	return fmt.Errorf("error creating postgres migration: %w", err)	×
190	}	×
191
192	// Populate the database with our set of schemas based on our embedded
193	// in-memory file system.
194	postgresFS := newReplacerFS(sqlSchemas, postgresSchemaReplacements)	1,533✔
195	return applyMigrations(	1,533✔
196	postgresFS, driver, "sqlc/migrations", dbName, target,	1,533✔
197	)	1,533✔
198	}

lightningnetwork / lnd / 12203658024

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous