15016440952

Committed 14 May 2025 08:51AM UTC coverage: 69.031% (+0.03%) from 68.997%

Build # 15016440952

Build Type

Pull #9692

github

Committed by

web-flow

Commit Message

Merge b7e72b2ef into b0cba7dd0

Pull Request Pull Request #9692: [graph-work-side-branch]: temp side branch for graph work

Run Details

292 of 349 new or added lines in 32 files covered. (83.67%)

45 existing lines in 13 files now uncovered.

134025 of 194151 relevant lines covered (69.03%)

22047.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

19.77

/kvdb/backend.go

//go:build !js
// +build !js

package kvdb

import (
        "context"
        "crypto/sha256"
        "encoding/binary"
        "encoding/hex"
        "fmt"
        "os"
        "path/filepath"
        "time"

        _ "github.com/btcsuite/btcwallet/walletdb/bdb" // Import to register backend.
)

const (
        // DefaultTempDBFileName is the default name of the temporary bolt DB
        // file that we'll use to atomically compact the primary DB file on
        // startup.
        DefaultTempDBFileName = "temp-dont-use.db"

        // LastCompactionFileNameSuffix is the suffix we append to the file name
        // of a database file to record the timestamp when the last compaction
        // occurred.
        LastCompactionFileNameSuffix = ".last-compacted"
)

var (
        byteOrder = binary.BigEndian
)

// fileExists returns true if the file exists, and false otherwise.
func fileExists(path string) bool {
        if _, err := os.Stat(path); err != nil {
                if os.IsNotExist(err) {
                        return false
                }
        }

        return true
}

// BoltBackendConfig is a struct that holds settings specific to the bolt
// database backend.
type BoltBackendConfig struct {
        // DBPath is the directory path in which the database file should be
        // stored.
        DBPath string

        // DBFileName is the name of the database file.
        DBFileName string

        // NoFreelistSync, if true, prevents the database from syncing its
        // freelist to disk, resulting in improved performance at the expense of
        // increased startup time.
        NoFreelistSync bool

        // AutoCompact specifies if a Bolt based database backend should be
        // automatically compacted on startup (if the minimum age of the
        // database file is reached). This will require additional disk space
        // for the compacted copy of the database but will result in an overall
        // lower database size after the compaction.
        AutoCompact bool

        // AutoCompactMinAge specifies the minimum time that must have passed
        // since a bolt database file was last compacted for the compaction to
        // be considered again.
        AutoCompactMinAge time.Duration

        // DBTimeout specifies the timeout value to use when opening the wallet
        // database.
        DBTimeout time.Duration

        // ReadOnly specifies if the database should be opened in read-only
        // mode.
        ReadOnly bool
}

// GetBoltBackend opens (or creates if doesn't exits) a bbolt backed database
// and returns a kvdb.Backend wrapping it.
func GetBoltBackend(cfg *BoltBackendConfig) (Backend, error) {
        dbFilePath := filepath.Join(cfg.DBPath, cfg.DBFileName)

        // Is this a new database?
        if !fileExists(dbFilePath) {
                if !fileExists(cfg.DBPath) {
                        if err := os.MkdirAll(cfg.DBPath, 0700); err != nil {
                                return nil, err
                        }
                }

                return Create(
                        BoltBackendName, dbFilePath,
                        cfg.NoFreelistSync, cfg.DBTimeout, cfg.ReadOnly,
                )
        }

        // This is an existing database. We might want to compact it on startup
        // to free up some space.
        if cfg.AutoCompact {
                if err := compactAndSwap(cfg); err != nil {
                        return nil, err
                }
        }

        return Open(
                BoltBackendName, dbFilePath,
                cfg.NoFreelistSync, cfg.DBTimeout, cfg.ReadOnly,
        )
}

// compactAndSwap will attempt to write a new temporary DB file to disk with
// the compacted database content, then atomically swap (via rename) the old
// file for the new file by updating the name of the new file to the old.
func compactAndSwap(cfg *BoltBackendConfig) error {
        sourceName := cfg.DBFileName

        // If the main DB file isn't set, then we can't proceed.
        if sourceName == "" {
                return fmt.Errorf("cannot compact DB with empty name")
        }
        sourceFilePath := filepath.Join(cfg.DBPath, sourceName)
        tempDestFilePath := filepath.Join(cfg.DBPath, DefaultTempDBFileName)

        // Let's find out how long ago the last compaction of the source file
        // occurred and possibly skip compacting it again now.
        lastCompactionDate, err := lastCompactionDate(sourceFilePath)
        if err != nil {
                return fmt.Errorf("cannot determine last compaction date of "+
                        "source DB file: %v", err)
        }
        compactAge := time.Since(lastCompactionDate)
        if cfg.AutoCompactMinAge != 0 && compactAge <= cfg.AutoCompactMinAge {
                log.Infof("Not compacting database file at %v, it was last "+
                        "compacted at %v (%v ago), min age is set to %v",
                        sourceFilePath, lastCompactionDate,
                        compactAge.Truncate(time.Second), cfg.AutoCompactMinAge)
                return nil
        }

        log.Infof("Compacting database file at %v", sourceFilePath)

        // If the old temporary DB file still exists, then we'll delete it
        // before proceeding.
        if _, err := os.Stat(tempDestFilePath); err == nil {
                log.Infof("Found old temp DB @ %v, removing before swap",
                        tempDestFilePath)

                err = os.Remove(tempDestFilePath)
                if err != nil {
                        return fmt.Errorf("unable to remove old temp DB file: "+
                                "%v", err)
                }
        }

        // Now that we know the staging area is clear, we'll create the new
        // temporary DB file and close it before we write the new DB to it.
        tempFile, err := os.Create(tempDestFilePath)
        if err != nil {
                return fmt.Errorf("unable to create temp DB file: %w", err)
        }
        if err := tempFile.Close(); err != nil {
                return fmt.Errorf("unable to close file: %w", err)
        }

        // With the file created, we'll start the compaction and remove the
        // temporary file all together once this method exits.
        defer func() {
                // This will only succeed if the rename below fails. If the
                // compaction is successful, the file won't exist on exit
                // anymore so no need to log an error here.
                _ = os.Remove(tempDestFilePath)
        }()
        c := &compacter{
                srcPath:   sourceFilePath,
                dstPath:   tempDestFilePath,
                dbTimeout: cfg.DBTimeout,
        }
        initialSize, newSize, err := c.execute()
        if err != nil {
                return fmt.Errorf("error during compact: %w", err)
        }

        log.Infof("DB compaction of %v successful, %d -> %d bytes (gain=%.2fx)",
                sourceFilePath, initialSize, newSize,
                float64(initialSize)/float64(newSize))

        // We try to store the current timestamp in a file with the suffix
        // .last-compacted so we can figure out how long ago the last compaction
        // was. But since this shouldn't fail the compaction process itself, we
        // only log the error. Worst case if this file cannot be written is that
        // we compact on every startup.
        err = updateLastCompactionDate(sourceFilePath)
        if err != nil {
                log.Warnf("Could not update last compaction timestamp in "+
                        "%s%s: %v", sourceFilePath,
                        LastCompactionFileNameSuffix, err)
        }

        log.Infof("Swapping old DB file from %v to %v", tempDestFilePath,
                sourceFilePath)

        // Finally, we'll attempt to atomically rename the temporary file to
        // the main back up file. If this succeeds, then we'll only have a
        // single file on disk once this method exits.
        return os.Rename(tempDestFilePath, sourceFilePath)
}

// lastCompactionDate returns the date the given database file was last
// compacted or a zero time.Time if no compaction was recorded before. The
// compaction date is read from a file in the same directory and with the same
// name as the DB file, but with the suffix ".last-compacted".
func lastCompactionDate(dbFile string) (time.Time, error) {
        zeroTime := time.Unix(0, 0)

        tsFile := fmt.Sprintf("%s%s", dbFile, LastCompactionFileNameSuffix)
        if !fileExists(tsFile) {
                return zeroTime, nil
        }

        tsBytes, err := os.ReadFile(tsFile)
        if err != nil {
                return zeroTime, err
        }

        tsNano := byteOrder.Uint64(tsBytes)
        return time.Unix(0, int64(tsNano)), nil
}

// updateLastCompactionDate stores the current time as a timestamp in a file
// in the same directory and with the same name as the DB file, but with the
// suffix ".last-compacted".
func updateLastCompactionDate(dbFile string) error {
        var tsBytes [8]byte
        byteOrder.PutUint64(tsBytes[:], uint64(time.Now().UnixNano()))

        tsFile := fmt.Sprintf("%s%s", dbFile, LastCompactionFileNameSuffix)
        return os.WriteFile(tsFile, tsBytes[:], 0600)
}

// GetTestBackend opens (or creates if doesn't exist) a bbolt or etcd
// backed database (for testing), and returns a kvdb.Backend and a cleanup
// func. Whether to create/open bbolt or embedded etcd database is based
// on the TestBackend constant which is conditionally compiled with build tag.
// The passed path is used to hold all db files, while the name is only used
// for bbolt.
func GetTestBackend(path, name string) (Backend, func(), error) {
        empty := func() {}

        // Note that for tests, we expect only one db backend build flag
        // (or none) to be set at a time and thus one of the following switch
        // cases should ever be true
        switch {
        case PostgresBackend:
                key := filepath.Join(path, name)
                keyHash := sha256.Sum256([]byte(key))

                f, err := NewPostgresFixture("test_" + hex.EncodeToString(
                        keyHash[:]),
                )
                if err != nil {
                        return nil, func() {}, err
                }
                return f.DB(), func() {
                        _ = f.DB().Close()
                }, nil

        case EtcdBackend:
                etcdConfig, cancel, err := StartEtcdTestBackend(path, 0, 0, "")
                if err != nil {
                        return nil, empty, err
                }
                backend, err := Open(
                        EtcdBackendName, context.Background(), etcdConfig,
                )
                return backend, cancel, err

        case SqliteBackend:
                dbPath := filepath.Join(path, name)
                keyHash := sha256.Sum256([]byte(dbPath))
                sqliteDb, err := StartSqliteTestBackend(
                        path, name, "test_"+hex.EncodeToString(keyHash[:]),
                )
                if err != nil {
                        return nil, empty, err
                }

                return sqliteDb, func() {
                        _ = sqliteDb.Close()
                }, nil

        default:
                db, err := GetBoltBackend(&BoltBackendConfig{
                        DBPath:         path,
                        DBFileName:     name,
                        NoFreelistSync: true,
                        DBTimeout:      DefaultDBTimeout,
                        ReadOnly:       false,
                })
                if err != nil {
                        return nil, nil, err
                }
                return db, empty, nil
        }
}

1	//go:build !js
2	// +build !js
3
4	package kvdb
5
6	import (
7	"context"
8	"crypto/sha256"
9	"encoding/binary"
10	"encoding/hex"
11	"fmt"
12	"os"
13	"path/filepath"
14	"time"
15
16	_ "github.com/btcsuite/btcwallet/walletdb/bdb" // Import to register backend.
17	)
18
19	const (
20	// DefaultTempDBFileName is the default name of the temporary bolt DB
21	// file that we'll use to atomically compact the primary DB file on
22	// startup.
23	DefaultTempDBFileName = "temp-dont-use.db"
24
25	// LastCompactionFileNameSuffix is the suffix we append to the file name
26	// of a database file to record the timestamp when the last compaction
27	// occurred.
28	LastCompactionFileNameSuffix = ".last-compacted"
29	)
30
31	var (
32	byteOrder = binary.BigEndian
33	)
34
35	// fileExists returns true if the file exists, and false otherwise.
36	func fileExists(path string) bool {	3,855✔
37	if _, err := os.Stat(path); err != nil {	5,922✔
38	if os.IsNotExist(err) {	4,134✔
39	return false	2,067✔
40	}	2,067✔
41	}
42
43	return true	1,788✔
44	}
45
46	// BoltBackendConfig is a struct that holds settings specific to the bolt
47	// database backend.
48	type BoltBackendConfig struct {
49	// DBPath is the directory path in which the database file should be
50	// stored.
51	DBPath string
52
53	// DBFileName is the name of the database file.
54	DBFileName string
55
56	// NoFreelistSync, if true, prevents the database from syncing its
57	// freelist to disk, resulting in improved performance at the expense of
58	// increased startup time.
59	NoFreelistSync bool
60
61	// AutoCompact specifies if a Bolt based database backend should be
62	// automatically compacted on startup (if the minimum age of the
63	// database file is reached). This will require additional disk space
64	// for the compacted copy of the database but will result in an overall
65	// lower database size after the compaction.
66	AutoCompact bool
67
68	// AutoCompactMinAge specifies the minimum time that must have passed
69	// since a bolt database file was last compacted for the compaction to
70	// be considered again.
71	AutoCompactMinAge time.Duration
72
73	// DBTimeout specifies the timeout value to use when opening the wallet
74	// database.
75	DBTimeout time.Duration
76
77	// ReadOnly specifies if the database should be opened in read-only
78	// mode.
79	ReadOnly bool
80	}
81
82	// GetBoltBackend opens (or creates if doesn't exits) a bbolt backed database
83	// and returns a kvdb.Backend wrapping it.
84	func GetBoltBackend(cfg *BoltBackendConfig) (Backend, error) {	2,045✔
85	dbFilePath := filepath.Join(cfg.DBPath, cfg.DBFileName)	2,045✔
86		2,045✔
87	// Is this a new database?	2,045✔
88	if !fileExists(dbFilePath) {	3,855✔
89	if !fileExists(cfg.DBPath) {	2,067✔
90	if err := os.MkdirAll(cfg.DBPath, 0700); err != nil {	257✔
91	return nil, err	×
92	}	×
93	}
94
95	return Create(	1,810✔
96	BoltBackendName, dbFilePath,	1,810✔
97	cfg.NoFreelistSync, cfg.DBTimeout, cfg.ReadOnly,	1,810✔
98	)	1,810✔
99	}
100
101	// This is an existing database. We might want to compact it on startup
102	// to free up some space.
103	if cfg.AutoCompact {	235✔
104	if err := compactAndSwap(cfg); err != nil {	×
105	return nil, err	×
106	}	×
107	}
108
109	return Open(	235✔
110	BoltBackendName, dbFilePath,	235✔
111	cfg.NoFreelistSync, cfg.DBTimeout, cfg.ReadOnly,	235✔
112	)	235✔
113	}
114
115	// compactAndSwap will attempt to write a new temporary DB file to disk with
116	// the compacted database content, then atomically swap (via rename) the old
117	// file for the new file by updating the name of the new file to the old.
118	func compactAndSwap(cfg *BoltBackendConfig) error {	×
119	sourceName := cfg.DBFileName	×
120		×
121	// If the main DB file isn't set, then we can't proceed.	×
122	if sourceName == "" {	×
123	return fmt.Errorf("cannot compact DB with empty name")	×
124	}	×
125	sourceFilePath := filepath.Join(cfg.DBPath, sourceName)	×
126	tempDestFilePath := filepath.Join(cfg.DBPath, DefaultTempDBFileName)	×
127		×
128	// Let's find out how long ago the last compaction of the source file	×
129	// occurred and possibly skip compacting it again now.	×
130	lastCompactionDate, err := lastCompactionDate(sourceFilePath)	×
131	if err != nil {	×
132	return fmt.Errorf("cannot determine last compaction date of "+	×
133	"source DB file: %v", err)	×
134	}	×
135	compactAge := time.Since(lastCompactionDate)	×
136	if cfg.AutoCompactMinAge != 0 && compactAge <= cfg.AutoCompactMinAge {	×
137	log.Infof("Not compacting database file at %v, it was last "+	×
138	"compacted at %v (%v ago), min age is set to %v",	×
139	sourceFilePath, lastCompactionDate,	×
140	compactAge.Truncate(time.Second), cfg.AutoCompactMinAge)	×
141	return nil	×
142	}	×
143
144	log.Infof("Compacting database file at %v", sourceFilePath)	×
145		×
146	// If the old temporary DB file still exists, then we'll delete it	×
147	// before proceeding.	×
148	if _, err := os.Stat(tempDestFilePath); err == nil {	×
149	log.Infof("Found old temp DB @ %v, removing before swap",	×
150	tempDestFilePath)	×
151		×
152	err = os.Remove(tempDestFilePath)	×
153	if err != nil {	×
154	return fmt.Errorf("unable to remove old temp DB file: "+	×
155	"%v", err)	×
156	}	×
157	}
158
159	// Now that we know the staging area is clear, we'll create the new
160	// temporary DB file and close it before we write the new DB to it.
161	tempFile, err := os.Create(tempDestFilePath)	×
162	if err != nil {	×
163	return fmt.Errorf("unable to create temp DB file: %w", err)	×
164	}	×
165	if err := tempFile.Close(); err != nil {	×
166	return fmt.Errorf("unable to close file: %w", err)	×
167	}	×
168
169	// With the file created, we'll start the compaction and remove the
170	// temporary file all together once this method exits.
171	defer func() {	×
172	// This will only succeed if the rename below fails. If the	×
173	// compaction is successful, the file won't exist on exit	×
174	// anymore so no need to log an error here.	×
175	_ = os.Remove(tempDestFilePath)	×
176	}()	×
177	c := &compacter{	×
178	srcPath: sourceFilePath,	×
179	dstPath: tempDestFilePath,	×
180	dbTimeout: cfg.DBTimeout,	×
181	}	×
182	initialSize, newSize, err := c.execute()	×
183	if err != nil {	×
184	return fmt.Errorf("error during compact: %w", err)	×
185	}	×
186
187	log.Infof("DB compaction of %v successful, %d -> %d bytes (gain=%.2fx)",	×
188	sourceFilePath, initialSize, newSize,	×
189	float64(initialSize)/float64(newSize))	×
190		×
191	// We try to store the current timestamp in a file with the suffix	×
192	// .last-compacted so we can figure out how long ago the last compaction	×
193	// was. But since this shouldn't fail the compaction process itself, we	×
194	// only log the error. Worst case if this file cannot be written is that	×
195	// we compact on every startup.	×
196	err = updateLastCompactionDate(sourceFilePath)	×
197	if err != nil {	×
198	log.Warnf("Could not update last compaction timestamp in "+	×
199	"%s%s: %v", sourceFilePath,	×
200	LastCompactionFileNameSuffix, err)	×
201	}	×
202
203	log.Infof("Swapping old DB file from %v to %v", tempDestFilePath,	×
204	sourceFilePath)	×
205		×
206	// Finally, we'll attempt to atomically rename the temporary file to	×
207	// the main back up file. If this succeeds, then we'll only have a	×
208	// single file on disk once this method exits.	×
209	return os.Rename(tempDestFilePath, sourceFilePath)	×
210	}
211
212	// lastCompactionDate returns the date the given database file was last
213	// compacted or a zero time.Time if no compaction was recorded before. The
214	// compaction date is read from a file in the same directory and with the same
215	// name as the DB file, but with the suffix ".last-compacted".
216	func lastCompactionDate(dbFile string) (time.Time, error) {	×
217	zeroTime := time.Unix(0, 0)	×
218		×
219	tsFile := fmt.Sprintf("%s%s", dbFile, LastCompactionFileNameSuffix)	×
220	if !fileExists(tsFile) {	×
221	return zeroTime, nil	×
222	}	×
223
224	tsBytes, err := os.ReadFile(tsFile)	×
225	if err != nil {	×
226	return zeroTime, err	×
227	}	×
228
229	tsNano := byteOrder.Uint64(tsBytes)	×
230	return time.Unix(0, int64(tsNano)), nil	×
231	}
232
233	// updateLastCompactionDate stores the current time as a timestamp in a file
234	// in the same directory and with the same name as the DB file, but with the
235	// suffix ".last-compacted".
236	func updateLastCompactionDate(dbFile string) error {	×
237	var tsBytes [8]byte	×
238	byteOrder.PutUint64(tsBytes[:], uint64(time.Now().UnixNano()))	×
239		×
240	tsFile := fmt.Sprintf("%s%s", dbFile, LastCompactionFileNameSuffix)	×
241	return os.WriteFile(tsFile, tsBytes[:], 0600)	×
242	}	×
243
244	// GetTestBackend opens (or creates if doesn't exist) a bbolt or etcd
245	// backed database (for testing), and returns a kvdb.Backend and a cleanup
246	// func. Whether to create/open bbolt or embedded etcd database is based
247	// on the TestBackend constant which is conditionally compiled with build tag.
248	// The passed path is used to hold all db files, while the name is only used
249	// for bbolt.
250	func GetTestBackend(path, name string) (Backend, func(), error) {	432✔
251	empty := func() {}	864✔
252
253	// Note that for tests, we expect only one db backend build flag
254	// (or none) to be set at a time and thus one of the following switch
255	// cases should ever be true
256	switch {	432✔
257	case PostgresBackend:	×
258	key := filepath.Join(path, name)	×
259	keyHash := sha256.Sum256([]byte(key))	×
260		×
261	f, err := NewPostgresFixture("test_" + hex.EncodeToString(	×
262	keyHash[:]),	×
263	)	×
264	if err != nil {	×
265	return nil, func() {}, err	×
266	}
267	return f.DB(), func() {	×
268	_ = f.DB().Close()	×
269	}, nil	×
270
271	case EtcdBackend:	×
272	etcdConfig, cancel, err := StartEtcdTestBackend(path, 0, 0, "")	×
273	if err != nil {	×
274	return nil, empty, err	×
275	}	×
276	backend, err := Open(	×
NEW 277	EtcdBackendName, context.Background(), etcdConfig,	×
278	)	×
279	return backend, cancel, err	×
280
281	case SqliteBackend:	×
282	dbPath := filepath.Join(path, name)	×
283	keyHash := sha256.Sum256([]byte(dbPath))	×
284	sqliteDb, err := StartSqliteTestBackend(	×
285	path, name, "test_"+hex.EncodeToString(keyHash[:]),	×
286	)	×
287	if err != nil {	×
288	return nil, empty, err	×
289	}	×
290
291	return sqliteDb, func() {	×
292	_ = sqliteDb.Close()	×
293	}, nil	×
294
295	default:	432✔
296	db, err := GetBoltBackend(&BoltBackendConfig{	432✔
297	DBPath: path,	432✔
298	DBFileName: name,	432✔
299	NoFreelistSync: true,	432✔
300	DBTimeout: DefaultDBTimeout,	432✔
301	ReadOnly: false,	432✔
302	})	432✔
303	if err != nil {	432✔
304	return nil, nil, err	×
305	}	×
306	return db, empty, nil	432✔
307	}
308	}

lightningnetwork / lnd / 15016440952

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous