15628278788

Committed 13 Jun 2025 06:45AM UTC coverage: 68.511% (+10.2%) from 58.333%

Build # 15628278788

Build Type

Pull #9945

github

Committed by

web-flow

Commit Message

Merge e78253ccc into 35102e7c3

Pull Request Pull Request #9945: Decayed log optional migration

Run Details

104 of 128 new or added lines in 10 files covered. (81.25%)

42 existing lines in 10 files now uncovered.

134495 of 196311 relevant lines covered (68.51%)

22245.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.47

/channeldb/meta.go

package channeldb

import (
        "bytes"
        "errors"
        "fmt"

        "github.com/lightningnetwork/lnd/kvdb"
        "github.com/lightningnetwork/lnd/tlv"
)

var (
        // metaBucket stores all the meta information concerning the state of
        // the database.
        metaBucket = []byte("metadata")

        // dbVersionKey is a boltdb key and it's used for storing/retrieving
        // current database version.
        dbVersionKey = []byte("dbp")

        // dbVersionKey is a boltdb key and it's used for storing/retrieving
        // a list of optional migrations that have been applied.
        optionalVersionKey = []byte("ovk")

        // TombstoneKey is the key under which we add a tag in the source DB
        // after we've successfully and completely migrated it to the target/
        // destination DB.
        TombstoneKey = []byte("data-migration-tombstone")

        // ErrMarkerNotPresent is the error that is returned if the queried
        // marker is not present in the given database.
        ErrMarkerNotPresent = errors.New("marker not present")

        // ErrInvalidOptionalVersion is the error that is returned if the
        // optional version persisted in the database is invalid.
        ErrInvalidOptionalVersion = errors.New("invalid optional version")
)

// Meta structure holds the database meta information.
type Meta struct {
        // DbVersionNumber is the current schema version of the database.
        DbVersionNumber uint32
}

// FetchMeta fetches the metadata from boltdb and returns filled meta structure.
func (d *DB) FetchMeta() (*Meta, error) {
        var meta *Meta

        err := kvdb.View(d, func(tx kvdb.RTx) error {
                return FetchMeta(meta, tx)
        }, func() {
                meta = &Meta{}
        })
        if err != nil {
                return nil, err
        }

        return meta, nil
}

// FetchMeta is a helper function used in order to allow callers to re-use a
// database transaction.
func FetchMeta(meta *Meta, tx kvdb.RTx) error {
        metaBucket := tx.ReadBucket(metaBucket)
        if metaBucket == nil {
                return ErrMetaNotFound
        }

        data := metaBucket.Get(dbVersionKey)
        if data == nil {
                meta.DbVersionNumber = getLatestDBVersion(dbVersions)
        } else {
                meta.DbVersionNumber = byteOrder.Uint32(data)
        }

        return nil
}

// PutMeta writes the passed instance of the database met-data struct to disk.
func (d *DB) PutMeta(meta *Meta) error {
        return kvdb.Update(d, func(tx kvdb.RwTx) error {
                return putMeta(meta, tx)
        }, func() {})
}

// putMeta is an internal helper function used in order to allow callers to
// re-use a database transaction. See the publicly exported PutMeta method for
// more information.
func putMeta(meta *Meta, tx kvdb.RwTx) error {
        metaBucket, err := tx.CreateTopLevelBucket(metaBucket)
        if err != nil {
                return err
        }

        return putDbVersion(metaBucket, meta)
}

func putDbVersion(metaBucket kvdb.RwBucket, meta *Meta) error {
        scratch := make([]byte, 4)
        byteOrder.PutUint32(scratch, meta.DbVersionNumber)
        return metaBucket.Put(dbVersionKey, scratch)
}

// OptionalMeta structure holds the database optional migration information.
type OptionalMeta struct {
        // Versions is a set that contains the versions that have been applied.
        // When saved to disk, only the indexes are stored.
        Versions map[uint64]string
}

func (om *OptionalMeta) String() string {
        s := ""
        for index, name := range om.Versions {
                s += fmt.Sprintf("%d: %s", index, name)
                if index != uint64(len(om.Versions)-1) {
                        s += ", "
                }
        }
        if s == "" {
                s = "empty"
        }
        return s
}

// fetchOptionalMeta reads the optional meta from the database.
func (d *DB) fetchOptionalMeta() (*OptionalMeta, error) {
        om := &OptionalMeta{
                Versions: make(map[uint64]string),
        }

        err := kvdb.View(d, func(tx kvdb.RTx) error {
                metaBucket := tx.ReadBucket(metaBucket)
                if metaBucket == nil {
                        return ErrMetaNotFound
                }

                vBytes := metaBucket.Get(optionalVersionKey)
                // Exit early if nothing found.
                if vBytes == nil {
                        return nil
                }

                // Read the versions' length.
                r := bytes.NewReader(vBytes)
                vLen, err := tlv.ReadVarInt(r, &[8]byte{})
                if err != nil {
                        return err
                }

                // Write the version index.
                for i := uint64(0); i < vLen; i++ {
                        version, err := tlv.ReadVarInt(r, &[8]byte{})
                        if err != nil {
                                return err
                        }

                        if version >= uint64(len(optionalVersions)) {
                                return ErrInvalidOptionalVersion
                        }

                        om.Versions[version] = optionalVersions[version].name
                }

                return nil
        }, func() {})
        if err != nil {
                return nil, err
        }

        return om, nil
}

// putOptionalMeta writes an optional meta to the database.
func (d *DB) putOptionalMeta(om *OptionalMeta) error {
        return kvdb.Update(d, func(tx kvdb.RwTx) error {
                metaBucket, err := tx.CreateTopLevelBucket(metaBucket)
                if err != nil {
                        return err
                }

                var b bytes.Buffer

                // Write the total length.
                err = tlv.WriteVarInt(&b, uint64(len(om.Versions)), &[8]byte{})
                if err != nil {
                        return err
                }

                // Write the version indexes.
                for v := range om.Versions {
                        if v >= uint64(len(optionalVersions)) {
                                return ErrInvalidOptionalVersion
                        }

                        err := tlv.WriteVarInt(&b, v, &[8]byte{})
                        if err != nil {
                                return err
                        }
                }

                return metaBucket.Put(optionalVersionKey, b.Bytes())
        }, func() {})
}

// CheckMarkerPresent returns the marker under the requested key or
// ErrMarkerNotFound if either the root bucket or the marker key within that
// bucket does not exist.
func CheckMarkerPresent(tx kvdb.RTx, markerKey []byte) ([]byte, error) {
        markerBucket := tx.ReadBucket(markerKey)
        if markerBucket == nil {
                return nil, ErrMarkerNotPresent
        }

        val := markerBucket.Get(markerKey)

        // If we wrote the marker correctly, we created a bucket _and_ created a
        // key with a non-empty value. It doesn't matter to us whether the key
        // exists or whether its value is empty, to us, it just means the marker
        // isn't there.
        if len(val) == 0 {
                return nil, ErrMarkerNotPresent
        }

        return val, nil
}

// EnsureNoTombstone returns an error if there is a tombstone marker in the DB
// of the given transaction.
func EnsureNoTombstone(tx kvdb.RTx) error {
        marker, err := CheckMarkerPresent(tx, TombstoneKey)
        if err == ErrMarkerNotPresent {
                // No marker present, so no tombstone. The DB is still alive.
                return nil
        }
        if err != nil {
                return err
        }

        // There was no error so there is a tombstone marker/tag. We cannot use
        // this DB anymore.
        return fmt.Errorf("refusing to use db, it was marked with a tombstone "+
                "after successful data migration; tombstone reads: %s",
                string(marker))
}

// AddMarker adds the marker with the given key into a top level bucket with the
// same name. So the structure will look like:
//
//        marker-key (top level bucket)
//            |->   marker-key:marker-value (key/value pair)
func AddMarker(tx kvdb.RwTx, markerKey, markerValue []byte) error {
        if len(markerValue) == 0 {
                return fmt.Errorf("marker value cannot be empty")
        }

        markerBucket, err := tx.CreateTopLevelBucket(markerKey)
        if err != nil {
                return err
        }

        return markerBucket.Put(markerKey, markerValue)
}

1	package channeldb
2
3	import (
4	"bytes"
5	"errors"
6	"fmt"
7
8	"github.com/lightningnetwork/lnd/kvdb"
9	"github.com/lightningnetwork/lnd/tlv"
10	)
11
12	var (
13	// metaBucket stores all the meta information concerning the state of
14	// the database.
15	metaBucket = []byte("metadata")
16
17	// dbVersionKey is a boltdb key and it's used for storing/retrieving
18	// current database version.
19	dbVersionKey = []byte("dbp")
20
21	// dbVersionKey is a boltdb key and it's used for storing/retrieving
22	// a list of optional migrations that have been applied.
23	optionalVersionKey = []byte("ovk")
24
25	// TombstoneKey is the key under which we add a tag in the source DB
26	// after we've successfully and completely migrated it to the target/
27	// destination DB.
28	TombstoneKey = []byte("data-migration-tombstone")
29
30	// ErrMarkerNotPresent is the error that is returned if the queried
31	// marker is not present in the given database.
32	ErrMarkerNotPresent = errors.New("marker not present")
33
34	// ErrInvalidOptionalVersion is the error that is returned if the
35	// optional version persisted in the database is invalid.
36	ErrInvalidOptionalVersion = errors.New("invalid optional version")
37	)
38
39	// Meta structure holds the database meta information.
40	type Meta struct {
41	// DbVersionNumber is the current schema version of the database.
42	DbVersionNumber uint32
43	}
44
45	// FetchMeta fetches the metadata from boltdb and returns filled meta structure.
46	func (d DB) FetchMeta() (Meta, error) {	1,762✔
47	var meta *Meta	1,762✔
48		1,762✔
49	err := kvdb.View(d, func(tx kvdb.RTx) error {	3,524✔
50	return FetchMeta(meta, tx)	1,762✔
51	}, func() {	3,524✔
52	meta = &Meta{}	1,762✔
53	})	1,762✔
54	if err != nil {	1,762✔
55	return nil, err	×
56	}	×
57
58	return meta, nil	1,762✔
59	}
60
61	// FetchMeta is a helper function used in order to allow callers to re-use a
62	// database transaction.
63	func FetchMeta(meta *Meta, tx kvdb.RTx) error {	3,689✔
64	metaBucket := tx.ReadBucket(metaBucket)	3,689✔
65	if metaBucket == nil {	5,411✔
66	return ErrMetaNotFound	1,722✔
67	}	1,722✔
68
69	data := metaBucket.Get(dbVersionKey)	1,970✔
70	if data == nil {	1,970✔
71	meta.DbVersionNumber = getLatestDBVersion(dbVersions)	×
72	} else {	1,970✔
73	meta.DbVersionNumber = byteOrder.Uint32(data)	1,970✔
74	}	1,970✔
75
76	return nil	1,970✔
77	}
78
79	// PutMeta writes the passed instance of the database met-data struct to disk.
80	func (d DB) PutMeta(meta Meta) error {	5✔
81	return kvdb.Update(d, func(tx kvdb.RwTx) error {	10✔
82	return putMeta(meta, tx)	5✔
83	}, func() {})	10✔
84	}
85
86	// putMeta is an internal helper function used in order to allow callers to
87	// re-use a database transaction. See the publicly exported PutMeta method for
88	// more information.
89	func putMeta(meta *Meta, tx kvdb.RwTx) error {	1,730✔
90	metaBucket, err := tx.CreateTopLevelBucket(metaBucket)	1,730✔
91	if err != nil {	1,730✔
92	return err	×
93	}	×
94
95	return putDbVersion(metaBucket, meta)	1,730✔
96	}
97
98	func putDbVersion(metaBucket kvdb.RwBucket, meta *Meta) error {	1,730✔
99	scratch := make([]byte, 4)	1,730✔
100	byteOrder.PutUint32(scratch, meta.DbVersionNumber)	1,730✔
101	return metaBucket.Put(dbVersionKey, scratch)	1,730✔
102	}	1,730✔
103
104	// OptionalMeta structure holds the database optional migration information.
105	type OptionalMeta struct {
106	// Versions is a set that contains the versions that have been applied.
107	// When saved to disk, only the indexes are stored.
108	Versions map[uint64]string
109	}
110
111	func (om *OptionalMeta) String() string {	1✔
112	s := ""	1✔
113	for index, name := range om.Versions {	3✔
114	s += fmt.Sprintf("%d: %s", index, name)	2✔
115	if index != uint64(len(om.Versions)-1) {	3✔
116	s += ", "	1✔
117	}	1✔
118	}
119	if s == "" {	1✔
UNCOV 120	s = "empty"	×
UNCOV 121	}	×
122	return s	1✔
123	}
124
125	// fetchOptionalMeta reads the optional meta from the database.
126	func (d DB) fetchOptionalMeta() (OptionalMeta, error) {	1,757✔
127	om := &OptionalMeta{	1,757✔
128	Versions: make(map[uint64]string),	1,757✔
129	}	1,757✔
130		1,757✔
131	err := kvdb.View(d, func(tx kvdb.RTx) error {	3,514✔
132	metaBucket := tx.ReadBucket(metaBucket)	1,757✔
133	if metaBucket == nil {	1,757✔
134	return ErrMetaNotFound	×
135	}	×
136
137	vBytes := metaBucket.Get(optionalVersionKey)	1,757✔
138	// Exit early if nothing found.	1,757✔
139	if vBytes == nil {	3,511✔
140	return nil	1,754✔
141	}	1,754✔
142
143	// Read the versions' length.
144	r := bytes.NewReader(vBytes)	6✔
145	vLen, err := tlv.ReadVarInt(r, &[8]byte{})	6✔
146	if err != nil {	6✔
147	return err	×
148	}	×
149
150	// Write the version index.
151	for i := uint64(0); i < vLen; i++ {	15✔
152	version, err := tlv.ReadVarInt(r, &[8]byte{})	9✔
153	if err != nil {	9✔
154	return err	×
155	}	×
156
157	if version >= uint64(len(optionalVersions)) {	9✔
NEW 158	return ErrInvalidOptionalVersion	×
NEW 159	}	×
160
161	om.Versions[version] = optionalVersions[version].name	9✔
162	}
163
164	return nil	6✔
165	}, func() {})	1,757✔
166	if err != nil {	1,757✔
167	return nil, err	×
168	}	×
169
170	return om, nil	1,757✔
171	}
172
173	// putOptionalMeta writes an optional meta to the database.
174	func (d DB) putOptionalMeta(om OptionalMeta) error {	6✔
175	return kvdb.Update(d, func(tx kvdb.RwTx) error {	12✔
176	metaBucket, err := tx.CreateTopLevelBucket(metaBucket)	6✔
177	if err != nil {	6✔
178	return err	×
179	}	×
180
181	var b bytes.Buffer	6✔
182		6✔
183	// Write the total length.	6✔
184	err = tlv.WriteVarInt(&b, uint64(len(om.Versions)), &[8]byte{})	6✔
185	if err != nil {	6✔
186	return err	×
187	}	×
188
189	// Write the version indexes.
190	for v := range om.Versions {	14✔
191	if v >= uint64(len(optionalVersions)) {	8✔
NEW 192	return ErrInvalidOptionalVersion	×
NEW 193	}	×
194
195	err := tlv.WriteVarInt(&b, v, &[8]byte{})	8✔
196	if err != nil {	8✔
197	return err	×
198	}	×
199	}
200
201	return metaBucket.Put(optionalVersionKey, b.Bytes())	6✔
202	}, func() {})	6✔
203	}
204
205	// CheckMarkerPresent returns the marker under the requested key or
206	// ErrMarkerNotFound if either the root bucket or the marker key within that
207	// bucket does not exist.
208	func CheckMarkerPresent(tx kvdb.RTx, markerKey []byte) ([]byte, error) {	1,937✔
209	markerBucket := tx.ReadBucket(markerKey)	1,937✔
210	if markerBucket == nil {	3,869✔
211	return nil, ErrMarkerNotPresent	1,932✔
212	}	1,932✔
213
214	val := markerBucket.Get(markerKey)	5✔
215		5✔
216	// If we wrote the marker correctly, we created a bucket _and_ created a	5✔
217	// key with a non-empty value. It doesn't matter to us whether the key	5✔
218	// exists or whether its value is empty, to us, it just means the marker	5✔
219	// isn't there.	5✔
220	if len(val) == 0 {	6✔
221	return nil, ErrMarkerNotPresent	1✔
222	}	1✔
223
224	return val, nil	4✔
225	}
226
227	// EnsureNoTombstone returns an error if there is a tombstone marker in the DB
228	// of the given transaction.
229	func EnsureNoTombstone(tx kvdb.RTx) error {	1,933✔
230	marker, err := CheckMarkerPresent(tx, TombstoneKey)	1,933✔
231	if err == ErrMarkerNotPresent {	3,864✔
232	// No marker present, so no tombstone. The DB is still alive.	1,931✔
233	return nil	1,931✔
234	}	1,931✔
235	if err != nil {	2✔
236	return err	×
237	}	×
238
239	// There was no error so there is a tombstone marker/tag. We cannot use
240	// this DB anymore.
241	return fmt.Errorf("refusing to use db, it was marked with a tombstone "+	2✔
242	"after successful data migration; tombstone reads: %s",	2✔
243	string(marker))	2✔
244	}
245
246	// AddMarker adds the marker with the given key into a top level bucket with the
247	// same name. So the structure will look like:
248	//
249	// marker-key (top level bucket)
250	// \|-> marker-key:marker-value (key/value pair)
251	func AddMarker(tx kvdb.RwTx, markerKey, markerValue []byte) error {	2✔
252	if len(markerValue) == 0 {	2✔
253	return fmt.Errorf("marker value cannot be empty")	×
254	}	×
255
256	markerBucket, err := tx.CreateTopLevelBucket(markerKey)	2✔
257	if err != nil {	2✔
258	return err	×
259	}	×
260
261	return markerBucket.Put(markerKey, markerValue)	2✔
262	}

lightningnetwork / lnd / 15628278788

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous