13440912774

Committed 20 Feb 2025 05:14PM UTC coverage: 57.697% (-1.1%) from 58.802%

Build # 13440912774

Build Type

Pull #9535

github

Committed by

guggero

Commit Message

GitHub: remove duplicate caching

Turns out that actions/setup-go starting with @v4 also adds caching.
With that, our cache size on disk has almost doubled, leading to the
GitHub runner running out of space in certain situation.
We fix that by disabling the automated caching since we already have our
own, custom-tailored version.

Pull Request Pull Request #9535: GitHub: remove duplicate caching

Run Details

103519 of 179417 relevant lines covered (57.7%)

24825.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.15

/channeldb/meta.go

package channeldb

import (
        "bytes"
        "errors"
        "fmt"

        "github.com/lightningnetwork/lnd/kvdb"
        "github.com/lightningnetwork/lnd/tlv"
)

var (
        // metaBucket stores all the meta information concerning the state of
        // the database.
        metaBucket = []byte("metadata")

        // dbVersionKey is a boltdb key and it's used for storing/retrieving
        // current database version.
        dbVersionKey = []byte("dbp")

        // dbVersionKey is a boltdb key and it's used for storing/retrieving
        // a list of optional migrations that have been applied.
        optionalVersionKey = []byte("ovk")

        // TombstoneKey is the key under which we add a tag in the source DB
        // after we've successfully and completely migrated it to the target/
        // destination DB.
        TombstoneKey = []byte("data-migration-tombstone")

        // ErrMarkerNotPresent is the error that is returned if the queried
        // marker is not present in the given database.
        ErrMarkerNotPresent = errors.New("marker not present")
)

// Meta structure holds the database meta information.
type Meta struct {
        // DbVersionNumber is the current schema version of the database.
        DbVersionNumber uint32
}

// FetchMeta fetches the metadata from boltdb and returns filled meta structure.
func (d *DB) FetchMeta() (*Meta, error) {
        var meta *Meta

        err := kvdb.View(d, func(tx kvdb.RTx) error {
                return FetchMeta(meta, tx)
        }, func() {
                meta = &Meta{}
        })
        if err != nil {
                return nil, err
        }

        return meta, nil
}

// FetchMeta is a helper function used in order to allow callers to re-use a
// database transaction.
func FetchMeta(meta *Meta, tx kvdb.RTx) error {
        metaBucket := tx.ReadBucket(metaBucket)
        if metaBucket == nil {
                return ErrMetaNotFound
        }

        data := metaBucket.Get(dbVersionKey)
        if data == nil {
                meta.DbVersionNumber = getLatestDBVersion(dbVersions)
        } else {
                meta.DbVersionNumber = byteOrder.Uint32(data)
        }

        return nil
}

// PutMeta writes the passed instance of the database met-data struct to disk.
func (d *DB) PutMeta(meta *Meta) error {
        return kvdb.Update(d, func(tx kvdb.RwTx) error {
                return putMeta(meta, tx)
        }, func() {})
}

// putMeta is an internal helper function used in order to allow callers to
// re-use a database transaction. See the publicly exported PutMeta method for
// more information.
func putMeta(meta *Meta, tx kvdb.RwTx) error {
        metaBucket, err := tx.CreateTopLevelBucket(metaBucket)
        if err != nil {
                return err
        }

        return putDbVersion(metaBucket, meta)
}

func putDbVersion(metaBucket kvdb.RwBucket, meta *Meta) error {
        scratch := make([]byte, 4)
        byteOrder.PutUint32(scratch, meta.DbVersionNumber)
        return metaBucket.Put(dbVersionKey, scratch)
}

// OptionalMeta structure holds the database optional migration information.
type OptionalMeta struct {
        // Versions is a set that contains the versions that have been applied.
        // When saved to disk, only the indexes are stored.
        Versions map[uint64]string
}

func (om *OptionalMeta) String() string {
        s := ""
        for index, name := range om.Versions {
                s += fmt.Sprintf("%d: %s", index, name)
        }
        if s == "" {
                s = "empty"
        }
        return s
}

// fetchOptionalMeta reads the optional meta from the database.
func (d *DB) fetchOptionalMeta() (*OptionalMeta, error) {
        om := &OptionalMeta{
                Versions: make(map[uint64]string),
        }

        err := kvdb.View(d, func(tx kvdb.RTx) error {
                metaBucket := tx.ReadBucket(metaBucket)
                if metaBucket == nil {
                        return ErrMetaNotFound
                }

                vBytes := metaBucket.Get(optionalVersionKey)
                // Exit early if nothing found.
                if vBytes == nil {
                        return nil
                }

                // Read the versions' length.
                r := bytes.NewReader(vBytes)
                vLen, err := tlv.ReadVarInt(r, &[8]byte{})
                if err != nil {
                        return err
                }

                // Write the version index.
                for i := uint64(0); i < vLen; i++ {
                        version, err := tlv.ReadVarInt(r, &[8]byte{})
                        if err != nil {
                                return err
                        }
                        om.Versions[version] = optionalVersions[i].name
                }

                return nil
        }, func() {})
        if err != nil {
                return nil, err
        }

        return om, nil
}

// putOptionalMeta writes an optional meta to the database.
func (d *DB) putOptionalMeta(om *OptionalMeta) error {
        return kvdb.Update(d, func(tx kvdb.RwTx) error {
                metaBucket, err := tx.CreateTopLevelBucket(metaBucket)
                if err != nil {
                        return err
                }

                var b bytes.Buffer

                // Write the total length.
                err = tlv.WriteVarInt(&b, uint64(len(om.Versions)), &[8]byte{})
                if err != nil {
                        return err
                }

                // Write the version indexes.
                for v := range om.Versions {
                        err := tlv.WriteVarInt(&b, v, &[8]byte{})
                        if err != nil {
                                return err
                        }
                }

                return metaBucket.Put(optionalVersionKey, b.Bytes())
        }, func() {})
}

// CheckMarkerPresent returns the marker under the requested key or
// ErrMarkerNotFound if either the root bucket or the marker key within that
// bucket does not exist.
func CheckMarkerPresent(tx kvdb.RTx, markerKey []byte) ([]byte, error) {
        markerBucket := tx.ReadBucket(markerKey)
        if markerBucket == nil {
                return nil, ErrMarkerNotPresent
        }

        val := markerBucket.Get(markerKey)

        // If we wrote the marker correctly, we created a bucket _and_ created a
        // key with a non-empty value. It doesn't matter to us whether the key
        // exists or whether its value is empty, to us, it just means the marker
        // isn't there.
        if len(val) == 0 {
                return nil, ErrMarkerNotPresent
        }

        return val, nil
}

// EnsureNoTombstone returns an error if there is a tombstone marker in the DB
// of the given transaction.
func EnsureNoTombstone(tx kvdb.RTx) error {
        marker, err := CheckMarkerPresent(tx, TombstoneKey)
        if err == ErrMarkerNotPresent {
                // No marker present, so no tombstone. The DB is still alive.
                return nil
        }
        if err != nil {
                return err
        }

        // There was no error so there is a tombstone marker/tag. We cannot use
        // this DB anymore.
        return fmt.Errorf("refusing to use db, it was marked with a tombstone "+
                "after successful data migration; tombstone reads: %s",
                string(marker))
}

// AddMarker adds the marker with the given key into a top level bucket with the
// same name. So the structure will look like:
//
//        marker-key (top level bucket)
//            |->   marker-key:marker-value (key/value pair)
func AddMarker(tx kvdb.RwTx, markerKey, markerValue []byte) error {
        if len(markerValue) == 0 {
                return fmt.Errorf("marker value cannot be empty")
        }

        markerBucket, err := tx.CreateTopLevelBucket(markerKey)
        if err != nil {
                return err
        }

        return markerBucket.Put(markerKey, markerValue)
}

1	package channeldb
2
3	import (
4	"bytes"
5	"errors"
6	"fmt"
7
8	"github.com/lightningnetwork/lnd/kvdb"
9	"github.com/lightningnetwork/lnd/tlv"
10	)
11
12	var (
13	// metaBucket stores all the meta information concerning the state of
14	// the database.
15	metaBucket = []byte("metadata")
16
17	// dbVersionKey is a boltdb key and it's used for storing/retrieving
18	// current database version.
19	dbVersionKey = []byte("dbp")
20
21	// dbVersionKey is a boltdb key and it's used for storing/retrieving
22	// a list of optional migrations that have been applied.
23	optionalVersionKey = []byte("ovk")
24
25	// TombstoneKey is the key under which we add a tag in the source DB
26	// after we've successfully and completely migrated it to the target/
27	// destination DB.
28	TombstoneKey = []byte("data-migration-tombstone")
29
30	// ErrMarkerNotPresent is the error that is returned if the queried
31	// marker is not present in the given database.
32	ErrMarkerNotPresent = errors.New("marker not present")
33	)
34
35	// Meta structure holds the database meta information.
36	type Meta struct {
37	// DbVersionNumber is the current schema version of the database.
38	DbVersionNumber uint32
39	}
40
41	// FetchMeta fetches the metadata from boltdb and returns filled meta structure.
42	func (d DB) FetchMeta() (Meta, error) {	1,754✔
43	var meta *Meta	1,754✔
44		1,754✔
45	err := kvdb.View(d, func(tx kvdb.RTx) error {	3,508✔
46	return FetchMeta(meta, tx)	1,754✔
47	}, func() {	3,508✔
48	meta = &Meta{}	1,754✔
49	})	1,754✔
50	if err != nil {	1,754✔
51	return nil, err	×
52	}	×
53
54	return meta, nil	1,754✔
55	}
56
57	// FetchMeta is a helper function used in order to allow callers to re-use a
58	// database transaction.
59	func FetchMeta(meta *Meta, tx kvdb.RTx) error {	3,676✔
60	metaBucket := tx.ReadBucket(metaBucket)	3,676✔
61	if metaBucket == nil {	5,390✔
62	return ErrMetaNotFound	1,714✔
63	}	1,714✔
64
65	data := metaBucket.Get(dbVersionKey)	1,962✔
66	if data == nil {	1,962✔
67	meta.DbVersionNumber = getLatestDBVersion(dbVersions)	×
68	} else {	1,962✔
69	meta.DbVersionNumber = byteOrder.Uint32(data)	1,962✔
70	}	1,962✔
71
72	return nil	1,962✔
73	}
74
75	// PutMeta writes the passed instance of the database met-data struct to disk.
76	func (d DB) PutMeta(meta Meta) error {	5✔
77	return kvdb.Update(d, func(tx kvdb.RwTx) error {	10✔
78	return putMeta(meta, tx)	5✔
79	}, func() {})	10✔
80	}
81
82	// putMeta is an internal helper function used in order to allow callers to
83	// re-use a database transaction. See the publicly exported PutMeta method for
84	// more information.
85	func putMeta(meta *Meta, tx kvdb.RwTx) error {	1,722✔
86	metaBucket, err := tx.CreateTopLevelBucket(metaBucket)	1,722✔
87	if err != nil {	1,722✔
88	return err	×
89	}	×
90
91	return putDbVersion(metaBucket, meta)	1,722✔
92	}
93
94	func putDbVersion(metaBucket kvdb.RwBucket, meta *Meta) error {	1,722✔
95	scratch := make([]byte, 4)	1,722✔
96	byteOrder.PutUint32(scratch, meta.DbVersionNumber)	1,722✔
97	return metaBucket.Put(dbVersionKey, scratch)	1,722✔
98	}	1,722✔
99
100	// OptionalMeta structure holds the database optional migration information.
101	type OptionalMeta struct {
102	// Versions is a set that contains the versions that have been applied.
103	// When saved to disk, only the indexes are stored.
104	Versions map[uint64]string
105	}
106
107	func (om *OptionalMeta) String() string {	1✔
108	s := ""	1✔
109	for index, name := range om.Versions {	2✔
110	s += fmt.Sprintf("%d: %s", index, name)	1✔
111	}	1✔
112	if s == "" {	1✔
113	s = "empty"	×
114	}	×
115	return s	1✔
116	}
117
118	// fetchOptionalMeta reads the optional meta from the database.
119	func (d DB) fetchOptionalMeta() (OptionalMeta, error) {	1,749✔
120	om := &OptionalMeta{	1,749✔
121	Versions: make(map[uint64]string),	1,749✔
122	}	1,749✔
123		1,749✔
124	err := kvdb.View(d, func(tx kvdb.RTx) error {	3,498✔
125	metaBucket := tx.ReadBucket(metaBucket)	1,749✔
126	if metaBucket == nil {	1,749✔
127	return ErrMetaNotFound	×
128	}	×
129
130	vBytes := metaBucket.Get(optionalVersionKey)	1,749✔
131	// Exit early if nothing found.	1,749✔
132	if vBytes == nil {	3,495✔
133	return nil	1,746✔
134	}	1,746✔
135
136	// Read the versions' length.
137	r := bytes.NewReader(vBytes)	3✔
138	vLen, err := tlv.ReadVarInt(r, &[8]byte{})	3✔
139	if err != nil {	3✔
140	return err	×
141	}	×
142
143	// Write the version index.
144	for i := uint64(0); i < vLen; i++ {	6✔
145	version, err := tlv.ReadVarInt(r, &[8]byte{})	3✔
146	if err != nil {	3✔
147	return err	×
148	}	×
149	om.Versions[version] = optionalVersions[i].name	3✔
150	}
151
152	return nil	3✔
153	}, func() {})	1,749✔
154	if err != nil {	1,749✔
155	return nil, err	×
156	}	×
157
158	return om, nil	1,749✔
159	}
160
161	// putOptionalMeta writes an optional meta to the database.
162	func (d DB) putOptionalMeta(om OptionalMeta) error {	2✔
163	return kvdb.Update(d, func(tx kvdb.RwTx) error {	4✔
164	metaBucket, err := tx.CreateTopLevelBucket(metaBucket)	2✔
165	if err != nil {	2✔
166	return err	×
167	}	×
168
169	var b bytes.Buffer	2✔
170		2✔
171	// Write the total length.	2✔
172	err = tlv.WriteVarInt(&b, uint64(len(om.Versions)), &[8]byte{})	2✔
173	if err != nil {	2✔
174	return err	×
175	}	×
176
177	// Write the version indexes.
178	for v := range om.Versions {	4✔
179	err := tlv.WriteVarInt(&b, v, &[8]byte{})	2✔
180	if err != nil {	2✔
181	return err	×
182	}	×
183	}
184
185	return metaBucket.Put(optionalVersionKey, b.Bytes())	2✔
186	}, func() {})	2✔
187	}
188
189	// CheckMarkerPresent returns the marker under the requested key or
190	// ErrMarkerNotFound if either the root bucket or the marker key within that
191	// bucket does not exist.
192	func CheckMarkerPresent(tx kvdb.RTx, markerKey []byte) ([]byte, error) {	1,929✔
193	markerBucket := tx.ReadBucket(markerKey)	1,929✔
194	if markerBucket == nil {	3,853✔
195	return nil, ErrMarkerNotPresent	1,924✔
196	}	1,924✔
197
198	val := markerBucket.Get(markerKey)	5✔
199		5✔
200	// If we wrote the marker correctly, we created a bucket _and_ created a	5✔
201	// key with a non-empty value. It doesn't matter to us whether the key	5✔
202	// exists or whether its value is empty, to us, it just means the marker	5✔
203	// isn't there.	5✔
204	if len(val) == 0 {	6✔
205	return nil, ErrMarkerNotPresent	1✔
206	}	1✔
207
208	return val, nil	4✔
209	}
210
211	// EnsureNoTombstone returns an error if there is a tombstone marker in the DB
212	// of the given transaction.
213	func EnsureNoTombstone(tx kvdb.RTx) error {	1,925✔
214	marker, err := CheckMarkerPresent(tx, TombstoneKey)	1,925✔
215	if err == ErrMarkerNotPresent {	3,848✔
216	// No marker present, so no tombstone. The DB is still alive.	1,923✔
217	return nil	1,923✔
218	}	1,923✔
219	if err != nil {	2✔
220	return err	×
221	}	×
222
223	// There was no error so there is a tombstone marker/tag. We cannot use
224	// this DB anymore.
225	return fmt.Errorf("refusing to use db, it was marked with a tombstone "+	2✔
226	"after successful data migration; tombstone reads: %s",	2✔
227	string(marker))	2✔
228	}
229
230	// AddMarker adds the marker with the given key into a top level bucket with the
231	// same name. So the structure will look like:
232	//
233	// marker-key (top level bucket)
234	// \|-> marker-key:marker-value (key/value pair)
235	func AddMarker(tx kvdb.RwTx, markerKey, markerValue []byte) error {	2✔
236	if len(markerValue) == 0 {	2✔
237	return fmt.Errorf("marker value cannot be empty")	×
238	}	×
239
240	markerBucket, err := tx.CreateTopLevelBucket(markerKey)	2✔
241	if err != nil {	2✔
242	return err	×
243	}	×
244
245	return markerBucket.Put(markerKey, markerValue)	2✔
246	}

lightningnetwork / lnd / 13440912774

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous