13566028875

Committed 27 Feb 2025 12:09PM UTC coverage: 49.396% (-9.4%) from 58.748%

Build # 13566028875

Build Type

Pull #9555

github

Committed by

ellemouton

Commit Message

graph/db: populate the graph cache in Start instead of during construction

In this commit, we move the graph cache population logic out of the
ChannelGraph constructor and into its Start method instead.

Pull Request Pull Request #9555: graph: extract cache from CRUD [6]

Run Details

34 of 54 new or added lines in 4 files covered. (62.96%)

27464 existing lines in 436 files now uncovered.

101095 of 204664 relevant lines covered (49.4%)

1.54 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/channeldb/migration23/migration.go

package migration23

import (
        "fmt"

        "github.com/lightningnetwork/lnd/kvdb"
)

var (
        // paymentsRootBucket is the name of the top-level bucket within the
        // database that stores all data related to payments.
        paymentsRootBucket = []byte("payments-root-bucket")

        // paymentHtlcsBucket is a bucket where we'll store the information
        // about the HTLCs that were attempted for a payment.
        paymentHtlcsBucket = []byte("payment-htlcs-bucket")

        // oldAttemptInfoKey is a key used in a HTLC's sub-bucket to store the
        // info about the attempt that was done for the HTLC in question.
        oldAttemptInfoKey = []byte("htlc-attempt-info")

        // oldSettleInfoKey is a key used in a HTLC's sub-bucket to store the
        // settle info, if any.
        oldSettleInfoKey = []byte("htlc-settle-info")

        // oldFailInfoKey is a key used in a HTLC's sub-bucket to store
        // failure information, if any.
        oldFailInfoKey = []byte("htlc-fail-info")

        // htlcAttemptInfoKey is the key used as the prefix of an HTLC attempt
        // to store the info about the attempt that was done for the HTLC in
        // question. The HTLC attempt ID is concatenated at the end.
        htlcAttemptInfoKey = []byte("ai")

        // htlcSettleInfoKey is the key used as the prefix of an HTLC attempt
        // settle info, if any. The HTLC attempt ID is concatenated at the end.
        htlcSettleInfoKey = []byte("si")

        // htlcFailInfoKey is the key used as the prefix of an HTLC attempt
        // failure information, if any.The  HTLC attempt ID is concatenated at
        // the end.
        htlcFailInfoKey = []byte("fi")
)

// htlcBucketKey creates a composite key from prefix and id where the result is
// simply the two concatenated. This is the exact copy from payments.go.
func htlcBucketKey(prefix, id []byte) []byte {
        key := make([]byte, len(prefix)+len(id))
        copy(key, prefix)
        copy(key[len(prefix):], id)
        return key
}

// MigrateHtlcAttempts will gather all htlc-attempt-info's, htlcs-settle-info's
// and htlcs-fail-info's from the attempt ID buckes and re-store them using the
// flattened keys to each payment's payment-htlcs-bucket.
func MigrateHtlcAttempts(tx kvdb.RwTx) error {
        payments := tx.ReadWriteBucket(paymentsRootBucket)
        if payments == nil {
                return nil
        }

        // Collect all payment hashes so we can migrate payments one-by-one to
        // avoid any bugs bbolt might have when invalidating cursors.
        // For 100 million payments, this would need about 3 GiB memory so we
        // should hopefully be fine for very large nodes too.
        var paymentHashes []string
        if err := payments.ForEach(func(hash, v []byte) error {
                // Get the bucket which contains the payment, fail if the key
                // does not have a bucket.
                bucket := payments.NestedReadBucket(hash)
                if bucket == nil {
                        return fmt.Errorf("key must be a bucket: '%v'",
                                string(paymentsRootBucket))
                }

                paymentHashes = append(paymentHashes, string(hash))
                return nil
        }); err != nil {
                return err
        }

        for _, paymentHash := range paymentHashes {
                payment := payments.NestedReadWriteBucket([]byte(paymentHash))
                if payment.Get(paymentHtlcsBucket) != nil {
                        return fmt.Errorf("key must be a bucket: '%v'",
                                string(paymentHtlcsBucket))
                }

                htlcs := payment.NestedReadWriteBucket(paymentHtlcsBucket)
                if htlcs == nil {
                        // Nothing to migrate for this payment.
                        continue
                }

                if err := migrateHtlcsBucket(htlcs); err != nil {
                        return err
                }
        }

        return nil
}

// migrateHtlcsBucket is a helper to gather, transform and re-store htlc attempt
// key/values.
func migrateHtlcsBucket(htlcs kvdb.RwBucket) error {
        // Collect attempt ids so that we can migrate attempts one-by-one
        // to avoid any bugs bbolt might have when invalidating cursors.
        var aids []string

        // First we collect all htlc attempt ids.
        if err := htlcs.ForEach(func(aid, v []byte) error {
                aids = append(aids, string(aid))
                return nil
        }); err != nil {
                return err
        }

        // Next we go over these attempts, fetch all data and migrate.
        for _, aid := range aids {
                aidKey := []byte(aid)
                attempt := htlcs.NestedReadWriteBucket(aidKey)
                if attempt == nil {
                        return fmt.Errorf("non bucket element '%v' in '%v' "+
                                "bucket", aidKey, string(paymentHtlcsBucket))
                }

                // Collect attempt/settle/fail infos.
                attemptInfo := attempt.Get(oldAttemptInfoKey)
                if len(attemptInfo) > 0 {
                        newKey := htlcBucketKey(htlcAttemptInfoKey, aidKey)
                        if err := htlcs.Put(newKey, attemptInfo); err != nil {
                                return err
                        }
                }

                settleInfo := attempt.Get(oldSettleInfoKey)
                if len(settleInfo) > 0 {
                        newKey := htlcBucketKey(htlcSettleInfoKey, aidKey)
                        if err := htlcs.Put(newKey, settleInfo); err != nil {
                                return err
                        }
                }

                failInfo := attempt.Get(oldFailInfoKey)
                if len(failInfo) > 0 {
                        newKey := htlcBucketKey(htlcFailInfoKey, aidKey)
                        if err := htlcs.Put(newKey, failInfo); err != nil {
                                return err
                        }
                }
        }

        // Finally we delete old attempt buckets.
        for _, aid := range aids {
                if err := htlcs.DeleteNestedBucket([]byte(aid)); err != nil {
                        return err
                }
        }

        return nil
}

1	package migration23
2
3	import (
4	"fmt"
5
6	"github.com/lightningnetwork/lnd/kvdb"
7	)
8
9	var (
10	// paymentsRootBucket is the name of the top-level bucket within the
11	// database that stores all data related to payments.
12	paymentsRootBucket = []byte("payments-root-bucket")
13
14	// paymentHtlcsBucket is a bucket where we'll store the information
15	// about the HTLCs that were attempted for a payment.
16	paymentHtlcsBucket = []byte("payment-htlcs-bucket")
17
18	// oldAttemptInfoKey is a key used in a HTLC's sub-bucket to store the
19	// info about the attempt that was done for the HTLC in question.
20	oldAttemptInfoKey = []byte("htlc-attempt-info")
21
22	// oldSettleInfoKey is a key used in a HTLC's sub-bucket to store the
23	// settle info, if any.
24	oldSettleInfoKey = []byte("htlc-settle-info")
25
26	// oldFailInfoKey is a key used in a HTLC's sub-bucket to store
27	// failure information, if any.
28	oldFailInfoKey = []byte("htlc-fail-info")
29
30	// htlcAttemptInfoKey is the key used as the prefix of an HTLC attempt
31	// to store the info about the attempt that was done for the HTLC in
32	// question. The HTLC attempt ID is concatenated at the end.
33	htlcAttemptInfoKey = []byte("ai")
34
35	// htlcSettleInfoKey is the key used as the prefix of an HTLC attempt
36	// settle info, if any. The HTLC attempt ID is concatenated at the end.
37	htlcSettleInfoKey = []byte("si")
38
39	// htlcFailInfoKey is the key used as the prefix of an HTLC attempt
40	// failure information, if any.The HTLC attempt ID is concatenated at
41	// the end.
42	htlcFailInfoKey = []byte("fi")
43	)
44
45	// htlcBucketKey creates a composite key from prefix and id where the result is
46	// simply the two concatenated. This is the exact copy from payments.go.
UNCOV 47	func htlcBucketKey(prefix, id []byte) []byte {	×
UNCOV 48	key := make([]byte, len(prefix)+len(id))	×
UNCOV 49	copy(key, prefix)	×
UNCOV 50	copy(key[len(prefix):], id)	×
UNCOV 51	return key	×
UNCOV 52	}	×
53
54	// MigrateHtlcAttempts will gather all htlc-attempt-info's, htlcs-settle-info's
55	// and htlcs-fail-info's from the attempt ID buckes and re-store them using the
56	// flattened keys to each payment's payment-htlcs-bucket.
UNCOV 57	func MigrateHtlcAttempts(tx kvdb.RwTx) error {	×
UNCOV 58	payments := tx.ReadWriteBucket(paymentsRootBucket)	×
UNCOV 59	if payments == nil {	×
60	return nil	×
61	}	×
62
63	// Collect all payment hashes so we can migrate payments one-by-one to
64	// avoid any bugs bbolt might have when invalidating cursors.
65	// For 100 million payments, this would need about 3 GiB memory so we
66	// should hopefully be fine for very large nodes too.
UNCOV 67	var paymentHashes []string	×
UNCOV 68	if err := payments.ForEach(func(hash, v []byte) error {	×
UNCOV 69	// Get the bucket which contains the payment, fail if the key	×
UNCOV 70	// does not have a bucket.	×
UNCOV 71	bucket := payments.NestedReadBucket(hash)	×
UNCOV 72	if bucket == nil {	×
UNCOV 73	return fmt.Errorf("key must be a bucket: '%v'",	×
UNCOV 74	string(paymentsRootBucket))	×
UNCOV 75	}	×
76
UNCOV 77	paymentHashes = append(paymentHashes, string(hash))	×
UNCOV 78	return nil	×
UNCOV 79	}); err != nil {	×
UNCOV 80	return err	×
UNCOV 81	}	×
82
UNCOV 83	for _, paymentHash := range paymentHashes {	×
UNCOV 84	payment := payments.NestedReadWriteBucket([]byte(paymentHash))	×
UNCOV 85	if payment.Get(paymentHtlcsBucket) != nil {	×
UNCOV 86	return fmt.Errorf("key must be a bucket: '%v'",	×
UNCOV 87	string(paymentHtlcsBucket))	×
UNCOV 88	}	×
89
UNCOV 90	htlcs := payment.NestedReadWriteBucket(paymentHtlcsBucket)	×
UNCOV 91	if htlcs == nil {	×
UNCOV 92	// Nothing to migrate for this payment.	×
UNCOV 93	continue	×
94	}
95
UNCOV 96	if err := migrateHtlcsBucket(htlcs); err != nil {	×
UNCOV 97	return err	×
UNCOV 98	}	×
99	}
100
UNCOV 101	return nil	×
102	}
103
104	// migrateHtlcsBucket is a helper to gather, transform and re-store htlc attempt
105	// key/values.
UNCOV 106	func migrateHtlcsBucket(htlcs kvdb.RwBucket) error {	×
UNCOV 107	// Collect attempt ids so that we can migrate attempts one-by-one	×
UNCOV 108	// to avoid any bugs bbolt might have when invalidating cursors.	×
UNCOV 109	var aids []string	×
UNCOV 110		×
UNCOV 111	// First we collect all htlc attempt ids.	×
UNCOV 112	if err := htlcs.ForEach(func(aid, v []byte) error {	×
UNCOV 113	aids = append(aids, string(aid))	×
UNCOV 114	return nil	×
UNCOV 115	}); err != nil {	×
116	return err	×
117	}	×
118
119	// Next we go over these attempts, fetch all data and migrate.
UNCOV 120	for _, aid := range aids {	×
UNCOV 121	aidKey := []byte(aid)	×
UNCOV 122	attempt := htlcs.NestedReadWriteBucket(aidKey)	×
UNCOV 123	if attempt == nil {	×
UNCOV 124	return fmt.Errorf("non bucket element '%v' in '%v' "+	×
UNCOV 125	"bucket", aidKey, string(paymentHtlcsBucket))	×
UNCOV 126	}	×
127
128	// Collect attempt/settle/fail infos.
UNCOV 129	attemptInfo := attempt.Get(oldAttemptInfoKey)	×
UNCOV 130	if len(attemptInfo) > 0 {	×
UNCOV 131	newKey := htlcBucketKey(htlcAttemptInfoKey, aidKey)	×
UNCOV 132	if err := htlcs.Put(newKey, attemptInfo); err != nil {	×
133	return err	×
134	}	×
135	}
136
UNCOV 137	settleInfo := attempt.Get(oldSettleInfoKey)	×
UNCOV 138	if len(settleInfo) > 0 {	×
UNCOV 139	newKey := htlcBucketKey(htlcSettleInfoKey, aidKey)	×
UNCOV 140	if err := htlcs.Put(newKey, settleInfo); err != nil {	×
141	return err	×
142	}	×
143	}
144
UNCOV 145	failInfo := attempt.Get(oldFailInfoKey)	×
UNCOV 146	if len(failInfo) > 0 {	×
UNCOV 147	newKey := htlcBucketKey(htlcFailInfoKey, aidKey)	×
UNCOV 148	if err := htlcs.Put(newKey, failInfo); err != nil {	×
149	return err	×
150	}	×
151	}
152	}
153
154	// Finally we delete old attempt buckets.
UNCOV 155	for _, aid := range aids {	×
UNCOV 156	if err := htlcs.DeleteNestedBucket([]byte(aid)); err != nil {	×
157	return err	×
158	}	×
159	}
160
UNCOV 161	return nil	×
162	}

lightningnetwork / lnd / 13566028875

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous