16034605897

Committed 02 Jul 2025 08:00PM UTC coverage: 67.593% (+0.004%) from 67.589%

Build # 16034605897

Build Type

Pull #10027

github

Committed by

web-flow

Commit Message

Merge 813baa2f7 into 8a0341419

Pull Request Pull Request #10027: Fix `ExtraData` field and use `BigSize` encodine

Run Details

133 of 145 new or added lines in 5 files covered. (91.72%)

99 existing lines in 28 files now uncovered.

135194 of 200011 relevant lines covered (67.59%)

21841.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.76

/lnwire/extra_bytes.go

package lnwire

import (
        "bytes"
        "fmt"
        "io"

        "github.com/lightningnetwork/lnd/fn/v2"
        "github.com/lightningnetwork/lnd/tlv"
)

// ExtraOpaqueData is the set of data that was appended to this message, some
// of which we may not actually know how to iterate or parse. By holding onto
// this data, we ensure that we're able to properly validate the set of
// signatures that cover these new fields, and ensure we're able to make
// upgrades to the network in a forwards compatible manner.
type ExtraOpaqueData []byte

// NewExtraOpaqueData creates a new ExtraOpaqueData instance from a tlv.TypeMap.
func NewExtraOpaqueData(tlvMap tlv.TypeMap) (ExtraOpaqueData, error) {
        // If the tlv map is empty, we'll want to mirror the behavior of
        // decoding an empty extra opaque data field (see Decode method).
        if len(tlvMap) == 0 {
                return make([]byte, 0), nil
        }

        // Convert the TLV map into a slice of records.
        records := TlvMapToRecords(tlvMap)

        // Encode the records into the extra data byte slice.
        return EncodeRecords(records)
}

// Encode attempts to encode the raw extra bytes into the passed io.Writer.
func (e *ExtraOpaqueData) Encode(w *bytes.Buffer) error {
        eBytes := []byte((*e)[:])
        if err := WriteBytes(w, eBytes); err != nil {
                return err
        }

        return nil
}

// Decode attempts to unpack the raw bytes encoded in the passed-in io.Reader as
// a set of extra opaque data.
func (e *ExtraOpaqueData) Decode(r io.Reader) error {
        // First, we'll attempt to read a set of bytes contained within the
        // passed io.Reader (if any exist).
        rawBytes, err := io.ReadAll(r)
        if err != nil {
                return err
        }

        // If we _do_ have some bytes, then we'll swap out our backing pointer.
        // This ensures that any struct that embeds this type will properly
        // store the bytes once this method exits.
        if len(rawBytes) > 0 {
                *e = rawBytes
        } else {
                *e = make([]byte, 0)
        }

        return nil
}

// ValidateTLV checks that the raw bytes that make up the ExtraOpaqueData
// instance are a valid TLV stream.
func (e *ExtraOpaqueData) ValidateTLV() error {
        // There is nothing to validate if the ExtraOpaqueData is nil or empty.
        if e == nil || len(*e) == 0 {
                return nil
        }

        tlvStream, err := tlv.NewStream()
        if err != nil {
                return err
        }

        // Ensure that the TLV stream is valid by attempting to decode it.
        _, err = tlvStream.DecodeWithParsedTypesP2P(bytes.NewReader(*e))
        if err != nil {
                return fmt.Errorf("invalid TLV stream: %w: %v", err, *e)
        }

        return nil
}

// PackRecords attempts to encode the set of tlv records into the target
// ExtraOpaqueData instance. The records will be encoded as a raw TLV stream
// and stored within the backing slice pointer.
func (e *ExtraOpaqueData) PackRecords(
        recordProducers ...tlv.RecordProducer) error {

        // Assemble all the records passed in series, then encode them.
        records := ProduceRecordsSorted(recordProducers...)
        encoded, err := EncodeRecords(records)
        if err != nil {
                return err
        }

        *e = encoded

        return nil
}

// ExtractRecords attempts to decode any types in the internal raw bytes as if
// it were a tlv stream. The set of raw parsed types is returned, and any
// passed records (if found in the stream) will be parsed into the proper
// tlv.Record.
func (e *ExtraOpaqueData) ExtractRecords(
        recordProducers ...tlv.RecordProducer) (tlv.TypeMap, error) {

        // First, assemble all the records passed in series.
        records := ProduceRecordsSorted(recordProducers...)
        extraBytesReader := bytes.NewReader(*e)

        // Since ExtraOpaqueData is provided by a potentially malicious peer,
        // pass it into the P2P decoding variant.
        return DecodeRecordsP2P(extraBytesReader, records...)
}

// RecordProducers parses ExtraOpaqueData into a slice of TLV record producers
// by interpreting it as a TLV map.
func (e *ExtraOpaqueData) RecordProducers() ([]tlv.RecordProducer, error) {
        var recordProducers []tlv.RecordProducer

        // If the instance is nil or empty, return an empty slice.
        if e == nil || len(*e) == 0 {
                return recordProducers, nil
        }

        // Parse the extra opaque data as a TLV map.
        tlvMap, err := e.ExtractRecords()
        if err != nil {
                return nil, err
        }

        // Convert the TLV map into a slice of record producers.
        records := TlvMapToRecords(tlvMap)

        return RecordsAsProducers(records), nil
}

// EncodeMessageExtraData encodes the given recordProducers into the given
// extraData.
func EncodeMessageExtraData(extraData *ExtraOpaqueData,
        recordProducers ...tlv.RecordProducer) error {

        // Treat extraData as a mutable reference.
        if extraData == nil {
                return fmt.Errorf("extra data cannot be nil")
        }

        // Pack in the series of TLV records into this message. The order we
        // pass them in doesn't matter, as the method will ensure that things
        // are all properly sorted.
        return extraData.PackRecords(recordProducers...)
}

// ParseAndExtractCustomRecords parses the given extra data into the passed-in
// records, then returns any remaining records split into custom records and
// extra data.
func ParseAndExtractCustomRecords(allExtraData ExtraOpaqueData,
        knownRecords ...tlv.RecordProducer) (CustomRecords,
        fn.Set[tlv.Type], ExtraOpaqueData, error) {

        extraDataTlvMap, err := allExtraData.ExtractRecords(knownRecords...)
        if err != nil {
                return nil, nil, nil, err
        }

        // Remove the known and now extracted records from the leftover extra
        // data map.
        parsedKnownRecords := make(fn.Set[tlv.Type], len(knownRecords))
        for _, producer := range knownRecords {
                r := producer.Record()

                // Only remove the records if it was parsed (remainder is nil).
                // We'll just store the type so we can tell the caller which
                // records were actually parsed fully.
                val, ok := extraDataTlvMap[r.Type()]
                if ok && val == nil {
                        parsedKnownRecords.Add(r.Type())
                        delete(extraDataTlvMap, r.Type())
                }
        }

        // Any records from the extra data TLV map which are in the custom
        // records TLV type range will be included in the custom records field
        // and removed from the extra data field.
        customRecordsTlvMap := make(tlv.TypeMap, len(extraDataTlvMap))
        for k, v := range extraDataTlvMap {
                // Skip records that are not in the custom records TLV type
                // range.
                if k < MinCustomRecordsTlvType {
                        continue
                }

                // Include the record in the custom records map.
                customRecordsTlvMap[k] = v

                // Now that the record is included in the custom records map,
                // we can remove it from the extra data TLV map.
                delete(extraDataTlvMap, k)
        }

        // Set the custom records field to the custom records specific TLV
        // record map.
        customRecords, err := NewCustomRecords(customRecordsTlvMap)
        if err != nil {
                return nil, nil, nil, err
        }

        // Encode the remaining records back into the extra data field. These
        // records are not in the custom records TLV type range and do not
        // have associated fields in the struct that produced the records.
        extraData, err := NewExtraOpaqueData(extraDataTlvMap)
        if err != nil {
                return nil, nil, nil, err
        }

        // Help with unit testing where we might have the empty value (nil) for
        // the extra data instead of the default that's returned by the
        // constructor (empty slice).
        if len(extraData) == 0 {
                extraData = nil
        }

        return customRecords, parsedKnownRecords, extraData, nil
}

// MergeAndEncode merges the known records with the extra data and custom
// records, then encodes the merged records into raw bytes.
func MergeAndEncode(knownRecords []tlv.RecordProducer,
        extraData ExtraOpaqueData, customRecords CustomRecords) ([]byte,
        error) {

        // Construct a slice of all the records that we should include in the
        // message extra data field. We will start by including any records from
        // the extra data field.
        mergedRecords, err := extraData.RecordProducers()
        if err != nil {
                return nil, err
        }

        // Merge the known and extra data records.
        mergedRecords = append(mergedRecords, knownRecords...)

        // Include custom records in the extra data wire field if they are
        // present. Ensure that the custom records are validated before encoding
        // them.
        if err := customRecords.Validate(); err != nil {
                return nil, fmt.Errorf("custom records validation error: %w",
                        err)
        }

        // Extend the message extra data records slice with TLV records from the
        // custom records field.
        mergedRecords = append(
                mergedRecords, customRecords.RecordProducers()...,
        )

        // Now we can sort the records and make sure there are no records with
        // the same type that would collide when encoding.
        sortedRecords := ProduceRecordsSorted(mergedRecords...)
        if err := AssertUniqueTypes(sortedRecords); err != nil {
                return nil, err
        }

        return EncodeRecords(sortedRecords)
}

// ParseAndExtractExtraData parses the given extra data into the passed-in
// records, then returns any remaining records as extra data.
func ParseAndExtractExtraData(allTlvData ExtraOpaqueData,
        knownRecords ...tlv.RecordProducer) (fn.Set[tlv.Type],
        ExtraOpaqueData, error) {

        extraDataTlvMap, err := allTlvData.ExtractRecords(knownRecords...)
        if err != nil {
                return nil, nil, err
        }

        // Remove the known and now extracted records from the leftover extra
        // data map.
        parsedKnownRecords := make(fn.Set[tlv.Type], len(knownRecords))
        for _, producer := range knownRecords {
                r := producer.Record()

                // Only remove the records if it was parsed (remainder is nil).
                // We'll just store the type so we can tell the caller which
                // records were actually parsed fully.
                val, ok := extraDataTlvMap[r.Type()]
                if ok && val == nil {
                        parsedKnownRecords.Add(r.Type())
                        delete(extraDataTlvMap, r.Type())
                }
        }

        // Encode the remaining records back into the extra data field. These
        // records are not in the custom records TLV type range and do not
        // have associated fields in the struct that produced the records.
        extraData, err := NewExtraOpaqueData(extraDataTlvMap)
        if err != nil {
                return nil, nil, err
        }

        return parsedKnownRecords, extraData, nil
}

1	package lnwire
2
3	import (
4	"bytes"
5	"fmt"
6	"io"
7
8	"github.com/lightningnetwork/lnd/fn/v2"
9	"github.com/lightningnetwork/lnd/tlv"
10	)
11
12	// ExtraOpaqueData is the set of data that was appended to this message, some
13	// of which we may not actually know how to iterate or parse. By holding onto
14	// this data, we ensure that we're able to properly validate the set of
15	// signatures that cover these new fields, and ensure we're able to make
16	// upgrades to the network in a forwards compatible manner.
17	type ExtraOpaqueData []byte
18
19	// NewExtraOpaqueData creates a new ExtraOpaqueData instance from a tlv.TypeMap.
20	func NewExtraOpaqueData(tlvMap tlv.TypeMap) (ExtraOpaqueData, error) {	6,661✔
21	// If the tlv map is empty, we'll want to mirror the behavior of	6,661✔
22	// decoding an empty extra opaque data field (see Decode method).	6,661✔
23	if len(tlvMap) == 0 {	12,854✔
24	return make([]byte, 0), nil	6,193✔
25	}	6,193✔
26
27	// Convert the TLV map into a slice of records.
28	records := TlvMapToRecords(tlvMap)	468✔
29		468✔
30	// Encode the records into the extra data byte slice.	468✔
31	return EncodeRecords(records)	468✔
32	}
33
34	// Encode attempts to encode the raw extra bytes into the passed io.Writer.
35	func (e ExtraOpaqueData) Encode(w bytes.Buffer) error {	177✔
36	eBytes := []byte((*e)[:])	177✔
37	if err := WriteBytes(w, eBytes); err != nil {	177✔
38	return err	×
39	}	×
40
41	return nil	177✔
42	}
43
44	// Decode attempts to unpack the raw bytes encoded in the passed-in io.Reader as
45	// a set of extra opaque data.
46	func (e *ExtraOpaqueData) Decode(r io.Reader) error {	13,165✔
47	// First, we'll attempt to read a set of bytes contained within the	13,165✔
48	// passed io.Reader (if any exist).	13,165✔
49	rawBytes, err := io.ReadAll(r)	13,165✔
50	if err != nil {	13,165✔
51	return err	×
52	}	×
53
54	// If we _do_ have some bytes, then we'll swap out our backing pointer.
55	// This ensures that any struct that embeds this type will properly
56	// store the bytes once this method exits.
57	if len(rawBytes) > 0 {	20,032✔
58	*e = rawBytes	6,867✔
59	} else {	13,168✔
60	*e = make([]byte, 0)	6,301✔
61	}	6,301✔
62
63	return nil	13,165✔
64	}
65
66	// ValidateTLV checks that the raw bytes that make up the ExtraOpaqueData
67	// instance are a valid TLV stream.
68	func (e *ExtraOpaqueData) ValidateTLV() error {	3,480✔
69	// There is nothing to validate if the ExtraOpaqueData is nil or empty.	3,480✔
70	if e == nil \|\| len(*e) == 0 {	5,806✔
71	return nil	2,326✔
72	}	2,326✔
73
74	tlvStream, err := tlv.NewStream()	1,157✔
75	if err != nil {	1,157✔
76	return err	×
77	}	×
78
79	// Ensure that the TLV stream is valid by attempting to decode it.
80	_, err = tlvStream.DecodeWithParsedTypesP2P(bytes.NewReader(*e))	1,157✔
81	if err != nil {	1,227✔
82	return fmt.Errorf("invalid TLV stream: %w: %v", err, *e)	70✔
83	}	70✔
84
85	return nil	1,087✔
86	}
87
88	// PackRecords attempts to encode the set of tlv records into the target
89	// ExtraOpaqueData instance. The records will be encoded as a raw TLV stream
90	// and stored within the backing slice pointer.
91	func (e *ExtraOpaqueData) PackRecords(
92	recordProducers ...tlv.RecordProducer) error {	35,038✔
93		35,038✔
94	// Assemble all the records passed in series, then encode them.	35,038✔
95	records := ProduceRecordsSorted(recordProducers...)	35,038✔
96	encoded, err := EncodeRecords(records)	35,038✔
97	if err != nil {	35,038✔
98	return err	×
99	}	×
100
101	*e = encoded	35,038✔
102		35,038✔
103	return nil	35,038✔
104	}
105
106	// ExtractRecords attempts to decode any types in the internal raw bytes as if
107	// it were a tlv stream. The set of raw parsed types is returned, and any
108	// passed records (if found in the stream) will be parsed into the proper
109	// tlv.Record.
110	func (e *ExtraOpaqueData) ExtractRecords(
111	recordProducers ...tlv.RecordProducer) (tlv.TypeMap, error) {	25,359✔
112		25,359✔
113	// First, assemble all the records passed in series.	25,359✔
114	records := ProduceRecordsSorted(recordProducers...)	25,359✔
115	extraBytesReader := bytes.NewReader(*e)	25,359✔
116		25,359✔
117	// Since ExtraOpaqueData is provided by a potentially malicious peer,	25,359✔
118	// pass it into the P2P decoding variant.	25,359✔
119	return DecodeRecordsP2P(extraBytesReader, records...)	25,359✔
120	}	25,359✔
121
122	// RecordProducers parses ExtraOpaqueData into a slice of TLV record producers
123	// by interpreting it as a TLV map.
124	func (e *ExtraOpaqueData) RecordProducers() ([]tlv.RecordProducer, error) {	9,301✔
125	var recordProducers []tlv.RecordProducer	9,301✔
126		9,301✔
127	// If the instance is nil or empty, return an empty slice.	9,301✔
128	if e == nil \|\| len(*e) == 0 {	18,445✔
129	return recordProducers, nil	9,144✔
130	}	9,144✔
131
132	// Parse the extra opaque data as a TLV map.
133	tlvMap, err := e.ExtractRecords()	157✔
134	if err != nil {	157✔
135	return nil, err	×
136	}	×
137
138	// Convert the TLV map into a slice of record producers.
139	records := TlvMapToRecords(tlvMap)	157✔
140		157✔
141	return RecordsAsProducers(records), nil	157✔
142	}
143
144	// EncodeMessageExtraData encodes the given recordProducers into the given
145	// extraData.
146	func EncodeMessageExtraData(extraData *ExtraOpaqueData,
147	recordProducers ...tlv.RecordProducer) error {	2,319✔
148		2,319✔
149	// Treat extraData as a mutable reference.	2,319✔
150	if extraData == nil {	2,319✔
151	return fmt.Errorf("extra data cannot be nil")	×
152	}	×
153
154	// Pack in the series of TLV records into this message. The order we
155	// pass them in doesn't matter, as the method will ensure that things
156	// are all properly sorted.
157	return extraData.PackRecords(recordProducers...)	2,319✔
158	}
159
160	// ParseAndExtractCustomRecords parses the given extra data into the passed-in
161	// records, then returns any remaining records split into custom records and
162	// extra data.
163	func ParseAndExtractCustomRecords(allExtraData ExtraOpaqueData,
164	knownRecords ...tlv.RecordProducer) (CustomRecords,
165	fn.Set[tlv.Type], ExtraOpaqueData, error) {	6,491✔
166		6,491✔
167	extraDataTlvMap, err := allExtraData.ExtractRecords(knownRecords...)	6,491✔
168	if err != nil {	6,633✔
169	return nil, nil, nil, err	142✔
170	}	142✔
171
172	// Remove the known and now extracted records from the leftover extra
173	// data map.
174	parsedKnownRecords := make(fn.Set[tlv.Type], len(knownRecords))	6,349✔
175	for _, producer := range knownRecords {	11,652✔
176	r := producer.Record()	5,303✔
177		5,303✔
178	// Only remove the records if it was parsed (remainder is nil).	5,303✔
179	// We'll just store the type so we can tell the caller which	5,303✔
180	// records were actually parsed fully.	5,303✔
181	val, ok := extraDataTlvMap[r.Type()]	5,303✔
182	if ok && val == nil {	6,179✔
183	parsedKnownRecords.Add(r.Type())	876✔
184	delete(extraDataTlvMap, r.Type())	876✔
185	}	876✔
186	}
187
188	// Any records from the extra data TLV map which are in the custom
189	// records TLV type range will be included in the custom records field
190	// and removed from the extra data field.
191	customRecordsTlvMap := make(tlv.TypeMap, len(extraDataTlvMap))	6,349✔
192	for k, v := range extraDataTlvMap {	10,805✔
193	// Skip records that are not in the custom records TLV type	4,456✔
194	// range.	4,456✔
195	if k < MinCustomRecordsTlvType {	5,874✔
196	continue	1,418✔
197	}
198
199	// Include the record in the custom records map.
200	customRecordsTlvMap[k] = v	3,038✔
201		3,038✔
202	// Now that the record is included in the custom records map,	3,038✔
203	// we can remove it from the extra data TLV map.	3,038✔
204	delete(extraDataTlvMap, k)	3,038✔
205	}
206
207	// Set the custom records field to the custom records specific TLV
208	// record map.
209	customRecords, err := NewCustomRecords(customRecordsTlvMap)	6,349✔
210	if err != nil {	6,349✔
211	return nil, nil, nil, err	×
212	}	×
213
214	// Encode the remaining records back into the extra data field. These
215	// records are not in the custom records TLV type range and do not
216	// have associated fields in the struct that produced the records.
217	extraData, err := NewExtraOpaqueData(extraDataTlvMap)	6,349✔
218	if err != nil {	6,349✔
219	return nil, nil, nil, err	×
220	}	×
221
222	// Help with unit testing where we might have the empty value (nil) for
223	// the extra data instead of the default that's returned by the
224	// constructor (empty slice).
225	if len(extraData) == 0 {	12,479✔
226	extraData = nil	6,130✔
227	}	6,130✔
228
229	return customRecords, parsedKnownRecords, extraData, nil	6,349✔
230	}
231
232	// MergeAndEncode merges the known records with the extra data and custom
233	// records, then encodes the merged records into raw bytes.
234	func MergeAndEncode(knownRecords []tlv.RecordProducer,
235	extraData ExtraOpaqueData, customRecords CustomRecords) ([]byte,
236	error) {	9,298✔
237		9,298✔
238	// Construct a slice of all the records that we should include in the	9,298✔
239	// message extra data field. We will start by including any records from	9,298✔
240	// the extra data field.	9,298✔
241	mergedRecords, err := extraData.RecordProducers()	9,298✔
242	if err != nil {	9,298✔
243	return nil, err	×
244	}	×
245
246	// Merge the known and extra data records.
247	mergedRecords = append(mergedRecords, knownRecords...)	9,298✔
248		9,298✔
249	// Include custom records in the extra data wire field if they are	9,298✔
250	// present. Ensure that the custom records are validated before encoding	9,298✔
251	// them.	9,298✔
252	if err := customRecords.Validate(); err != nil {	9,301✔
253	return nil, fmt.Errorf("custom records validation error: %w",	3✔
254	err)	3✔
255	}	3✔
256
257	// Extend the message extra data records slice with TLV records from the
258	// custom records field.
259	mergedRecords = append(	9,295✔
260	mergedRecords, customRecords.RecordProducers()...,	9,295✔
261	)	9,295✔
262		9,295✔
263	// Now we can sort the records and make sure there are no records with	9,295✔
264	// the same type that would collide when encoding.	9,295✔
265	sortedRecords := ProduceRecordsSorted(mergedRecords...)	9,295✔
266	if err := AssertUniqueTypes(sortedRecords); err != nil {	9,296✔
267	return nil, err	1✔
268	}	1✔
269
270	return EncodeRecords(sortedRecords)	9,294✔
271	}
272
273	// ParseAndExtractExtraData parses the given extra data into the passed-in
274	// records, then returns any remaining records as extra data.
275	func ParseAndExtractExtraData(allTlvData ExtraOpaqueData,
276	knownRecords ...tlv.RecordProducer) (fn.Set[tlv.Type],
277	ExtraOpaqueData, error) {	432✔
278		432✔
279	extraDataTlvMap, err := allTlvData.ExtractRecords(knownRecords...)	432✔
280	if err != nil {	555✔
281	return nil, nil, err	123✔
282	}	123✔
283
284	// Remove the known and now extracted records from the leftover extra
285	// data map.
286	parsedKnownRecords := make(fn.Set[tlv.Type], len(knownRecords))	309✔
287	for _, producer := range knownRecords {	1,967✔
288	r := producer.Record()	1,658✔
289		1,658✔
290	// Only remove the records if it was parsed (remainder is nil).	1,658✔
291	// We'll just store the type so we can tell the caller which	1,658✔
292	// records were actually parsed fully.	1,658✔
293	val, ok := extraDataTlvMap[r.Type()]	1,658✔
294	if ok && val == nil {	2,401✔
295	parsedKnownRecords.Add(r.Type())	743✔
296	delete(extraDataTlvMap, r.Type())	743✔
297	}	743✔
298	}
299
300	// Encode the remaining records back into the extra data field. These
301	// records are not in the custom records TLV type range and do not
302	// have associated fields in the struct that produced the records.
303	extraData, err := NewExtraOpaqueData(extraDataTlvMap)	309✔
304	if err != nil {	309✔
NEW 305	return nil, nil, err	×
NEW 306	}	×
307
308	return parsedKnownRecords, extraData, nil	309✔
309	}

lightningnetwork / lnd / 16034605897

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous