15039553421

Committed 15 May 2025 07:49AM UTC coverage: 58.62% (-10.4%) from 68.997%

Build # 15039553421

Build Type

Pull #9801

github

Committed by

web-flow

Commit Message

Merge d6d25a946 into b0cba7dd0

Pull Request Pull Request #9801: peer+lnd: add new CLI option to control if we D/C on slow pongs

Run Details

5 of 80 new or added lines in 3 files covered. (6.25%)

28140 existing lines in 449 files now uncovered.

97474 of 166282 relevant lines covered (58.62%)

1.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

29.19

/peer/ping_manager.go

package peer

import (
        "errors"
        "fmt"
        "sync"
        "sync/atomic"
        "time"

        "github.com/lightningnetwork/lnd/fn/v2"
        "github.com/lightningnetwork/lnd/lnwire"
)

// PingManagerConfig is a structure containing various parameters that govern
// how the PingManager behaves.
type PingManagerConfig struct {
        // NewPingPayload is a closure that returns the payload to be packaged
        // in the Ping message.
        NewPingPayload func() []byte

        // NewPongSize is a closure that returns a random value between
        // [0, lnwire.MaxPongBytes]. This random value helps to more effectively
        // pair Pong messages with Ping.
        NewPongSize func() uint16

        // IntervalDuration is the Duration between attempted pings.
        IntervalDuration time.Duration

        // TimeoutDuration is the Duration we wait before declaring a ping
        // attempt failed.
        TimeoutDuration time.Duration

        // SendPing is a closure that is responsible for sending the Ping
        // message out to our peer
        SendPing func(ping *lnwire.Ping)

        // OnPongFailure is a closure that is responsible for executing the
        // logic when a Pong message is either late or does not match our
        // expectations for that Pong
        OnPongFailure func(failureReason error, timeWaitedForPong time.Duration,
                lastKnownRTT time.Duration)
}

// PingManager is a structure that is designed to manage the internal state
// of the ping pong lifecycle with the remote peer. We assume there is only one
// ping outstanding at once.
//
// NOTE: This structure MUST be initialized with NewPingManager.
type PingManager struct {
        cfg *PingManagerConfig

        // pingTime is a rough estimate of the RTT (round-trip-time) between us
        // and the connected peer.
        // To be used atomically.
        // TODO(roasbeef): also use a WMA or EMA?
        pingTime atomic.Pointer[time.Duration]

        // pingLastSend is the time when we sent our last ping message.
        // To be used atomically.
        pingLastSend *time.Time

        // outstandingPongSize is the current size of the requested pong
        // payload.  This value can only validly range from [0,65531]. Any
        // value < 0 is interpreted as if there is no outstanding ping message.
        outstandingPongSize int32

        // pingTicker is a pointer to a Ticker that fires on every ping
        // interval.
        pingTicker *time.Ticker

        // pingTimeout is a Timer that will fire when we want to time out a
        // ping
        pingTimeout *time.Timer

        // pongChan is the channel on which the pingManager will write Pong
        // messages it is evaluating
        pongChan chan *lnwire.Pong

        started sync.Once
        stopped sync.Once

        quit chan struct{}
        wg   sync.WaitGroup
}

// NewPingManager constructs a pingManager in a valid state. It must be started
// before it does anything useful, though.
func NewPingManager(cfg *PingManagerConfig) *PingManager {
        m := PingManager{
                cfg:                 cfg,
                outstandingPongSize: -1,
                pongChan:            make(chan *lnwire.Pong, 1),
                quit:                make(chan struct{}),
        }

        return &m
}

// Start launches the primary goroutine that is owned by the pingManager.
func (m *PingManager) Start() error {
        var err error
        m.started.Do(func() {
                m.pingTicker = time.NewTicker(m.cfg.IntervalDuration)
                m.pingTimeout = time.NewTimer(0)

                m.wg.Add(1)
                go m.pingHandler()
        })

        return err
}

// getLastRTT safely retrieves the last known RTT, returning 0 if none exists.
func (m *PingManager) getLastRTT() time.Duration {
        rttPtr := m.pingTime.Load()
        if rttPtr == nil {
                return 0
        }

        return *rttPtr
}

// pendingPingWait calculates the time waited since the last ping was sent. If
// no ping time is reported, None is returned. defaultDuration.
func (m *PingManager) pendingPingWait() fn.Option[time.Duration] {
        if m.pingLastSend != nil {
                return fn.Some(time.Since(*m.pingLastSend))
        }

        return fn.None[time.Duration]()
}

// pingHandler is the main goroutine responsible for enforcing the ping/pong
// protocol.
func (m *PingManager) pingHandler() {
        defer m.wg.Done()
        defer m.pingTimeout.Stop()

        // Ensure that the pingTimeout channel is empty.
        if !m.pingTimeout.Stop() {
                <-m.pingTimeout.C
        }

        // Because we don't know if the OnPingFailure callback actually
        // disconnects a peer (dependent on user config), we should never return
        // from this loop unless the ping manager is stopped explicitly (which
        // happens on disconnect).
        for {
                select {
                case <-m.pingTicker.C:
                        // If this occurs it means that the new ping cycle has
                        // begun while there is still an outstanding ping
                        // awaiting a pong response.  This should never occur,
                        // but if it does, it implies a timeout.
                        if m.outstandingPongSize >= 0 {
                                // Ping was outstanding, meaning it timed out by
                                // the arrival of the next ping interval.
                                timeWaited := m.pendingPingWait().UnwrapOr(
                                        m.cfg.IntervalDuration,
                                )
                                lastRTT := m.getLastRTT()

                                m.cfg.OnPongFailure(
                                        errors.New("ping timed "+
                                                "out by next interval"),
                                        timeWaited, lastRTT,
                                )

                                m.resetPingState()
                        }

                        pongSize := m.cfg.NewPongSize()
                        ping := &lnwire.Ping{
                                NumPongBytes: pongSize,
                                PaddingBytes: m.cfg.NewPingPayload(),
                        }

                        // Set up our bookkeeping for the new Ping.
                        if err := m.setPingState(pongSize); err != nil {
                                // This is an internal error related to timer
                                // reset. Pass it to OnPongFailure as it's
                                // critical. Current and last RTT are not
                                // directly applicable here.
                                m.cfg.OnPongFailure(err, 0, 0)

                                continue
                        }

                        m.cfg.SendPing(ping)

                case <-m.pingTimeout.C:
                        timeWaited := m.pendingPingWait().UnwrapOr(
                                m.cfg.TimeoutDuration,
                        )
                        lastRTT := m.getLastRTT()

                        m.cfg.OnPongFailure(
                                errors.New("timeout while waiting for "+
                                        "pong response"),
                                timeWaited, lastRTT,
                        )

                        m.resetPingState()

                case pong := <-m.pongChan:
                        pongSize := int32(len(pong.PongBytes))

                        // Save off values we are about to override when we call
                        // resetPingState.
                        expected := m.outstandingPongSize
                        lastPingTime := m.pingLastSend

                        // This is an unexpected pong, we'll continue.
                        if lastPingTime == nil {
                                continue
                        }

                        actualRTT := time.Since(*lastPingTime)

                        // If the pong we receive doesn't match the ping we sent
                        // out, then we fail out.
                        if pongSize != expected {
                                e := fmt.Errorf("pong response does not match "+
                                        "expected size. Expected: %d, Got: %d",
                                        expected, pongSize)

                                lastRTT := m.getLastRTT()
                                m.cfg.OnPongFailure(e, actualRTT, lastRTT)

                                m.resetPingState()

                                continue
                        }

                        // Pong is good, update RTT and reset state.
                        m.pingTime.Store(&actualRTT)
                        m.resetPingState()

                case <-m.quit:
                        return
                }
        }
}

// Stop interrupts the goroutines that the PingManager owns.
func (m *PingManager) Stop() {
        if m.pingTicker == nil {
                return
        }

        m.stopped.Do(func() {
                close(m.quit)
                m.wg.Wait()

                m.pingTicker.Stop()
                m.pingTimeout.Stop()
        })
}

// setPingState is a private method to keep track of all of the fields we need
// to set when we send out a Ping.
func (m *PingManager) setPingState(pongSize uint16) error {
        t := time.Now()
        m.pingLastSend = &t
        m.outstandingPongSize = int32(pongSize)
        if m.pingTimeout.Reset(m.cfg.TimeoutDuration) {
                return fmt.Errorf(
                        "impossible: ping timeout reset when already active",
                )
        }

        return nil
}

// resetPingState is a private method that resets all of the bookkeeping that
// is tracking a currently outstanding Ping.
func (m *PingManager) resetPingState() {
        m.pingLastSend = nil
        m.outstandingPongSize = -1

        if !m.pingTimeout.Stop() {
                select {
                case <-m.pingTimeout.C:
                default:
                }
        }
}

// GetPingTimeMicroSeconds reports back the RTT calculated by the pingManager.
func (m *PingManager) GetPingTimeMicroSeconds() int64 {
        rtt := m.pingTime.Load()

        if rtt == nil {
                return -1
        }

        return rtt.Microseconds()
}

// ReceivedPong is called to evaluate a Pong message against the expectations
// we have for it. It will cause the PingManager to invoke the supplied
// OnPongFailure function if the Pong argument supplied violates expectations.
func (m *PingManager) ReceivedPong(msg *lnwire.Pong) {
        select {
        case m.pongChan <- msg:
        case <-m.quit:
        }
}

1	package peer
2
3	import (
4	"errors"
5	"fmt"
6	"sync"
7	"sync/atomic"
8	"time"
9
10	"github.com/lightningnetwork/lnd/fn/v2"
11	"github.com/lightningnetwork/lnd/lnwire"
12	)
13
14	// PingManagerConfig is a structure containing various parameters that govern
15	// how the PingManager behaves.
16	type PingManagerConfig struct {
17	// NewPingPayload is a closure that returns the payload to be packaged
18	// in the Ping message.
19	NewPingPayload func() []byte
20
21	// NewPongSize is a closure that returns a random value between
22	// [0, lnwire.MaxPongBytes]. This random value helps to more effectively
23	// pair Pong messages with Ping.
24	NewPongSize func() uint16
25
26	// IntervalDuration is the Duration between attempted pings.
27	IntervalDuration time.Duration
28
29	// TimeoutDuration is the Duration we wait before declaring a ping
30	// attempt failed.
31	TimeoutDuration time.Duration
32
33	// SendPing is a closure that is responsible for sending the Ping
34	// message out to our peer
35	SendPing func(ping *lnwire.Ping)
36
37	// OnPongFailure is a closure that is responsible for executing the
38	// logic when a Pong message is either late or does not match our
39	// expectations for that Pong
40	OnPongFailure func(failureReason error, timeWaitedForPong time.Duration,
41	lastKnownRTT time.Duration)
42	}
43
44	// PingManager is a structure that is designed to manage the internal state
45	// of the ping pong lifecycle with the remote peer. We assume there is only one
46	// ping outstanding at once.
47	//
48	// NOTE: This structure MUST be initialized with NewPingManager.
49	type PingManager struct {
50	cfg *PingManagerConfig
51
52	// pingTime is a rough estimate of the RTT (round-trip-time) between us
53	// and the connected peer.
54	// To be used atomically.
55	// TODO(roasbeef): also use a WMA or EMA?
56	pingTime atomic.Pointer[time.Duration]
57
58	// pingLastSend is the time when we sent our last ping message.
59	// To be used atomically.
60	pingLastSend *time.Time
61
62	// outstandingPongSize is the current size of the requested pong
63	// payload. This value can only validly range from [0,65531]. Any
64	// value < 0 is interpreted as if there is no outstanding ping message.
65	outstandingPongSize int32
66
67	// pingTicker is a pointer to a Ticker that fires on every ping
68	// interval.
69	pingTicker *time.Ticker
70
71	// pingTimeout is a Timer that will fire when we want to time out a
72	// ping
73	pingTimeout *time.Timer
74
75	// pongChan is the channel on which the pingManager will write Pong
76	// messages it is evaluating
77	pongChan chan *lnwire.Pong
78
79	started sync.Once
80	stopped sync.Once
81
82	quit chan struct{}
83	wg sync.WaitGroup
84	}
85
86	// NewPingManager constructs a pingManager in a valid state. It must be started
87	// before it does anything useful, though.
88	func NewPingManager(cfg PingManagerConfig) PingManager {	3✔
89	m := PingManager{	3✔
90	cfg: cfg,	3✔
91	outstandingPongSize: -1,	3✔
92	pongChan: make(chan *lnwire.Pong, 1),	3✔
93	quit: make(chan struct{}),	3✔
94	}	3✔
95		3✔
96	return &m	3✔
97	}	3✔
98
99	// Start launches the primary goroutine that is owned by the pingManager.
100	func (m *PingManager) Start() error {	3✔
101	var err error	3✔
102	m.started.Do(func() {	6✔
103	m.pingTicker = time.NewTicker(m.cfg.IntervalDuration)	3✔
104	m.pingTimeout = time.NewTimer(0)	3✔
105		3✔
106	m.wg.Add(1)	3✔
107	go m.pingHandler()	3✔
108	})	3✔
109
110	return err	3✔
111	}
112
113	// getLastRTT safely retrieves the last known RTT, returning 0 if none exists.
NEW 114	func (m *PingManager) getLastRTT() time.Duration {	×
NEW 115	rttPtr := m.pingTime.Load()	×
NEW 116	if rttPtr == nil {	×
NEW 117	return 0	×
NEW 118	}	×
119
NEW 120	return *rttPtr	×
121	}
122
123	// pendingPingWait calculates the time waited since the last ping was sent. If
124	// no ping time is reported, None is returned. defaultDuration.
NEW 125	func (m *PingManager) pendingPingWait() fn.Option[time.Duration] {	×
NEW 126	if m.pingLastSend != nil {	×
NEW 127	return fn.Some(time.Since(*m.pingLastSend))	×
NEW 128	}	×
129
NEW 130	return fn.None[time.Duration]()	×
131	}
132
133	// pingHandler is the main goroutine responsible for enforcing the ping/pong
134	// protocol.
135	func (m *PingManager) pingHandler() {	3✔
136	defer m.wg.Done()	3✔
137	defer m.pingTimeout.Stop()	3✔
138		3✔
139	// Ensure that the pingTimeout channel is empty.	3✔
140	if !m.pingTimeout.Stop() {	3✔
141	<-m.pingTimeout.C	×
142	}	×
143
144	// Because we don't know if the OnPingFailure callback actually
145	// disconnects a peer (dependent on user config), we should never return
146	// from this loop unless the ping manager is stopped explicitly (which
147	// happens on disconnect).
148	for {	6✔
149	select {	3✔
UNCOV 150	case <-m.pingTicker.C:	×
UNCOV 151	// If this occurs it means that the new ping cycle has	×
UNCOV 152	// begun while there is still an outstanding ping	×
UNCOV 153	// awaiting a pong response. This should never occur,	×
UNCOV 154	// but if it does, it implies a timeout.	×
UNCOV 155	if m.outstandingPongSize >= 0 {	×
NEW 156	// Ping was outstanding, meaning it timed out by	×
NEW 157	// the arrival of the next ping interval.	×
NEW 158	timeWaited := m.pendingPingWait().UnwrapOr(	×
NEW 159	m.cfg.IntervalDuration,	×
NEW 160	)	×
NEW 161	lastRTT := m.getLastRTT()	×
NEW 162		×
NEW 163	m.cfg.OnPongFailure(	×
NEW 164	errors.New("ping timed "+	×
NEW 165	"out by next interval"),	×
NEW 166	timeWaited, lastRTT,	×
167	)	×
168		×
NEW 169	m.resetPingState()	×
170	}	×
171
UNCOV 172	pongSize := m.cfg.NewPongSize()	×
UNCOV 173	ping := &lnwire.Ping{	×
UNCOV 174	NumPongBytes: pongSize,	×
UNCOV 175	PaddingBytes: m.cfg.NewPingPayload(),	×
UNCOV 176	}	×
UNCOV 177		×
UNCOV 178	// Set up our bookkeeping for the new Ping.	×
UNCOV 179	if err := m.setPingState(pongSize); err != nil {	×
NEW 180	// This is an internal error related to timer	×
NEW 181	// reset. Pass it to OnPongFailure as it's	×
NEW 182	// critical. Current and last RTT are not	×
NEW 183	// directly applicable here.	×
NEW 184	m.cfg.OnPongFailure(err, 0, 0)	×
185		×
NEW 186	continue	×
187	}
188
UNCOV 189	m.cfg.SendPing(ping)	×
190
UNCOV 191	case <-m.pingTimeout.C:	×
NEW 192	timeWaited := m.pendingPingWait().UnwrapOr(	×
NEW 193	m.cfg.TimeoutDuration,	×
UNCOV 194	)	×
NEW 195	lastRTT := m.getLastRTT()	×
UNCOV 196		×
NEW 197	m.cfg.OnPongFailure(	×
NEW 198	errors.New("timeout while waiting for "+	×
NEW 199	"pong response"),	×
NEW 200	timeWaited, lastRTT,	×
NEW 201	)	×
UNCOV 202		×
NEW 203	m.resetPingState()	×
204
UNCOV 205	case pong := <-m.pongChan:	×
UNCOV 206	pongSize := int32(len(pong.PongBytes))	×
UNCOV 207		×
NEW 208	// Save off values we are about to override when we call	×
NEW 209	// resetPingState.	×
UNCOV 210	expected := m.outstandingPongSize	×
NEW 211	lastPingTime := m.pingLastSend	×
UNCOV 212		×
NEW 213	// This is an unexpected pong, we'll continue.	×
NEW 214	if lastPingTime == nil {	×
NEW 215	continue	×
216	}
217
NEW 218	actualRTT := time.Since(*lastPingTime)	×
UNCOV 219		×
NEW 220	// If the pong we receive doesn't match the ping we sent	×
NEW 221	// out, then we fail out.	×
UNCOV 222	if pongSize != expected {	×
NEW 223	e := fmt.Errorf("pong response does not match "+	×
NEW 224	"expected size. Expected: %d, Got: %d",	×
NEW 225	expected, pongSize)	×
UNCOV 226		×
NEW 227	lastRTT := m.getLastRTT()	×
NEW 228	m.cfg.OnPongFailure(e, actualRTT, lastRTT)	×
UNCOV 229		×
NEW 230	m.resetPingState()	×
UNCOV 231		×
NEW 232	continue	×
233	}
234
235	// Pong is good, update RTT and reset state.
NEW 236	m.pingTime.Store(&actualRTT)	×
NEW 237	m.resetPingState()	×
238
239	case <-m.quit:	3✔
240	return	3✔
241	}
242	}
243	}
244
245	// Stop interrupts the goroutines that the PingManager owns.
246	func (m *PingManager) Stop() {	3✔
247	if m.pingTicker == nil {	6✔
248	return	3✔
249	}	3✔
250
251	m.stopped.Do(func() {	6✔
252	close(m.quit)	3✔
253	m.wg.Wait()	3✔
254		3✔
255	m.pingTicker.Stop()	3✔
256	m.pingTimeout.Stop()	3✔
257	})	3✔
258	}
259
260	// setPingState is a private method to keep track of all of the fields we need
261	// to set when we send out a Ping.
UNCOV 262	func (m *PingManager) setPingState(pongSize uint16) error {	×
UNCOV 263	t := time.Now()	×
UNCOV 264	m.pingLastSend = &t	×
UNCOV 265	m.outstandingPongSize = int32(pongSize)	×
UNCOV 266	if m.pingTimeout.Reset(m.cfg.TimeoutDuration) {	×
267	return fmt.Errorf(	×
268	"impossible: ping timeout reset when already active",	×
269	)	×
270	}	×
271
UNCOV 272	return nil	×
273	}
274
275	// resetPingState is a private method that resets all of the bookkeeping that
276	// is tracking a currently outstanding Ping.
UNCOV 277	func (m *PingManager) resetPingState() {	×
UNCOV 278	m.pingLastSend = nil	×
UNCOV 279	m.outstandingPongSize = -1	×
NEW 280		×
UNCOV 281	if !m.pingTimeout.Stop() {	×
UNCOV 282	select {	×
283	case <-m.pingTimeout.C:	×
UNCOV 284	default:	×
285	}
286	}
287	}
288
289	// GetPingTimeMicroSeconds reports back the RTT calculated by the pingManager.
290	func (m *PingManager) GetPingTimeMicroSeconds() int64 {	3✔
291	rtt := m.pingTime.Load()	3✔
292		3✔
293	if rtt == nil {	6✔
294	return -1	3✔
295	}	3✔
296
297	return rtt.Microseconds()	×
298	}
299
300	// ReceivedPong is called to evaluate a Pong message against the expectations
301	// we have for it. It will cause the PingManager to invoke the supplied
302	// OnPongFailure function if the Pong argument supplied violates expectations.
UNCOV 303	func (m PingManager) ReceivedPong(msg lnwire.Pong) {	×
UNCOV 304	select {	×
UNCOV 305	case m.pongChan <- msg:	×
306	case <-m.quit:	×
307	}
308	}

lightningnetwork / lnd / 15039553421

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous