Fixed DHCP concurrency exit issues

Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
This commit is contained in:
Patryk Strusiewicz-Surmacki
2026-09-09 17:43:51 +02:00
committed by Marcel Fest
parent 3771ccee29
commit b4771c5319
7 changed files with 240 additions and 141 deletions

View File

@@ -343,6 +343,9 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
return nil, err
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("context error while starting DHCPv4 for %s/%s: error: %w",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, ctx.Err())
case err := <-instance.DHCPv4Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv4 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
@@ -358,6 +361,9 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
return nil, err
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("context error while starting DHCPv6 for %s/%s: error: %w",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, ctx.Err())
case err := <-instance.DHCPv6Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv6 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
@@ -654,7 +660,8 @@ func (i *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uin
wg.Go(func() {
if err := client.Start(ctx); err != nil {
log.Error("[instance] DHCP client error: %w")
log.Error("[instance] DHCP client", "error", err)
client.Stop()
}
})

View File

@@ -213,7 +213,7 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
}
if svcInstance == nil {
svcInstance, err = instance.NewInstance(ctx, svc, p.config, p.intfMgr, p.arpMgr, p.routeMgr, p.nodeLabelManager, wg)
svcInstance, err = instance.NewInstance(svcCtx.Ctx, svc, p.config, p.intfMgr, p.arpMgr, p.routeMgr, p.nodeLabelManager, wg)
if err != nil {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "new_instance").Inc()
return fmt.Errorf("unable to create instance for service %s/%s", svc.Namespace, svc.Name)

View File

@@ -239,7 +239,6 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
}
log.Debug("IPv4 update channel closed, stopping")
}
})
}
@@ -256,7 +255,7 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv6")
} else {
for ip := range inst.DHCPv4Client.IPChannel() {
for ip := range inst.DHCPv6Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
inst.VIPConfigs[index].VIP = ip
inst.DHCPInterfaceIPv6 = ip

View File

@@ -93,33 +93,41 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc *Callback,
}
// Used for tracking an active endpoint / pod
EventLoop:
for event := range ch {
metrics.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err := p.AddOrModify(watcherCtx, event, serviceFunc, forcedOnly, &wg, cancelWatcher); err != nil {
if utils.IsPanicError(err) {
return fmt.Errorf("add/modify service error: %w", err)
}
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
case watch.Deleted:
if err := p.Delete(event, forcedOnly); err != nil {
if utils.IsPanicError(err) {
return fmt.Errorf("delete service error: %w", err)
}
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
watchErr := utils.WatchError(event.Object)
log.Error("services", "err", watchErr)
return utils.WrapPanicError(watchErr, "service watch failed")
select {
case <-ctx.Done():
log.Info("global context done")
case <-watcherCtx.Done():
log.Info("WatcheConotext done")
break EventLoop
default:
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err := p.AddOrModify(watcherCtx, event, serviceFunc, forcedOnly, &wg, cancelWatcher); err != nil {
if utils.IsPanicError(err) {
return fmt.Errorf("add/modify service error: %w", err)
}
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
case watch.Deleted:
if err := p.Delete(event, forcedOnly); err != nil {
if utils.IsPanicError(err) {
return fmt.Errorf("delete service error: %w", err)
}
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
watchErr := utils.WatchError(event.Object)
log.Error("services", "err", watchErr)
return utils.WrapPanicError(watchErr, "service watch failed")
default:
}
}
}

View File

@@ -27,12 +27,24 @@ type DHCPv4Client struct {
initRebootFlag bool
requestedIP net.IP
broadcastFlag bool
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
releasedChan chan struct{} // indicate that the IP has been released
stopChan chan struct{} // is used by external clients to stop DHCP
errorChan chan error // indicates there was an error on the IP request
ipChan chan string
backoffAttempts uint
stop sync.Once
stopOnce sync.Once
mtx sync.RWMutex
}
func (c *DHCPv4Client) storeLease(lease *nclient4.Lease) {
c.mtx.Lock()
defer c.mtx.Unlock()
c.lease = lease
}
func (c *DHCPv4Client) loadLease() *nclient4.Lease {
c.mtx.RLock()
defer c.mtx.RUnlock()
return c.lease
}
// NewDHCPv4Client returns a new DHCP Client.
@@ -40,7 +52,6 @@ func NewDHCPv4Client(iface *net.Interface, initRebootFlag bool, requestedIP stri
return &DHCPv4Client{
iface: iface,
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
errorChan: make(chan error),
initRebootFlag: initRebootFlag,
requestedIP: net.ParseIP(requestedIP),
@@ -57,11 +68,14 @@ func (c *DHCPv4Client) WithHostName(hostname string) DHCPClient {
// Stop state-transition process and close dhcp client
func (c *DHCPv4Client) Stop() {
c.stop.Do(func() {
c.close()
}
func (c *DHCPv4Client) close() {
c.stopOnce.Do(func() {
close(c.ipChan)
close(c.stopChan)
})
<-c.releasedChan
}
// Gets the IPChannel for consumption
@@ -129,80 +143,93 @@ func (c *DHCPv4Client) ErrorChannel() chan error {
// ----------
// Figure: State-transition diagram for DHCP clients
func (c *DHCPv4Client) Start(ctx context.Context) error {
dhcpCtx, cancel := context.WithCancel(ctx)
defer cancel()
lease, err := c.requestWithBackoff(dhcpCtx)
lease, err := c.requestWithBackoff(ctx)
if err != nil {
return fmt.Errorf("DHCPv4 client failed: %w", err)
}
c.initRebootFlag = false
c.lease = lease
// Set up two ticker to renew/rebind regularly
t1Timeout := c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 2
t2Timeout := (c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 8) * 7
c.storeLease(lease)
// Set up two timers to renew/rebind regularly
t1Timeout, t2Timeout := getLeaseTimeouts(lease)
log.Debug("[DHCPv4] timeouts", "timeout1", t1Timeout, "timeout2", t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
t1, t2 := time.NewTimer(t1Timeout), time.NewTimer(t2Timeout)
for {
select {
case <-c.stopChan:
return c.killProcessing(t1, t2)
case <-ctx.Done():
c.close()
return c.killProcessing(t1, t2)
case <-t1.C:
// renew is a unicast request of the IP renewal
// A point on renew is: the library does not return the right message (NAK)
// on renew error due to IP Change, but instead it returns a different error
// This way there's not much to do other than log and continue, as the renew error
// may be an offline server, or may be an incorrect package match
lease, err := c.renew(dhcpCtx)
lease, err := c.renew(ctx)
if err == nil {
c.lease = lease
c.storeLease(lease)
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
log.Info("[DHCPv4] renew", "lease", lease)
t2.Reset(t2Timeout)
} else {
log.Error("[DHCPv4] renew failed", "err", err)
}
t1.Reset(t1Timeout)
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
lease, err := c.request(dhcpCtx, true)
lease, err := c.request(ctx, true)
if err == nil {
c.lease = lease
c.storeLease(lease)
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
log.Info("[DHCPv4] rebind", "lease", lease)
} else {
if _, ok := err.(*nclient4.ErrNak); !ok {
t1.Stop()
t2.Stop()
log.Error("[DHCPv4] rebind failed", "err", err)
}
log.Warn("[DHCPv4] ip may have changed", "ip", c.lease.ACK.YourIPAddr, "err", err)
lease = c.loadLease()
log.Warn("[DHCPv4] ip may have changed", "ip", lease.ACK.YourIPAddr, "err", err)
c.initRebootFlag = false
lease, backoffErr := c.requestWithBackoff(dhcpCtx)
lease, backoffErr := c.requestWithBackoff(ctx)
if backoffErr != nil {
log.Error("[DHCPv4] failed to reacquire lease", "err", backoffErr)
continue
}
c.lease = lease
c.storeLease(lease)
t1Timeout, t2Timeout = getLeaseTimeouts(lease)
}
t1.Reset(t1Timeout)
t2.Reset(t2Timeout)
case <-c.stopChan:
// release is a unicast request of the IP release.
var err error
if err = c.release(); err != nil {
log.Error("[DHCPv4] release lease failed", "lease", lease, "err", err)
} else {
log.Info("[DHCPv4] release", "lease", lease)
}
t1.Stop()
t2.Stop()
close(c.releasedChan)
return err
}
}
}
func getLeaseTimeouts(lease *nclient4.Lease) (time.Duration, time.Duration) {
t1Timeout, t2Timeout := lease.ACK.IPAddressLeaseTime(defaultDHCPRenew)/2, (lease.ACK.IPAddressLeaseTime(defaultDHCPRenew)/8)*7
log.Debug("[DHCPv4] timeouts", "address", lease.ACK.YourIPAddr.String(), "T1", t1Timeout, "T2", t2Timeout)
return t1Timeout, t2Timeout
}
func (c *DHCPv4Client) killProcessing(t1, t2 *time.Timer) error {
// release is a unicast request of the IP release.
var err error
lease := c.loadLease()
if lease != nil {
if err = c.release(); err != nil {
log.Error("[DHCPv4] release lease failed", "lease", lease, "err", err)
} else {
log.Info("[DHCPv4] release", "lease", lease)
}
}
t1.Stop()
t2.Stop()
return err
}
// --------------------------------------------------------
// | |INIT-REBOOT | RENEWING |REBINDING |
// --------------------------------------------------------
@@ -225,27 +252,38 @@ func (c *DHCPv4Client) requestWithBackoff(ctx context.Context) (*nclient4.Lease,
log.Debug("[DHCPv4]", "attempts", c.backoffAttempts)
RequestLoop:
for {
log.Debug("[DHCPv4] trying to get a new IP", "attempt", backoff.Attempt()+1)
lease, err = c.request(ctx, false)
if err != nil {
dur := backoff.Duration()
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
log.Error(fmt.Sprintf("[DHCPv4] %s", errMsg.Error()))
c.errorChan <- errMsg
return nil, errMsg
select {
case <-ctx.Done():
return nil, fmt.Errorf("[DHCPv4] context error: %w", ctx.Err())
default:
log.Debug("[DHCPv4] trying to get a new IP", "attempt", backoff.Attempt()+1)
lease, err = c.request(ctx, false)
if err != nil {
dur := backoff.Duration()
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
log.Error(fmt.Sprintf("[DHCPv4] %s", errMsg.Error()))
c.errorChan <- errMsg
return nil, errMsg
}
log.Error("[DHCPv4] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
t := time.NewTimer(dur)
select {
case <-t.C:
t.Stop()
case <-ctx.Done():
}
continue RequestLoop
}
log.Error("[DHCPv4] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
time.Sleep(dur)
continue
backoff.Reset()
break RequestLoop
}
backoff.Reset()
break
}
if c.ipChan != nil {
log.Debug("[DHCPv4] using channel")
c.ipChan <- lease.ACK.YourIPAddr.String()
}
@@ -296,7 +334,12 @@ func (c *DHCPv4Client) release() error {
defer dhclient.Close()
// TODO modify lease
return dhclient.Release(c.lease)
err = dhclient.Release(c.lease)
if err != nil {
return fmt.Errorf("DHCPv4 release failed: %w", err)
}
return nil
}
func (c *DHCPv4Client) renew(ctx context.Context) (*nclient4.Lease, error) {

View File

@@ -95,13 +95,25 @@ type DHCPv6Client struct {
initRebootFlag bool
requestedIP net.IP
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
releasedChan chan struct{} // indicate that the IP has been released
errorChan chan error // indicates there was an error on the IP request
ipChan chan string
ic *DHCPv6InternalClient
addr *dhcpv6.OptIAAddress
backoffAttempts uint
stop sync.Once
mtx sync.RWMutex
}
func (c *DHCPv6Client) storeAddr(addr *dhcpv6.OptIAAddress) {
c.mtx.Lock()
defer c.mtx.Unlock()
c.addr = addr
}
func (c *DHCPv6Client) loadAddr() *dhcpv6.OptIAAddress {
c.mtx.RLock()
defer c.mtx.RUnlock()
return c.addr
}
// NewDHCPv6Client returns a new DHCP6 Client.
@@ -120,7 +132,6 @@ func NewDHCPv6Client(iface *net.Interface, parent netlink.Link, initRebootFlag b
iface: iface,
managerKey: name,
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
errorChan: make(chan error),
initRebootFlag: initRebootFlag,
requestedIP: net.ParseIP(requestedIP),
@@ -137,11 +148,15 @@ func (c *DHCPv6Client) WithHostName(hostname string) DHCPClient {
// Stop state-transition process and close dhcp client
func (c *DHCPv6Client) Stop() {
c.close()
}
// Close dhcp client channels
func (c *DHCPv6Client) close() {
c.stop.Do(func() {
close(c.ipChan)
close(c.stopChan)
})
<-c.releasedChan
dhcpv6ClientManager.Delete(c.managerKey)
}
@@ -156,27 +171,27 @@ func (c *DHCPv6Client) ErrorChannel() chan error {
}
func (c *DHCPv6Client) Start(ctx context.Context) error {
dhcpCtx, cancel := context.WithCancel(ctx)
defer cancel()
addr, err := c.requestWithBackoff(dhcpCtx)
addr, err := c.requestWithBackoff(ctx)
if err != nil {
return fmt.Errorf("DHCPv6 client failed: %w", err)
}
c.addr = addr
c.initRebootFlag = false
c.storeAddr(addr)
// Set up two ticker to renew/rebind regularly
t1Timeout := c.addr.PreferredLifetime / 2
t2Timeout := (c.addr.ValidLifetime / 8) * 7
log.Debug("[DHCPv6] timeouts", "timeout1", t1Timeout, "timeout2", t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
t1Timeout, t2Timeout := getAddrTimeouts(addr)
t1, t2 := time.NewTimer(t1Timeout), time.NewTimer(t2Timeout)
for {
select {
case <-c.stopChan:
return c.killProcessing(t1, t2)
case <-ctx.Done():
c.close()
return c.killProcessing(t1, t2)
case <-t1.C:
// renew is a unicast request of the IP renewal
// A point on renew is: the library does not return the right message (NAK)
@@ -184,49 +199,65 @@ func (c *DHCPv6Client) Start(ctx context.Context) error {
// This way there's not much to do other than log and continue, as the renew error
// may be an offline server, or may be an incorrect package match
addr, err := c.renew(dhcpCtx)
addr, err := c.renew(ctx)
if err == nil {
c.addr = addr
c.storeAddr(addr)
log.Info("[DHCPv6] renew", "addr", addr.IPv6Addr.String())
t1Timeout, t2Timeout = getAddrTimeouts(addr)
t2.Reset(t2Timeout)
} else {
log.Error("[DHCPv6] renew failed", "err", err)
}
t1.Reset(t1Timeout)
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
addr, err := c.request(dhcpCtx, true)
addr, err := c.request(ctx, true)
if err == nil {
c.addr = addr
c.storeAddr(addr)
log.Info("[DHCPv6] rebind", "lease", addr)
t1Timeout, t2Timeout = getAddrTimeouts(addr)
} else {
addr = c.loadAddr()
log.Warn("[DHCPv6] ip may have changed", "ip", addr.IPv6Addr.String(), "err", err)
c.initRebootFlag = false
c.addr, err = c.requestWithBackoff(dhcpCtx)
log.Error("[DHCPv6] rebind failed", "err", err)
addr, backoffErr := c.requestWithBackoff(ctx)
if backoffErr != nil {
log.Error("[DHCPv6] failed to reacquire lease", "err", backoffErr)
continue
}
c.storeAddr(addr)
t1Timeout, t2Timeout = getAddrTimeouts(addr)
}
t1.Reset(t1Timeout)
t2.Reset(t2Timeout)
case <-c.stopChan:
// create new context for DHCP cleanup (independent)
dhcpStopCtx, cancel := context.WithCancel(context.Background())
defer cancel()
// IP address release.
var err error
if err = c.release(dhcpStopCtx); err != nil {
log.Error("[DHCPv6] release failed", "err", err)
} else {
log.Info("[DHCPv6] released", "address", c.addr.String())
}
t1.Stop()
t2.Stop()
close(c.releasedChan)
return err
}
}
}
func getAddrTimeouts(addr *dhcpv6.OptIAAddress) (time.Duration, time.Duration) {
t1Timeout, t2Timeout := addr.PreferredLifetime/2, (addr.ValidLifetime/8)*7
log.Debug("[DHCPv6] timeouts", "address", addr.IPv6Addr.String(), "T1", t1Timeout, "T2", t2Timeout)
return t1Timeout, t2Timeout
}
func (c *DHCPv6Client) killProcessing(t1, t2 *time.Timer) error {
// create new context for DHCP cleanup (independent)
dhcpStopCtx, cancel := context.WithCancel(context.Background())
defer cancel()
// IP address release.
var err error
if c.loadAddr() != nil {
if err = c.release(dhcpStopCtx); err != nil {
log.Error("[DHCPv6] release failed", "err", err)
} else {
log.Info("[DHCPv6] released", "address", c.addr.String())
}
}
t1.Stop()
t2.Stop()
return err
}
func (c *DHCPv6Client) requestWithBackoff(ctx context.Context) (*dhcpv6.OptIAAddress, error) {
backoff := backoff.Backoff{
Factor: 2,
@@ -235,32 +266,44 @@ func (c *DHCPv6Client) requestWithBackoff(ctx context.Context) (*dhcpv6.OptIAAdd
Max: 1 * time.Minute,
}
var err error
var addr *dhcpv6.OptIAAddress
var err error
log.Debug("[DHCPv6]", "attempts", c.backoffAttempts)
RequestLoop:
for {
log.Debug("[DHCPv6] trying to get a new IP", "attempt", backoff.Attempt()+1)
select {
case <-ctx.Done():
return nil, fmt.Errorf("[DHCPv6] context error: %w", ctx.Err())
default:
log.Debug("[DHCPv6] trying to get a new IP", "attempt", backoff.Attempt()+1)
addr, err = c.request(ctx, false)
addr, err = c.request(ctx, false)
if err != nil {
dur := backoff.Duration()
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
log.Error(fmt.Sprintf("[DHCPv6] %s", errMsg.Error()))
c.errorChan <- errMsg
return nil, fmt.Errorf("failed to get IPv6 address: %w", err)
if err != nil {
dur := backoff.Duration()
if c.backoffAttempts > 0 && backoff.Attempt() > float64(c.backoffAttempts)-1 {
errMsg := fmt.Errorf("failed to get an IPv4 address after %d attempt(s), giving up, error: %s", c.backoffAttempts, err.Error())
log.Error(fmt.Sprintf("[DHCPv6] %s", errMsg.Error()))
c.errorChan <- errMsg
return nil, fmt.Errorf("failed to get IPv6 address: %w", err)
}
log.Error("[DHCPv6] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
t := time.NewTimer(dur)
select {
case <-t.C:
t.Stop()
case <-ctx.Done():
}
continue RequestLoop
}
log.Error("[DHCPv6] request failed", "attempt", backoff.Attempt(), "err", err.Error(), "waiting", dur)
time.Sleep(dur)
continue
backoff.Reset()
break RequestLoop
}
backoff.Reset()
break
}
if c.ipChan != nil {
log.Debug("[DHCPv6] using channel")
c.ipChan <- addr.IPv6Addr.String()
}

View File

@@ -21,14 +21,13 @@ func TestDHCPv6StopReleasesManagerReferenceForParentInterface(t *testing.T) {
}
client := &DHCPv6Client{
iface: &net.Interface{Name: "vlan-child"},
managerKey: "parent0",
ipChan: make(chan string),
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
ic: shared,
iface: &net.Interface{Name: "vlan-child"},
managerKey: "parent0",
ipChan: make(chan string),
stopChan: make(chan struct{}),
ic: shared,
addr: &dhcpv6.OptIAAddress{},
}
close(client.releasedChan)
client.Stop()