feat(ntp): enhances time sync with DHCP NTP and custom servers (#625)

* Ensure the mDNS mode is set every time network state changes

Eliminates (mostly) duplicate code

* Add custom NTP and HTTP time sync servers

Since the ordering may have been previously defaulted and saved as "ntp,http", but that was being ignored and fallback-defaults were being used, in Ordering, `ntp` means use the fallback NTP servers, and `http` means use the fallback HTTP URLs. Thus `ntp_user_provided` and `http_user_provided` are the user specified static lists.

* Add support for using DHCP-provided NTP server

internal/timesync/http.go

@@ -19,9 +19,9 @@ var defaultHTTPUrls = []string{
 	// "http://www.msftconnecttest.com/connecttest.txt",
 }
 
-func (t *TimeSync) queryAllHttpTime() (now *time.Time) {
-	chunkSize := 4
-	httpUrls := t.httpUrls
+func (t *TimeSync) queryAllHttpTime(httpUrls []string) (now *time.Time) {
+	chunkSize := int(t.networkConfig.TimeSyncParallel.ValueOr(4))
+	t.l.Info().Strs("httpUrls", httpUrls).Int("chunkSize", chunkSize).Msg("querying HTTP URLs")
 
 	// shuffle the http urls to avoid always querying the same servers
 	rand.Shuffle(len(httpUrls), func(i, j int) { httpUrls[i], httpUrls[j] = httpUrls[j], httpUrls[i] })

internal/timesync/metrics.go

@@ -73,6 +73,7 @@ var (
 		},
 		[]string{"url"},
 	)
+
 	metricNtpServerInfo = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "jetkvm_timesync_ntp_server_info",

internal/timesync/ntp.go

@@ -1,6 +1,7 @@
 package timesync
 
 import (
+	"context"
 	"math/rand/v2"
 	"strconv"
 	"time"
@@ -21,9 +22,9 @@ var defaultNTPServers = []string{
 	"3.pool.ntp.org",
 }
 
-func (t *TimeSync) queryNetworkTime() (now *time.Time, offset *time.Duration) {
-	chunkSize := 4
-	ntpServers := t.ntpServers
+func (t *TimeSync) queryNetworkTime(ntpServers []string) (now *time.Time, offset *time.Duration) {
+	chunkSize := int(t.networkConfig.TimeSyncParallel.ValueOr(4))
+	t.l.Info().Strs("servers", ntpServers).Int("chunkSize", chunkSize).Msg("querying NTP servers")
 
 	// shuffle the ntp servers to avoid always querying the same servers
 	rand.Shuffle(len(ntpServers), func(i, j int) { ntpServers[i], ntpServers[j] = ntpServers[j], ntpServers[i] })
@@ -46,6 +47,10 @@ type ntpResult struct {
 
 func (t *TimeSync) queryMultipleNTP(servers []string, timeout time.Duration) (now *time.Time, offset *time.Duration) {
 	results := make(chan *ntpResult, len(servers))
+
+	_, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
 	for _, server := range servers {
 		go func(server string) {
 			scopedLogger := t.l.With().
@@ -66,15 +71,25 @@ func (t *TimeSync) queryMultipleNTP(servers []string, timeout time.Duration) (no
 				return
 			}
 
+			if response.IsKissOfDeath() {
+				scopedLogger.Warn().
+					Str("kiss_code", response.KissCode).
+					Msg("ignoring NTP server kiss of death")
+				results <- nil
+				return
+			}
+
+			rtt := float64(response.RTT.Milliseconds())
+
 			// set the last RTT
 			metricNtpServerLastRTT.WithLabelValues(
 				server,
-			).Set(float64(response.RTT.Milliseconds()))
+			).Set(rtt)
 
 			// set the RTT histogram
 			metricNtpServerRttHistogram.WithLabelValues(
 				server,
-			).Observe(float64(response.RTT.Milliseconds()))
+			).Observe(rtt)
 
 			// set the server info
 			metricNtpServerInfo.WithLabelValues(
@@ -91,10 +106,13 @@ func (t *TimeSync) queryMultipleNTP(servers []string, timeout time.Duration) (no
 			scopedLogger.Info().
 				Str("time", now.Format(time.RFC3339)).
 				Str("reference", response.ReferenceString()).
-				Str("rtt", response.RTT.String()).
+				Float64("rtt", rtt).
 				Str("clockOffset", response.ClockOffset.String()).
 				Uint8("stratum", response.Stratum).
 				Msg("NTP server returned time")
+
+			cancel()
+
 			results <- &ntpResult{
 				now:    now,
 				offset: &response.ClockOffset,

internal/timesync/timesync.go

@@ -28,9 +28,8 @@ type TimeSync struct {
 	syncLock *sync.Mutex
 	l        *zerolog.Logger
 
-	ntpServers    []string
-	httpUrls      []string
-	networkConfig *network.NetworkConfig
+	networkConfig    *network.NetworkConfig
+	dhcpNtpAddresses []string
 
 	rtcDevicePath string
 	rtcDevice     *os.File //nolint:unused
@@ -64,14 +63,13 @@ func NewTimeSync(opts *TimeSyncOptions) *TimeSync {
 	}
 
 	t := &TimeSync{
-		syncLock:      &sync.Mutex{},
-		l:             opts.Logger,
-		rtcDevicePath: rtcDevice,
-		rtcLock:       &sync.Mutex{},
-		preCheckFunc:  opts.PreCheckFunc,
-		ntpServers:    defaultNTPServers,
-		httpUrls:      defaultHTTPUrls,
-		networkConfig: opts.NetworkConfig,
+		syncLock:         &sync.Mutex{},
+		l:                opts.Logger,
+		dhcpNtpAddresses: []string{},
+		rtcDevicePath:    rtcDevice,
+		rtcLock:          &sync.Mutex{},
+		preCheckFunc:     opts.PreCheckFunc,
+		networkConfig:    opts.NetworkConfig,
 	}
 
 	if t.rtcDevicePath != "" {
@@ -82,34 +80,42 @@ func NewTimeSync(opts *TimeSyncOptions) *TimeSync {
 	return t
 }
 
+func (t *TimeSync) SetDhcpNtpAddresses(addresses []string) {
+	t.dhcpNtpAddresses = addresses
+}
+
 func (t *TimeSync) getSyncMode() SyncMode {
 	syncMode := SyncMode{
+		Ntp:             true,
+		Http:            true,
+		Ordering:        []string{"ntp_dhcp", "ntp", "http"},
 		NtpUseFallback:  true,
 		HttpUseFallback: true,
 	}
-	var syncModeString string
 
 	if t.networkConfig != nil {
-		syncModeString = t.networkConfig.TimeSyncMode.String
+		switch t.networkConfig.TimeSyncMode.String {
+		case "ntp_only":
+			syncMode.Http = false
+		case "http_only":
+			syncMode.Ntp = false
+		}
+
 		if t.networkConfig.TimeSyncDisableFallback.Bool {
 			syncMode.NtpUseFallback = false
 			syncMode.HttpUseFallback = false
 		}
+
+		var syncOrdering = t.networkConfig.TimeSyncOrdering
+		if len(syncOrdering) > 0 {
+			syncMode.Ordering = syncOrdering
+		}
 	}
 
-	switch syncModeString {
-	case "ntp_only":
-		syncMode.Ntp = true
-	case "http_only":
-		syncMode.Http = true
-	default:
-		syncMode.Ntp = true
-		syncMode.Http = true
-	}
+	t.l.Debug().Strs("Ordering", syncMode.Ordering).Bool("Ntp", syncMode.Ntp).Bool("Http", syncMode.Http).Bool("NtpUseFallback", syncMode.NtpUseFallback).Bool("HttpUseFallback", syncMode.HttpUseFallback).Msg("sync mode")
 
 	return syncMode
 }
-
 func (t *TimeSync) doTimeSync() {
 	metricTimeSyncStatus.Set(0)
 	for {
@@ -154,16 +160,61 @@ func (t *TimeSync) Sync() error {
 		offset *time.Duration
 	)
 
-	syncMode := t.getSyncMode()
-
 	metricTimeSyncCount.Inc()
 
-	if syncMode.Ntp {
-		now, offset = t.queryNetworkTime()
-	}
+	syncMode := t.getSyncMode()
 
-	if syncMode.Http && now == nil {
-		now = t.queryAllHttpTime()
+Orders:
+	for _, mode := range syncMode.Ordering {
+		switch mode {
+		case "ntp_user_provided":
+			if syncMode.Ntp {
+				t.l.Info().Msg("using NTP custom servers")
+				now, offset = t.queryNetworkTime(t.networkConfig.TimeSyncNTPServers)
+				if now != nil {
+					t.l.Info().Str("source", "NTP").Time("now", *now).Msg("time obtained")
+					break Orders
+				}
+			}
+		case "ntp_dhcp":
+			if syncMode.Ntp {
+				t.l.Info().Msg("using NTP servers from DHCP")
+				now, offset = t.queryNetworkTime(t.dhcpNtpAddresses)
+				if now != nil {
+					t.l.Info().Str("source", "NTP DHCP").Time("now", *now).Msg("time obtained")
+					break Orders
+				}
+			}
+		case "ntp":
+			if syncMode.Ntp && syncMode.NtpUseFallback {
+				t.l.Info().Msg("using NTP fallback")
+				now, offset = t.queryNetworkTime(defaultNTPServers)
+				if now != nil {
+					t.l.Info().Str("source", "NTP fallback").Time("now", *now).Msg("time obtained")
+					break Orders
+				}
+			}
+		case "http_user_provided":
+			if syncMode.Http {
+				t.l.Info().Msg("using HTTP custom URLs")
+				now = t.queryAllHttpTime(t.networkConfig.TimeSyncHTTPUrls)
+				if now != nil {
+					t.l.Info().Str("source", "HTTP").Time("now", *now).Msg("time obtained")
+					break Orders
+				}
+			}
+		case "http":
+			if syncMode.Http && syncMode.HttpUseFallback {
+				t.l.Info().Msg("using HTTP fallback")
+				now = t.queryAllHttpTime(defaultHTTPUrls)
+				if now != nil {
+					t.l.Info().Str("source", "HTTP fallback").Time("now", *now).Msg("time obtained")
+					break Orders
+				}
+			}
+		default:
+			t.l.Warn().Str("mode", mode).Msg("unknown time sync mode, skipping")
+		}
 	}
 
 	if now == nil {