mirror of
https://github.com/luckfox-eng29/kvm.git
synced 2026-01-18 03:28:19 +01:00
network enhanecment / refactor (#361)
* chore(network): improve connectivity check * refactor(network): rewrite network and timesync component * feat(display): show cloud connection status * chore: change logging verbosity * chore(websecure): update log message * fix(ota): validate root certificate when downloading update * feat(ui): add network settings tab * fix(display): cloud connecting animation * fix: golintci issues * feat: add network settings tab * feat(timesync): query servers in parallel * refactor(network): move to internal/network package * feat(timesync): add metrics * refactor(log): move log to internal/logging package * refactor(mdms): move mdns to internal/mdns package * feat(developer): add pprof endpoint * feat(logging): add a simple logging streaming endpoint * fix(mdns): do not start mdns until network is up * feat(network): allow users to update network settings from ui * fix(network): handle errors when net.IPAddr is nil * fix(mdns): scopedLogger SIGSEGV * fix(dhcp): watch directory instead of file to catch fsnotify.Create event * refactor(nbd): move platform-specific code to different files * refactor(native): move platform-specific code to different files * chore: fix linter issues * chore(dev_deploy): allow to override PION_LOG_TRACE
This commit is contained in:
132
internal/timesync/http.go
Normal file
132
internal/timesync/http.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package timesync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var defaultHTTPUrls = []string{
|
||||
"http://www.gstatic.com/generate_204",
|
||||
"http://cp.cloudflare.com/",
|
||||
"http://edge-http.microsoft.com/captiveportal/generate_204",
|
||||
// Firefox, Apple, and Microsoft have inconsistent results, so we don't use it
|
||||
// "http://detectportal.firefox.com/",
|
||||
// "http://www.apple.com/library/test/success.html",
|
||||
// "http://www.msftconnecttest.com/connecttest.txt",
|
||||
}
|
||||
|
||||
func (t *TimeSync) queryAllHttpTime() (now *time.Time) {
|
||||
chunkSize := 4
|
||||
httpUrls := t.httpUrls
|
||||
|
||||
// shuffle the http urls to avoid always querying the same servers
|
||||
rand.Shuffle(len(httpUrls), func(i, j int) { httpUrls[i], httpUrls[j] = httpUrls[j], httpUrls[i] })
|
||||
|
||||
for i := 0; i < len(httpUrls); i += chunkSize {
|
||||
chunk := httpUrls[i:min(i+chunkSize, len(httpUrls))]
|
||||
results := t.queryMultipleHttp(chunk, timeSyncTimeout)
|
||||
if results != nil {
|
||||
return results
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TimeSync) queryMultipleHttp(urls []string, timeout time.Duration) (now *time.Time) {
|
||||
results := make(chan *time.Time, len(urls))
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
for _, url := range urls {
|
||||
go func(url string) {
|
||||
scopedLogger := t.l.With().
|
||||
Str("http_url", url).
|
||||
Logger()
|
||||
|
||||
metricHttpRequestCount.WithLabelValues(url).Inc()
|
||||
metricHttpTotalRequestCount.Inc()
|
||||
|
||||
startTime := time.Now()
|
||||
now, response, err := queryHttpTime(
|
||||
ctx,
|
||||
url,
|
||||
timeout,
|
||||
)
|
||||
duration := time.Since(startTime)
|
||||
|
||||
metricHttpServerLastRTT.WithLabelValues(url).Set(float64(duration.Milliseconds()))
|
||||
metricHttpServerRttHistogram.WithLabelValues(url).Observe(float64(duration.Milliseconds()))
|
||||
|
||||
status := 0
|
||||
if response != nil {
|
||||
status = response.StatusCode
|
||||
}
|
||||
metricHttpServerInfo.WithLabelValues(
|
||||
url,
|
||||
strconv.Itoa(status),
|
||||
).Set(1)
|
||||
|
||||
if err == nil {
|
||||
metricHttpTotalSuccessCount.Inc()
|
||||
metricHttpSuccessCount.WithLabelValues(url).Inc()
|
||||
|
||||
requestId := response.Header.Get("X-Request-Id")
|
||||
if requestId != "" {
|
||||
requestId = response.Header.Get("X-Msedge-Ref")
|
||||
}
|
||||
if requestId == "" {
|
||||
requestId = response.Header.Get("Cf-Ray")
|
||||
}
|
||||
scopedLogger.Info().
|
||||
Str("time", now.Format(time.RFC3339)).
|
||||
Int("status", status).
|
||||
Str("request_id", requestId).
|
||||
Str("time_taken", duration.String()).
|
||||
Msg("HTTP server returned time")
|
||||
|
||||
cancel()
|
||||
results <- now
|
||||
} else if errors.Is(err, context.Canceled) {
|
||||
metricHttpCancelCount.WithLabelValues(url).Inc()
|
||||
metricHttpTotalCancelCount.Inc()
|
||||
} else {
|
||||
scopedLogger.Warn().
|
||||
Str("error", err.Error()).
|
||||
Int("status", status).
|
||||
Msg("failed to query HTTP server")
|
||||
}
|
||||
}(url)
|
||||
}
|
||||
|
||||
return <-results
|
||||
}
|
||||
|
||||
func queryHttpTime(
|
||||
ctx context.Context,
|
||||
url string,
|
||||
timeout time.Duration,
|
||||
) (now *time.Time, response *http.Response, err error) {
|
||||
client := http.Client{
|
||||
Timeout: timeout,
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
dateStr := resp.Header.Get("Date")
|
||||
parsedTime, err := time.Parse(time.RFC1123, dateStr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return &parsedTime, resp, nil
|
||||
}
|
||||
147
internal/timesync/metrics.go
Normal file
147
internal/timesync/metrics.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package timesync
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
metricTimeSyncStatus = promauto.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "jetkvm_timesync_status",
|
||||
Help: "The status of the timesync, 1 if successful, 0 if not",
|
||||
},
|
||||
)
|
||||
metricTimeSyncCount = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_count",
|
||||
Help: "The number of times the timesync has been run",
|
||||
},
|
||||
)
|
||||
metricTimeSyncSuccessCount = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_success_count",
|
||||
Help: "The number of times the timesync has been successful",
|
||||
},
|
||||
)
|
||||
metricRTCUpdateCount = promauto.NewCounter( //nolint:unused
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_rtc_update_count",
|
||||
Help: "The number of times the RTC has been updated",
|
||||
},
|
||||
)
|
||||
metricNtpTotalSuccessCount = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_ntp_total_success_count",
|
||||
Help: "The total number of successful NTP requests",
|
||||
},
|
||||
)
|
||||
metricNtpTotalRequestCount = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_ntp_total_request_count",
|
||||
Help: "The total number of NTP requests sent",
|
||||
},
|
||||
)
|
||||
metricNtpSuccessCount = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_ntp_success_count",
|
||||
Help: "The number of successful NTP requests",
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricNtpRequestCount = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_ntp_request_count",
|
||||
Help: "The number of NTP requests sent to the server",
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricNtpServerLastRTT = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "jetkvm_timesync_ntp_server_last_rtt",
|
||||
Help: "The last RTT of the NTP server in milliseconds",
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricNtpServerRttHistogram = promauto.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "jetkvm_timesync_ntp_server_rtt",
|
||||
Help: "The histogram of the RTT of the NTP server in milliseconds",
|
||||
Buckets: []float64{
|
||||
10, 25, 50, 100, 200, 300, 500, 1000,
|
||||
},
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricNtpServerInfo = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "jetkvm_timesync_ntp_server_info",
|
||||
Help: "The info of the NTP server",
|
||||
},
|
||||
[]string{"url", "reference", "stratum", "precision"},
|
||||
)
|
||||
|
||||
metricHttpTotalSuccessCount = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_http_total_success_count",
|
||||
Help: "The total number of successful HTTP requests",
|
||||
},
|
||||
)
|
||||
metricHttpTotalRequestCount = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_http_total_request_count",
|
||||
Help: "The total number of HTTP requests sent",
|
||||
},
|
||||
)
|
||||
metricHttpTotalCancelCount = promauto.NewCounter(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_http_total_cancel_count",
|
||||
Help: "The total number of HTTP requests cancelled",
|
||||
},
|
||||
)
|
||||
metricHttpSuccessCount = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_http_success_count",
|
||||
Help: "The number of successful HTTP requests",
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricHttpRequestCount = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_http_request_count",
|
||||
Help: "The number of HTTP requests sent to the server",
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricHttpCancelCount = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "jetkvm_timesync_http_cancel_count",
|
||||
Help: "The number of HTTP requests cancelled",
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricHttpServerLastRTT = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "jetkvm_timesync_http_server_last_rtt",
|
||||
Help: "The last RTT of the HTTP server in milliseconds",
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricHttpServerRttHistogram = promauto.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Name: "jetkvm_timesync_http_server_rtt",
|
||||
Help: "The histogram of the RTT of the HTTP server in milliseconds",
|
||||
Buckets: []float64{
|
||||
10, 25, 50, 100, 200, 300, 500, 1000,
|
||||
},
|
||||
},
|
||||
[]string{"url"},
|
||||
)
|
||||
metricHttpServerInfo = promauto.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "jetkvm_timesync_http_server_info",
|
||||
Help: "The info of the HTTP server",
|
||||
},
|
||||
[]string{"url", "http_code"},
|
||||
)
|
||||
)
|
||||
113
internal/timesync/ntp.go
Normal file
113
internal/timesync/ntp.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package timesync
|
||||
|
||||
import (
|
||||
"math/rand/v2"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/beevik/ntp"
|
||||
)
|
||||
|
||||
var defaultNTPServers = []string{
|
||||
"time.apple.com",
|
||||
"time.aws.com",
|
||||
"time.windows.com",
|
||||
"time.google.com",
|
||||
"162.159.200.123", // time.cloudflare.com
|
||||
"0.pool.ntp.org",
|
||||
"1.pool.ntp.org",
|
||||
"2.pool.ntp.org",
|
||||
"3.pool.ntp.org",
|
||||
}
|
||||
|
||||
func (t *TimeSync) queryNetworkTime() (now *time.Time, offset *time.Duration) {
|
||||
chunkSize := 4
|
||||
ntpServers := t.ntpServers
|
||||
|
||||
// shuffle the ntp servers to avoid always querying the same servers
|
||||
rand.Shuffle(len(ntpServers), func(i, j int) { ntpServers[i], ntpServers[j] = ntpServers[j], ntpServers[i] })
|
||||
|
||||
for i := 0; i < len(ntpServers); i += chunkSize {
|
||||
chunk := ntpServers[i:min(i+chunkSize, len(ntpServers))]
|
||||
now, offset := t.queryMultipleNTP(chunk, timeSyncTimeout)
|
||||
if now != nil {
|
||||
return now, offset
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type ntpResult struct {
|
||||
now *time.Time
|
||||
offset *time.Duration
|
||||
}
|
||||
|
||||
func (t *TimeSync) queryMultipleNTP(servers []string, timeout time.Duration) (now *time.Time, offset *time.Duration) {
|
||||
results := make(chan *ntpResult, len(servers))
|
||||
for _, server := range servers {
|
||||
go func(server string) {
|
||||
scopedLogger := t.l.With().
|
||||
Str("server", server).
|
||||
Logger()
|
||||
|
||||
// increase request count
|
||||
metricNtpTotalRequestCount.Inc()
|
||||
metricNtpRequestCount.WithLabelValues(server).Inc()
|
||||
|
||||
// query the server
|
||||
now, response, err := queryNtpServer(server, timeout)
|
||||
|
||||
// set the last RTT
|
||||
metricNtpServerLastRTT.WithLabelValues(
|
||||
server,
|
||||
).Set(float64(response.RTT.Milliseconds()))
|
||||
|
||||
// set the RTT histogram
|
||||
metricNtpServerRttHistogram.WithLabelValues(
|
||||
server,
|
||||
).Observe(float64(response.RTT.Milliseconds()))
|
||||
|
||||
// set the server info
|
||||
metricNtpServerInfo.WithLabelValues(
|
||||
server,
|
||||
response.ReferenceString(),
|
||||
strconv.Itoa(int(response.Stratum)),
|
||||
strconv.Itoa(int(response.Precision)),
|
||||
).Set(1)
|
||||
|
||||
if err == nil {
|
||||
// increase success count
|
||||
metricNtpTotalSuccessCount.Inc()
|
||||
metricNtpSuccessCount.WithLabelValues(server).Inc()
|
||||
|
||||
scopedLogger.Info().
|
||||
Str("time", now.Format(time.RFC3339)).
|
||||
Str("reference", response.ReferenceString()).
|
||||
Str("rtt", response.RTT.String()).
|
||||
Str("clockOffset", response.ClockOffset.String()).
|
||||
Uint8("stratum", response.Stratum).
|
||||
Msg("NTP server returned time")
|
||||
results <- &ntpResult{
|
||||
now: now,
|
||||
offset: &response.ClockOffset,
|
||||
}
|
||||
} else {
|
||||
scopedLogger.Warn().
|
||||
Str("error", err.Error()).
|
||||
Msg("failed to query NTP server")
|
||||
}
|
||||
}(server)
|
||||
}
|
||||
|
||||
result := <-results
|
||||
return result.now, result.offset
|
||||
}
|
||||
|
||||
func queryNtpServer(server string, timeout time.Duration) (now *time.Time, response *ntp.Response, err error) {
|
||||
resp, err := ntp.QueryWithOptions(server, ntp.QueryOptions{Timeout: timeout})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return &resp.Time, resp, nil
|
||||
}
|
||||
26
internal/timesync/rtc.go
Normal file
26
internal/timesync/rtc.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package timesync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
var (
|
||||
rtcDeviceSearchPaths = []string{
|
||||
"/dev/rtc",
|
||||
"/dev/rtc0",
|
||||
"/dev/rtc1",
|
||||
"/dev/misc/rtc",
|
||||
"/dev/misc/rtc0",
|
||||
"/dev/misc/rtc1",
|
||||
}
|
||||
)
|
||||
|
||||
func getRtcDevicePath() (string, error) {
|
||||
for _, path := range rtcDeviceSearchPaths {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("rtc device not found")
|
||||
}
|
||||
105
internal/timesync/rtc_linux.go
Normal file
105
internal/timesync/rtc_linux.go
Normal file
@@ -0,0 +1,105 @@
|
||||
//go:build linux
|
||||
|
||||
package timesync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func TimetoRtcTime(t time.Time) unix.RTCTime {
|
||||
return unix.RTCTime{
|
||||
Sec: int32(t.Second()),
|
||||
Min: int32(t.Minute()),
|
||||
Hour: int32(t.Hour()),
|
||||
Mday: int32(t.Day()),
|
||||
Mon: int32(t.Month() - 1),
|
||||
Year: int32(t.Year() - 1900),
|
||||
Wday: int32(0),
|
||||
Yday: int32(0),
|
||||
Isdst: int32(0),
|
||||
}
|
||||
}
|
||||
|
||||
func RtcTimetoTime(t unix.RTCTime) time.Time {
|
||||
return time.Date(
|
||||
int(t.Year)+1900,
|
||||
time.Month(t.Mon+1),
|
||||
int(t.Mday),
|
||||
int(t.Hour),
|
||||
int(t.Min),
|
||||
int(t.Sec),
|
||||
0,
|
||||
time.UTC,
|
||||
)
|
||||
}
|
||||
|
||||
func (t *TimeSync) getRtcDevice() (*os.File, error) {
|
||||
if t.rtcDevice == nil {
|
||||
file, err := os.OpenFile(t.rtcDevicePath, os.O_RDWR, 0666)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.rtcDevice = file
|
||||
}
|
||||
return t.rtcDevice, nil
|
||||
}
|
||||
|
||||
func (t *TimeSync) getRtcDeviceFd() (int, error) {
|
||||
device, err := t.getRtcDevice()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int(device.Fd()), nil
|
||||
}
|
||||
|
||||
// Read implements Read for the Linux RTC
|
||||
func (t *TimeSync) readRtcTime() (time.Time, error) {
|
||||
fd, err := t.getRtcDeviceFd()
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("failed to get RTC device fd: %w", err)
|
||||
}
|
||||
|
||||
rtcTime, err := unix.IoctlGetRTCTime(fd)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("failed to get RTC time: %w", err)
|
||||
}
|
||||
|
||||
date := RtcTimetoTime(*rtcTime)
|
||||
|
||||
return date, nil
|
||||
}
|
||||
|
||||
// Set implements Set for the Linux RTC
|
||||
// ...
|
||||
// It might be not accurate as the time consumed by the system call is not taken into account
|
||||
// but it's good enough for our purposes
|
||||
func (t *TimeSync) setRtcTime(tu time.Time) error {
|
||||
rt := TimetoRtcTime(tu)
|
||||
|
||||
fd, err := t.getRtcDeviceFd()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get RTC device fd: %w", err)
|
||||
}
|
||||
|
||||
currentRtcTime, err := t.readRtcTime()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read RTC time: %w", err)
|
||||
}
|
||||
|
||||
t.l.Info().
|
||||
Interface("rtc_time", tu).
|
||||
Str("offset", tu.Sub(currentRtcTime).String()).
|
||||
Msg("set rtc time")
|
||||
|
||||
if err := unix.IoctlSetRTCTime(fd, &rt); err != nil {
|
||||
return fmt.Errorf("failed to set RTC time: %w", err)
|
||||
}
|
||||
|
||||
metricRTCUpdateCount.Inc()
|
||||
|
||||
return nil
|
||||
}
|
||||
16
internal/timesync/rtc_notlinux.go
Normal file
16
internal/timesync/rtc_notlinux.go
Normal file
@@ -0,0 +1,16 @@
|
||||
//go:build !linux
|
||||
|
||||
package timesync
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (t *TimeSync) readRtcTime() (time.Time, error) {
|
||||
return time.Now(), nil
|
||||
}
|
||||
|
||||
func (t *TimeSync) setRtcTime(tu time.Time) error {
|
||||
return errors.New("not supported")
|
||||
}
|
||||
208
internal/timesync/timesync.go
Normal file
208
internal/timesync/timesync.go
Normal file
@@ -0,0 +1,208 @@
|
||||
package timesync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jetkvm/kvm/internal/network"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
const (
|
||||
timeSyncRetryStep = 5 * time.Second
|
||||
timeSyncRetryMaxInt = 1 * time.Minute
|
||||
timeSyncWaitNetChkInt = 100 * time.Millisecond
|
||||
timeSyncWaitNetUpInt = 3 * time.Second
|
||||
timeSyncInterval = 1 * time.Hour
|
||||
timeSyncTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
timeSyncRetryInterval = 0 * time.Second
|
||||
)
|
||||
|
||||
type TimeSync struct {
|
||||
syncLock *sync.Mutex
|
||||
l *zerolog.Logger
|
||||
|
||||
ntpServers []string
|
||||
httpUrls []string
|
||||
networkConfig *network.NetworkConfig
|
||||
|
||||
rtcDevicePath string
|
||||
rtcDevice *os.File //nolint:unused
|
||||
rtcLock *sync.Mutex
|
||||
|
||||
syncSuccess bool
|
||||
|
||||
preCheckFunc func() (bool, error)
|
||||
}
|
||||
|
||||
type TimeSyncOptions struct {
|
||||
PreCheckFunc func() (bool, error)
|
||||
Logger *zerolog.Logger
|
||||
NetworkConfig *network.NetworkConfig
|
||||
}
|
||||
|
||||
type SyncMode struct {
|
||||
Ntp bool
|
||||
Http bool
|
||||
Ordering []string
|
||||
NtpUseFallback bool
|
||||
HttpUseFallback bool
|
||||
}
|
||||
|
||||
func NewTimeSync(opts *TimeSyncOptions) *TimeSync {
|
||||
rtcDevice, err := getRtcDevicePath()
|
||||
if err != nil {
|
||||
opts.Logger.Error().Err(err).Msg("failed to get RTC device path")
|
||||
} else {
|
||||
opts.Logger.Info().Str("path", rtcDevice).Msg("RTC device found")
|
||||
}
|
||||
|
||||
t := &TimeSync{
|
||||
syncLock: &sync.Mutex{},
|
||||
l: opts.Logger,
|
||||
rtcDevicePath: rtcDevice,
|
||||
rtcLock: &sync.Mutex{},
|
||||
preCheckFunc: opts.PreCheckFunc,
|
||||
ntpServers: defaultNTPServers,
|
||||
httpUrls: defaultHTTPUrls,
|
||||
networkConfig: opts.NetworkConfig,
|
||||
}
|
||||
|
||||
if t.rtcDevicePath != "" {
|
||||
rtcTime, _ := t.readRtcTime()
|
||||
t.l.Info().Interface("rtc_time", rtcTime).Msg("read RTC time")
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *TimeSync) getSyncMode() SyncMode {
|
||||
syncMode := SyncMode{
|
||||
NtpUseFallback: true,
|
||||
HttpUseFallback: true,
|
||||
}
|
||||
var syncModeString string
|
||||
|
||||
if t.networkConfig != nil {
|
||||
syncModeString = t.networkConfig.TimeSyncMode.String
|
||||
if t.networkConfig.TimeSyncDisableFallback.Bool {
|
||||
syncMode.NtpUseFallback = false
|
||||
syncMode.HttpUseFallback = false
|
||||
}
|
||||
}
|
||||
|
||||
switch syncModeString {
|
||||
case "ntp_only":
|
||||
syncMode.Ntp = true
|
||||
case "http_only":
|
||||
syncMode.Http = true
|
||||
default:
|
||||
syncMode.Ntp = true
|
||||
syncMode.Http = true
|
||||
}
|
||||
|
||||
return syncMode
|
||||
}
|
||||
|
||||
func (t *TimeSync) doTimeSync() {
|
||||
metricTimeSyncStatus.Set(0)
|
||||
for {
|
||||
if ok, err := t.preCheckFunc(); !ok {
|
||||
if err != nil {
|
||||
t.l.Error().Err(err).Msg("pre-check failed")
|
||||
}
|
||||
time.Sleep(timeSyncWaitNetChkInt)
|
||||
continue
|
||||
}
|
||||
|
||||
t.l.Info().Msg("syncing system time")
|
||||
start := time.Now()
|
||||
err := t.Sync()
|
||||
if err != nil {
|
||||
t.l.Error().Str("error", err.Error()).Msg("failed to sync system time")
|
||||
|
||||
// retry after a delay
|
||||
timeSyncRetryInterval += timeSyncRetryStep
|
||||
time.Sleep(timeSyncRetryInterval)
|
||||
// reset the retry interval if it exceeds the max interval
|
||||
if timeSyncRetryInterval > timeSyncRetryMaxInt {
|
||||
timeSyncRetryInterval = 0
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
t.syncSuccess = true
|
||||
t.l.Info().Str("now", time.Now().Format(time.RFC3339)).
|
||||
Str("time_taken", time.Since(start).String()).
|
||||
Msg("time sync successful")
|
||||
|
||||
metricTimeSyncStatus.Set(1)
|
||||
|
||||
time.Sleep(timeSyncInterval) // after the first sync is done
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TimeSync) Sync() error {
|
||||
var (
|
||||
now *time.Time
|
||||
offset *time.Duration
|
||||
)
|
||||
|
||||
syncMode := t.getSyncMode()
|
||||
|
||||
metricTimeSyncCount.Inc()
|
||||
|
||||
if syncMode.Ntp {
|
||||
now, offset = t.queryNetworkTime()
|
||||
}
|
||||
|
||||
if syncMode.Http && now == nil {
|
||||
now = t.queryAllHttpTime()
|
||||
}
|
||||
|
||||
if now == nil {
|
||||
return fmt.Errorf("failed to get time from any source")
|
||||
}
|
||||
|
||||
if offset != nil {
|
||||
newNow := time.Now().Add(*offset)
|
||||
now = &newNow
|
||||
}
|
||||
|
||||
err := t.setSystemTime(*now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set system time: %w", err)
|
||||
}
|
||||
|
||||
metricTimeSyncSuccessCount.Inc()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TimeSync) IsSyncSuccess() bool {
|
||||
return t.syncSuccess
|
||||
}
|
||||
|
||||
func (t *TimeSync) Start() {
|
||||
go t.doTimeSync()
|
||||
}
|
||||
|
||||
func (t *TimeSync) setSystemTime(now time.Time) error {
|
||||
nowStr := now.Format("2006-01-02 15:04:05")
|
||||
output, err := exec.Command("date", "-s", nowStr).CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to run date -s: %w, %s", err, string(output))
|
||||
}
|
||||
|
||||
if t.rtcDevicePath != "" {
|
||||
return t.setRtcTime(now)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user