network enhanecment / refactor (#361)

* chore(network): improve connectivity check

* refactor(network): rewrite network and timesync component

* feat(display): show cloud connection status

* chore: change logging verbosity

* chore(websecure): update log message

* fix(ota): validate root certificate when downloading update

* feat(ui): add network settings tab

* fix(display): cloud connecting animation

* fix: golintci issues

* feat: add network settings tab

* feat(timesync): query servers in parallel

* refactor(network): move to internal/network package

* feat(timesync): add metrics

* refactor(log): move log to internal/logging package

* refactor(mdms): move mdns to internal/mdns package

* feat(developer): add pprof endpoint

* feat(logging): add a simple logging streaming endpoint

* fix(mdns): do not start mdns until network is up

* feat(network): allow users to update network settings from ui

* fix(network): handle errors when net.IPAddr is nil

* fix(mdns): scopedLogger SIGSEGV

* fix(dhcp): watch directory instead of file to catch fsnotify.Create event

* refactor(nbd): move platform-specific code to different files

* refactor(native): move platform-specific code to different files

* chore: fix linter issues

* chore(dev_deploy): allow to override PION_LOG_TRACE
This commit is contained in:
Aveline
2025-04-16 01:39:23 +02:00
committed by GitHub
parent 2b2a14204d
commit 189b84380b
71 changed files with 4938 additions and 825 deletions

132
internal/timesync/http.go Normal file
View File

@@ -0,0 +1,132 @@
package timesync
import (
"context"
"errors"
"math/rand"
"net/http"
"strconv"
"time"
)
var defaultHTTPUrls = []string{
"http://www.gstatic.com/generate_204",
"http://cp.cloudflare.com/",
"http://edge-http.microsoft.com/captiveportal/generate_204",
// Firefox, Apple, and Microsoft have inconsistent results, so we don't use it
// "http://detectportal.firefox.com/",
// "http://www.apple.com/library/test/success.html",
// "http://www.msftconnecttest.com/connecttest.txt",
}
func (t *TimeSync) queryAllHttpTime() (now *time.Time) {
chunkSize := 4
httpUrls := t.httpUrls
// shuffle the http urls to avoid always querying the same servers
rand.Shuffle(len(httpUrls), func(i, j int) { httpUrls[i], httpUrls[j] = httpUrls[j], httpUrls[i] })
for i := 0; i < len(httpUrls); i += chunkSize {
chunk := httpUrls[i:min(i+chunkSize, len(httpUrls))]
results := t.queryMultipleHttp(chunk, timeSyncTimeout)
if results != nil {
return results
}
}
return nil
}
func (t *TimeSync) queryMultipleHttp(urls []string, timeout time.Duration) (now *time.Time) {
results := make(chan *time.Time, len(urls))
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
for _, url := range urls {
go func(url string) {
scopedLogger := t.l.With().
Str("http_url", url).
Logger()
metricHttpRequestCount.WithLabelValues(url).Inc()
metricHttpTotalRequestCount.Inc()
startTime := time.Now()
now, response, err := queryHttpTime(
ctx,
url,
timeout,
)
duration := time.Since(startTime)
metricHttpServerLastRTT.WithLabelValues(url).Set(float64(duration.Milliseconds()))
metricHttpServerRttHistogram.WithLabelValues(url).Observe(float64(duration.Milliseconds()))
status := 0
if response != nil {
status = response.StatusCode
}
metricHttpServerInfo.WithLabelValues(
url,
strconv.Itoa(status),
).Set(1)
if err == nil {
metricHttpTotalSuccessCount.Inc()
metricHttpSuccessCount.WithLabelValues(url).Inc()
requestId := response.Header.Get("X-Request-Id")
if requestId != "" {
requestId = response.Header.Get("X-Msedge-Ref")
}
if requestId == "" {
requestId = response.Header.Get("Cf-Ray")
}
scopedLogger.Info().
Str("time", now.Format(time.RFC3339)).
Int("status", status).
Str("request_id", requestId).
Str("time_taken", duration.String()).
Msg("HTTP server returned time")
cancel()
results <- now
} else if errors.Is(err, context.Canceled) {
metricHttpCancelCount.WithLabelValues(url).Inc()
metricHttpTotalCancelCount.Inc()
} else {
scopedLogger.Warn().
Str("error", err.Error()).
Int("status", status).
Msg("failed to query HTTP server")
}
}(url)
}
return <-results
}
func queryHttpTime(
ctx context.Context,
url string,
timeout time.Duration,
) (now *time.Time, response *http.Response, err error) {
client := http.Client{
Timeout: timeout,
}
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, nil, err
}
dateStr := resp.Header.Get("Date")
parsedTime, err := time.Parse(time.RFC1123, dateStr)
if err != nil {
return nil, nil, err
}
return &parsedTime, resp, nil
}

View File

@@ -0,0 +1,147 @@
package timesync
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
metricTimeSyncStatus = promauto.NewGauge(
prometheus.GaugeOpts{
Name: "jetkvm_timesync_status",
Help: "The status of the timesync, 1 if successful, 0 if not",
},
)
metricTimeSyncCount = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_timesync_count",
Help: "The number of times the timesync has been run",
},
)
metricTimeSyncSuccessCount = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_timesync_success_count",
Help: "The number of times the timesync has been successful",
},
)
metricRTCUpdateCount = promauto.NewCounter( //nolint:unused
prometheus.CounterOpts{
Name: "jetkvm_timesync_rtc_update_count",
Help: "The number of times the RTC has been updated",
},
)
metricNtpTotalSuccessCount = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_timesync_ntp_total_success_count",
Help: "The total number of successful NTP requests",
},
)
metricNtpTotalRequestCount = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_timesync_ntp_total_request_count",
Help: "The total number of NTP requests sent",
},
)
metricNtpSuccessCount = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "jetkvm_timesync_ntp_success_count",
Help: "The number of successful NTP requests",
},
[]string{"url"},
)
metricNtpRequestCount = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "jetkvm_timesync_ntp_request_count",
Help: "The number of NTP requests sent to the server",
},
[]string{"url"},
)
metricNtpServerLastRTT = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "jetkvm_timesync_ntp_server_last_rtt",
Help: "The last RTT of the NTP server in milliseconds",
},
[]string{"url"},
)
metricNtpServerRttHistogram = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "jetkvm_timesync_ntp_server_rtt",
Help: "The histogram of the RTT of the NTP server in milliseconds",
Buckets: []float64{
10, 25, 50, 100, 200, 300, 500, 1000,
},
},
[]string{"url"},
)
metricNtpServerInfo = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "jetkvm_timesync_ntp_server_info",
Help: "The info of the NTP server",
},
[]string{"url", "reference", "stratum", "precision"},
)
metricHttpTotalSuccessCount = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_timesync_http_total_success_count",
Help: "The total number of successful HTTP requests",
},
)
metricHttpTotalRequestCount = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_timesync_http_total_request_count",
Help: "The total number of HTTP requests sent",
},
)
metricHttpTotalCancelCount = promauto.NewCounter(
prometheus.CounterOpts{
Name: "jetkvm_timesync_http_total_cancel_count",
Help: "The total number of HTTP requests cancelled",
},
)
metricHttpSuccessCount = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "jetkvm_timesync_http_success_count",
Help: "The number of successful HTTP requests",
},
[]string{"url"},
)
metricHttpRequestCount = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "jetkvm_timesync_http_request_count",
Help: "The number of HTTP requests sent to the server",
},
[]string{"url"},
)
metricHttpCancelCount = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "jetkvm_timesync_http_cancel_count",
Help: "The number of HTTP requests cancelled",
},
[]string{"url"},
)
metricHttpServerLastRTT = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "jetkvm_timesync_http_server_last_rtt",
Help: "The last RTT of the HTTP server in milliseconds",
},
[]string{"url"},
)
metricHttpServerRttHistogram = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "jetkvm_timesync_http_server_rtt",
Help: "The histogram of the RTT of the HTTP server in milliseconds",
Buckets: []float64{
10, 25, 50, 100, 200, 300, 500, 1000,
},
},
[]string{"url"},
)
metricHttpServerInfo = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "jetkvm_timesync_http_server_info",
Help: "The info of the HTTP server",
},
[]string{"url", "http_code"},
)
)

113
internal/timesync/ntp.go Normal file
View File

@@ -0,0 +1,113 @@
package timesync
import (
"math/rand/v2"
"strconv"
"time"
"github.com/beevik/ntp"
)
var defaultNTPServers = []string{
"time.apple.com",
"time.aws.com",
"time.windows.com",
"time.google.com",
"162.159.200.123", // time.cloudflare.com
"0.pool.ntp.org",
"1.pool.ntp.org",
"2.pool.ntp.org",
"3.pool.ntp.org",
}
func (t *TimeSync) queryNetworkTime() (now *time.Time, offset *time.Duration) {
chunkSize := 4
ntpServers := t.ntpServers
// shuffle the ntp servers to avoid always querying the same servers
rand.Shuffle(len(ntpServers), func(i, j int) { ntpServers[i], ntpServers[j] = ntpServers[j], ntpServers[i] })
for i := 0; i < len(ntpServers); i += chunkSize {
chunk := ntpServers[i:min(i+chunkSize, len(ntpServers))]
now, offset := t.queryMultipleNTP(chunk, timeSyncTimeout)
if now != nil {
return now, offset
}
}
return nil, nil
}
type ntpResult struct {
now *time.Time
offset *time.Duration
}
func (t *TimeSync) queryMultipleNTP(servers []string, timeout time.Duration) (now *time.Time, offset *time.Duration) {
results := make(chan *ntpResult, len(servers))
for _, server := range servers {
go func(server string) {
scopedLogger := t.l.With().
Str("server", server).
Logger()
// increase request count
metricNtpTotalRequestCount.Inc()
metricNtpRequestCount.WithLabelValues(server).Inc()
// query the server
now, response, err := queryNtpServer(server, timeout)
// set the last RTT
metricNtpServerLastRTT.WithLabelValues(
server,
).Set(float64(response.RTT.Milliseconds()))
// set the RTT histogram
metricNtpServerRttHistogram.WithLabelValues(
server,
).Observe(float64(response.RTT.Milliseconds()))
// set the server info
metricNtpServerInfo.WithLabelValues(
server,
response.ReferenceString(),
strconv.Itoa(int(response.Stratum)),
strconv.Itoa(int(response.Precision)),
).Set(1)
if err == nil {
// increase success count
metricNtpTotalSuccessCount.Inc()
metricNtpSuccessCount.WithLabelValues(server).Inc()
scopedLogger.Info().
Str("time", now.Format(time.RFC3339)).
Str("reference", response.ReferenceString()).
Str("rtt", response.RTT.String()).
Str("clockOffset", response.ClockOffset.String()).
Uint8("stratum", response.Stratum).
Msg("NTP server returned time")
results <- &ntpResult{
now: now,
offset: &response.ClockOffset,
}
} else {
scopedLogger.Warn().
Str("error", err.Error()).
Msg("failed to query NTP server")
}
}(server)
}
result := <-results
return result.now, result.offset
}
func queryNtpServer(server string, timeout time.Duration) (now *time.Time, response *ntp.Response, err error) {
resp, err := ntp.QueryWithOptions(server, ntp.QueryOptions{Timeout: timeout})
if err != nil {
return nil, nil, err
}
return &resp.Time, resp, nil
}

26
internal/timesync/rtc.go Normal file
View File

@@ -0,0 +1,26 @@
package timesync
import (
"fmt"
"os"
)
var (
rtcDeviceSearchPaths = []string{
"/dev/rtc",
"/dev/rtc0",
"/dev/rtc1",
"/dev/misc/rtc",
"/dev/misc/rtc0",
"/dev/misc/rtc1",
}
)
func getRtcDevicePath() (string, error) {
for _, path := range rtcDeviceSearchPaths {
if _, err := os.Stat(path); err == nil {
return path, nil
}
}
return "", fmt.Errorf("rtc device not found")
}

View File

@@ -0,0 +1,105 @@
//go:build linux
package timesync
import (
"fmt"
"os"
"time"
"golang.org/x/sys/unix"
)
func TimetoRtcTime(t time.Time) unix.RTCTime {
return unix.RTCTime{
Sec: int32(t.Second()),
Min: int32(t.Minute()),
Hour: int32(t.Hour()),
Mday: int32(t.Day()),
Mon: int32(t.Month() - 1),
Year: int32(t.Year() - 1900),
Wday: int32(0),
Yday: int32(0),
Isdst: int32(0),
}
}
func RtcTimetoTime(t unix.RTCTime) time.Time {
return time.Date(
int(t.Year)+1900,
time.Month(t.Mon+1),
int(t.Mday),
int(t.Hour),
int(t.Min),
int(t.Sec),
0,
time.UTC,
)
}
func (t *TimeSync) getRtcDevice() (*os.File, error) {
if t.rtcDevice == nil {
file, err := os.OpenFile(t.rtcDevicePath, os.O_RDWR, 0666)
if err != nil {
return nil, err
}
t.rtcDevice = file
}
return t.rtcDevice, nil
}
func (t *TimeSync) getRtcDeviceFd() (int, error) {
device, err := t.getRtcDevice()
if err != nil {
return 0, err
}
return int(device.Fd()), nil
}
// Read implements Read for the Linux RTC
func (t *TimeSync) readRtcTime() (time.Time, error) {
fd, err := t.getRtcDeviceFd()
if err != nil {
return time.Time{}, fmt.Errorf("failed to get RTC device fd: %w", err)
}
rtcTime, err := unix.IoctlGetRTCTime(fd)
if err != nil {
return time.Time{}, fmt.Errorf("failed to get RTC time: %w", err)
}
date := RtcTimetoTime(*rtcTime)
return date, nil
}
// Set implements Set for the Linux RTC
// ...
// It might be not accurate as the time consumed by the system call is not taken into account
// but it's good enough for our purposes
func (t *TimeSync) setRtcTime(tu time.Time) error {
rt := TimetoRtcTime(tu)
fd, err := t.getRtcDeviceFd()
if err != nil {
return fmt.Errorf("failed to get RTC device fd: %w", err)
}
currentRtcTime, err := t.readRtcTime()
if err != nil {
return fmt.Errorf("failed to read RTC time: %w", err)
}
t.l.Info().
Interface("rtc_time", tu).
Str("offset", tu.Sub(currentRtcTime).String()).
Msg("set rtc time")
if err := unix.IoctlSetRTCTime(fd, &rt); err != nil {
return fmt.Errorf("failed to set RTC time: %w", err)
}
metricRTCUpdateCount.Inc()
return nil
}

View File

@@ -0,0 +1,16 @@
//go:build !linux
package timesync
import (
"errors"
"time"
)
func (t *TimeSync) readRtcTime() (time.Time, error) {
return time.Now(), nil
}
func (t *TimeSync) setRtcTime(tu time.Time) error {
return errors.New("not supported")
}

View File

@@ -0,0 +1,208 @@
package timesync
import (
"fmt"
"os"
"os/exec"
"sync"
"time"
"github.com/jetkvm/kvm/internal/network"
"github.com/rs/zerolog"
)
const (
timeSyncRetryStep = 5 * time.Second
timeSyncRetryMaxInt = 1 * time.Minute
timeSyncWaitNetChkInt = 100 * time.Millisecond
timeSyncWaitNetUpInt = 3 * time.Second
timeSyncInterval = 1 * time.Hour
timeSyncTimeout = 2 * time.Second
)
var (
timeSyncRetryInterval = 0 * time.Second
)
type TimeSync struct {
syncLock *sync.Mutex
l *zerolog.Logger
ntpServers []string
httpUrls []string
networkConfig *network.NetworkConfig
rtcDevicePath string
rtcDevice *os.File //nolint:unused
rtcLock *sync.Mutex
syncSuccess bool
preCheckFunc func() (bool, error)
}
type TimeSyncOptions struct {
PreCheckFunc func() (bool, error)
Logger *zerolog.Logger
NetworkConfig *network.NetworkConfig
}
type SyncMode struct {
Ntp bool
Http bool
Ordering []string
NtpUseFallback bool
HttpUseFallback bool
}
func NewTimeSync(opts *TimeSyncOptions) *TimeSync {
rtcDevice, err := getRtcDevicePath()
if err != nil {
opts.Logger.Error().Err(err).Msg("failed to get RTC device path")
} else {
opts.Logger.Info().Str("path", rtcDevice).Msg("RTC device found")
}
t := &TimeSync{
syncLock: &sync.Mutex{},
l: opts.Logger,
rtcDevicePath: rtcDevice,
rtcLock: &sync.Mutex{},
preCheckFunc: opts.PreCheckFunc,
ntpServers: defaultNTPServers,
httpUrls: defaultHTTPUrls,
networkConfig: opts.NetworkConfig,
}
if t.rtcDevicePath != "" {
rtcTime, _ := t.readRtcTime()
t.l.Info().Interface("rtc_time", rtcTime).Msg("read RTC time")
}
return t
}
func (t *TimeSync) getSyncMode() SyncMode {
syncMode := SyncMode{
NtpUseFallback: true,
HttpUseFallback: true,
}
var syncModeString string
if t.networkConfig != nil {
syncModeString = t.networkConfig.TimeSyncMode.String
if t.networkConfig.TimeSyncDisableFallback.Bool {
syncMode.NtpUseFallback = false
syncMode.HttpUseFallback = false
}
}
switch syncModeString {
case "ntp_only":
syncMode.Ntp = true
case "http_only":
syncMode.Http = true
default:
syncMode.Ntp = true
syncMode.Http = true
}
return syncMode
}
func (t *TimeSync) doTimeSync() {
metricTimeSyncStatus.Set(0)
for {
if ok, err := t.preCheckFunc(); !ok {
if err != nil {
t.l.Error().Err(err).Msg("pre-check failed")
}
time.Sleep(timeSyncWaitNetChkInt)
continue
}
t.l.Info().Msg("syncing system time")
start := time.Now()
err := t.Sync()
if err != nil {
t.l.Error().Str("error", err.Error()).Msg("failed to sync system time")
// retry after a delay
timeSyncRetryInterval += timeSyncRetryStep
time.Sleep(timeSyncRetryInterval)
// reset the retry interval if it exceeds the max interval
if timeSyncRetryInterval > timeSyncRetryMaxInt {
timeSyncRetryInterval = 0
}
continue
}
t.syncSuccess = true
t.l.Info().Str("now", time.Now().Format(time.RFC3339)).
Str("time_taken", time.Since(start).String()).
Msg("time sync successful")
metricTimeSyncStatus.Set(1)
time.Sleep(timeSyncInterval) // after the first sync is done
}
}
func (t *TimeSync) Sync() error {
var (
now *time.Time
offset *time.Duration
)
syncMode := t.getSyncMode()
metricTimeSyncCount.Inc()
if syncMode.Ntp {
now, offset = t.queryNetworkTime()
}
if syncMode.Http && now == nil {
now = t.queryAllHttpTime()
}
if now == nil {
return fmt.Errorf("failed to get time from any source")
}
if offset != nil {
newNow := time.Now().Add(*offset)
now = &newNow
}
err := t.setSystemTime(*now)
if err != nil {
return fmt.Errorf("failed to set system time: %w", err)
}
metricTimeSyncSuccessCount.Inc()
return nil
}
func (t *TimeSync) IsSyncSuccess() bool {
return t.syncSuccess
}
func (t *TimeSync) Start() {
go t.doTimeSync()
}
func (t *TimeSync) setSystemTime(now time.Time) error {
nowStr := now.Format("2006-01-02 15:04:05")
output, err := exec.Command("date", "-s", nowStr).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to run date -s: %w, %s", err, string(output))
}
if t.rtcDevicePath != "" {
return t.setRtcTime(now)
}
return nil
}