feat: add simple retries for status requests

Author: pboros

db/conn.go

@@ -17,6 +17,7 @@ package db
 import (
 	"bufio"
 	"encoding/json"
+	"fmt"
 	"os"
 	"regexp"
 	"strconv"
@@ -100,7 +101,18 @@ func getFdb() fdb.Database {
 	return db
 }
 
-func GetStatus() (*models.FullStatus, error) {
+func IsStatusIncomplete(status *models.FullStatus) bool {
+	if status.Cluster == nil ||
+		status.Cluster.DatabaseLockState == nil ||
+		status.Cluster.FaultTolerance == nil ||
+		status.Cluster.Data == nil ||
+		status.Cluster.DatabaseAvailable == false {
+		return true
+	}
+	return false
+}
+
+func getStatusOnce() (*models.FullStatus, error) {
 	start := time.Now()
 	conn := getFdb()
 	var status models.FullStatus
@@ -122,11 +134,7 @@ func GetStatus() (*models.FullStatus, error) {
 	}
 
 	if os.Getenv("DEBUG_LOG_INCOMPLETE_STATUS") == "true" {
-		if status.Cluster == nil ||
-			status.Cluster.DatabaseLockState == nil ||
-			status.Cluster.FaultTolerance == nil ||
-			status.Cluster.Data == nil ||
-			status.Cluster.DatabaseAvailable == false {
+		if IsStatusIncomplete(&status) {
 			statusString := string(statusJson.([]byte))
 			log.Debug().Str("status_json", statusString).Msg("status json is missing cluster fields")
 
@@ -135,13 +143,25 @@ func GetStatus() (*models.FullStatus, error) {
 				Dur("took", d).
 				Str("status_json", statusString).
 				Msg("status json is missing cluster fields")
-
+			return nil, fmt.Errorf("incomplete status")
 		}
 	}
-
 	return &status, nil
 }
 
+func GetStatus() (*models.FullStatus, error) {
+	status, err := getStatusOnce()
+	if err != nil {
+		log.Error().Msg("failed to get status, retrying")
+		time.Sleep(1 * time.Second)
+		status, err = getStatusOnce()
+		if err != nil {
+			log.Error().Msg("failed to get status, retry failed")
+		}
+	}
+	return status, err
+}
+
 func isTLSmode(c *string) bool {
 	// Find if must run in TLS mode
 	file, err := os.Open(*c)

metrics/metric_provider.go

@@ -15,8 +15,10 @@
 package metrics
 
 import (
+	"context"
 	"net/http"
 	"os"
+	"sync"
 	"time"
 
 	ulog "github.com/tigrisdata/fdb-exporter/util/log"
@@ -56,24 +58,46 @@ func (m *MetricProvider) Close() {
 
 // Periodic data collection, called from main in a goroutine
 func (mp *MetricProvider) Collect() {
-	// TODO make this configurable
-	interval := 10 * time.Second
+	var reporterSwap sync.Mutex
+
+	interval := 3 * time.Second
 	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
 
 	if err := mp.reporter.collectOnce(); err != nil {
 		ulog.E(err, "failed to collect metrics")
 	}
+
 	for range ticker.C {
-		newReporter := NewMetricReporter()
-		if err := newReporter.collectOnce(); err != nil {
-			ulog.E(err, "failed to collect metrics in a tick")
-		}
-		time.Sleep(1 * time.Second) // Wait a bit before serving new tally's data (otherwise the first query will return 0)
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+
+		done := make(chan struct{})
+
+		go func() {
+			defer close(done)
 
-		oldReporter := mp.reporter
-		mp.reporter = newReporter
+			newReporter := NewMetricReporter()
+			if err := newReporter.collectOnce(); err != nil {
+				ulog.E(err, "failed to collect metrics in a tick")
+				return
+			}
 
-		oldReporter.Close()
+			time.Sleep(1 * time.Second)
+
+			reporterSwap.Lock()
+			oldReporter := mp.reporter
+			mp.reporter = newReporter
+			oldReporter.Close()
+			reporterSwap.Unlock()
+		}()
+
+		select {
+		case <-done:
+			// finished within timeout
+		case <-ctx.Done():
+			ulog.E(ctx.Err(), "metric collection tick timed out")
+		}
+
+		cancel()
 	}
-	defer ticker.Stop()
 }