localstore: improve metrics (#1427)

pkg/localstore/gc.go

@@ -79,12 +79,12 @@ func (db *DB) collectGarbageWorker() {
 // This function is called in collectGarbageWorker.
 func (db *DB) collectGarbage() (collectedCount uint64, done bool, err error) {
 	db.metrics.GCCounter.Inc()
-	defer totalTimeMetric(db.metrics.TotalTimeCollectGarbage, time.Now())
-	defer func() {
+	defer func(start time.Time) {
 		if err != nil {
 			db.metrics.GCErrorCounter.Inc()
 		}
-	}()
+		totalTimeMetric(db.metrics.TotalTimeCollectGarbage, start)
+	}(time.Now())
 
 	batch := new(leveldb.Batch)
 	target := db.gcTarget()
@@ -107,7 +107,13 @@ func (db *DB) collectGarbage() (collectedCount uint64, done bool, err error) {
 	db.metrics.GCSize.Set(float64(gcSize))
 
 	done = true
+	first := true
+	start := time.Now()
 	err = db.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) {
+		if first {
+			totalTimeMetric(db.metrics.TotalTimeGCFirstItem, start)
+			first = false
+		}
 		if gcSize-collectedCount <= target {
 			return true, nil
 		}
@@ -144,13 +150,12 @@ func (db *DB) collectGarbage() (collectedCount uint64, done bool, err error) {
 	if err != nil {
 		return 0, false, err
 	}
-	db.metrics.GCCollectedCounter.Inc()
-
-	db.metrics.GCSize.Set(float64(gcSize - collectedCount))
+	db.metrics.GCCollectedCounter.Add(float64(collectedCount))
 	db.gcSize.PutInBatch(batch, gcSize-collectedCount)
+
 	err = db.shed.WriteBatch(batch)
 	if err != nil {
-		db.metrics.GCExcludeWriteBatchError.Inc()
+		db.metrics.GCErrorCounter.Inc()
 		return 0, false, err
 	}
 	return collectedCount, done, nil
@@ -216,7 +221,7 @@ func (db *DB) removeChunksInExcludeIndexFromGC() (err error) {
 		return err
 	}
 
-	db.metrics.GCExcludeCounter.Inc()
+	db.metrics.GCExcludeCounter.Add(float64(excludedCount))
 	err = db.shed.WriteBatch(batch)
 	if err != nil {
 		db.metrics.GCExcludeWriteBatchError.Inc()

pkg/localstore/metrics.go

@@ -10,10 +10,7 @@ import (
 )
 
 type metrics struct {
-	// all metrics fields must be exported
-	// to be able to return them by Metrics()
-	// using reflection
-
+	TotalTimeGCFirstItem            prometheus.Counter
 	TotalTimeCollectGarbage         prometheus.Counter
 	TotalTimeGCExclude              prometheus.Counter
 	TotalTimeGet                    prometheus.Counter
@@ -29,10 +26,8 @@ type metrics struct {
 	GCCounter                prometheus.Counter
 	GCErrorCounter           prometheus.Counter
 	GCCollectedCounter       prometheus.Counter
-	GCWriteBatchError        prometheus.Counter
 	GCExcludeCounter         prometheus.Counter
 	GCExcludeError           prometheus.Counter
-	GCExcludedCounter        prometheus.Counter
 	GCExcludeWriteBatchError prometheus.Counter
 	GCUpdate                 prometheus.Counter
 	GCUpdateError            prometheus.Counter
@@ -68,6 +63,12 @@ func newMetrics() metrics {
 	subsystem := "localstore"
 
 	return metrics{
+		TotalTimeGCFirstItem: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: m.Namespace,
+			Subsystem: subsystem,
+			Name:      "gc_first_item_time",
+			Help:      "Total time taken till first item in gc comes out of gcIndex iterator.",
+		}),
 		TotalTimeCollectGarbage: prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: m.Namespace,
 			Subsystem: subsystem,
@@ -152,12 +153,6 @@ func newMetrics() metrics {
 			Name:      "gc_collected_count",
 			Help:      "Number of times the GC_COLLECTED operation is done.",
 		}),
-		GCWriteBatchError: prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: m.Namespace,
-			Subsystem: subsystem,
-			Name:      "gc_write_batch_error_count",
-			Help:      "Number of times the GC_WRITE_BATCH operation failed.",
-		}),
 		GCExcludeCounter: prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: m.Namespace,
 			Subsystem: subsystem,
@@ -170,12 +165,6 @@ func newMetrics() metrics {
 			Name:      "gc_exclude_fail_count",
 			Help:      "Number of times the GC_EXCLUDE operation failed.",
 		}),
-		GCExcludedCounter: prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: m.Namespace,
-			Subsystem: subsystem,
-			Name:      "gc_excluded_count",
-			Help:      "Number of times the GC_EXCLUDED operation is done.",
-		}),
 		GCExcludeWriteBatchError: prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: m.Namespace,
 			Subsystem: subsystem,
@@ -191,7 +180,7 @@ func newMetrics() metrics {
 		GCUpdateError: prometheus.NewCounter(prometheus.CounterOpts{
 			Namespace: m.Namespace,
 			Subsystem: subsystem,
-			Name:      "fc_update_error_count",
+			Name:      "gc_update_error_count",
 			Help:      "Number of times the gc update had error.",
 		}),