remove IPVS metrics (#1133)

* remove IPVS metrics

Remove metrics for IPVS services when the IPVS service is deleted so
that the number of metrics does not grow without bound.

Fixes #734

* delete metricsMap key when IPVS service is removed

Delete the key in NetworkServicesController.metricsMap when the
respective IPVS configuration is removed.

Remove a period from a comment to conform to kube-router norms

* cleanup stale metrics in a distinct method

* remove unnecessary error return value on cleanupStaleMetrics

pkg/controllers/proxy/network_services_controller.go

@@ -218,6 +218,7 @@ type NetworkServicesController struct {
 	client              kubernetes.Interface
 	nodeportBindOnAllIP bool
 	MetricsEnabled      bool
+	metricsMap          map[string][]string
 	ln                  LinuxNetworking
 	readyForUpdates     bool
 	ProxyFirewallSetup  *sync.Cond
@@ -817,17 +818,30 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM
 			}
 
 			if pushMetric {
+
 				klog.V(3).Infof("Publishing metrics for %s/%s (%s:%d/%s)", svc.namespace, svc.name, svcVip, svc.port, svc.protocol)
-				metrics.ServiceBpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSIn))
-				metrics.ServiceBpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BPSOut))
-				metrics.ServiceBytesIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesIn))
-				metrics.ServiceBytesOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.BytesOut))
-				metrics.ServiceCPS.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.CPS))
-				metrics.ServicePacketsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PacketsIn))
-				metrics.ServicePacketsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PacketsOut))
-				metrics.ServicePpsIn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSIn))
-				metrics.ServicePpsOut.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.PPSOut))
-				metrics.ServiceTotalConn.WithLabelValues(svc.namespace, svc.name, svcVip, svc.protocol, strconv.Itoa(svc.port)).Set(float64(ipvsSvc.Stats.Connections))
+
+				labelValues := []string{
+					svc.namespace,
+					svc.name,
+					svcVip,
+					svc.protocol,
+					strconv.Itoa(svc.port),
+				}
+
+				key := generateIPPortID(svcVip, svc.protocol, strconv.Itoa(svc.port))
+				nsc.metricsMap[key] = labelValues
+				// these same metrics should be deleted when the service is deleted.
+				metrics.ServiceBpsIn.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.BPSIn))
+				metrics.ServiceBpsOut.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.BPSOut))
+				metrics.ServiceBytesIn.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.BytesIn))
+				metrics.ServiceBytesOut.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.BytesOut))
+				metrics.ServiceCPS.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.CPS))
+				metrics.ServicePacketsIn.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.PacketsIn))
+				metrics.ServicePacketsOut.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.PacketsOut))
+				metrics.ServicePpsIn.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.PPSIn))
+				metrics.ServicePpsOut.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.PPSOut))
+				metrics.ServiceTotalConn.WithLabelValues(labelValues...).Set(float64(ipvsSvc.Stats.Connections))
 				metrics.ControllerIpvsServices.Set(float64(len(ipvsSvcs)))
 			}
 		}
@@ -2491,10 +2505,10 @@ func NewNetworkServicesController(clientset kubernetes.Interface,
 		return nil, err
 	}
 
-	nsc := NetworkServicesController{ln: ln, ipsetMutex: ipsetMutex}
+	nsc := NetworkServicesController{ln: ln, ipsetMutex: ipsetMutex, metricsMap: make(map[string][]string)}
 
 	if config.MetricsEnabled {
-		//Register the metrics for this controller
+		// Register the metrics for this controller
 		prometheus.MustRegister(metrics.ControllerIpvsServices)
 		prometheus.MustRegister(metrics.ControllerIpvsServicesSyncTime)
 		prometheus.MustRegister(metrics.ServiceBpsIn)

pkg/controllers/proxy/service_endpoints_sync.go

@@ -63,6 +63,9 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
 		syncErrors = true
 		klog.Errorf("Error cleaning up stale IPVS services and servers: %s", err.Error())
 	}
+
+	nsc.cleanupStaleMetrics(activeServiceEndpointMap)
+
 	err = nsc.syncIpvsFirewall()
 	if err != nil {
 		syncErrors = true
@@ -575,3 +578,24 @@ func (nsc *NetworkServicesController) cleanupStaleIPVSConfig(activeServiceEndpoi
 	}
 	return nil
 }
+
+func (nsc *NetworkServicesController) cleanupStaleMetrics(activeServiceEndpointMap map[string][]string) {
+	for k, v := range nsc.metricsMap {
+		if _, ok := activeServiceEndpointMap[k]; ok {
+			continue
+		}
+
+		metrics.ServiceBpsIn.DeleteLabelValues(v...)
+		metrics.ServiceBpsOut.DeleteLabelValues(v...)
+		metrics.ServiceBytesIn.DeleteLabelValues(v...)
+		metrics.ServiceBytesOut.DeleteLabelValues(v...)
+		metrics.ServiceCPS.DeleteLabelValues(v...)
+		metrics.ServicePacketsIn.DeleteLabelValues(v...)
+		metrics.ServicePacketsOut.DeleteLabelValues(v...)
+		metrics.ServicePpsIn.DeleteLabelValues(v...)
+		metrics.ServicePpsOut.DeleteLabelValues(v...)
+		metrics.ServiceTotalConn.DeleteLabelValues(v...)
+		metrics.ControllerIpvsServices.Dec()
+		delete(nsc.metricsMap, k)
+	}
+}