Add task run next/prev time metrics

Author: dispensable

metrics.go

@@ -7,6 +7,8 @@ var (
 	prometheusMetricTaskRunCount    *prometheus.CounterVec
 	prometheusMetricTaskRunResult   *prometheus.GaugeVec
 	prometheusMetricTaskRunTime     *prometheus.GaugeVec
+	prometheusMetricTaskRunPrevTs   *prometheus.GaugeVec
+	prometheusMetricTaskRunNextTs   *prometheus.GaugeVec
 	prometheusMetricTaskRunDuration *prometheus.GaugeVec
 )
 
@@ -55,6 +57,24 @@ func initMetrics() {
 		[]string{"cronSpec", "cronUser", "cronCommand"},
 	)
 	prometheus.MustRegister(prometheusMetricTaskRunDuration)
+
+	prometheusMetricTaskRunNextTs = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "gocrond_task_run_next_time",
+			Help: "gocrond task next run ts",
+		},
+		[]string{"cronSpec", "cronUser", "cronCommand"},
+	)
+	prometheus.MustRegister(prometheusMetricTaskRunNextTs)
+
+	prometheusMetricTaskRunPrevTs = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "gocrond_task_run_prev_time",
+			Help: "gocrond task prev run ts",
+		},
+		[]string{"cronSpec", "cronUser", "cronCommand"},
+	)
+	prometheus.MustRegister(prometheusMetricTaskRunPrevTs)
 }
 
 func resetMetrics() {
@@ -63,4 +83,6 @@ func resetMetrics() {
 	prometheusMetricTaskRunResult.Reset()
 	prometheusMetricTaskRunTime.Reset()
 	prometheusMetricTaskRunDuration.Reset()
+	prometheusMetricTaskRunNextTs.Reset()
+	prometheusMetricTaskRunPrevTs.Reset()
 }

parser.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/robfig/cron/v3"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -35,6 +36,7 @@ type CrontabEntry struct {
 	Env         []string
 	Shell       string
 	CrontabPath string
+	EntryId     cron.EntryID
 }
 
 type Parser struct {
@@ -65,6 +67,10 @@ func NewCronjobSystemParser(path string) (*Parser, error) {
 	return p, nil
 }
 
+func (e *CrontabEntry) SetEntryId(eid cron.EntryID) {
+	(*e).EntryId = eid
+}
+
 // Parse crontab
 func (p *Parser) Parse() []CrontabEntry {
 	entries := p.parseLines()

runner.go

@@ -14,7 +14,8 @@ import (
 )
 
 type Runner struct {
-	cron *cron.Cron
+	cron     *cron.Cron
+	cronjobs map[cron.EntryID]*CrontabEntry
 }
 
 func NewRunner() *Runner {
@@ -26,13 +27,14 @@ func NewRunner() *Runner {
 				),
 			),
 		),
+		cronjobs: map[cron.EntryID]*CrontabEntry{},
 	}
 	return r
 }
 
 // Add crontab entry
 func (r *Runner) Add(cronjob CrontabEntry) error {
-	_, err := r.cron.AddFunc(cronjob.Spec, r.cmdFunc(cronjob, func(execCmd *exec.Cmd) bool {
+	eid, err := r.cron.AddFunc(cronjob.Spec, r.cmdFunc(&cronjob, func(execCmd *exec.Cmd) bool {
 		// before exec callback
 		log.WithFields(LogCronjobToFields(cronjob)).Infof("executing")
 		return true
@@ -42,6 +44,8 @@ func (r *Runner) Add(cronjob CrontabEntry) error {
 		prometheusMetricTask.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
 		log.WithFields(LogCronjobToFields(cronjob)).Errorf("cronjob failed adding:%v", err)
 	} else {
+		cronjob.SetEntryId(eid)
+		r.cronjobs[eid] = &cronjob
 		prometheusMetricTask.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
 		log.WithFields(LogCronjobToFields(cronjob)).Infof("cronjob added")
 	}
@@ -51,7 +55,7 @@ func (r *Runner) Add(cronjob CrontabEntry) error {
 
 // Add crontab entry with user
 func (r *Runner) AddWithUser(cronjob CrontabEntry) error {
-	_, err := r.cron.AddFunc(cronjob.Spec, r.cmdFunc(cronjob, func(execCmd *exec.Cmd) bool {
+	eid, err := r.cron.AddFunc(cronjob.Spec, r.cmdFunc(&cronjob, func(execCmd *exec.Cmd) bool {
 		// before exec callback
 		log.WithFields(LogCronjobToFields(cronjob)).Debugf("executing")
 
@@ -86,6 +90,8 @@ func (r *Runner) AddWithUser(cronjob CrontabEntry) error {
 		prometheusMetricTask.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
 		log.WithFields(LogCronjobToFields(cronjob)).Errorf("cronjob failed adding: %v", err)
 	} else {
+		cronjob.SetEntryId(eid)
+		r.cronjobs[eid] = &cronjob
 		prometheusMetricTask.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
 		log.WithFields(LogCronjobToFields(cronjob)).Infof("cronjob added")
 	}
@@ -102,6 +108,7 @@ func (r *Runner) Len() int {
 func (r *Runner) Start() {
 	log.Infof("start runner with %d jobs\n", r.Len())
 	r.cron.Start()
+	r.initAllCronEntryMetrics()
 }
 
 // Stop runner
@@ -111,7 +118,7 @@ func (r *Runner) Stop() {
 }
 
 // Execute crontab command
-func (r *Runner) cmdFunc(cronjob CrontabEntry, cmdCallback func(*exec.Cmd) bool) func() {
+func (r *Runner) cmdFunc(cronjob *CrontabEntry, cmdCallback func(*exec.Cmd) bool) func() {
 	cmdFunc := func() {
 		// fall back to normal shell if not specified
 		taskShell := cronjob.Shell
@@ -137,25 +144,27 @@ func (r *Runner) cmdFunc(cronjob CrontabEntry, cmdCallback func(*exec.Cmd) bool)
 
 			elapsed := time.Since(start)
 
-			prometheusMetricTaskRunDuration.With(r.cronjobToPrometheusLabels(cronjob)).Set(elapsed.Seconds())
-			prometheusMetricTaskRunTime.With(r.cronjobToPrometheusLabels(cronjob)).SetToCurrentTime()
+			cronjobMetricCommonLables := r.cronjobToPrometheusLabels(*cronjob)
+			prometheusMetricTaskRunDuration.With(cronjobMetricCommonLables).Set(elapsed.Seconds())
+			prometheusMetricTaskRunTime.With(cronjobMetricCommonLables).SetToCurrentTime()
 
-			logFields := LogCronjobToFields(cronjob)
+			logFields := LogCronjobToFields(*cronjob)
 			logFields["elapsed_s"] = elapsed.Seconds()
 			if execCmd.ProcessState != nil {
 				logFields["exitCode"] = execCmd.ProcessState.ExitCode()
 			}
 
 			if err != nil {
-				prometheusMetricTaskRunCount.With(r.cronjobToPrometheusLabels(cronjob, prometheus.Labels{"result": "error"})).Inc()
-				prometheusMetricTaskRunResult.With(r.cronjobToPrometheusLabels(cronjob)).Set(0)
+				prometheusMetricTaskRunCount.With(r.cronjobToPrometheusLabels(*cronjob, prometheus.Labels{"result": "error"})).Inc()
+				prometheusMetricTaskRunResult.With(cronjobMetricCommonLables).Set(0)
 				logFields["result"] = "error"
 			} else {
-				prometheusMetricTaskRunCount.With(r.cronjobToPrometheusLabels(cronjob, prometheus.Labels{"result": "success"})).Inc()
-				prometheusMetricTaskRunResult.With(r.cronjobToPrometheusLabels(cronjob)).Set(1)
+				prometheusMetricTaskRunCount.With(r.cronjobToPrometheusLabels(*cronjob, prometheus.Labels{"result": "success"})).Inc()
+				prometheusMetricTaskRunResult.With(cronjobMetricCommonLables).Set(1)
 				logFields["result"] = "success"
 			}
 
+			r.updateCronEntryMetrics(cronjob)
 			log.WithFields(logFields).Info("finished")
 			if len(cmdStdout) > 0 {
 				log.Debugln(string(cmdStdout))
@@ -178,3 +187,26 @@ func (r *Runner) cronjobToPrometheusLabels(cronjob CrontabEntry, additionalLabel
 	}
 	return
 }
+
+func (r *Runner) updateCronEntryMetrics(cronjob *CrontabEntry) {
+	cronjobMetricCommonLables := r.cronjobToPrometheusLabels(*cronjob)
+	entry := r.cron.Entry(cronjob.EntryId)
+
+	if entry.Next.IsZero() {
+		prometheusMetricTaskRunNextTs.With(cronjobMetricCommonLables).Set(0)
+	} else {
+		prometheusMetricTaskRunNextTs.With(cronjobMetricCommonLables).Set(float64(entry.Next.Unix()))
+	}
+
+	if entry.Prev.IsZero() {
+		prometheusMetricTaskRunPrevTs.With(cronjobMetricCommonLables).Set(0)
+	} else {
+		prometheusMetricTaskRunPrevTs.With(cronjobMetricCommonLables).Set(float64(entry.Prev.Unix()))
+	}
+}
+
+func (r *Runner) initAllCronEntryMetrics() {
+	for _, cronjob := range r.cronjobs {
+		r.updateCronEntryMetrics(cronjob)
+	}
+}