feat: Add metric to export last duration job spent waiting in queue i…

…n otherwords waiting for a runner

cmd/ghalistener/metrics/metrics.go

@@ -46,6 +46,7 @@ var (
 	lastJobExecutionDurationLabels = append(jobLabels, labelKeyJobResult)
 	startedJobsTotalLabels         = jobLabels
 	lastJobStartupDurationLabels   = jobLabels
+	jobQueueDurationLabels         = jobLabels
 )
 
 var (
@@ -139,6 +140,15 @@ var (
 		completedJobsTotalLabels,
 	)
 
+	jobLastQueueDurationSeconds = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Subsystem: githubScaleSetSubsystem,
+			Name:      "job_last_queue_duration_seconds",
+			Help:      "Last duration spent in the queue by the job (in seconds).",
+		},
+		jobQueueDurationLabels,
+	)
+
 	jobLastStartupDurationSeconds = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Subsystem: githubScaleSetSubsystem,
@@ -253,6 +263,7 @@ func NewExporter(config ExporterConfig) ServerPublisher {
 		idleRunners,
 		startedJobsTotal,
 		completedJobsTotal,
+		jobLastQueueDurationSeconds,
 		jobLastStartupDurationSeconds,
 		jobLastExecutionDurationSeconds,
 	)
@@ -316,6 +327,11 @@ func (e *exporter) PublishJobStarted(msg *actions.JobStarted) {
 		startupDuration := msg.JobMessageBase.RunnerAssignTime.Unix() - msg.JobMessageBase.ScaleSetAssignTime.Unix()
 		jobLastStartupDurationSeconds.With(l).Set(float64(startupDuration))
 	}
+
+	if !msg.JobMessageBase.QueueTime.IsZero() && !msg.JobMessageBase.RunnerAssignTime.IsZero() {
+		queueDuration := msg.JobMessageBase.RunnerAssignTime.Unix() - msg.JobMessageBase.QueueTime.Unix()
+		jobLastQueueDurationSeconds.With(l).Set(float64(queueDuration))
+	}
 }
 
 func (e *exporter) PublishJobCompleted(msg *actions.JobCompleted) {