From 8e8b752524e1c1558a0858a121bb56ac1b78456c Mon Sep 17 00:00:00 2001 From: Nayana Bidari Date: Fri, 14 May 2021 17:20:13 -0700 Subject: Add stuck tasks and startup stuck tasks to weirdness metric Weirdness metric will replace the below two metrics: - watchdog/stuck_startup_detected - watchdog/stuck_tasks_detected PiperOrigin-RevId: 373895696 --- pkg/metric/metric.go | 8 ++++---- pkg/sentry/watchdog/watchdog.go | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index 528910d7c..fdeee3a5f 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -40,8 +40,8 @@ var ( createdSentryMetrics = false // WeirdnessMetric is a metric with fields created to track the number - // of weird occurrences such as time fallback, partial_result and - // vsyscall count. + // of weird occurrences such as time fallback, partial_result, vsyscall + // count, watchdog startup timeouts and stuck tasks. WeirdnessMetric *Uint64Metric // SuspiciousOperationsMetric is a metric with fields created to detect @@ -401,10 +401,10 @@ func CreateSentryMetrics() { createdSentryMetrics = true - WeirdnessMetric = MustCreateNewUint64Metric("/weirdness", true /* sync */, "Increment for weird occurrences of problems such as time fallback, partial result and vsyscalls invoked in the sandbox.", + WeirdnessMetric = MustCreateNewUint64Metric("/weirdness", true /* sync */, "Increment for weird occurrences of problems such as time fallback, partial result, vsyscalls invoked in the sandbox, watchdog startup timeouts and stuck tasks.", Field{ name: "weirdness_type", - allowedValues: []string{"time_fallback", "partial_result", "vsyscall_count"}, + allowedValues: []string{"time_fallback", "partial_result", "vsyscall_count", "watchdog_stuck_startup", "watchdog_stuck_tasks"}, }) SuspiciousOperationsMetric = MustCreateNewUint64Metric("/suspicious_operations", true /* sync */, "Increment for suspicious operations such as opening an executable file to write from a gofer.", diff --git a/pkg/sentry/watchdog/watchdog.go b/pkg/sentry/watchdog/watchdog.go index bd31464d0..dfe85f31d 100644 --- a/pkg/sentry/watchdog/watchdog.go +++ b/pkg/sentry/watchdog/watchdog.go @@ -243,6 +243,7 @@ func (w *Watchdog) waitForStart() { } stuckStartup.Increment() + metric.WeirdnessMetric.Increment("watchdog_stuck_startup") var buf bytes.Buffer buf.WriteString(fmt.Sprintf("Watchdog.Start() not called within %s", w.StartupTimeout)) @@ -316,6 +317,7 @@ func (w *Watchdog) runTurn() { // Task.UninterruptibleSleepStart/Finish. tc = &offender{lastUpdateTime: lastUpdateTime} stuckTasks.Increment() + metric.WeirdnessMetric.Increment("watchdog_stuck_tasks") newTaskFound = true } newOffenders[t] = tc -- cgit v1.2.3