Browse code

show warning when pod's containers are restarting

deads2k authored on 2015/11/12 01:30:28
Showing 5 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,78 @@
0
+apiVersion: v1
1
+items:
2
+- apiVersion: v1
3
+  kind: Pod
4
+  metadata:
5
+    annotations:
6
+      kubernetes.io/created-by: '{"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"ReplicationController","namespace":"example","name":"frontend-app-1","uid":"666e12d7-2996-11e5-b9e2-28d2447dc82b","apiVersion":"v1","resourceVersion":"32416"}}'
7
+      openshift.io/scc: restricted
8
+    creationTimestamp: 2015-07-13T19:36:04Z
9
+    generateName: frontend-app-1-
10
+    labels:
11
+      deconflict: frontend.app
12
+      name: frontend
13
+      template: ruby-helloworld-sample
14
+    name: frontend-app-1-bjwh8
15
+    namespace: example
16
+    resourceVersion: "32467"
17
+    selfLink: /api/v1/namespaces/example/pods/frontend-app-1-bjwh8
18
+    uid: 669a3b5d-2996-11e5-b9e2-28d2447dc82b
19
+  spec:
20
+    containers:
21
+    - env:
22
+      - name: ADMIN_USERNAME
23
+        value: admin6TM
24
+      - name: ADMIN_PASSWORD
25
+        value: xImx1tHR
26
+      - name: MYSQL_ROOT_PASSWORD
27
+        value: rQHfVnTo
28
+      - name: MYSQL_DATABASE
29
+        value: root
30
+      image: openshift/ruby-hello-world
31
+      imagePullPolicy: IfNotPresent
32
+      name: ruby-helloworld
33
+      ports:
34
+      - containerPort: 8080
35
+        protocol: TCP
36
+      resources: {}
37
+      securityContext:
38
+        capabilities: {}
39
+        privileged: false
40
+        runAsUser: 1000060000
41
+        seLinuxOptions:
42
+          level: s0:c8,c2
43
+      terminationMessagePath: /dev/termination-log
44
+      volumeMounts:
45
+      - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
46
+        name: default-token-mmv27
47
+        readOnly: true
48
+    dnsPolicy: ClusterFirst
49
+    host: deads-dev-01
50
+    imagePullSecrets:
51
+    - name: default-dockercfg-yr2e3
52
+    nodeName: deads-dev-01
53
+    restartPolicy: Always
54
+    serviceAccount: default
55
+    serviceAccountName: default
56
+    volumes:
57
+    - name: default-token-mmv27
58
+      secret:
59
+        secretName: default-token-mmv27
60
+  status:
61
+    conditions:
62
+    - status: "False"
63
+      type: Ready
64
+    containerStatuses:
65
+    - image: openshift/ruby-hello-world
66
+      imageID: ""
67
+      lastState: {}
68
+      name: ruby-helloworld
69
+      ready: false
70
+      restartCount: 8
71
+      state:
72
+        waiting:
73
+          reason: 'Image: openshift/ruby-hello-world is not ready on the node'
74
+    phase: Pending
75
+    startTime: 2015-07-13T19:36:05Z
76
+kind: List
77
+metadata: {}
0 78
new file mode 100644
... ...
@@ -0,0 +1,44 @@
0
+package analysis
1
+
2
+import (
3
+	"fmt"
4
+
5
+	kapi "k8s.io/kubernetes/pkg/api"
6
+
7
+	osgraph "github.com/openshift/origin/pkg/api/graph"
8
+	kubegraph "github.com/openshift/origin/pkg/api/kubegraph/nodes"
9
+)
10
+
11
+const (
12
+	RestartingPodWarning = "RestartingPod"
13
+
14
+	RestartThreshold = 3
15
+)
16
+
17
+// FindRestartingPods inspects all Pods to see if they've restarted more than the threshold
18
+func FindRestartingPods(g osgraph.Graph) []osgraph.Marker {
19
+	markers := []osgraph.Marker{}
20
+
21
+	for _, uncastPodNode := range g.NodesByKind(kubegraph.PodNodeKind) {
22
+		podNode := uncastPodNode.(*kubegraph.PodNode)
23
+		pod, ok := podNode.Object().(*kapi.Pod)
24
+		if !ok {
25
+			continue
26
+		}
27
+
28
+		for _, containerStatus := range pod.Status.ContainerStatuses {
29
+			if containerStatus.RestartCount >= RestartThreshold {
30
+				markers = append(markers, osgraph.Marker{
31
+					Node: podNode,
32
+
33
+					Severity: osgraph.WarningSeverity,
34
+					Key:      RestartingPodWarning,
35
+					Message: fmt.Sprintf("container %q in %s has restarted %d times", containerStatus.Name,
36
+						podNode.ResourceString(), containerStatus.RestartCount),
37
+				})
38
+			}
39
+		}
40
+	}
41
+
42
+	return markers
43
+}
0 44
new file mode 100644
... ...
@@ -0,0 +1,22 @@
0
+package analysis
1
+
2
+import (
3
+	"testing"
4
+
5
+	osgraphtest "github.com/openshift/origin/pkg/api/graph/test"
6
+)
7
+
8
+func TestRestartingPodWarning(t *testing.T) {
9
+	g, _, err := osgraphtest.BuildGraph("../../../api/graph/test/restarting-pod.yaml")
10
+	if err != nil {
11
+		t.Fatalf("unexpected error: %v", err)
12
+	}
13
+
14
+	markers := FindRestartingPods(g)
15
+	if e, a := 1, len(markers); e != a {
16
+		t.Fatalf("expected %v, got %v", e, a)
17
+	}
18
+	if e, a := RestartingPodWarning, markers[0].Key; e != a {
19
+		t.Fatalf("expected %v, got %v", e, a)
20
+	}
21
+}
... ...
@@ -251,6 +251,7 @@ func createForbiddenMarkers(forbiddenResources sets.String) []osgraph.Marker {
251 251
 
252 252
 func getMarkerScanners() []osgraph.MarkerScanner {
253 253
 	return []osgraph.MarkerScanner{
254
+		kubeanalysis.FindRestartingPods,
254 255
 		kubeanalysis.FindDuelingReplicationControllers,
255 256
 		kubeanalysis.FindUnmountableSecrets,
256 257
 		kubeanalysis.FindMissingSecrets,
... ...
@@ -246,6 +246,18 @@ func TestProjectStatus(t *testing.T) {
246 246
 			},
247 247
 			Time: mustParseTime("2015-04-07T04:12:25Z"),
248 248
 		},
249
+		"restarting pod": {
250
+			Path: "../../../api/graph/test/restarting-pod.yaml",
251
+			Extra: []runtime.Object{
252
+				&projectapi.Project{
253
+					ObjectMeta: kapi.ObjectMeta{Name: "example", Namespace: ""},
254
+				},
255
+			},
256
+			ErrFn: func(err error) bool { return err == nil },
257
+			Contains: []string{
258
+				`container "ruby-helloworld" in pod/frontend-app-1-bjwh8 has restarted 8 times`,
259
+			},
260
+		},
249 261
 	}
250 262
 	oldTimeFn := timeNowFn
251 263
 	defer func() { timeNowFn = oldTimeFn }()