@@ -4,6 +4,7 @@

 /.project

 /.vagrant

 /.vscode

+/.settings

 /cpu.pprof

 /assets/app/config.local.js

 /assets/nbproject

@@ -53,8 +53,19 @@ diagnostics to run which regular users cannot.

 are skipped.

 .PP

 Diagnostics may be individually run by passing diagnostic name as arguments.

+

+.PP

+.RS

+

+.nf

+oadm diagnostics <DiagnosticName>

+

+.fi

+.RE

+

+.PP

 The available diagnostic names are:

-AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

+AggregatedLogging AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

 .SH OPTIONS

@@ -53,8 +53,19 @@ diagnostics to run which regular users cannot.

 are skipped.

 .PP

 Diagnostics may be individually run by passing diagnostic name as arguments.

+

+.PP

+.RS

+

+.nf

+oc adm diagnostics <DiagnosticName>

+

+.fi

+.RE

+

+.PP

 The available diagnostic names are:

-AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

+AggregatedLogging AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

 .SH OPTIONS

@@ -53,8 +53,19 @@ diagnostics to run which regular users cannot.

 are skipped.

 .PP

 Diagnostics may be individually run by passing diagnostic name as arguments.

+

+.PP

+.RS

+

+.nf

+openshift admin diagnostics <DiagnosticName>

+

+.fi

+.RE

+

+.PP

 The available diagnostic names are:

-AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

+AggregatedLogging AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

 .SH OPTIONS

@@ -53,8 +53,19 @@ diagnostics to run which regular users cannot.

 are skipped.

 .PP

 Diagnostics may be individually run by passing diagnostic name as arguments.

+

+.PP

+.RS

+

+.nf

+openshift cli adm diagnostics <DiagnosticName>

+

+.fi

+.RE

+

+.PP

 The available diagnostic names are:

-AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

+AggregatedLogging AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

 .SH OPTIONS

@@ -53,8 +53,19 @@ diagnostics to run which regular users cannot.

 are skipped.

 .PP

 Diagnostics may be individually run by passing diagnostic name as arguments.

+

+.PP

+.RS

+

+.nf

+openshift ex diagnostics <DiagnosticName>

+

+.fi

+.RE

+

+.PP

 The available diagnostic names are:

-AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

+AggregatedLogging AnalyzeLogs ClusterRegistry ClusterRoleBindings ClusterRoles ClusterRouter ConfigContexts DiagnosticPod MasterConfigCheck MasterNode MetricsApiProxy NodeConfigCheck NodeDefinitions ServiceExternalIPs UnitStatus

 .SH OPTIONS

@@ -14,13 +14,24 @@ import (

 	"github.com/openshift/origin/pkg/client"

 	osclientcmd "github.com/openshift/origin/pkg/cmd/util/clientcmd"

 	clustdiags "github.com/openshift/origin/pkg/diagnostics/cluster"

+	agldiags "github.com/openshift/origin/pkg/diagnostics/cluster/aggregated_logging"

 	"github.com/openshift/origin/pkg/diagnostics/types"

 )

 var (

 	// availableClusterDiagnostics contains the names of cluster diagnostics that can be executed

 	// during a single run of diagnostics. Add more diagnostics to the list as they are defined.

-	availableClusterDiagnostics = sets.NewString(clustdiags.NodeDefinitionsName, clustdiags.ClusterRegistryName, clustdiags.ClusterRouterName, clustdiags.ClusterRolesName, clustdiags.ClusterRoleBindingsName, clustdiags.MasterNodeName, clustdiags.MetricsApiProxyName, clustdiags.ServiceExternalIPsName)

+	availableClusterDiagnostics = sets.NewString(

+		agldiags.AggregatedLoggingName,

+		clustdiags.ClusterRegistryName,

+		clustdiags.ClusterRouterName,

+		clustdiags.ClusterRolesName,

+		clustdiags.ClusterRoleBindingsName,

+		clustdiags.MasterNodeName,

+		clustdiags.MetricsApiProxyName,

+		clustdiags.NodeDefinitionsName,

+		clustdiags.ServiceExternalIPsName,

+	)

 )

 // buildClusterDiagnostics builds cluster Diagnostic objects if a cluster-admin client can be extracted from the rawConfig passed in.

@@ -46,6 +57,8 @@ func (o DiagnosticsOptions) buildClusterDiagnostics(rawConfig *clientcmdapi.Conf

 	for _, diagnosticName := range requestedDiagnostics {

 		var d types.Diagnostic

 		switch diagnosticName {

+		case agldiags.AggregatedLoggingName:

+			d = agldiags.NewAggregatedLogging(o.MasterConfigLocation, kclusterClient, clusterClient)

 		case clustdiags.NodeDefinitionsName:

 			d = &clustdiags.NodeDefinitions{KubeClient: kclusterClient, OsClient: clusterClient}

 		case clustdiags.MasterNodeName:

@@ -80,6 +80,9 @@ you will receive an error if they are not found. For example:

   are skipped.

 Diagnostics may be individually run by passing diagnostic name as arguments.

+

+    %[1]s <DiagnosticName>

+    

 The available diagnostic names are:

 %[2]s

 `

@@ -95,7 +95,7 @@ save may be your own.

 A diagnostic is an object that conforms to the Diagnostic interface

 (see pkg/diagnostics/types/diagnostic.go). The diagnostic object should

-be built in one of the builders in the pkg/cmd/experimental/diagnostics

+be built in one of the builders in the pkg/cmd/admin/diagnostics

 package (based on whether it depends on client, cluster-admin, or host

 configuration). When executed, the diagnostic logs its findings into

 a result object. It should be assumed that they may run in parallel.

new file mode 100644

@@ -0,0 +1,41 @@

+package aggregated_logging

+

+import (

+	"fmt"

+

+	"k8s.io/kubernetes/pkg/apis/rbac"

+	"k8s.io/kubernetes/pkg/util/sets"

+)

+

+const clusterReaderRoleBindingName = "cluster-readers"

+

+var clusterReaderRoleBindingNames = sets.NewString(fluentdServiceAccountName)

+

+const clusterReaderUnboundServiceAccount = `

+The ServiceAccount '%[1]s' is not a cluster-reader in the '%[2]s' project.  This

+is required to enable Fluentd to look up pod metadata for the logs it gathers.

+As a user with a cluster-admin role, you can grant the permissions by running

+the following:

+

+  oadm policy add-cluster-role-to-user cluster-reader system:serviceaccount:%[2]s:%[1]s

+`

+

+func checkClusterRoleBindings(r diagnosticReporter, adapter clusterRoleBindingsAdapter, project string) {

+	r.Debug("AGL0600", "Checking ClusterRoleBindings...")

+	crb, err := adapter.getClusterRoleBinding(clusterReaderRoleBindingName)

+	if err != nil {

+		r.Error("AGL0605", err, fmt.Sprintf("There was an error while trying to retrieve the ClusterRoleBindings for the logging stack: %s", err))

+		return

+	}

+	boundServiceAccounts := sets.NewString()

+	for _, subject := range crb.Subjects {

+		if subject.Kind == rbac.ServiceAccountKind && subject.Namespace == project {

+			boundServiceAccounts.Insert(subject.Name)

+		}

+	}

+	for _, name := range clusterReaderRoleBindingNames.List() {

+		if !boundServiceAccounts.Has(name) {

+			r.Error("AGL0610", nil, fmt.Sprintf(clusterReaderUnboundServiceAccount, name, project))

+		}

+	}

+}

new file mode 100644

@@ -0,0 +1,70 @@

+package aggregated_logging

+

+import (

+	"errors"

+	"testing"

+

+	kapi "k8s.io/kubernetes/pkg/api"

+	"k8s.io/kubernetes/pkg/apis/rbac"

+

+	authapi "github.com/openshift/origin/pkg/authorization/api"

+	"github.com/openshift/origin/pkg/diagnostics/log"

+)

+

+type fakeRoleBindingDiagnostic struct {

+	fakeDiagnostic

+	fakeClusterRoleBinding authapi.ClusterRoleBinding

+}

+

+func newFakeRoleBindingDiagnostic(t *testing.T) *fakeRoleBindingDiagnostic {

+	return &fakeRoleBindingDiagnostic{

+		fakeDiagnostic: *newFakeDiagnostic(t),

+	}

+}

+

+func (f *fakeRoleBindingDiagnostic) getClusterRoleBinding(name string) (*authapi.ClusterRoleBinding, error) {

+	if f.err != nil {

+		return nil, f.err

+	}

+	return &f.fakeClusterRoleBinding, nil

+}

+func (f *fakeRoleBindingDiagnostic) addBinding(name string, namespace string) {

+	ref := kapi.ObjectReference{

+		Name:      name,

+		Kind:      rbac.ServiceAccountKind,

+		Namespace: namespace,

+	}

+	f.fakeClusterRoleBinding.Subjects = append(f.fakeClusterRoleBinding.Subjects, ref)

+}

+

+//test error when client error

+func TestCheckClusterRoleBindingsWhenErrorFromClientRetrievingRoles(t *testing.T) {

+	d := newFakeRoleBindingDiagnostic(t)

+	d.err = errors.New("client error")

+

+	checkClusterRoleBindings(d, d, fakeProject)

+

+	d.assertMessage("AGL0605", "Exp. an error message if client error retrieving ClusterRoleBindings", log.ErrorLevel)

+	d.dumpMessages()

+}

+

+func TestCheckClusterRoleBindingsWhenClusterReaderIsNotInProject(t *testing.T) {

+	d := newFakeRoleBindingDiagnostic(t)

+	d.addBinding("someName", "someRandomProject")

+	d.addBinding(fluentdServiceAccountName, fakeProject)

+

+	checkClusterRoleBindings(d, d, fakeProject)

+

+	d.assertNoErrors()

+	d.dumpMessages()

+}

+

+func TestCheckClusterRoleBindingsWhenUnboundServiceAccounts(t *testing.T) {

+	d := newFakeRoleBindingDiagnostic(t)

+	d.addBinding(fluentdServiceAccountName, "someRandomProject")

+

+	checkClusterRoleBindings(d, d, fakeProject)

+

+	d.assertMessage("AGL0610", "Exp. an error when the exp service-accounts dont have cluster-reader access", log.ErrorLevel)

+	d.dumpMessages()

+}

new file mode 100644

@@ -0,0 +1,118 @@

+package aggregated_logging

+

+import (

+	"fmt"

+

+	kapi "k8s.io/kubernetes/pkg/api"

+	kapisext "k8s.io/kubernetes/pkg/apis/extensions"

+	"k8s.io/kubernetes/pkg/labels"

+)

+

+const daemonSetNoLabeledNodes = `

+There are no nodes that match the selector for DaemonSet '%[1]s'. This

+means Fluentd is not running and is not gathering logs from any nodes.

+An example of a command to target a specific node for this DaemonSet:

+

+  oc label node/node1.example.com %[2]s

+

+or to label them all:

+

+  oc label node --all %[2]s

+`

+

+const daemonSetPartialNodesLabeled = `

+There are some nodes that match the selector for DaemonSet '%s'.  

+A list of matching nodes can be discovered by running:

+

+  oc get nodes -l %s

+`

+const daemonSetNoPodsFound = `

+There were no pods found that match DaemonSet '%s' with matchLabels '%s'

+`

+const daemonSetPodsNotRunning = `

+The Pod '%[1]s' matched by DaemonSet '%[2]s' is not in '%[3]s' status: %[4]s. 

+

+Depending upon the state, this could mean there is an error running the image 

+for one or more pod containers, the node could be pulling images, etc.  Try running

+the following commands to get additional information:

+

+  oc describe pod %[1]s -n %[5]s

+  oc logs %[1]s -n %[5]s

+  oc get events -n %[5]s

+`

+const daemonSetNotFound = `

+There were no DaemonSets in project '%s' that included label '%s'.  This implies

+the Fluentd pods are not deployed or the logging stack needs to be upgraded.  Try

+running the installer to upgrade the logging stack.

+`

+

+var loggingInfraFluentdSelector = labels.Set{loggingInfraKey: "fluentd"}

+

+func checkDaemonSets(r diagnosticReporter, adapter daemonsetAdapter, project string) {

+	r.Debug("AGL0400", fmt.Sprintf("Checking DaemonSets in project '%s'...", project))

+	dsList, err := adapter.daemonsets(project, kapi.ListOptions{LabelSelector: loggingInfraFluentdSelector.AsSelector()})

+	if err != nil {

+		r.Error("AGL0405", err, fmt.Sprintf("There was an error while trying to retrieve the logging DaemonSets in project '%s' which is most likely transient: %s", project, err))

+		return

+	}

+	if len(dsList.Items) == 0 {

+		r.Error("AGL0407", err, fmt.Sprintf(daemonSetNotFound, project, loggingInfraFluentdSelector.AsSelector()))

+		return

+	}

+	nodeList, err := adapter.nodes(kapi.ListOptions{})

+	if err != nil {

+		r.Error("AGL0410", err, fmt.Sprintf("There was an error while trying to retrieve the list of Nodes which is most likely transient: %s", err))

+		return

+	}

+	for _, ds := range dsList.Items {

+		labeled := 0

+		nodeSelector := labels.Set(ds.Spec.Template.Spec.NodeSelector).AsSelector()

+		r.Debug("AGL0415", fmt.Sprintf("Checking DaemonSet '%s' nodeSelector '%s'", ds.ObjectMeta.Name, nodeSelector))

+		for _, node := range nodeList.Items {

+			if nodeSelector.Matches(labels.Set(node.Labels)) {

+				labeled = labeled + 1

+			}

+		}

+		switch {

+		case labeled == 0:

+			r.Error("AGL0420", nil, fmt.Sprintf(daemonSetNoLabeledNodes, ds.ObjectMeta.Name, nodeSelector))

+			break

+		case labeled < len(nodeList.Items):

+			r.Warn("AGL0425", nil, fmt.Sprintf(daemonSetPartialNodesLabeled, ds.ObjectMeta.Name, nodeSelector))

+			break

+		default:

+			r.Debug("AGL0430", fmt.Sprintf("DaemonSet '%s' matches all nodes", ds.ObjectMeta.Name))

+		}

+		if labeled > 0 {

+			checkDaemonSetPods(r, adapter, ds, project, labeled)

+		}

+	}

+}

+

+func checkDaemonSetPods(r diagnosticReporter, adapter daemonsetAdapter, ds kapisext.DaemonSet, project string, numLabeledNodes int) {

+	if ds.Spec.Selector == nil {

+		r.Debug("AGL0455", "DaemonSet selector is nil. Unable to verify a pod is running")

+		return

+	}

+	podSelector := labels.Set(ds.Spec.Selector.MatchLabels).AsSelector()

+	r.Debug("AGL0435", fmt.Sprintf("Checking for running pods for DaemonSet '%s' with matchLabels '%s'", ds.ObjectMeta.Name, podSelector))

+	podList, err := adapter.pods(project, kapi.ListOptions{LabelSelector: podSelector})

+	if err != nil {

+		r.Error("AGL0438", err, fmt.Sprintf("There was an error retrieving pods matched to DaemonSet '%s' that is most likely transient: %s", ds.ObjectMeta.Name, err))

+		return

+	}

+	if len(podList.Items) == 0 {

+		r.Error("AGL0440", nil, fmt.Sprintf(daemonSetNoPodsFound, ds.ObjectMeta.Name, podSelector))

+		return

+	}

+	if len(podList.Items) != numLabeledNodes {

+		r.Error("AGL0443", nil, fmt.Sprintf("The number of deployed pods %s does not match the number of labeled nodes %d", len(podList.Items), numLabeledNodes))

+	}

+	for _, pod := range podList.Items {

+		if pod.Status.Phase != kapi.PodRunning {

+			podName := pod.ObjectMeta.Name

+			r.Error("AGL0445", nil, fmt.Sprintf(daemonSetPodsNotRunning, podName, ds.ObjectMeta.Name, kapi.PodRunning, pod.Status.Phase, project))

+		}

+

+	}

+}

new file mode 100644

@@ -0,0 +1,188 @@

+package aggregated_logging

+

+import (

+	"errors"

+	"testing"

+

+	kapi "k8s.io/kubernetes/pkg/api"

+	"k8s.io/kubernetes/pkg/api/unversioned"

+	kapisext "k8s.io/kubernetes/pkg/apis/extensions"

+

+	"github.com/openshift/origin/pkg/diagnostics/log"

+)

+

+const (

+	testPodsKey  = "pods"

+	testNodesKey = "nodes"

+	testDsKey    = "daemonsets"

+)

+

+type fakeDaemonSetDiagnostic struct {

+	fakeDiagnostic

+	fakePods       kapi.PodList

+	fakeNodes      kapi.NodeList

+	fakeDaemonsets kapisext.DaemonSetList

+	clienterrors   map[string]error

+}

+

+func newFakeDaemonSetDiagnostic(t *testing.T) *fakeDaemonSetDiagnostic {

+	return &fakeDaemonSetDiagnostic{

+		fakeDiagnostic: *newFakeDiagnostic(t),

+		clienterrors:   map[string]error{},

+	}

+}

+

+func (f *fakeDaemonSetDiagnostic) addDsPodWithPhase(state kapi.PodPhase) {

+	pod := kapi.Pod{

+		Spec: kapi.PodSpec{},

+		Status: kapi.PodStatus{

+			Phase: state,

+		},

+	}

+	f.fakePods.Items = append(f.fakePods.Items, pod)

+}

+

+func (f *fakeDaemonSetDiagnostic) addDaemonSetWithSelector(key string, value string) {

+	selector := map[string]string{key: value}

+	ds := kapisext.DaemonSet{

+		Spec: kapisext.DaemonSetSpec{

+			Template: kapi.PodTemplateSpec{

+				Spec: kapi.PodSpec{

+					NodeSelector: selector,

+				},

+			},

+			Selector: &unversioned.LabelSelector{MatchLabels: selector},

+		},

+	}

+	f.fakeDaemonsets.Items = append(f.fakeDaemonsets.Items, ds)

+}

+

+func (f *fakeDaemonSetDiagnostic) addNodeWithLabel(key string, value string) {

+	labels := map[string]string{key: value}

+	node := kapi.Node{

+		ObjectMeta: kapi.ObjectMeta{

+			Labels: labels,

+		},

+	}

+	f.fakeNodes.Items = append(f.fakeNodes.Items, node)

+}

+

+func (f *fakeDaemonSetDiagnostic) daemonsets(project string, options kapi.ListOptions) (*kapisext.DaemonSetList, error) {

+	value, ok := f.clienterrors[testDsKey]

+	if ok {

+		return nil, value

+	}

+	return &f.fakeDaemonsets, nil

+}

+

+func (f *fakeDaemonSetDiagnostic) nodes(options kapi.ListOptions) (*kapi.NodeList, error) {

+	value, ok := f.clienterrors[testNodesKey]

+	if ok {

+		return nil, value

+	}

+	return &f.fakeNodes, nil

+}

+

+func (f *fakeDaemonSetDiagnostic) pods(project string, options kapi.ListOptions) (*kapi.PodList, error) {

+	value, ok := f.clienterrors[testPodsKey]

+	if ok {

+		return nil, value

+	}

+	return &f.fakePods, nil

+}

+

+func TestCheckDaemonsetsWhenErrorResponseFromClientRetrievingDaemonsets(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+	d.clienterrors[testDsKey] = errors.New("someerror")

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertMessage("AGL0405", "Exp. error when client errors on retrieving DaemonSets", log.ErrorLevel)

+}

+

+func TestCheckDaemonsetsWhenNoDaemonsetsFound(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertMessage("AGL0407", "Exp. error when client retrieves no DaemonSets", log.ErrorLevel)

+}

+

+func TestCheckDaemonsetsWhenErrorResponseFromClientRetrievingNodes(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+	d.clienterrors[testNodesKey] = errors.New("someerror")

+	d.addDaemonSetWithSelector("foo", "bar")

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertMessage("AGL0410", "Exp. error when client errors on retrieving Nodes", log.ErrorLevel)

+}

+

+func TestCheckDaemonsetsWhenDaemonsetsMatchNoNodes(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+	d.addDaemonSetWithSelector("foo", "bar")

+	d.addNodeWithLabel("foo", "xyz")

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertMessage("AGL0420", "Exp. error when daemonsets do not match any nodes", log.ErrorLevel)

+}

+

+func TestCheckDaemonsetsWhenDaemonsetsMatchPartialNodes(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+	d.addDaemonSetWithSelector("foo", "bar")

+	d.addNodeWithLabel("foo", "bar")

+	d.addNodeWithLabel("foo", "xyz")

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertMessage("AGL0425", "Exp. warning when daemonsets matches less then all the nodes", log.WarnLevel)

+}

+

+func TestCheckDaemonsetsWhenClientErrorsFetchingPods(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+	d.clienterrors[testPodsKey] = errors.New("some error")

+	d.addDaemonSetWithSelector("foo", "bar")

+	d.addNodeWithLabel("foo", "bar")

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertMessage("AGL0438", "Exp. error when there is an error retrieving pods for a daemonset", log.ErrorLevel)

+

+	d.dumpMessages()

+}

+

+func TestCheckDaemonsetsWhenNoPodsMatchDaemonSet(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+	d.addDaemonSetWithSelector("foo", "bar")

+	d.addNodeWithLabel("foo", "bar")

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertMessage("AGL0440", "Exp. error when there are no pods that match a daemonset", log.ErrorLevel)

+	d.dumpMessages()

+}

+

+func TestCheckDaemonsetsWhenNoPodsInRunningState(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+	d.addDaemonSetWithSelector("foo", "bar")

+	d.addNodeWithLabel("foo", "bar")

+	d.addDsPodWithPhase(kapi.PodPending)

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertMessage("AGL0445", "Exp. error when there are no pods in running state", log.ErrorLevel)

+	d.dumpMessages()

+}

+

+func TestCheckDaemonsetsWhenAllPodsInRunningState(t *testing.T) {

+	d := newFakeDaemonSetDiagnostic(t)

+	d.addDaemonSetWithSelector("foo", "bar")

+	d.addNodeWithLabel("foo", "bar")

+	d.addDsPodWithPhase(kapi.PodRunning)

+

+	checkDaemonSets(d, d, fakeProject)

+

+	d.assertNoErrors()

+	d.dumpMessages()

+}

new file mode 100644

@@ -0,0 +1,128 @@

+package aggregated_logging

+

+import (

+	"fmt"

+	"strings"

+

+	kapi "k8s.io/kubernetes/pkg/api"

+	"k8s.io/kubernetes/pkg/labels"

+	"k8s.io/kubernetes/pkg/selection"

+	"k8s.io/kubernetes/pkg/util/sets"

+

+	deployapi "github.com/openshift/origin/pkg/deploy/api"

+)

+

+const (

+	componentNameEs        = "es"

+	componentNameEsOps     = "es-ops"

+	componentNameKibana    = "kibana"

+	componentNameKibanaOps = "kibana-ops"

+	componentNameCurator   = "curator"

+)

+

+// loggingComponents are those 'managed' by rep controllers (e.g. fluentd is deployed with a DaemonSet)

+var loggingComponents = sets.NewString(componentNameEs, componentNameEsOps, componentNameKibana, componentNameKibanaOps, componentNameCurator)

+

+const deploymentConfigWarnMissingForOps = `

+Did not find a DeploymentConfig to support component '%s'.  If you require

+a separate ElasticSearch cluster to aggregate operations logs, please re-install

+or update logging and specify the appropriate switch to enable the ops cluster.

+`

+

+const deploymentConfigZeroPodsFound = `

+There were no Pods found that support logging.  Try running

+the following commands for additional information:

+

+  oc describe dc -n %[1]s

+  oc get events -n %[1]s

+`

+const deploymentConfigNoPodsFound = `

+There were no Pods found for DeploymentConfig '%[1]s'.  Try running

+the following commands for additional information:

+

+  oc describe dc %[1]s -n %[2]s

+  oc get events -n %[2]s

+`

+const deploymentConfigPodsNotRunning = `

+The Pod '%[1]s' matched by DeploymentConfig '%[2]s' is not in '%[3]s' status: %[4]s. 

+

+Depending upon the state, this could mean there is an error running the image 

+for one or more pod containers, the node could be pulling images, etc.  Try running

+the following commands for additional information:

+

+  oc describe pod %[1]s -n %[5]s

+  oc logs %[1]s -n %[5]s

+  oc get events -n %[5]s

+`

+

+func checkDeploymentConfigs(r diagnosticReporter, adapter deploymentConfigAdapter, project string) {

+	req, _ := labels.NewRequirement(loggingInfraKey, selection.Exists, nil)

+	selector := labels.NewSelector().Add(*req)

+	r.Debug("AGL0040", fmt.Sprintf("Checking for DeploymentConfigs in project '%s' with selector '%s'", project, selector))

+	dcList, err := adapter.deploymentconfigs(project, kapi.ListOptions{LabelSelector: selector})

+	if err != nil {

+		r.Error("AGL0045", err, fmt.Sprintf("There was an error while trying to retrieve the DeploymentConfigs in project '%s': %s", project, err))

+		return

+	}

+	if len(dcList.Items) == 0 {

+		r.Error("AGL0047", nil, fmt.Sprintf("Did not find any matching DeploymentConfigs in project '%s' which means no logging components were deployed.  Try running the installer.", project))

+		return

+	}

+	found := sets.NewString()

+	for _, entry := range dcList.Items {

+		comp := labels.Set(entry.ObjectMeta.Labels).Get(componentKey)

+		found.Insert(comp)

+		r.Debug("AGL0050", fmt.Sprintf("Found DeploymentConfig '%s' for component '%s'", entry.ObjectMeta.Name, comp))

+	}

+	for _, entry := range loggingComponents.List() {

+		exists := found.Has(entry)

+		if !exists {

+			if strings.HasSuffix(entry, "-ops") {

+				r.Info("AGL0060", fmt.Sprintf(deploymentConfigWarnMissingForOps, entry))

+			} else {

+				r.Error("AGL0065", nil, fmt.Sprintf("Did not find a DeploymentConfig to support component '%s'", entry))

+			}

+		}

+	}

+	checkDeploymentConfigPods(r, adapter, *dcList, project)

+}

+

+func checkDeploymentConfigPods(r diagnosticReporter, adapter deploymentConfigAdapter, dcs deployapi.DeploymentConfigList, project string) {

+	compReq, _ := labels.NewRequirement(componentKey, selection.In, loggingComponents)

+	provReq, _ := labels.NewRequirement(providerKey, selection.Equals, sets.NewString(openshiftValue))

+	podSelector := labels.NewSelector().Add(*compReq, *provReq)

+	r.Debug("AGL0070", fmt.Sprintf("Getting pods that match selector '%s'", podSelector))

+	podList, err := adapter.pods(project, kapi.ListOptions{LabelSelector: podSelector})

+	if err != nil {

+		r.Error("AGL0075", err, fmt.Sprintf("There was an error while trying to retrieve the pods for the AggregatedLogging stack: %s", err))

+		return

+	}

+	if len(podList.Items) == 0 {

+		r.Error("AGL0080", nil, fmt.Sprintf(deploymentConfigZeroPodsFound, project))

+		return

+	}

+	dcPodCount := make(map[string]int, len(dcs.Items))

+	for _, dc := range dcs.Items {

+		dcPodCount[dc.ObjectMeta.Name] = 0

+	}

+

+	for _, pod := range podList.Items {

+		r.Debug("AGL0082", fmt.Sprintf("Checking status of Pod '%s'...", pod.ObjectMeta.Name))

+		dcName, hasDcName := pod.ObjectMeta.Annotations[deployapi.DeploymentConfigAnnotation]

+		if !hasDcName {

+			r.Warn("AGL0085", nil, fmt.Sprintf("Found Pod '%s' that that does not reference a logging deployment config which may be acceptable. Skipping check to see if its running.", pod.ObjectMeta.Name))

+			continue

+		}

+		if pod.Status.Phase != kapi.PodRunning {

+			podName := pod.ObjectMeta.Name

+			r.Error("AGL0090", nil, fmt.Sprintf(deploymentConfigPodsNotRunning, podName, dcName, kapi.PodRunning, pod.Status.Phase, project))

+		}

+		count, _ := dcPodCount[dcName]

+		dcPodCount[dcName] = count + 1

+	}

+	for name, count := range dcPodCount {

+		if count == 0 {

+			r.Error("AGL0095", nil, fmt.Sprintf(deploymentConfigNoPodsFound, name, project))

+		}

+	}

+}

new file mode 100644

@@ -0,0 +1,153 @@

+package aggregated_logging

+

+import (

+	"errors"

+	"testing"

+

+	kapi "k8s.io/kubernetes/pkg/api"

+

+	deployapi "github.com/openshift/origin/pkg/deploy/api"

+	"github.com/openshift/origin/pkg/diagnostics/log"

+)

+

+const (

+	testDcPodsKey      = "pods"

+	testDcKey          = "deploymentconfigs"

+	testSkipAnnotation = "skipAddAnnoation"

+)

+

+type fakeDeploymentConfigsDiagnostic struct {

+	fakeDiagnostic

+	fakePods     kapi.PodList

+	fakeDcs      deployapi.DeploymentConfigList

+	clienterrors map[string]error

+}

+

+func newFakeDeploymentConfigsDiagnostic(t *testing.T) *fakeDeploymentConfigsDiagnostic {

+	return &fakeDeploymentConfigsDiagnostic{

+		fakeDiagnostic: *newFakeDiagnostic(t),

+		clienterrors:   map[string]error{},

+	}

+}

+func (f *fakeDeploymentConfigsDiagnostic) addDeployConfigFor(component string) {

+	labels := map[string]string{componentKey: component}

+	dc := deployapi.DeploymentConfig{

+		ObjectMeta: kapi.ObjectMeta{

+			Name:   component + "Name",

+			Labels: labels,

+		},

+	}

+	f.fakeDcs.Items = append(f.fakeDcs.Items, dc)

+}

+

+func (f *fakeDeploymentConfigsDiagnostic) addPodFor(comp string, state kapi.PodPhase) {

+	annotations := map[string]string{}

+	if comp != testSkipAnnotation {

+		annotations[deployapi.DeploymentConfigAnnotation] = comp

+	}

+	pod := kapi.Pod{

+		ObjectMeta: kapi.ObjectMeta{

+			Name:        comp,

+			Annotations: annotations,

+		},

+		Spec: kapi.PodSpec{},

+		Status: kapi.PodStatus{

+			Phase: state,

+		},

+	}

+	f.fakePods.Items = append(f.fakePods.Items, pod)

+}

+

+func (f *fakeDeploymentConfigsDiagnostic) deploymentconfigs(project string, options kapi.ListOptions) (*deployapi.DeploymentConfigList, error) {

+	f.test.Logf(">> calling deploymentconfigs: %s", f.clienterrors)

+	value, ok := f.clienterrors[testDcKey]

+	if ok {

+		f.test.Logf(">> error key found..returning %s", value)

+		return nil, value

+	}

+	f.test.Logf(">> error key not found..")

+	return &f.fakeDcs, nil

+}

+

+func (f *fakeDeploymentConfigsDiagnostic) pods(project string, options kapi.ListOptions) (*kapi.PodList, error) {

+	value, ok := f.clienterrors[testDcPodsKey]

+	if ok {

+		return nil, value

+	}

+	return &f.fakePods, nil

+}

+

+//test client error listing dcs

+func TestCheckDcWhenErrorResponseFromClientRetrievingDc(t *testing.T) {

+	d := newFakeDeploymentConfigsDiagnostic(t)

+	d.clienterrors[testDcKey] = errors.New("error")

+

+	checkDeploymentConfigs(d, d, fakeProject)

+

+	d.assertMessage("AGL0045", "Exp. an error when client returns error retrieving dcs", log.ErrorLevel)

+	d.dumpMessages()

+}

+

+func TestCheckDcWhenNoDeployConfigsFound(t *testing.T) {

+	d := newFakeDeploymentConfigsDiagnostic(t)

+

+	checkDeploymentConfigs(d, d, fakeProject)

+

+	d.assertMessage("AGL0047", "Exp. an error when no DeploymentConfigs are found", log.ErrorLevel)

+	d.dumpMessages()

+}

+

+func TestCheckDcWhenOpsOrOtherDeployConfigsMissing(t *testing.T) {

+	d := newFakeDeploymentConfigsDiagnostic(t)

+	d.addDeployConfigFor(componentNameEs)

+

+	checkDeploymentConfigs(d, d, fakeProject)

+

+	d.assertMessage("AGL0060", "Exp. a warning when ops DeploymentConfigs are missing", log.InfoLevel)

+	d.assertMessage("AGL0065", "Exp. an error when non-ops DeploymentConfigs are missing", log.ErrorLevel)

+	d.dumpMessages()

+}

+

+func TestCheckDcWhenClientErrorListingPods(t *testing.T) {

+	d := newFakeDeploymentConfigsDiagnostic(t)

+	d.clienterrors[testDcPodsKey] = errors.New("New pod error")

+	for _, comp := range loggingComponents.List() {

+		d.addDeployConfigFor(comp)

+	}

+

+	checkDeploymentConfigs(d, d, fakeProject)

+

+	d.assertMessage("AGL0075", "Exp. an error when retrieving pods errors", log.ErrorLevel)

+	d.dumpMessages()

+}

+

+func TestCheckDcWhenNoPodsFoundMatchingDeployConfig(t *testing.T) {

+	d := newFakeDeploymentConfigsDiagnostic(t)

+	for _, comp := range loggingComponents.List() {

+		d.addDeployConfigFor(comp)

+	}

+

+	checkDeploymentConfigs(d, d, fakeProject)

+

+	d.assertMessage("AGL0080", "Exp. an error when retrieving pods errors", log.ErrorLevel)

+	d.dumpMessages()