package diagnostics
import (
"fmt"
"io"
"os"
"runtime/debug"
"strings"
"github.com/spf13/cobra"
flag "github.com/spf13/pflag"
kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
kutilerrors "k8s.io/kubernetes/pkg/util/errors"
"k8s.io/kubernetes/pkg/util/sets"
"github.com/openshift/origin/pkg/cmd/admin/diagnostics/options"
"github.com/openshift/origin/pkg/cmd/cli/config"
"github.com/openshift/origin/pkg/cmd/flagtypes"
"github.com/openshift/origin/pkg/cmd/templates"
osclientcmd "github.com/openshift/origin/pkg/cmd/util/clientcmd"
"github.com/openshift/origin/pkg/cmd/util/variable"
"github.com/openshift/origin/pkg/diagnostics/log"
"github.com/openshift/origin/pkg/diagnostics/types"
)
// DiagnosticsOptions holds values received from command line flags as well as
// other objects generated for the command to operate.
type DiagnosticsOptions struct {
// list of diagnostic names to limit what is run
RequestedDiagnostics []string
// specify locations of host config files
MasterConfigLocation string
NodeConfigLocation string
// specify context name to be used for cluster-admin access
ClientClusterContext string
// indicate this is an openshift host despite lack of other indicators
IsHost bool
// specify the image template to use for DiagnosticPod
ImageTemplate variable.ImageTemplate
// When true, prevent diagnostics from changing API state (e.g. creating something)
PreventModification bool
// We need a factory for creating clients. Creating a factory
// creates flags as a byproduct, most of which we don't want.
// The command creates these and binds only the flags we want.
ClientFlags *flag.FlagSet
Factory *osclientcmd.Factory
// LogOptions determine globally what the user wants to see and how.
LogOptions *log.LoggerOptions
// The Logger is built with the options and should be used for all diagnostic output.
Logger *log.Logger
}
const (
DiagnosticsRecommendedName = "diagnostics"
// Standard locations for the host config files OpenShift uses.
StandardMasterConfigPath string = "/etc/origin/master/master-config.yaml"
StandardNodeConfigPath string = "/etc/origin/node/node-config.yaml"
)
var (
longDescription = templates.LongDesc(`
This utility helps troubleshoot and diagnose known problems. It runs
diagnostics using a client and/or the state of a running master /
node host.
%[1]s
If run without flags, it will check for standard config files for
client, master, and node, and if found, use them for diagnostics.
You may also specify config files explicitly with flags, in which case
you will receive an error if they are not found. For example:
%[1]s --master-config=/etc/origin/master/master-config.yaml
* If master/node config files are not found and the --host flag is not
present, host diagnostics are skipped.
* If the client has cluster-admin access, this access enables cluster
diagnostics to run which regular users cannot.
* If a client config file is not found, client and cluster diagnostics
are skipped.
Diagnostics may be individually run by passing diagnostic name as arguments.
%[1]s <DiagnosticName>
The available diagnostic names are: %[2]s.`)
)
// NewCmdDiagnostics is the base command for running any diagnostics.
func NewCmdDiagnostics(name string, fullName string, out io.Writer) *cobra.Command {
o := &DiagnosticsOptions{
RequestedDiagnostics: []string{},
LogOptions: &log.LoggerOptions{Out: out},
ImageTemplate: variable.NewDefaultImageTemplate(),
}
cmd := &cobra.Command{
Use: name,
Short: "Diagnose common cluster problems",
Long: fmt.Sprintf(longDescription, fullName, strings.Join(availableDiagnostics().List(), ", ")),
Run: func(c *cobra.Command, args []string) {
kcmdutil.CheckErr(o.Complete(args))
kcmdutil.CheckErr(o.Validate())
failed, err, warnCount, errorCount := o.RunDiagnostics()
o.Logger.Summary(warnCount, errorCount)
kcmdutil.CheckErr(err)
if failed {
os.Exit(255)
}
},
}
cmd.SetOutput(out) // for output re: usage / help
o.ClientFlags = flag.NewFlagSet("client", flag.ContinueOnError) // hide the extensive set of client flags
o.Factory = osclientcmd.New(o.ClientFlags) // that would otherwise be added to this command
cmd.Flags().AddFlag(o.ClientFlags.Lookup(config.OpenShiftConfigFlagName))
cmd.Flags().AddFlag(o.ClientFlags.Lookup("context")) // TODO: find k8s constant
cmd.Flags().StringVar(&o.ClientClusterContext, options.FlagClusterContextName, "", "Client context to use for cluster administrator")
cmd.Flags().StringVar(&o.MasterConfigLocation, options.FlagMasterConfigName, "", "Path to master config file (implies --host)")
cmd.Flags().StringVar(&o.NodeConfigLocation, options.FlagNodeConfigName, "", "Path to node config file (implies --host)")
cmd.Flags().BoolVar(&o.IsHost, options.FlagIsHostName, false, "Look for systemd and journald units even without master/node config")
cmd.Flags().StringVar(&o.ImageTemplate.Format, options.FlagImageTemplateName, o.ImageTemplate.Format, "Image template for DiagnosticPod to use in creating a pod")
cmd.Flags().BoolVar(&o.ImageTemplate.Latest, options.FlagLatestImageName, false, "When expanding the image template, use latest version, not release version")
cmd.Flags().BoolVar(&o.PreventModification, options.FlagPreventModificationName, false, "May be set to prevent diagnostics making any changes via the API")
flagtypes.GLog(cmd.Flags())
options.BindLoggerOptionFlags(cmd.Flags(), o.LogOptions, options.RecommendedLoggerOptionFlags())
return cmd
}
// Complete fills in DiagnosticsOptions needed if the command is actually invoked.
func (o *DiagnosticsOptions) Complete(args []string) error {
var err error
o.Logger, err = o.LogOptions.NewLogger()
if err != nil {
return err
}
// If not given master/client config file locations, check if the defaults exist
// and adjust the options accordingly:
if len(o.MasterConfigLocation) == 0 {
if _, err := os.Stat(StandardMasterConfigPath); !os.IsNotExist(err) {
o.MasterConfigLocation = StandardMasterConfigPath
}
}
if len(o.NodeConfigLocation) == 0 {
if _, err := os.Stat(StandardNodeConfigPath); !os.IsNotExist(err) {
o.NodeConfigLocation = StandardNodeConfigPath
}
}
o.RequestedDiagnostics = append(o.RequestedDiagnostics, args...)
if len(o.RequestedDiagnostics) == 0 {
o.RequestedDiagnostics = availableDiagnostics().List()
}
return nil
}
func (o *DiagnosticsOptions) Validate() error {
available := availableDiagnostics()
if common := available.Intersection(sets.NewString(o.RequestedDiagnostics...)); len(common) == 0 {
o.Logger.Error("CED3012", log.EvalTemplate("CED3012", "None of the requested diagnostics are available:\n {{.requested}}\nPlease try from the following:\n {{.available}}",
log.Hash{"requested": o.RequestedDiagnostics, "available": available.List()}))
return fmt.Errorf("No requested diagnostics are available: requested=%s available=%s", strings.Join(o.RequestedDiagnostics, " "), strings.Join(available.List(), " "))
} else if len(common) < len(o.RequestedDiagnostics) {
o.Logger.Error("CED3013", log.EvalTemplate("CED3013", `
Of the requested diagnostics:
{{.requested}}
only these are available:
{{.common}}
The list of all possible is:
{{.available}}
`, log.Hash{"requested": o.RequestedDiagnostics, "common": common.List(), "available": available.List()}))
return fmt.Errorf("Not all requested diagnostics are available: missing=%s requested=%s available=%s",
strings.Join(sets.NewString(o.RequestedDiagnostics...).Difference(available).List(), " "),
strings.Join(o.RequestedDiagnostics, " "),
strings.Join(available.List(), " "))
}
return nil
}
func availableDiagnostics() sets.String {
available := sets.NewString()
available.Insert(availableClientDiagnostics.List()...)
available.Insert(availableClusterDiagnostics.List()...)
available.Insert(availableHostDiagnostics.List()...)
return available
}
// RunDiagnostics builds diagnostics based on the options and executes them, returning a summary.
func (o DiagnosticsOptions) RunDiagnostics() (bool, error, int, int) {
failed := false
warnings := []error{}
errors := []error{}
diagnostics := []types.Diagnostic{}
func() { // don't trust discovery/build of diagnostics; wrap panic nicely in case of developer error
defer func() {
if r := recover(); r != nil {
failed = true
stack := debug.Stack()
errors = append(errors, fmt.Errorf("While building the diagnostics, a panic was encountered.\nThis is a bug in diagnostics. Error and stack trace follow: \n%v\n%s", r, stack))
}
}()
detected, detectWarnings, detectErrors := o.detectClientConfig() // may log and return problems
for _, warn := range detectWarnings {
warnings = append(warnings, warn)
}
for _, err := range detectErrors {
errors = append(errors, err)
}
if !detected { // there just plain isn't any client config file available
o.Logger.Notice("CED3014", "No client configuration specified; skipping client and cluster diagnostics.")
} else if rawConfig, err := o.buildRawConfig(); err != nil { // client config is totally broken - won't parse etc (problems may have been detected and logged)
o.Logger.Error("CED3015", fmt.Sprintf("Client configuration failed to load; skipping client and cluster diagnostics due to error: %s", err.Error()))
errors = append(errors, err)
} else {
clientDiags, ok, err := o.buildClientDiagnostics(rawConfig)
failed = failed || !ok
if ok {
diagnostics = append(diagnostics, clientDiags...)
}
if err != nil {
errors = append(errors, err)
}
clusterDiags, ok, err := o.buildClusterDiagnostics(rawConfig)
failed = failed || !ok
if ok {
diagnostics = append(diagnostics, clusterDiags...)
}
if err != nil {
errors = append(errors, err)
}
}
hostDiags, ok, err := o.buildHostDiagnostics()
failed = failed || !ok
if ok {
diagnostics = append(diagnostics, hostDiags...)
}
if err != nil {
errors = append(errors, err)
}
}()
if failed {
return failed, kutilerrors.NewAggregate(errors), len(warnings), len(errors)
}
failed, err, numWarnings, numErrors := o.Run(diagnostics)
numWarnings += len(warnings)
numErrors += len(errors)
return failed, err, numWarnings, numErrors
}
// Run performs the actual execution of diagnostics once they're built.
func (o DiagnosticsOptions) Run(diagnostics []types.Diagnostic) (bool, error, int, int) {
warnCount := 0
errorCount := 0
for _, diagnostic := range diagnostics {
func() { // wrap diagnostic panic nicely in case of developer error
defer func() {
if r := recover(); r != nil {
errorCount += 1
stack := debug.Stack()
o.Logger.Error("CED3017",
fmt.Sprintf("While running the %s diagnostic, a panic was encountered.\nThis is a bug in diagnostics. Error and stack trace follow: \n%s\n%s",
diagnostic.Name(), fmt.Sprintf("%v", r), stack))
}
}()
if canRun, reason := diagnostic.CanRun(); !canRun {
if reason == nil {
o.Logger.Notice("CED3018", fmt.Sprintf("Skipping diagnostic: %s\nDescription: %s", diagnostic.Name(), diagnostic.Description()))
} else {
o.Logger.Notice("CED3019", fmt.Sprintf("Skipping diagnostic: %s\nDescription: %s\nBecause: %s", diagnostic.Name(), diagnostic.Description(), reason.Error()))
}
return
}
o.Logger.Notice("CED3020", fmt.Sprintf("Running diagnostic: %s\nDescription: %s", diagnostic.Name(), diagnostic.Description()))
r := diagnostic.Check()
for _, entry := range r.Logs() {
o.Logger.LogEntry(entry)
}
warnCount += len(r.Warnings())
errorCount += len(r.Errors())
}()
}
return errorCount > 0, nil, warnCount, errorCount
}