package start import ( "errors" "fmt" "io" "io/ioutil" "os" "path" "path/filepath" "strings" "github.com/coreos/go-systemd/daemon" "github.com/golang/glog" "github.com/spf13/cobra" kerrors "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/capabilities" "k8s.io/kubernetes/pkg/kubelet" "k8s.io/kubernetes/pkg/util" "github.com/openshift/origin/pkg/cmd/server/admin" configapi "github.com/openshift/origin/pkg/cmd/server/api" configapilatest "github.com/openshift/origin/pkg/cmd/server/api/latest" "github.com/openshift/origin/pkg/cmd/server/api/validation" "github.com/openshift/origin/pkg/cmd/server/bootstrappolicy" "github.com/openshift/origin/pkg/cmd/server/etcd" "github.com/openshift/origin/pkg/cmd/server/kubernetes" "github.com/openshift/origin/pkg/cmd/server/origin" cmdutil "github.com/openshift/origin/pkg/cmd/util" "github.com/openshift/origin/pkg/version" ) type MasterOptions struct { MasterArgs *MasterArgs CreateCertificates bool ConfigFile string Output io.Writer DisabledFeatures []string } const masterLong = `Start a master server This command helps you launch a master server. Running $ %[1]s start master will start a master listening on all interfaces, launch an etcd server to store persistent data, and launch the Kubernetes system components. The server will run in the foreground until you terminate the process. Note: starting the master without passing the --master address will attempt to find the IP address that will be visible inside running Docker containers. This is not always successful, so if you have problems tell the master what public address it should use via --master=<ip>. You may also pass an optional argument to the start command to start in one of the following roles: // Launches the server and control plane. You may pass a list of the node // hostnames you want to use, or create nodes via the REST API or '%[1]s kube'. $ %[1]s start master --nodes=<host1,host2,host3,...> You may also pass --etcd=<address> to connect to an external etcd server. You may also pass --kubeconfig=<path> to connect to an external Kubernetes cluster.` // NewCommandStartMaster provides a CLI handler for 'start master' command func NewCommandStartMaster(fullName string, out io.Writer) (*cobra.Command, *MasterOptions) { options := &MasterOptions{Output: out} switch fullName { case "atomic-enterprise": options.DisabledFeatures = configapi.AtomicDisabledFeatures } cmd := &cobra.Command{ Use: "master", Short: "Launch OpenShift master", Long: fmt.Sprintf(masterLong, fullName), Run: func(c *cobra.Command, args []string) { if err := options.Complete(); err != nil { fmt.Println(err.Error()) c.Help() return } if err := options.Validate(args); err != nil { fmt.Println(err.Error()) c.Help() return } startProfiler() if err := options.StartMaster(); err != nil { if kerrors.IsInvalid(err) { if details := err.(*kerrors.StatusError).ErrStatus.Details; details != nil { fmt.Fprintf(options.Output, "Invalid %s %s\n", details.Kind, details.Name) for _, cause := range details.Causes { fmt.Fprintf(options.Output, " %s: %s\n", cause.Field, cause.Message) } os.Exit(255) } } glog.Fatal(err) } }, } options.MasterArgs = NewDefaultMasterArgs() flags := cmd.Flags() flags.Var(options.MasterArgs.ConfigDir, "write-config", "Directory to write an initial config into. After writing, exit without starting the server.") flags.StringVar(&options.ConfigFile, "config", "", "Location of the master configuration file to run from. When running from a configuration file, all other command-line arguments are ignored.") flags.BoolVar(&options.CreateCertificates, "create-certs", true, "Indicates whether missing certs should be created") BindMasterArgs(options.MasterArgs, flags, "") BindListenArg(options.MasterArgs.ListenArg, flags, "") BindImageFormatArgs(options.MasterArgs.ImageFormatArgs, flags, "") BindKubeConnectionArgs(options.MasterArgs.KubeConnectionArgs, flags, "") BindNetworkArgs(options.MasterArgs.NetworkArgs, flags, "") // autocompletion hints cmd.MarkFlagFilename("write-config") cmd.MarkFlagFilename("config", "yaml", "yml") return cmd, options } func (o MasterOptions) Validate(args []string) error { if len(args) != 0 { return errors.New("no arguments are supported for start master") } if o.IsWriteConfigOnly() { if o.IsRunFromConfig() { return errors.New("--config may not be set if --write-config is set") } } if len(o.MasterArgs.ConfigDir.Value()) == 0 { return errors.New("configDir must have a value") } // if we are not starting up using a config file, run the argument validation if !o.IsRunFromConfig() { if err := o.MasterArgs.Validate(); err != nil { return err } } return nil } func (o *MasterOptions) Complete() error { if !o.MasterArgs.ConfigDir.Provided() { o.MasterArgs.ConfigDir.Default("openshift.local.config/master") } nodeList := util.NewStringSet() // take everything toLower for _, s := range o.MasterArgs.NodeList { nodeList.Insert(strings.ToLower(s)) } o.MasterArgs.NodeList = nodeList.List() return nil } // StartMaster calls RunMaster and then waits forever func (o MasterOptions) StartMaster() error { if err := o.RunMaster(); err != nil { return err } if o.IsWriteConfigOnly() { return nil } go daemon.SdNotify("READY=1") select {} return nil } // RunMaster takes the options and: // 1. Creates certs if needed // 2. Reads fully specified master config OR builds a fully specified master config from the args // 3. Writes the fully specified master config and exits if needed // 4. Starts the master based on the fully specified config func (o MasterOptions) RunMaster() error { startUsingConfigFile := !o.IsWriteConfigOnly() && o.IsRunFromConfig() if !startUsingConfigFile && o.CreateCertificates { glog.V(2).Infof("Generating master configuration") if err := o.CreateCerts(); err != nil { return err } if err := o.CreateBootstrapPolicy(); err != nil { return err } } var masterConfig *configapi.MasterConfig var err error if startUsingConfigFile { masterConfig, err = configapilatest.ReadAndResolveMasterConfig(o.ConfigFile) } else { masterConfig, err = o.MasterArgs.BuildSerializeableMasterConfig() } if err != nil { return err } if o.IsWriteConfigOnly() { // Resolve relative to CWD cwd, err := os.Getwd() if err != nil { return err } if err := configapi.ResolveMasterConfigPaths(masterConfig, cwd); err != nil { return err } // Relativize to config file dir base, err := cmdutil.MakeAbs(filepath.Dir(o.MasterArgs.GetConfigFileToWrite()), cwd) if err != nil { return err } if err := configapi.RelativizeMasterConfigPaths(masterConfig, base); err != nil { return err } content, err := configapilatest.WriteYAML(masterConfig) if err != nil { return err } if err := os.MkdirAll(path.Dir(o.MasterArgs.GetConfigFileToWrite()), os.FileMode(0755)); err != nil { return err } if err := ioutil.WriteFile(o.MasterArgs.GetConfigFileToWrite(), content, 0644); err != nil { return err } fmt.Fprintf(o.Output, "Wrote master config to: %s\n", o.MasterArgs.GetConfigFileToWrite()) return nil } // Inject disabled feature flags based on distribution being used and // regardless of configuration. They aren't written to config file to // prevent upgrade path issues. masterConfig.DisabledFeatures.Add(o.DisabledFeatures...) validationResults := validation.ValidateMasterConfig(masterConfig) if len(validationResults.Warnings) != 0 { for _, warning := range validationResults.Warnings { glog.Warningf("%v", warning) } } if len(validationResults.Errors) != 0 { return kerrors.NewInvalid("MasterConfig", o.ConfigFile, validationResults.Errors) } if err := StartMaster(masterConfig); err != nil { return err } return nil } func (o MasterOptions) CreateBootstrapPolicy() error { writeBootstrapPolicy := admin.CreateBootstrapPolicyFileOptions{ File: o.MasterArgs.GetPolicyFile(), OpenShiftSharedResourcesNamespace: bootstrappolicy.DefaultOpenShiftSharedResourcesNamespace, } return writeBootstrapPolicy.CreateBootstrapPolicyFile() } func (o MasterOptions) CreateCerts() error { masterAddr, err := o.MasterArgs.GetMasterAddress() if err != nil { return err } publicMasterAddr, err := o.MasterArgs.GetMasterPublicAddress() if err != nil { return err } signerName := admin.DefaultSignerName() hostnames, err := o.MasterArgs.GetServerCertHostnames() if err != nil { return err } mintAllCertsOptions := admin.CreateMasterCertsOptions{ CertDir: o.MasterArgs.ConfigDir.Value(), SignerName: signerName, Hostnames: hostnames.List(), APIServerURL: masterAddr.String(), PublicAPIServerURL: publicMasterAddr.String(), Output: o.Output, } if err := mintAllCertsOptions.Validate(nil); err != nil { return err } if err := mintAllCertsOptions.CreateMasterCerts(); err != nil { return err } return nil } func StartMaster(openshiftMasterConfig *configapi.MasterConfig) error { glog.Infof("Starting master on %s (%s)", openshiftMasterConfig.ServingInfo.BindAddress, version.Get().String()) glog.Infof("Public master address is %s", openshiftMasterConfig.AssetConfig.MasterPublicURL) if len(openshiftMasterConfig.DisabledFeatures) > 0 { glog.V(4).Infof("Disabled features: %s", strings.Join(openshiftMasterConfig.DisabledFeatures, ", ")) } if openshiftMasterConfig.EtcdConfig != nil { etcd.RunEtcd(openshiftMasterConfig.EtcdConfig) } // Allow privileged containers // TODO: make this configurable and not the default https://github.com/openshift/origin/issues/662 capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: true, HostNetworkSources: []string{kubelet.ApiserverSource, kubelet.FileSource}, }) openshiftConfig, err := origin.BuildMasterConfig(*openshiftMasterConfig) if err != nil { return err } go func() { openshiftConfig.ControllerPlug.WaitForStop() glog.Fatalf("Master shutdown requested") }() // Must start policy caching immediately openshiftConfig.RunGroupCache() openshiftConfig.RunPolicyCache() openshiftConfig.RunProjectCache() unprotectedInstallers := []origin.APIInstaller{} if openshiftMasterConfig.OAuthConfig != nil { authConfig, err := origin.BuildAuthConfig(*openshiftMasterConfig) if err != nil { return err } unprotectedInstallers = append(unprotectedInstallers, authConfig) } var standaloneAssetConfig *origin.AssetConfig if openshiftMasterConfig.AssetConfig != nil { config, err := origin.BuildAssetConfig(*openshiftMasterConfig.AssetConfig) if err != nil { return err } if openshiftMasterConfig.AssetConfig.ServingInfo.BindAddress == openshiftMasterConfig.ServingInfo.BindAddress { unprotectedInstallers = append(unprotectedInstallers, config) } else { standaloneAssetConfig = config } } var kubeConfig *kubernetes.MasterConfig if openshiftMasterConfig.KubernetesMasterConfig != nil { kubeConfig, err = kubernetes.BuildKubernetesMasterConfig(*openshiftMasterConfig, openshiftConfig.RequestContextMapper, openshiftConfig.KubeClient()) if err != nil { return err } openshiftConfig.Run([]origin.APIInstaller{kubeConfig}, unprotectedInstallers) } else { _, kubeConfig, err := configapi.GetKubeClient(openshiftMasterConfig.MasterClients.ExternalKubernetesKubeConfig) if err != nil { return err } proxy := &kubernetes.ProxyConfig{ ClientConfig: kubeConfig, } openshiftConfig.Run([]origin.APIInstaller{proxy}, unprotectedInstallers) } glog.Infof("Using images from %q", openshiftConfig.ImageFor("<component>")) if standaloneAssetConfig != nil { standaloneAssetConfig.Run() } if openshiftMasterConfig.DNSConfig != nil { openshiftConfig.RunDNSServer() } openshiftConfig.RunProjectAuthorizationCache() if openshiftMasterConfig.Controllers != configapi.ControllersDisabled { go func() { openshiftConfig.ControllerPlug.WaitForStart() glog.Infof("Master controllers starting (%s)", openshiftMasterConfig.Controllers) // Start these first, because they provide credentials for other controllers' clients openshiftConfig.RunServiceAccountsController() openshiftConfig.RunServiceAccountTokensController() // used by admission controllers openshiftConfig.RunServiceAccountPullSecretsControllers() openshiftConfig.RunSecurityAllocationController() if kubeConfig != nil { _, rcClient, err := openshiftConfig.GetServiceAccountClients(openshiftConfig.ReplicationControllerServiceAccount) if err != nil { glog.Fatalf("Could not get client for replication controller: %v", err) } // called by admission control kubeConfig.RunResourceQuotaManager() // no special order kubeConfig.RunNodeController() kubeConfig.RunScheduler() kubeConfig.RunReplicationController(rcClient) kubeConfig.RunEndpointController() kubeConfig.RunNamespaceController() kubeConfig.RunPersistentVolumeClaimBinder() kubeConfig.RunPersistentVolumeClaimRecycler(openshiftConfig.ImageFor("deployer")) } // no special order openshiftConfig.RunBuildController() openshiftConfig.RunBuildPodController() openshiftConfig.RunBuildImageChangeTriggerController() openshiftConfig.RunDeploymentController() openshiftConfig.RunDeployerPodController() openshiftConfig.RunDeploymentConfigController() openshiftConfig.RunDeploymentConfigChangeController() openshiftConfig.RunDeploymentImageChangeTriggerController() openshiftConfig.RunImageImportController() openshiftConfig.RunOriginNamespaceController() openshiftConfig.RunSDNController() }() } return nil } func (o MasterOptions) IsWriteConfigOnly() bool { return o.MasterArgs.ConfigDir.Provided() } func (o MasterOptions) IsRunFromConfig() bool { return (len(o.ConfigFile) > 0) }