From 6de826b35ced3b7cadc809d7ea778ce6a50aff43 Mon Sep 17 00:00:00 2001 From: DheerajSShetty Date: Wed, 22 Aug 2018 16:30:37 -0700 Subject: [PATCH] VKE patch for k8s version 1.9 (fbdcc5c) --- api/swagger-spec/apps_v1alpha1.json | Bin 135734 -> 136495 bytes api/swagger-spec/apps_v1beta1.json | Bin 311661 -> 312422 bytes api/swagger-spec/apps_v1beta2.json | Bin 425385 -> 426146 bytes api/swagger-spec/batch_v1.json | Bin 182308 -> 183069 bytes api/swagger-spec/batch_v1beta1.json | Bin 185598 -> 186359 bytes api/swagger-spec/batch_v2alpha1.json | Bin 185625 -> 186386 bytes api/swagger-spec/extensions_v1beta1.json | Bin 473155 -> 473916 bytes api/swagger-spec/settings.k8s.io_v1alpha1.json | Bin 128260 -> 129021 bytes api/swagger-spec/v1.json | Bin 988673 -> 989618 bytes cmd/kube-apiserver/app/options/plugins.go | 2 + cmd/kube-controller-manager/app/BUILD | 1 + cmd/kube-controller-manager/app/plugins.go | 4 + cmd/kubelet/app/BUILD | 1 + cmd/kubelet/app/plugins.go | 2 + pkg/apis/core/types.go | 14 + pkg/apis/core/validation/validation.go | 25 + pkg/apis/extensions/types.go | 1 + pkg/cloudprovider/providers/BUILD | 2 + pkg/cloudprovider/providers/cascade/BUILD | 56 ++ pkg/cloudprovider/providers/cascade/OWNERS | 3 + pkg/cloudprovider/providers/cascade/apitypes.go | 230 +++++ pkg/cloudprovider/providers/cascade/auth.go | 145 ++++ pkg/cloudprovider/providers/cascade/cascade.go | 219 +++++ .../providers/cascade/cascade_disks.go | 226 +++++ .../providers/cascade/cascade_instances.go | 91 ++ .../providers/cascade/cascade_instances_test.go | 43 + .../providers/cascade/cascade_loadbalancer.go | 284 +++++++ pkg/cloudprovider/providers/cascade/client.go | 399 +++++++++ pkg/cloudprovider/providers/cascade/oidcclient.go | 297 +++++++ pkg/cloudprovider/providers/cascade/restclient.go | 262 ++++++ pkg/cloudprovider/providers/cascade/tests_owed | 5 + pkg/cloudprovider/providers/cascade/utils.go | 29 + pkg/cloudprovider/providers/providers.go | 1 + pkg/kubeapiserver/authorizer/config.go | 8 +- pkg/kubeapiserver/authorizer/modes/modes.go | 3 +- pkg/printers/internalversion/describe.go | 11 + pkg/security/podsecuritypolicy/util/util.go | 3 + pkg/volume/cascade_disk/BUILD | 43 + pkg/volume/cascade_disk/OWNERS | 2 + pkg/volume/cascade_disk/attacher.go | 265 ++++++ pkg/volume/cascade_disk/azure_disk_util.go | 136 +++ pkg/volume/cascade_disk/cascade_disk.go | 391 +++++++++ pkg/volume/cascade_disk/cascade_util.go | 201 +++++ .../admission/persistentvolume/label/admission.go | 54 ++ plugin/pkg/admission/vke/BUILD | 61 ++ plugin/pkg/admission/vke/admission.go | 587 +++++++++++++ plugin/pkg/admission/vke/admission_test.go | 941 +++++++++++++++++++++ plugin/pkg/auth/authorizer/vke/BUILD | 40 + plugin/pkg/auth/authorizer/vke/OWNERS | 3 + plugin/pkg/auth/authorizer/vke/vke_authorizer.go | 123 +++ .../pkg/auth/authorizer/vke/vke_authorizer_test.go | 230 +++++ staging/src/k8s.io/api/core/v1/generated.pb.go | Bin 1241955 -> 1248240 bytes staging/src/k8s.io/api/core/v1/types.go | 26 +- 53 files changed, 5462 insertions(+), 8 deletions(-) diff --git a/api/swagger-spec/apps_v1alpha1.json b/api/swagger-spec/apps_v1alpha1.json index aa3fbdc..0189f38 100644 --- a/api/swagger-spec/apps_v1alpha1.json +++ b/api/swagger-spec/apps_v1alpha1.json @@ -1459,6 +1459,10 @@ "photonPersistentDisk": { "$ref": "v1.PhotonPersistentDiskVolumeSource", "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" + }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" } } }, @@ -2109,6 +2113,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.Container": { "id": "v1.Container", "description": "A single application container that you want to run within a pod.", diff --git a/api/swagger-spec/apps_v1beta1.json b/api/swagger-spec/apps_v1beta1.json index e253317..c1fa812 100644 --- a/api/swagger-spec/apps_v1beta1.json +++ b/api/swagger-spec/apps_v1beta1.json @@ -4479,6 +4479,10 @@ "$ref": "v1.PhotonPersistentDiskVolumeSource", "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" + }, "projected": { "$ref": "v1.ProjectedVolumeSource", "description": "Items for all in one resources secrets, configmaps, and downward API" @@ -5206,6 +5210,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.ProjectedVolumeSource": { "id": "v1.ProjectedVolumeSource", "description": "Represents a projected volume source", diff --git a/api/swagger-spec/apps_v1beta2.json b/api/swagger-spec/apps_v1beta2.json index be42788..5abb9f5 100644 --- a/api/swagger-spec/apps_v1beta2.json +++ b/api/swagger-spec/apps_v1beta2.json @@ -6845,6 +6845,10 @@ "$ref": "v1.PhotonPersistentDiskVolumeSource", "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" + }, "projected": { "$ref": "v1.ProjectedVolumeSource", "description": "Items for all in one resources secrets, configmaps, and downward API" @@ -7572,6 +7576,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.ProjectedVolumeSource": { "id": "v1.ProjectedVolumeSource", "description": "Represents a projected volume source", diff --git a/api/swagger-spec/batch_v1.json b/api/swagger-spec/batch_v1.json index 28787d8..28fcb65 100644 --- a/api/swagger-spec/batch_v1.json +++ b/api/swagger-spec/batch_v1.json @@ -1819,6 +1819,10 @@ "$ref": "v1.PhotonPersistentDiskVolumeSource", "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" + }, "projected": { "$ref": "v1.ProjectedVolumeSource", "description": "Items for all in one resources secrets, configmaps, and downward API" @@ -2546,6 +2550,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.ProjectedVolumeSource": { "id": "v1.ProjectedVolumeSource", "description": "Represents a projected volume source", diff --git a/api/swagger-spec/batch_v1beta1.json b/api/swagger-spec/batch_v1beta1.json index bb9b870..f67a014 100644 --- a/api/swagger-spec/batch_v1beta1.json +++ b/api/swagger-spec/batch_v1beta1.json @@ -1874,6 +1874,10 @@ "$ref": "v1.PhotonPersistentDiskVolumeSource", "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" + }, "projected": { "$ref": "v1.ProjectedVolumeSource", "description": "Items for all in one resources secrets, configmaps, and downward API" @@ -2601,6 +2605,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.ProjectedVolumeSource": { "id": "v1.ProjectedVolumeSource", "description": "Represents a projected volume source", diff --git a/api/swagger-spec/batch_v2alpha1.json b/api/swagger-spec/batch_v2alpha1.json index cde6619..78888be 100644 --- a/api/swagger-spec/batch_v2alpha1.json +++ b/api/swagger-spec/batch_v2alpha1.json @@ -1889,6 +1889,10 @@ "storageos": { "$ref": "v1.StorageOSVolumeSource", "description": "StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes." + }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" } } }, @@ -2793,6 +2797,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.Container": { "id": "v1.Container", "description": "A single application container that you want to run within a pod.", diff --git a/api/swagger-spec/extensions_v1beta1.json b/api/swagger-spec/extensions_v1beta1.json index d8b20a3..73342a1 100644 --- a/api/swagger-spec/extensions_v1beta1.json +++ b/api/swagger-spec/extensions_v1beta1.json @@ -7502,6 +7502,10 @@ "storageos": { "$ref": "v1.StorageOSVolumeSource", "description": "StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes." + }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" } } }, @@ -8214,6 +8218,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.ProjectedVolumeSource": { "id": "v1.ProjectedVolumeSource", "description": "Represents a projected volume source", diff --git a/api/swagger-spec/settings.k8s.io_v1alpha1.json b/api/swagger-spec/settings.k8s.io_v1alpha1.json index dc442a8..8c1f100 100644 --- a/api/swagger-spec/settings.k8s.io_v1alpha1.json +++ b/api/swagger-spec/settings.k8s.io_v1alpha1.json @@ -1676,6 +1676,10 @@ "storageos": { "$ref": "v1.StorageOSVolumeSource", "description": "StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes." + }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" } } }, @@ -2350,6 +2354,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.ProjectedVolumeSource": { "id": "v1.ProjectedVolumeSource", "description": "Represents a projected volume source", diff --git a/api/swagger-spec/v1.json b/api/swagger-spec/v1.json index 97be62b..df544c8 100644 --- a/api/swagger-spec/v1.json +++ b/api/swagger-spec/v1.json @@ -20629,6 +20629,10 @@ "$ref": "v1.PhotonPersistentDiskVolumeSource", "description": "PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine" }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" + }, "portworxVolume": { "$ref": "v1.PortworxVolumeSource", "description": "PortworxVolume represents a portworx volume attached and mounted on kubelets host machine" @@ -21200,6 +21204,23 @@ } } }, + "v1.CascadeDiskVolumeSource": { + "id": "v1.CascadeDiskVolumeSource", + "description": "Represents a Cascade persistent disk resource.", + "required": [ + "diskID" + ], + "properties": { + "diskID": { + "type": "string", + "description": "ID that identifies Cascade persistent disk" + }, + "fsType": { + "type": "string", + "description": "Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified." + } + } + }, "v1.PortworxVolumeSource": { "id": "v1.PortworxVolumeSource", "description": "PortworxVolumeSource represents a Portworx volume resource.", @@ -21657,6 +21678,10 @@ "storageos": { "$ref": "v1.StorageOSVolumeSource", "description": "StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes." + }, + "vkeDisk": { + "$ref": "v1.CascadeDiskVolumeSource", + "description": "vkeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine" } } }, diff --git a/cmd/kube-apiserver/app/options/plugins.go b/cmd/kube-apiserver/app/options/plugins.go index a0d2502..4fe32e4 100644 --- a/cmd/kube-apiserver/app/options/plugins.go +++ b/cmd/kube-apiserver/app/options/plugins.go @@ -52,6 +52,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/admission/securitycontext/scdeny" "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount" "k8s.io/kubernetes/plugin/pkg/admission/storageclass/setdefault" + "k8s.io/kubernetes/plugin/pkg/admission/vke" ) // RegisterAllAdmissionPlugins registers all admission plugins @@ -83,4 +84,5 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { setdefault.Register(plugins) resize.Register(plugins) pvcprotection.Register(plugins) + vke.Register(plugins) } diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index c518b36..f1a91f6 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -86,6 +86,7 @@ go_library( "//pkg/volume/aws_ebs:go_default_library", "//pkg/volume/azure_dd:go_default_library", "//pkg/volume/azure_file:go_default_library", + "//pkg/volume/cascade_disk:go_default_library", "//pkg/volume/cinder:go_default_library", "//pkg/volume/csi:go_default_library", "//pkg/volume/fc:go_default_library", diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index 170c366..5fa1cb1 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/volume/aws_ebs" "k8s.io/kubernetes/pkg/volume/azure_dd" "k8s.io/kubernetes/pkg/volume/azure_file" + "k8s.io/kubernetes/pkg/volume/cascade_disk" "k8s.io/kubernetes/pkg/volume/cinder" "k8s.io/kubernetes/pkg/volume/csi" "k8s.io/kubernetes/pkg/volume/fc" @@ -77,6 +78,7 @@ func ProbeAttachableVolumePlugins() []volume.VolumePlugin { allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...) allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, cascade_disk.ProbeVolumePlugins()...) if utilfeature.DefaultFeatureGate.Enabled(features.CSIPersistentVolume) { allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) } @@ -106,6 +108,7 @@ func ProbeExpandableVolumePlugins(config componentconfig.VolumeConfiguration) [] allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, cascade_disk.ProbeVolumePlugins()...) if !utilfeature.DefaultFeatureGate.Enabled(features.CSIPersistentVolume) { allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) } @@ -165,6 +168,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...) allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...) allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, cascade_disk.ProbeVolumePlugins()...) return allPlugins } diff --git a/cmd/kubelet/app/BUILD b/cmd/kubelet/app/BUILD index c7e482f..0d3645a 100644 --- a/cmd/kubelet/app/BUILD +++ b/cmd/kubelet/app/BUILD @@ -74,6 +74,7 @@ go_library( "//pkg/volume/aws_ebs:go_default_library", "//pkg/volume/azure_dd:go_default_library", "//pkg/volume/azure_file:go_default_library", + "//pkg/volume/cascade_disk:go_default_library", "//pkg/volume/cephfs:go_default_library", "//pkg/volume/cinder:go_default_library", "//pkg/volume/configmap:go_default_library", diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index ef41bb8..c9806f7 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/pkg/volume/aws_ebs" "k8s.io/kubernetes/pkg/volume/azure_dd" "k8s.io/kubernetes/pkg/volume/azure_file" + "k8s.io/kubernetes/pkg/volume/cascade_disk" "k8s.io/kubernetes/pkg/volume/cephfs" "k8s.io/kubernetes/pkg/volume/cinder" "k8s.io/kubernetes/pkg/volume/configmap" @@ -100,6 +101,7 @@ func ProbeVolumePlugins() []volume.VolumePlugin { allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...) allPlugins = append(allPlugins, local.ProbeVolumePlugins()...) allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...) + allPlugins = append(allPlugins, cascade_disk.ProbeVolumePlugins()...) if utilfeature.DefaultFeatureGate.Enabled(features.CSIPersistentVolume) { allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) } diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index b6b570b..1d9db5e 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -316,6 +316,8 @@ type VolumeSource struct { // StorageOS represents a StorageOS volume that is attached to the kubelet's host machine and mounted into the pod // +optional StorageOS *StorageOSVolumeSource + // CascadeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine + CascadeDisk *CascadeDiskVolumeSource } // Similar to VolumeSource but meant for the administrator who creates PVs. @@ -394,6 +396,8 @@ type PersistentVolumeSource struct { // CSI (Container Storage Interface) represents storage that handled by an external CSI driver // +optional CSI *CSIPersistentVolumeSource + // CascadeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine + CascadeDisk *CascadeDiskVolumeSource } type PersistentVolumeClaimVolumeSource struct { @@ -1471,6 +1475,16 @@ type StorageOSPersistentVolumeSource struct { SecretRef *ObjectReference } +// Represents a Cascade persistent disk resource. +type CascadeDiskVolumeSource struct { + // ID that identifies Cascade persistent disk + DiskID string + // Filesystem type to mount. + // Must be a filesystem type supported by the host operating system. + // Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + FSType string +} + // Adapts a ConfigMap into a volume. // // The contents of the target ConfigMap's Data field will be presented in a diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index f6ab55f..2fa447d 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -681,6 +681,14 @@ func validateVolumeSource(source *core.VolumeSource, fldPath *field.Path, volNam allErrs = append(allErrs, validateScaleIOVolumeSource(source.ScaleIO, fldPath.Child("scaleIO"))...) } } + if source.CascadeDisk != nil { + if numVolumes > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("cascadeDisk"), "may not specify more than 1 volume type")) + } else { + numVolumes++ + allErrs = append(allErrs, validateCascadeDiskVolumeSource(source.CascadeDisk, fldPath.Child("cascadeDisk"))...) + } + } if numVolumes == 0 { allErrs = append(allErrs, field.Required(fldPath, "must specify a volume type")) @@ -1440,6 +1448,14 @@ func validateCSIPersistentVolumeSource(csi *core.CSIPersistentVolumeSource, fldP return allErrs } +func validateCascadeDiskVolumeSource(cd *core.CascadeDiskVolumeSource, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if len(cd.DiskID) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("diskID"), "")) + } + return allErrs +} + // ValidatePersistentVolumeName checks that a name is appropriate for a // PersistentVolumeName object. var ValidatePersistentVolumeName = NameIsDNSSubdomain @@ -1674,6 +1690,15 @@ func ValidatePersistentVolume(pv *core.PersistentVolume) field.ErrorList { } } + if pv.Spec.CascadeDisk != nil { + if numVolumes > 0 { + allErrs = append(allErrs, field.Forbidden(specPath.Child("cascadeDisk"), "may not specify more than 1 volume type")) + } else { + numVolumes++ + allErrs = append(allErrs, validateCascadeDiskVolumeSource(pv.Spec.CascadeDisk, specPath.Child("cascadeDisk"))...) + } + } + if numVolumes == 0 { allErrs = append(allErrs, field.Required(specPath, "must specify a volume type")) } diff --git a/pkg/apis/extensions/types.go b/pkg/apis/extensions/types.go index e369728..a5406ab 100644 --- a/pkg/apis/extensions/types.go +++ b/pkg/apis/extensions/types.go @@ -925,6 +925,7 @@ var ( PortworxVolume FSType = "portworxVolume" ScaleIO FSType = "scaleIO" CSI FSType = "csi" + CascadeDisk FSType = "cascadeDisk" All FSType = "*" ) diff --git a/pkg/cloudprovider/providers/BUILD b/pkg/cloudprovider/providers/BUILD index aeccfa1..4313576 100644 --- a/pkg/cloudprovider/providers/BUILD +++ b/pkg/cloudprovider/providers/BUILD @@ -12,6 +12,7 @@ go_library( deps = [ "//pkg/cloudprovider/providers/aws:go_default_library", "//pkg/cloudprovider/providers/azure:go_default_library", + "//pkg/cloudprovider/providers/cascade:go_default_library", "//pkg/cloudprovider/providers/cloudstack:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", "//pkg/cloudprovider/providers/openstack:go_default_library", @@ -34,6 +35,7 @@ filegroup( ":package-srcs", "//pkg/cloudprovider/providers/aws:all-srcs", "//pkg/cloudprovider/providers/azure:all-srcs", + "//pkg/cloudprovider/providers/cascade:all-srcs", "//pkg/cloudprovider/providers/cloudstack:all-srcs", "//pkg/cloudprovider/providers/fake:all-srcs", "//pkg/cloudprovider/providers/gce:all-srcs", diff --git a/pkg/cloudprovider/providers/cascade/BUILD b/pkg/cloudprovider/providers/cascade/BUILD new file mode 100644 index 0000000..4089166 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/BUILD @@ -0,0 +1,56 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = [ + "apitypes.go", + "auth.go", + "cascade.go", + "cascade_disks.go", + "cascade_instances.go", + "cascade_loadbalancer.go", + "client.go", + "oidcclient.go", + "restclient.go", + "utils.go" + ], + deps = [ + "//pkg/api/v1/helper:go_default_library", + "//pkg/cloudprovider:go_default_library", + "//pkg/controller:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/gopkg.in/gcfg.v1:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "cascade_instances_test.go", + ], + embed = [":go_default_library"], + deps = [ + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/cloudprovider/providers/cascade/OWNERS b/pkg/cloudprovider/providers/cascade/OWNERS new file mode 100644 index 0000000..70efc9d --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/OWNERS @@ -0,0 +1,3 @@ +maintainers: +- ashokc +- ysheng diff --git a/pkg/cloudprovider/providers/cascade/apitypes.go b/pkg/cloudprovider/providers/cascade/apitypes.go new file mode 100644 index 0000000..d437394 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/apitypes.go @@ -0,0 +1,230 @@ +package cascade + +import "fmt" + +const ( + NotFoundError = 1408 + VMNotFoundError = 2006 + DiskNotFoundError = 3011 + DiskInUseError = 3012 + + DiskStateAttached = "ATTACHED" +) + +// Represents APIError returned by the API in case of an error. +type APIError struct { + Code *string `json:"code"` + Data map[string]string `json:"data"` + ErrorCode int32 `json:"errorCode,omitempty"` + Message *string `json:"message"` + HttpStatusCode int `json:"-"` // Not part of API contract +} + +// Implement Go error interface for ApiError. +func (e APIError) Error() string { + return fmt.Sprintf( + "Cascade: { HTTP status: '%d', code: '%s', message: '%s', data: '%v', errorcode: '%d' }", + e.HttpStatusCode, StringVal(e.Code), StringVal(e.Message), e.Data, e.ErrorCode) +} + +// Used to represent a generic HTTP error, i.e. an unexpected HTTP 500. +type HttpError struct { + StatusCode int + Message string +} + +// Implementation of error interface for HttpError. +func (e HttpError) Error() string { + return fmt.Sprintf("Cascade: HTTP %d: %v", e.StatusCode, e.Message) +} + +// Represents a task which gets returned for long running API calls. +type Task struct { + EndTime int64 `json:"endTime,omitempty"` + Entity *Entity `json:"entity,omitempty"` + ID *string `json:"id"` + Operation string `json:"operation,omitempty"` + QueuedTime *int64 `json:"queuedTime"` + ResourceProperties interface{} `json:"resourceProperties,omitempty"` + SelfLink string `json:"selfLink,omitempty"` + StartedTime *int64 `json:"startedTime"` + State *string `json:"state"` + Steps []*Step `json:"steps"` +} + +// Represents the entity associated with the task. +type Entity struct { + ID *string `json:"id"` + Kind *string `json:"kind"` +} + +// Represents a task that has entered into an error state. Task errors can be caught and type-checked against with the +// usual Go idiom. +type TaskError struct { + ID string `json:"id"` + Step Step `json:"step,omitempty"` +} + +// Implement Go error interface for TaskError. +func (e TaskError) Error() string { + return fmt.Sprintf("Cascade: Task '%s' is in error state: {@step==%s}", e.ID, GetStep(e.Step)) +} + +// An error representing a timeout while waiting for a task to complete. +type TaskTimeoutError struct { + ID string +} + +// Implement Go error interface for TaskTimeoutError. +func (e TaskTimeoutError) Error() string { + return fmt.Sprintf("Cascade: Timed out waiting for task '%s'. "+ + "Task may not be in error state, examine task for full details.", e.ID) +} + +// Represents a step in a task. +type Step struct { + EndTime int64 `json:"endTime,omitempty"` + Errors []*APIError `json:"errors"` + Operation string `json:"operation,omitempty"` + Options map[string]string `json:"options,omitempty"` + QueuedTime *int64 `json:"queuedTime"` + Sequence int32 `json:"sequence,omitempty"` + StartedTime *int64 `json:"startedTime"` + State *string `json:"state"` + Warnings []*APIError `json:"warnings"` +} + +// Implement Go error interface for Step. +func GetStep(s Step) string { + return fmt.Sprintf("{\"operation\"=>\"%s\",\"state\"=>\"%s}", s.Operation, StringVal(s.State)) +} + +// Represents the VM response returned by the API. +type VM struct { + AttachedDisks []*AttachedDisk `json:"attachedDisks"` + Cost []*QuotaLineItem `json:"cost"` + Flavor *string `json:"flavor"` + FloatingIP string `json:"floatingIp,omitempty"` + HighAvailableVMGroupID string `json:"highAvailableVMGroupID,omitempty"` + ID *string `json:"id"` + Kind string `json:"kind"` + Name *string `json:"name"` + SelfLink string `json:"selfLink,omitempty"` + SourceImageID string `json:"sourceImageId,omitempty"` + State *string `json:"state"` + Subnets []string `json:"subnets"` + Tags []string `json:"tags"` +} + +// Represents the listVMs response returned by the API. +type VMList struct { + Items []*VM `json:"items"` + NextPageLink string `json:"nextPageLink,omitempty"` + PreviousPageLink string `json:"previousPageLink,omitempty"` +} + +// Represents multiple VMs returned by the API. +type VMs struct { + Items []VM `json:"items"` +} + +// Represents the disks attached to the VMs. +type AttachedDisk struct { + BootDisk *bool `json:"bootDisk"` + CapacityGb *int32 `json:"capacityGb"` + Flavor *string `json:"flavor"` + ID *string `json:"id"` + Kind *string `json:"kind"` + Name *string `json:"name"` + State *string `json:"state"` +} + +// Represents an attach disk operation request. +type VMDiskOperation struct { + Arguments map[string]string `json:"arguments,omitempty"` + DiskID *string `json:"diskId"` +} + +// Represents the quota line items for the VM. +type QuotaLineItem struct { + Key *string `json:"key"` + Unit *string `json:"unit"` + Value *float64 `json:"value"` +} + +// Represents a persistent disk +type PersistentDisk struct { + CapacityGB int32 `json:"capacityGb,omitempty"` + Cost []*QuotaLineItem `json:"cost"` + Datastore string `json:"datastore,omitempty"` + Flavor *string `json:"flavor"` + ID *string `json:"id"` + Kind string `json:"kind"` + Name *string `json:"name"` + SelfLink string `json:"selfLink,omitempty"` + State *string `json:"state"` + Tags []string `json:"tags"` + VM string `json:"vm"` + MountDevice string `json:"mountDevice,omitempty"` + Zone *string `json:"zone"` +} + +// Represents the spec for creating a disk. +type DiskCreateSpec struct { + Affinities []*LocalitySpec `json:"affinities"` + CapacityGB *int32 `json:"capacityGb"` + Flavor *string `json:"flavor"` + Kind *string `json:"kind"` + Name *string `json:"name"` + Tags []string `json:"tags"` + Zone *string `json:"zone"` + Encrypted *bool `json:"encrypted"` +} + +// Represents the spec for specifying affinity for a disk with another entity. +type LocalitySpec struct { + ID *string `json:"id"` + Kind *string `json:"kind"` +} + +// Represens the LoadBalancer response returned by the API. +type LoadBalancer struct { + Endpoint *string `json:"endpoint"` +} + +// Represents the spec for creating a LoadBalancer. +type LoadBalancerCreateSpec struct { + HealthCheck *LoadBalancerHealthCheck `json:"healthCheck"` + Name *string `json:"name"` + PortMaps []*LoadBalancerPortMap `json:"portMaps"` + Type *string `json:"type"` + SubDomain *string `json:"subDomain"` +} + +// Represents the health check spec for a load balancer. +type LoadBalancerHealthCheck struct { + HealthyThreshold int64 `json:"healthyThreshold,omitempty"` + IntervalInSeconds int64 `json:"intervalInSeconds,omitempty"` + Path *string `json:"path,omitempty"` + Port *int64 `json:"port"` + Protocol *string `json:"protocol"` +} + +// Represents a port mapping spec for a load balancer. +type LoadBalancerPortMap struct { + AllowedCidrs []*string `json:"allowedCidrs"` + InstancePort *int64 `json:"instancePort"` + InstanceProtocol *string `json:"instanceProtocol"` + LoadBalancerPort *int64 `json:"loadBalancerPort"` + LoadBalancerProtocol *string `json:"loadBalancerProtocol"` +} + +// Represents a VM to be registered with or deregistered from the load balancer. +type LoadBalancerVM struct { + ID *string `json:"id"` +} + +// Represents a list of VMs to be registered with or deregistered from the load balancer. +type LoadBalancerVMUpdate struct { + VMIds []*LoadBalancerVM `json:"vmIds"` +} diff --git a/pkg/cloudprovider/providers/cascade/auth.go b/pkg/cloudprovider/providers/cascade/auth.go new file mode 100644 index 0000000..fc92377 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/auth.go @@ -0,0 +1,145 @@ +package cascade + +import ( + "fmt" + "strings" + "github.com/golang/glog" + "os/exec" +) + +const ( + tScope = "openid offline_access rs_admin_server at_groups rs_vmdir" + + afdCli = "/opt/vmware/bin/vmafd-cli" + afdCliMachineAccountCmd = "get-machine-account-info" + afdCliPasswordPrefix = "Password: " + afdCliSeparator = "\n" +) + +// AuthConfig contains configuration information for the authentication client. +type AuthConfig struct { + tenantName string + authEndpoint string + machineAccountName string +} + +// AuthClient defines functions related to authentication. +type AuthClient struct { + cfg *AuthConfig +} + +// NewAuthClient creates a new authentication client +func NewAuthClient(cascadeCfg *CascadeConfig) (*AuthClient, error) { + return &AuthClient{ + cfg: &AuthConfig{ + tenantName: cascadeCfg.Global.TenantName, + authEndpoint: cascadeCfg.Global.AuthEndpoint, + machineAccountName: fmt.Sprintf("%s@%s", cascadeCfg.Global.DNSName, cascadeCfg.Global.DomainName), + }, + }, nil +} + +func (c *AuthClient) GetTokensByMachineAccount() (*TokenOptions, error) { + // Use the VMAFD CLI to get the machine account password + cmd := exec.Command(afdCli, afdCliMachineAccountCmd) + output, err := cmd.Output() + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to get machine account credentials. Cannot create Client.") + return nil, fmt.Errorf("Failed to get machine account credentials, err: %v", err) + } + + password, err := parseMachineAccountInfo(output) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to parse machine account credentials. Cannot create Client.") + return nil, fmt.Errorf("Failed to parse machine account credentials, err: %v", err) + } + + return c.GetTokensByCredentials(c.cfg.machineAccountName, password) +} + +// GetTokensByPassword gets tokens using username and password +func (c *AuthClient) GetTokensByCredentials(username, password string) (*TokenOptions, error) { + // Parse tenant part from username + parts := strings.Split(username, "@") + if len(parts) != 2 { + return nil, fmt.Errorf("Invalid full user name '%s': expected user@tenant", username) + } + tenant := parts[1] + + oidcClient, err := buildOIDCClient(c.cfg.authEndpoint) + if err != nil { + return nil, err + } + + tokenResponse, err := oidcClient.GetTokenByPasswordGrant(tenant, username, password) + if err != nil { + return nil, err + } + + return toTokenOptions(tokenResponse), nil +} + +// GetTokensByRefreshToken gets tokens using refresh token +func (c *AuthClient) GetTokensByRefreshToken(refreshtoken string) (*TokenOptions, error) { + oidcClient, err := buildOIDCClient(c.cfg.authEndpoint) + if err != nil { + return nil, err + } + + tokenResponse, err := oidcClient.GetTokenByRefreshTokenGrant(c.cfg.tenantName, refreshtoken) + if err != nil { + return nil, err + } + + return toTokenOptions(tokenResponse), nil +} + +func buildOIDCClient(authEndpoint string) (*OIDCClient, error) { + options := &OIDCClientOptions{ + IgnoreCertificate: false, + RootCAs: nil, + TokenScope: tScope, + } + + return NewOIDCClient(authEndpoint, options, nil), nil +} + +func toTokenOptions(response *OIDCTokenResponse) *TokenOptions { + return &TokenOptions{ + AccessToken: response.AccessToken, + ExpiresIn: response.ExpiresIn, + RefreshToken: response.RefreshToken, + IDToken: response.IDToken, + TokenType: response.TokenType, + } +} + +// parseMachineAccountInfo parses the machine account password from the machine-account-info output which looks like +// this: +//MachineAccount: photon-8rwdscr1.lw-testdom.com +//Password: FT`])}]d/3\EPwRpz9k1 +func parseMachineAccountInfo(output []byte) (string, error) { + if len(output) <= 0 { + return "", fmt.Errorf("account info is not specified") + } + + strOut := string(output) + strOutLen := len(strOut) + + pwdStart := strings.Index(strOut, afdCliPasswordPrefix) + if pwdStart < 0 { + return "", fmt.Errorf("account info is not in expected format") + } + pwdStart = pwdStart + len(afdCliPasswordPrefix) + if pwdStart >= strOutLen { + return "", fmt.Errorf("account info is not in expected format") + } + pwdEnd := strings.LastIndex(strOut, afdCliSeparator) + if pwdEnd < 0 || pwdEnd <= pwdStart || pwdEnd >= strOutLen { + return "", fmt.Errorf("account info is not in expected format") + } + + pwd := strOut[pwdStart:pwdEnd] + + return pwd, nil +} \ No newline at end of file diff --git a/pkg/cloudprovider/providers/cascade/cascade.go b/pkg/cloudprovider/providers/cascade/cascade.go new file mode 100644 index 0000000..ba42576 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/cascade.go @@ -0,0 +1,219 @@ +// The use of Cascade cloud provider requires the kubelet, kube-apiserver, and kube-controller-manager to be started +// with config flag: '--cloud-provider=cascade --cloud-config=[path_to_config_file]'. +package cascade + +import ( + "errors" + "fmt" + "github.com/golang/glog" + "gopkg.in/gcfg.v1" + "io" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/controller" + "os" + "strings" +) + +const ( + ProviderName = "vke" + DiskSpecKind = "persistent-disk" + MasterPrefix = "master" +) + +// CascadeCloud is an implementation of the cloud provider interface for Cascade Controller. +type CascadeCloud struct { + cfg *CascadeConfig + // Authentication client to get token for Cascade API calls + authClient *AuthClient + // API Client to make Cascade API calls + apiClient *Client + // local $HOSTNAME + localHostname string + // hostname from K8S, could be overridden + localK8sHostname string +} + +// CascadeCloud represents Cascade cloud provider's configuration. +type CascadeConfig struct { + Global struct { + // the Cascade Controller endpoint + CloudTarget string `gcfg:"target"` + // Cascade Controller tenantName name + TenantName string `gcfg:"tenantName"` + // Cascade Controller cluster ID + ClusterID string `gcfg:"clusterID"` + // Authentication server endpoint for Cascade Controller + AuthEndpoint string `gcfg:"authEndpoint"` + // Lightwave domain name for the node + DomainName string `gcfg:"domainName"` + // DNS name of the node. + DNSName string `gcfg:"dnsName"` + // Region in which the cluster is in + Region string `gcfg:"region"` + // Availability zone in which the cluster is in + Zone string `gcfg:"zone"` + // IP address of the node. + IPAddress string `gcfg:"ipAddress"` + } +} + +// Disks is interface for manipulation with Cascade Controller Persistent Disks. +type Disks interface { + // AttachDisk attaches given disk to given node. Current node + // is used when nodeName is empty string. + AttachDisk(diskID string, nodeName k8stypes.NodeName) (string, error) + + // DetachDisk detaches given disk to given node. Current node + // is used when nodeName is empty string. + DetachDisk(diskID string, nodeName k8stypes.NodeName) error + + // DiskIsAttached checks if a disk is attached to the given node. + DiskIsAttached(diskID string, nodeName k8stypes.NodeName) (bool, error) + + // DisksAreAttached is a batch function to check if a list of disks are attached + // to the node with the specified NodeName. + DisksAreAttached(diskID []string, nodeName k8stypes.NodeName) (map[string]bool, error) + + // CreateDisk creates a new PD with given properties. + CreateDisk(volumeOptions *VolumeOptions) (diskID string, err error) + + // DeleteDisk deletes PD. + DeleteDisk(diskID string) error + + // Get labels to apply to volume on creation. + GetVolumeLabels(diskID string) (map[string]string, error) +} + +// VolumeOptions specifies capacity, tags, name and flavorID for a volume. +type VolumeOptions struct { + CapacityGB int + Tags map[string]string + Name string + Flavor string + Encrypted bool +} + +func readConfig(config io.Reader) (*CascadeConfig, error) { + if config == nil { + err := fmt.Errorf("Cascade Cloud Provider: config file is missing. Please restart with " + + "--cloud-provider=cascade --cloud-config=[path_to_config_file]") + return nil, err + } + + var cfg CascadeConfig + err := gcfg.ReadInto(&cfg, config) + return &cfg, err +} + +func init() { + cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { + cfg, err := readConfig(config) + if err != nil { + glog.Errorf("Cascade Cloud Provider: failed to read in cloud provider config file. Error[%v]", err) + return nil, err + } + return newCascadeCloud(cfg) + }) +} + +func newCascadeCloud(cfg *CascadeConfig) (*CascadeCloud, error) { + if len(cfg.Global.CloudTarget) == 0 { + return nil, fmt.Errorf("Cascade Controller endpoint was not specified.") + } + + // Get local hostname + hostname, err := os.Hostname() + if err != nil { + glog.Errorf("Cascade Cloud Provider: get hostname failed. Error[%v]", err) + return nil, err + } + + cc := CascadeCloud{ + cfg: cfg, + localHostname: hostname, + localK8sHostname: "", + } + + // Instantiate the auth and API clients only on the master nodes. Kubelets running on the workers don't need them as + // they are used primarily for making API calls to Cascade. + if strings.HasPrefix(hostname, MasterPrefix) { + if cc.authClient, err = NewAuthClient(cfg); err != nil { + return nil, err + } + + if cc.apiClient, err = NewClient(cfg, cc.authClient); err != nil { + return nil, err + } + } + + return &cc, nil +} + +// Initialize passes a Kubernetes clientBuilder interface to the cloud provider +func (cc *CascadeCloud) Initialize(clientBuilder controller.ControllerClientBuilder) {} + +// Instances returns an implementation of Instances for Cascade Controller. +func (cc *CascadeCloud) Instances() (cloudprovider.Instances, bool) { + return cc, true +} + +// List is an implementation of Instances.List. +func (cc *CascadeCloud) List(filter string) ([]k8stypes.NodeName, error) { + return nil, errors.New("unimplemented") +} + +func (cc *CascadeCloud) Clusters() (cloudprovider.Clusters, bool) { + return nil, true +} + +// ProviderName returns the cloud provider ID. +func (cc *CascadeCloud) ProviderName() string { + return ProviderName +} + +// LoadBalancer returns an implementation of LoadBalancer for Cascade Controller. +func (cc *CascadeCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) { + return cc, true +} + +// Zones returns an implementation of Zones for Cascade Controller. +func (cc *CascadeCloud) Zones() (cloudprovider.Zones, bool) { + return cc, true +} + +func (cc *CascadeCloud) GetZone() (cloudprovider.Zone, error) { + return cloudprovider.Zone{ + Region: cc.cfg.Global.Region, + FailureDomain: cc.cfg.Global.Zone, + }, nil +} + +// GetZoneByProviderID implements Zones.GetZoneByProviderID +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (cc *CascadeCloud) GetZoneByProviderID(providerID string) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("unimplemented") +} + +// GetZoneByNodeName implements Zones.GetZoneByNodeName +// This is particularly useful in external cloud providers where the kubelet +// does not initialize node data. +func (cc *CascadeCloud) GetZoneByNodeName(nodeName k8stypes.NodeName) (cloudprovider.Zone, error) { + return cloudprovider.Zone{}, errors.New("unimeplemented") +} + +// Routes returns a false since the interface is not supported for Cascade controller. +func (cc *CascadeCloud) Routes() (cloudprovider.Routes, bool) { + return nil, false +} + +// ScrubDNS filters DNS settings for pods. +func (cc *CascadeCloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut []string) { + return nameservers, searches +} + +// HasClusterID returns true if the cluster has a clusterID +func (cc *CascadeCloud) HasClusterID() bool { + return true +} diff --git a/pkg/cloudprovider/providers/cascade/cascade_disks.go b/pkg/cloudprovider/providers/cascade/cascade_disks.go new file mode 100644 index 0000000..4df1ab9 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/cascade_disks.go @@ -0,0 +1,226 @@ +package cascade + +import ( + "github.com/golang/glog" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/kubelet/apis" + "k8s.io/kubernetes/pkg/volume" +) + +// Attaches given virtual disk volume to the node running kubelet. +func (cc *CascadeCloud) AttachDisk(diskID string, nodeName k8stypes.NodeName) (string, error) { + // Check if disk is already attached to that node. + attached, err := cc.DiskIsAttached(diskID, nodeName) + if err != nil { + glog.Errorf("Cascade Cloud Provider: cc.DiskIsAttached failed during AttachDisk. Error[%v]", err) + return "", err + } + + // If not already attached, attach the disk. + if !attached { + operation := &VMDiskOperation{ + DiskID: StringPtr(diskID), + } + + vmID, err := cc.InstanceID(nodeName) + if err != nil { + glog.Errorf("Cascade Cloud Provider: cc.InstanceID failed for AttachDisk. Error[%v]", err) + return "", err + } + + task, err := cc.apiClient.AttachDisk(vmID, operation) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to attach disk with ID %s. Error[%v]", diskID, err) + return "", err + } + + _, err = cc.apiClient.WaitForTask(StringVal(task.ID)) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to wait for task to attach disk with ID %s. Error[%v]", + diskID, err) + return "", err + } + } + + // Get mount device of the attached disk. + disk, err := cc.apiClient.GetDisk(diskID) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to Get disk with diskID %s. Error[%v]", diskID, err) + return "", err + } + + return disk.MountDevice, nil +} + +// Detaches given virtual disk volume from the node running kubelet. +func (cc *CascadeCloud) DetachDisk(diskID string, nodeName k8stypes.NodeName) error { + operation := &VMDiskOperation{ + DiskID: StringPtr(diskID), + } + + vmID, err := cc.InstanceID(nodeName) + if err != nil { + glog.Errorf("Cascade Cloud Provider: cc.InstanceID failed for DetachDisk. Error[%v]", err) + return err + } + + task, err := cc.apiClient.DetachDisk(vmID, operation) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to detach disk with pdID %s. Error[%v]", diskID, err) + return err + } + + _, err = cc.apiClient.WaitForTask(StringVal(task.ID)) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to wait for task to detach disk with pdID %s. Error[%v]", + diskID, err) + return err + } + + return nil +} + +// DiskIsAttached returns if disk is attached to the VM using controllers supported by the plugin. +func (cc *CascadeCloud) DiskIsAttached(diskID string, nodeName k8stypes.NodeName) (bool, error) { + vmID, err := cc.InstanceID(nodeName) + if err != nil { + glog.Errorf("Cascade Cloud Provider: cc.InstanceID failed for DiskIsAttached. Error[%v]", err) + return false, err + } + + _, err = cc.apiClient.GetVM(vmID) + if err != nil { + switch err.(type) { + case APIError: + if err.(APIError).ErrorCode == VMNotFoundError { + // If instance no longer exists, we will assume that the volume is not attached. + glog.Warningf("Cascade Cloud Provider: Instance %s does not exist. DiskIsAttached will assume"+ + " disk %s is not attached to it.", nodeName, diskID) + return false, nil + } + } + return false, err + } + + disk, err := cc.apiClient.GetDisk(diskID) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to Get disk with diskID %s. Error[%v]", diskID, err) + return false, err + } + + if disk.VM == vmID && StringVal(disk.State) == DiskStateAttached { + return true, nil + } + + return false, nil +} + +// DisksAreAttached returns if disks are attached to the VM using controllers supported by the plugin. +func (cc *CascadeCloud) DisksAreAttached(diskIDs []string, nodeName k8stypes.NodeName) (map[string]bool, error) { + attached := make(map[string]bool) + for _, diskID := range diskIDs { + attached[diskID] = false + } + + vmID, err := cc.InstanceID(nodeName) + if err != nil { + glog.Errorf("Cascade Cloud Provider: cc.InstanceID failed for DiskIsAttached. Error[%v]", err) + return attached, err + } + + for _, diskID := range diskIDs { + disk, err := cc.apiClient.GetDisk(diskID) + if err != nil { + glog.Warningf("Cascade Cloud Provider: failed to get VMs for persistent disk %s, err [%v]", + diskID, err) + } else { + if disk.VM == vmID && StringVal(disk.State) == DiskStateAttached { + attached[diskID] = true + } + } + } + + return attached, nil +} + +// Create a volume of given size (in GB). +func (cc *CascadeCloud) CreateDisk(volumeOptions *VolumeOptions) (diskID string, err error) { + // Get Zones for the cluster + zones, err := cc.apiClient.GetZones() + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to Get zones for the cluster. Error[%v]", err) + return "", err + } + + // Pick a zone to place the disk in. + zoneSet := sets.NewString() + for _, zone := range zones { + zoneSet.Insert(zone) + } + zone := volume.ChooseZoneForVolume(zoneSet, volumeOptions.Name) + + diskSpec := DiskCreateSpec{} + diskSpec.Name = StringPtr(volumeOptions.Name) + diskSpec.Flavor = StringPtr(volumeOptions.Flavor) + diskSpec.CapacityGB = Int32Ptr(int32(volumeOptions.CapacityGB)) + diskSpec.Kind = StringPtr(DiskSpecKind) + diskSpec.Zone = StringPtr(zone) + diskSpec.Encrypted = BoolPtr(volumeOptions.Encrypted) + + task, err := cc.apiClient.CreateDisk(&diskSpec) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to CreateDisk. Error[%v]", err) + return "", err + } + + waitTask, err := cc.apiClient.WaitForTask(StringVal(task.ID)) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to wait for task to CreateDisk. Error[%v]", err) + return "", err + } + + return StringVal(waitTask.Entity.ID), nil +} + +// Deletes a volume given volume name. +func (cc *CascadeCloud) DeleteDisk(diskID string) error { + task, err := cc.apiClient.DeleteDisk(diskID) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to DeleteDisk. Error[%v]", err) + // If we get a DiskNotFound error, we assume that the disk is already deleted. So we don't return an error here. + switch err.(type) { + case APIError: + if err.(APIError).ErrorCode == DiskNotFoundError { + return nil + } + if err.(APIError).ErrorCode == DiskInUseError { + return volume.NewDeletedVolumeInUseError(err.Error()) + } + } + return err + } + + _, err = cc.apiClient.WaitForTask(StringVal(task.ID)) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to wait for task to DeleteDisk. Error[%v]", err) + return err + } + + return nil +} + +// Gets the zone and region for the volume. +func (cc *CascadeCloud) GetVolumeLabels(diskID string) (map[string]string, error) { + disk, err := cc.apiClient.GetDisk(diskID) + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to GetDisk for GetVolumeLabels. Error[%v]", err) + return nil, err + } + + labels := make(map[string]string) + labels[apis.LabelZoneFailureDomain] = StringVal(disk.Zone) + labels[apis.LabelZoneRegion] = cc.cfg.Global.Region + + return labels, nil +} diff --git a/pkg/cloudprovider/providers/cascade/cascade_instances.go b/pkg/cloudprovider/providers/cascade/cascade_instances.go new file mode 100644 index 0000000..0172151 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/cascade_instances.go @@ -0,0 +1,91 @@ +package cascade + +import ( + "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "errors" + "strings" +) + +// NodeAddresses is an implementation of Instances.NodeAddresses. In the future, private IP address, external IP, etc. +// will be added based on need. +func (cc *CascadeCloud) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress, error) { + return []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: cc.cfg.Global.IPAddress}, + {Type: v1.NodeInternalDNS, Address: cc.cfg.Global.DNSName}, + }, nil +} + +// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (cc *CascadeCloud) NodeAddressesByProviderID(providerID string) ([]v1.NodeAddress, error) { + // Get the name of the VM using the ID and generate the DNS name based on the VM name. + vm, err := cc.apiClient.GetVM(providerID) + if err != nil { + return nil, err + } + // Get the DNS name for the master VM and replace the VM name portion with the requested VM name. + dnsNameParts := strings.SplitN(cc.cfg.Global.DNSName, ".", 2) + if len(dnsNameParts) != 2 { + return nil, errors.New("Cascade cloud provider: Invalid DNS name specified in the configuation. " + + "Cannot get NodeAddressByProviderID.") + } + dnsAddress := StringVal(vm.Name) + dnsNameParts[1] + addresses := []v1.NodeAddress{} + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalDNS, Address: dnsAddress}) + return addresses, nil +} + +func (cc *CascadeCloud) AddSSHKeyToAllInstances(user string, keyData []byte) error { + return errors.New("unimplemented") +} + +// Current node name returns node name based on host name. For Cascade Kubernetes nodes, we will use host name as the +// node name. +func (cc *CascadeCloud) CurrentNodeName(hostname string) (k8stypes.NodeName, error) { + cc.localK8sHostname = hostname + return k8stypes.NodeName(hostname), nil +} + +// ExternalID returns the cloud provider ID of the specified instance (deprecated). +// Note: We do not call Cascade Controller here to check if the instance is alive or not because that requires the +// worker nodes to also login to Cascade Controller. That check is used by Kubernetes to proactively remove nodes that +// the cloud provider believes is no longer available. Even otherwise, Kubernetes will remove those nodes eventually. +// So we are not losing much by not doing that check. +func (cc *CascadeCloud) ExternalID(nodeName k8stypes.NodeName) (string, error) { + return getInstanceIDFromNodeName(nodeName) +} + +// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. +// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. +func (cc *CascadeCloud) InstanceExistsByProviderID(providerID string) (bool, error) { + return false, errors.New("unimplemented") +} + +// InstanceID returns the cloud provider ID of the specified instance. +func (cc *CascadeCloud) InstanceID(nodeName k8stypes.NodeName) (string, error) { + return getInstanceIDFromNodeName(nodeName) +} + +// This gets the Cascade VM ID from the Kubernetes node name. +func getInstanceIDFromNodeName(nodeName k8stypes.NodeName) (string, error) { + // nodeName is of the format master-instance-id or worker-instance-id. To compute the instance ID, we need to just + // get the portion after master- or worker-. That is what we do below. + nodeParts := strings.SplitN(string(nodeName), "-", 2) + if len(nodeParts) != 2 { + return "", errors.New("Cascade cloud provider: Invalid node name. Cannot fetch instance ID.") + } + return nodeParts[1], nil +} + +// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID +// This method will not be called from the node that is requesting this ID. i.e. metadata service +// and other local methods cannot be used here +func (cc *CascadeCloud) InstanceTypeByProviderID(providerID string) (string, error) { + return "", errors.New("unimplemented") +} + +func (cc *CascadeCloud) InstanceType(nodeName k8stypes.NodeName) (string, error) { + return "", nil +} diff --git a/pkg/cloudprovider/providers/cascade/cascade_instances_test.go b/pkg/cloudprovider/providers/cascade/cascade_instances_test.go new file mode 100644 index 0000000..bec5491 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/cascade_instances_test.go @@ -0,0 +1,43 @@ +package cascade + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" +) + +const configFile = ` +[Global] +target = target +tenantName = tenant +clusterID = cluster-id +authEndpoint = auth-endpoint +domainName = domain.test +dnsName = node.domain.test +region = region +zone = zone +ipAddress = 1.1.1.1 +` + +func getCascadeInstance() (*CascadeCloud, error) { + cfg, err := readConfig(strings.NewReader(configFile)) + return &CascadeCloud{ + cfg: cfg, + localHostname: "hostname", + localK8sHostname: "", + }, err +} + +func TestNodeAddresses(t *testing.T) { + cc, err := getCascadeInstance() + assert.Nil(t, err) + expectedNodeAddresses := []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "1.1.1.1"}, + {Type: v1.NodeInternalDNS, Address: "node.domain.test"}, + } + actualNodeAddresses, err := cc.NodeAddresses("node") + assert.Nil(t, err) + assert.Equal(t, expectedNodeAddresses, actualNodeAddresses) +} diff --git a/pkg/cloudprovider/providers/cascade/cascade_loadbalancer.go b/pkg/cloudprovider/providers/cascade/cascade_loadbalancer.go new file mode 100644 index 0000000..fac37e5 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/cascade_loadbalancer.go @@ -0,0 +1,284 @@ +package cascade + +import ( + "fmt" + "github.com/golang/glog" + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/api/v1/service" + "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/apimachinery/pkg/types" +) + +const TCP_PROTOCOL = "TCP" + +const HTTP_PROTOCOL = "HTTP" + +// EnsureLoadBalancer creates or updates a Cascade load balancer +func (cc *CascadeCloud) EnsureLoadBalancer(clusterName string, k8sService *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { + logger := newLoadBalancerLogger(clusterName, k8sService, "EnsureLoadBalancer") + + loadBalancerName := cloudprovider.GetLoadBalancerName(k8sService) + logger.Infof("Load balancer name: %s", loadBalancerName) + + // Sanity checks + if k8sService.Spec.SessionAffinity != v1.ServiceAffinityNone { + logger.Errorf("Unsupported load balancer session affinity: %+v", k8sService.Spec.SessionAffinity) + return nil, fmt.Errorf("Unsupported load balancer session affinity: %+v", k8sService.Spec.SessionAffinity) + } + + if len(k8sService.Spec.Ports) == 0 { + logger.Errorf("No port mapping is specified") + return nil, fmt.Errorf("No port mapping is specified") + } + + // Create load balancer port maps + portMaps := []*LoadBalancerPortMap{} + for _, port := range k8sService.Spec.Ports { + if port.Protocol != v1.ProtocolTCP { + logger.Warningf("Ignoring port that does not use TCP protocol: %+v", port) + continue + } + + if port.NodePort == 0 { + logger.Warningf("Ignoring port without node port defined: %+v", port) + continue + } + + // TODO: For now we only support SSL pass through. All port mappings are using TCP protocol. + // Also note that we allow all external traffic to access the ports. + portMap := &LoadBalancerPortMap{ + InstancePort: Int64Ptr(int64(port.NodePort)), + InstanceProtocol: StringPtr(TCP_PROTOCOL), + LoadBalancerPort: Int64Ptr(int64(port.Port)), + LoadBalancerProtocol: StringPtr(TCP_PROTOCOL), + } + portMaps = append(portMaps, portMap) + } + + // Create load balancer health check + healthCheck := &LoadBalancerHealthCheck{ + HealthyThreshold: 5, + IntervalInSeconds: 10, + } + if healthCheckPath, healthCheckNodePort := service.GetServiceHealthCheckPathPort(k8sService); healthCheckPath != "" { + logger.Infof("HTTP health checks on: %s:%d", healthCheckPath, healthCheckNodePort) + healthCheck.Path = StringPtr(healthCheckPath) + healthCheck.Port = Int64Ptr(int64(healthCheckNodePort)) + healthCheck.Protocol = StringPtr(HTTP_PROTOCOL) + } else { + logger.Infof("TCP health check on port: %d", Int64Val(portMaps[0].InstancePort)) + healthCheck.Port = portMaps[0].InstancePort + healthCheck.Protocol = StringPtr(TCP_PROTOCOL) + } + + // Create load balancer + createSpec := &LoadBalancerCreateSpec{ + Name: StringPtr(loadBalancerName), + Type: StringPtr("PUBLIC"), + PortMaps: portMaps, + HealthCheck: healthCheck, + SubDomain: StringPtr(k8sService.Name), + } + logger.Infof("Load balancer create spec: %+v", *createSpec) + + task, err := cc.apiClient.CreateOrUpdateLoadBalancer(createSpec) + if err != nil { + logger.Errorf("Failed to create or update load balancer. Error: [%v]", err) + return nil, err + } + + _, err = cc.apiClient.WaitForTask(StringVal(task.ID)) + if err != nil { + logger.Errorf("Failed to poll task status of creating or updating load balancer. Error: [%v]", err) + return nil, err + } + + // Apply VM update to load balancer + err = cc.updateLoadBalancerVMs(nodes, loadBalancerName, logger) + if err != nil { + // The private function already did logging. No need to log again. + return nil, err + } + + // Get load balancer + loadBalancer, err := cc.apiClient.GetLoadBalancer(StringPtr(loadBalancerName)) + if err != nil { + glog.Errorf("Failed to get load balancer. Error: [%v]", err) + return nil, err + } + + return toLoadBalancerStatus(loadBalancer), nil +} + +// GetLoadBalancer returns the information about a Cascade load balancer +func (cc *CascadeCloud) GetLoadBalancer(clusterName string, k8sService *v1.Service) (*v1.LoadBalancerStatus, bool, error) { + logger := newLoadBalancerLogger(clusterName, k8sService, "GetLoadBalancer") + + loadBalancerName := cloudprovider.GetLoadBalancerName(k8sService) + logger.Infof("Load balancer name: %s", loadBalancerName) + + // Get load balancer + loadBalancer, err := cc.apiClient.GetLoadBalancer(StringPtr(loadBalancerName)) + if err != nil { + logger.Errorf("Failed to get load balancer. Error: [%v]", err) + // Do not return error here because we want the caller of this function to determine + // what to do with the not-found situation. + switch err.(type) { + case APIError: + if err.(APIError).ErrorCode == NotFoundError { + return nil, false, nil + } + } + return nil, false, err + } + + return toLoadBalancerStatus(loadBalancer), true, nil +} + +// UpdateLoadBalancer updates the node information of a Cascade load balancer +func (cc *CascadeCloud) UpdateLoadBalancer(clusterName string, k8sService *v1.Service, nodes []*v1.Node) error { + logger := newLoadBalancerLogger(clusterName, k8sService, "UpdateLoadBalancer") + + loadBalancerName := cloudprovider.GetLoadBalancerName(k8sService) + logger.Infof("Load balancer name: %s", loadBalancerName) + + err := cc.updateLoadBalancerVMs(nodes, loadBalancerName, logger) + if err != nil { + // The private function already did logging. No need to log again. + return err + } + + return nil +} + +// EnsureLoadBalancerDeleted deletes a Cascade load balancer +func (cc *CascadeCloud) EnsureLoadBalancerDeleted(clusterName string, k8sService *v1.Service) error { + logger := newLoadBalancerLogger(clusterName, k8sService, "EnsureLoadBalancerDeleted") + + loadBalancerName := cloudprovider.GetLoadBalancerName(k8sService) + logger.Infof("Load balancer name: %s", loadBalancerName) + + task, err := cc.apiClient.DeleteLoadBalancer(StringPtr(loadBalancerName), k8sService.Name) + if err != nil { + logger.Errorf("Failed to delete load balancer. Error: [%v]", err) + // If we get a NotFound error, we assume that the load balancer is already deleted. So we don't return an error + // here. + switch err.(type) { + case APIError: + if err.(APIError).ErrorCode == NotFoundError { + return nil + } + } + return err + } + + _, err = cc.apiClient.WaitForTask(StringVal(task.ID)) + if err != nil { + logger.Errorf("Failed to poll task status of deleting load balancer. Error: [%v]", err) + return err + } + + return nil +} + +func (cc *CascadeCloud) updateLoadBalancerVMs( + nodes []*v1.Node, loadBalancerName string, logger *loadBalancerLogger) error { + + // Apply VM update to the load balancer + loadBalancerVMs := make([]*LoadBalancerVM, 0) + + for _, node := range(nodes) { + // If the node does not have a name, we cannot derive its instance ID. Therefore we skip this node. + if len(node.Name) == 0 { + logger.Warningf("Node %s does not have a name. Skip updating this VM for load balancer", node.UID) + continue + } + + // If we cannot get the instance ID, something is wrong on the Cascade Controller side. + // However, we should tolerate such failure and continue the load balancer VM update + // by skipping this VM. + instanceID, err := cc.InstanceID(types.NodeName(node.Name)) + if err != nil { + logger.Warningf("Unable to get instance ID for node %s, skip updating this VM for load balancer. Error [%v]", node.Name, err) + continue + } + + loadBalancerVMs = append(loadBalancerVMs, &LoadBalancerVM{ + ID: StringPtr(instanceID), + }) + } + + if len(loadBalancerVMs) == 0 { + logger.Infof("No nodes to be added to the load balancer. Skip updating load balancer VMs") + return nil + } + + vmUpdate := &LoadBalancerVMUpdate{ + VMIds: loadBalancerVMs, + } + logger.Infof("Load balancer VM update spec: %+v", vmUpdate.VMIds) + + task, err := cc.apiClient.ApplyVMsToLoadBalancer(StringPtr(loadBalancerName), vmUpdate) + if err != nil { + logger.Errorf("Failed to update load balancer VMs. Error: [%v]", err) + return err + } + + _, err = cc.apiClient.WaitForTask(StringVal(task.ID)) + if err != nil { + logger.Errorf("Failed to poll task status of updating load balancer VMs. Error: [%v]", err) + return err + } + + return nil +} + +func toLoadBalancerStatus(lb *LoadBalancer) *v1.LoadBalancerStatus { + var endpoint string + if lb != nil && lb.Endpoint != nil { + endpoint = StringVal(lb.Endpoint) + } + + return &v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + Hostname: endpoint, + }, + }, + } +} + +type loadBalancerLogger struct { + clusterName string + k8sService *v1.Service + callingFunc string +} + +func newLoadBalancerLogger(clusterName string, k8sService *v1.Service, callingFunc string) *loadBalancerLogger { + return &loadBalancerLogger{ + clusterName: clusterName, + k8sService: k8sService, + callingFunc: callingFunc, + } +} + +func (l *loadBalancerLogger) getLogMsg( + msgTemplate string, args ...interface{}) string { + + errorMsg := fmt.Sprintf("Cascade Cloud Provider::%s::Cluster [%s] Service [%s]: %s", + l.callingFunc, l.clusterName, l.k8sService.Name, + msgTemplate) + return fmt.Sprintf(errorMsg, args) +} + +func (l *loadBalancerLogger) Errorf(msgTemplate string, args ...interface{}) { + glog.Errorln(l.getLogMsg(msgTemplate, args)) +} + +func (l *loadBalancerLogger) Warningf(msgTemplate string, args ...interface{}) { + glog.Warningln(l.getLogMsg(msgTemplate, args)) +} + +func (l *loadBalancerLogger) Infof(msgTemplate string, args ...interface{}) { + glog.Infoln(l.getLogMsg(msgTemplate, args)) +} \ No newline at end of file diff --git a/pkg/cloudprovider/providers/cascade/client.go b/pkg/cloudprovider/providers/cascade/client.go new file mode 100644 index 0000000..e4494e4 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/client.go @@ -0,0 +1,399 @@ +package cascade + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "github.com/golang/glog" + "net/http" + "strings" + "time" +) + +// Represents stateless context needed to call Cascade APIs. +// Note that we are implementing the Cascade APIs manually instead of using the swagger generated code +// because swagger uses a different version of openapi library than kubernetes. It is difficult to +// address the version conflict to make it compile. +type Client struct { + cfg *ClientConfig + options ClientOptions + restClient *restClient +} + +type ClientConfig struct { + tenantName string + clusterID string + region string + endpoint string +} + +// Represents Tokens +type TokenOptions struct { + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + RefreshToken string `json:"refresh_token,omitempty"` + IDToken string `json:"id_token"` + TokenType string `json:"token_type"` +} + +type TokenCallback func(string) + +// Options for Client +type ClientOptions struct { + // When using the Tasks.Wait APIs, defines the duration of how long + // we should continue to poll the server. Default is 30 minutes. + // TasksAPI.WaitTimeout() can be used to specify timeout on + // individual calls. + TaskPollTimeout time.Duration + + // Whether or not to ignore any TLS errors when talking to Cascade, + // false by default. + IgnoreCertificate bool + + // List of root CA's to use for server validation + // nil by default. + RootCAs *x509.CertPool + + // For tasks APIs, defines the number of retries to make in the event + // of an error. Default is 3. + TaskRetryCount int + + // Tokens for user authentication. Default is empty. + TokenOptions *TokenOptions +} + +const minimumTaskPollDelay = 500 * time.Millisecond + +// Creates a new Cascade client which can be used to make API calls to Cascade. +func NewClient(cfg *CascadeConfig, authClient *AuthClient) (c *Client, err error) { + tokenOptions, err := authClient.GetTokensByMachineAccount() + if err != nil { + glog.Errorf("Cascade Cloud Provider: Failed to create new client due to error: %+v", err) + return + } + + options := &ClientOptions{ + TaskPollTimeout: 30 * time.Minute, + TaskRetryCount: 3, + TokenOptions: tokenOptions, + IgnoreCertificate: false, + RootCAs: nil, + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: options.IgnoreCertificate, + RootCAs: options.RootCAs}, + } + + tokenCallback := func(newToken string) { + c.options.TokenOptions.AccessToken = newToken + } + + restClient := &restClient{ + authClient: authClient, + httpClient: &http.Client{Transport: tr}, + UpdateAccessTokenCallback: tokenCallback, + } + + clientConfig := &ClientConfig{ + tenantName: cfg.Global.TenantName, + clusterID: cfg.Global.ClusterID, + region: cfg.Global.Region, + endpoint: strings.TrimRight(cfg.Global.CloudTarget, "/"), + } + + c = &Client{ + cfg: clientConfig, + restClient: restClient, + // Ensure a copy of options is made, rather than using a pointer + // which may change out from underneath if misused by the caller. + options: *options, + } + + return +} + +// Gets VM with the specified ID. +func (api *Client) GetVM(vmID string) (vm *VM, err error) { + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/vms/%s", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID, vmID) + res, err := api.restClient.Get(uri, api.options.TokenOptions) + if err != nil { + return + } + defer res.Body.Close() + res, err = getError(res) + if err != nil { + return + } + vm = &VM{} + err = json.NewDecoder(res.Body).Decode(vm) + return +} + +// Gets disk with the specified ID. +func (api *Client) GetDisk(diskID string) (disk *PersistentDisk, err error) { + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/disks/%s", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID, diskID) + res, err := api.restClient.Get(uri, api.options.TokenOptions) + if err != nil { + return + } + defer res.Body.Close() + res, err = getError(res) + if err != nil { + return + } + disk = &PersistentDisk{} + err = json.NewDecoder(res.Body).Decode(disk) + return +} + +// Creates a disk under the cluster. +func (api *Client) CreateDisk(spec *DiskCreateSpec) (task *Task, err error) { + body, err := json.Marshal(spec) + if err != nil { + return + } + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/disks", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID) + res, err := api.restClient.Post(uri, "application/json", bytes.NewReader(body), api.options.TokenOptions) + if err != nil { + return + } + defer res.Body.Close() + task, err = getTask(getError(res)) + return +} + +// Deletes a disk with the specified ID. +func (api *Client) DeleteDisk(diskID string) (task *Task, err error) { + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/disks/%s", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID, diskID) + res, err := api.restClient.Delete(uri, api.options.TokenOptions) + if err != nil { + return + } + defer res.Body.Close() + task, err = getTask(getError(res)) + return +} + +// Attaches a disk to the specified VM. +func (api *Client) AttachDisk(vmID string, op *VMDiskOperation) (task *Task, err error) { + body, err := json.Marshal(op) + if err != nil { + return + } + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/vms/%s/attach_disk", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID, vmID) + res, err := api.restClient.Post(uri, "application/json", bytes.NewReader(body), api.options.TokenOptions) + if err != nil { + return + } + defer res.Body.Close() + task, err = getTask(getError(res)) + return +} + +// Detaches a disk from the specified VM. +func (api *Client) DetachDisk(vmID string, op *VMDiskOperation) (task *Task, err error) { + body, err := json.Marshal(op) + if err != nil { + return + } + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/vms/%s/detach_disk", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID, vmID) + res, err := api.restClient.Post(uri, "application/json", bytes.NewReader(body), api.options.TokenOptions) + if err != nil { + return + } + defer res.Body.Close() + task, err = getTask(getError(res)) + return +} + +// Gets a task by ID. +func (api *Client) GetTask(taskID string) (task *Task, err error) { + uri := fmt.Sprintf("%s/v1/tenants/%s/tasks/%s?region=%s", api.cfg.endpoint, api.cfg.tenantName, + taskID, api.cfg.region) + res, err := api.restClient.Get(uri, api.options.TokenOptions) + if err != nil { + return + } + defer res.Body.Close() + result, err := getTask(getError(res)) + return result, err +} + +// Waits for a task to complete by polling the tasks API until a task returns with the state COMPLETED or ERROR. +func (api *Client) WaitForTask(taskID string) (task *Task, err error) { + start := time.Now() + numErrors := 0 + maxErrors := api.options.TaskRetryCount + backoffMultiplier := 1 + + for time.Since(start) < api.options.TaskPollTimeout { + task, err = api.GetTask(taskID) + if err != nil { + switch err.(type) { + // If an ApiError comes back, something is wrong, return the error to the caller + case APIError: + return + // For other errors, retry before giving up + default: + numErrors++ + if numErrors > maxErrors { + return + } + } + } else { + // Reset the error count any time a successful call is made + numErrors = 0 + if StringVal(task.State) == "COMPLETED" { + return + } + if StringVal(task.State) == "ERROR" { + err = TaskError{StringVal(task.ID), getFailedStep(task)} + return + } + } + + // Perform backoff based on how long it has been since we started polling. The logic is as follows: + // For the first 10 seconds, poll every 500 milliseconds. + // From there till the first 1 minute, poll every 1 second. + // From there till the first 10 minutes, poll every 5 seconds. + // From there till the timeout (30 minutes), poll every 10 seconds. + elapsedTime := time.Since(start) + if elapsedTime > 10*time.Second && elapsedTime <= 60*time.Second { + backoffMultiplier = 2 + } else if elapsedTime > 60*time.Second && elapsedTime <= 600*time.Second { + backoffMultiplier = 10 + } else if elapsedTime > 600*time.Second && elapsedTime <= api.options.TaskPollTimeout { + backoffMultiplier = 20 + } + time.Sleep(time.Duration(backoffMultiplier) * minimumTaskPollDelay) + } + err = TaskTimeoutError{taskID} + return +} + +// CreateOrUpdateLoadBalancer creates a load balancer if not existed, or update one otherwise +func (api *Client) CreateOrUpdateLoadBalancer(spec *LoadBalancerCreateSpec) (*Task, error) { + body, err := json.Marshal(spec) + if err != nil { + return nil, err + } + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/loadbalancers", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID) + res, err := api.restClient.Post(uri, "application/json", bytes.NewReader(body), api.options.TokenOptions) + if err != nil { + return nil, err + } + defer res.Body.Close() + return getTask(getError(res)) +} + +// GetLoadBalancer returns a load balancer by name +func (api *Client) GetLoadBalancer(loadBalancerName *string) (*LoadBalancer, error) { + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/loadbalancers/%s", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID, StringVal(loadBalancerName)) + res, err := api.restClient.Get(uri, api.options.TokenOptions) + if err != nil { + return nil, err + } + defer res.Body.Close() + res, err = getError(res) + if err != nil { + return nil, err + } + loadBalancer := &LoadBalancer{} + err = json.NewDecoder(res.Body).Decode(loadBalancer) + return loadBalancer, err +} + +// DeleteLoadBalancer deletes a load balancer by name +func (api *Client) DeleteLoadBalancer(loadBalancerName *string, subDomain string) (*Task, error) { + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/loadbalancers/%s", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID, StringVal(loadBalancerName)) + + if len(subDomain) > 0 { + uri = fmt.Sprintf(uri + "?sub-domain=%s", subDomain) + } + + res, err := api.restClient.Delete(uri, api.options.TokenOptions) + if err != nil { + return nil, err + } + return getTask(getError(res)) +} + +// ApplyVMsToLoadBalancer updates the instances that are registered with the load balancer +func (api *Client) ApplyVMsToLoadBalancer(loadBalancerName *string, update *LoadBalancerVMUpdate) (*Task, error) { + body, err := json.Marshal(update) + if err != nil { + return nil, err + } + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/loadbalancers/%s/update_vms", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID, StringVal(loadBalancerName)) + res, err := api.restClient.Post(uri, "application/json", bytes.NewReader(body), api.options.TokenOptions) + if err != nil { + return nil, err + } + defer res.Body.Close() + return getTask(getError(res)) +} + +// Gets all the zones in which the cluster has the VMs in. +func (api *Client) GetZones() (zones []string, err error) { + uri := fmt.Sprintf("%s/v1/tenants/%s/clusters/%s/zones", api.cfg.endpoint, api.cfg.tenantName, + api.cfg.clusterID) + res, err := api.restClient.Get(uri, api.options.TokenOptions) + if err != nil { + return + } + defer res.Body.Close() + res, err = getError(res) + if err != nil { + return + } + err = json.NewDecoder(res.Body).Decode(&zones) + return +} + +// Reads a task object out of the HTTP response. Takes an error argument +// so that GetTask can easily wrap GetError. This function will do nothing +// if e is not nil. +// e.g. res, err := getTask(getError(someApi.Get())) +func getTask(res *http.Response, e error) (*Task, error) { + if e != nil { + return nil, e + } + var task Task + err := json.NewDecoder(res.Body).Decode(&task) + if err != nil { + return nil, err + } + if StringVal(task.State) == "ERROR" { + // Critical: return task as well, so that it can be examined + // for error details. + return &task, TaskError{StringVal(task.ID), getFailedStep(&task)} + } + return &task, nil +} + +// Gets the failed step in the task to get error details for failed task. +func getFailedStep(task *Task) (step Step) { + var errorStep Step + for _, s := range task.Steps { + if StringVal(s.State) == "ERROR" { + errorStep = *s + break + } + } + + return errorStep +} diff --git a/pkg/cloudprovider/providers/cascade/oidcclient.go b/pkg/cloudprovider/providers/cascade/oidcclient.go new file mode 100644 index 0000000..6a71cc1 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/oidcclient.go @@ -0,0 +1,297 @@ +package cascade + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "strings" +) + +const tokenScope string = "openid offline_access" + +// OIDCClient is client for OIDC +type OIDCClient struct { + httpClient *http.Client + logger *log.Logger + + Endpoint string + Options *OIDCClientOptions +} + +// OIDCClientOptions is OIDC client options +type OIDCClientOptions struct { + // Whether or not to ignore any TLS errors when talking to Cascade, + // false by default. + IgnoreCertificate bool + + // List of root CA's to use for server validation + // nil by default. + RootCAs *x509.CertPool + + // The scope values to use when requesting tokens + TokenScope string +} + +// NewOIDCClient creates an instance of OIDCClient +func NewOIDCClient(endpoint string, options *OIDCClientOptions, logger *log.Logger) (c *OIDCClient) { + if logger == nil { + logger = log.New(ioutil.Discard, "", log.LstdFlags) + } + + options = buildOptions(options) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: options.IgnoreCertificate, + RootCAs: options.RootCAs}, + } + + c = &OIDCClient{ + httpClient: &http.Client{Transport: tr}, + logger: logger, + Endpoint: strings.TrimRight(endpoint, "/"), + Options: options, + } + return +} + +func buildOptions(options *OIDCClientOptions) (result *OIDCClientOptions) { + result = &OIDCClientOptions{ + TokenScope: tokenScope, + } + + if options == nil { + return + } + + result.IgnoreCertificate = options.IgnoreCertificate + + if options.RootCAs != nil { + result.RootCAs = options.RootCAs + } + + if options.TokenScope != "" { + result.TokenScope = options.TokenScope + } + + return +} + +func (client *OIDCClient) buildURL(path string) (url string) { + return fmt.Sprintf("%s%s", client.Endpoint, path) +} + +// Cert download helper + +const certDownloadPath string = "/afd/vecs/ssl" + +type lightWaveCert struct { + Value string `json:"encoded"` +} + +// GetRootCerts gets root certs +func (client *OIDCClient) GetRootCerts() (certList []*x509.Certificate, err error) { + // turn TLS verification off for + originalTr := client.httpClient.Transport + defer client.setTransport(originalTr) + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: false, + }, + } + client.setTransport(tr) + + // get the certs + resp, err := client.httpClient.Get(client.buildURL(certDownloadPath)) + if err != nil { + return + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + err = fmt.Errorf("Unexpected error retrieving auth server certs: %v %s", resp.StatusCode, resp.Status) + return + } + + // parse the certs + certsData := &[]lightWaveCert{} + err = json.NewDecoder(resp.Body).Decode(certsData) + if err != nil { + return + } + + certList = make([]*x509.Certificate, len(*certsData)) + for idx, cert := range *certsData { + block, _ := pem.Decode([]byte(cert.Value)) + if block == nil { + err = fmt.Errorf("Unexpected response format: %v", certsData) + return nil, err + } + + decodedCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + + certList[idx] = decodedCert + } + + return +} + +func (client *OIDCClient) setTransport(tr http.RoundTripper) { + client.httpClient.Transport = tr +} + +// Metadata request helpers +const metadataPathFormat string = "/openidconnect/%s/.well-known/openid-configuration" + +// OIDCMetadataResponse is the response for Metadata request +type OIDCMetadataResponse struct { + TokenEndpoint string `json:"token_endpoint"` + AuthorizationEndpoint string `json:"authorization_endpoint"` + EndSessionEndpoint string `json:"end_session_endpoint"` +} + +func (client *OIDCClient) getMetadata(domain string) (metadata *OIDCMetadataResponse, err error) { + metadataPath := fmt.Sprintf(metadataPathFormat, domain) + request, err := http.NewRequest("GET", client.buildURL(metadataPath), nil) + if err != nil { + return nil, err + } + + resp, err := client.httpClient.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + err = client.checkResponse(resp) + if err != nil { + return nil, err + } + + metadata = &OIDCMetadataResponse{} + err = json.NewDecoder(resp.Body).Decode(metadata) + if err != nil { + return nil, err + } + + return +} + +// Token request helpers + +const passwordGrantFormatString = "grant_type=password&username=%s&password=%s&scope=%s" +const refreshTokenGrantFormatString = "grant_type=refresh_token&refresh_token=%s" +const clientGrantFormatString = "grant_type=password&username=%s&password=%s&scope=%s&client_id=%s" + +// OIDCTokenResponse is the response for OIDC request +type OIDCTokenResponse struct { + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + RefreshToken string `json:"refresh_token,omitempty"` + IDToken string `json:"id_token"` + TokenType string `json:"token_type"` +} + +// GetTokenByPasswordGrant gets OIDC tokens by password +func (client *OIDCClient) GetTokenByPasswordGrant(domain, username, password string) (tokens *OIDCTokenResponse, err error) { + metadata, err := client.getMetadata(domain) + if err != nil { + return nil, err + } + + username = url.QueryEscape(username) + password = url.QueryEscape(password) + body := fmt.Sprintf(passwordGrantFormatString, username, password, client.Options.TokenScope) + return client.getToken(metadata.TokenEndpoint, body) +} + +// GetClientTokenByPasswordGrant gets OIDC tokens by password +func (client *OIDCClient) GetClientTokenByPasswordGrant(domain, username, password, clientID string) (tokens *OIDCTokenResponse, err error) { + metadata, err := client.getMetadata(domain) + if err != nil { + return nil, err + } + + username = url.QueryEscape(username) + password = url.QueryEscape(password) + clientID = url.QueryEscape(clientID) + body := fmt.Sprintf(clientGrantFormatString, username, password, client.Options.TokenScope, clientID) + return client.getToken(metadata.TokenEndpoint, body) +} + +// GetTokenByRefreshTokenGrant gets OIDC tokens by refresh token +func (client *OIDCClient) GetTokenByRefreshTokenGrant(domain, refreshToken string) (tokens *OIDCTokenResponse, err error) { + metadata, err := client.getMetadata(domain) + if err != nil { + return nil, err + } + + body := fmt.Sprintf(refreshTokenGrantFormatString, refreshToken) + return client.getToken(metadata.TokenEndpoint, body) +} + +func (client *OIDCClient) getToken(tokenEndpoint, body string) (tokens *OIDCTokenResponse, err error) { + request, err := http.NewRequest("POST", tokenEndpoint, strings.NewReader(body)) + if err != nil { + return nil, err + } + request.Header.Add("Content-Type", "application/x-www-form-urlencoded") + + resp, err := client.httpClient.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + err = client.checkResponse(resp) + if err != nil { + return nil, err + } + + tokens = &OIDCTokenResponse{} + err = json.NewDecoder(resp.Body).Decode(tokens) + if err != nil { + return nil, err + } + + return +} + +// OIDCError is OIDC error +type OIDCError struct { + Code string `json:"error"` + Message string `json:"error_description"` +} + +func (e OIDCError) Error() string { + return fmt.Sprintf("%v: %v", e.Code, e.Message) +} + +func (client *OIDCClient) checkResponse(response *http.Response) (err error) { + if response.StatusCode/100 == 2 { + return + } + + respBody, readErr := ioutil.ReadAll(response.Body) + if readErr != nil { + return fmt.Errorf( + "Status: %v, Body: %v [%v]", response.Status, string(respBody[:]), readErr) + } + + var oidcErr OIDCError + err = json.Unmarshal(respBody, &oidcErr) + if err != nil || oidcErr.Code == "" { + return fmt.Errorf( + "Status: %v, Body: %v [%v]", response.Status, string(respBody[:]), readErr) + } + + return oidcErr +} diff --git a/pkg/cloudprovider/providers/cascade/restclient.go b/pkg/cloudprovider/providers/cascade/restclient.go new file mode 100644 index 0000000..71d8d1c --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/restclient.go @@ -0,0 +1,262 @@ +package cascade + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "net/http" +) + +type restClient struct { + httpClient *http.Client + authClient *AuthClient + UpdateAccessTokenCallback TokenCallback +} + +type request struct { + Method string + URL string + ContentType string + Body io.Reader + Tokens *TokenOptions +} + +type page struct { + Items []interface{} `json:"items"` + NextPageLink string `json:"nextPageLink"` + PreviousPageLink string `json:"previousPageLink"` +} + +type documentList struct { + Items []interface{} +} + +type bodyRewinder func() io.Reader + +const appJson string = "application/json" +const expiredAuthToken int32 = 1904 + +func (client *restClient) AppendSlice(origSlice []interface{}, dataToAppend []interface{}) []interface{} { + origLen := len(origSlice) + newLen := origLen + len(dataToAppend) + + if newLen > cap(origSlice) { + newSlice := make([]interface{}, (newLen+1)*2) + copy(newSlice, origSlice) + origSlice = newSlice + } + + origSlice = origSlice[0:newLen] + copy(origSlice[origLen:newLen], dataToAppend) + + return origSlice +} + +func (client *restClient) Get(url string, tokens *TokenOptions) (res *http.Response, err error) { + req := request{"GET", url, "", nil, tokens} + res, err = client.SendRequest(&req, nil) + return +} + +func (client *restClient) GetList(endpoint string, url string, tokens *TokenOptions) (result []byte, err error) { + req := request{"GET", url, "", nil, tokens} + res, err := client.SendRequest(&req, nil) + if err != nil { + return + } + res, err = getError(res) + if err != nil { + return + } + + decoder := json.NewDecoder(res.Body) + decoder.UseNumber() + + page := &page{} + err = decoder.Decode(page) + if err != nil { + return + } + + documentList := &documentList{} + documentList.Items = client.AppendSlice(documentList.Items, page.Items) + + for page.NextPageLink != "" { + req = request{"GET", endpoint + page.NextPageLink, "", nil, tokens} + res, err = client.SendRequest(&req, nil) + if err != nil { + return + } + res, err = getError(res) + if err != nil { + return + } + + decoder = json.NewDecoder(res.Body) + decoder.UseNumber() + + page.NextPageLink = "" + page.PreviousPageLink = "" + + err = decoder.Decode(page) + if err != nil { + return + } + + documentList.Items = client.AppendSlice(documentList.Items, page.Items) + } + + result, err = json.Marshal(documentList) + + return +} + +func (client *restClient) Post(url string, contentType string, body io.ReadSeeker, tokens *TokenOptions) (res *http.Response, err error) { + if contentType == "" { + contentType = appJson + } + + req := request{"POST", url, contentType, body, tokens} + rewinder := func() io.Reader { + body.Seek(0, 0) + return body + } + res, err = client.SendRequest(&req, rewinder) + return +} + +func (client *restClient) Patch(url string, contentType string, body io.ReadSeeker, tokens *TokenOptions) (res *http.Response, err error) { + if contentType == "" { + contentType = appJson + } + + req := request{"PATCH", url, contentType, body, tokens} + rewinder := func() io.Reader { + body.Seek(0, 0) + return body + } + res, err = client.SendRequest(&req, rewinder) + return +} + +func (client *restClient) Put(url string, contentType string, body io.ReadSeeker, tokens *TokenOptions) (res *http.Response, err error) { + if contentType == "" { + contentType = appJson + } + + req := request{"PUT", url, contentType, body, tokens} + rewinder := func() io.Reader { + body.Seek(0, 0) + return body + } + res, err = client.SendRequest(&req, rewinder) + return +} + +func (client *restClient) Delete(url string, tokens *TokenOptions) (res *http.Response, err error) { + req := request{"DELETE", url, "", nil, tokens} + res, err = client.SendRequest(&req, nil) + return +} + +func (client *restClient) SendRequest(req *request, bodyRewinder bodyRewinder) (res *http.Response, err error) { + res, err = client.sendRequestHelper(req) + // In most cases, we'll return immediately + // If the operation succeeded, but we got a 401 response and if we're using + // authentication, then we'll look into the body to see if the token expired + if err != nil { + return res, err + } + if res.StatusCode != 401 { + // It's not a 401, so the token didn't expire + return res, err + } + if req.Tokens == nil || req.Tokens.AccessToken == "" { + // We don't have a token, so we can't renew the token, no need to proceed + return res, err + } + + // We're going to look in the body to see if it failed because the token expired + // This means we need to read the body, but the functions that call us also + // expect to read the body. So we read the body, then create a new reader + // so they can read the body as normal. + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return res, err + } + res.Body = ioutil.NopCloser(bytes.NewReader(body)) + + // Now see if we had an expired token or not + var apiError APIError + err = json.Unmarshal(body, &apiError) + if err != nil { + return res, err + } + if apiError.ErrorCode != expiredAuthToken { + return res, nil + } + + // We were told that the access token expired, so we acquire a new token using the refresh token. + newTokens, err := client.authClient.GetTokensByRefreshToken(req.Tokens.RefreshToken) + // If there is an error during token refresh, we assume that the refresh token also expired. So we login again using + // the machine account. + if err != nil { + newTokens, err = client.authClient.GetTokensByMachineAccount() + if err != nil { + return res, err + } + } + req.Tokens.AccessToken = newTokens.AccessToken + if client.UpdateAccessTokenCallback != nil { + client.UpdateAccessTokenCallback(newTokens.AccessToken) + } + if req.Body != nil && bodyRewinder != nil { + req.Body = bodyRewinder() + } + res, err = client.sendRequestHelper(req) + return res, nil +} + +func (client *restClient) sendRequestHelper(req *request) (res *http.Response, err error) { + r, err := http.NewRequest(req.Method, req.URL, req.Body) + if err != nil { + return + } + if req.ContentType != "" { + r.Header.Add("Content-Type", req.ContentType) + } + if req.Tokens != nil && req.Tokens.AccessToken != "" { + r.Header.Add("Authorization", "Bearer "+req.Tokens.AccessToken) + } + res, err = client.httpClient.Do(r) + if err != nil { + return + } + + return +} + +// Reads an error out of the HTTP response, or does nothing if +// no error occured. +func getError(res *http.Response) (*http.Response, error) { + // Do nothing if the response is a successful 2xx + if res.StatusCode/100 == 2 { + return res, nil + } + var apiError APIError + // ReadAll is usually a bad practice, but here we need to read the response all + // at once because we may attempt to use the data twice. It's preferable to use + // methods that take io.Reader, e.g. json.NewDecoder + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + err = json.Unmarshal(body, &apiError) + if err != nil { + // If deserializing into ApiError fails, return a generic HttpError instead + return nil, HttpError{res.StatusCode, string(body[:])} + } + apiError.HttpStatusCode = res.StatusCode + return nil, apiError +} diff --git a/pkg/cloudprovider/providers/cascade/tests_owed b/pkg/cloudprovider/providers/cascade/tests_owed new file mode 100644 index 0000000..dff5ab1 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/tests_owed @@ -0,0 +1,5 @@ + +Yu Sheng +Change-Id: Ifc11818f65a3e018aeea6988d9e2c0719b592920 + + diff --git a/pkg/cloudprovider/providers/cascade/utils.go b/pkg/cloudprovider/providers/cascade/utils.go new file mode 100644 index 0000000..866f853 --- /dev/null +++ b/pkg/cloudprovider/providers/cascade/utils.go @@ -0,0 +1,29 @@ +package cascade + +func StringPtr(s string) *string { + return &s +} + +// StringVal returns string from string pointer, nil returns "" +func StringVal(p *string) (s string) { + if p != nil { + s = *p + } + return +} + +func Int64Ptr(s int64) *int64 { + return &s +} + +func Int64Val(s *int64) int64 { + return *s +} + +func Int32Ptr(s int32) *int32 { + return &s +} + +func BoolPtr(s bool) *bool { + return &s +} diff --git a/pkg/cloudprovider/providers/providers.go b/pkg/cloudprovider/providers/providers.go index 7de9ca9..6d8a1d2 100644 --- a/pkg/cloudprovider/providers/providers.go +++ b/pkg/cloudprovider/providers/providers.go @@ -20,6 +20,7 @@ import ( // Cloud providers _ "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" _ "k8s.io/kubernetes/pkg/cloudprovider/providers/azure" + _ "k8s.io/kubernetes/pkg/cloudprovider/providers/cascade" _ "k8s.io/kubernetes/pkg/cloudprovider/providers/cloudstack" _ "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" _ "k8s.io/kubernetes/pkg/cloudprovider/providers/openstack" diff --git a/pkg/kubeapiserver/authorizer/config.go b/pkg/kubeapiserver/authorizer/config.go index 659f2ae..ed1f5f1 100644 --- a/pkg/kubeapiserver/authorizer/config.go +++ b/pkg/kubeapiserver/authorizer/config.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/auth/authorizer/node" "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac" "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" + "k8s.io/kubernetes/plugin/pkg/auth/authorizer/vke" ) type AuthorizationConfig struct { @@ -82,7 +83,12 @@ func (config AuthorizationConfig) New() (authorizer.Authorizer, authorizer.RuleR ) nodeAuthorizer := node.NewAuthorizer(graph, nodeidentifier.NewDefaultNodeIdentifier(), bootstrappolicy.NodeRules()) authorizers = append(authorizers, nodeAuthorizer) - + case modes.ModeVKE: + vkeAuthorizer, err := vke.NewAuthorizer() + if err != nil { + return nil, nil, err + } + authorizers = append(authorizers, vkeAuthorizer) case modes.ModeAlwaysAllow: alwaysAllowAuthorizer := authorizerfactory.NewAlwaysAllowAuthorizer() authorizers = append(authorizers, alwaysAllowAuthorizer) diff --git a/pkg/kubeapiserver/authorizer/modes/modes.go b/pkg/kubeapiserver/authorizer/modes/modes.go index 56a708a..6eb920f 100644 --- a/pkg/kubeapiserver/authorizer/modes/modes.go +++ b/pkg/kubeapiserver/authorizer/modes/modes.go @@ -23,9 +23,10 @@ const ( ModeWebhook string = "Webhook" ModeRBAC string = "RBAC" ModeNode string = "Node" + ModeVKE string = "VKE" ) -var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABAC, ModeWebhook, ModeRBAC, ModeNode} +var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABAC, ModeWebhook, ModeRBAC, ModeNode, ModeVKE} // IsValidAuthorizationMode returns true if the given authorization mode is a valid one for the apiserver func IsValidAuthorizationMode(authzMode string) bool { diff --git a/pkg/printers/internalversion/describe.go b/pkg/printers/internalversion/describe.go index c33b1c6..ef39b3b 100644 --- a/pkg/printers/internalversion/describe.go +++ b/pkg/printers/internalversion/describe.go @@ -751,6 +751,8 @@ func describeVolumes(volumes []api.Volume, w PrefixWriter, space string) { printFlexVolumeSource(volume.VolumeSource.FlexVolume, w) case volume.VolumeSource.Flocker != nil: printFlockerVolumeSource(volume.VolumeSource.Flocker, w) + case volume.VolumeSource.CascadeDisk != nil: + printCascadeDiskVolumeSource(volume.VolumeSource.CascadeDisk, w) default: w.Write(LEVEL_1, "\n") } @@ -1101,6 +1103,13 @@ func printCSIPersistentVolumeSource(csi *api.CSIPersistentVolumeSource, w Prefix csi.Driver, csi.VolumeHandle, csi.ReadOnly) } +func printCascadeDiskVolumeSource(cascade *api.CascadeDiskVolumeSource, w PrefixWriter) { + w.Write(LEVEL_2, "Type:\tVKEDisk (a Persistent Disk resource in VKE)\n"+ + " DiskID:\t%v\n"+ + " FSType:\t%v\n", + cascade.DiskID, cascade.FSType) +} + type PersistentVolumeDescriber struct { clientset.Interface } @@ -1189,6 +1198,8 @@ func describePersistentVolume(pv *api.PersistentVolume, events *api.EventList) ( printFlockerVolumeSource(pv.Spec.Flocker, w) case pv.Spec.CSI != nil: printCSIPersistentVolumeSource(pv.Spec.CSI, w) + case pv.Spec.CascadeDisk != nil: + printCascadeDiskVolumeSource(pv.Spec.CascadeDisk, w) default: w.Write(LEVEL_1, "\n") } diff --git a/pkg/security/podsecuritypolicy/util/util.go b/pkg/security/podsecuritypolicy/util/util.go index d654f88..422fe0d 100644 --- a/pkg/security/podsecuritypolicy/util/util.go +++ b/pkg/security/podsecuritypolicy/util/util.go @@ -68,6 +68,7 @@ func GetAllFSTypesAsSet() sets.String { string(extensions.PortworxVolume), string(extensions.ScaleIO), string(extensions.CSI), + string(extensions.CascadeDisk), ) return fstypes } @@ -129,6 +130,8 @@ func GetVolumeFSType(v api.Volume) (extensions.FSType, error) { return extensions.PortworxVolume, nil case v.ScaleIO != nil: return extensions.ScaleIO, nil + case v.CascadeDisk != nil: + return extensions.CascadeDisk, nil } return "", fmt.Errorf("unknown volume type for volume: %#v", v) diff --git a/pkg/volume/cascade_disk/BUILD b/pkg/volume/cascade_disk/BUILD new file mode 100644 index 0000000..3386612 --- /dev/null +++ b/pkg/volume/cascade_disk/BUILD @@ -0,0 +1,43 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_library( + name = "go_default_library", + srcs = [ + "attacher.go", + "cascade_disk.go", + "cascade_util.go", + ], + deps = [ + "//pkg/cloudprovider:go_default_library", + "//pkg/cloudprovider/providers/cascade:go_default_library", + "//pkg/util/mount:go_default_library", + "//pkg/util/strings:go_default_library", + "//pkg/volume:go_default_library", + "//pkg/volume/util:go_default_library", + "//pkg/volume/util/volumehelper:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/volume/cascade_disk/OWNERS b/pkg/volume/cascade_disk/OWNERS new file mode 100644 index 0000000..c3a4ed7 --- /dev/null +++ b/pkg/volume/cascade_disk/OWNERS @@ -0,0 +1,2 @@ +maintainers: +- ashokc diff --git a/pkg/volume/cascade_disk/attacher.go b/pkg/volume/cascade_disk/attacher.go new file mode 100644 index 0000000..66b5836 --- /dev/null +++ b/pkg/volume/cascade_disk/attacher.go @@ -0,0 +1,265 @@ +package cascade_disk + +import ( + "fmt" + "os" + "path" + "time" + + "github.com/golang/glog" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/cloudprovider/providers/cascade" + "k8s.io/kubernetes/pkg/util/mount" + "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" +) + +type cascadeDiskAttacher struct { + host volume.VolumeHost + cascadeDisks cascade.Disks +} + +var _ volume.Attacher = &cascadeDiskAttacher{} +var _ volume.AttachableVolumePlugin = &cascadeDiskPlugin{} + +func (plugin *cascadeDiskPlugin) NewAttacher() (volume.Attacher, error) { + cascadeCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) + if err != nil { + glog.Errorf("Cascade attacher: NewAttacher failed to get cloud provider") + return nil, err + } + + return &cascadeDiskAttacher{ + host: plugin.host, + cascadeDisks: cascadeCloud, + }, nil +} + +// Attach attaches the volume specified by the given spec to the given host. On success, returns the device path where +// the device was attached on the node. +func (attacher *cascadeDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) { + hostName := string(nodeName) + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Cascade attacher: Attach failed to get volume source") + return "", err + } + + // cascadeDisks.AttachDisk checks if disk is already attached to the node. So we don't have to do that separately + // here. + glog.V(4).Infof("Cascade: Attach disk called for host %s", hostName) + devicePath, err := attacher.cascadeDisks.AttachDisk(volumeSource.DiskID, nodeName) + if err != nil { + glog.Errorf("Error attaching volume %q to node %q: %+v", volumeSource.DiskID, nodeName, err) + return "", err + } + return devicePath, nil +} + +// VolumesAreAttached verifies whether the volumes specified in the spec are attached to the specified node. +func (attacher *cascadeDiskAttacher) VolumesAreAttached(specs []*volume.Spec, + nodeName types.NodeName) (map[*volume.Spec]bool, error) { + volumesAttachedCheck := make(map[*volume.Spec]bool) + volumeSpecMap := make(map[string]*volume.Spec) + diskIDList := []string{} + for _, spec := range specs { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Error getting volume (%q) source : %v", spec.Name(), err) + continue + } + + diskIDList = append(diskIDList, volumeSource.DiskID) + volumesAttachedCheck[spec] = true + volumeSpecMap[volumeSource.DiskID] = spec + } + attachedResult, err := attacher.cascadeDisks.DisksAreAttached(diskIDList, nodeName) + if err != nil { + glog.Errorf( + "Error checking if volumes (%v) are attached to current node (%q). err=%v", + diskIDList, nodeName, err) + return volumesAttachedCheck, err + } + + for diskID, attached := range attachedResult { + if !attached { + spec := volumeSpecMap[diskID] + volumesAttachedCheck[spec] = false + glog.V(2).Infof("VolumesAreAttached: check volume %q (specName: %q) is no longer attached", + diskID, spec.Name()) + } + } + return volumesAttachedCheck, nil +} + +// WaitForAttach waits until the devicePath returned by the Attach call is available. +func (attacher *cascadeDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, + timeout time.Duration) (string, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Cascade attacher: WaitForAttach failed to get volume source") + return "", err + } + + if devicePath == "" { + return "", fmt.Errorf("WaitForAttach failed for disk %s: devicePath is empty.", volumeSource.DiskID) + } + + ticker := time.NewTicker(checkSleepDuration) + defer ticker.Stop() + + timer := time.NewTimer(timeout) + defer timer.Stop() + + for { + select { + case <-ticker.C: + glog.V(4).Infof("Checking disk %s is attached", volumeSource.DiskID) + devicePath := getDiskByIdPath(devicePath) + checkPath, err := verifyDevicePath(devicePath) + if err != nil { + // Log error, if any, and continue checking periodically. See issue #11321 + glog.Warningf("VKE attacher: WaitForAttach with devicePath %s Checking PD %s Error verify "+ + "path", devicePath, volumeSource.DiskID) + } else if checkPath != "" { + // A device path has successfully been created for the disk + glog.V(4).Infof("Successfully found attached disk %s.", volumeSource.DiskID) + return checkPath, nil + } + case <-timer.C: + return "", fmt.Errorf("Could not find attached disk %s. Timeout waiting for mount paths to be "+ + "created.", volumeSource.DiskID) + } + } +} + +// GetDeviceMountPath returns a path where the device should point which should be bind mounted for individual volumes. +func (attacher *cascadeDiskAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Cascade attacher: GetDeviceMountPath failed to get volume source") + return "", err + } + + return makeGlobalPDPath(attacher.host, volumeSource.DiskID), nil +} + +// GetMountDeviceRefs finds all other references to the device referenced by deviceMountPath; returns a list of paths. +func (plugin *cascadeDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { + mounter := plugin.host.GetMounter(plugin.GetPluginName()) + return mount.GetMountRefs(mounter, deviceMountPath) +} + +// MountDevice mounts device to global mount point. +func (attacher *cascadeDiskAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string) error { + mounter := attacher.host.GetMounter(cascadeDiskPluginName) + notMnt, err := mounter.IsLikelyNotMountPoint(deviceMountPath) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(deviceMountPath, 0750); err != nil { + glog.Errorf("Failed to create directory at %#v. err: %s", deviceMountPath, err) + return err + } + notMnt = true + } else { + return err + } + } + + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Cascade attacher: MountDevice failed to get volume source. err: %s", err) + return err + } + + options := []string{} + + if notMnt { + diskMounter := volumehelper.NewSafeFormatAndMountFromHost(cascadeDiskPluginName, attacher.host) + mountOptions := volume.MountOptionFromSpec(spec) + err = diskMounter.FormatAndMount(devicePath, deviceMountPath, volumeSource.FSType, mountOptions) + if err != nil { + os.Remove(deviceMountPath) + return err + } + glog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", + spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options) + } + return nil +} + +type cascadeDiskDetacher struct { + mounter mount.Interface + cascadeDisks cascade.Disks +} + +var _ volume.Detacher = &cascadeDiskDetacher{} + +// NewDetacher returns the detacher associated with the Cascade volume plugin. +func (plugin *cascadeDiskPlugin) NewDetacher() (volume.Detacher, error) { + cascadeCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) + if err != nil { + glog.Errorf("Cascade attacher: NewDetacher failed to get cloud provider. err: %s", err) + return nil, err + } + + return &cascadeDiskDetacher{ + mounter: plugin.host.GetMounter(plugin.GetPluginName()), + cascadeDisks: cascadeCloud, + }, nil +} + +// Detach detaches the given device from the given host. +func (detacher *cascadeDiskDetacher) Detach(deviceMountPath string, nodeName types.NodeName) error { + hostName := string(nodeName) + diskID := path.Base(deviceMountPath) + attached, err := detacher.cascadeDisks.DiskIsAttached(diskID, nodeName) + if err != nil { + // Log error and continue with detach + glog.Errorf( + "Error checking if persistent disk (%q) is already attached to current node (%q). "+ + "Will continue and try detach anyway. err=%v", diskID, hostName, err) + } + + if err == nil && !attached { + // Volume is already detached from node. + glog.V(4).Infof("detach operation was successful. persistent disk %q is already detached "+ + "from node %q.", diskID, hostName) + return nil + } + + if err := detacher.cascadeDisks.DetachDisk(diskID, nodeName); err != nil { + glog.Errorf("Error detaching volume %q: %v", diskID, err) + return err + } + return nil +} + +// WaitForDetach waits for the devicePath to become unavailable. +func (detacher *cascadeDiskDetacher) WaitForDetach(devicePath string, timeout time.Duration) error { + ticker := time.NewTicker(checkSleepDuration) + defer ticker.Stop() + timer := time.NewTimer(timeout) + defer timer.Stop() + + for { + select { + case <-ticker.C: + glog.V(4).Infof("Checking device %q is detached.", devicePath) + if pathExists, err := volumeutil.PathExists(devicePath); err != nil { + return fmt.Errorf("Error checking if device path exists: %v", err) + } else if !pathExists { + return nil + } + case <-timer.C: + return fmt.Errorf("Timeout reached; Device %v is still attached", devicePath) + } + } +} + +// UnmountDevice unmounts the disk specified by the device mount path. +func (detacher *cascadeDiskDetacher) UnmountDevice(deviceMountPath string) error { + return volumeutil.UnmountPath(deviceMountPath, detacher.mounter) +} diff --git a/pkg/volume/cascade_disk/azure_disk_util.go b/pkg/volume/cascade_disk/azure_disk_util.go new file mode 100644 index 0000000..036b420 --- /dev/null +++ b/pkg/volume/cascade_disk/azure_disk_util.go @@ -0,0 +1,136 @@ +package cascade_disk + +import ( + "io/ioutil" + "os" + "path" + "strconv" + "strings" + + "github.com/golang/glog" +) + +const ( + environmentFilePath = "/etc/kubernetes/environment" + diskPath = "/dev/disk/cloud/" + sysPath = "/sys/bus/scsi/devices" + lunIndex = 3 + maxOsScsiHostNo = 3 + vendor = "vendor" + msftVendor = "MSFT" + model = "model" + vhdModel = "VIRTUAL DISK" + block = "block" +) + + +func findAzureVolume(lun int) (device string, err error) { + azureDisks := listAzureDiskPath() + return findDiskByLunWithConstraint(lun, azureDisks) +} + +// List all the devices that are used by azure. +// All these devices are the symbolic link under /dev/disk/cloud/ +// eg. azure_resource -> ../../sdb +func listAzureDiskPath() []string { + var azureDiskList []string + if dirs, err := ioutil.ReadDir(diskPath); err == nil { + for _, f := range dirs { + name := f.Name() + diskPath := diskPath + name + if link, linkErr := os.Readlink(diskPath); linkErr == nil { + sd := link[(strings.LastIndex(link, "/") + 1):] + azureDiskList = append(azureDiskList, sd) + } + } + } + glog.V(12).Infof("Azure sys disks paths: %v", azureDiskList) + return azureDiskList +} + +// Azure attach/detach APIs return the disk model that does not have disk +// device path name (eg. /dev/sde) instead it gives the lun number of the disk +// getting attached/detached. +// This function maps that lun number to device path name of the +// device mounted to "current" node. +// Refrence : https://github.com/kubernetes/kubernetes/blob/master/pkg/volume/azure_dd/azure_common_linux.go#L69 +// https://docs.microsoft.com/en-us/azure/virtual-machines/linux/troubleshoot-device-names-problems +func findDiskByLunWithConstraint(lun int, azureDisks []string) (string, error) { + var err error + if dirs, err := ioutil.ReadDir(sysPath); err == nil { + for _, f := range dirs { + name := f.Name() + // look for path like /sys/bus/scsi/devices/3:0:0:1 + arr := strings.Split(name, ":") + if len(arr) < 4 { + continue + } + if len(azureDisks) == 0 { + glog.V(4).Infof("/dev/disk/cloud is not populated, now try to parse %v directly", name) + target, err := strconv.Atoi(arr[0]) + if err != nil { + glog.Errorf("failed to parse target from %v (%v), err %v", arr[0], name, err) + continue + } + // as observed, targets 0-3 are used by OS disks. Skip them + if target <= maxOsScsiHostNo { + continue + } + } + + // extract LUN from the path. + // LUN is the last index of the array, i.e. 1 in /sys/bus/scsi/devices/3:0:0:1 + l, err := strconv.Atoi(arr[lunIndex]) + if err != nil { + // unknown path format, continue to read the next one + glog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[lunIndex], name, err) + continue + } + if lun == l { + // find the matching LUN + // read vendor and model to ensure it is a VHD disk + vendorPath := path.Join(sysPath, name, vendor) + vendorBytes, err := ioutil.ReadFile(vendorPath) + if err != nil { + glog.Errorf("failed to read device vendor, err: %v", err) + continue + } + vendor := strings.TrimSpace(string(vendorBytes)) + if strings.ToUpper(vendor) != msftVendor { + glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor) + continue + } + + modelPath := path.Join(sysPath, name, model) + modelBytes, err := ioutil.ReadFile(modelPath) + if err != nil { + glog.Errorf("failed to read device model, err: %v", err) + continue + } + model := strings.TrimSpace(string(modelBytes)) + if strings.ToUpper(model) != vhdModel { + glog.V(4).Infof("model doesn't match VHD, got %s", model) + continue + } + + // find a disk, validate name + dir := path.Join(sysPath, name, block) + if dev, err := ioutil.ReadDir(dir); err == nil { + found := false + devName := dev[0].Name() + for _, diskName := range azureDisks { + glog.V(12).Infof("azureDisk - validating disk %q with sys disk %q", devName, diskName) + if devName == diskName { + found = true + break + } + } + if !found { + return "/dev/" + devName, nil + } + } + } + } + } + return "", err +} diff --git a/pkg/volume/cascade_disk/cascade_disk.go b/pkg/volume/cascade_disk/cascade_disk.go new file mode 100644 index 0000000..a25f224 --- /dev/null +++ b/pkg/volume/cascade_disk/cascade_disk.go @@ -0,0 +1,391 @@ +package cascade_disk + +import ( + "fmt" + "os" + "path" + + "github.com/golang/glog" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/util/mount" + utilstrings "k8s.io/kubernetes/pkg/util/strings" + "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/util" + "k8s.io/kubernetes/pkg/volume/util/volumehelper" +) + +// This is the primary entrypoint for volume plugins. +func ProbeVolumePlugins() []volume.VolumePlugin { + return []volume.VolumePlugin{&cascadeDiskPlugin{}} +} + +type cascadeDiskPlugin struct { + host volume.VolumeHost +} + +var _ volume.VolumePlugin = &cascadeDiskPlugin{} +var _ volume.PersistentVolumePlugin = &cascadeDiskPlugin{} +var _ volume.DeletableVolumePlugin = &cascadeDiskPlugin{} +var _ volume.ProvisionableVolumePlugin = &cascadeDiskPlugin{} + +const ( + cascadeDiskPluginName = "kubernetes.io/vke-disk" +) + +// Init initializes the Cascade volume plugin. +func (plugin *cascadeDiskPlugin) Init(host volume.VolumeHost) error { + plugin.host = host + return nil +} + +// GetPluginName returns the name of the Cascade volume plugin. +func (plugin *cascadeDiskPlugin) GetPluginName() string { + return cascadeDiskPluginName +} + +// GetVolumeName returns the name of the volume which is the diskID in our case. +func (plugin *cascadeDiskPlugin) GetVolumeName(spec *volume.Spec) (string, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Cascade volume plugin: GetVolumeName failed to get volume source") + return "", err + } + + return volumeSource.DiskID, nil +} + +// CanSupport specifies whether the Cascade volume plguin can support the specific resource type. +// Cascade plugin only supports the persistent volume and volume resource which has the Cascade disk annotation. +func (plugin *cascadeDiskPlugin) CanSupport(spec *volume.Spec) bool { + return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CascadeDisk != nil) || + (spec.Volume != nil && spec.Volume.CascadeDisk != nil) +} + +// RequiresRemount specifies whether remount is required for the disk. +func (plugin *cascadeDiskPlugin) RequiresRemount() bool { + return false +} + +// SupportsMountOption specifies whether the Cascade volume plugin supports the mount operation. +func (plugin *cascadeDiskPlugin) SupportsMountOption() bool { + return true +} + +// SupportsBulkVolumeVerification specifies whether bulk volume verification is supported. +func (plugin *cascadeDiskPlugin) SupportsBulkVolumeVerification() bool { + return false +} + +// NewMounter returns the mounter associated with the Cascade volume plugin. +func (plugin *cascadeDiskPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, + _ volume.VolumeOptions) (volume.Mounter, error) { + return plugin.newMounterInternal(spec, pod.UID, &CascadeDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName())) +} + +// NewUnmounter returns the unmounter associated with the Cascade volume plugin. +func (plugin *cascadeDiskPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) { + return plugin.newUnmounterInternal(volName, podUID, &CascadeDiskUtil{}, + plugin.host.GetMounter(plugin.GetPluginName())) +} + +func (plugin *cascadeDiskPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, + mounter mount.Interface) (volume.Mounter, error) { + volumeSource, _, err := getVolumeSource(spec) + if err != nil { + glog.Errorf("Cascade volume plugin: newMounterInternal failed to get volume source") + return nil, err + } + + diskID := volumeSource.DiskID + fsType := volumeSource.FSType + + return &cascadeDiskMounter{ + cascadeDisk: &cascadeDisk{ + podUID: podUID, + volName: spec.Name(), + diskID: diskID, + manager: manager, + mounter: mounter, + plugin: plugin, + }, + fsType: fsType, + diskMounter: volumehelper.NewSafeFormatAndMountFromHost(plugin.GetPluginName(), plugin.host)}, nil +} + +func (plugin *cascadeDiskPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, + mounter mount.Interface) (volume.Unmounter, error) { + return &cascadeDiskUnmounter{ + &cascadeDisk{ + podUID: podUID, + volName: volName, + manager: manager, + mounter: mounter, + plugin: plugin, + }}, nil +} + +// ConstructVolumeSpec constructs a Cascade volume spec based on the name and mount path. +func (plugin *cascadeDiskPlugin) ConstructVolumeSpec(volumeSpecName, mountPath string) (*volume.Spec, error) { + mounter := plugin.host.GetMounter(plugin.GetPluginName()) + pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName()) + diskID, err := mounter.GetDeviceNameFromMount(mountPath, pluginDir) + if err != nil { + return nil, err + } + + cascadeDisk := &v1.Volume{ + Name: volumeSpecName, + VolumeSource: v1.VolumeSource{ + CascadeDisk: &v1.CascadeDiskVolumeSource{ + DiskID: diskID, + }, + }, + } + return volume.NewSpecFromVolume(cascadeDisk), nil +} + +// Abstract interface to disk operations. +type diskManager interface { + // Creates a volume + CreateVolume(provisioner *cascadeDiskProvisioner) (diskID string, volumeSizeGB int, fstype string, err error) + // Deletes a volume + DeleteVolume(deleter *cascadeDiskDeleter) error +} + +// cascadeDisk volumes are disk resources attached to the kubelet's host machine and exposed to the pod. +type cascadeDisk struct { + volName string + podUID types.UID + diskID string + fsType string + manager diskManager + mounter mount.Interface + plugin *cascadeDiskPlugin + volume.MetricsNil +} + +var _ volume.Mounter = &cascadeDiskMounter{} + +type cascadeDiskMounter struct { + *cascadeDisk + fsType string + diskMounter *mount.SafeFormatAndMount +} + +// GetAttributes returns the attributes associated with a Cascade disk. +func (b *cascadeDiskMounter) GetAttributes() volume.Attributes { + return volume.Attributes{ + SupportsSELinux: true, + } +} + +// CanMount checks prior to mount operations to verify that the required components (binaries, etc.) to mount the +// volume are available on the underlying node. If not, it returns an error. +func (b *cascadeDiskMounter) CanMount() error { + return nil +} + +// SetUp attaches the disk and bind mounts to the volume path. +func (b *cascadeDiskMounter) SetUp(fsGroup *int64) error { + return b.SetUpAt(b.GetPath(), fsGroup) +} + +// SetUpAt attaches the disk and bind mounts to the volume path. +func (b *cascadeDiskMounter) SetUpAt(dir string, fsGroup *int64) error { + glog.V(4).Infof("Cascade Persistent Disk setup %s to %s", b.diskID, dir) + + // TODO: handle failed mounts here. + notmnt, err := b.mounter.IsLikelyNotMountPoint(dir) + if err != nil && !os.IsNotExist(err) { + glog.Errorf("cannot validate mount point: %s %v", dir, err) + return err + } + if !notmnt { + return nil + } + + if err := os.MkdirAll(dir, 0750); err != nil { + glog.Errorf("mkdir failed on disk %s (%v)", dir, err) + return err + } + + options := []string{"bind"} + + // Perform a bind mount to the full path to allow duplicate mounts of the same PD. + globalPDPath := makeGlobalPDPath(b.plugin.host, b.diskID) + glog.V(4).Infof("attempting to mount %s", dir) + + err = b.mounter.Mount(globalPDPath, dir, "", options) + if err != nil { + notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) + if mntErr != nil { + glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + return err + } + if !notmnt { + if mntErr = b.mounter.Unmount(dir); mntErr != nil { + glog.Errorf("Failed to unmount: %v", mntErr) + return err + } + notmnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) + if mntErr != nil { + glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) + return err + } + if !notmnt { + glog.Errorf("%s is still mounted, despite call to unmount(). Will try again next sync loop.", + b.GetPath()) + return err + } + } + os.Remove(dir) + glog.Errorf("Mount of disk %s failed: %v", dir, err) + return err + } + volume.SetVolumeOwnership(b, fsGroup) + + return nil +} + +var _ volume.Unmounter = &cascadeDiskUnmounter{} + +type cascadeDiskUnmounter struct { + *cascadeDisk +} + +// TearDown unmounts the bind mount, and detaches the disk only if the disk resource was the last reference to that +// disk on the kubelet. +func (c *cascadeDiskUnmounter) TearDown() error { + return c.TearDownAt(c.GetPath()) +} + +// TearDownAt unmounts the bind mount, and detaches the disk only if the disk resource was the last reference to that +// disk on the kubelet. +func (c *cascadeDiskUnmounter) TearDownAt(dir string) error { + return util.UnmountPath(dir, c.mounter) +} + +func makeGlobalPDPath(host volume.VolumeHost, diskID string) string { + return path.Join(host.GetPluginDir(cascadeDiskPluginName), mount.MountsInGlobalPDPath, diskID) +} + +func (cd *cascadeDisk) GetPath() string { + name := cascadeDiskPluginName + return cd.plugin.host.GetPodVolumeDir(cd.podUID, utilstrings.EscapeQualifiedNameForDisk(name), cd.volName) +} + +func (plugin *cascadeDiskPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { + return []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + } +} + +type cascadeDiskDeleter struct { + *cascadeDisk +} + +var _ volume.Deleter = &cascadeDiskDeleter{} + +// NewDeleter returns the deleter associated with the Cascade volume plugin. +func (plugin *cascadeDiskPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) { + return plugin.newDeleterInternal(spec, &CascadeDiskUtil{}) +} + +func (plugin *cascadeDiskPlugin) newDeleterInternal(spec *volume.Spec, manager diskManager) (volume.Deleter, error) { + if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CascadeDisk == nil { + return nil, fmt.Errorf("spec.PersistentVolumeSource.CascadeDisk is nil") + } + return &cascadeDiskDeleter{ + &cascadeDisk{ + volName: spec.Name(), + diskID: spec.PersistentVolume.Spec.CascadeDisk.DiskID, + manager: manager, + plugin: plugin, + }}, nil +} + +func (r *cascadeDiskDeleter) Delete() error { + return r.manager.DeleteVolume(r) +} + +type cascadeDiskProvisioner struct { + *cascadeDisk + options volume.VolumeOptions +} + +var _ volume.Provisioner = &cascadeDiskProvisioner{} + +// NewProvisioner returns the provisioner associated with the Cascade volume plugin. +func (plugin *cascadeDiskPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) { + return plugin.newProvisionerInternal(options, &CascadeDiskUtil{}) +} + +func (plugin *cascadeDiskPlugin) newProvisionerInternal(options volume.VolumeOptions, + manager diskManager) (volume.Provisioner, error) { + return &cascadeDiskProvisioner{ + cascadeDisk: &cascadeDisk{ + manager: manager, + plugin: plugin, + }, + options: options, + }, nil +} + +// Provision provisions the persistent volume by making a CreateDisk call to Cascade Controller. +func (p *cascadeDiskProvisioner) Provision() (*v1.PersistentVolume, error) { + if !volume.AccessModesContainedInAll(p.plugin.GetAccessModes(), p.options.PVC.Spec.AccessModes) { + return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", + p.options.PVC.Spec.AccessModes, p.plugin.GetAccessModes()) + } + + diskID, sizeGB, fstype, err := p.manager.CreateVolume(p) + if err != nil { + return nil, err + } + + if fstype == "" { + fstype = "ext4" + } + + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: p.options.PVName, + Labels: map[string]string{}, + Annotations: map[string]string{ + volumehelper.VolumeDynamicallyCreatedByKey: "vke-volume-dynamic-provisioner", + }, + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: p.options.PersistentVolumeReclaimPolicy, + AccessModes: p.options.PVC.Spec.AccessModes, + Capacity: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dGi", sizeGB)), + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + CascadeDisk: &v1.CascadeDiskVolumeSource{ + DiskID: diskID, + FSType: fstype, + }, + }, + MountOptions: p.options.MountOptions, + }, + } + if len(p.options.PVC.Spec.AccessModes) == 0 { + pv.Spec.AccessModes = p.plugin.GetAccessModes() + } + + return pv, nil +} + +func getVolumeSource(spec *volume.Spec) (*v1.CascadeDiskVolumeSource, bool, error) { + if spec.Volume != nil && spec.Volume.CascadeDisk != nil { + return spec.Volume.CascadeDisk, spec.ReadOnly, nil + } else if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CascadeDisk != nil { + return spec.PersistentVolume.Spec.CascadeDisk, spec.ReadOnly, nil + } + + return nil, false, fmt.Errorf("Spec does not reference a Cascade disk type") +} \ No newline at end of file diff --git a/pkg/volume/cascade_disk/cascade_util.go b/pkg/volume/cascade_disk/cascade_util.go new file mode 100644 index 0000000..2604b89 --- /dev/null +++ b/pkg/volume/cascade_disk/cascade_util.go @@ -0,0 +1,201 @@ +package cascade_disk + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/golang/glog" + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/cascade" + "k8s.io/kubernetes/pkg/volume" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +const ( + checkSleepDuration = time.Second +) + +type CascadeDiskUtil struct{} + +func verifyDevicePath(path string) (string, error) { + if pathExists, err := volumeutil.PathExists(path); err != nil { + return "", fmt.Errorf("Error checking if path exists: %v", err) + } else if pathExists { + return path, nil + } + + glog.V(4).Infof("verifyDevicePath: path does not exist yet") + return "", nil +} + +// Returns path for given VKE disk mount +func getDiskByIdPath(devicePath string) string { + provider, err := getProvider() + if err != nil { + glog.Warningf("error getting provider name from %s: %v", environmentFilePath, err) + return devicePath + } + + switch provider { + case "azure": + // devicePath is an integer then it is lun returned from azure attach disk. + if diskLun, err := strconv.Atoi(devicePath); err == nil { + azDevicePath, err := findAzureVolume(diskLun) + if err != nil { + glog.Warningf("error looking for azure volume %q: %v", devicePath, err) + } else if azDevicePath != "" { + devicePath = azDevicePath + } + } else { + glog.Warningf("The device path for azure expected to be an integer but it is %s. Error : %v", devicePath, err) + } + default: + nvmePath, err := findNvmeVolume(devicePath) + if err != nil { + glog.Warningf("error looking for nvme volume %q: %v", devicePath, err) + } else if nvmePath != "" { + devicePath = nvmePath + } + } + return devicePath +} + +// CreateVolume creates a Cascade persistent disk. +func (util *CascadeDiskUtil) CreateVolume(p *cascadeDiskProvisioner) (diskID string, capacityGB int, fstype string, + err error) { + cloud, err := getCloudProvider(p.plugin.host.GetCloudProvider()) + if err != nil { + glog.Errorf("Cascade Util: CreateVolume failed to get cloud provider. Error [%v]", err) + return "", 0, "", err + } + + capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] + volSizeBytes := capacity.Value() + // Cascade works with GB, convert to GB with rounding up + volSizeGB := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024)) + name := volume.GenerateVolumeName(p.options.ClusterName, p.options.PVName, 255) + volumeOptions := &cascade.VolumeOptions{ + CapacityGB: volSizeGB, + Tags: *p.options.CloudTags, + Name: name, + } + + // enabling encryption by default if not provided from within storage class + volumeOptions.Encrypted = true + for parameter, value := range p.options.Parameters { + switch strings.ToLower(parameter) { + case "flavor": + volumeOptions.Flavor = value + case "encrypted": + volumeOptions.Encrypted, err = strconv.ParseBool(value) + if err != nil { + glog.Errorf("Cascade Util: invalid value %q for encryption of volume plugin %s.", value, + p.plugin.GetPluginName()) + return "", 0, "", fmt.Errorf("invalid encrypted boolean value %q, must be true or false: %v", value, err) + } + case volume.VolumeParameterFSType: + fstype = value + glog.V(4).Infof("Cascade Util: Setting fstype to %s", fstype) + default: + glog.Errorf("Cascade Util: invalid option %s for volume plugin %s.", parameter, + p.plugin.GetPluginName()) + return "", 0, "", fmt.Errorf("Cascade Util: invalid option %s for volume plugin %s.", parameter, + p.plugin.GetPluginName()) + } + } + + diskID, err = cloud.CreateDisk(volumeOptions) + if err != nil { + glog.Errorf("Cascade Util: failed to CreateDisk. Error [%v]", err) + return "", 0, "", err + } + + glog.V(4).Infof("Successfully created Cascade persistent disk %s", name) + return diskID, volSizeGB, "", nil +} + +// DeleteVolume deletes a Cascade volume. +func (util *CascadeDiskUtil) DeleteVolume(disk *cascadeDiskDeleter) error { + cloud, err := getCloudProvider(disk.plugin.host.GetCloudProvider()) + if err != nil { + glog.Errorf("Cascade Util: DeleteVolume failed to get cloud provider. Error [%v]", err) + return err + } + + if err = cloud.DeleteDisk(disk.diskID); err != nil { + glog.Errorf("Cascade Util: failed to DeleteDisk for diskID %s. Error [%v]", disk.diskID, err) + return err + } + + glog.V(4).Infof("Successfully deleted Cascade persistent disk %s", disk.diskID) + return nil +} + +func getCloudProvider(cloud cloudprovider.Interface) (*cascade.CascadeCloud, error) { + if cloud == nil { + glog.Errorf("Cascade Util: Cloud provider not initialized properly") + return nil, fmt.Errorf("Cascade Util: Cloud provider not initialized properly") + } + + cc := cloud.(*cascade.CascadeCloud) + if cc == nil { + glog.Errorf("Invalid cloud provider: expected Cascade") + return nil, fmt.Errorf("Invalid cloud provider: expected Cascade") + } + return cc, nil +} + +// findNvmeVolume looks for the nvme volume with the specified name +// It follows the symlink (if it exists) and returns the absolute path to the device +func findNvmeVolume(findName string) (device string, err error) { + stat, err := os.Lstat(findName) + if err != nil { + if os.IsNotExist(err) { + glog.V(6).Infof("nvme path not found %q", findName) + return "", nil + } + return "", fmt.Errorf("error getting stat of %q: %v", findName, err) + } + + if stat.Mode()&os.ModeSymlink != os.ModeSymlink { + glog.Warningf("nvme file %q found, but was not a symlink", findName) + return "", nil + } + + // Find the target, resolving to an absolute path + // For example, /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol0fab1d5e3f72a5e23 -> ../../nvme2n1 + resolved, err := filepath.EvalSymlinks(findName) + if err != nil { + return "", fmt.Errorf("error reading target of symlink %q: %v", findName, err) + } + + if !strings.HasPrefix(resolved, "/dev") { + return "", fmt.Errorf("resolved symlink for %q was unexpected: %q", findName, resolved) + } + + return resolved, nil +} + +func getProvider() (provider string, err error) { + file, err := os.Open(environmentFilePath) + if err != nil { + return "", err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + lineContent := scanner.Text() + if strings.Contains(lineContent, "PROVIDER=") { + return strings.SplitAfter(lineContent, "=")[1], err + } + } + return "", err +} diff --git a/plugin/pkg/admission/persistentvolume/label/admission.go b/plugin/pkg/admission/persistentvolume/label/admission.go index 86e1921..bf2912b 100644 --- a/plugin/pkg/admission/persistentvolume/label/admission.go +++ b/plugin/pkg/admission/persistentvolume/label/admission.go @@ -27,6 +27,7 @@ import ( api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" + "k8s.io/kubernetes/pkg/cloudprovider/providers/cascade" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" @@ -50,6 +51,7 @@ type persistentVolumeLabel struct { ebsVolumes aws.Volumes cloudConfig []byte gceCloudProvider *gce.GCECloud + cascadeDisks cascade.Disks } var _ admission.MutationInterface = &persistentVolumeLabel{} @@ -102,6 +104,13 @@ func (l *persistentVolumeLabel) Admit(a admission.Attributes) (err error) { } volumeLabels = labels } + if volume.Spec.CascadeDisk != nil { + labels, err := l.findCascadeDiskLabels(volume) + if err != nil { + return admission.NewForbidden(a, fmt.Errorf("error querying Cascade volume %s: %v", volume.Spec.CascadeDisk.DiskID, err)) + } + volumeLabels = labels + } if len(volumeLabels) != 0 { if volume.Labels == nil { @@ -214,3 +223,48 @@ func (l *persistentVolumeLabel) getGCECloudProvider() (*gce.GCECloud, error) { } return l.gceCloudProvider, nil } + +func (l *persistentVolumeLabel) findCascadeDiskLabels(volume *api.PersistentVolume) (map[string]string, error) { + // Ignore any volumes that are being provisioned + if volume.Spec.CascadeDisk.DiskID == vol.ProvisionedVolumeName { + return nil, nil + } + cascadeDisks, err := l.getCascadeDisks() + if err != nil { + return nil, err + } + if cascadeDisks == nil { + return nil, fmt.Errorf("unable to build Cascade cloud provider for volumes") + } + + labels, err := cascadeDisks.GetVolumeLabels(volume.Spec.CascadeDisk.DiskID) + if err != nil { + return nil, err + } + + return labels, nil +} + +// getCascadeDisks returns the Cascade Disks interface +func (l *persistentVolumeLabel) getCascadeDisks() (cascade.Disks, error) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.cascadeDisks == nil { + var cloudConfigReader io.Reader + if len(l.cloudConfig) > 0 { + cloudConfigReader = bytes.NewReader(l.cloudConfig) + } + cloudProvider, err := cloudprovider.GetCloudProvider("vke", cloudConfigReader) + if err != nil || cloudProvider == nil { + return nil, err + } + provider, ok := cloudProvider.(*cascade.CascadeCloud) + if !ok { + // GetCloudProvider has gone very wrong + return nil, fmt.Errorf("error retrieving Cascade cloud provider") + } + l.cascadeDisks = provider + } + return l.cascadeDisks, nil +} diff --git a/plugin/pkg/admission/vke/BUILD b/plugin/pkg/admission/vke/BUILD new file mode 100644 index 0000000..7d66036 --- /dev/null +++ b/plugin/pkg/admission/vke/BUILD @@ -0,0 +1,61 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["admission.go"], + importpath = "k8s.io/kubernetes/plugin/pkg/admission/vke", + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/apis/extensions:go_default_library", + "//pkg/apis/extensions/v1beta1:go_default_library", + "//pkg/apis/rbac:go_default_library", + "//pkg/registry/rbac:go_default_library", + "//pkg/security/podsecuritypolicy:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/api/extensions/v1beta1:go_default_library" + "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["admission_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/apis/extensions:go_default_library", + "//pkg/apis/rbac:go_default_library", + "//pkg/registry/rbac:go_default_library", + "//pkg/security/podsecuritypolicy:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) \ No newline at end of file diff --git a/plugin/pkg/admission/vke/admission.go b/plugin/pkg/admission/vke/admission.go new file mode 100644 index 0000000..192f384 --- /dev/null +++ b/plugin/pkg/admission/vke/admission.go @@ -0,0 +1,587 @@ +package vke + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/golang/glog" + "k8s.io/api/extensions/v1beta1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/apiserver/pkg/admission" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/apis/extensions" + policybeta "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" + "k8s.io/kubernetes/pkg/apis/rbac" + rbacregistry "k8s.io/kubernetes/pkg/registry/rbac" + "k8s.io/kubernetes/pkg/security/podsecuritypolicy" +) + +const ( + // PluginName indicates name of admission plugin. + PluginName = "VMwareAdmissionController" + + systemUnsecuredUser = "system:unsecured" + systemNodesGroup = "system:nodes" + privilegedNamespace = "vke-system" + privilegedServiceAccount = "system:serviceaccount:" + privilegedNamespace + ":" + reservedPrefix = "vke" + reservedTolerationKey = "Dedicated" + reservedTolerationValue = "Master" + masterNodePrefix = "master" + etcSslCerts = "/etc/ssl/certs" +) + +// Register registers a plugin. +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { + return NewVMwareAdmissionController(config) + }) +} + +// vmwareAdmissionController is an implementation of admission.Interface. +// It restricts access to VKE namespace for users and prevents the users from creating privileged pods. +type vmwareAdmissionController struct { + psp *extensions.PodSecurityPolicy + strategyFactory podsecuritypolicy.StrategyFactory + privilegedGroup string + clusterID string +} + +// vmwareAdmissionControllerConfig holds config data for VMwareAdmissionController. +type vmwareAdmissionControllerConfig struct { + ClusterID string `yaml:"clusterID"` + PrivilegedGroup string `yaml:"privilegedGroup"` + PodSecurityPolicyFile string `yaml:"podSecurityPolicyFile"` +} + +// AdmissionConfig holds config data for admission controllers. +type AdmissionConfig struct { + VMwareAdmissionController vmwareAdmissionControllerConfig `yaml:"vmwareAdmissionController"` +} + +var _ admission.MutationInterface = &vmwareAdmissionController{} + +// Admit makes an admission decision based on the request attributes. +// Note: We implement the mutation interface for this admission controller, which means we have the ability to modify +// the resources. But we do not modify them because we just need to validate the requests to either let them through or +// block them. We have implemented this as a mutation interface because we need access to oldObjects during update +// operations to find out what values are being modified. For node update operation, the old and new objects are always +// identical in the validation phase (possibly due to some initializer modifying it). So, we are performing our +// validation during the mutation phase. +func (vac *vmwareAdmissionController) Admit(a admission.Attributes) (err error) { + if isPrivilegedUser(vac, a) { + return nil + } + + if isSystemUnsecuredUser(a) { + return validateSystemUnsecuredUser(vac, a) + } + + if isCertificateFromNode(a) { + return validateCertificateFromNode(a) + } + + if isPrivilegedServiceAccount(a) { + return validatePrivilegedServiceAccount(a) + } + + if isPrivilegedNamespace(a) { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: cannot modify resources in namespace %s", PluginName, a.GetNamespace())) + } + + switch a.GetResource().GroupResource() { + case api.Resource("pods"): + err = validatePods(vac, a) + case api.Resource("nodes"): + err = validateNodes(a) + case rbac.Resource("clusterroles"): + err = validateClusterRoles(a) + case rbac.Resource("clusterrolebindings"): + err = validateClusterRoleBindings(a) + } + + return err +} + +// Handles returns true if this admission controller can handle the given operation +// where operation can be one of CREATE, UPDATE, DELETE, or CONNECT. +func (vac *vmwareAdmissionController) Handles(operation admission.Operation) bool { + return true +} + +// NewVMwareAdmissionController creates a new VMwareAdmissionController. +func NewVMwareAdmissionController(configFile io.Reader) (*vmwareAdmissionController, error) { + glog.V(2).Infof("%s is enabled", PluginName) + if configFile == nil { + glog.Warningf("No config specified for %s. Using default configuration", PluginName) + return nil, fmt.Errorf("no config file specified for %s", PluginName) + } + + var config AdmissionConfig + d := yaml.NewYAMLOrJSONDecoder(configFile, 4096) + err := d.Decode(&config) + if err != nil { + return nil, err + } + + // Load PSP from file. If it fails, use default. + psp := getPSPFromFile(config.VMwareAdmissionController.PodSecurityPolicyFile) + if psp == nil { + psp = getDefaultPSP() + } + + return &vmwareAdmissionController{ + psp: psp, + strategyFactory: podsecuritypolicy.NewSimpleStrategyFactory(), + privilegedGroup: config.VMwareAdmissionController.PrivilegedGroup, + clusterID: config.VMwareAdmissionController.ClusterID, + }, nil +} + +func getDefaultPSP() *extensions.PodSecurityPolicy { + return &extensions.PodSecurityPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: "PodSecurityPolicy", + APIVersion: "extensions/v1beta1", + }, + Spec: extensions.PodSecurityPolicySpec{ + Privileged: false, + HostNetwork: false, + HostIPC: false, + HostPID: false, + AllowPrivilegeEscalation: false, + HostPorts: []extensions.HostPortRange{ + {1, 65535}, + }, + Volumes: []extensions.FSType{ + "emptyDir", + "secret", + "downwardAPI", + "configMap", + "persistentVolumeClaim", + "projected", + "hostPath", + }, + // We allow /etc/ssl/certs to be mounted in read only mode as a hack to allow Wavefront pods to be deployed. + // TODO(ashokc): Once we have support for users to create pods using privileged mode and host path, remove this. + AllowedHostPaths: []extensions.AllowedHostPath{ + { + etcSslCerts, + }, + }, + FSGroup: extensions.FSGroupStrategyOptions{ + Rule: extensions.FSGroupStrategyRunAsAny, + }, + RunAsUser: extensions.RunAsUserStrategyOptions{ + Rule: extensions.RunAsUserStrategyRunAsAny, + }, + SELinux: extensions.SELinuxStrategyOptions{ + Rule: extensions.SELinuxStrategyRunAsAny, + }, + SupplementalGroups: extensions.SupplementalGroupsStrategyOptions{ + Rule: extensions.SupplementalGroupsStrategyRunAsAny, + }, + }, + } +} + +func getPSPFromFile(pspFile string) *extensions.PodSecurityPolicy { + pspBeta := v1beta1.PodSecurityPolicy{} + pspExtensions := extensions.PodSecurityPolicy{} + + if pspFile == "" { + glog.V(2).Infof("%s: PSP file not specified, using default PSP", PluginName) + return nil + } + + pspConfig, err := os.Open(pspFile) + if err != nil { + glog.V(2).Infof("%s: cannot open PSP file, using default PSP: %v", PluginName, err) + return nil + } + + // We load the PSP that we read from file into pspBeta because this is the struct to which we can decode yaml to. + d := yaml.NewYAMLOrJSONDecoder(pspConfig, 4096) + err = d.Decode(&pspBeta) + if err != nil { + glog.V(2).Infof("%s: cannot decode PSP file, using default PSP: %v", PluginName, err) + return nil + } + + // We convert pspBeta object into pspExtensions object because this is the one that pod validation uses. + err = policybeta.Convert_v1beta1_PodSecurityPolicy_To_extensions_PodSecurityPolicy(&pspBeta, &pspExtensions, nil) + if err != nil { + glog.V(2).Infof("%s: cannot convert v1beta1.PSP to extensions.PSP, using default PSP: %v", PluginName, err) + return nil + } + + return &pspExtensions +} + +func isPrivilegedUser(vac *vmwareAdmissionController, a admission.Attributes) bool { + // If the request comes from a user belonging to a privileged group, then we allow it. Only calls from Cascade + // controller will belong to this privileged group. + groups := a.GetUserInfo().GetGroups() + for _, group := range groups { + if group == vac.privilegedGroup { + return true + } + } + + return false +} + +func isSystemUnsecuredUser(a admission.Attributes) bool { + return a.GetUserInfo().GetName() == systemUnsecuredUser +} + +func validateSystemUnsecuredUser(vac *vmwareAdmissionController, a admission.Attributes) (err error) { + // Currently the insecure port 8080 is exposed to only localhost inside the Kubernetes master VMs. So it can be used + // only by kube-controller-manager, kube-scheduler and cloud-init script which creates our pods and other resources. + // When a call comes on insecure port 8080, Kubernetes assigns them system:unsecured user name. We need to allow + // this so that our master components can be started successfully and kube-controller-manager and kube-scheduler can + // work as expected. + // But this needs to be allowed only inside our privileged namespace. If the request comes to any other namespace, + // we need to make it go through our pod validation. This is needed because a user can create a deployment or + // replica set which has a privileged pod. Since our admission controller does not look at deployments or replica + // sets, we will allow it. The actual pod inside the deployment or replica set will be created by the + // controller-manager and if we allow it to create pods in any namespace, then a user can create a privileged pod + // totally bypassing our security checks. + if !isPrivilegedNamespace(a) && (a.GetResource().GroupResource() == api.Resource("pods")) { + return validatePods(vac, a) + } + + // For all other resources, allow. + return nil +} + +func isCertificateFromNode(a admission.Attributes) bool { + // If the request came from a user with group = systemNodesGroup, then we assume that the request comes from a node + // which uses a certificate for authentication. + groups := a.GetUserInfo().GetGroups() + for _, group := range groups { + if group == systemNodesGroup { + return true + } + } + return false +} + +func validateCertificateFromNode(a admission.Attributes) error { + // Block exec operations into pods for nodes. This is needed to block someone from using Kubelet's certificate to + // exec into privileged pods running on the master. Other operations with the node certificate like modifying master + // node, creating pods on master node, etc. are blocked by the node restriction admission controller. + if a.GetResource().GroupResource() == api.Resource("pods") && a.GetOperation() == admission.Connect { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: cannot modify pods in namespace %s", PluginName, a.GetNamespace())) + } + return nil +} + +func isPrivilegedNamespace(a admission.Attributes) bool { + // If the namespace mentioned in the resource is privileged, return true. We will hit this for calls made to all + // resources in this namespace and during delete and update operation on the namespace itself. + if a.GetNamespace() == privilegedNamespace { + return true + } + + // If the resource is a namespace and if its name matched the privileged namespace, return true. We will hit this + // during creation of the namespace. + if a.GetResource().GroupResource() == api.Resource("namespaces") { + if namespace, ok := a.GetObject().(*api.Namespace); ok { + if namespace.Name == privilegedNamespace { + return true + } + } + } + + return false +} + +func isPrivilegedServiceAccount(a admission.Attributes) bool { + user := a.GetUserInfo().GetName() + return strings.HasPrefix(user, privilegedServiceAccount) +} + +func validatePrivilegedServiceAccount(a admission.Attributes) error { + // If the privileged service account performs any operation on the pods resource like create, update or connect, + // we block it. This is needed so that we can block exec access into privileged pods running on the master. Also, + // privileged service account does not need to perform these operations. So, just to be extra cautious we also block + // off create and update pods. + if a.GetResource().GroupResource() == api.Resource("pods") { + // Allow Delete operation on pods + if a.GetOperation() == admission.Delete { + return nil + } + // Allow subresources other than exec on pods + if len(a.GetSubresource()) != 0 && a.GetSubresource() != "exec" { + return nil + } + // Block all other operations + if a.GetOperation() == admission.Create || a.GetOperation() == admission.Update || a.GetOperation() == admission.Connect { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: cannot modify pods in namespace %s", PluginName, a.GetNamespace())) + } + } + + // If the privileged service account tries to update taints on a node, we block. We need to do this so that a user + // cannot use a privileged service account to untaint the node and run pods on a master. + if a.GetResource().GroupResource() == api.Resource("nodes") { + if a.GetOperation() == admission.Update { + node, ok := a.GetObject().(*api.Node) + if !ok { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: unexpected type %T", PluginName, a.GetObject())) + } + oldNode, ok := a.GetOldObject().(*api.Node) + if !ok { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: unexpected type %T", PluginName, a.GetOldObject())) + } + + if !apiequality.Semantic.DeepEqual(node.Spec.Taints, oldNode.Spec.Taints) { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: cannot modify taints on a node", PluginName)) + } + } + } + + return nil +} + +func validateNodes(a admission.Attributes) error { + // If it is a connect operation on the sub resource, allow it. Sub resources of nodes are nodes/nodename/proxy and + // nodes/nodename/status. Prometheus needs read access to nodes/nodename/proxy/metrics. To support that and other + // pods which might need read access to node metrics, we need to allow this. But the VKE authorizer blocks off any + // access to perform exec, attach or run on the proxy resource on a master which will prevent the users from getting + // access to the privileged pods on master. + if a.GetOperation() == admission.Connect && len(a.GetSubresource()) != 0 { + return nil + } + + // If the operation is Delete, fail. Deleting a node is not something that is useful to the user. Also, by deleting + // a node, they can potentially make their cluster useless. + if a.GetOperation() == admission.Delete { + return admission.NewForbidden(a, fmt.Errorf("%s validation failed: cannot delete nodes", PluginName)) + } + + // If the operation is on a master node, fail. We do not want to allow the users to modify labels and taints on the + // master node because it can compromise the security of the cluster. + if strings.HasPrefix(a.GetName(), masterNodePrefix) { + return admission.NewForbidden(a, fmt.Errorf("%s validation failed: cannot modify master nodes", PluginName)) + } + + return nil +} + +func validateClusterRoles(a admission.Attributes) error { + // If the name in the request is not empty and has the reserved prefix, then fail. We will hit this during delete + // and update operations on the cluster roles. If it does not have the reserved prefix, allow it. If the name is + // empty then proceed to read it from the object in the request. + if a.GetName() != "" { + return checkReservedPrefix(a.GetName(), a) + } + + clusterRole, ok := a.GetObject().(*rbac.ClusterRole) + // If we cannot get the cluster role binding object, fail. + if !ok { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: unexpected type %T", PluginName, a.GetObject())) + } + // If we get the object and the name has the reserved prefix, fail. We will hit this when someone tries to create a + // cluster role with the reserved prefix. + return checkReservedPrefix(clusterRole.Name, a) +} + +func validateClusterRoleBindings(a admission.Attributes) error { + // If the name in the request is not empty and has the reserved prefix, then fail. We will hit this during delete + // and update operations on the cluster role bindings. If it does not have the reserved prefix, allow it. If the + // name is empty then proceed to read it from the object in the request. + if a.GetName() != "" { + return checkReservedPrefix(a.GetName(), a) + } + + clusterRoleBinding, ok := a.GetObject().(*rbac.ClusterRoleBinding) + // If we cannot get the cluster role binding object, fail. + if !ok { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: unexpected type %T", PluginName, a.GetObject())) + } + // If we get the object and the name has the reserved prefix, fail. We will hit this when someone tries to create a + // cluster role binding with the reserved prefix. + return checkReservedPrefix(clusterRoleBinding.Name, a) +} + +func validatePods(vac *vmwareAdmissionController, a admission.Attributes) error { + // If the request is acting on a sub resource of a pod then allow it. This request is not directly coming to a pod, + // but to a sub-resource like pods/foo/status. So, this does not have to be blocked. + if len(a.GetSubresource()) != 0 { + return nil + } + + // If it is a Connect or Delete operation, allow it. We restrict access to connect to any pods in the vke-system + // namespace. So it is OK to allow this. + if a.GetOperation() == admission.Connect || a.GetOperation() == admission.Delete { + return nil + } + + // If we cannot get the pod object, fail. + if _, ok := a.GetObject().(*api.Pod); !ok { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: unexpected type %T", PluginName, a.GetObject())) + } + + // If this is an update, see if we are only updating the ownerRef/finalizers. Garbage collection does this + // and we should allow it in general, since you had the power to update and the power to delete. + // The worst that happens is that you delete something, but you aren't controlling the privileged object itself + if a.GetOperation() == admission.Update && + rbacregistry.IsOnlyMutatingGCFields(a.GetObject(), a.GetOldObject(), apiequality.Semantic) { + return nil + } + + errs := field.ErrorList{} + originalPod := a.GetObject().(*api.Pod) + + // Generate a copy of the pod object because we are not allowed to mutate the pod object. + pod := originalPod.DeepCopy() + + provider, err := podsecuritypolicy.NewSimpleProvider(vac.psp, pod.Namespace, vac.strategyFactory) + if err != nil { + return admission.NewForbidden(a, fmt.Errorf("%s validation failed: %v", PluginName, err)) + } + + // Set default security context for the pod. This fills in the defaults for the security context values that are not + // provided. This is needed to validate the security context correctly. + pod.Spec.SecurityContext, _, err = provider.CreatePodSecurityContext(pod) + if err != nil { + errs = append(errs, field.Invalid(field.NewPath("spec", "securityContext"), + pod.Spec.SecurityContext, err.Error())) + } + + // Validate the pod. + errs = append(errs, provider.ValidatePodSecurityContext(pod, field.NewPath("spec", "securityContext"))...) + + // Validate the pod's tolerations. + fieldErr := validatePodToleration(pod) + if fieldErr != nil { + errs = append(errs, fieldErr) + } + + // Validate the initContainers that are part of the pod. + for i := range pod.Spec.InitContainers { + pod.Spec.InitContainers[i].SecurityContext, _, err = provider.CreateContainerSecurityContext(pod, &pod.Spec.InitContainers[i]) + if err != nil { + errs = append(errs, field.Invalid(field.NewPath("spec", "initContainers").Index(i). + Child("securityContext"), "", err.Error())) + continue + } + errs = append(errs, provider.ValidateContainerSecurityContext(pod, &pod.Spec.InitContainers[i], + field.NewPath("spec", "initContainers").Index(i).Child("securityContext"))...) + } + + // Validate the containers that are part of the pod. + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].SecurityContext, _, err = provider.CreateContainerSecurityContext(pod, &pod.Spec.Containers[i]) + if err != nil { + errs = append(errs, field.Invalid(field.NewPath("spec", "containers").Index(i). + Child("securityContext"), "", err.Error())) + continue + } + errs = append(errs, provider.ValidateContainerSecurityContext(pod, &pod.Spec.Containers[i], + field.NewPath("spec", "containers").Index(i).Child("securityContext"))...) + } + + // Validate that /etc/ssl/certs if mounted using hostPath volume mount is readOnly. + fieldErr = validateEtcSslCertsHostPath(pod) + if fieldErr != nil { + errs = append(errs, fieldErr) + } + + if len(errs) > 0 { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: %v", PluginName, errs)) + } + + return nil +} + +func validatePodToleration(pod *api.Pod) *field.Error { + // Master nodes are tainted with "Dedicated=Master:NoSchedule". Only vke-system pods are allowed to tolerate + // this taint and to run on master nodes. A user's pod will be rejected if its spec has toleration for this taint. + for _, t := range pod.Spec.Tolerations { + reject := false + + if t.Key == reservedTolerationKey && t.Value == reservedTolerationValue { + // Reject pod that has the reserved toleration "Dedicated=Master" + reject = true + } else if t.Operator == api.TolerationOpExists && (t.Key == reservedTolerationKey || t.Key == "") { + // Reject pod that has wildcard toleration matching the reserved toleration + reject = true + } + + if reject { + return field.Invalid(field.NewPath("spec", "toleration"), fmt.Sprintf("%+v", t), + fmt.Sprintf("%s validation failed: should not tolerate master node taint", PluginName)) + } + } + return nil +} + +// Validate that /etc/ssl/certs if mounted using hostPath volume mount is readOnly. If not, fail. +// This is a hack that is needed to get Wavefront pods to work. +// TODO(ashokc): Once we have support for users to create pods using privileged mode and host path, remove this. +func validateEtcSslCertsHostPath(pod *api.Pod) *field.Error { + // Get volumes which mount /etc/ssl/certs and put them in a map. + volumes := map[string]struct{}{} + for _, vol := range pod.Spec.Volumes { + if vol.HostPath != nil && strings.HasPrefix(vol.HostPath.Path, etcSslCerts) { + volumes[vol.Name] = struct{}{} + } + } + + // For every initContainer, get all volumeMounts and verify if it matches any of the volumes in the volumes map. + // If yes, then check if they are read only. If not, return an error. + err := checkVolumeReadOnly(pod.Spec.InitContainers, volumes, "initContainers") + if err != nil { + return err + } + + // For every container, get all volumeMounts and verify if it matches any of the volumes in the volumes map. + // If yes, then check if they are read only. If not, return an error. + err = checkVolumeReadOnly(pod.Spec.Containers, volumes, "containers") + if err != nil { + return err + } + + return nil +} + +// Checks if the container has a volumeMount belonging to the volumes map. If yes, it has to be read only. If not, +// return error. +func checkVolumeReadOnly(containers []api.Container, volumes map[string]struct{}, containerType string) *field.Error { + for i, container := range containers { + for _, vol := range container.VolumeMounts { + if _, ok := volumes[vol.Name]; ok { + if !vol.ReadOnly { + return field.Invalid(field.NewPath("spec", containerType).Index(i).Child("volumeMounts"), + fmt.Sprintf("%+v", vol), fmt.Sprintf("%s has to be mount as readOnly", etcSslCerts)) + } + } + } + } + return nil +} + +func checkReservedPrefix(resourceName string, a admission.Attributes) error { + if strings.HasPrefix(resourceName, reservedPrefix) { + return admission.NewForbidden(a, + fmt.Errorf("%s validation failed: cannot modify resources with prefix %s", PluginName, reservedPrefix)) + } + return nil +} diff --git a/plugin/pkg/admission/vke/admission_test.go b/plugin/pkg/admission/vke/admission_test.go new file mode 100644 index 0000000..3fb4674 --- /dev/null +++ b/plugin/pkg/admission/vke/admission_test.go @@ -0,0 +1,941 @@ +package vke + +import ( + "fmt" + "os" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kadmission "k8s.io/apiserver/pkg/admission" + kapi "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/apis/rbac" + "k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/authentication/user" +) + +const ( + testServiceAccountsGroup = "system.test\\cascade-controller-service-accounts" + clusterID = "cluster-id" + defaultConfigFileFormat = ` +vmwareAdmissionController: + privilegedGroup: %s + podSecurityPolicyFile: %s + clusterID: %s +` + pspFileName = "/tmp/psp.yaml" + pspConfigFile = ` +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: vmware-pod-security-policy-restricted +spec: + privileged: true + fsGroup: + rule: RunAsAny + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'emptyDir' + - 'secret' + - 'downwardAPI' + - 'configMap' + - 'persistentVolumeClaim' + - 'projected' + - 'hostPath' + hostPID: false + hostIPC: false + hostNetwork: true + hostPorts: + - min: 1 + max: 65536 +` +) + +func TestAdmitPrivileged(t *testing.T) { + tests := map[string]struct { + operation kadmission.Operation + pod *kapi.Pod + name string + userInfo user.Info + shouldPassValidate bool + }{ + "create pod with Privileged=nil allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with Privileged=false allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withPrivileged(false).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with Privileged=true denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withPrivileged(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with multiple containers, one has Privileged=true denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withPrivileged(true).withInitContainer().withContainer().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "update pod with Privileged=true denied": { + operation: kadmission.Update, + pod: newTestPodBuilder().withPrivileged(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with HostNetwork=true denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostNetwork(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with HostIPC=true denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostIPC(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with HostPID=true denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostPID(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with HostPort allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostPort().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostVolume denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostVolume("/", false).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with HostVolume /etc/ssl/certs in read-only mode allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostVolume("/etc/ssl/certs", true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostVolume /etc/ssl/certs in read-write mode denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostVolume("/etc/ssl/certs", false).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with CascadeDisk allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withCascadeDisk().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostVolume and CascadeDisk denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostVolume("/", false).withCascadeDisk().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "connect pod allowed": { + operation: kadmission.Connect, + pod: newTestPodBuilder().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "delete pod allowed": { + operation: kadmission.Delete, + pod: nil, + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + } + + for k, v := range tests { + testPodValidation(k, v.operation, v.pod, v.name, v.userInfo, v.shouldPassValidate, t) + } +} + +func TestAdmitPrivilegedWithCustomPSP(t *testing.T) { + tests := map[string]struct { + operation kadmission.Operation + pod *kapi.Pod + name string + userInfo user.Info + shouldPassValidate bool + }{ + "create pod with Privileged=nil allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with Privileged=false allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withPrivileged(false).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with Privileged=true allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withPrivileged(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with multiple containers, one has Privileged=true allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withPrivileged(true).withInitContainer().withContainer().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "update pod with Privileged=true allowed": { + operation: kadmission.Update, + pod: newTestPodBuilder().withPrivileged(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostNetwork=true allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostNetwork(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostIPC=true denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostIPC(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with HostPID=true denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostPID(true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with HostPort allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostPort().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostVolume allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostVolume("/", false).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostVolume /etc/ssl/certs in read-only mode allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostVolume("/etc/ssl/certs", true).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostVolume /etc/ssl/certs in read-write mode denied": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostVolume("/etc/ssl/certs", false).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "create pod with CascadeDisk allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withCascadeDisk().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "create pod with HostVolume and CascadeDisk allowed": { + operation: kadmission.Create, + pod: newTestPodBuilder().withHostVolume("/", false).withCascadeDisk().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "connect pod allowed": { + operation: kadmission.Connect, + pod: newTestPodBuilder().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "delete pod allowed": { + operation: kadmission.Delete, + pod: nil, + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + } + + // Setup custom PSP file. + file, err := os.Create(pspFileName) + if err != nil { + t.Errorf("TestAdmitPrivilegedWithCustomPSP: failed to open custom PSP file %v", err) + return + } + _, err = file.WriteString(pspConfigFile) + if err != nil { + t.Errorf("TestAdmitPrivilegedWithCustomPSP: failed to write to custom PSP file %v", err) + return + } + + for k, v := range tests { + testPodValidation(k, v.operation, v.pod, v.name, v.userInfo, v.shouldPassValidate, t) + } + + // Delete custom PSP file. + err = os.Remove(pspFileName) + if err != nil { + t.Errorf("TestAdmitPrivilegedWithCustomPSP: failed to delete custom PSP file %v", err) + } +} + +func TestPrivilegedNamespace(t *testing.T) { + tests := map[string]struct { + operation kadmission.Operation + pod *kapi.Pod + name string + userInfo user.Info + shouldPassValidate bool + }{ + "denied: regular lightwave user creates pod in vke-system namespace": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user cannot escalate privilege using service account": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).withServiceAccount(privilegedServiceAccount + "default").build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular service account creates pod in vke-system namespace": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withName("system:serviceaccount:kube-system:default").build(), + shouldPassValidate: false, + }, + "allowed: privileged service account deletes a pod in privileged namespace": { + operation: kadmission.Delete, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withName(privilegedServiceAccount + "default").build(), + shouldPassValidate: true, + }, + "denied: privileged service account creates a pod in privileged namespace": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withName(privilegedServiceAccount + "default").build(), + shouldPassValidate: false, + }, + "denied: privileged service account execs into a pod in privileged namespace": { + operation: kadmission.Connect, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withName(privilegedServiceAccount + "default").build(), + shouldPassValidate: false, + }, + "denied: privileged service account updates a pod in privileged namespace": { + operation: kadmission.Update, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withName(privilegedServiceAccount + "default").build(), + shouldPassValidate: false, + }, + "allowed: regular user creates pod in other namespace": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace("default").build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "allowed: Cascade Controller Service Account creates pod in vke-system namespace": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withGroup(testServiceAccountsGroup).build(), + shouldPassValidate: true, + }, + "allowed: systemUnsecuredUser creates pod in vke-system namespace": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withName(systemUnsecuredUser).build(), + shouldPassValidate: true, + }, + "allowed: kubelet group creates pod in vke-system namespace": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withGroup(systemNodesGroup).withName("system:node:worker").build(), + shouldPassValidate: true, + }, + "denied: regular lightwave group does not grant privileged access": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withGroup("test1\\group1").build(), + shouldPassValidate: false, + }, + "allowed: if user has multiple groups, any privileged group can grant privileged access": { + operation: kadmission.Create, + pod: newTestPodBuilder().withNamespace(privilegedNamespace).build(), + userInfo: newTestUserBuilder().withGroup("test1\\group1").withGroup(testServiceAccountsGroup).build(), + shouldPassValidate: true, + }, + "denied: kubelet exec into pod": { + operation: kadmission.Connect, + pod: newTestPodBuilder().build(), + userInfo: newTestUserBuilder().withGroup("system:nodes").build(), + shouldPassValidate: false, + }, + } + for k, v := range tests { + testPodValidation(k, v.operation, v.pod, v.name, v.userInfo, v.shouldPassValidate, t) + } +} + +func TestToleration(t *testing.T) { + tests := map[string]struct { + operation kadmission.Operation + pod *kapi.Pod + name string + userInfo user.Info + shouldPassValidate bool + }{ + "allowed: create pod with no toleration": { + operation: kadmission.Create, + pod: newTestPodBuilder().build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "allowed: create pod with normal toleration key": { + operation: kadmission.Create, + pod: newTestPodBuilder().withToleration("mykey", reservedTolerationValue, kapi.TolerationOpEqual, kapi.TaintEffectNoSchedule).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "allowed: create pod with normal toleration value": { + operation: kadmission.Create, + pod: newTestPodBuilder().withToleration(reservedTolerationKey, "myval", kapi.TolerationOpEqual, kapi.TaintEffectNoSchedule).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "denied: create pod with reserved toleration": { + operation: kadmission.Create, + pod: newTestPodBuilder().withToleration(reservedTolerationKey, reservedTolerationValue, kapi.TolerationOpEqual, kapi.TaintEffectNoSchedule).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: create pod with wildcard toleration": { + operation: kadmission.Create, + pod: newTestPodBuilder().withToleration("", "", kapi.TolerationOpExists, "").build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: create pod with value wildcard toleration": { + operation: kadmission.Create, + pod: newTestPodBuilder().withToleration(reservedTolerationKey, "", kapi.TolerationOpExists, kapi.TaintEffectNoSchedule).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "allowed: create pod with value wildcard and normal key": { + operation: kadmission.Create, + pod: newTestPodBuilder().withToleration("mykey", "", kapi.TolerationOpExists, kapi.TaintEffectNoSchedule).build(), + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + } + + for k, v := range tests { + testPodValidation(k, v.operation, v.pod, v.name, v.userInfo, v.shouldPassValidate, t) + } +} + +func TestClusterLevelResources(t *testing.T) { + tests := map[string]struct { + operation kadmission.Operation + resource string + subresource string + name string + namespace string + userInfo user.Info + object runtime.Object + oldObject runtime.Object + shouldPassValidate bool + }{ + "denied: regular lightwave user update configmaps in vke-system namespace": { + operation: kadmission.Update, + resource: "configmaps", + namespace: privilegedNamespace, + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user delete daemonsets in vke-system namespace": { + operation: kadmission.Delete, + resource: "daemonsets", + namespace: privilegedNamespace, + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user create deployments in vke-system namespace": { + operation: kadmission.Create, + resource: "deployments", + namespace: privilegedNamespace, + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user create rolebindings in vke-system namespace": { + operation: kadmission.Create, + resource: "rolebindings", + namespace: privilegedNamespace, + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "allowed: regular lightwave user create rolebindings in other namespace": { + operation: kadmission.Create, + resource: "rolebindings", + namespace: "default", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "allowed: regular lightwave user create clusterroles": { + operation: kadmission.Create, + resource: "clusterroles", + name: "cluster-role", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "denied: regular lightwave user create clusterroles with vke: prefix": { + operation: kadmission.Create, + resource: "clusterroles", + name: "vke:clusterrole", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user delete clusterroles with vke: prefix": { + operation: kadmission.Delete, + resource: "clusterroles", + name: "vke:clusterrole", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "allowed: systemUnsecuredUser update clusterroles with vke: prefix": { + operation: kadmission.Update, + resource: "clusterroles", + name: "vke:clusterrole", + namespace: "", + userInfo: newTestUserBuilder().withName(systemUnsecuredUser).build(), + shouldPassValidate: true, + }, + "allowed: regular lightwave user create clusterrolebindings": { + operation: kadmission.Create, + resource: "clusterrolebindings", + name: "cluster-role-binding", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "denied: regular lightwave user create clusterrolebindings with vke: prefix": { + operation: kadmission.Create, + resource: "clusterrolebindings", + name: "vke:clusterrolebinding", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user update clusterrolebindings with vke: prefix": { + operation: kadmission.Delete, + resource: "clusterrolebindings", + name: "vke:clusterrolebinding", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "allowed: systemUnsecuredUser update clusterrolebindings with vke: prefix": { + operation: kadmission.Update, + resource: "clusterrolebindings", + name: "vke:clusterrolebinding", + namespace: "", + userInfo: newTestUserBuilder().withName(systemUnsecuredUser).build(), + shouldPassValidate: true, + }, + "allowed: regular lightwave user update worker nodes": { + operation: kadmission.Update, + resource: "nodes", + name: "worker-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "allowed: regular lightwave user get worker nodes proxy subresource": { + operation: kadmission.Connect, + resource: "nodes", + subresource: "proxy", + name: "worker-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "allowed: regular lightwave user patch worker nodes proxy subresource": { + operation: kadmission.Update, + resource: "nodes", + subresource: "proxy", + name: "worker-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "allowed: regular lightwave user create worker nodes proxy subresource": { + operation: kadmission.Create, + resource: "nodes", + subresource: "proxy", + name: "worker-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "allowed: regular lightwave user get master nodes proxy subresource": { + operation: kadmission.Connect, + resource: "nodes", + subresource: "proxy", + name: "master-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: true, + }, + "denied: regular lightwave user patch master nodes proxy subresource": { + operation: kadmission.Update, + resource: "nodes", + subresource: "proxy", + name: "master-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user create master nodes proxy subresource": { + operation: kadmission.Create, + resource: "nodes", + subresource: "proxy", + name: "master-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user update master nodes": { + operation: kadmission.Update, + resource: "nodes", + name: "master-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user delete master nodes": { + operation: kadmission.Delete, + resource: "nodes", + name: "master-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "denied: regular lightwave user delete worker nodes": { + operation: kadmission.Delete, + resource: "nodes", + name: "worker-guid", + namespace: "", + userInfo: newTestUserBuilder().build(), + shouldPassValidate: false, + }, + "allowed: systemUnsecuredUser update nodes": { + operation: kadmission.Update, + resource: "nodes", + namespace: "", + userInfo: newTestUserBuilder().withName(systemUnsecuredUser).build(), + shouldPassValidate: true, + }, + "allowed: kubelet update node": { + operation: kadmission.Update, + resource: "nodes", + namespace: "", + userInfo: newTestUserBuilder().withGroup(systemNodesGroup).build(), + shouldPassValidate: true, + }, + "denied: privileged service account update node taint": { + operation: kadmission.Update, + resource: "nodes", + namespace: "", + oldObject: newTestNodeBuilder().build(), + object: newTestNodeBuilder().withTaint(nil).build(), + userInfo: newTestUserBuilder().withName(privilegedServiceAccount + "default").build(), + shouldPassValidate: false, + }, + } + for k, v := range tests { + testResourceValidation(k, v.operation, v.resource, v.subresource, v.name, v.namespace, v.userInfo, v.object, + v.oldObject, v.shouldPassValidate, t) + } +} + +func testPodValidation(testCaseName string, op kadmission.Operation, pod *kapi.Pod, name string, userInfo user.Info, + shouldPassValidate bool, t *testing.T) { + + defaultConfigFile := fmt.Sprintf(defaultConfigFileFormat, testServiceAccountsGroup, pspFileName, clusterID) + configFile := strings.NewReader(defaultConfigFile) + plugin, err := NewVMwareAdmissionController(configFile) + if err != nil { + t.Errorf("%s: failed to create admission controller %v", testCaseName, err) + } + + namespace := "default" + if pod != nil { + namespace = pod.Namespace + } + + attrs := kadmission.NewAttributesRecord(pod, nil, kapi.Kind("Pod").WithVersion("version"), + namespace, name, kapi.Resource("pods").WithVersion("version"), "", op, userInfo) + + err = plugin.Admit(attrs) + if shouldPassValidate && err != nil { + t.Errorf("%s: expected no errors on Validate but received %v", testCaseName, err) + } else if !shouldPassValidate && err == nil { + t.Errorf("%s: expected errors on Validate but received none", testCaseName) + } +} + +func testResourceValidation(testCaseName string, op kadmission.Operation, resource, subresource, name, namespace string, + userInfo user.Info, object runtime.Object, oldObject runtime.Object, shouldPassValidate bool, t *testing.T) { + + defaultConfigFile := fmt.Sprintf(defaultConfigFileFormat, testServiceAccountsGroup, pspFileName, clusterID) + configFile := strings.NewReader(defaultConfigFile) + plugin, err := NewVMwareAdmissionController(configFile) + if err != nil { + t.Errorf("%s: failed to create admission controller %v", testCaseName, err) + } + + groupResource := kapi.Resource(resource).WithVersion("version") + if resource == "clusterroles" || resource == "clusterrolebindings" { + groupResource = rbac.Resource(resource).WithVersion("version") + } + + attrs := kadmission.NewAttributesRecord(object, oldObject, kapi.Kind("kind").WithVersion("version"), + namespace, name, groupResource, subresource, op, userInfo) + + err = plugin.Admit(attrs) + if shouldPassValidate && err != nil { + t.Errorf("%s: expected no errors on Validate but received %v", testCaseName, err) + } else if !shouldPassValidate && err == nil { + t.Errorf("%s: expected errors on Validate but received none", testCaseName) + } +} + +// testPodBuilder +type testPodBuilder struct { + pod kapi.Pod +} + +func newTestPodBuilder() *testPodBuilder { + builder := new(testPodBuilder) + return builder.init() +} + +func (p *testPodBuilder) init() *testPodBuilder { + p.pod = kapi.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "namespace", + Annotations: map[string]string{}, + }, + Spec: kapi.PodSpec{ + ServiceAccountName: "default", + SecurityContext: &kapi.PodSecurityContext{}, + Containers: []kapi.Container{ + { + Name: "test-container-1", + SecurityContext: &kapi.SecurityContext{}, + }, + }, + }, + } + return p +} + +func (p *testPodBuilder) build() *kapi.Pod { + return &p.pod +} + +func (p *testPodBuilder) withNamespace(namespace string) *testPodBuilder { + p.pod.ObjectMeta.Namespace = namespace + return p +} + +func (p *testPodBuilder) withServiceAccount(sa string) *testPodBuilder { + p.pod.Spec.ServiceAccountName = sa + return p +} + +func (p *testPodBuilder) withPrivileged(v bool) *testPodBuilder { + p.pod.Spec.Containers[0].SecurityContext.Privileged = &v + return p +} + +func (p *testPodBuilder) withHostNetwork(v bool) *testPodBuilder { + p.pod.Spec.SecurityContext.HostNetwork = v + return p +} + +func (p *testPodBuilder) withHostIPC(v bool) *testPodBuilder { + p.pod.Spec.SecurityContext.HostIPC = v + return p +} + +func (p *testPodBuilder) withHostPID(v bool) *testPodBuilder { + p.pod.Spec.SecurityContext.HostPID = v + return p +} + +func (p *testPodBuilder) withHostPort() *testPodBuilder { + containerPorts := []kapi.ContainerPort{ + { + HostPort: 3000, + ContainerPort: 80, + }, + } + + p.pod.Spec.Containers[0].Ports = containerPorts + return p +} + +func (p *testPodBuilder) withHostVolume(hostPath string, readOnly bool) *testPodBuilder { + volume := kapi.Volume{ + Name: "host", + VolumeSource: kapi.VolumeSource{ + HostPath: &kapi.HostPathVolumeSource{ + Path: hostPath, + }, + }, + } + volumeMount := kapi.VolumeMount{Name: "host", MountPath: "/data", ReadOnly: readOnly} + + p.pod.Spec.Volumes = append(p.pod.Spec.Volumes, volume) + p.pod.Spec.Containers[0].VolumeMounts = append(p.pod.Spec.Containers[0].VolumeMounts, volumeMount) + return p +} + +func (p *testPodBuilder) withCascadeDisk() *testPodBuilder { + volume := kapi.Volume{ + Name: "cascadeDisk", + VolumeSource: kapi.VolumeSource{ + PersistentVolumeClaim: &kapi.PersistentVolumeClaimVolumeSource{ + ClaimName: "00000000-0000-0000-0000-000000000001", + ReadOnly: false, + }, + }, + } + device := kapi.VolumeDevice{Name: "cascadeDisk", DevicePath: "/cascadeDisk"} + + p.pod.Spec.Volumes = append(p.pod.Spec.Volumes, volume) + p.pod.Spec.Containers[0].VolumeDevices = append(p.pod.Spec.Containers[0].VolumeDevices, device) + return p +} + +func (p *testPodBuilder) withContainer() *testPodBuilder { + container := kapi.Container{ + Name: "test-container-2", + SecurityContext: &kapi.SecurityContext{}, + } + + p.pod.Spec.Containers = append(p.pod.Spec.Containers, container) + return p +} + +func (p *testPodBuilder) withInitContainer() *testPodBuilder { + container := kapi.Container{ + Name: "test-init-container", + SecurityContext: &kapi.SecurityContext{}, + } + + p.pod.Spec.Containers = append(p.pod.Spec.Containers, container) + return p +} + +func (p *testPodBuilder) withToleration(key, value string, operator kapi.TolerationOperator, effect kapi.TaintEffect) *testPodBuilder { + p.pod.Spec.Tolerations = append(p.pod.Spec.Tolerations, kapi.Toleration{ + Key: key, + Value: value, + Operator: operator, + Effect: effect, + }) + return p +} + +// testUserBuilder +type testUserBuilder struct { + user *user.DefaultInfo +} + +func newTestUserBuilder() *testUserBuilder { + builder := new(testUserBuilder) + return builder.init() +} + +func (p *testUserBuilder) init() *testUserBuilder { + p.user = &user.DefaultInfo{ + Name: "https://lightwave.cascade-cloud.com/openidconnect/00000000-0000-0000-0000-000000000001#joe@vmware.com", + UID: "10001", + Groups: []string{}, + } + return p +} + +func (p *testUserBuilder) build() *user.DefaultInfo { + return p.user +} + +func (p *testUserBuilder) withName(name string) *testUserBuilder { + p.user.Name = name + return p +} + +func (p *testUserBuilder) withGroup(group string) *testUserBuilder { + p.user.Groups = append(p.user.Groups, group) + return p +} + +// testNodeBuilder +type testNodeBuilder struct { + node kapi.Node +} + +func newTestNodeBuilder() *testNodeBuilder { + builder := new(testNodeBuilder) + return builder.init() +} + +func (n *testNodeBuilder) init() *testNodeBuilder { + n.node = kapi.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + Namespace: "", + Annotations: nil, + }, + Spec: kapi.NodeSpec{ + Taints: []kapi.Taint{ + { + "Dedicated", "Master", "NoSchedule", nil, + }, + }, + }, + } + return n +} + +func (n *testNodeBuilder) build() *kapi.Node { + return &n.node +} + +func (n *testNodeBuilder) withTaint(taints []kapi.Taint) *testNodeBuilder { + n.node.Spec.Taints = taints + return n +} diff --git a/plugin/pkg/auth/authorizer/vke/BUILD b/plugin/pkg/auth/authorizer/vke/BUILD new file mode 100644 index 0000000..4b984f1 --- /dev/null +++ b/plugin/pkg/auth/authorizer/vke/BUILD @@ -0,0 +1,40 @@ +package(default_visibility = ["//visibility:public"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_test( + name = "go_default_test", + srcs = ["vke_authorizer_test.go"], + embed = [":go_default_library"], + deps = [ + "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library", + "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + ], +) + +go_library( + name = "go_default_library", + srcs = [ + "vke_authorizer.go", + ], + importpath = "k8s.io/kubernetes/plugin/pkg/auth/authorizer/vke", + deps = [ + "//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/plugin/pkg/auth/authorizer/vke/OWNERS b/plugin/pkg/auth/authorizer/vke/OWNERS new file mode 100644 index 0000000..48f5944 --- /dev/null +++ b/plugin/pkg/auth/authorizer/vke/OWNERS @@ -0,0 +1,3 @@ +maintainers: +- ashokc +- vivekgoyal \ No newline at end of file diff --git a/plugin/pkg/auth/authorizer/vke/vke_authorizer.go b/plugin/pkg/auth/authorizer/vke/vke_authorizer.go new file mode 100644 index 0000000..5f3103b --- /dev/null +++ b/plugin/pkg/auth/authorizer/vke/vke_authorizer.go @@ -0,0 +1,123 @@ +package vke + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/golang/glog" + "k8s.io/apiserver/pkg/authorization/authorizer" +) + +const ( + systemClusterPrefix = "system:clusterID:" + systemNodePrefix = "system:node:" + systemWorkerGroup = "system:worker" + masterPrefix = "master-" +) + +// VKEAuthorizer authorizes requests which comes from nodes using certificates. +// If a request is from a node which is not a part of the cluster, reject. +type VKEAuthorizer struct { + clusterID string +} + +// NewAuthorizer returns a new node authorizer +func NewAuthorizer() (authorizer.Authorizer, error) { + file, err := os.Open("/etc/kubernetes/cc_cloud.config") + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + lineContent := scanner.Text() + if strings.Contains(lineContent, "clusterID = ") { + clusterID := strings.SplitAfter(lineContent, "= ")[1] + return &VKEAuthorizer{clusterID: clusterID}, nil + } + } + + return nil, fmt.Errorf("Cluster ID cannot be parsed from config file successfully") +} + +// Authorize authorizes requests based on the VKE authorizer. +func (v *VKEAuthorizer) Authorize(attrs authorizer.Attributes) (authorizer.Decision, string, error) { + // If the request comes from a non cluster node, then deny. + if isNonClusterNode(attrs, v.clusterID) { + glog.V(2).Infof("VKE authorizer: DENY the request because it is from a different cluster") + return authorizer.DecisionDeny, "", nil + } + + // If a worker node name does not have the node prefix, then deny. This is needed for the request to go through node + // authorizer and node restriction admission controller. If it is not set, then a user can bypass node authorizer + // and the node restriction admission controller and modify the master node. + if isWorkerWithoutNodeNameRestriction(attrs) { + glog.V(2).Infof("VKE authorizer: DENY the request because the node name restriction is not met") + return authorizer.DecisionDeny, "", nil + } + + // If it is a proxy request to the master node to exec, run or attach to a container, then deny. + if isProxyRequestToMasterNode(attrs) { + glog.V(2).Infof("VKE authorizer: DENY the request because it tried to execute commands on master pods") + return authorizer.DecisionDeny, "", nil + } + + return authorizer.DecisionNoOpinion, "", nil +} + +// isNonClusterNode verifies that the request is not from a node which does not belong to this cluster. This is needed +// to prevent cross cluster attacks where a user can use the kubelet certificate of one cluster to access the resources +// in another cluster. The reason we have this check is because, when a certificate is presented for authentication, +// Kubernetes just verifies that the certificate is signed by the CA that the cluster trusts. Since, in our case, the CA +// is same for all clusters, authentication with certificate is not enough. So, we make sure that the request is not +// from another cluster. Lightwave will make sure that any certificates generated on a worker will always have the +// "system:clusterID:" group. This way, we can just check the cluster ID in the group and reject if it is not the +// same as this cluster's ID. +func isNonClusterNode(attrs authorizer.Attributes, clusterID string) bool { + groups := attrs.GetUser().GetGroups() + for _, group := range groups { + if strings.HasPrefix(group, systemClusterPrefix) { + groupParts := strings.Split(group, ":") + if clusterID != "" && groupParts[len(groupParts)-1] != clusterID { + return true + } + } + } + + return false +} + +// isWorkerWithoutNodeNameRestriction verifies that the certificate presented by the worker nodes also have the +// appropriate name "system:node:". We need to validate this because it is possible for a user to generate +// certificates with CN other than "system:node:". This is because lightwave does group validation and that +// the CNs can be one of system:node:, kubernetes-master and etcd. When that happens, that certificate will be +// able to bypass the NodeRestriction admission controller which will allow access to modify master node and create pods +// on master. To prevent that from happening, we need to verify that if a request comes from a worker node, then it has +// to have the expected name so that NodeRestriction admission controller is enforced. +func isWorkerWithoutNodeNameRestriction(attrs authorizer.Attributes) bool { + groups := attrs.GetUser().GetGroups() + name := attrs.GetUser().GetName() + for _, group := range groups { + if group == systemWorkerGroup && !strings.HasPrefix(name, systemNodePrefix) { + return true + } + } + return false +} + +// isProxyRequestToMasterNode checks if the request is made to exec, run or attach to a container on the master node. We +// need this check because we do not want to allow a user to exec into a privileged pod on the master node. +func isProxyRequestToMasterNode(attrs authorizer.Attributes) bool { + if attrs.GetResource() == "nodes" && strings.HasPrefix(attrs.GetName(), masterPrefix) { + if strings.Contains(attrs.GetPath(), "proxy/run") || + strings.Contains(attrs.GetPath(), "proxy/exec") || + strings.Contains(attrs.GetPath(), "proxy/attach") { + return true + } + } + return false +} diff --git a/plugin/pkg/auth/authorizer/vke/vke_authorizer_test.go b/plugin/pkg/auth/authorizer/vke/vke_authorizer_test.go new file mode 100644 index 0000000..6aba9ec --- /dev/null +++ b/plugin/pkg/auth/authorizer/vke/vke_authorizer_test.go @@ -0,0 +1,230 @@ +package vke + +import ( + "testing" + + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" +) + +const ( + masterProxyAttachPath = "/api/v1/nodes/master-0/proxy/attach/vke-system/pod-name/container-name" + masterProxyExecPath = "/api/v1/nodes/master-0/proxy/exec/vke-system/pod-name/container-name" + masterProxyRunPath = "/api/v1/nodes/master-0/proxy/run/vke-system/pod-name/container-name" + workerProxyAttachPath = "/api/v1/nodes/worker-0/proxy/attach/vke-system/pod-name/container-name" + workerProxyExecPath = "/api/v1/nodes/worker-0/proxy/exec/vke-system/pod-name/container-name" + workerProxyRunPath = "/api/v1/nodes/worker-0/proxy/run/vke-system/pod-name/container-name" +) + +func TestAuthorizer(t *testing.T) { + authz := &VKEAuthorizer{clusterID: "cluster-id"} + + clusterNode := &user.DefaultInfo{Name: "system:node:worker-0", Groups: []string{"system:nodes", "system:clusterID:cluster-id", "system:worker"}} + crossClusterNode := &user.DefaultInfo{Name: "system:node:worker-0", Groups: []string{"system:nodes", "system:clusterID:invalid", "system:worker"}} + invalidWorker := &user.DefaultInfo{Name: "worker-0", Groups: []string{"system:nodes", "system:clusterID:cluster-id", "system:worker"}} + + tests := []struct { + name string + attrs authorizer.AttributesRecord + expect authorizer.Decision + }{ + // Do not deny requests from a normal cluster node user. + { + name: "allowed: cluster node get pod", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "get", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node list pod", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "list", Resource: "pods", Namespace: "ns"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node create pod", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "create", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node update pod", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "update", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node delete pod", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "delete", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node get node", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "get", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node list nodes", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "list", Resource: "nodes"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node create nodes", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node update nodes", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "update", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: cluster node delete nodes", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "delete", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionNoOpinion, + }, + + // Deny requests from another cluster node. + { + name: "denied: cross cluster node get pod", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "get", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node list pod", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "list", Resource: "pods", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node create pod", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "create", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node update pod", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "update", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node delete pod", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "delete", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node get node", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "get", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node list nodes", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "list", Resource: "nodes"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node create nodes", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node update nodes", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "update", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: cross cluster node delete nodes", + attrs: authorizer.AttributesRecord{User: crossClusterNode, ResourceRequest: true, Verb: "delete", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionDeny, + }, + + // Deny requests from invalid node. + { + name: "denied: invalid worker get pod", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "get", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker list pod", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "list", Resource: "pods", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker create pod", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "create", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker update pod", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "update", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker delete pod", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "delete", Resource: "pods", Name: "pod", Namespace: "ns"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker get node", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "get", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker list nodes", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "list", Resource: "nodes"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker create nodes", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker update nodes", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "update", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: invalid worker delete nodes", + attrs: authorizer.AttributesRecord{User: invalidWorker, ResourceRequest: true, Verb: "delete", Resource: "nodes", Name: "node"}, + expect: authorizer.DecisionDeny, + }, + + // Deny exec, run and attach operation on master nodes using proxy resource. + { + name: "denied: attach using proxy resource on master", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "master-0", Path: masterProxyAttachPath}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: exec using proxy resource on master", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "master-0", Path: masterProxyExecPath}, + expect: authorizer.DecisionDeny, + }, + { + name: "denied: run using proxy resource on master", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "master-0", Path: masterProxyRunPath}, + expect: authorizer.DecisionDeny, + }, + + // Do not deny exec, run and attach operation on worker nodes using proxy resource. + { + name: "allowed: attach using proxy resource on worker", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "worker-0", Path: workerProxyAttachPath}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: exec using proxy resource on worker", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "worker-0", Path: workerProxyExecPath}, + expect: authorizer.DecisionNoOpinion, + }, + { + name: "allowed: run using proxy resource on worker", + attrs: authorizer.AttributesRecord{User: clusterNode, ResourceRequest: true, Verb: "create", Resource: "nodes", Name: "worker-0", Path: workerProxyRunPath}, + expect: authorizer.DecisionNoOpinion, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + decision, _, _ := authz.Authorize(tc.attrs) + if decision != tc.expect { + t.Errorf("expected %v, got %v", tc.expect, decision) + } + }) + } +} diff --git a/staging/src/k8s.io/api/core/v1/generated.pb.go b/staging/src/k8s.io/api/core/v1/generated.pb.go index 5aeae2c..a7d5b12 100644 --- a/staging/src/k8s.io/api/core/v1/generated.pb.go +++ b/staging/src/k8s.io/api/core/v1/generated.pb.go @@ -35,6 +35,7 @@ limitations under the License. Binding CSIPersistentVolumeSource Capabilities + CascadeDiskVolumeSource CephFSPersistentVolumeSource CephFSVolumeSource CinderVolumeSource @@ -260,9 +261,11 @@ func (m *AvoidPods) Reset() { *m = AvoidPods{} } func (*AvoidPods) ProtoMessage() {} func (*AvoidPods) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{3} } -func (m *AzureDiskVolumeSource) Reset() { *m = AzureDiskVolumeSource{} } -func (*AzureDiskVolumeSource) ProtoMessage() {} -func (*AzureDiskVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{4} } +func (m *CascadeDiskVolumeSource) Reset() { *m = CascadeDiskVolumeSource{} } +func (*CascadeDiskVolumeSource) ProtoMessage() {} +func (*CascadeDiskVolumeSource) Descriptor() ([]byte, []int) { + return fileDescriptorGenerated, []int{4} +} func (m *AzureFilePersistentVolumeSource) Reset() { *m = AzureFilePersistentVolumeSource{} } func (*AzureFilePersistentVolumeSource) ProtoMessage() {} @@ -1040,6 +1043,11 @@ func (*WeightedPodAffinityTerm) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{185} } +func (m *AzureDiskVolumeSource) Reset() { *m = AzureDiskVolumeSource{} } +func (*AzureDiskVolumeSource) ProtoMessage() {} +func (*AzureDiskVolumeSource) Descriptor() ([]byte, []int) { return fileDescriptorGenerated, []int{186} } + + func init() { proto.RegisterType((*AWSElasticBlockStoreVolumeSource)(nil), "k8s.io.api.core.v1.AWSElasticBlockStoreVolumeSource") proto.RegisterType((*Affinity)(nil), "k8s.io.api.core.v1.Affinity") @@ -1051,6 +1059,7 @@ func init() { proto.RegisterType((*Binding)(nil), "k8s.io.api.core.v1.Binding") proto.RegisterType((*CSIPersistentVolumeSource)(nil), "k8s.io.api.core.v1.CSIPersistentVolumeSource") proto.RegisterType((*Capabilities)(nil), "k8s.io.api.core.v1.Capabilities") + proto.RegisterType((*CascadeDiskVolumeSource)(nil), "k8s.io.api.core.v1.CascadeDiskVolumeSource") proto.RegisterType((*CephFSPersistentVolumeSource)(nil), "k8s.io.api.core.v1.CephFSPersistentVolumeSource") proto.RegisterType((*CephFSVolumeSource)(nil), "k8s.io.api.core.v1.CephFSVolumeSource") proto.RegisterType((*CinderVolumeSource)(nil), "k8s.io.api.core.v1.CinderVolumeSource") @@ -1613,6 +1622,32 @@ func (m *Capabilities) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *CascadeDiskVolumeSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CascadeDiskVolumeSource) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.DiskID))) + i += copy(dAtA[i:], m.DiskID) + dAtA[i] = 0x12 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(len(m.FSType))) + i += copy(dAtA[i:], m.FSType) + return i, nil +} + func (m *CephFSPersistentVolumeSource) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6283,13 +6318,13 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { } i += n120 } - if m.AzureDisk != nil { + if m.CascadeDisk != nil { dAtA[i] = 0x82 i++ dAtA[i] = 0x1 i++ - i = encodeVarintGenerated(dAtA, i, uint64(m.AzureDisk.Size())) - n121, err := m.AzureDisk.MarshalTo(dAtA[i:]) + i = encodeVarintGenerated(dAtA, i, uint64(m.CascadeDisk.Size())) + n121, err := m.CascadeDisk.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -6367,6 +6402,18 @@ func (m *PersistentVolumeSource) MarshalTo(dAtA []byte) (int, error) { } i += n127 } + if m.AzureDisk != nil { + dAtA[i] = 0xba + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.AzureDisk.Size())) + n128, err := m.AzureDisk.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n128 + } return i, nil } @@ -10316,13 +10363,13 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { } i += n223 } - if m.AzureDisk != nil { + if m.CascadeDisk != nil { dAtA[i] = 0xb2 i++ dAtA[i] = 0x1 i++ - i = encodeVarintGenerated(dAtA, i, uint64(m.AzureDisk.Size())) - n224, err := m.AzureDisk.MarshalTo(dAtA[i:]) + i = encodeVarintGenerated(dAtA, i, uint64(m.CascadeDisk.Size())) + n224, err := m.CascadeDisk.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -10388,6 +10435,18 @@ func (m *VolumeSource) MarshalTo(dAtA []byte) (int, error) { } i += n229 } + if m.AzureDisk != nil { + dAtA[i] = 0xe2 + i++ + dAtA[i] = 0x1 + i++ + i = encodeVarintGenerated(dAtA, i, uint64(m.AzureDisk.Size())) + n230, err := m.AzureDisk.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n230 + } return i, nil } @@ -10623,6 +10682,16 @@ func (m *Capabilities) Size() (n int) { return n } +func (m *CascadeDiskVolumeSource) Size() (n int) { + var l int + _ = l + l = len(m.DiskID) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.FSType) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + func (m *CephFSPersistentVolumeSource) Size() (n int) { var l int _ = l @@ -12331,8 +12400,8 @@ func (m *PersistentVolumeSource) Size() (n int) { l = m.Quobyte.Size() n += 1 + l + sovGenerated(uint64(l)) } - if m.AzureDisk != nil { - l = m.AzureDisk.Size() + if m.CascadeDisk != nil { + l = m.CascadeDisk.Size() n += 2 + l + sovGenerated(uint64(l)) } if m.PhotonPersistentDisk != nil { @@ -12359,6 +12428,10 @@ func (m *PersistentVolumeSource) Size() (n int) { l = m.CSI.Size() n += 2 + l + sovGenerated(uint64(l)) } + if m.AzureDisk != nil { + l = m.AzureDisk.Size() + n += 2 + l + sovGenerated(uint64(l)) + } return n } @@ -13788,8 +13861,8 @@ func (m *VolumeSource) Size() (n int) { l = m.Quobyte.Size() n += 2 + l + sovGenerated(uint64(l)) } - if m.AzureDisk != nil { - l = m.AzureDisk.Size() + if m.CascadeDisk != nil { + l = m.CascadeDisk.Size() n += 2 + l + sovGenerated(uint64(l)) } if m.PhotonPersistentDisk != nil { @@ -13812,6 +13885,10 @@ func (m *VolumeSource) Size() (n int) { l = m.StorageOS.Size() n += 2 + l + sovGenerated(uint64(l)) } + if m.AzureDisk != nil { + l = m.AzureDisk.Size() + n += 2 + l + sovGenerated(uint64(l)) + } return n } @@ -13971,6 +14048,17 @@ func (this *Capabilities) String() string { }, "") return s } +func (this *CascadeDiskVolumeSource) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&CascadeDiskVolumeSource{`, + `DiskID:` + fmt.Sprintf("%v", this.DiskID) + `,`, + `FSType:` + fmt.Sprintf("%v", this.FSType) + `,`, + `}`, + }, "") + return s +} func (this *CephFSPersistentVolumeSource) String() string { if this == nil { return "nil" @@ -15335,13 +15423,14 @@ func (this *PersistentVolumeSource) String() string { `AzureFile:` + strings.Replace(fmt.Sprintf("%v", this.AzureFile), "AzureFilePersistentVolumeSource", "AzureFilePersistentVolumeSource", 1) + `,`, `VsphereVolume:` + strings.Replace(fmt.Sprintf("%v", this.VsphereVolume), "VsphereVirtualDiskVolumeSource", "VsphereVirtualDiskVolumeSource", 1) + `,`, `Quobyte:` + strings.Replace(fmt.Sprintf("%v", this.Quobyte), "QuobyteVolumeSource", "QuobyteVolumeSource", 1) + `,`, - `AzureDisk:` + strings.Replace(fmt.Sprintf("%v", this.AzureDisk), "AzureDiskVolumeSource", "AzureDiskVolumeSource", 1) + `,`, + `CascadeDisk:` + strings.Replace(fmt.Sprintf("%v", this.CascadeDisk), "CascadeDiskVolumeSource", "CascadeDiskVolumeSource", 1) + `,`, `PhotonPersistentDisk:` + strings.Replace(fmt.Sprintf("%v", this.PhotonPersistentDisk), "PhotonPersistentDiskVolumeSource", "PhotonPersistentDiskVolumeSource", 1) + `,`, `PortworxVolume:` + strings.Replace(fmt.Sprintf("%v", this.PortworxVolume), "PortworxVolumeSource", "PortworxVolumeSource", 1) + `,`, `ScaleIO:` + strings.Replace(fmt.Sprintf("%v", this.ScaleIO), "ScaleIOPersistentVolumeSource", "ScaleIOPersistentVolumeSource", 1) + `,`, `Local:` + strings.Replace(fmt.Sprintf("%v", this.Local), "LocalVolumeSource", "LocalVolumeSource", 1) + `,`, `StorageOS:` + strings.Replace(fmt.Sprintf("%v", this.StorageOS), "StorageOSPersistentVolumeSource", "StorageOSPersistentVolumeSource", 1) + `,`, `CSI:` + strings.Replace(fmt.Sprintf("%v", this.CSI), "CSIPersistentVolumeSource", "CSIPersistentVolumeSource", 1) + `,`, + `AzureDisk:` + strings.Replace(fmt.Sprintf("%v", this.AzureDisk), "AzureDiskVolumeSource", "AzureDiskVolumeSource", 1) + `,`, `}`, }, "") return s @@ -16468,12 +16557,13 @@ func (this *VolumeSource) String() string { `ConfigMap:` + strings.Replace(fmt.Sprintf("%v", this.ConfigMap), "ConfigMapVolumeSource", "ConfigMapVolumeSource", 1) + `,`, `VsphereVolume:` + strings.Replace(fmt.Sprintf("%v", this.VsphereVolume), "VsphereVirtualDiskVolumeSource", "VsphereVirtualDiskVolumeSource", 1) + `,`, `Quobyte:` + strings.Replace(fmt.Sprintf("%v", this.Quobyte), "QuobyteVolumeSource", "QuobyteVolumeSource", 1) + `,`, - `AzureDisk:` + strings.Replace(fmt.Sprintf("%v", this.AzureDisk), "AzureDiskVolumeSource", "AzureDiskVolumeSource", 1) + `,`, + `CascadeDisk:` + strings.Replace(fmt.Sprintf("%v", this.CascadeDisk), "CascadeDiskVolumeSource", "CascadeDiskVolumeSource", 1) + `,`, `PhotonPersistentDisk:` + strings.Replace(fmt.Sprintf("%v", this.PhotonPersistentDisk), "PhotonPersistentDiskVolumeSource", "PhotonPersistentDiskVolumeSource", 1) + `,`, `PortworxVolume:` + strings.Replace(fmt.Sprintf("%v", this.PortworxVolume), "PortworxVolumeSource", "PortworxVolumeSource", 1) + `,`, `ScaleIO:` + strings.Replace(fmt.Sprintf("%v", this.ScaleIO), "ScaleIOVolumeSource", "ScaleIOVolumeSource", 1) + `,`, `Projected:` + strings.Replace(fmt.Sprintf("%v", this.Projected), "ProjectedVolumeSource", "ProjectedVolumeSource", 1) + `,`, `StorageOS:` + strings.Replace(fmt.Sprintf("%v", this.StorageOS), "StorageOSVolumeSource", "StorageOSVolumeSource", 1) + `,`, + `AzureDisk:` + strings.Replace(fmt.Sprintf("%v", this.AzureDisk), "AzureDiskVolumeSource", "AzureDiskVolumeSource", 1) + `,`, `}`, }, "") return s @@ -34322,7 +34412,7 @@ func (m *PersistentVolumeSource) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 16: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AzureDisk", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CascadeDisk", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -34346,10 +34436,10 @@ func (m *PersistentVolumeSource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.AzureDisk == nil { - m.AzureDisk = &AzureDiskVolumeSource{} + if m.CascadeDisk == nil { + m.CascadeDisk = &CascadeDiskVolumeSource{} } - if err := m.AzureDisk.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.CascadeDisk.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -34551,6 +34641,39 @@ func (m *PersistentVolumeSource) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 23: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AzureDisk", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.AzureDisk == nil { + m.AzureDisk = &AzureDiskVolumeSource{} + } + if err := m.AzureDisk.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -35089,6 +35212,114 @@ func (m *PersistentVolumeStatus) Unmarshal(dAtA []byte) error { } return nil } +func (m *CascadeDiskVolumeSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CascadeDiskVolumeSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CascadeDiskVolumeSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DiskID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DiskID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FSType", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FSType = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *PhotonPersistentDiskVolumeSource) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -48522,7 +48753,7 @@ func (m *VolumeSource) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 22: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field AzureDisk", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CascadeDisk", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -48546,10 +48777,10 @@ func (m *VolumeSource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.AzureDisk == nil { - m.AzureDisk = &AzureDiskVolumeSource{} + if m.CascadeDisk == nil { + m.CascadeDisk = &CascadeDiskVolumeSource{} } - if err := m.AzureDisk.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.CascadeDisk.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -48718,6 +48949,39 @@ func (m *VolumeSource) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 28: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AzureDisk", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.AzureDisk == nil { + m.AzureDisk = &AzureDiskVolumeSource{} + } + if err := m.AzureDisk.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 728cbd5..87ba6a4 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -333,9 +333,8 @@ type VolumeSource struct { // Quobyte represents a Quobyte mount on the host that shares a pod's lifetime // +optional Quobyte *QuobyteVolumeSource `json:"quobyte,omitempty" protobuf:"bytes,21,opt,name=quobyte"` - // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - // +optional - AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty" protobuf:"bytes,22,opt,name=azureDisk"` + // CascadeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine + CascadeDisk *CascadeDiskVolumeSource `json:"vkeDisk,omitempty" protobuf:"bytes,22,opt,name=cascadeDisk"` // PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine PhotonPersistentDisk *PhotonPersistentDiskVolumeSource `json:"photonPersistentDisk,omitempty" protobuf:"bytes,23,opt,name=photonPersistentDisk"` // Items for all in one resources secrets, configmaps, and downward API @@ -349,6 +348,9 @@ type VolumeSource struct { // StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. // +optional StorageOS *StorageOSVolumeSource `json:"storageos,omitempty" protobuf:"bytes,27,opt,name=storageos"` + // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + // +optional + AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty" protobuf:"bytes,28,opt,name=azureDisk"` } // PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. @@ -428,9 +430,8 @@ type PersistentVolumeSource struct { // Quobyte represents a Quobyte mount on the host that shares a pod's lifetime // +optional Quobyte *QuobyteVolumeSource `json:"quobyte,omitempty" protobuf:"bytes,15,opt,name=quobyte"` - // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. - // +optional - AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty" protobuf:"bytes,16,opt,name=azureDisk"` + // CascadeDisk represents a Cascade persistent disk attached and mounted on kubelets host machine + CascadeDisk *CascadeDiskVolumeSource `json:"vkeDisk,omitempty" protobuf:"bytes,16,opt,name=cascadeDisk"` // PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine PhotonPersistentDisk *PhotonPersistentDiskVolumeSource `json:"photonPersistentDisk,omitempty" protobuf:"bytes,17,opt,name=photonPersistentDisk"` // PortworxVolume represents a portworx volume attached and mounted on kubelets host machine @@ -449,6 +450,9 @@ type PersistentVolumeSource struct { // CSI represents storage that handled by an external CSI driver // +optional CSI *CSIPersistentVolumeSource `json:"csi,omitempty" protobuf:"bytes,22,opt,name=csi"` + // AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + // +optional + AzureDisk *AzureDiskVolumeSource `json:"azureDisk,omitempty" protobuf:"bytes,23,opt,name=azureDisk"` } const ( @@ -1578,6 +1582,16 @@ type StorageOSPersistentVolumeSource struct { SecretRef *ObjectReference `json:"secretRef,omitempty" protobuf:"bytes,5,opt,name=secretRef"` } +// Represents a Photon Controller persistent disk resource. +type CascadeDiskVolumeSource struct { + // ID that identifies Cascade persistent disk + DiskID string `json:"diskID" protobuf:"bytes,1,opt,name=diskID"` + // Filesystem type to mount. + // Must be a filesystem type supported by the host operating system. + // Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + FSType string `json:"fsType,omitempty" protobuf:"bytes,2,opt,name=fsType"` +} + // Adapts a ConfigMap into a volume. // // The contents of the target ConfigMap's Data field will be presented in a -- 2.7.4