Browse code

deploy: make activeDeadlineSeconds configurable for deployer pods

Michail Kargakis authored on 2016/12/16 00:08:36
Showing 12 changed files
... ...
@@ -329,10 +329,15 @@ func fuzzInternalObject(t *testing.T, forVersion unversioned.GroupVersion, item
329 329
 			}
330 330
 		},
331 331
 		func(j *deploy.DeploymentStrategy, c fuzz.Continue) {
332
+			randInt64 := func() *int64 {
333
+				p := int64(c.RandUint64())
334
+				return &p
335
+			}
332 336
 			c.FuzzNoCustom(j)
333 337
 			j.RecreateParams, j.RollingParams, j.CustomParams = nil, nil, nil
334 338
 			strategyTypes := []deploy.DeploymentStrategyType{deploy.DeploymentStrategyTypeRecreate, deploy.DeploymentStrategyTypeRolling, deploy.DeploymentStrategyTypeCustom}
335 339
 			j.Type = strategyTypes[c.Rand.Intn(len(strategyTypes))]
340
+			j.ActiveDeadlineSeconds = randInt64()
336 341
 			switch j.Type {
337 342
 			case deploy.DeploymentStrategyTypeRecreate:
338 343
 				params := &deploy.RecreateDeploymentStrategyParams{}
... ...
@@ -344,10 +349,6 @@ func fuzzInternalObject(t *testing.T, forVersion unversioned.GroupVersion, item
344 344
 				j.RecreateParams = params
345 345
 			case deploy.DeploymentStrategyTypeRolling:
346 346
 				params := &deploy.RollingDeploymentStrategyParams{}
347
-				randInt64 := func() *int64 {
348
-					p := int64(c.RandUint64())
349
-					return &p
350
-				}
351 347
 				params.TimeoutSeconds = randInt64()
352 348
 				params.IntervalSeconds = randInt64()
353 349
 				params.UpdatePeriodSeconds = randInt64()
... ...
@@ -80,6 +80,7 @@ func OkStrategy() deployapi.DeploymentStrategy {
80 80
 		RecreateParams: &deployapi.RecreateDeploymentStrategyParams{
81 81
 			TimeoutSeconds: mkintp(20),
82 82
 		},
83
+		ActiveDeadlineSeconds: mkintp(int(deployapi.MaxDeploymentDurationSeconds)),
83 84
 	}
84 85
 }
85 86
 
... ...
@@ -195,6 +195,10 @@ type DeploymentStrategy struct {
195 195
 	Labels map[string]string
196 196
 	// Annotations is a set of key, value pairs added to custom deployer and lifecycle pre/post hook pods.
197 197
 	Annotations map[string]string
198
+
199
+	// ActiveDeadlineSeconds is the duration in seconds that the deployer pods for this deployment
200
+	// config may be active on a node before the system actively tries to terminate them.
201
+	ActiveDeadlineSeconds *int64
198 202
 }
199 203
 
200 204
 // DeploymentStrategyType refers to a specific DeploymentStrategy implementation.
... ...
@@ -64,6 +64,10 @@ func SetDefaults_DeploymentStrategy(obj *DeploymentStrategy) {
64 64
 	if obj.Type == DeploymentStrategyTypeRecreate && obj.RecreateParams == nil {
65 65
 		obj.RecreateParams = &RecreateDeploymentStrategyParams{}
66 66
 	}
67
+
68
+	if obj.ActiveDeadlineSeconds == nil {
69
+		obj.ActiveDeadlineSeconds = mkintp(deployapi.MaxDeploymentDurationSeconds)
70
+	}
67 71
 }
68 72
 
69 73
 func SetDefaults_RecreateDeploymentStrategyParams(obj *RecreateDeploymentStrategyParams) {
... ...
@@ -42,6 +42,7 @@ func TestDefaults(t *testing.T) {
42 42
 							MaxSurge:            &defaultIntOrString,
43 43
 							MaxUnavailable:      &defaultIntOrString,
44 44
 						},
45
+						ActiveDeadlineSeconds: newInt64(deployapi.MaxDeploymentDurationSeconds),
45 46
 					},
46 47
 					Triggers: []deployv1.DeploymentTriggerPolicy{
47 48
 						{
... ...
@@ -127,6 +128,7 @@ func TestDefaults(t *testing.T) {
127 127
 							MaxSurge:            &differentIntOrString,
128 128
 							MaxUnavailable:      &differentIntOrString,
129 129
 						},
130
+						ActiveDeadlineSeconds: newInt64(deployapi.MaxDeploymentDurationSeconds),
130 131
 					},
131 132
 					Triggers: []deployv1.DeploymentTriggerPolicy{
132 133
 						{
... ...
@@ -165,6 +167,7 @@ func TestDefaults(t *testing.T) {
165 165
 							TimeoutSeconds:      newInt64(7),
166 166
 							MaxSurge:            newIntOrString(intstr.FromString("50%")),
167 167
 						},
168
+						ActiveDeadlineSeconds: newInt64(3600),
168 169
 					},
169 170
 					Triggers: []deployv1.DeploymentTriggerPolicy{
170 171
 						{
... ...
@@ -184,6 +187,7 @@ func TestDefaults(t *testing.T) {
184 184
 							MaxSurge:            newIntOrString(intstr.FromString("50%")),
185 185
 							MaxUnavailable:      newIntOrString(intstr.FromInt(0)),
186 186
 						},
187
+						ActiveDeadlineSeconds: newInt64(3600),
187 188
 					},
188 189
 					Triggers: []deployv1.DeploymentTriggerPolicy{
189 190
 						{
... ...
@@ -223,6 +227,7 @@ func TestDefaults(t *testing.T) {
223 223
 							MaxSurge:            newIntOrString(intstr.FromInt(0)),
224 224
 							MaxUnavailable:      newIntOrString(intstr.FromString("25%")),
225 225
 						},
226
+						ActiveDeadlineSeconds: newInt64(deployapi.MaxDeploymentDurationSeconds),
226 227
 					},
227 228
 					Triggers: []deployv1.DeploymentTriggerPolicy{
228 229
 						{
... ...
@@ -260,6 +265,7 @@ func TestDefaults(t *testing.T) {
260 260
 							MaxUnavailable:      newIntOrString(intstr.FromString("25%")),
261 261
 							MaxSurge:            newIntOrString(intstr.FromInt(0)),
262 262
 						},
263
+						ActiveDeadlineSeconds: newInt64(deployapi.MaxDeploymentDurationSeconds),
263 264
 					},
264 265
 					Triggers: []deployv1.DeploymentTriggerPolicy{
265 266
 						{},
... ...
@@ -90,6 +90,10 @@ type DeploymentStrategy struct {
90 90
 	Labels map[string]string `json:"labels,omitempty" protobuf:"bytes,6,rep,name=labels"`
91 91
 	// Annotations is a set of key, value pairs added to custom deployer and lifecycle pre/post hook pods.
92 92
 	Annotations map[string]string `json:"annotations,omitempty" protobuf:"bytes,7,rep,name=annotations"`
93
+
94
+	// ActiveDeadlineSeconds is the duration in seconds that the deployer pods for this deployment
95
+	// config may be active on a node before the system actively tries to terminate them.
96
+	ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
93 97
 }
94 98
 
95 99
 // DeploymentStrategyType refers to a specific DeploymentStrategy implementation.
... ...
@@ -44,9 +44,17 @@ func ValidateDeploymentConfigSpec(spec deployapi.DeploymentConfigSpec) field.Err
44 44
 		allErrs = append(allErrs, kapivalidation.ValidateNonnegativeField(int64(*spec.RevisionHistoryLimit), specPath.Child("revisionHistoryLimit"))...)
45 45
 	}
46 46
 	allErrs = append(allErrs, kapivalidation.ValidateNonnegativeField(int64(spec.MinReadySeconds), specPath.Child("minReadySeconds"))...)
47
-	if int64(spec.MinReadySeconds) >= deployapi.DefaultRollingTimeoutSeconds {
48
-		allErrs = append(allErrs, field.Invalid(specPath.Child("minReadySeconds"), spec.MinReadySeconds,
49
-			fmt.Sprintf("must be less than the deployment timeout (%ds)", deployapi.DefaultRollingTimeoutSeconds)))
47
+	timeoutSeconds := deployapi.DefaultRollingTimeoutSeconds
48
+	if spec.Strategy.RollingParams != nil && spec.Strategy.RollingParams.TimeoutSeconds != nil {
49
+		timeoutSeconds = *(spec.Strategy.RollingParams.TimeoutSeconds)
50
+	} else if spec.Strategy.RecreateParams != nil && spec.Strategy.RecreateParams.TimeoutSeconds != nil {
51
+		timeoutSeconds = *(spec.Strategy.RecreateParams.TimeoutSeconds)
52
+	}
53
+	if timeoutSeconds > 0 && int64(spec.MinReadySeconds) >= timeoutSeconds {
54
+		allErrs = append(allErrs, field.Invalid(specPath.Child("minReadySeconds"), spec.MinReadySeconds, fmt.Sprintf("must be less than the deployment timeout (%ds)", timeoutSeconds)))
55
+	}
56
+	if spec.Strategy.ActiveDeadlineSeconds != nil && int64(spec.MinReadySeconds) >= *(spec.Strategy.ActiveDeadlineSeconds) {
57
+		allErrs = append(allErrs, field.Invalid(specPath.Child("minReadySeconds"), spec.MinReadySeconds, fmt.Sprintf("must be less than activeDeadlineSeconds (%ds - used by the deployer pod)", *(spec.Strategy.ActiveDeadlineSeconds))))
50 58
 	}
51 59
 	if spec.Template == nil {
52 60
 		allErrs = append(allErrs, field.Required(specPath.Child("template"), ""))
... ...
@@ -239,6 +247,19 @@ func validateDeploymentStrategy(strategy *deployapi.DeploymentStrategy, pod *kap
239 239
 
240 240
 	errs = append(errs, validation.ValidateResourceRequirements(&strategy.Resources, fldPath.Child("resources"))...)
241 241
 
242
+	if strategy.ActiveDeadlineSeconds != nil {
243
+		errs = append(errs, kapivalidation.ValidateNonnegativeField(*strategy.ActiveDeadlineSeconds, fldPath.Child("activeDeadlineSeconds"))...)
244
+		var timeoutSeconds *int64
245
+		if strategy.RollingParams != nil {
246
+			timeoutSeconds = strategy.RollingParams.TimeoutSeconds
247
+		} else if strategy.RecreateParams != nil {
248
+			timeoutSeconds = strategy.RecreateParams.TimeoutSeconds
249
+		}
250
+		if timeoutSeconds != nil && *strategy.ActiveDeadlineSeconds <= *timeoutSeconds {
251
+			errs = append(errs, field.Invalid(fldPath.Child("activeDeadlineSeconds"), *strategy.ActiveDeadlineSeconds, "activeDeadlineSeconds must be greater than timeoutSeconds"))
252
+		}
253
+	}
254
+
242 255
 	return errs
243 256
 }
244 257
 
... ...
@@ -34,6 +34,7 @@ func rollingConfig(interval, updatePeriod, timeout int) api.DeploymentConfig {
34 34
 					TimeoutSeconds:      mkint64p(timeout),
35 35
 					MaxSurge:            intstr.FromInt(1),
36 36
 				},
37
+				ActiveDeadlineSeconds: mkint64p(3600),
37 38
 			},
38 39
 			Template: test.OkPodTemplate(),
39 40
 			Selector: test.OkSelector(),
... ...
@@ -55,6 +56,7 @@ func rollingConfigMax(maxSurge, maxUnavailable intstr.IntOrString) api.Deploymen
55 55
 					MaxSurge:            maxSurge,
56 56
 					MaxUnavailable:      maxUnavailable,
57 57
 				},
58
+				ActiveDeadlineSeconds: mkint64p(3600),
58 59
 			},
59 60
 			Template: test.OkPodTemplate(),
60 61
 			Selector: test.OkSelector(),
... ...
@@ -254,7 +256,8 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
254 254
 					Triggers: manualTrigger(),
255 255
 					Selector: test.OkSelector(),
256 256
 					Strategy: api.DeploymentStrategy{
257
-						CustomParams: test.OkCustomParams(),
257
+						CustomParams:          test.OkCustomParams(),
258
+						ActiveDeadlineSeconds: mkint64p(3600),
258 259
 					},
259 260
 					Template: test.OkPodTemplate(),
260 261
 				},
... ...
@@ -271,6 +274,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
271 271
 					Selector: test.OkSelector(),
272 272
 					Strategy: api.DeploymentStrategy{
273 273
 						Type: api.DeploymentStrategyTypeCustom,
274
+						ActiveDeadlineSeconds: mkint64p(3600),
274 275
 					},
275 276
 					Template: test.OkPodTemplate(),
276 277
 				},
... ...
@@ -292,6 +296,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
292 292
 								{Name: "A=B"},
293 293
 							},
294 294
 						},
295
+						ActiveDeadlineSeconds: mkint64p(3600),
295 296
 					},
296 297
 					Template: test.OkPodTemplate(),
297 298
 				},
... ...
@@ -314,6 +319,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
314 314
 								},
315 315
 							},
316 316
 						},
317
+						ActiveDeadlineSeconds: mkint64p(3600),
317 318
 					},
318 319
 					Template: test.OkPodTemplate(),
319 320
 					Selector: test.OkSelector(),
... ...
@@ -334,6 +340,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
334 334
 								FailurePolicy: api.LifecycleHookFailurePolicyRetry,
335 335
 							},
336 336
 						},
337
+						ActiveDeadlineSeconds: mkint64p(3600),
337 338
 					},
338 339
 					Template: test.OkPodTemplate(),
339 340
 					Selector: test.OkSelector(),
... ...
@@ -357,6 +364,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
357 357
 								},
358 358
 							},
359 359
 						},
360
+						ActiveDeadlineSeconds: mkint64p(3600),
360 361
 					},
361 362
 					Template: test.OkPodTemplate(),
362 363
 					Selector: test.OkSelector(),
... ...
@@ -380,6 +388,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
380 380
 								},
381 381
 							},
382 382
 						},
383
+						ActiveDeadlineSeconds: mkint64p(3600),
383 384
 					},
384 385
 					Template: test.OkPodTemplate(),
385 386
 					Selector: test.OkSelector(),
... ...
@@ -405,6 +414,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
405 405
 								},
406 406
 							},
407 407
 						},
408
+						ActiveDeadlineSeconds: mkint64p(3600),
408 409
 					},
409 410
 					Template: test.OkPodTemplate(),
410 411
 					Selector: test.OkSelector(),
... ...
@@ -425,6 +435,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
425 425
 								FailurePolicy: api.LifecycleHookFailurePolicyRetry,
426 426
 							},
427 427
 						},
428
+						ActiveDeadlineSeconds: mkint64p(3600),
428 429
 					},
429 430
 					Template: test.OkPodTemplate(),
430 431
 					Selector: test.OkSelector(),
... ...
@@ -445,6 +456,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
445 445
 								FailurePolicy: api.LifecycleHookFailurePolicyRetry,
446 446
 							},
447 447
 						},
448
+						ActiveDeadlineSeconds: mkint64p(3600),
448 449
 					},
449 450
 					Template: test.OkPodTemplate(),
450 451
 					Selector: test.OkSelector(),
... ...
@@ -471,6 +483,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
471 471
 								},
472 472
 							},
473 473
 						},
474
+						ActiveDeadlineSeconds: mkint64p(3600),
474 475
 					},
475 476
 					Template: test.OkPodTemplate(),
476 477
 					Selector: test.OkSelector(),
... ...
@@ -497,6 +510,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
497 497
 								},
498 498
 							},
499 499
 						},
500
+						ActiveDeadlineSeconds: mkint64p(3600),
500 501
 					},
501 502
 					Template: test.OkPodTemplate(),
502 503
 					Selector: test.OkSelector(),
... ...
@@ -523,6 +537,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
523 523
 								},
524 524
 							},
525 525
 						},
526
+						ActiveDeadlineSeconds: mkint64p(3600),
526 527
 					},
527 528
 					Template: test.OkPodTemplate(),
528 529
 					Selector: test.OkSelector(),
... ...
@@ -545,6 +560,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
545 545
 								TagImages:     []api.TagImageHook{{}},
546 546
 							},
547 547
 						},
548
+						ActiveDeadlineSeconds: mkint64p(3600),
548 549
 					},
549 550
 					Template: test.OkPodTemplate(),
550 551
 					Selector: test.OkSelector(),
... ...
@@ -587,6 +603,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
587 587
 								},
588 588
 							},
589 589
 						},
590
+						ActiveDeadlineSeconds: mkint64p(3600),
590 591
 					},
591 592
 					Template: test.OkPodTemplate(),
592 593
 					Selector: test.OkSelector(),
... ...
@@ -638,6 +655,7 @@ func TestValidateDeploymentConfigMissingFields(t *testing.T) {
638 638
 	}
639 639
 
640 640
 	for testName, v := range errorCases {
641
+		t.Logf("running scenario %q", testName)
641 642
 		errs := ValidateDeploymentConfig(&v.DeploymentConfig)
642 643
 		if len(v.ErrorType) == 0 {
643 644
 			if len(errs) > 0 {
... ...
@@ -287,6 +287,9 @@ func (c *DeploymentController) makeDeployerPod(deployment *kapi.ReplicationContr
287 287
 
288 288
 	// Assigning to a variable since its address is required
289 289
 	maxDeploymentDurationSeconds := deployapi.MaxDeploymentDurationSeconds
290
+	if deploymentConfig.Spec.Strategy.ActiveDeadlineSeconds != nil {
291
+		maxDeploymentDurationSeconds = *(deploymentConfig.Spec.Strategy.ActiveDeadlineSeconds)
292
+	}
290 293
 
291 294
 	gracePeriod := int64(10)
292 295
 
... ...
@@ -332,7 +332,11 @@ func makeHookPod(hook *deployapi.LifecycleHook, rc *kapi.ReplicationController,
332 332
 	}
333 333
 
334 334
 	// Assigning to a variable since its address is required
335
-	maxDeploymentDurationSeconds := deployapi.MaxDeploymentDurationSeconds - int64(time.Since(deployerPod.Status.StartTime.Time).Seconds())
335
+	defaultActiveDeadline := deployapi.MaxDeploymentDurationSeconds
336
+	if strategy.ActiveDeadlineSeconds != nil {
337
+		defaultActiveDeadline = *(strategy.ActiveDeadlineSeconds)
338
+	}
339
+	maxDeploymentDurationSeconds := defaultActiveDeadline - int64(time.Since(deployerPod.Status.StartTime.Time).Seconds())
336 340
 
337 341
 	// Let the kubelet manage retries if requested
338 342
 	restartPolicy := kapi.RestartPolicyNever
... ...
@@ -363,9 +363,9 @@ os::cmd::try_until_text 'oc get pods -l openshift.io/deployer-pod.type=hook-post
363 363
 # test the pre hook on a rolling deployment
364 364
 os::cmd::expect_success 'oc create -f test/testdata/failing-dc.yaml'
365 365
 os::cmd::try_until_success 'oc get rc/failing-dc-1'
366
-os::cmd::expect_success 'oc logs -f dc/failing-dc'
367 366
 os::cmd::expect_failure 'oc rollout status dc/failing-dc'
368 367
 os::cmd::expect_success_and_text 'oc logs dc/failing-dc' 'test pre hook executed'
368
+os::cmd::expect_success_and_text 'oc get po failing-dc-1-deploy -o jsonpath={.spec.activeDeadlineSeconds}' '3600'
369 369
 os::cmd::try_until_text 'oc rollout latest failing-dc --again -o revision' '2'
370 370
 os::cmd::expect_success_and_text 'oc logs --version=1 dc/failing-dc' 'test pre hook executed'
371 371
 os::cmd::expect_success_and_text 'oc logs --previous dc/failing-dc'  'test pre hook executed'
... ...
@@ -10,6 +10,7 @@ spec:
10 10
   strategy:
11 11
     type: Rolling
12 12
     resources: {}
13
+    activeDeadlineSeconds: 3600
13 14
     rollingParams:
14 15
       intervalSeconds: 1
15 16
       maxSurge: 25%