Browse code

Forced endpoint delete

Signed-off-by: Madhu Venugopal <madhu@docker.com>

Madhu Venugopal authored on 2016/01/09 04:24:14
Showing 10 changed files
... ...
@@ -529,7 +529,7 @@ func procDeleteEndpoint(c libnetwork.NetworkController, vars map[string]string,
529 529
 		return nil, errRsp
530 530
 	}
531 531
 
532
-	err := ep.Delete()
532
+	err := ep.Delete(false)
533 533
 	if err != nil {
534 534
 		return nil, convertNetworkError(err)
535 535
 	}
... ...
@@ -641,13 +641,22 @@ func procPublishService(c libnetwork.NetworkController, vars map[string]string,
641 641
 }
642 642
 
643 643
 func procUnpublishService(c libnetwork.NetworkController, vars map[string]string, body []byte) (interface{}, *responseStatus) {
644
+	var sd serviceDelete
645
+
646
+	if body != nil {
647
+		err := json.Unmarshal(body, &sd)
648
+		if err != nil {
649
+			return "", &responseStatus{Status: "Invalid body: " + err.Error(), StatusCode: http.StatusBadRequest}
650
+		}
651
+	}
652
+
644 653
 	epT, epBy := detectEndpointTarget(vars)
645 654
 	sv, errRsp := findService(c, epT, epBy)
646 655
 	if !errRsp.isOK() {
647 656
 		return nil, errRsp
648 657
 	}
649
-	err := sv.Delete()
650
-	if err != nil {
658
+
659
+	if err := sv.Delete(sd.Force); err != nil {
651 660
 		return nil, endpointToService(convertNetworkError(err))
652 661
 	}
653 662
 	return nil, &successResponse
... ...
@@ -690,15 +690,15 @@ func TestProcGetServices(t *testing.T) {
690 690
 	}
691 691
 
692 692
 	delete(vars, urlEpPID)
693
-	err = ep11.Delete()
693
+	err = ep11.Delete(false)
694 694
 	if err != nil {
695 695
 		t.Fatal(err)
696 696
 	}
697
-	err = ep12.Delete()
697
+	err = ep12.Delete(false)
698 698
 	if err != nil {
699 699
 		t.Fatal(err)
700 700
 	}
701
-	err = ep21.Delete()
701
+	err = ep21.Delete(false)
702 702
 	if err != nil {
703 703
 		t.Fatal(err)
704 704
 	}
... ...
@@ -1014,7 +1014,7 @@ func TestAttachDetachBackend(t *testing.T) {
1014 1014
 		t.Fatalf("Did not find expected sandbox. Got %v", sb)
1015 1015
 	}
1016 1016
 
1017
-	err = ep1.Delete()
1017
+	err = ep1.Delete(false)
1018 1018
 	if err != nil {
1019 1019
 		t.Fatal(err)
1020 1020
 	}
... ...
@@ -1495,7 +1495,7 @@ func TestFindEndpointUtil(t *testing.T) {
1495 1495
 		t.Fatalf("Diffenrent queries returned different endpoints")
1496 1496
 	}
1497 1497
 
1498
-	ep.Delete()
1498
+	ep.Delete(false)
1499 1499
 
1500 1500
 	_, errRsp = findEndpoint(c, nid, "secondEp", byID, byName)
1501 1501
 	if errRsp == &successResponse {
... ...
@@ -76,6 +76,12 @@ type servicePublish struct {
76 76
 	PortMapping  []types.PortBinding   `json:"port_mapping"`
77 77
 }
78 78
 
79
+// serviceDelete represents the body of the "unpublish service" http request message
80
+type serviceDelete struct {
81
+	Name  string `json:"name"`
82
+	Force bool   `json:"force"`
83
+}
84
+
79 85
 // extraHost represents the extra host object
80 86
 type extraHost struct {
81 87
 	Name    string `json:"name"`
... ...
@@ -216,6 +216,31 @@ func (c *controller) validateHostDiscoveryConfig() bool {
216 216
 	return true
217 217
 }
218 218
 
219
+func (c *controller) clusterHostID() string {
220
+	c.Lock()
221
+	defer c.Unlock()
222
+	if c.cfg == nil || c.cfg.Cluster.Address == "" {
223
+		return ""
224
+	}
225
+	addr := strings.Split(c.cfg.Cluster.Address, ":")
226
+	return addr[0]
227
+}
228
+
229
+func (c *controller) isNodeAlive(node string) bool {
230
+	if c.discovery == nil {
231
+		return false
232
+	}
233
+
234
+	nodes := c.discovery.Fetch()
235
+	for _, n := range nodes {
236
+		if n.String() == node {
237
+			return true
238
+		}
239
+	}
240
+
241
+	return false
242
+}
243
+
219 244
 func (c *controller) initDiscovery(watcher discovery.Watcher) error {
220 245
 	if c.cfg == nil {
221 246
 		return fmt.Errorf("discovery initialization requires a valid configuration")
... ...
@@ -87,7 +87,7 @@ func (sb *sandbox) clearDefaultGW() error {
87 87
 	if err := ep.sbLeave(sb); err != nil {
88 88
 		return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err)
89 89
 	}
90
-	if err := ep.Delete(); err != nil {
90
+	if err := ep.Delete(false); err != nil {
91 91
 		return fmt.Errorf("container %s: deleting endpoint on GW Network failed: %v", sb.containerID, err)
92 92
 	}
93 93
 	return nil
... ...
@@ -41,7 +41,7 @@ type Endpoint interface {
41 41
 	DriverInfo() (map[string]interface{}, error)
42 42
 
43 43
 	// Delete and detaches this endpoint from the network.
44
-	Delete() error
44
+	Delete(force bool) error
45 45
 }
46 46
 
47 47
 // EndpointOption is a option setter function type used to pass varios options to Network
... ...
@@ -56,6 +56,7 @@ type endpoint struct {
56 56
 	iface             *endpointInterface
57 57
 	joinInfo          *endpointJoinInfo
58 58
 	sandboxID         string
59
+	locator           string
59 60
 	exposedPorts      []types.TransportPort
60 61
 	anonymous         bool
61 62
 	disableResolution bool
... ...
@@ -84,6 +85,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
84 84
 		epMap["generic"] = ep.generic
85 85
 	}
86 86
 	epMap["sandbox"] = ep.sandboxID
87
+	epMap["locator"] = ep.locator
87 88
 	epMap["anonymous"] = ep.anonymous
88 89
 	epMap["disableResolution"] = ep.disableResolution
89 90
 	epMap["myAliases"] = ep.myAliases
... ...
@@ -167,6 +169,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
167 167
 	if v, ok := epMap["disableResolution"]; ok {
168 168
 		ep.disableResolution = v.(bool)
169 169
 	}
170
+	if l, ok := epMap["locator"]; ok {
171
+		ep.locator = l.(string)
172
+	}
170 173
 	ma, _ := json.Marshal(epMap["myAliases"])
171 174
 	var myAliases []string
172 175
 	json.Unmarshal(ma, &myAliases)
... ...
@@ -186,6 +191,7 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
186 186
 	dstEp.name = ep.name
187 187
 	dstEp.id = ep.id
188 188
 	dstEp.sandboxID = ep.sandboxID
189
+	dstEp.locator = ep.locator
189 190
 	dstEp.dbIndex = ep.dbIndex
190 191
 	dstEp.dbExists = ep.dbExists
191 192
 	dstEp.anonymous = ep.anonymous
... ...
@@ -600,7 +606,23 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
600 600
 	return sb.clearDefaultGW()
601 601
 }
602 602
 
603
-func (ep *endpoint) Delete() error {
603
+func (n *network) validateForceDelete(locator string) error {
604
+	if n.Scope() == datastore.LocalScope {
605
+		return nil
606
+	}
607
+
608
+	if locator == "" {
609
+		return fmt.Errorf("invalid endpoint locator identifier")
610
+	}
611
+
612
+	if n.getController().isNodeAlive(locator) {
613
+		return fmt.Errorf("the remote host %s hosting the container is alive", locator)
614
+	}
615
+
616
+	return nil
617
+}
618
+
619
+func (ep *endpoint) Delete(force bool) error {
604 620
 	var err error
605 621
 	n, err := ep.getNetworkFromStore()
606 622
 	if err != nil {
... ...
@@ -615,18 +637,33 @@ func (ep *endpoint) Delete() error {
615 615
 	ep.Lock()
616 616
 	epid := ep.id
617 617
 	name := ep.name
618
-	sb, _ := n.getController().SandboxByID(ep.sandboxID)
619
-	if sb != nil {
620
-		ep.Unlock()
618
+	sbid := ep.sandboxID
619
+	locator := ep.locator
620
+	ep.Unlock()
621
+
622
+	if force {
623
+		if err = n.validateForceDelete(locator); err != nil {
624
+			return fmt.Errorf("unable to force delete endpoint %s: %v", name, err)
625
+		}
626
+	}
627
+
628
+	sb, _ := n.getController().SandboxByID(sbid)
629
+	if sb != nil && !force {
621 630
 		return &ActiveContainerError{name: name, id: epid}
622 631
 	}
623
-	ep.Unlock()
632
+
633
+	if sb != nil {
634
+		if e := ep.sbLeave(sb); e != nil {
635
+			log.Warnf("failed to leave sandbox for endpoint %s : %v", name, e)
636
+		}
637
+	}
624 638
 
625 639
 	if err = n.getController().deleteFromStore(ep); err != nil {
626 640
 		return err
627 641
 	}
642
+
628 643
 	defer func() {
629
-		if err != nil {
644
+		if err != nil && !force {
630 645
 			ep.dbExists = false
631 646
 			if e := n.getController().updateToStore(ep); e != nil {
632 647
 				log.Warnf("failed to recreate endpoint in store %s : %v", name, e)
... ...
@@ -634,11 +671,11 @@ func (ep *endpoint) Delete() error {
634 634
 		}
635 635
 	}()
636 636
 
637
-	if err = n.getEpCnt().DecEndpointCnt(); err != nil {
637
+	if err = n.getEpCnt().DecEndpointCnt(); err != nil && !force {
638 638
 		return err
639 639
 	}
640 640
 	defer func() {
641
-		if err != nil {
641
+		if err != nil && !force {
642 642
 			if e := n.getEpCnt().IncEndpointCnt(); e != nil {
643 643
 				log.Warnf("failed to update network %s : %v", n.name, e)
644 644
 			}
... ...
@@ -648,7 +685,7 @@ func (ep *endpoint) Delete() error {
648 648
 	// unwatch for service records
649 649
 	n.getController().unWatchSvcRecord(ep)
650 650
 
651
-	if err = ep.deleteEndpoint(); err != nil {
651
+	if err = ep.deleteEndpoint(); err != nil && !force {
652 652
 		return err
653 653
 	}
654 654
 
... ...
@@ -923,7 +960,7 @@ func (c *controller) cleanupLocalEndpoints() {
923 923
 		}
924 924
 
925 925
 		for _, ep := range epl {
926
-			if err := ep.Delete(); err != nil {
926
+			if err := ep.Delete(false); err != nil {
927 927
 				log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
928 928
 			}
929 929
 		}
... ...
@@ -407,7 +407,7 @@ func TestIpamReleaseOnNetDriverFailures(t *testing.T) {
407 407
 	if err != nil {
408 408
 		t.Fatal(err)
409 409
 	}
410
-	defer ep.Delete()
410
+	defer ep.Delete(false)
411 411
 
412 412
 	expectedIP, _ := types.ParseCIDR("10.34.0.1/16")
413 413
 	if !types.CompareIPNet(ep.Info().Iface().Address(), expectedIP) {
... ...
@@ -135,7 +135,7 @@ func TestNull(t *testing.T) {
135 135
 		t.Fatal(err)
136 136
 	}
137 137
 
138
-	if err := ep.Delete(); err != nil {
138
+	if err := ep.Delete(false); err != nil {
139 139
 		t.Fatal(err)
140 140
 	}
141 141
 
... ...
@@ -213,11 +213,11 @@ func TestHost(t *testing.T) {
213 213
 		t.Fatal(err)
214 214
 	}
215 215
 
216
-	if err := ep1.Delete(); err != nil {
216
+	if err := ep1.Delete(false); err != nil {
217 217
 		t.Fatal(err)
218 218
 	}
219 219
 
220
-	if err := ep2.Delete(); err != nil {
220
+	if err := ep2.Delete(false); err != nil {
221 221
 		t.Fatal(err)
222 222
 	}
223 223
 
... ...
@@ -249,7 +249,7 @@ func TestHost(t *testing.T) {
249 249
 		t.Fatal(err)
250 250
 	}
251 251
 
252
-	if err := ep3.Delete(); err != nil {
252
+	if err := ep3.Delete(false); err != nil {
253 253
 		t.Fatal(err)
254 254
 	}
255 255
 
... ...
@@ -305,7 +305,7 @@ func TestBridge(t *testing.T) {
305 305
 		t.Fatalf("Incomplete data for port mapping in endpoint operational data: %d", len(pm))
306 306
 	}
307 307
 
308
-	if err := ep.Delete(); err != nil {
308
+	if err := ep.Delete(false); err != nil {
309 309
 		t.Fatal(err)
310 310
 	}
311 311
 
... ...
@@ -358,7 +358,7 @@ func TestBridgeIpv6FromMac(t *testing.T) {
358 358
 		t.Fatalf("Expected %v. Got: %v", expIP, iface.AddressIPv6())
359 359
 	}
360 360
 
361
-	if err := ep.Delete(); err != nil {
361
+	if err := ep.Delete(false); err != nil {
362 362
 		t.Fatal(err)
363 363
 	}
364 364
 
... ...
@@ -514,7 +514,7 @@ func TestDeleteNetworkWithActiveEndpoints(t *testing.T) {
514 514
 	}
515 515
 
516 516
 	// Done testing. Now cleanup.
517
-	if err := ep.Delete(); err != nil {
517
+	if err := ep.Delete(false); err != nil {
518 518
 		t.Fatal(err)
519 519
 	}
520 520
 
... ...
@@ -586,7 +586,7 @@ func TestUnknownEndpoint(t *testing.T) {
586 586
 		t.Fatal(err)
587 587
 	}
588 588
 
589
-	err = ep.Delete()
589
+	err = ep.Delete(false)
590 590
 	if err != nil {
591 591
 		t.Fatal(err)
592 592
 	}
... ...
@@ -624,7 +624,7 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
624 624
 		t.Fatal(err)
625 625
 	}
626 626
 	defer func() {
627
-		if err := ep11.Delete(); err != nil {
627
+		if err := ep11.Delete(false); err != nil {
628 628
 			t.Fatal(err)
629 629
 		}
630 630
 	}()
... ...
@@ -634,7 +634,7 @@ func TestNetworkEndpointsWalkers(t *testing.T) {
634 634
 		t.Fatal(err)
635 635
 	}
636 636
 	defer func() {
637
-		if err := ep12.Delete(); err != nil {
637
+		if err := ep12.Delete(false); err != nil {
638 638
 			t.Fatal(err)
639 639
 		}
640 640
 	}()
... ...
@@ -752,7 +752,7 @@ func TestDuplicateEndpoint(t *testing.T) {
752 752
 		t.Fatal(err)
753 753
 	}
754 754
 	defer func() {
755
-		if err := ep.Delete(); err != nil {
755
+		if err := ep.Delete(false); err != nil {
756 756
 			t.Fatal(err)
757 757
 		}
758 758
 	}()
... ...
@@ -761,7 +761,7 @@ func TestDuplicateEndpoint(t *testing.T) {
761 761
 	defer func() {
762 762
 		// Cleanup ep2 as well, else network cleanup might fail for failure cases
763 763
 		if ep2 != nil {
764
-			if err := ep2.Delete(); err != nil {
764
+			if err := ep2.Delete(false); err != nil {
765 765
 				t.Fatal(err)
766 766
 			}
767 767
 		}
... ...
@@ -904,7 +904,7 @@ func TestNetworkQuery(t *testing.T) {
904 904
 		t.Fatal(err)
905 905
 	}
906 906
 	defer func() {
907
-		if err := ep11.Delete(); err != nil {
907
+		if err := ep11.Delete(false); err != nil {
908 908
 			t.Fatal(err)
909 909
 		}
910 910
 	}()
... ...
@@ -914,7 +914,7 @@ func TestNetworkQuery(t *testing.T) {
914 914
 		t.Fatal(err)
915 915
 	}
916 916
 	defer func() {
917
-		if err := ep12.Delete(); err != nil {
917
+		if err := ep12.Delete(false); err != nil {
918 918
 			t.Fatal(err)
919 919
 		}
920 920
 	}()
... ...
@@ -1032,7 +1032,7 @@ func TestEndpointJoin(t *testing.T) {
1032 1032
 		t.Fatal(err)
1033 1033
 	}
1034 1034
 	defer func() {
1035
-		if err := ep1.Delete(); err != nil {
1035
+		if err := ep1.Delete(false); err != nil {
1036 1036
 			t.Fatal(err)
1037 1037
 		}
1038 1038
 	}()
... ...
@@ -1150,7 +1150,7 @@ func TestEndpointJoin(t *testing.T) {
1150 1150
 		t.Fatal(err)
1151 1151
 	}
1152 1152
 	defer func() {
1153
-		if err := ep2.Delete(); err != nil {
1153
+		if err := ep2.Delete(false); err != nil {
1154 1154
 			t.Fatal(err)
1155 1155
 		}
1156 1156
 	}()
... ...
@@ -1253,7 +1253,7 @@ func externalKeyTest(t *testing.T, reexec bool) {
1253 1253
 		t.Fatal(err)
1254 1254
 	}
1255 1255
 	defer func() {
1256
-		err = ep.Delete()
1256
+		err = ep.Delete(false)
1257 1257
 		if err != nil {
1258 1258
 			t.Fatal(err)
1259 1259
 		}
... ...
@@ -1264,7 +1264,7 @@ func externalKeyTest(t *testing.T, reexec bool) {
1264 1264
 		t.Fatal(err)
1265 1265
 	}
1266 1266
 	defer func() {
1267
-		err = ep2.Delete()
1267
+		err = ep2.Delete(false)
1268 1268
 		if err != nil {
1269 1269
 			t.Fatal(err)
1270 1270
 		}
... ...
@@ -1402,7 +1402,7 @@ func TestEndpointDeleteWithActiveContainer(t *testing.T) {
1402 1402
 		t.Fatal(err)
1403 1403
 	}
1404 1404
 	defer func() {
1405
-		err = ep.Delete()
1405
+		err = ep.Delete(false)
1406 1406
 		if err != nil {
1407 1407
 			t.Fatal(err)
1408 1408
 		}
... ...
@@ -1431,7 +1431,7 @@ func TestEndpointDeleteWithActiveContainer(t *testing.T) {
1431 1431
 		}
1432 1432
 	}()
1433 1433
 
1434
-	err = ep.Delete()
1434
+	err = ep.Delete(false)
1435 1435
 	if err == nil {
1436 1436
 		t.Fatal("Expected to fail. But instead succeeded")
1437 1437
 	}
... ...
@@ -1465,7 +1465,7 @@ func TestEndpointMultipleJoins(t *testing.T) {
1465 1465
 		t.Fatal(err)
1466 1466
 	}
1467 1467
 	defer func() {
1468
-		if err := ep.Delete(); err != nil {
1468
+		if err := ep.Delete(false); err != nil {
1469 1469
 			t.Fatal(err)
1470 1470
 		}
1471 1471
 	}()
... ...
@@ -1589,7 +1589,7 @@ func TestontainerInvalidLeave(t *testing.T) {
1589 1589
 		t.Fatal(err)
1590 1590
 	}
1591 1591
 	defer func() {
1592
-		if err := ep.Delete(); err != nil {
1592
+		if err := ep.Delete(false); err != nil {
1593 1593
 			t.Fatal(err)
1594 1594
 		}
1595 1595
 	}()
... ...
@@ -2334,7 +2334,7 @@ func runParallelTests(t *testing.T, thrNumber int) {
2334 2334
 			t.Fatal(err)
2335 2335
 		}
2336 2336
 	} else {
2337
-		err = ep.Delete()
2337
+		err = ep.Delete(false)
2338 2338
 		if err != nil {
2339 2339
 			t.Fatal(err)
2340 2340
 		}
... ...
@@ -681,6 +681,7 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
681 681
 	// Initialize ep.network with a possibly stale copy of n. We need this to get network from
682 682
 	// store. But once we get it from store we will have the most uptodate copy possible.
683 683
 	ep.network = n
684
+	ep.locator = n.getController().clusterHostID()
684 685
 	ep.network, err = ep.getNetworkFromStore()
685 686
 	if err != nil {
686 687
 		return nil, fmt.Errorf("failed to get network during CreateEndpoint: %v", err)
... ...
@@ -198,7 +198,7 @@ func (sb *sandbox) Delete() error {
198 198
 			log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
199 199
 		}
200 200
 
201
-		if err := ep.Delete(); err != nil {
201
+		if err := ep.Delete(false); err != nil {
202 202
 			log.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err)
203 203
 		}
204 204
 	}