Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)
Michael Crosby authored on 2014/01/30 11:34:43... | ... |
@@ -5,6 +5,11 @@ import ( |
5 | 5 |
"net" |
6 | 6 |
) |
7 | 7 |
|
8 |
+const ( |
|
9 |
+ DefaultNetworkMtu = 1500 |
|
10 |
+ DisableNetworkBridge = "none" |
|
11 |
+) |
|
12 |
+ |
|
8 | 13 |
// FIXME: separate runtime configuration from http api configuration |
9 | 14 |
type DaemonConfig struct { |
10 | 15 |
Pidfile string |
... | ... |
@@ -13,12 +18,13 @@ type DaemonConfig struct { |
13 | 13 |
Dns []string |
14 | 14 |
EnableIptables bool |
15 | 15 |
EnableIpForward bool |
16 |
- BridgeIface string |
|
17 |
- BridgeIp string |
|
18 | 16 |
DefaultIp net.IP |
17 |
+ BridgeIface string |
|
18 |
+ BridgeIP string |
|
19 | 19 |
InterContainerCommunication bool |
20 | 20 |
GraphDriver string |
21 | 21 |
Mtu int |
22 |
+ DisableNetwork bool |
|
22 | 23 |
} |
23 | 24 |
|
24 | 25 |
// ConfigFromJob creates and returns a new DaemonConfig object |
... | ... |
@@ -30,7 +36,7 @@ func DaemonConfigFromJob(job *engine.Job) *DaemonConfig { |
30 | 30 |
AutoRestart: job.GetenvBool("AutoRestart"), |
31 | 31 |
EnableIptables: job.GetenvBool("EnableIptables"), |
32 | 32 |
EnableIpForward: job.GetenvBool("EnableIpForward"), |
33 |
- BridgeIp: job.Getenv("BridgeIp"), |
|
33 |
+ BridgeIP: job.Getenv("BridgeIp"), |
|
34 | 34 |
DefaultIp: net.ParseIP(job.Getenv("DefaultIp")), |
35 | 35 |
InterContainerCommunication: job.GetenvBool("InterContainerCommunication"), |
36 | 36 |
GraphDriver: job.Getenv("GraphDriver"), |
... | ... |
@@ -38,16 +44,12 @@ func DaemonConfigFromJob(job *engine.Job) *DaemonConfig { |
38 | 38 |
if dns := job.GetenvList("Dns"); dns != nil { |
39 | 39 |
config.Dns = dns |
40 | 40 |
} |
41 |
- if br := job.Getenv("BridgeIface"); br != "" { |
|
42 |
- config.BridgeIface = br |
|
43 |
- } else { |
|
44 |
- config.BridgeIface = DefaultNetworkBridge |
|
45 |
- } |
|
46 | 41 |
if mtu := job.GetenvInt("Mtu"); mtu != 0 { |
47 | 42 |
config.Mtu = mtu |
48 | 43 |
} else { |
49 | 44 |
config.Mtu = DefaultNetworkMtu |
50 | 45 |
} |
46 |
+ config.DisableNetwork = job.Getenv("BridgeIface") == DisableNetworkBridge |
|
51 | 47 |
|
52 | 48 |
return config |
53 | 49 |
} |
... | ... |
@@ -8,7 +8,6 @@ import ( |
8 | 8 |
"github.com/dotcloud/docker/engine" |
9 | 9 |
"github.com/dotcloud/docker/execdriver" |
10 | 10 |
"github.com/dotcloud/docker/graphdriver" |
11 |
- "github.com/dotcloud/docker/networkdriver/ipallocator" |
|
12 | 11 |
"github.com/dotcloud/docker/pkg/mount" |
13 | 12 |
"github.com/dotcloud/docker/pkg/term" |
14 | 13 |
"github.com/dotcloud/docker/utils" |
... | ... |
@@ -16,7 +15,6 @@ import ( |
16 | 16 |
"io" |
17 | 17 |
"io/ioutil" |
18 | 18 |
"log" |
19 |
- "net" |
|
20 | 19 |
"os" |
21 | 20 |
"path" |
22 | 21 |
"path/filepath" |
... | ... |
@@ -47,7 +45,6 @@ type Container struct { |
47 | 47 |
State State |
48 | 48 |
Image string |
49 | 49 |
|
50 |
- network *NetworkInterface |
|
51 | 50 |
NetworkSettings *NetworkSettings |
52 | 51 |
|
53 | 52 |
ResolvConfPath string |
... | ... |
@@ -558,6 +555,7 @@ func populateCommand(c *Container) { |
558 | 558 |
en *execdriver.Network |
559 | 559 |
driverConfig []string |
560 | 560 |
) |
561 |
+ |
|
561 | 562 |
if !c.Config.NetworkDisabled { |
562 | 563 |
network := c.NetworkSettings |
563 | 564 |
en = &execdriver.Network{ |
... | ... |
@@ -603,15 +601,18 @@ func (container *Container) Start() (err error) { |
603 | 603 |
if container.State.IsRunning() { |
604 | 604 |
return fmt.Errorf("The container %s is already running.", container.ID) |
605 | 605 |
} |
606 |
+ |
|
606 | 607 |
defer func() { |
607 | 608 |
if err != nil { |
608 | 609 |
container.cleanup() |
609 | 610 |
} |
610 | 611 |
}() |
612 |
+ |
|
611 | 613 |
if err := container.Mount(); err != nil { |
612 | 614 |
return err |
613 | 615 |
} |
614 |
- if container.runtime.networkManager.disabled { |
|
616 |
+ |
|
617 |
+ if container.runtime.config.DisableNetwork { |
|
615 | 618 |
container.Config.NetworkDisabled = true |
616 | 619 |
container.buildHostnameAndHostsFiles("127.0.1.1") |
617 | 620 |
} else { |
... | ... |
@@ -669,34 +670,39 @@ func (container *Container) Start() (err error) { |
669 | 669 |
} |
670 | 670 |
|
671 | 671 |
if len(children) > 0 { |
672 |
- container.activeLinks = make(map[string]*Link, len(children)) |
|
673 |
- |
|
674 |
- // If we encounter an error make sure that we rollback any network |
|
675 |
- // config and ip table changes |
|
676 |
- rollback := func() { |
|
677 |
- for _, link := range container.activeLinks { |
|
678 |
- link.Disable() |
|
679 |
- } |
|
680 |
- container.activeLinks = nil |
|
681 |
- } |
|
682 |
- |
|
683 |
- for p, child := range children { |
|
684 |
- link, err := NewLink(container, child, p, runtime.networkManager.bridgeIface) |
|
685 |
- if err != nil { |
|
686 |
- rollback() |
|
687 |
- return err |
|
688 |
- } |
|
689 |
- |
|
690 |
- container.activeLinks[link.Alias()] = link |
|
691 |
- if err := link.Enable(); err != nil { |
|
692 |
- rollback() |
|
693 |
- return err |
|
694 |
- } |
|
672 |
+ panic("todo crosbymichael") |
|
673 |
+ /* |
|
674 |
+ linking is specific to iptables and the bridge we need to move this to a job |
|
675 |
+ |
|
676 |
+ container.activeLinks = make(map[string]*Link, len(children)) |
|
677 |
+ |
|
678 |
+ // If we encounter an error make sure that we rollback any network |
|
679 |
+ // config and ip table changes |
|
680 |
+ rollback := func() { |
|
681 |
+ for _, link := range container.activeLinks { |
|
682 |
+ link.Disable() |
|
683 |
+ } |
|
684 |
+ container.activeLinks = nil |
|
685 |
+ } |
|
695 | 686 |
|
696 |
- for _, envVar := range link.ToEnv() { |
|
697 |
- env = append(env, envVar) |
|
698 |
- } |
|
699 |
- } |
|
687 |
+ for p, child := range children { |
|
688 |
+ link, err := NewLink(container, child, p, runtime.networkManager.bridgeIface) |
|
689 |
+ if err != nil { |
|
690 |
+ rollback() |
|
691 |
+ return err |
|
692 |
+ } |
|
693 |
+ |
|
694 |
+ container.activeLinks[link.Alias()] = link |
|
695 |
+ if err := link.Enable(); err != nil { |
|
696 |
+ rollback() |
|
697 |
+ return err |
|
698 |
+ } |
|
699 |
+ |
|
700 |
+ for _, envVar := range link.ToEnv() { |
|
701 |
+ env = append(env, envVar) |
|
702 |
+ } |
|
703 |
+ } |
|
704 |
+ */ |
|
700 | 705 |
} |
701 | 706 |
|
702 | 707 |
for _, elem := range container.Config.Env { |
... | ... |
@@ -1102,34 +1108,44 @@ func (container *Container) allocateNetwork() error { |
1102 | 1102 |
} |
1103 | 1103 |
|
1104 | 1104 |
var ( |
1105 |
- iface *NetworkInterface |
|
1106 |
- err error |
|
1105 |
+ env *engine.Env |
|
1106 |
+ eng = container.runtime.srv.Eng |
|
1107 | 1107 |
) |
1108 | 1108 |
if container.State.IsGhost() { |
1109 |
- if manager := container.runtime.networkManager; manager.disabled { |
|
1110 |
- iface = &NetworkInterface{disabled: true} |
|
1109 |
+ if container.runtime.config.DisableNetwork { |
|
1110 |
+ env = &engine.Env{} |
|
1111 | 1111 |
} else { |
1112 |
- iface = &NetworkInterface{ |
|
1113 |
- IPNet: net.IPNet{IP: net.ParseIP(container.NetworkSettings.IPAddress), Mask: manager.bridgeNetwork.Mask}, |
|
1114 |
- Gateway: manager.bridgeNetwork.IP, |
|
1115 |
- manager: manager, |
|
1116 |
- } |
|
1117 |
- if iface != nil && iface.IPNet.IP != nil { |
|
1118 |
- if _, err := ipallocator.RequestIP(manager.bridgeNetwork, &iface.IPNet.IP); err != nil { |
|
1119 |
- return err |
|
1112 |
+ // TODO: @crosbymichael |
|
1113 |
+ panic("not implemented") |
|
1114 |
+ /* |
|
1115 |
+ iface = &NetworkInterface{ |
|
1116 |
+ IPNet: net.IPNet{IP: net.ParseIP(container.NetworkSettings.IPAddress), Mask: manager.bridgeNetwork.Mask}, |
|
1117 |
+ Gateway: manager.bridgeNetwork.IP, |
|
1120 | 1118 |
} |
1121 |
- } else { |
|
1122 |
- iface, err = container.runtime.networkManager.Allocate() |
|
1123 |
- if err != nil { |
|
1124 |
- return err |
|
1119 |
+ |
|
1120 |
+ // request an existing ip |
|
1121 |
+ if iface != nil && iface.IPNet.IP != nil { |
|
1122 |
+ if _, err := ipallocator.RequestIP(manager.bridgeNetwork, &iface.IPNet.IP); err != nil { |
|
1123 |
+ return err |
|
1124 |
+ } |
|
1125 |
+ } else { |
|
1126 |
+ job = eng.Job("allocate_interface", container.ID) |
|
1127 |
+ if err := job.Run(); err != nil { |
|
1128 |
+ return err |
|
1129 |
+ } |
|
1125 | 1130 |
} |
1126 |
- } |
|
1131 |
+ */ |
|
1127 | 1132 |
} |
1128 | 1133 |
} else { |
1129 |
- iface, err = container.runtime.networkManager.Allocate() |
|
1134 |
+ job := eng.Job("allocate_interface", container.ID) |
|
1135 |
+ var err error |
|
1136 |
+ env, err = job.Stdout.AddEnv() |
|
1130 | 1137 |
if err != nil { |
1131 | 1138 |
return err |
1132 | 1139 |
} |
1140 |
+ if err := job.Run(); err != nil { |
|
1141 |
+ return err |
|
1142 |
+ } |
|
1133 | 1143 |
} |
1134 | 1144 |
|
1135 | 1145 |
if container.Config.PortSpecs != nil { |
... | ... |
@@ -1171,37 +1187,43 @@ func (container *Container) allocateNetwork() error { |
1171 | 1171 |
if container.hostConfig.PublishAllPorts && len(binding) == 0 { |
1172 | 1172 |
binding = append(binding, PortBinding{}) |
1173 | 1173 |
} |
1174 |
+ |
|
1174 | 1175 |
for i := 0; i < len(binding); i++ { |
1175 | 1176 |
b := binding[i] |
1176 |
- nat, err := iface.AllocatePort(port, b) |
|
1177 |
- if err != nil { |
|
1178 |
- iface.Release() |
|
1177 |
+ |
|
1178 |
+ portJob := eng.Job("allocate_port", container.ID) |
|
1179 |
+ portJob.Setenv("HostIP", b.HostIp) |
|
1180 |
+ portJob.Setenv("HostPort", b.HostPort) |
|
1181 |
+ portJob.Setenv("Proto", port.Proto()) |
|
1182 |
+ portJob.Setenv("ContainerPort", port.Port()) |
|
1183 |
+ |
|
1184 |
+ if err := portJob.Run(); err != nil { |
|
1185 |
+ eng.Job("release_interface", container.ID).Run() |
|
1179 | 1186 |
return err |
1180 | 1187 |
} |
1181 |
- utils.Debugf("Allocate port: %s:%s->%s", nat.Binding.HostIp, port, nat.Binding.HostPort) |
|
1182 |
- binding[i] = nat.Binding |
|
1183 | 1188 |
} |
1184 | 1189 |
bindings[port] = binding |
1185 | 1190 |
} |
1186 | 1191 |
container.writeHostConfig() |
1187 | 1192 |
|
1188 | 1193 |
container.NetworkSettings.Ports = bindings |
1189 |
- container.network = iface |
|
1190 | 1194 |
|
1191 |
- container.NetworkSettings.Bridge = container.runtime.networkManager.bridgeIface |
|
1192 |
- container.NetworkSettings.IPAddress = iface.IPNet.IP.String() |
|
1193 |
- container.NetworkSettings.IPPrefixLen, _ = iface.IPNet.Mask.Size() |
|
1194 |
- container.NetworkSettings.Gateway = iface.Gateway.String() |
|
1195 |
+ container.NetworkSettings.Bridge = env.Get("Bridge") |
|
1196 |
+ container.NetworkSettings.IPAddress = env.Get("IP") |
|
1197 |
+ container.NetworkSettings.IPPrefixLen = env.GetInt("IPPrefixLen") |
|
1198 |
+ container.NetworkSettings.Gateway = env.Get("Gateway") |
|
1199 |
+ fmt.Printf("\n-----> %#v\n", container.NetworkSettings) |
|
1195 | 1200 |
|
1196 | 1201 |
return nil |
1197 | 1202 |
} |
1198 | 1203 |
|
1199 | 1204 |
func (container *Container) releaseNetwork() { |
1200 |
- if container.Config.NetworkDisabled || container.network == nil { |
|
1205 |
+ if container.Config.NetworkDisabled { |
|
1201 | 1206 |
return |
1202 | 1207 |
} |
1203 |
- container.network.Release() |
|
1204 |
- container.network = nil |
|
1208 |
+ eng := container.runtime.srv.Eng |
|
1209 |
+ |
|
1210 |
+ eng.Job("release_interface", container.ID).Run() |
|
1205 | 1211 |
container.NetworkSettings = &NetworkSettings{} |
1206 | 1212 |
} |
1207 | 1213 |
|
... | ... |
@@ -19,8 +19,6 @@ import ( |
19 | 19 |
|
20 | 20 |
const ( |
21 | 21 |
DefaultNetworkBridge = "docker0" |
22 |
- DisableNetworkBridge = "none" |
|
23 |
- DefaultNetworkMtu = 1500 |
|
24 | 22 |
siocBRADDBR = 0x89a0 |
25 | 23 |
) |
26 | 24 |
|
... | ... |
@@ -70,17 +68,24 @@ func InitDriver(job *engine.Job) engine.Status { |
70 | 70 |
enableIPTables = job.GetenvBool("EnableIptables") |
71 | 71 |
icc = job.GetenvBool("InterContainerCommunication") |
72 | 72 |
ipForward = job.GetenvBool("EnableIpForward") |
73 |
+ bridgeIP = job.Getenv("BridgeIP") |
|
73 | 74 |
) |
75 |
+ |
|
74 | 76 |
bridgeIface = job.Getenv("BridgeIface") |
77 |
+ if bridgeIface == "" { |
|
78 |
+ bridgeIface = DefaultNetworkBridge |
|
79 |
+ } |
|
75 | 80 |
|
76 | 81 |
addr, err := networkdriver.GetIfaceAddr(bridgeIface) |
77 | 82 |
if err != nil { |
78 | 83 |
// If the iface is not found, try to create it |
79 |
- if err := createBridgeIface(bridgeIface); err != nil { |
|
84 |
+ job.Logf("creating new bridge for %s", bridgeIface) |
|
85 |
+ if err := createBridge(bridgeIP); err != nil { |
|
80 | 86 |
job.Error(err) |
81 | 87 |
return engine.StatusErr |
82 | 88 |
} |
83 | 89 |
|
90 |
+ job.Logf("getting iface addr") |
|
84 | 91 |
addr, err = networkdriver.GetIfaceAddr(bridgeIface) |
85 | 92 |
if err != nil { |
86 | 93 |
job.Error(err) |
... | ... |
@@ -122,9 +127,14 @@ func InitDriver(job *engine.Job) engine.Status { |
122 | 122 |
} |
123 | 123 |
|
124 | 124 |
bridgeNetwork = network |
125 |
+ |
|
126 |
+ // https://github.com/dotcloud/docker/issues/2768 |
|
127 |
+ job.Eng.Hack_SetGlobalVar("httpapi.bridgeIP", bridgeNetwork.IP) |
|
128 |
+ |
|
125 | 129 |
for name, f := range map[string]engine.Handler{ |
126 | 130 |
"allocate_interface": Allocate, |
127 | 131 |
"release_interface": Release, |
132 |
+ "allocate_port": AllocatePort, |
|
128 | 133 |
} { |
129 | 134 |
if err := job.Eng.Register(name, f); err != nil { |
130 | 135 |
job.Error(err) |
... | ... |
@@ -304,6 +314,10 @@ func Allocate(job *engine.Job) engine.Status { |
304 | 304 |
out.Set("IP", string(*ip)) |
305 | 305 |
out.Set("Mask", string(bridgeNetwork.Mask)) |
306 | 306 |
out.Set("Gateway", string(bridgeNetwork.IP)) |
307 |
+ out.Set("Bridge", bridgeIface) |
|
308 |
+ |
|
309 |
+ size, _ := bridgeNetwork.Mask.Size() |
|
310 |
+ out.SetInt("IPPrefixLen", size) |
|
307 | 311 |
|
308 | 312 |
currentInterfaces[id] = &networkInterface{ |
309 | 313 |
IP: *ip, |
... | ... |
@@ -4,6 +4,7 @@ import ( |
4 | 4 |
"container/list" |
5 | 5 |
"fmt" |
6 | 6 |
"github.com/dotcloud/docker/archive" |
7 |
+ "github.com/dotcloud/docker/engine" |
|
7 | 8 |
"github.com/dotcloud/docker/execdriver" |
8 | 9 |
"github.com/dotcloud/docker/execdriver/chroot" |
9 | 10 |
"github.com/dotcloud/docker/execdriver/lxc" |
... | ... |
@@ -12,6 +13,7 @@ import ( |
12 | 12 |
_ "github.com/dotcloud/docker/graphdriver/btrfs" |
13 | 13 |
_ "github.com/dotcloud/docker/graphdriver/devmapper" |
14 | 14 |
_ "github.com/dotcloud/docker/graphdriver/vfs" |
15 |
+ _ "github.com/dotcloud/docker/networkdriver/lxc" |
|
15 | 16 |
"github.com/dotcloud/docker/networkdriver/portallocator" |
16 | 17 |
"github.com/dotcloud/docker/pkg/graphdb" |
17 | 18 |
"github.com/dotcloud/docker/pkg/sysinfo" |
... | ... |
@@ -42,7 +44,6 @@ type Runtime struct { |
42 | 42 |
repository string |
43 | 43 |
sysInitPath string |
44 | 44 |
containers *list.List |
45 |
- networkManager *NetworkManager |
|
46 | 45 |
graph *Graph |
47 | 46 |
repositories *TagStore |
48 | 47 |
idIndex *utils.TruncIndex |
... | ... |
@@ -609,15 +610,15 @@ func (runtime *Runtime) RegisterLink(parent, child *Container, alias string) err |
609 | 609 |
} |
610 | 610 |
|
611 | 611 |
// FIXME: harmonize with NewGraph() |
612 |
-func NewRuntime(config *DaemonConfig) (*Runtime, error) { |
|
613 |
- runtime, err := NewRuntimeFromDirectory(config) |
|
612 |
+func NewRuntime(config *DaemonConfig, eng *engine.Engine) (*Runtime, error) { |
|
613 |
+ runtime, err := NewRuntimeFromDirectory(config, eng) |
|
614 | 614 |
if err != nil { |
615 | 615 |
return nil, err |
616 | 616 |
} |
617 | 617 |
return runtime, nil |
618 | 618 |
} |
619 | 619 |
|
620 |
-func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) { |
|
620 |
+func NewRuntimeFromDirectory(config *DaemonConfig, eng *engine.Engine) (*Runtime, error) { |
|
621 | 621 |
|
622 | 622 |
// Set the default driver |
623 | 623 |
graphdriver.DefaultDriver = config.GraphDriver |
... | ... |
@@ -664,12 +665,19 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) { |
664 | 664 |
if err != nil { |
665 | 665 |
return nil, fmt.Errorf("Couldn't create Tag store: %s", err) |
666 | 666 |
} |
667 |
- if config.BridgeIface == "" { |
|
668 |
- config.BridgeIface = DefaultNetworkBridge |
|
669 |
- } |
|
670 |
- netManager, err := newNetworkManager(config) |
|
671 |
- if err != nil { |
|
672 |
- return nil, err |
|
667 |
+ |
|
668 |
+ if !config.DisableNetwork { |
|
669 |
+ job := eng.Job("init_networkdriver") |
|
670 |
+ |
|
671 |
+ job.SetenvBool("EnableIptables", config.EnableIptables) |
|
672 |
+ job.SetenvBool("InterContainerCommunication", config.InterContainerCommunication) |
|
673 |
+ job.SetenvBool("EnableIpForward", config.EnableIpForward) |
|
674 |
+ job.Setenv("BridgeIface", config.BridgeIface) |
|
675 |
+ job.Setenv("BridgeIP", config.BridgeIP) |
|
676 |
+ |
|
677 |
+ if err := job.Run(); err != nil { |
|
678 |
+ return nil, err |
|
679 |
+ } |
|
673 | 680 |
} |
674 | 681 |
|
675 | 682 |
graphdbPath := path.Join(config.Root, "linkgraph.db") |
... | ... |
@@ -721,7 +729,6 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) { |
721 | 721 |
runtime := &Runtime{ |
722 | 722 |
repository: runtimeRepo, |
723 | 723 |
containers: list.New(), |
724 |
- networkManager: netManager, |
|
725 | 724 |
graph: g, |
726 | 725 |
repositories: repositories, |
727 | 726 |
idIndex: utils.NewTruncIndex(), |
... | ... |
@@ -65,10 +65,7 @@ func jobInitApi(job *engine.Job) engine.Status { |
65 | 65 |
}() |
66 | 66 |
job.Eng.Hack_SetGlobalVar("httpapi.server", srv) |
67 | 67 |
job.Eng.Hack_SetGlobalVar("httpapi.runtime", srv.runtime) |
68 |
- // https://github.com/dotcloud/docker/issues/2768 |
|
69 |
- if srv.runtime.networkManager.bridgeNetwork != nil { |
|
70 |
- job.Eng.Hack_SetGlobalVar("httpapi.bridgeIP", srv.runtime.networkManager.bridgeNetwork.IP) |
|
71 |
- } |
|
68 |
+ |
|
72 | 69 |
for name, handler := range map[string]engine.Handler{ |
73 | 70 |
"export": srv.ContainerExport, |
74 | 71 |
"create": srv.ContainerCreate, |
... | ... |
@@ -2354,7 +2351,7 @@ func (srv *Server) ContainerCopy(job *engine.Job) engine.Status { |
2354 | 2354 |
} |
2355 | 2355 |
|
2356 | 2356 |
func NewServer(eng *engine.Engine, config *DaemonConfig) (*Server, error) { |
2357 |
- runtime, err := NewRuntime(config) |
|
2357 |
+ runtime, err := NewRuntime(config, eng) |
|
2358 | 2358 |
if err != nil { |
2359 | 2359 |
return nil, err |
2360 | 2360 |
} |