@@ -29,7 +29,7 @@ clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837

 clone git github.com/imdario/mergo 0.2.1

 #get libnetwork packages

-clone git github.com/docker/libnetwork v0.7.0-dev.10

+clone git github.com/docker/libnetwork v0.7.0-rc.1

 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

 clone git github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b

 clone git github.com/hashicorp/memberlist 9a1e242e454d2443df330bdd51a436d5a9058fc4

@@ -1,5 +1,11 @@

 # Changelog

+## 0.7.0-rc.1 (2016-03-30)

+- Fixes https://github.com/docker/libnetwork/issues/985

+- Fixes https://github.com/docker/libnetwork/issues/945

+- Log time taken to set sandbox key

+- Limit number of concurrent DNS queries

+

 ## 0.7.0-dev.10 (2016-03-21)

 - Add IPv6 service discovery (AAAA records) in embedded DNS server

 - Honor enableIPv6 flag in network create for the IP allocation

@@ -218,6 +218,9 @@ func (c *controller) ReloadConfiguration(cfgOptions ...config.Option) error {

 				return types.ForbiddenErrorf("cannot accept new configuration because it modifies an existing datastore client")

 			}

 		} else {

+			if err := c.initScopedStore(s, nSCfg); err != nil {

+				return err

+			}

 			update = true

 		}

 	}

@@ -229,10 +232,6 @@ func (c *controller) ReloadConfiguration(cfgOptions ...config.Option) error {

 	c.cfg = cfg

 	c.Unlock()

-	if err := c.initStores(); err != nil {

-		return err

-	}

-

 	if c.discovery == nil && c.cfg.Cluster.Watcher != nil {

 		if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil {

 			log.Errorf("Failed to Initialize Discovery after configuration update: %v", err)

@@ -129,8 +129,8 @@ func (d *driver) Leave(nid, eid string) error {

 	if d.notifyCh != nil {

 		d.notifyCh <- ovNotify{

 			action: "leave",

-			nid:    nid,

-			eid:    eid,

+			nw:     n,

+			ep:     ep,

 		}

 	}

@@ -149,9 +149,9 @@ func (n *network) joinSubnetSandbox(s *subnet) error {

 func (n *network) leaveSandbox() {

 	n.Lock()

+	defer n.Unlock()

 	n.joinCnt--

 	if n.joinCnt != 0 {

-		n.Unlock()

 		return

 	}

@@ -162,15 +162,14 @@ func (n *network) leaveSandbox() {

 	for _, s := range n.subnets {

 		s.once = &sync.Once{}

 	}

-	n.Unlock()

 	n.destroySandbox()

 }

+// to be called while holding network lock

 func (n *network) destroySandbox() {

-	sbox := n.sandbox()

-	if sbox != nil {

-		for _, iface := range sbox.Info().Interfaces() {

+	if n.sbox != nil {

+		for _, iface := range n.sbox.Info().Interfaces() {

 			if err := iface.Remove(); err != nil {

 				logrus.Debugf("Remove interface %s failed: %v", iface.SrcName(), err)

 			}

@@ -197,8 +196,8 @@ func (n *network) destroySandbox() {

 			}

 		}

-		sbox.Destroy()

-		n.setSandbox(nil)

+		n.sbox.Destroy()

+		n.sbox = nil

 	}

 }

@@ -12,8 +12,8 @@ import (

 type ovNotify struct {

 	action string

-	eid    string

-	nid    string

+	ep     *endpoint

+	nw     *network

 }

 type logWriter struct{}

@@ -81,13 +81,12 @@ func (d *driver) serfJoin(neighIP string) error {

 }

 func (d *driver) notifyEvent(event ovNotify) {

-	n := d.network(event.nid)

-	ep := n.endpoint(event.eid)

+	ep := event.ep

 	ePayload := fmt.Sprintf("%s %s %s %s", event.action, ep.addr.IP.String(),

 		net.IP(ep.addr.Mask).String(), ep.mac.String())

 	eName := fmt.Sprintf("jl %s %s %s", d.serfInstance.LocalMember().Addr.String(),

-		event.nid, event.eid)

+		event.nw.id, ep.id)

 	if err := d.serfInstance.UserEvent(eName, []byte(ePayload), true); err != nil {

 		logrus.Errorf("Sending user event failed: %v\n", err)

@@ -180,13 +180,24 @@ func (d *driver) nodeJoin(node string, self bool) {

 }

 func (d *driver) pushLocalEndpointEvent(action, nid, eid string) {

+	n := d.network(nid)

+	if n == nil {

+		logrus.Debugf("Error pushing local endpoint event for network %s", nid)

+		return

+	}

+	ep := n.endpoint(eid)

+	if ep == nil {

+		logrus.Debugf("Error pushing local endpoint event for ep %s / %s", nid, eid)

+		return

+	}

+

 	if !d.isSerfAlive() {

 		return

 	}

 	d.notifyCh <- ovNotify{

 		action: "join",

-		nid:    nid,

-		eid:    eid,

+		nw:     n,

+		ep:     ep,

 	}

 }

@@ -547,6 +547,10 @@ func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {

 	sb.joinLeaveStart()

 	defer sb.joinLeaveEnd()

+	if sb.resolver != nil {

+		sb.resolver.FlushExtServers()

+	}

+

 	return ep.sbLeave(sb, false, options...)

 }

@@ -28,8 +28,12 @@ type Resolver interface {

 	// NameServer() returns the IP of the DNS resolver for the

 	// containers.

 	NameServer() string

-	// To configure external name servers the resolver should use

+	// SetExtServers configures the external nameservers the resolver

+	// should use to forward queries

 	SetExtServers([]string)

+	// FlushExtServers clears the cached UDP connections to external

+	// nameservers

+	FlushExtServers()

 	// ResolverOptions returns resolv.conf options that should be set

 	ResolverOptions() []string

 }

@@ -43,6 +47,8 @@ const (

 	maxExtDNS       = 3 //max number of external servers to try

 	extIOTimeout    = 3 * time.Second

 	defaultRespSize = 512

+	maxConcurrent   = 50

+	logInterval     = 2 * time.Second

 )

 type extDNSEntry struct {

@@ -60,6 +66,9 @@ type resolver struct {

 	tcpServer  *dns.Server

 	tcpListen  *net.TCPListener

 	err        error

+	count      int32

+	tStamp     time.Time

+	queryLock  sync.Mutex

 }

 func init() {

@@ -139,11 +148,15 @@ func (r *resolver) Start() error {

 	return nil

 }

-func (r *resolver) Stop() {

+func (r *resolver) FlushExtServers() {

 	for i := 0; i < maxExtDNS; i++ {

 		r.extDNSList[i].extConn = nil

 		r.extDNSList[i].extOnce = sync.Once{}

 	}

+}

+

+func (r *resolver) Stop() {

+	r.FlushExtServers()

 	if r.server != nil {

 		r.server.Shutdown()

@@ -154,6 +167,9 @@ func (r *resolver) Stop() {

 	r.conn = nil

 	r.tcpServer = nil

 	r.err = fmt.Errorf("setup not done yet")

+	r.tStamp = time.Time{}

+	r.count = 0

+	r.queryLock = sync.Mutex{}

 }

 func (r *resolver) SetExtServers(dns []string) {

@@ -320,7 +336,8 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {

 			if extDNS.ipStr == "" {

 				break

 			}

-			log.Debugf("Querying ext dns %s:%s for %s[%d]", proto, extDNS.ipStr, name, query.Question[0].Qtype)

+			log.Debugf("Query %s[%d] from %s, forwarding to %s:%s", name, query.Question[0].Qtype,

+				w.LocalAddr().String(), proto, extDNS.ipStr)

 			extConnect := func() {

 				addr := fmt.Sprintf("%s:%d", extDNS.ipStr, 53)

@@ -358,6 +375,15 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {

 			extConn.SetDeadline(time.Now().Add(extIOTimeout))

 			co := &dns.Conn{Conn: extConn}

+			if r.concurrentQueryInc() == false {

+				old := r.tStamp

+				r.tStamp = time.Now()

+				if r.tStamp.Sub(old) > logInterval {

+					log.Errorf("More than %v concurrent queries from %s", maxConcurrent, w.LocalAddr().String())

+				}

+				continue

+			}

+

 			defer func() {

 				if proto == "tcp" {

 					co.Close()

@@ -365,11 +391,13 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {

 			}()

 			err = co.WriteMsg(query)

 			if err != nil {

+				r.concurrentQueryDec()

 				log.Debugf("Send to DNS server failed, %s", err)

 				continue

 			}

 			resp, err = co.ReadMsg()

+			r.concurrentQueryDec()

 			if err != nil {

 				log.Debugf("Read from DNS server failed, %s", err)

 				continue

@@ -389,3 +417,23 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {

 		log.Errorf("error writing resolver resp, %s", err)

 	}

 }

+

+func (r *resolver) concurrentQueryInc() bool {

+	r.queryLock.Lock()

+	defer r.queryLock.Unlock()

+	if r.count == maxConcurrent {

+		return false

+	}

+	r.count++

+	return true

+}

+

+func (r *resolver) concurrentQueryDec() bool {

+	r.queryLock.Lock()

+	defer r.queryLock.Unlock()

+	if r.count == 0 {

+		return false

+	}

+	r.count--

+	return true

+}

@@ -7,6 +7,7 @@ import (

 	"net"

 	"strings"

 	"sync"

+	"time"

 	log "github.com/Sirupsen/logrus"

 	"github.com/docker/libnetwork/etchosts"

@@ -536,6 +537,11 @@ func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoin

 }

 func (sb *sandbox) SetKey(basePath string) error {

+	start := time.Now()

+	defer func() {

+		log.Debugf("sandbox set key processing took %s for container %s", time.Now().Sub(start), sb.ContainerID())

+	}()

+

 	if basePath == "" {

 		return types.BadRequestErrorf("invalid sandbox key")

 	}

@@ -7,6 +7,18 @@ import (

 	"github.com/docker/libnetwork/datastore"

 )

+func (c *controller) initScopedStore(scope string, scfg *datastore.ScopeCfg) error {

+	store, err := datastore.NewDataStore(scope, scfg)

+	if err != nil {

+		return err

+	}

+	c.Lock()

+	c.stores = append(c.stores, store)

+	c.Unlock()

+

+	return nil

+}

+

 func (c *controller) initStores() error {

 	c.Lock()

 	if c.cfg == nil {

@@ -18,13 +30,9 @@ func (c *controller) initStores() error {

 	c.Unlock()

 	for scope, scfg := range scopeConfigs {

-		store, err := datastore.NewDataStore(scope, scfg)

-		if err != nil {

+		if err := c.initScopedStore(scope, scfg); err != nil {

 			return err

 		}

-		c.Lock()

-		c.stores = append(c.stores, store)

-		c.Unlock()

 	}

 	c.startWatch()