Browse code

Spew debugging

Signed-off-by: John Howard <jhoward@microsoft.com>

John Howard authored on 2016/10/13 07:29:47
Showing 6 changed files
... ...
@@ -476,10 +476,6 @@ func NewDaemon(config *Config, registryService registry.Service, containerdRemot
476 476
 		return nil, err
477 477
 	}
478 478
 
479
-	// set up SIGUSR1 handler on Unix-like systems, or a Win32 global event
480
-	// on Windows to dump Go routine stacks
481
-	setupDumpStackTrap(config.Root)
482
-
483 479
 	uidMaps, gidMaps, err := setupRemappedRoot(config)
484 480
 	if err != nil {
485 481
 		return nil, err
... ...
@@ -699,6 +695,10 @@ func NewDaemon(config *Config, registryService registry.Service, containerdRemot
699 699
 	engineCpus.Set(float64(info.NCPU))
700 700
 	engineMemory.Set(float64(info.MemTotal))
701 701
 
702
+	// set up SIGUSR1 handler on Unix-like systems, or a Win32 global event
703
+	// on Windows to dump Go routine stacks
704
+	d.setupDumpStackTrap(config.Root)
705
+
702 706
 	return d, nil
703 707
 }
704 708
 
705 709
new file mode 100644
... ...
@@ -0,0 +1,33 @@
0
+package daemon
1
+
2
+import (
3
+	"fmt"
4
+	"os"
5
+	"path/filepath"
6
+	"strings"
7
+	"time"
8
+
9
+	"github.com/davecgh/go-spew/spew"
10
+	"github.com/pkg/errors"
11
+)
12
+
13
+const dataStructuresLogNameTemplate = "daemon-data-%s.log"
14
+
15
+// dumpDaemon appends the daemon datastructures into file in dir and returns full path
16
+// to that file.
17
+func (d *Daemon) dumpDaemon(dir string) (string, error) {
18
+	// Ensure we recover from a panic as we are doing this without any locking
19
+	defer func() {
20
+		recover()
21
+	}()
22
+
23
+	path := filepath.Join(dir, fmt.Sprintf(dataStructuresLogNameTemplate, strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1)))
24
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
25
+	if err != nil {
26
+		return "", errors.Wrap(err, "failed to open file to write the daemon datastructure dump")
27
+	}
28
+	defer f.Close()
29
+	spew.Fdump(f, d) // Does not return an error
30
+	f.Sync()
31
+	return path, nil
32
+}
... ...
@@ -7,12 +7,11 @@ import (
7 7
 	"os/signal"
8 8
 	"syscall"
9 9
 
10
-	stackdump "github.com/docker/docker/pkg/signal"
11
-
12 10
 	"github.com/Sirupsen/logrus"
11
+	stackdump "github.com/docker/docker/pkg/signal"
13 12
 )
14 13
 
15
-func setupDumpStackTrap(root string) {
14
+func (d *Daemon) setupDumpStackTrap(root string) {
16 15
 	c := make(chan os.Signal, 1)
17 16
 	signal.Notify(c, syscall.SIGUSR1)
18 17
 	go func() {
... ...
@@ -20,9 +19,15 @@ func setupDumpStackTrap(root string) {
20 20
 			path, err := stackdump.DumpStacks(root)
21 21
 			if err != nil {
22 22
 				logrus.WithError(err).Error("failed to write goroutines dump")
23
-				continue
23
+			} else {
24
+				logrus.Infof("goroutine stacks written to %s", path)
25
+			}
26
+			path, err = d.dumpDaemon(root)
27
+			if err != nil {
28
+				logrus.WithError(err).Error("failed to write daemon datastructure dump")
29
+			} else {
30
+				logrus.Infof("daemon datastructure dump written to %s", path)
24 31
 			}
25
-			logrus.Infof("goroutine stacks written to %s", path)
26 32
 		}
27 33
 	}()
28 34
 }
... ...
@@ -2,6 +2,6 @@
2 2
 
3 3
 package daemon
4 4
 
5
-func setupDumpStackTrap(_ string) {
5
+func (d *Daemon) setupDumpStackTrap(_ string) {
6 6
 	return
7 7
 }
... ...
@@ -12,7 +12,7 @@ import (
12 12
 	"github.com/docker/docker/pkg/system"
13 13
 )
14 14
 
15
-func setupDumpStackTrap(root string) {
15
+func (d *Daemon) setupDumpStackTrap(root string) {
16 16
 	// Windows does not support signals like *nix systems. So instead of
17 17
 	// trapping on SIGUSR1 to dump stacks, we wait on a Win32 event to be
18 18
 	// signaled. ACL'd to builtin administrators and local system
... ...
@@ -38,9 +38,15 @@ func setupDumpStackTrap(root string) {
38 38
 			path, err := signal.DumpStacks(root)
39 39
 			if err != nil {
40 40
 				logrus.WithError(err).Error("failed to write goroutines dump")
41
-				continue
41
+			} else {
42
+				logrus.Infof("goroutine stacks written to %s", path)
43
+			}
44
+			path, err = d.dumpDaemon(root)
45
+			if err != nil {
46
+				logrus.WithError(err).Error("failed to write daemon datastructure dump")
47
+			} else {
48
+				logrus.Infof("daemon datastructure dump written to %s", path)
42 49
 			}
43
-			logrus.Infof("goroutine stacks written to %s", path)
44 50
 		}
45 51
 	}()
46 52
 }
... ...
@@ -6,6 +6,7 @@ import (
6 6
 	gosignal "os/signal"
7 7
 	"path/filepath"
8 8
 	"runtime"
9
+	"strings"
9 10
 	"sync/atomic"
10 11
 	"syscall"
11 12
 	"time"
... ...
@@ -82,7 +83,7 @@ func DumpStacks(dir string) (string, error) {
82 82
 		bufferLen *= 2
83 83
 	}
84 84
 	buf = buf[:stackSize]
85
-	path := filepath.Join(dir, fmt.Sprintf(stacksLogNameTemplate, time.Now().Format(time.RFC3339)))
85
+	path := filepath.Join(dir, fmt.Sprintf(stacksLogNameTemplate, strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1)))
86 86
 	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
87 87
 	if err != nil {
88 88
 		return "", errors.Wrap(err, "failed to open file to write the goroutine stacks")