Browse code

daemon: always dump stack to file

Dumping to log is unusable in 90% of cases and inspecting file is much
more convenient.

Signed-off-by: Alexander Morozov <lk4d4@docker.com>

Alexander Morozov authored on 2016/11/01 23:32:55
Showing 3 changed files
... ...
@@ -7,15 +7,22 @@ import (
7 7
 	"os/signal"
8 8
 	"syscall"
9 9
 
10
-	psignal "github.com/docker/docker/pkg/signal"
10
+	stackdump "github.com/docker/docker/pkg/signal"
11
+
12
+	"github.com/Sirupsen/logrus"
11 13
 )
12 14
 
13
-func setupDumpStackTrap(_ string) {
15
+func setupDumpStackTrap(root string) {
14 16
 	c := make(chan os.Signal, 1)
15 17
 	signal.Notify(c, syscall.SIGUSR1)
16 18
 	go func() {
17 19
 		for range c {
18
-			psignal.DumpStacks("")
20
+			path, err := stackdump.DumpStacks(root)
21
+			if err != nil {
22
+				logrus.WithError(err).Error("failed to write goroutines dump")
23
+				continue
24
+			}
25
+			logrus.Infof("goroutine stacks written to %s", path)
19 26
 		}
20 27
 	}()
21 28
 }
... ...
@@ -35,7 +35,12 @@ func setupDumpStackTrap(root string) {
35 35
 		logrus.Debugf("Stackdump - waiting signal at %s", ev)
36 36
 		for {
37 37
 			syscall.WaitForSingleObject(h, syscall.INFINITE)
38
-			signal.DumpStacks(root)
38
+			path, err := signal.DumpStacks(root)
39
+			if err != nil {
40
+				logrus.WithError(err).Error("failed to write goroutines dump")
41
+				continue
42
+			}
43
+			logrus.Infof("goroutine stacks written to %s", path)
39 44
 		}
40 45
 	}()
41 46
 }
... ...
@@ -1,6 +1,7 @@
1 1
 package signal
2 2
 
3 3
 import (
4
+	"fmt"
4 5
 	"os"
5 6
 	gosignal "os/signal"
6 7
 	"path/filepath"
... ...
@@ -10,6 +11,7 @@ import (
10 10
 	"time"
11 11
 
12 12
 	"github.com/Sirupsen/logrus"
13
+	"github.com/pkg/errors"
13 14
 )
14 15
 
15 16
 // Trap sets up a simplified signal "trap", appropriate for common
... ...
@@ -64,8 +66,11 @@ func Trap(cleanup func()) {
64 64
 	}()
65 65
 }
66 66
 
67
-// DumpStacks dumps the runtime stack.
68
-func DumpStacks(root string) {
67
+const stacksLogNameTemplate = "goroutine-stacks-%s.log"
68
+
69
+// DumpStacks appends the runtime stack into file in dir and returns full path
70
+// to that file.
71
+func DumpStacks(dir string) (string, error) {
69 72
 	var (
70 73
 		buf       []byte
71 74
 		stackSize int
... ...
@@ -77,32 +82,15 @@ func DumpStacks(root string) {
77 77
 		bufferLen *= 2
78 78
 	}
79 79
 	buf = buf[:stackSize]
80
-	// Note that if the daemon is started with a less-verbose log-level than "info" (the default), the goroutine
81
-	// traces won't show up in the log.
82
-	if root == "" {
83
-		logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
84
-	} else {
85
-		// Dumps the stacks to a file in the root directory of the daemon
86
-		// On Windows, this overcomes two issues - one being that if the stack is too big, it doesn't
87
-		// get written to the event log when the Windows daemon is running as a service.
88
-		// Second, using logrus, the tabs and new-lines end up getting written as literal
89
-		// \t and \n's, meaning you need to use something like notepad++ to convert the
90
-		// output into something readable using 'type' from a command line or notepad/notepad++ etc.
91
-		path := filepath.Join(root, "goroutine-stacks.log")
92
-		f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
93
-		if err != nil {
94
-			logrus.Warnf("Could not open %s to write the goroutine stacks: %v", path, err)
95
-			return
96
-		}
97
-		defer f.Close()
98
-		f.WriteString("=== BEGIN goroutine stack dump ===\n")
99
-		f.WriteString(time.Now().String() + "\n")
100
-		if _, err := f.Write(buf); err != nil {
101
-			logrus.Warnf("Could not write goroutine stacks to %s: %v", path, err)
102
-			return
103
-		}
104
-		f.WriteString("=== END goroutine stack dump ===\n")
105
-		f.Sync()
106
-		logrus.Infof("goroutine stacks written to %s", path)
80
+	path := filepath.Join(dir, fmt.Sprintf(stacksLogNameTemplate, time.Now().Format(time.RFC3339)))
81
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0666)
82
+	if err != nil {
83
+		return "", errors.Wrap(err, "failed to open file to write the goroutine stacks")
84
+	}
85
+	defer f.Close()
86
+	if _, err := f.Write(buf); err != nil {
87
+		return "", errors.Wrap(err, "failed to write goroutine stacks")
107 88
 	}
89
+	f.Sync()
90
+	return path, nil
108 91
 }