Browse code

Implement client side display for stats

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Michael Crosby authored on 2015/01/08 09:22:42
Showing 7 changed files
... ...
@@ -16,6 +16,7 @@ import (
16 16
 	"path"
17 17
 	"path/filepath"
18 18
 	"runtime"
19
+	"sort"
19 20
 	"strconv"
20 21
 	"strings"
21 22
 	"text/tabwriter"
... ...
@@ -42,6 +43,7 @@ import (
42 42
 	"github.com/docker/docker/pkg/urlutil"
43 43
 	"github.com/docker/docker/registry"
44 44
 	"github.com/docker/docker/runconfig"
45
+	"github.com/docker/docker/stats"
45 46
 	"github.com/docker/docker/utils"
46 47
 	"github.com/docker/libtrust"
47 48
 )
... ...
@@ -2618,3 +2620,106 @@ func (cli *DockerCli) CmdExec(args ...string) error {
2618 2618
 
2619 2619
 	return nil
2620 2620
 }
2621
+
2622
+type containerStats struct {
2623
+	Name             string
2624
+	CpuPercentage    float64
2625
+	Memory           float64
2626
+	MemoryPercentage float64
2627
+	NetworkRx        int
2628
+	NetworkTx        int
2629
+}
2630
+
2631
+type statSorter struct {
2632
+	stats []containerStats
2633
+}
2634
+
2635
+func (s *statSorter) Len() int {
2636
+	return len(s.stats)
2637
+}
2638
+
2639
+func (s *statSorter) Swap(i, j int) {
2640
+	s.stats[i], s.stats[j] = s.stats[j], s.stats[i]
2641
+}
2642
+
2643
+func (s *statSorter) Less(i, j int) bool {
2644
+	return s.stats[i].Name < s.stats[j].Name
2645
+}
2646
+
2647
+func (cli *DockerCli) CmdStats(args ...string) error {
2648
+	cmd := cli.Subcmd("stats", "CONTAINER", "Stream the stats of a container", true)
2649
+	cmd.Require(flag.Min, 1)
2650
+	utils.ParseFlags(cmd, args, true)
2651
+
2652
+	cStats := map[string]containerStats{}
2653
+	for _, name := range cmd.Args() {
2654
+		go cli.streamStats(name, cStats)
2655
+	}
2656
+	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
2657
+	for _ = range time.Tick(1000 * time.Millisecond) {
2658
+		fmt.Fprint(cli.out, "\033[2J")
2659
+		fmt.Fprint(cli.out, "\033[H")
2660
+		fmt.Fprintln(w, "CONTAINER\tCPU %\tMEM\tMEM %\tNET I/O")
2661
+		sStats := []containerStats{}
2662
+		for _, s := range cStats {
2663
+			sStats = append(sStats, s)
2664
+		}
2665
+		sorter := &statSorter{sStats}
2666
+		sort.Sort(sorter)
2667
+		for _, s := range sStats {
2668
+			fmt.Fprintf(w, "%s\t%f%%\t%s\t%f%%\t%d/%d\n",
2669
+				s.Name,
2670
+				s.CpuPercentage,
2671
+				units.HumanSize(s.Memory),
2672
+				s.MemoryPercentage,
2673
+				s.NetworkRx, s.NetworkTx)
2674
+		}
2675
+		w.Flush()
2676
+	}
2677
+	return nil
2678
+}
2679
+
2680
+func (cli *DockerCli) streamStats(name string, data map[string]containerStats) error {
2681
+	stream, _, err := cli.call("GET", "/containers/"+name+"/stats", nil, false)
2682
+	if err != nil {
2683
+		return err
2684
+	}
2685
+
2686
+	var (
2687
+		previousCpu    uint64
2688
+		previousSystem uint64
2689
+		start          = true
2690
+		dec            = json.NewDecoder(stream)
2691
+	)
2692
+	for {
2693
+		var v *stats.Stats
2694
+		if err := dec.Decode(&v); err != nil {
2695
+			return err
2696
+		}
2697
+		memPercent := float64(v.MemoryStats.Usage) / float64(v.MemoryStats.Limit) * 100.0
2698
+		cpuPercent := 0.0
2699
+
2700
+		if !start {
2701
+			cpuDelta := float64(v.CpuStats.CpuUsage.TotalUsage) - float64(previousCpu)
2702
+			systemDelta := float64(int(v.CpuStats.SystemUsage)/v.ClockTicks) - float64(int(previousSystem)/v.ClockTicks)
2703
+
2704
+			if systemDelta > 0.0 {
2705
+				cpuPercent = (cpuDelta / systemDelta) * float64(v.ClockTicks*len(v.CpuStats.CpuUsage.PercpuUsage))
2706
+			}
2707
+		}
2708
+		start = false
2709
+		d := data[name]
2710
+		d.Name = name
2711
+		d.CpuPercentage = cpuPercent
2712
+		d.Memory = float64(v.MemoryStats.Usage)
2713
+		d.MemoryPercentage = memPercent
2714
+		d.NetworkRx = int(v.Network.RxBytes)
2715
+		d.NetworkTx = int(v.Network.TxBytes)
2716
+		data[name] = d
2717
+
2718
+		previousCpu = v.CpuStats.CpuUsage.TotalUsage
2719
+		previousSystem = v.CpuStats.SystemUsage
2720
+	}
2721
+	return nil
2722
+
2723
+}
... ...
@@ -106,8 +106,10 @@ type Resources struct {
106 106
 
107 107
 type ResourceStats struct {
108 108
 	*libcontainer.ContainerStats
109
-	Read       time.Time `json:"read"`
110
-	ClockTicks int       `json:"clock_ticks"`
109
+	Read        time.Time `json:"read"`
110
+	ClockTicks  int       `json:"clock_ticks"`
111
+	MemoryLimit int64     `json:"memory_limit"`
112
+	SystemUsage uint64    `json:"system_usage"`
111 113
 }
112 114
 
113 115
 type Mount struct {
... ...
@@ -2,14 +2,21 @@ package execdrivers
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"path"
6
+
5 7
 	"github.com/docker/docker/daemon/execdriver"
6 8
 	"github.com/docker/docker/daemon/execdriver/lxc"
7 9
 	"github.com/docker/docker/daemon/execdriver/native"
8 10
 	"github.com/docker/docker/pkg/sysinfo"
9
-	"path"
11
+	"github.com/docker/docker/pkg/system"
10 12
 )
11 13
 
12 14
 func NewDriver(name, root, initPath string, sysInfo *sysinfo.SysInfo) (execdriver.Driver, error) {
15
+	meminfo, err := system.ReadMemInfo()
16
+	if err != nil {
17
+		return nil, err
18
+	}
19
+
13 20
 	switch name {
14 21
 	case "lxc":
15 22
 		// we want to give the lxc driver the full docker root because it needs
... ...
@@ -17,7 +24,7 @@ func NewDriver(name, root, initPath string, sysInfo *sysinfo.SysInfo) (execdrive
17 17
 		// to be backwards compatible
18 18
 		return lxc.NewDriver(root, initPath, sysInfo.AppArmor)
19 19
 	case "native":
20
-		return native.NewDriver(path.Join(root, "execdriver", "native"), initPath)
20
+		return native.NewDriver(path.Join(root, "execdriver", "native"), initPath, meminfo.MemTotal/1000)
21 21
 	}
22 22
 	return nil, fmt.Errorf("unknown exec driver %s", name)
23 23
 }
... ...
@@ -42,23 +42,23 @@ type driver struct {
42 42
 	root             string
43 43
 	initPath         string
44 44
 	activeContainers map[string]*activeContainer
45
+	machineMemory    int64
45 46
 	sync.Mutex
46 47
 }
47 48
 
48
-func NewDriver(root, initPath string) (*driver, error) {
49
+func NewDriver(root, initPath string, machineMemory int64) (*driver, error) {
49 50
 	if err := os.MkdirAll(root, 0700); err != nil {
50 51
 		return nil, err
51 52
 	}
52
-
53 53
 	// native driver root is at docker_root/execdriver/native. Put apparmor at docker_root
54 54
 	if err := apparmor.InstallDefaultProfile(); err != nil {
55 55
 		return nil, err
56 56
 	}
57
-
58 57
 	return &driver{
59 58
 		root:             root,
60 59
 		initPath:         initPath,
61 60
 		activeContainers: make(map[string]*activeContainer),
61
+		machineMemory:    machineMemory,
62 62
 	}, nil
63 63
 }
64 64
 
... ...
@@ -281,6 +281,7 @@ func (d *driver) Clean(id string) error {
281 281
 }
282 282
 
283 283
 func (d *driver) Stats(id string) (*execdriver.ResourceStats, error) {
284
+	c := d.activeContainers[id]
284 285
 	state, err := libcontainer.GetState(filepath.Join(d.root, id))
285 286
 	if err != nil {
286 287
 		return nil, err
... ...
@@ -290,10 +291,15 @@ func (d *driver) Stats(id string) (*execdriver.ResourceStats, error) {
290 290
 	if err != nil {
291 291
 		return nil, err
292 292
 	}
293
+	memoryLimit := c.container.Cgroups.Memory
294
+	if memoryLimit == 0 {
295
+		memoryLimit = d.machineMemory
296
+	}
293 297
 	return &execdriver.ResourceStats{
294 298
 		ContainerStats: stats,
295 299
 		ClockTicks:     system.GetClockTicks(),
296 300
 		Read:           now,
301
+		MemoryLimit:    memoryLimit,
297 302
 	}, nil
298 303
 }
299 304
 
... ...
@@ -8,6 +8,7 @@ import (
8 8
 
9 9
 	"github.com/docker/docker/engine"
10 10
 	"github.com/docker/docker/runconfig"
11
+	"github.com/docker/docker/stats"
11 12
 )
12 13
 
13 14
 func (daemon *Daemon) ContainerStart(job *engine.Job) engine.Status {
... ...
@@ -80,15 +81,24 @@ func (daemon *Daemon) setHostConfig(container *Container, hostConfig *runconfig.
80 80
 }
81 81
 
82 82
 func (daemon *Daemon) ContainerStats(job *engine.Job) engine.Status {
83
-	stats, err := daemon.SubscribeToContainerStats(job.Args[0])
83
+	s, err := daemon.SubscribeToContainerStats(job.Args[0])
84 84
 	if err != nil {
85 85
 		return job.Error(err)
86 86
 	}
87 87
 	enc := json.NewEncoder(job.Stdout)
88
-	for update := range stats {
89
-		if err := enc.Encode(update); err != nil {
88
+	for update := range s {
89
+		ss := stats.ToStats(update.ContainerStats)
90
+		ss.MemoryStats.Limit = uint64(update.MemoryLimit)
91
+		ss.Read = update.Read
92
+		ss.ClockTicks = update.ClockTicks
93
+		ss.CpuStats.SystemUsage = update.SystemUsage
94
+		if err := enc.Encode(ss); err != nil {
90 95
 			return job.Error(err)
91 96
 		}
92 97
 	}
93 98
 	return engine.StatusOK
94 99
 }
100
+
101
+func mapToAPIStats() {
102
+
103
+}
... ...
@@ -1,6 +1,11 @@
1 1
 package daemon
2 2
 
3 3
 import (
4
+	"bufio"
5
+	"fmt"
6
+	"os"
7
+	"strconv"
8
+	"strings"
4 9
 	"sync"
5 10
 	"time"
6 11
 
... ...
@@ -55,12 +60,18 @@ func (s *statsCollector) start() {
55 55
 			log.Debugf("starting collection of container stats")
56 56
 			s.m.Lock()
57 57
 			for id, d := range s.containers {
58
+				systemUsage, err := getSystemCpuUsage()
59
+				if err != nil {
60
+					log.Errorf("collecting system cpu usage for %s: %v", id, err)
61
+					continue
62
+				}
58 63
 				stats, err := d.c.Stats()
59 64
 				if err != nil {
60 65
 					// TODO: @crosbymichael evict container depending on error
61 66
 					log.Errorf("collecting stats for %s: %v", id, err)
62 67
 					continue
63 68
 				}
69
+				stats.SystemUsage = systemUsage
64 70
 				for _, sub := range s.containers[id].subs {
65 71
 					sub <- stats
66 72
 				}
... ...
@@ -69,3 +80,36 @@ func (s *statsCollector) start() {
69 69
 		}
70 70
 	}()
71 71
 }
72
+
73
+// returns value in nanoseconds
74
+func getSystemCpuUsage() (uint64, error) {
75
+	f, err := os.Open("/proc/stat")
76
+	if err != nil {
77
+		return 0, err
78
+	}
79
+	defer f.Close()
80
+
81
+	sc := bufio.NewScanner(f)
82
+	for sc.Scan() {
83
+		parts := strings.Fields(sc.Text())
84
+		switch parts[0] {
85
+		case "cpu":
86
+			if len(parts) < 8 {
87
+				return 0, fmt.Errorf("invalid number of cpu fields")
88
+			}
89
+
90
+			var total uint64
91
+			for _, i := range parts[1:8] {
92
+				v, err := strconv.ParseUint(i, 10, 64)
93
+				if err != nil {
94
+					return 0.0, fmt.Errorf("Unable to convert value %s to int: %s", i, err)
95
+				}
96
+				total += v
97
+			}
98
+			return total * 1000000000, nil
99
+		default:
100
+			continue
101
+		}
102
+	}
103
+	return 0, fmt.Errorf("invalid stat format")
104
+}
72 105
new file mode 100644
... ...
@@ -0,0 +1,156 @@
0
+package stats
1
+
2
+import (
3
+	"time"
4
+
5
+	"github.com/docker/libcontainer"
6
+	"github.com/docker/libcontainer/cgroups"
7
+)
8
+
9
+type ThrottlingData struct {
10
+	// Number of periods with throttling active
11
+	Periods uint64 `json:"periods,omitempty"`
12
+	// Number of periods when the container hit its throttling limit.
13
+	ThrottledPeriods uint64 `json:"throttled_periods,omitempty"`
14
+	// Aggregate time the container was throttled for in nanoseconds.
15
+	ThrottledTime uint64 `json:"throttled_time,omitempty"`
16
+}
17
+
18
+// All CPU stats are aggregate since container inception.
19
+type CpuUsage struct {
20
+	// Total CPU time consumed.
21
+	// Units: nanoseconds.
22
+	TotalUsage uint64 `json:"total_usage,omitempty"`
23
+	// Total CPU time consumed per core.
24
+	// Units: nanoseconds.
25
+	PercpuUsage []uint64 `json:"percpu_usage,omitempty"`
26
+	// Time spent by tasks of the cgroup in kernel mode.
27
+	// Units: nanoseconds.
28
+	UsageInKernelmode uint64 `json:"usage_in_kernelmode"`
29
+	// Time spent by tasks of the cgroup in user mode.
30
+	// Units: nanoseconds.
31
+	UsageInUsermode uint64 `json:"usage_in_usermode"`
32
+}
33
+
34
+type CpuStats struct {
35
+	CpuUsage       CpuUsage       `json:"cpu_usage,omitempty"`
36
+	SystemUsage    uint64         `json:"system_cpu_usage"`
37
+	ThrottlingData ThrottlingData `json:"throttling_data,omitempty"`
38
+}
39
+
40
+type MemoryStats struct {
41
+	// current res_counter usage for memory
42
+	Usage uint64 `json:"usage,omitempty"`
43
+	// maximum usage ever recorded.
44
+	MaxUsage uint64 `json:"max_usage,omitempty"`
45
+	// TODO(vishh): Export these as stronger types.
46
+	// all the stats exported via memory.stat.
47
+	Stats map[string]uint64 `json:"stats,omitempty"`
48
+	// number of times memory usage hits limits.
49
+	Failcnt uint64 `json:"failcnt"`
50
+	Limit   uint64 `json:"limit"`
51
+}
52
+
53
+type BlkioStatEntry struct {
54
+	Major uint64 `json:"major,omitempty"`
55
+	Minor uint64 `json:"minor,omitempty"`
56
+	Op    string `json:"op,omitempty"`
57
+	Value uint64 `json:"value,omitempty"`
58
+}
59
+
60
+type BlkioStats struct {
61
+	// number of bytes tranferred to and from the block device
62
+	IoServiceBytesRecursive []BlkioStatEntry `json:"io_service_bytes_recursive,omitempty"`
63
+	IoServicedRecursive     []BlkioStatEntry `json:"io_serviced_recursive,omitempty"`
64
+	IoQueuedRecursive       []BlkioStatEntry `json:"io_queue_recursive,omitempty"`
65
+	IoServiceTimeRecursive  []BlkioStatEntry `json:"io_service_time_recursive,omitempty"`
66
+	IoWaitTimeRecursive     []BlkioStatEntry `json:"io_wait_time_recursive,omitempty"`
67
+	IoMergedRecursive       []BlkioStatEntry `json:"io_merged_recursive,omitempty"`
68
+	IoTimeRecursive         []BlkioStatEntry `json:"io_time_recursive,omitempty"`
69
+	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive,omitempty"`
70
+}
71
+
72
+type Network struct {
73
+	RxBytes   uint64 `json:"rx_bytes"`
74
+	RxPackets uint64 `json:"rx_packets"`
75
+	RxErrors  uint64 `json:"rx_errors"`
76
+	RxDropped uint64 `json:"rx_dropped"`
77
+	TxBytes   uint64 `json:"tx_bytes"`
78
+	TxPackets uint64 `json:"tx_packets"`
79
+	TxErrors  uint64 `json:"tx_errors"`
80
+	TxDropped uint64 `json:"tx_dropped"`
81
+}
82
+
83
+type Stats struct {
84
+	Read        time.Time   `json:"read"`
85
+	ClockTicks  int         `json:"clock_ticks"`
86
+	Interval    int         `json:"interval"` // in ms
87
+	Network     Network     `json:"network,omitempty"`
88
+	CpuStats    CpuStats    `json:"cpu_stats,omitempty"`
89
+	MemoryStats MemoryStats `json:"memory_stats,omitempty"`
90
+	BlkioStats  BlkioStats  `json:"blkio_stats,omitempty"`
91
+}
92
+
93
+func ToStats(ls *libcontainer.ContainerStats) *Stats {
94
+	s := &Stats{}
95
+	if ls.NetworkStats != nil {
96
+		s.Network = Network{
97
+			RxBytes:   ls.NetworkStats.RxBytes,
98
+			RxPackets: ls.NetworkStats.RxPackets,
99
+			RxErrors:  ls.NetworkStats.RxErrors,
100
+			RxDropped: ls.NetworkStats.RxDropped,
101
+			TxBytes:   ls.NetworkStats.TxBytes,
102
+			TxPackets: ls.NetworkStats.TxPackets,
103
+			TxErrors:  ls.NetworkStats.TxErrors,
104
+			TxDropped: ls.NetworkStats.TxDropped,
105
+		}
106
+	}
107
+	cs := ls.CgroupStats
108
+	if cs != nil {
109
+		s.BlkioStats = BlkioStats{
110
+			IoServiceBytesRecursive: copyBlkioEntry(cs.BlkioStats.IoServiceBytesRecursive),
111
+			IoServicedRecursive:     copyBlkioEntry(cs.BlkioStats.IoServicedRecursive),
112
+			IoQueuedRecursive:       copyBlkioEntry(cs.BlkioStats.IoQueuedRecursive),
113
+			IoServiceTimeRecursive:  copyBlkioEntry(cs.BlkioStats.IoServiceTimeRecursive),
114
+			IoWaitTimeRecursive:     copyBlkioEntry(cs.BlkioStats.IoWaitTimeRecursive),
115
+			IoMergedRecursive:       copyBlkioEntry(cs.BlkioStats.IoMergedRecursive),
116
+			IoTimeRecursive:         copyBlkioEntry(cs.BlkioStats.IoTimeRecursive),
117
+			SectorsRecursive:        copyBlkioEntry(cs.BlkioStats.SectorsRecursive),
118
+		}
119
+		cpu := cs.CpuStats
120
+		s.CpuStats = CpuStats{
121
+			CpuUsage: CpuUsage{
122
+				TotalUsage:        cpu.CpuUsage.TotalUsage,
123
+				PercpuUsage:       cpu.CpuUsage.PercpuUsage,
124
+				UsageInKernelmode: cpu.CpuUsage.UsageInKernelmode,
125
+				UsageInUsermode:   cpu.CpuUsage.UsageInUsermode,
126
+			},
127
+			ThrottlingData: ThrottlingData{
128
+				Periods:          cpu.ThrottlingData.Periods,
129
+				ThrottledPeriods: cpu.ThrottlingData.ThrottledPeriods,
130
+				ThrottledTime:    cpu.ThrottlingData.ThrottledTime,
131
+			},
132
+		}
133
+		mem := cs.MemoryStats
134
+		s.MemoryStats = MemoryStats{
135
+			Usage:    mem.Usage,
136
+			MaxUsage: mem.MaxUsage,
137
+			Stats:    mem.Stats,
138
+			Failcnt:  mem.Failcnt,
139
+		}
140
+	}
141
+	return s
142
+}
143
+
144
+func copyBlkioEntry(entries []cgroups.BlkioStatEntry) []BlkioStatEntry {
145
+	out := make([]BlkioStatEntry, len(entries))
146
+	for i, re := range entries {
147
+		out[i] = BlkioStatEntry{
148
+			Major: re.Major,
149
+			Minor: re.Minor,
150
+			Op:    re.Op,
151
+			Value: re.Value,
152
+		}
153
+	}
154
+	return out
155
+}