Browse code

daemon/logger: Add logging driver for Google Cloud Logging

Signed-off-by: Mike Danese <mikedanese@google.com>

Mike Danese authored on 2015/12/19 02:43:32
Showing 9 changed files
... ...
@@ -397,6 +397,7 @@ __docker_complete_log_drivers() {
397 397
 		awslogs
398 398
 		etwlogs
399 399
 		fluentd
400
+		gcplogs
400 401
 		gelf
401 402
 		journald
402 403
 		json-file
... ...
@@ -410,13 +411,14 @@ __docker_complete_log_options() {
410 410
 	# see docs/reference/logging/index.md
411 411
 	local awslogs_options="awslogs-region awslogs-group awslogs-stream"
412 412
 	local fluentd_options="env fluentd-address labels tag"
413
+	local gcplogs_options="env gcp-log-cmd gcp-project labels"
413 414
 	local gelf_options="env gelf-address labels tag"
414 415
 	local journald_options="env labels tag"
415 416
 	local json_file_options="env labels max-file max-size"
416 417
 	local syslog_options="syslog-address syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify syslog-facility tag"
417 418
 	local splunk_options="env labels splunk-caname splunk-capath splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url tag"
418 419
 
419
-	local all_options="$fluentd_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
420
+	local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
420 421
 
421 422
 	case $(__docker_value_of_option --log-driver) in
422 423
 		'')
... ...
@@ -428,6 +430,9 @@ __docker_complete_log_options() {
428 428
 		fluentd)
429 429
 			COMPREPLY=( $( compgen -W "$fluentd_options" -S = -- "$cur" ) )
430 430
 			;;
431
+		gcplogs)
432
+			COMPREPLY=( $( compgen -W "$gcplogs_options" -S = -- "$cur" ) )
433
+			;;
431 434
 		gelf)
432 435
 			COMPREPLY=( $( compgen -W "$gelf_options" -S = -- "$cur" ) )
433 436
 			;;
... ...
@@ -201,6 +201,7 @@ __docker_get_log_options() {
201 201
 
202 202
     awslogs_options=("awslogs-region" "awslogs-group" "awslogs-stream")
203 203
     fluentd_options=("env" "fluentd-address" "labels" "tag")
204
+    gcplogs_options=("env" "gcp-log-cmd" "gcp-project" "labels")
204 205
     gelf_options=("env" "gelf-address" "labels" "tag")
205 206
     journald_options=("env" "labels")
206 207
     json_file_options=("env" "labels" "max-file" "max-size")
... ...
@@ -209,6 +210,7 @@ __docker_get_log_options() {
209 209
 
210 210
     [[ $log_driver = (awslogs|all) ]] && _describe -t awslogs-options "awslogs options" awslogs_options "$@" && ret=0
211 211
     [[ $log_driver = (fluentd|all) ]] && _describe -t fluentd-options "fluentd options" fluentd_options "$@" && ret=0
212
+    [[ $log_driver = (gcplogs|all) ]] && _describe -t gcplogs-options "gcplogs options" gcplogs_options "$@" && ret=0
212 213
     [[ $log_driver = (gelf|all) ]] && _describe -t gelf-options "gelf options" gelf_options "$@" && ret=0
213 214
     [[ $log_driver = (journald|all) ]] && _describe -t journald-options "journald options" journald_options "$@" && ret=0
214 215
     [[ $log_driver = (json-file|all) ]] && _describe -t json-file-options "json-file options" json_file_options "$@" && ret=0
... ...
@@ -5,6 +5,7 @@ import (
5 5
 	// therefore they register themselves to the logdriver factory.
6 6
 	_ "github.com/docker/docker/daemon/logger/awslogs"
7 7
 	_ "github.com/docker/docker/daemon/logger/fluentd"
8
+	_ "github.com/docker/docker/daemon/logger/gcplogs"
8 9
 	_ "github.com/docker/docker/daemon/logger/gelf"
9 10
 	_ "github.com/docker/docker/daemon/logger/journald"
10 11
 	_ "github.com/docker/docker/daemon/logger/jsonfilelog"
11 12
new file mode 100644
... ...
@@ -0,0 +1,181 @@
0
+package gcplogs
1
+
2
+import (
3
+	"fmt"
4
+	"sync/atomic"
5
+	"time"
6
+
7
+	"github.com/docker/docker/daemon/logger"
8
+
9
+	"github.com/Sirupsen/logrus"
10
+	"golang.org/x/net/context"
11
+	"google.golang.org/cloud/compute/metadata"
12
+	"google.golang.org/cloud/logging"
13
+)
14
+
15
+const (
16
+	name = "gcplogs"
17
+
18
+	projectOptKey = "gcp-project"
19
+	logLabelsKey  = "labels"
20
+	logEnvKey     = "env"
21
+	logCmdKey     = "gcp-log-cmd"
22
+)
23
+
24
+var (
25
+	// The number of logs the gcplogs driver has dropped.
26
+	droppedLogs uint64
27
+
28
+	onGCE = metadata.OnGCE()
29
+
30
+	// instance metadata populated from the metadata server if available
31
+	projectID    string
32
+	zone         string
33
+	instanceName string
34
+	instanceID   string
35
+)
36
+
37
+func init() {
38
+	if onGCE {
39
+		// These will fail on instances if the metadata service is
40
+		// down or the client is compiled with an API version that
41
+		// has been removed. Since these are not vital, let's ignore
42
+		// them and make their fields in the dockeLogEntry ,omitempty
43
+		projectID, _ = metadata.ProjectID()
44
+		zone, _ = metadata.Zone()
45
+		instanceName, _ = metadata.InstanceName()
46
+		instanceID, _ = metadata.InstanceID()
47
+	}
48
+
49
+	if err := logger.RegisterLogDriver(name, New); err != nil {
50
+		logrus.Fatal(err)
51
+	}
52
+
53
+	if err := logger.RegisterLogOptValidator(name, ValidateLogOpts); err != nil {
54
+		logrus.Fatal(err)
55
+	}
56
+}
57
+
58
+type gcplogs struct {
59
+	client    *logging.Client
60
+	instance  *instanceInfo
61
+	container *containerInfo
62
+}
63
+
64
+type dockerLogEntry struct {
65
+	Instance  *instanceInfo  `json:"instance,omitempty"`
66
+	Container *containerInfo `json:"container,omitempty"`
67
+	Data      string         `json:"data,omitempty"`
68
+}
69
+
70
+type instanceInfo struct {
71
+	Zone string `json:"zone,omitempty"`
72
+	Name string `json:"name,omitempty"`
73
+	ID   string `json:"id,omitempty"`
74
+}
75
+
76
+type containerInfo struct {
77
+	Name      string            `json:"name,omitempty"`
78
+	ID        string            `json:"id,omitempty"`
79
+	ImageName string            `json:"imageName,omitempty"`
80
+	ImageID   string            `json:"imageId,omitempty"`
81
+	Created   time.Time         `json:"created,omitempty"`
82
+	Command   string            `json:"command,omitempty"`
83
+	Metadata  map[string]string `json:"metadata,omitempty"`
84
+}
85
+
86
+// New creates a new logger that logs to Google Cloud Logging using the application
87
+// default credentials.
88
+//
89
+// See https://developers.google.com/identity/protocols/application-default-credentials
90
+func New(ctx logger.Context) (logger.Logger, error) {
91
+
92
+	var project string
93
+	if projectID != "" {
94
+		project = projectID
95
+	}
96
+	if projectID, found := ctx.Config[projectOptKey]; found {
97
+		project = projectID
98
+	}
99
+	if project == "" {
100
+		return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project")
101
+	}
102
+
103
+	c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver")
104
+	if err != nil {
105
+		return nil, err
106
+	}
107
+
108
+	if err := c.Ping(); err != nil {
109
+		return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err)
110
+	}
111
+
112
+	l := &gcplogs{
113
+		client: c,
114
+		container: &containerInfo{
115
+			Name:      ctx.ContainerName,
116
+			ID:        ctx.ContainerID,
117
+			ImageName: ctx.ContainerImageName,
118
+			ImageID:   ctx.ContainerImageID,
119
+			Created:   ctx.ContainerCreated,
120
+			Metadata:  ctx.ExtraAttributes(nil),
121
+		},
122
+	}
123
+
124
+	if ctx.Config[logCmdKey] == "true" {
125
+		l.container.Command = ctx.Command()
126
+	}
127
+
128
+	if onGCE {
129
+		l.instance = &instanceInfo{
130
+			Zone: zone,
131
+			Name: instanceName,
132
+			ID:   instanceID,
133
+		}
134
+	}
135
+
136
+	// The logger "overflows" at a rate of 10,000 logs per second and this
137
+	// overflow func is called. We want to surface the error to the user
138
+	// without overly spamming /var/log/docker.log so we log the first time
139
+	// we overflow and every 1000th time after.
140
+	c.Overflow = func(_ *logging.Client, _ logging.Entry) error {
141
+		if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 {
142
+			logrus.Errorf("gcplogs driver has dropped %v logs", i)
143
+		}
144
+		return nil
145
+	}
146
+
147
+	return l, nil
148
+}
149
+
150
+// ValidateLogOpts validates the opts passed to the gcplogs driver. Currently, the gcplogs
151
+// driver doesn't take any arguments.
152
+func ValidateLogOpts(cfg map[string]string) error {
153
+	for k := range cfg {
154
+		switch k {
155
+		case projectOptKey, logLabelsKey, logEnvKey, logCmdKey:
156
+		default:
157
+			return fmt.Errorf("%q is not a valid option for the gcplogs driver", k)
158
+		}
159
+	}
160
+	return nil
161
+}
162
+
163
+func (l *gcplogs) Log(m *logger.Message) error {
164
+	return l.client.Log(logging.Entry{
165
+		Time: m.Timestamp,
166
+		Payload: &dockerLogEntry{
167
+			Instance:  l.instance,
168
+			Container: l.container,
169
+			Data:      string(m.Line),
170
+		},
171
+	})
172
+}
173
+
174
+func (l *gcplogs) Close() error {
175
+	return l.client.Flush()
176
+}
177
+
178
+func (l *gcplogs) Name() string {
179
+	return name
180
+}
0 181
new file mode 100644
... ...
@@ -0,0 +1,70 @@
0
+<!--[metadata]>
1
+title = "Google Cloud Logging driver"
2
+description = "Describes how to use the Google Cloud Logging driver."
3
+keywords = ["gcplogs, google, docker, logging, driver"]
4
+[menu.main]
5
+parent = "smn_logging"
6
+weight = 2
7
+<![end-metadata]-->
8
+
9
+# Google Cloud Logging driver
10
+
11
+The Google Cloud Logging driver sends container logs to <a href="https://cloud.google.com/logging/docs/" target="_blank">Google Cloud
12
+Logging</a>.
13
+
14
+## Usage
15
+
16
+You can configure the default logging driver by passing the `--log-driver`
17
+option to the Docker daemon:
18
+
19
+    docker daemon --log-driver=gcplogs
20
+
21
+You can set the logging driver for a specific container by using the
22
+`--log-driver` option to `docker run`:
23
+
24
+    docker run --log-driver=gcplogs ...
25
+
26
+This log driver does not implement a reader so it is incompatible with
27
+`docker logs`.
28
+
29
+If Docker detects that it is running in a Google Cloud Project, it will discover configuration
30
+from the <a href="https://cloud.google.com/compute/docs/metadata" target="_blank">instance metadata service</a>.
31
+Otherwise, the user must specify which project to log to using the `--gcp-project`
32
+log option and Docker will attempt to obtain credentials from the
33
+<a href="https://developers.google.com/identity/protocols/application-default-credentials" target="_blank">Google Application Default Credential</a>.
34
+The `--gcp-project` takes precedence over information discovered from the metadata server
35
+so a Docker daemon running in a Google Cloud Project can be overriden to log to a different
36
+Google Cloud Project using `--gcp-project`.
37
+
38
+## gcplogs options
39
+
40
+You can use the `--log-opt NAME=VALUE` flag to specify these additional Google
41
+Cloud Logging driver options:
42
+
43
+| Option                      | Required | Description                                                                                                                                 |
44
+|-----------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------|
45
+| `gcp-project`               | optional | Which GCP project to log to. Defaults to discovering this value from the GCE metadata service.                                              |
46
+| `gcp-log-cmd`               | optional | Whether to log the command that the container was started with. Defaults to false.                                                          |
47
+| `labels`                    | optional | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container.                   |
48
+| `env`                       | optional | Comma-separated list of keys of environment variables, which should be included in message, if these variables are specified for container. |
49
+
50
+If there is collision between `label` and `env` keys, the value of the `env`
51
+takes precedence. Both options add additional fields to the attributes of a
52
+logging message.
53
+
54
+Below is an example of the logging options required to log to the default
55
+logging destination which is discovered by querying the GCE metadata server.
56
+
57
+    docker run --log-driver=gcplogs \
58
+        --log-opt labels=location
59
+        --log-opt env=TEST
60
+        --log-opt gcp-log-cmd=true
61
+        --env "TEST=false"
62
+        --label location=west
63
+        your/application
64
+
65
+This configuration also directs the driver to include in the payload the label
66
+`location`, the environment variable `ENV`, and the command used to start the
67
+container.
... ...
@@ -27,6 +27,7 @@ container's logging driver. The following options are supported:
27 27
 | `awslogs`   | Amazon CloudWatch Logs logging driver for Docker. Writes log messages to Amazon CloudWatch Logs.                              |
28 28
 | `splunk`    | Splunk logging driver for Docker. Writes log messages to `splunk` using HTTP Event Collector.                                 |
29 29
 | `etwlogs`   | ETW logging driver for Docker on Windows. Writes log messages as ETW events.                                                  |
30
+| `gcplogs`   | Google Cloud Logging driver for Docker. Writes log messages to Google Cloud Logging.                                          |
30 31
 
31 32
 The `docker logs`command is available only for the `json-file` and `journald`
32 33
 logging drivers.
... ...
@@ -213,4 +214,14 @@ as an ETW event. An ETW listener can then be created to listen for these events.
213 213
 
214 214
 For detailed information on working with this logging driver, see [the ETW logging driver](etwlogs.md) reference documentation.
215 215
 
216
+## Google Cloud Logging
216 217
 
218
+The Google Cloud Logging driver supports the following options:
219
+
220
+    --log-opt gcp-project=<gcp_projext>
221
+    --log-opt labels=<label1>,<label2>
222
+    --log-opt env=<envvar1>,<envvar2>
223
+    --log-opt log-cmd=true
224
+
225
+For detailed information about working with this logging driver, see the [Google Cloud Logging driver](gcplogs.md).
226
+reference documentation.
... ...
@@ -214,7 +214,7 @@ millions of trillions.
214 214
    Add link to another container in the form of <name or id>:alias or just
215 215
    <name or id> in which case the alias will match the name.
216 216
 
217
-**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
217
+**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
218 218
   Logging driver for container. Default is defined by daemon `--log-driver` flag.
219 219
   **Warning**: the `docker logs` command works only for the `json-file` and
220 220
   `journald` logging drivers.
... ...
@@ -185,7 +185,7 @@ unix://[/path/to/socket] to use.
185 185
 **--label**="[]"
186 186
   Set key=value labels to the daemon (displayed in `docker info`)
187 187
 
188
-**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
188
+**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
189 189
   Default driver for container logs. Default is `json-file`.
190 190
   **Warning**: `docker logs` command works only for `json-file` logging driver.
191 191
 
... ...
@@ -320,7 +320,7 @@ container can access the exposed port via a private networking interface. Docker
320 320
 will set some environment variables in the client container to help indicate
321 321
 which interface and port to use.
322 322
 
323
-**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
323
+**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
324 324
   Logging driver for container. Default is defined by daemon `--log-driver` flag.
325 325
   **Warning**: the `docker logs` command works only for the `json-file` and
326 326
   `journald` logging drivers.