Browse code

Use a regex to match environment variables #27565

Signed-off-by: Joseph Rothrock <rothrock@rothrock.org>

Joseph Rothrock authored on 2016/11/09 09:34:47
Showing 11 changed files
... ...
@@ -78,7 +78,10 @@ func New(info logger.Info) (logger.Logger, error) {
78 78
 		return nil, err
79 79
 	}
80 80
 
81
-	extra := info.ExtraAttributes(nil)
81
+	extra, err := info.ExtraAttributes(nil)
82
+	if err != nil {
83
+		return nil, err
84
+	}
82 85
 
83 86
 	bufferLimit := defaultBufferLimit
84 87
 	if info.Config[bufferLimitKey] != "" {
... ...
@@ -169,6 +172,7 @@ func ValidateLogOpt(cfg map[string]string) error {
169 169
 	for key := range cfg {
170 170
 		switch key {
171 171
 		case "env":
172
+		case "env-regex":
172 173
 		case "labels":
173 174
 		case "tag":
174 175
 		case addressKey:
... ...
@@ -17,13 +17,14 @@ import (
17 17
 const (
18 18
 	name = "gcplogs"
19 19
 
20
-	projectOptKey = "gcp-project"
21
-	logLabelsKey  = "labels"
22
-	logEnvKey     = "env"
23
-	logCmdKey     = "gcp-log-cmd"
24
-	logZoneKey    = "gcp-meta-zone"
25
-	logNameKey    = "gcp-meta-name"
26
-	logIDKey      = "gcp-meta-id"
20
+	projectOptKey  = "gcp-project"
21
+	logLabelsKey   = "labels"
22
+	logEnvKey      = "env"
23
+	logEnvRegexKey = "env-regex"
24
+	logCmdKey      = "gcp-log-cmd"
25
+	logZoneKey     = "gcp-meta-zone"
26
+	logNameKey     = "gcp-meta-name"
27
+	logIDKey       = "gcp-meta-id"
27 28
 )
28 29
 
29 30
 var (
... ...
@@ -133,6 +134,11 @@ func New(info logger.Info) (logger.Logger, error) {
133 133
 		return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err)
134 134
 	}
135 135
 
136
+	extraAttributes, err := info.ExtraAttributes(nil)
137
+	if err != nil {
138
+		return nil, err
139
+	}
140
+
136 141
 	l := &gcplogs{
137 142
 		logger: lg,
138 143
 		container: &containerInfo{
... ...
@@ -141,7 +147,7 @@ func New(info logger.Info) (logger.Logger, error) {
141 141
 			ImageName: info.ContainerImageName,
142 142
 			ImageID:   info.ContainerImageID,
143 143
 			Created:   info.ContainerCreated,
144
-			Metadata:  info.ExtraAttributes(nil),
144
+			Metadata:  extraAttributes,
145 145
 		},
146 146
 	}
147 147
 
... ...
@@ -185,7 +191,7 @@ func New(info logger.Info) (logger.Logger, error) {
185 185
 func ValidateLogOpts(cfg map[string]string) error {
186 186
 	for k := range cfg {
187 187
 		switch k {
188
-		case projectOptKey, logLabelsKey, logEnvKey, logCmdKey, logZoneKey, logNameKey, logIDKey:
188
+		case projectOptKey, logLabelsKey, logEnvKey, logEnvRegexKey, logCmdKey, logZoneKey, logNameKey, logIDKey:
189 189
 		default:
190 190
 			return fmt.Errorf("%q is not a valid option for the gcplogs driver", k)
191 191
 		}
... ...
@@ -69,12 +69,17 @@ func New(info logger.Info) (logger.Logger, error) {
69 69
 		"_created":        info.ContainerCreated,
70 70
 	}
71 71
 
72
-	extraAttrs := info.ExtraAttributes(func(key string) string {
72
+	extraAttrs, err := info.ExtraAttributes(func(key string) string {
73 73
 		if key[0] == '_' {
74 74
 			return key
75 75
 		}
76 76
 		return "_" + key
77 77
 	})
78
+
79
+	if err != nil {
80
+		return nil, err
81
+	}
82
+
78 83
 	for k, v := range extraAttrs {
79 84
 		extra[k] = v
80 85
 	}
... ...
@@ -156,6 +161,7 @@ func ValidateLogOpt(cfg map[string]string) error {
156 156
 		case "tag":
157 157
 		case "labels":
158 158
 		case "env":
159
+		case "env-regex":
159 160
 		case "gelf-compression-level":
160 161
 			i, err := strconv.Atoi(val)
161 162
 			if err != nil || i < flate.DefaultCompression || i > flate.BestCompression {
... ...
@@ -75,7 +75,10 @@ func New(info logger.Info) (logger.Logger, error) {
75 75
 		"CONTAINER_NAME":    info.Name(),
76 76
 		"CONTAINER_TAG":     tag,
77 77
 	}
78
-	extraAttrs := info.ExtraAttributes(sanitizeKeyMod)
78
+	extraAttrs, err := info.ExtraAttributes(sanitizeKeyMod)
79
+	if err != nil {
80
+		return nil, err
81
+	}
79 82
 	for k, v := range extraAttrs {
80 83
 		vars[k] = v
81 84
 	}
... ...
@@ -89,6 +92,7 @@ func validateLogOpt(cfg map[string]string) error {
89 89
 		switch key {
90 90
 		case "labels":
91 91
 		case "env":
92
+		case "env-regex":
92 93
 		case "tag":
93 94
 		default:
94 95
 			return fmt.Errorf("unknown log opt '%s' for journald log driver", key)
... ...
@@ -67,7 +67,11 @@ func New(info logger.Info) (logger.Logger, error) {
67 67
 	}
68 68
 
69 69
 	var extra []byte
70
-	if attrs := info.ExtraAttributes(nil); len(attrs) > 0 {
70
+	attrs, err := info.ExtraAttributes(nil)
71
+	if err != nil {
72
+		return nil, err
73
+	}
74
+	if len(attrs) > 0 {
71 75
 		var err error
72 76
 		extra, err = json.Marshal(attrs)
73 77
 		if err != nil {
... ...
@@ -121,6 +125,7 @@ func ValidateLogOpt(cfg map[string]string) error {
121 121
 		case "max-size":
122 122
 		case "labels":
123 123
 		case "env":
124
+		case "env-regex":
124 125
 		default:
125 126
 			return fmt.Errorf("unknown log opt '%s' for json-file log driver", key)
126 127
 		}
... ...
@@ -160,13 +160,13 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
160 160
 	}
161 161
 	defer os.RemoveAll(tmp)
162 162
 	filename := filepath.Join(tmp, "container.log")
163
-	config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl"}
163
+	config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl", "env-regex": "^dc"}
164 164
 	l, err := New(logger.Info{
165 165
 		ContainerID:     cid,
166 166
 		LogPath:         filename,
167 167
 		Config:          config,
168 168
 		ContainerLabels: map[string]string{"rack": "101", "dc": "lhr"},
169
-		ContainerEnv:    []string{"environ=production", "debug=false", "port=10001", "ssl=true"},
169
+		ContainerEnv:    []string{"environ=production", "debug=false", "port=10001", "ssl=true", "dc_region=west"},
170 170
 	})
171 171
 	if err != nil {
172 172
 		t.Fatal(err)
... ...
@@ -189,11 +189,12 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
189 189
 		t.Fatal(err)
190 190
 	}
191 191
 	expected := map[string]string{
192
-		"rack":    "101",
193
-		"dc":      "lhr",
194
-		"environ": "production",
195
-		"debug":   "false",
196
-		"ssl":     "true",
192
+		"rack":      "101",
193
+		"dc":        "lhr",
194
+		"environ":   "production",
195
+		"debug":     "false",
196
+		"ssl":       "true",
197
+		"dc_region": "west",
197 198
 	}
198 199
 	if !reflect.DeepEqual(extra, expected) {
199 200
 		t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected)
... ...
@@ -78,6 +78,7 @@ func ValidateLogOpt(cfg map[string]string) error {
78 78
 	for key := range cfg {
79 79
 		switch key {
80 80
 		case "env":
81
+		case "env-regex":
81 82
 		case "labels":
82 83
 		case "tag":
83 84
 		case key:
... ...
@@ -3,6 +3,7 @@ package logger
3 3
 import (
4 4
 	"fmt"
5 5
 	"os"
6
+	"regexp"
6 7
 	"strings"
7 8
 	"time"
8 9
 )
... ...
@@ -26,7 +27,7 @@ type Info struct {
26 26
 // ExtraAttributes returns the user-defined extra attributes (labels,
27 27
 // environment variables) in key-value format. This can be used by log drivers
28 28
 // that support metadata to add more context to a log.
29
-func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string {
29
+func (info *Info) ExtraAttributes(keyMod func(string) string) (map[string]string, error) {
30 30
 	extra := make(map[string]string)
31 31
 	labels, ok := info.Config["labels"]
32 32
 	if ok && len(labels) > 0 {
... ...
@@ -40,14 +41,15 @@ func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string
40 40
 		}
41 41
 	}
42 42
 
43
+	envMapping := make(map[string]string)
44
+	for _, e := range info.ContainerEnv {
45
+		if kv := strings.SplitN(e, "=", 2); len(kv) == 2 {
46
+			envMapping[kv[0]] = kv[1]
47
+		}
48
+	}
49
+
43 50
 	env, ok := info.Config["env"]
44 51
 	if ok && len(env) > 0 {
45
-		envMapping := make(map[string]string)
46
-		for _, e := range info.ContainerEnv {
47
-			if kv := strings.SplitN(e, "=", 2); len(kv) == 2 {
48
-				envMapping[kv[0]] = kv[1]
49
-			}
50
-		}
51 52
 		for _, l := range strings.Split(env, ",") {
52 53
 			if v, ok := envMapping[l]; ok {
53 54
 				if keyMod != nil {
... ...
@@ -58,7 +60,23 @@ func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string
58 58
 		}
59 59
 	}
60 60
 
61
-	return extra
61
+	envRegex, ok := info.Config["env-regex"]
62
+	if ok && len(envRegex) > 0 {
63
+		re, err := regexp.Compile(envRegex)
64
+		if err != nil {
65
+			return nil, err
66
+		}
67
+		for k, v := range envMapping {
68
+			if re.MatchString(k) {
69
+				if keyMod != nil {
70
+					k = keyMod(k)
71
+				}
72
+				extra[k] = v
73
+			}
74
+		}
75
+	}
76
+
77
+	return extra, nil
62 78
 }
63 79
 
64 80
 // Hostname returns the hostname from the underlying OS.
... ...
@@ -39,6 +39,7 @@ const (
39 39
 	splunkGzipCompressionKey      = "splunk-gzip"
40 40
 	splunkGzipCompressionLevelKey = "splunk-gzip-level"
41 41
 	envKey                        = "env"
42
+	envRegexKey                   = "env-regex"
42 43
 	labelsKey                     = "labels"
43 44
 	tagKey                        = "tag"
44 45
 )
... ...
@@ -235,7 +236,10 @@ func New(info logger.Info) (logger.Logger, error) {
235 235
 		}
236 236
 	}
237 237
 
238
-	attrs := info.ExtraAttributes(nil)
238
+	attrs, err := info.ExtraAttributes(nil)
239
+	if err != nil {
240
+		return nil, err
241
+	}
239 242
 
240 243
 	var (
241 244
 		postMessagesFrequency = getAdvancedOptionDuration(envVarPostMessagesFrequency, defaultPostMessagesFrequency)
... ...
@@ -538,6 +542,7 @@ func ValidateLogOpt(cfg map[string]string) error {
538 538
 		case splunkGzipCompressionKey:
539 539
 		case splunkGzipCompressionLevelKey:
540 540
 		case envKey:
541
+		case envRegexKey:
541 542
 		case labelsKey:
542 543
 		case tagKey:
543 544
 		default:
... ...
@@ -25,9 +25,10 @@ func TestValidateLogOpt(t *testing.T) {
25 25
 		splunkVerifyConnectionKey:     "true",
26 26
 		splunkGzipCompressionKey:      "true",
27 27
 		splunkGzipCompressionLevelKey: "1",
28
-		envKey:    "a",
29
-		labelsKey: "b",
30
-		tagKey:    "c",
28
+		envKey:      "a",
29
+		envRegexKey: "^foo",
30
+		labelsKey:   "b",
31
+		tagKey:      "c",
31 32
 	})
32 33
 	if err != nil {
33 34
 		t.Fatal(err)
... ...
@@ -215,8 +216,9 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) {
215 215
 			splunkIndexKey:           "myindex",
216 216
 			splunkFormatKey:          splunkFormatInline,
217 217
 			splunkGzipCompressionKey: "true",
218
-			tagKey:    "{{.ImageName}}/{{.Name}}",
219
-			labelsKey: "a",
218
+			tagKey:      "{{.ImageName}}/{{.Name}}",
219
+			labelsKey:   "a",
220
+			envRegexKey: "^foo",
220 221
 		},
221 222
 		ContainerID:        "containeriid",
222 223
 		ContainerName:      "/container_name",
... ...
@@ -225,6 +227,7 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) {
225 225
 		ContainerLabels: map[string]string{
226 226
 			"a": "b",
227 227
 		},
228
+		ContainerEnv: []string{"foo_finder=bar"},
228 229
 	}
229 230
 
230 231
 	hostname, err := info.Hostname()
... ...
@@ -295,6 +298,7 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) {
295 295
 			event["source"] != "stdout" ||
296 296
 			event["tag"] != "container_image_name/container_name" ||
297 297
 			event["attrs"].(map[string]interface{})["a"] != "b" ||
298
+			event["attrs"].(map[string]interface{})["foo_finder"] != "bar" ||
298 299
 			len(event) != 4 {
299 300
 			t.Fatalf("Unexpected event in message %v", event)
300 301
 		}
... ...
@@ -184,6 +184,7 @@ func ValidateLogOpt(cfg map[string]string) error {
184 184
 	for key := range cfg {
185 185
 		switch key {
186 186
 		case "env":
187
+		case "env-regex":
187 188
 		case "labels":
188 189
 		case "syslog-address":
189 190
 		case "syslog-facility":