Browse code

Generator for protobuf

Clayton Coleman authored on 2016/06/16 10:27:11
Showing 9 changed files
... ...
@@ -52,6 +52,7 @@ rsync -av \
52 52
 /test/fixtures/***
53 53
 /test/integration/***
54 54
 /third_party/golang/***
55
+/third_party/protobuf/***
55 56
 /README.md
56 57
 EOF
57 58
 
... ...
@@ -19,7 +19,7 @@ if [[ ! "$clientgen" ]]; then
19 19
     echo "It looks as if you don't have a compiled client-gen binary"
20 20
     echo
21 21
     echo "If you are running from a clone of the git repo, please run"
22
-    echo "'./hack/build-go.sh Godeps/_workspace/src/k8s.io/kubernetes/cmd/libs/go2idl/client-gen'."
22
+    echo "'./hack/build-go.sh vendor/k8s.io/kubernetes/cmd/libs/go2idl/client-gen'."
23 23
   } >&2
24 24
   exit 1
25 25
 fi
26 26
new file mode 100755
... ...
@@ -0,0 +1,35 @@
0
+#!/bin/bash
1
+
2
+set -o errexit
3
+set -o nounset
4
+set -o pipefail
5
+
6
+OS_ROOT=$(dirname "${BASH_SOURCE}")/..
7
+source "${OS_ROOT}/hack/lib/init.sh"
8
+
9
+# Go to the top of the tree.
10
+cd "${OS_ROOT}"
11
+
12
+if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
13
+  echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
14
+  echo "install the platform appropriate Protobuf package for your OS: "
15
+  echo
16
+  echo "  https://github.com/google/protobuf/releases"
17
+  echo
18
+  exit 1
19
+fi
20
+
21
+os::build::setup_env
22
+
23
+hack/build-go.sh tools/genprotobuf vendor/k8s.io/kubernetes/cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo
24
+genprotobuf="$( os::build::find-binary genprotobuf )"
25
+
26
+if [[ -z "${genprotobuf}" ]]; then
27
+	echo "It looks as if you don't have a compiled genprotobuf binary."
28
+	echo
29
+	echo "If you are running from a clone of the git repo, please run"
30
+	echo "'./hack/build-go.sh tools/genprotobuf'."
31
+	exit 1
32
+fi
33
+
34
+${genprotobuf} --output-base="${GOPATH}/src" "$@"
0 35
\ No newline at end of file
1 36
new file mode 100755
... ...
@@ -0,0 +1,38 @@
0
+#!/bin/bash
1
+
2
+set -o errexit
3
+set -o nounset
4
+set -o pipefail
5
+
6
+OS_ROOT=$(dirname "${BASH_SOURCE}")/..
7
+source "${OS_ROOT}/hack/lib/init.sh"
8
+
9
+APIROOTS=${APIROOTS:-pkg}
10
+_tmp="${OS_ROOT}/_output/diff"
11
+
12
+cleanup() {
13
+  echo rm -rf "${_tmp}"
14
+}
15
+
16
+trap "cleanup" EXIT SIGINT
17
+
18
+cleanup
19
+for APIROOT in ${APIROOTS}; do
20
+  mkdir -p "${_tmp}/${APIROOT%/*}"
21
+  cp -rf "${OS_ROOT}/${APIROOT}" "${_tmp}/"
22
+done
23
+
24
+"${OS_ROOT}/hack/update-generated-protobuf.sh"
25
+for APIROOT in ${APIROOTS}; do
26
+  TMP_APIROOT="${_tmp}/${APIROOT}"
27
+  echo "diffing ${APIROOT} against freshly generated protobuf"
28
+  ret=0
29
+  diff -Naupr -I 'Auto generated by' "${OS_ROOT}/${APIROOT}" "${TMP_APIROOT}" || ret=$?
30
+  # cp -rf "${TMP_APIROOT}" "${OS_ROOT}/${APIROOT%/*}"
31
+  if [[ $ret -eq 0 ]]; then
32
+    echo "${APIROOT} up to date."
33
+  else
34
+    echo "${APIROOT} is out of date. Please run hack/update-generated-protobuf.sh"
35
+    exit 1
36
+  fi
37
+done
... ...
@@ -50,6 +50,7 @@ DIR_BLACKLIST='./hack
50 50
 ./pkg/project
51 51
 ./pkg/quota
52 52
 ./pkg/router
53
+./pkg/sdn
53 54
 ./pkg/security
54 55
 ./pkg/serviceaccounts
55 56
 ./pkg/template
... ...
@@ -1,6 +1,8 @@
1 1
 package api_test
2 2
 
3 3
 import (
4
+	"bytes"
5
+	"encoding/hex"
4 6
 	"fmt"
5 7
 	"math/rand"
6 8
 	"reflect"
... ...
@@ -11,10 +13,12 @@ import (
11 11
 	"github.com/google/gofuzz"
12 12
 	kapi "k8s.io/kubernetes/pkg/api"
13 13
 	"k8s.io/kubernetes/pkg/api/meta"
14
+	"k8s.io/kubernetes/pkg/api/testapi"
14 15
 	apitesting "k8s.io/kubernetes/pkg/api/testing"
15 16
 	"k8s.io/kubernetes/pkg/api/unversioned"
16 17
 	"k8s.io/kubernetes/pkg/api/validation"
17 18
 	"k8s.io/kubernetes/pkg/runtime"
19
+	"k8s.io/kubernetes/pkg/runtime/serializer/protobuf"
18 20
 	"k8s.io/kubernetes/pkg/types"
19 21
 	"k8s.io/kubernetes/pkg/util/diff"
20 22
 	"k8s.io/kubernetes/pkg/util/intstr"
... ...
@@ -42,6 +46,16 @@ import (
42 42
 	_ "k8s.io/kubernetes/pkg/api/install"
43 43
 )
44 44
 
45
+var codecsToTest = []func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error){
46
+	func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error) {
47
+		return kapi.Codecs.LegacyCodec(version), nil
48
+	},
49
+	func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error) {
50
+		s := protobuf.NewSerializer(kapi.Scheme, kapi.Scheme, "application/arbitrary.content.type")
51
+		return kapi.Codecs.CodecForVersions(s, s, testapi.ExternalGroupVersions(), nil), nil
52
+	},
53
+}
54
+
45 55
 func fuzzInternalObject(t *testing.T, forVersion unversioned.GroupVersion, item runtime.Object, seed int64) runtime.Object {
46 56
 	f := apitesting.FuzzerFor(t, forVersion, rand.NewSource(seed))
47 57
 	f.Funcs(
... ...
@@ -439,6 +453,21 @@ func defaultHookContainerName(hook *deploy.LifecycleHook, containerName string)
439 439
 	}
440 440
 }
441 441
 
442
+func roundTripWithAllCodecs(t *testing.T, version unversioned.GroupVersion, item runtime.Object) {
443
+	var codecs []runtime.Codec
444
+	for _, fn := range codecsToTest {
445
+		codec, err := fn(version, item)
446
+		if err != nil {
447
+			t.Errorf("unable to get codec: %v", err)
448
+			return
449
+		}
450
+		codecs = append(codecs, codec)
451
+	}
452
+	for _, codec := range codecs {
453
+		roundTrip(t, codec, item)
454
+	}
455
+}
456
+
442 457
 func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
443 458
 	// Make a copy of the originalItem to give to conversion functions
444 459
 	// This lets us know if conversion messed with the input object
... ...
@@ -453,7 +482,8 @@ func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
453 453
 	data, err := runtime.Encode(codec, item)
454 454
 	if err != nil {
455 455
 		if runtime.IsNotRegisteredError(err) {
456
-			t.Logf("%v is not registered", name)
456
+			t.Logf("%v skipped: not registered: %v", name, err)
457
+			return
457 458
 		}
458 459
 		t.Errorf("%v: %v (%#v)", name, err, item)
459 460
 		return
... ...
@@ -474,7 +504,7 @@ func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
474 474
 	}
475 475
 
476 476
 	if !kapi.Semantic.DeepEqual(originalItem, obj2) {
477
-		t.Errorf("1: %v: diff: %v\nCodec: %v\nData: %s", name, diff.ObjectReflectDiff(originalItem, obj2), codec, string(data))
477
+		t.Errorf("1: %v: diff: %v\nCodec: %v\nData: %s", name, diff.ObjectReflectDiff(originalItem, obj2), codec, dataToString(data))
478 478
 		return
479 479
 	}
480 480
 
... ...
@@ -489,6 +519,13 @@ func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
489 489
 	}
490 490
 }
491 491
 
492
+func dataToString(s []byte) string {
493
+	if bytes.HasPrefix(s, []byte("k8s")) {
494
+		return "\n" + hex.Dump(s)
495
+	}
496
+	return string(s)
497
+}
498
+
492 499
 // skipStandardVersions is a map of Kind to a list of API versions to test with.
493 500
 var skipStandardVersions = map[string][]unversioned.GroupVersion{
494 501
 	// The API versions here are to test our object that serializes from/into
... ...
@@ -503,17 +540,20 @@ func TestSpecificKind(t *testing.T) {
503 503
 	kapi.Scheme.Log(t)
504 504
 	defer kapi.Scheme.Log(nil)
505 505
 
506
-	kind := "DeploymentConfig"
506
+	kind := "ClusterRole"
507 507
 	item, err := kapi.Scheme.New(osapi.SchemeGroupVersion.WithKind(kind))
508 508
 	if err != nil {
509
-		t.Errorf("Couldn't make a %v? %v", kind, err)
510
-		return
509
+		t.Fatalf("Couldn't make a %v? %v", kind, err)
510
+	}
511
+	codec, err := codecsToTest[1](v1.SchemeGroupVersion, nil)
512
+	if err != nil {
513
+		t.Fatal(err)
511 514
 	}
512
-	seed := int64(2703387474910584091) //rand.Int63()
515
+	seed := int64(2703387474910584091)
513 516
 	for i := 0; i < fuzzIters; i++ {
514 517
 		//t.Logf(`About to test %v with "v1"`, kind)
515 518
 		fuzzInternalObject(t, v1.SchemeGroupVersion, item, seed)
516
-		roundTrip(t, kapi.Codecs.LegacyCodec(v1.SchemeGroupVersion), item)
519
+		roundTrip(t, codec, item)
517 520
 	}
518 521
 }
519 522
 
... ...
@@ -558,7 +598,7 @@ func TestTypes(t *testing.T) {
558 558
 						continue
559 559
 					}
560 560
 					fuzzInternalObject(t, externalVersion, item, seed)
561
-					roundTrip(t, kapi.Codecs.LegacyCodec(externalVersion), item)
561
+					roundTripWithAllCodecs(t, externalVersion, item)
562 562
 				}
563 563
 			}
564 564
 		}
... ...
@@ -271,6 +271,7 @@ func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
271 271
 	if err != nil {
272 272
 		if runtime.IsNotRegisteredError(err) {
273 273
 			t.Logf("%v is not registered", name)
274
+			return
274 275
 		}
275 276
 		t.Errorf("%v: %v (%#v)", name, err, item)
276 277
 		return
277 278
new file mode 100644
... ...
@@ -0,0 +1,60 @@
0
+// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
1
+// existing IDL tags on the Go struct.
2
+package main
3
+
4
+import (
5
+	"path/filepath"
6
+	"strings"
7
+
8
+	"k8s.io/kubernetes/cmd/libs/go2idl/args"
9
+	"k8s.io/kubernetes/cmd/libs/go2idl/go-to-protobuf/protobuf"
10
+
11
+	flag "github.com/spf13/pflag"
12
+)
13
+
14
+var g = protobuf.New()
15
+
16
+func init() {
17
+	sourceTree := args.DefaultSourceTree()
18
+	g.Common.GoHeaderFilePath = filepath.Join("hack", "boilerplate.txt")
19
+	g.ProtoImport = []string{
20
+		filepath.Join("vendor"),
21
+		filepath.Join("vendor", "k8s.io", "kubernetes", "third_party", "protobuf"),
22
+	}
23
+	g.OutputBase = sourceTree
24
+	g.Packages = strings.Join([]string{
25
+		`-k8s.io/kubernetes/pkg/util/intstr`,
26
+		`-k8s.io/kubernetes/pkg/api/resource`,
27
+		`-k8s.io/kubernetes/pkg/runtime`,
28
+		`-k8s.io/kubernetes/pkg/watch/versioned`,
29
+		`-k8s.io/kubernetes/pkg/api/unversioned`,
30
+		`-k8s.io/kubernetes/pkg/api/v1`,
31
+		`-k8s.io/kubernetes/pkg/apis/policy/v1alpha1`,
32
+		`-k8s.io/kubernetes/pkg/apis/extensions/v1beta1`,
33
+		`-k8s.io/kubernetes/pkg/apis/autoscaling/v1`,
34
+		`-k8s.io/kubernetes/pkg/apis/batch/v1`,
35
+		`-k8s.io/kubernetes/pkg/apis/batch/v2alpha1`,
36
+		`-k8s.io/kubernetes/pkg/apis/apps/v1alpha1`,
37
+		`-k8s.io/kubernetes/federation/apis/federation/v1beta1`,
38
+
39
+		`github.com/openshift/origin/pkg/authorization/api/v1`,
40
+		`github.com/openshift/origin/pkg/build/api/v1`,
41
+		`github.com/openshift/origin/pkg/deploy/api/v1`,
42
+		`github.com/openshift/origin/pkg/image/api/v1`,
43
+		`github.com/openshift/origin/pkg/oauth/api/v1`,
44
+		`github.com/openshift/origin/pkg/project/api/v1`,
45
+		`github.com/openshift/origin/pkg/quota/api/v1`,
46
+		`github.com/openshift/origin/pkg/route/api/v1`,
47
+		`github.com/openshift/origin/pkg/sdn/api/v1`,
48
+		`github.com/openshift/origin/pkg/security/api/v1`,
49
+		`github.com/openshift/origin/pkg/template/api/v1`,
50
+		`github.com/openshift/origin/pkg/user/api/v1`,
51
+	}, ",")
52
+
53
+	g.BindFlags(flag.CommandLine)
54
+}
55
+
56
+func main() {
57
+	flag.Parse()
58
+	protobuf.Run(g)
59
+}
0 60
new file mode 100644
... ...
@@ -0,0 +1,32 @@
0
+/*
1
+Copyright 2015 The Kubernetes Authors All rights reserved.
2
+
3
+Licensed under the Apache License, Version 2.0 (the "License");
4
+you may not use this file except in compliance with the License.
5
+You may obtain a copy of the License at
6
+
7
+    http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+Unless required by applicable law or agreed to in writing, software
10
+distributed under the License is distributed on an "AS IS" BASIS,
11
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+See the License for the specific language governing permissions and
13
+limitations under the License.
14
+*/
15
+
16
+// Package main defines the protoc-gen-gogo binary we use to generate our proto go files,
17
+// as well as takes dependencies on the correct gogo/protobuf packages for godeps.
18
+package main
19
+
20
+import (
21
+	"github.com/gogo/protobuf/vanity/command"
22
+
23
+	// dependencies that are required for our packages
24
+	_ "github.com/gogo/protobuf/gogoproto"
25
+	_ "github.com/gogo/protobuf/proto"
26
+	_ "github.com/gogo/protobuf/sortkeys"
27
+)
28
+
29
+func main() {
30
+	command.Write(command.Generate(command.Read()))
31
+}