Browse code

Merge pull request #6803 from smarterclayton/build_speed

Merged by openshift-bot

OpenShift Bot authored on 2016/01/26 14:02:36
Showing 13 changed files
... ...
@@ -25,6 +25,7 @@ import (
25 25
 	"net/http/httptest"
26 26
 	"net/url"
27 27
 	"reflect"
28
+	"runtime"
28 29
 	"strings"
29 30
 	"sync"
30 31
 	"testing"
... ...
@@ -280,6 +281,9 @@ func fakePortForwardServer(t *testing.T, testName string, serverSends, expectedF
280 280
 }
281 281
 
282 282
 func TestForwardPorts(t *testing.T) {
283
+	if runtime.GOOS == "darwin" {
284
+		t.Skipf("does not work on IPv6 systems")
285
+	}
283 286
 	tests := map[string]struct {
284 287
 		ports       []string
285 288
 		clientSends map[uint16]string
... ...
@@ -21,6 +21,7 @@ import (
21 21
 	"net/http"
22 22
 	"net/http/httptest"
23 23
 	"net/url"
24
+	"runtime"
24 25
 	"strconv"
25 26
 	"strings"
26 27
 	"testing"
... ...
@@ -39,6 +40,9 @@ func containsAny(s string, substrs []string) bool {
39 39
 }
40 40
 
41 41
 func TestTcpHealthChecker(t *testing.T) {
42
+	if runtime.GOOS == "darwin" {
43
+		t.Skipf("doesn't work on Macs")
44
+	}
42 45
 	prober := New()
43 46
 	tests := []struct {
44 47
 		expectedStatus probe.Result
... ...
@@ -31,31 +31,20 @@ all build:
31 31
 	hack/build-go.sh $(WHAT)
32 32
 .PHONY: all build
33 33
 
34
-# Build and run unit tests
35
-#
36
-# Args:
37
-#   WHAT: Directory names to test.  All *_test.go files under these
38
-#     directories will be run.  If not specified, "everything" will be tested.
39
-#   TESTS: Same as WHAT.
40
-#   GOFLAGS: Extra flags to pass to 'go' when building.
41
-#   TESTFLAGS: Extra flags that should only be passed to hack/test-go.sh
34
+# Run core verification and all self contained tests.
42 35
 #
43 36
 # Example:
44 37
 #   make check
45
-#   make check WHAT=pkg/build GOFLAGS=-v
46
-check:
47
-	TEST_KUBE=1 hack/test-go.sh $(WHAT) $(TESTS) $(TESTFLAGS)
38
+check: | build verify
39
+	$(MAKE) test-unit test-cmd -o build -o verify
48 40
 .PHONY: check
49 41
 
50
-# Verify code is properly organized.
42
+
43
+# Verify code conventions are properly setup.
51 44
 #
52 45
 # Example:
53 46
 #   make verify
54
-ifeq ($(SKIP_BUILD), true)
55
-verify:
56
-else
57 47
 verify: build
58
-endif
59 48
 	hack/verify-upstream-commits.sh
60 49
 	hack/verify-gofmt.sh
61 50
 	hack/verify-govet.sh
... ...
@@ -66,91 +55,76 @@ endif
66 66
 	hack/verify-generated-swagger-spec.sh
67 67
 .PHONY: verify
68 68
 
69
-# check and verify can't run concurently because of strange concurrent build issues.
70
-check-verify:
71
-	# delegate to another make process that runs serially against the check and verify targets
72
-	$(MAKE) -j1 check verify
73
-.PHONY: check-verify
74
-
75
-# Install travis dependencies
76
-#
77
-# Args:
78
-#   TEST_ASSETS: Instead of running tests, test assets only.
79
-ifeq ($(TEST_ASSETS), true)
80
-install-travis:
81
-	hack/install-assets.sh
82
-else
83
-install-travis:
84
-	hack/install-etcd.sh
85
-	hack/install-tools.sh
86
-endif
87
-.PHONY: install-travis
88
-
89
-# Run unit and integration tests that don't require Docker.
69
+# Run unit tests.
90 70
 #
91 71
 # Args:
72
+#   WHAT: Directory names to test.  All *_test.go files under these
73
+#     directories will be run.  If not specified, "everything" will be tested.
74
+#   TESTS: Same as WHAT.
92 75
 #   GOFLAGS: Extra flags to pass to 'go' when building.
93 76
 #   TESTFLAGS: Extra flags that should only be passed to hack/test-go.sh
94
-#   TEST_ASSETS: Instead of running tests, test assets only.
95 77
 #
96 78
 # Example:
97
-#   make check-test
98
-check-test: export KUBE_COVER= -cover -covermode=atomic
99
-check-test: export KUBE_RACE=  -race
100
-ifeq ($(TEST_ASSETS), true)
101
-check-test:
102
-	hack/test-assets.sh
103
-else
104
-check-test: 
105
-	check-verify
106
-	hack/test-cmd.sh
107
-	KUBE_RACE=" " hack/test-integration.sh
108
-endif
109
-.PHONY: check-test
79
+#   make test-unit
80
+#   make test-unit WHAT=pkg/build GOFLAGS=-v
81
+test-unit:
82
+	TEST_KUBE=1 hack/test-go.sh $(WHAT) $(TESTS) $(TESTFLAGS)
83
+.PHONY: test-unit
110 84
 
111
-# Build and run the complete test-suite.
112
-#
113
-# Args:
114
-#   GOFLAGS: Extra flags to pass to 'go' when building.
115
-#   TESTFLAGS: Extra flags that should only be passed to hack/test-go.sh
85
+# Run integration tests. Compiles its own tests, cannot be run
86
+# in parallel with any other go compilation.
116 87
 #
117 88
 # Example:
118
-#   make test
119
-#   make test GOFLAGS=-v
120
-test: export KUBE_COVER= -cover -covermode=atomic
121
-test: export KUBE_RACE=  -race
122
-ifeq ($(SKIP_BUILD), true)
123
-test: check-verify test-int-plus
124
-else
125
-test: build check-verify test-int-plus
126
-endif
127
-.PHONY: test
89
+#   make test-integration
90
+test-integration:
91
+	KUBE_COVER=" " KUBE_RACE=" " hack/test-integration.sh
92
+.PHONY: test-integration
128 93
 
129
-# Split out of `test`.  This allows `make -j --output-sync=recurse test` to parallelize as expected
130
-test-int-plus: export KUBE_COVER= -cover -covermode=atomic
131
-test-int-plus: export KUBE_RACE=  -race
132
-ifeq ($(SKIP_BUILD), true)
133
-test-int-plus: 
134
-else
135
-test-int-plus: build
136
-endif
137
-test-int-plus:
94
+# Run command tests. Uses whatever binaries are currently built.
95
+#
96
+# Example:
97
+#   make test-cmd
98
+test-cmd: build
138 99
 	hack/test-cmd.sh
100
+.PHONY: test-cmd
101
+
102
+# Run end to end tests. Uses whatever binaries are currently built.
103
+#
104
+# Example:
105
+#   make test-cmd
106
+test-end-to-end: build
107
+	hack/test-end-to-end.sh
108
+.PHONY: test-end-to-end
109
+
110
+# Run tools tests.
111
+#
112
+# Example:
113
+#   make test-cmd
114
+test-tools:
139 115
 	hack/test-tools.sh
140
-	KUBE_RACE=" " hack/test-integration-docker.sh
141
-	hack/test-end-to-end-docker.sh
142
-ifeq ($(EXTENDED),true)
143
-	hack/test-extended.sh
116
+.PHONY: test-tools
117
+
118
+test-assets:
119
+ifeq ($(TEST_ASSETS),true)
120
+	hack/test-assets.sh
144 121
 endif
145
-.PHONY: test-int-plus
122
+.PHONY: test-assets
146 123
 
124
+# Build and run the complete test-suite.
125
+#
126
+# Example:
127
+#   make test
128
+test: check
129
+	$(MAKE) test-tools test-integration test-assets -o build
130
+	$(MAKE) test-end-to-end -o build
131
+.PHONY: test
147 132
 
148 133
 # Run All-in-one OpenShift server.
149 134
 #
150 135
 # Example:
151 136
 #   make run
152
-OS_OUTPUT_BINPATH=$(shell bash -c 'source hack/common.sh; echo $${OS_OUTPUT_BINPATH}')
153
-PLATFORM=$(shell bash -c 'source hack/common.sh; os::build::host_platform')
137
+run: export OS_OUTPUT_BINPATH=$(shell bash -c 'source hack/common.sh; echo $${OS_OUTPUT_BINPATH}')
138
+run: export PLATFORM=$(shell bash -c 'source hack/common.sh; os::build::host_platform')
154 139
 run: build
155 140
 	$(OS_OUTPUT_BINPATH)/$(PLATFORM)/openshift start
156 141
 .PHONY: run
... ...
@@ -181,3 +155,10 @@ release-binaries: clean
181 181
 	hack/build-release.sh
182 182
 	hack/extract-release.sh
183 183
 .PHONY: release-binaries
184
+
185
+# Install travis dependencies
186
+#
187
+install-travis:
188
+	hack/install-tools.sh
189
+.PHONY: install-travis
190
+
... ...
@@ -12,6 +12,22 @@ source "${OS_ROOT}/hack/common.sh"
12 12
 source "${OS_ROOT}/hack/util.sh"
13 13
 os::log::install_errexit
14 14
 
15
+# only works on Linux for now, all other platforms must build binaries themselves
16
+if [[ -z "$@" ]]; then
17
+  if os::build::detect_local_release_tars $(os::build::host_platform_friendly) >/dev/null; then
18
+    platform=$(os::build::host_platform)
19
+    echo "++ Using release artifacts from ${OS_RELEASE_COMMIT} for ${platform} instead of building"
20
+    mkdir -p "${OS_OUTPUT_BINPATH}/${platform}"
21
+    tar mxzf "${OS_PRIMARY_RELEASE_TAR}" --strip-components=1 -C "${OS_OUTPUT_BINPATH}/${platform}"
22
+    tar mxzf "${OS_CLIENT_RELEASE_TAR}" --strip-components=1 -C "${OS_OUTPUT_BINPATH}/${platform}"
23
+    tar mxzf "${OS_IMAGE_RELEASE_TAR}" --strip-components=1 -C "${OS_OUTPUT_BINPATH}/${platform}"
24
+
25
+    os::build::make_openshift_binary_symlinks
26
+
27
+    ret=$?; ENDTIME=$(date +%s); echo "$0 took $(($ENDTIME - $STARTTIME)) seconds"; exit "$ret"
28
+  fi
29
+fi
30
+
15 31
 os::build::build_binaries "$@"
16 32
 os::build::place_bins "$@"
17 33
 os::build::make_openshift_binary_symlinks
... ...
@@ -124,6 +124,25 @@ os::build::host_platform() {
124 124
   echo "$(go env GOHOSTOS)/$(go env GOHOSTARCH)"
125 125
 }
126 126
 
127
+# Create a user friendly version of host_platform for end users
128
+os::build::host_platform_friendly() {
129
+  local platform=${1:-}
130
+  if [[ -z "${platform}" ]]; then
131
+    platform=$(os::build::host_platform)
132
+  fi
133
+  if [[ $platform == "windows/amd64" ]]; then
134
+    echo "windows"
135
+  elif [[ $platform == "darwin/amd64" ]]; then
136
+    echo "mac"
137
+  elif [[ $platform == "linux/386" ]]; then
138
+    echo "linux-32bit"
139
+  elif [[ $platform == "linux/amd64" ]]; then
140
+    echo "linux-64bit"
141
+  else
142
+    echo "$(go env GOHOSTOS)-$(go env GOHOSTARCH)"
143
+  fi
144
+}
145
+
127 146
 # Build binaries targets specified
128 147
 #
129 148
 # Input:
... ...
@@ -444,26 +463,33 @@ os::build::detect_local_release_tars() {
444 444
 
445 445
   if [[ ! -d "${OS_LOCAL_RELEASEPATH}" ]]; then
446 446
     echo "There are no release artifacts in ${OS_LOCAL_RELEASEPATH}"
447
-    exit 2
447
+    return 2
448 448
   fi
449 449
   if [[ ! -f "${OS_LOCAL_RELEASEPATH}/.commit" ]]; then
450 450
     echo "There is no release .commit identifier ${OS_LOCAL_RELEASEPATH}"
451
-    exit 2
451
+    return 2
452 452
   fi
453 453
   local primary=$(find ${OS_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name openshift-origin-server-*-${platform}*)
454 454
   if [[ $(echo "${primary}" | wc -l) -ne 1 || -z "${primary}" ]]; then
455 455
     echo "There should be exactly one ${platform} server tar in $OS_LOCAL_RELEASEPATH"
456
-    exit 2
456
+    return 2
457
+  fi
458
+
459
+  local client=$(find ${OS_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name openshift-origin-client-tools-*-${platform}*)
460
+  if [[ $(echo "${client}" | wc -l) -ne 1 || -z "${primary}" ]]; then
461
+    echo "There should be exactly one ${platform} client tar in $OS_LOCAL_RELEASEPATH"
462
+    return 2
457 463
   fi
458 464
 
459 465
   local image=$(find ${OS_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name openshift-origin-image*-${platform}*)
460 466
   if [[ $(echo "${image}" | wc -l) -ne 1 || -z "${image}" ]]; then
461 467
     echo "There should be exactly one ${platform} image tar in $OS_LOCAL_RELEASEPATH"
462
-    exit 3
468
+    return 2
463 469
   fi
464 470
 
465 471
   export OS_PRIMARY_RELEASE_TAR="${primary}"
466 472
   export OS_IMAGE_RELEASE_TAR="${image}"
473
+  export OS_CLIENT_RELEASE_TAR="${client}"
467 474
   export OS_RELEASE_COMMIT="$(cat ${OS_LOCAL_RELEASEPATH}/.commit)"
468 475
 }
469 476
 
... ...
@@ -15,10 +15,11 @@ cd "${OS_ROOT}"
15 15
 
16 16
 # Copy the linux release archives release back to the local _output/local/bin/linux/amd64 directory.
17 17
 # TODO: support different OS's?
18
-os::build::detect_local_release_tars "linux-64bit"
18
+os::build::detect_local_release_tars $(os::build::host_platform_friendly)
19 19
 
20 20
 mkdir -p "${OS_OUTPUT_BINPATH}/linux/amd64"
21 21
 tar mxzf "${OS_PRIMARY_RELEASE_TAR}" --strip-components=1 -C "${OS_OUTPUT_BINPATH}/linux/amd64"
22
+tar mxzf "${OS_CLIENT_RELEASE_TAR}" --strip-components=1 -C "${OS_OUTPUT_BINPATH}/linux/amd64"
22 23
 tar mxzf "${OS_IMAGE_RELEASE_TAR}" --strip-components=1 -C "${OS_OUTPUT_BINPATH}/linux/amd64"
23 24
 
24 25
 os::build::make_openshift_binary_symlinks
... ...
@@ -27,33 +27,14 @@ function cmd() {
27 27
   exit $rc
28 28
 }
29 29
 
30
-# If we are running inside of Travis then do not run the rest of this
31
-# script unless we want to TEST_ASSETS
32
-if [[ "${TRAVIS-}" == "true" && "${TEST_ASSETS-}" == "false" ]]; then
33
-  exit
34
-fi
35
-
36
-# Lock version of npm to work around https://github.com/npm/npm/issues/6309
37
-if [[ "${TRAVIS-}" == "true" ]]; then
38
-  cmd "npm install -g npm@2.1.14"
39
-fi
40
-
41 30
 # Install bower if needed
42 31
 if ! which bower > /dev/null 2>&1 ; then
43
-  if [[ "${TRAVIS-}" == "true" ]]; then
44
-    cmd "npm install -g bower"
45
-  else
46
-    cmd "sudo npm install -g bower"
47
-  fi
32
+  cmd "sudo npm install -g bower"
48 33
 fi
49 34
 
50 35
 # Install grunt if needed
51 36
 if ! which grunt > /dev/null 2>&1 ; then
52
-  if [[ "${TRAVIS-}" == "true" ]]; then
53
-    cmd "npm install -g grunt-cli"
54
-  else
55
-    cmd "sudo npm install -g grunt-cli"
56
-  fi
37
+  cmd "sudo npm install -g grunt-cli"
57 38
 fi
58 39
 
59 40
 pushd ${OS_ROOT}/assets > /dev/null
... ...
@@ -11,12 +11,6 @@ source "${OS_ROOT}/hack/common.sh"
11 11
 # Go to the top of the tree.
12 12
 cd "${OS_ROOT}"
13 13
 
14
-# If we are running inside of Travis then do not run the rest of this
15
-# script unless we want to TEST_ASSETS
16
-if [[ "${TRAVIS-}" == "true" && "${TEST_ASSETS-}" == "false" ]]; then
17
-  exit
18
-fi
19
-
20 14
 pushd "${OS_ROOT}/assets" > /dev/null
21 15
   grunt test
22 16
   grunt build
... ...
@@ -9,6 +9,16 @@ set -o pipefail
9 9
 
10 10
 STARTTIME=$(date +%s)
11 11
 OS_ROOT=$(dirname "${BASH_SOURCE}")/..
12
+
13
+if [[ "${TEST_END_TO_END:-}" != "direct" ]]; then
14
+	if docker version >/dev/null 2>&1; then
15
+		echo "++ Docker is installed, running hack/test-end-to-end-docker.sh instead."
16
+		"${OS_ROOT}/hack/test-end-to-end-docker.sh"
17
+		exit $?
18
+	fi
19
+	echo "++ Docker is not installed, running end-to-end against local binaries"
20
+fi
21
+
12 22
 source "${OS_ROOT}/hack/util.sh"
13 23
 source "${OS_ROOT}/hack/lib/log.sh"
14 24
 source "${OS_ROOT}/hack/lib/util/environment.sh"
15 25
deleted file mode 100755
... ...
@@ -1,12 +0,0 @@
1
-#!/bin/bash
2
-
3
-set -o errexit
4
-set -o nounset
5
-set -o pipefail
6
-
7
-OS_ROOT=$(dirname "${BASH_SOURCE}")/..
8
-
9
-# Go to the top of the tree.
10
-cd "${OS_ROOT}"
11
-
12
-OS_TEST_TAGS="integration docker etcd" hack/test-integration.sh $@
... ...
@@ -37,7 +37,13 @@ function cleanup() {
37 37
 trap cleanup EXIT SIGINT
38 38
 
39 39
 package="${OS_TEST_PACKAGE:-test/integration}"
40
-tags="${OS_TEST_TAGS:-integration !docker etcd}"
40
+
41
+if docker version >/dev/null 2>&1; then
42
+	tags="${OS_TEST_TAGS:-integration docker etcd}"
43
+else
44
+	echo "++ Docker not available, running only integration tests without the 'docker' tag"
45
+	tags="${OS_TEST_TAGS:-integration !docker etcd}"
46
+fi
41 47
 
42 48
 export GOMAXPROCS="$(grep "processor" -c /proc/cpuinfo 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || 1)"
43 49
 
... ...
@@ -776,6 +776,14 @@ os::util::sed() {
776 776
   fi
777 777
 }
778 778
 
779
+os::util::base64decode() {
780
+  if [[ "$(go env GOHOSTOS)" == "darwin" ]]; then
781
+  	base64 -D $@
782
+  else
783
+  	base64 -d $@
784
+  fi
785
+}
786
+
779 787
 os::util::get_object_assert() {
780 788
   local object=$1
781 789
   local request=$2
... ...
@@ -111,6 +111,6 @@ os::cmd::expect_failure 'oc get dc/ruby-hello-world'
111 111
 echo "delete all: ok"
112 112
 
113 113
 # service accounts should not be allowed to request new projects
114
-SA_TOKEN=`oc get sa/builder --template='{{range .secrets}}{{ .name }} {{end}}' | xargs -n 1 oc get secret --template='{{ if .data.token }}{{ .data.token }}{{end}}' | base64 -d -`
114
+SA_TOKEN=`oc get sa/builder --template='{{range .secrets}}{{ .name }} {{end}}' | xargs -n 1 oc get secret --template='{{ if .data.token }}{{ .data.token }}{{end}}' | os::util::base64decode -`
115 115
 os::cmd::expect_failure_and_text "oc new-project --token=${SA_TOKEN} will-fail" 'Error from server: You may not request a new project via this API'
116 116