Browse code

go {build,test}: rm -i option, add go cache volume

Looks like -i (together with DOCKER_INCREMENTAL_BINARY etc)
were used to get faster incremental builds.

Nowdays (since Go 1.10) this is no longer the case, as
go build cache is used [1]. Here's a quote:

> You do not have to use "go test -i" or "go build -i" or
> "go install" just to get fast incremental builds. We will
> not have to teach new users those workarounds anymore.
> Everything will just be fast.

To enable go cache between builds, add a volume for /root/.cache.

[1] https://groups.google.com/forum/#!msg/golang-dev/qfa3mHN4ZPA/X2UzjNV1BAAJ

Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>

Kir Kolyshkin authored on 2018/09/19 05:28:49
Showing 5 changed files
... ...
@@ -1,10 +1,8 @@
1
-.PHONY: all binary dynbinary build cross help init-go-pkg-cache install manpages run shell test test-docker-py test-integration test-unit validate win
1
+.PHONY: all binary dynbinary build cross help install manpages run shell test test-docker-py test-integration test-unit validate win
2 2
 
3 3
 # set the graph driver as the current graphdriver if not set
4 4
 DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //'))
5 5
 export DOCKER_GRAPHDRIVER
6
-DOCKER_INCREMENTAL_BINARY := $(if $(DOCKER_INCREMENTAL_BINARY),$(DOCKER_INCREMENTAL_BINARY),1)
7
-export DOCKER_INCREMENTAL_BINARY
8 6
 
9 7
 # get OS/Arch of docker engine
10 8
 DOCKER_OSARCH := $(shell bash -c 'source hack/make/.detect-daemon-osarch && echo $${DOCKER_ENGINE_OSARCH}')
... ...
@@ -38,7 +36,6 @@ DOCKER_ENVS := \
38 38
 	-e DOCKER_EXPERIMENTAL \
39 39
 	-e DOCKER_GITCOMMIT \
40 40
 	-e DOCKER_GRAPHDRIVER \
41
-	-e DOCKER_INCREMENTAL_BINARY \
42 41
 	-e DOCKER_LDFLAGS \
43 42
 	-e DOCKER_PORT \
44 43
 	-e DOCKER_REMAP_ROOT \
... ...
@@ -75,14 +72,10 @@ DOCKER_MOUNT := $(if $(DOCKER_MOUNT),$(DOCKER_MOUNT),-v /go/src/github.com/docke
75 75
 # This allows to set the docker-dev container name
76 76
 DOCKER_CONTAINER_NAME := $(if $(CONTAINER_NAME),--name $(CONTAINER_NAME),)
77 77
 
78
-# enable package cache if DOCKER_INCREMENTAL_BINARY and DOCKER_MOUNT (i.e.DOCKER_HOST) are set
79
-PKGCACHE_MAP := gopath:/go/pkg goroot-linux_amd64:/usr/local/go/pkg/linux_amd64 goroot-linux_amd64_netgo:/usr/local/go/pkg/linux_amd64_netgo
80
-PKGCACHE_VOLROOT := dockerdev-go-pkg-cache
81
-PKGCACHE_VOL := $(if $(PKGCACHE_DIR),$(CURDIR)/$(PKGCACHE_DIR)/,$(PKGCACHE_VOLROOT)-)
82
-DOCKER_MOUNT_PKGCACHE := $(if $(DOCKER_INCREMENTAL_BINARY),$(shell echo $(PKGCACHE_MAP) | sed -E 's@([^ ]*)@-v "$(PKGCACHE_VOL)\1"@g'),)
78
+DOCKER_MOUNT_CACHE := -v docker-dev-cache:/root/.cache
83 79
 DOCKER_MOUNT_CLI := $(if $(DOCKER_CLI_PATH),-v $(shell dirname $(DOCKER_CLI_PATH)):/usr/local/cli,)
84 80
 DOCKER_MOUNT_BASH_COMPLETION := $(if $(DOCKER_BASH_COMPLETION_PATH),-v $(shell dirname $(DOCKER_BASH_COMPLETION_PATH)):/usr/local/completion/bash,)
85
-DOCKER_MOUNT := $(DOCKER_MOUNT) $(DOCKER_MOUNT_PKGCACHE) $(DOCKER_MOUNT_CLI) $(DOCKER_MOUNT_BASH_COMPLETION)
81
+DOCKER_MOUNT := $(DOCKER_MOUNT) $(DOCKER_MOUNT_CACHE) $(DOCKER_MOUNT_CLI) $(DOCKER_MOUNT_BASH_COMPLETION)
86 82
 
87 83
 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
88 84
 GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g")
... ...
@@ -124,19 +117,19 @@ binary: build ## build the linux binaries
124 124
 dynbinary: build ## build the linux dynbinaries
125 125
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary
126 126
 
127
-build: bundles init-go-pkg-cache
127
+build: bundles
128 128
 	$(warning The docker client CLI has moved to github.com/docker/cli. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n})
129 129
 	docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" .
130 130
 
131 131
 bundles:
132 132
 	mkdir bundles
133 133
 
134
-clean: clean-pkg-cache-vol ## clean up cached resources
134
+.PHONY: clean
135
+clean: clean-cache
135 136
 
136
-clean-pkg-cache-vol:
137
-	@- $(foreach mapping,$(PKGCACHE_MAP), \
138
-		$(shell docker volume rm $(PKGCACHE_VOLROOT)-$(shell echo $(mapping) | awk -F':/' '{ print $$1 }') > /dev/null 2>&1) \
139
-	)
137
+.PHONY: clean-cache
138
+clean-cache:
139
+	docker volume rm -f docker-dev-cache
140 140
 
141 141
 cross: build ## cross build the binaries for darwin, freebsd and\nwindows
142 142
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary binary cross
... ...
@@ -144,9 +137,6 @@ cross: build ## cross build the binaries for darwin, freebsd and\nwindows
144 144
 help: ## this help
145 145
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
146 146
 
147
-init-go-pkg-cache:
148
-	$(if $(PKGCACHE_DIR), mkdir -p $(shell echo $(PKGCACHE_MAP) | sed -E 's@([^: ]*):[^ ]*@$(PKGCACHE_DIR)/\1@g'))
149
-
150 147
 install: ## install the linux binaries
151 148
 	KEEPBUNDLE=1 hack/make.sh install-binary
152 149
 
... ...
@@ -197,12 +187,11 @@ build-integration-cli-on-swarm: build ## build images and binary for running int
197 197
 	go build -buildmode=pie -o ./hack/integration-cli-on-swarm/integration-cli-on-swarm ./hack/integration-cli-on-swarm/host
198 198
 	@echo "Building $(INTEGRATION_CLI_MASTER_IMAGE)"
199 199
 	docker build -t $(INTEGRATION_CLI_MASTER_IMAGE) hack/integration-cli-on-swarm/agent
200
-# For worker, we don't use `docker build` so as to enable DOCKER_INCREMENTAL_BINARY and so on
201 200
 	@echo "Building $(INTEGRATION_CLI_WORKER_IMAGE) from $(DOCKER_IMAGE)"
202 201
 	$(eval tmp := integration-cli-worker-tmp)
203 202
 # We mount pkgcache, but not bundle (bundle needs to be baked into the image)
204 203
 # For avoiding bakings DOCKER_GRAPHDRIVER and so on to image, we cannot use $(DOCKER_ENVS) here
205
-	docker run -t -d --name $(tmp) -e DOCKER_GITCOMMIT -e BUILDFLAGS -e DOCKER_INCREMENTAL_BINARY --privileged $(DOCKER_MOUNT_PKGCACHE) $(DOCKER_IMAGE) top
204
+	docker run -t -d --name $(tmp) -e DOCKER_GITCOMMIT -e BUILDFLAGS --privileged $(DOCKER_IMAGE) top
206 205
 	docker exec $(tmp) hack/make.sh build-integration-test-binary dynbinary
207 206
 	docker exec $(tmp) go build -buildmode=pie -o /worker github.com/docker/docker/hack/integration-cli-on-swarm/agent/worker
208 207
 	docker commit -c 'ENTRYPOINT ["/worker"]' $(tmp) $(INTEGRATION_CLI_WORKER_IMAGE)
... ...
@@ -130,7 +130,7 @@ can take over 15 minutes to complete.
130 130
    ```none
131 131
    Successfully built 3d872560918e
132 132
    Successfully tagged docker-dev:dry-run-test
133
-   docker run --rm -i --privileged -e BUILDFLAGS -e KEEPBUNDLE -e DOCKER_BUILD_GOGC -e DOCKER_BUILD_PKGS -e DOCKER_CLIENTONLY -e DOCKER_DEBUG -e DOCKER_EXPERIMENTAL -e DOCKER_GITCOMMIT -e DOCKER_GRAPHDRIVER=devicemapper -e DOCKER_INCREMENTAL_BINARY -e DOCKER_REMAP_ROOT -e DOCKER_STORAGE_OPTS -e DOCKER_USERLANDPROXY -e TESTDIRS -e TESTFLAGS -e TIMEOUT -v "home/ubuntu/repos/docker/bundles:/go/src/github.com/docker/docker/bundles" -t "docker-dev:dry-run-test" bash
133
+   docker run --rm -i --privileged -e BUILDFLAGS -e KEEPBUNDLE -e DOCKER_BUILD_GOGC -e DOCKER_BUILD_PKGS -e DOCKER_CLIENTONLY -e DOCKER_DEBUG -e DOCKER_EXPERIMENTAL -e DOCKER_GITCOMMIT -e DOCKER_GRAPHDRIVER=devicemapper -e DOCKER_REMAP_ROOT -e DOCKER_STORAGE_OPTS -e DOCKER_USERLANDPROXY -e TESTDIRS -e TESTFLAGS -e TIMEOUT -v "home/ubuntu/repos/docker/bundles:/go/src/github.com/docker/docker/bundles" -t "docker-dev:dry-run-test" bash
134 134
    #
135 135
    ```
136 136
 
... ...
@@ -36,7 +36,6 @@ while the client is supposed to be running on a laptop, e.g. Docker for Mac/Wind
36 36
 Following environment variables are known to work in this step:
37 37
 
38 38
  - `BUILDFLAGS`
39
- - `DOCKER_INCREMENTAL_BINARY`
40 39
 
41 40
 Note: during the transition into Moby Project, you might need to create a symbolic link `$GOPATH/src/github.com/docker/docker` to `$GOPATH/src/github.com/moby/moby`. 
42 41
 
... ...
@@ -148,14 +148,6 @@ EXTLDFLAGS_STATIC='-static'
148 148
 ORIG_BUILDFLAGS=( -tags "autogen netgo osusergo static_build $DOCKER_BUILDTAGS" -installsuffix netgo )
149 149
 # see https://github.com/golang/go/issues/9369#issuecomment-69864440 for why -installsuffix is necessary here
150 150
 
151
-# When $DOCKER_INCREMENTAL_BINARY is set in the environment, enable incremental
152
-# builds by installing dependent packages to the GOPATH.
153
-REBUILD_FLAG="-a"
154
-if [ "$DOCKER_INCREMENTAL_BINARY" == "1" ] || [ "$DOCKER_INCREMENTAL_BINARY" == "true" ]; then
155
-	REBUILD_FLAG="-i"
156
-fi
157
-ORIG_BUILDFLAGS+=( $REBUILD_FLAG )
158
-
159 151
 BUILDFLAGS=( $BUILDFLAGS "${ORIG_BUILDFLAGS[@]}" )
160 152
 
161 153
 # Test timeout.
... ...
@@ -19,10 +19,6 @@ TESTDIRS="${TESTDIRS:-"./..."}"
19 19
 exclude_paths="/vendor/|/integration"
20 20
 pkg_list=$(go list $TESTDIRS | grep -vE "($exclude_paths)")
21 21
 
22
-# install test dependencies once before running tests for each package. This
23
-# significantly reduces the runtime.
24
-go test -i "${BUILDFLAGS[@]}" $pkg_list
25
-
26 22
 for pkg in $pkg_list; do
27 23
     go test "${BUILDFLAGS[@]}" \
28 24
         -cover \