@@ -149,6 +149,18 @@ RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/

 FROM runtime-dev-cross-${CROSS} AS runtime-dev

+FROM base AS delve

+# DELVE_VERSION specifies the version of the Delve debugger binary

+# from the https://github.com/go-delve/delve repository.

+# It can be used to run Docker with a possibility of

+# attaching debugger to it.

+#

+ARG DELVE_VERSION=v1.8.1

+RUN --mount=type=cache,target=/root/.cache/go-build \

+    --mount=type=cache,target=/go/pkg/mod \

+        GOBIN=/build/ GO111MODULE=on go install "github.com/go-delve/delve/cmd/dlv@${DELVE_VERSION}" \

+     && /build/dlv --help

+

 FROM base AS tomll

 # GOTOML_VERSION specifies the version of the tomll binary to build and install

 # from the https://github.com/pelletier/go-toml repository. This binary is used

@@ -301,6 +313,7 @@ RUN pip3 install yamllint==1.26.1

 COPY --from=dockercli     /build/ /usr/local/cli

 COPY --from=frozen-images /build/ /docker-frozen-images

 COPY --from=swagger       /build/ /usr/local/bin/

+COPY --from=delve         /build/ /usr/local/bin/

 COPY --from=tomll         /build/ /usr/local/bin/

 COPY --from=tini          /build/ /usr/local/bin/

 COPY --from=registry      /build/ /usr/local/bin/

@@ -68,6 +68,7 @@ DOCKER_ENVS := \

 	-e DOCKER_TEST_HOST \

 	-e DOCKER_USERLANDPROXY \

 	-e DOCKERD_ARGS \

+	-e DELVE_PORT \

 	-e TEST_FORCE_VALIDATE \

 	-e TEST_INTEGRATION_DIR \

 	-e TEST_SKIP_INTEGRATION \

@@ -114,8 +115,9 @@ DOCKER_CONTAINER_NAME := $(if $(CONTAINER_NAME),--name $(CONTAINER_NAME),)

 DOCKER_IMAGE := docker-dev

 DOCKER_PORT_FORWARD := $(if $(DOCKER_PORT),-p "$(DOCKER_PORT)",)

+DELVE_PORT_FORWARD := $(if $(DELVE_PORT),-p "$(DELVE_PORT)",)

-DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD)

+DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD) $(DELVE_PORT_FORWARD)

 BUILD_APT_MIRROR := $(if $(DOCKER_BUILD_APT_MIRROR),--build-arg APT_MIRROR=$(DOCKER_BUILD_APT_MIRROR))

 export BUILD_APT_MIRROR

@@ -77,6 +77,8 @@ if [ -z "$mtu" ]; then

 	mtu=1500

 fi

+dockerd="${DOCKERD:-dockerd}"

+

 if [ -z "$_DOCKERD_ROOTLESS_CHILD" ]; then

 	_DOCKERD_ROOTLESS_CHILD=1

 	export _DOCKERD_ROOTLESS_CHILD

@@ -128,5 +130,7 @@ else

 		mount --rbind ${realpath_etc_ssl} /etc/ssl

 	fi

-	exec dockerd $@

+	# shellcheck disable=SC2068

+	# shellcheck disable=SC2086

+	exec $dockerd $@

 fi

@@ -6,3 +6,4 @@

  * [Configure Git for contributing](set-up-git.md)

  * [Work with a development container](set-up-dev-env.md)

  * [Run tests and test documentation](test.md)

+ * [Debugging the daemon](debug.md)

new file mode 100644

@@ -0,0 +1,66 @@

+### Debugging the daemon

+

+The Docker daemon inside the development container can be debugged with [Delve](https://github.com/go-delve/delve).

+

+Delve debugger listens on a port, which has to be exposed outside the development container.

+Also, in order to be able to debug the daemon, it has to be compiled with the debugging symbols.

+This can be done by launching the development container with the following command:

+

+```bash

+$ make BIND_DIR=. DOCKER_DEBUG=1 DELVE_PORT=127.0.0.1:2345:2345 shell

+```

+

+The `DOCKER_DEBUG` variable disables build optimizations, allowing to debug the binary,

+while `DELVE_PORT` publishes the specified port for use with the debugger.

+

+The `DELVE_PORT` variable accepts the port in the same format as Docker CLI's `--publish` (`-p`) option.

+This means that the port can be published in multiple ways:

+

+1. `DELVE_PORT=127.0.0.1:2345:2345` - exposes debugger on port `2345` for local development only (recommended)

+2. `DELVE_PORT=2345:2345` - exposes debugger on port `2345` without binding to specific IP

+3. `DELVE_PORT=2345` - same as above

+

+**IMPORTANT:** Publishing the port without binding it to localhost (127.0.0.1) might expose the debugger

+outside the developer's machine and is not recommended.

+

+## Running Docker daemon with debugger attached

+

+1. Run development container with build optimizations disabled and Delve enabled:

+   ```bash

+   $ make BIND_DIR=. DOCKER_DEBUG=1 DELVE_PORT=127.0.0.1:2345:2345 shell

+   ```

+2. Inside the development container:

+   1. Build the Docker daemon:

+      ```bash

+      $ ./hack/make.sh binary

+      ```

+   2. Install the newly-built daemon:

+      ```bash

+      $ make install

+      ```

+   3. Run the daemon through the `make.sh` script:

+      ```bash

+      $ ./hack/make.sh run

+      ```

+      The execution will stop and wait for the IDE or Delve CLI to attach

+      to the port, specified with the `DELVE_PORT` variable.

+      Once the IDE or Delve CLI is attached, the execution will continue.

+   

+## Debugging from IDE (on example of GoLand 2021.3)

+

+1. Open the project in GoLand

+2. Click *Add Configuration* on the taskbar

+   ![GoLand - adding configuration](images/goland_add_config.png)

+3. Create the *Go Remote* configuration. 

+   No changes are necessary, unless a different port is to be used.

+   ![GoLand - adding remote configuration](images/goland_debug_config.png)

+4. Run the Docker binary in the development container, as described in the previous section.

+   Make sure that the port in the `DELVE_PORT` variable corresponds to one, used in the *Go Remote* configuration.

+5. Run the *Go Remote* configuration.

+   The Docker daemon will continue execution inside the container and debugger will stop it on the breakpoints.

+   ![GoLand - run Go Remote configuration](images/goland_run_debug_config.png)

+

+## Where to go next

+

+Congratulations, you have experienced how to use Delve to debug the Docker daemon

+and how to configure an IDE to make use of it.

\ No newline at end of file

new file mode 100644

Binary files /dev/null and b/docs/contributing/images/goland_add_config.png differ

new file mode 100644

Binary files /dev/null and b/docs/contributing/images/goland_debug_config.png differ

new file mode 100644

Binary files /dev/null and b/docs/contributing/images/goland_run_debug_config.png differ

@@ -250,3 +250,4 @@ jobs can be triggered and re-ran by the Moby maintainers

 Congratulations, you have successfully completed the basics you need to

 understand the Moby test framework.

+In the next section you'll [learn how to debug Docker daemon, running inside the development container](debug.md).

@@ -27,6 +27,11 @@ if [ -n "$DOCKER_PORT" ]; then

 	listen_port="${ports[-1]}"

 fi

+if [ -n "$DELVE_PORT" ]; then

+	IFS=':' read -r -a ports <<< "$DELVE_PORT"

+	delve_listen_port="${ports[-1]}"

+fi

+

 extra_params="$DOCKERD_ARGS"

 if [ "$DOCKER_REMAP_ROOT" ]; then

 	extra_params="$extra_params --userns-remap $DOCKER_REMAP_ROOT"

@@ -36,7 +41,7 @@ if [ -n "$DOCKER_EXPERIMENTAL" ]; then

 	extra_params="$extra_params --experimental"

 fi

-dockerd="dockerd"

+dockerd="$(command -v dockerd)"

 socket=/var/run/docker.sock

 if [ -n "$DOCKER_ROOTLESS" ]; then

 	user="unprivilegeduser"

@@ -44,7 +49,6 @@ if [ -n "$DOCKER_ROOTLESS" ]; then

 	# shellcheck disable=SC2174

 	mkdir -p -m 700 "/tmp/docker-${uid}"

 	chown $user "/tmp/docker-${uid}"

-	dockerd="sudo -u $user -E XDG_RUNTIME_DIR=/tmp/docker-${uid} -E HOME=/home/${user} -- dockerd-rootless.sh"

 	socket=/tmp/docker-${uid}/docker.sock

 fi

@@ -55,6 +59,29 @@ args="--debug \

 	$storage_params \

 	$extra_params"

+if [ -n "$DELVE_PORT" ]; then

+	dockerd="dlv --listen=0.0.0.0:$delve_listen_port \

+		--headless=true \

+		--log \

+		--api-version=2 \

+		--only-same-user=false \

+		--check-go-version=false \

+		--accept-multiclient \

+		exec ${dockerd} --"

+fi

+

 echo "${dockerd} ${args}"

-# shellcheck disable=SC2086

-exec "${dockerd}" ${args}

+

+if [ -n "$DOCKER_ROOTLESS" ]; then

+	# shellcheck disable=SC2068

+	# shellcheck disable=SC2086

+	exec sudo -u $user \

+		-E DOCKERD="$dockerd" \

+		-E XDG_RUNTIME_DIR=/tmp/docker-${uid} \

+		-E XDG_CONFIG_HOME=/home/${user}/.config \

+		-E HOME=/home/${user} \

+		-- /go/src/github.com/docker/docker/contrib/dockerd-rootless.sh ${args}

+else

+	# shellcheck disable=SC2086

+	exec ${dockerd} ${args}

+fi