@@ -148,6 +148,18 @@ RUN --mount=type=cache,sharing=locked,id=moby-cross-true-aptlib,target=/var/lib/

 FROM runtime-dev-cross-${CROSS} AS runtime-dev

+FROM base AS delve

+# DELVE_VERSION specifies the version of the Delve debugger binary

+# from the https://github.com/go-delve/delve repository.

+# It can be used to run Docker with a possibility of

+# attaching debugger to it.

+#

+ARG DELVE_VERSION=v1.8.1

+RUN --mount=type=cache,target=/root/.cache/go-build \

+    --mount=type=cache,target=/go/pkg/mod \

+        GOBIN=/build/ GO111MODULE=on go install "github.com/go-delve/delve/cmd/dlv@${DELVE_VERSION}" \

+     && /build/dlv --help

+

 FROM base AS tomll

 # GOTOML_VERSION specifies the version of the tomll binary to build and install

 # from the https://github.com/pelletier/go-toml repository. This binary is used

@@ -308,6 +320,7 @@ RUN pip3 install yamllint==1.26.1

 COPY --from=dockercli     /build/ /usr/local/cli

 COPY --from=frozen-images /build/ /docker-frozen-images

 COPY --from=swagger       /build/ /usr/local/bin/

+COPY --from=delve         /build/ /usr/local/bin/

 COPY --from=tomll         /build/ /usr/local/bin/

 COPY --from=gowinres      /build/ /usr/local/bin/

 COPY --from=tini          /build/ /usr/local/bin/

@@ -68,6 +68,7 @@ DOCKER_ENVS := \

 	-e DOCKER_TEST_HOST \

 	-e DOCKER_USERLANDPROXY \

 	-e DOCKERD_ARGS \

+	-e DELVE_PORT \

 	-e TEST_FORCE_VALIDATE \

 	-e TEST_INTEGRATION_DIR \

 	-e TEST_SKIP_INTEGRATION \

@@ -115,8 +116,9 @@ DOCKER_CONTAINER_NAME := $(if $(CONTAINER_NAME),--name $(CONTAINER_NAME),)

 DOCKER_IMAGE := docker-dev

 DOCKER_PORT_FORWARD := $(if $(DOCKER_PORT),-p "$(DOCKER_PORT)",)

+DELVE_PORT_FORWARD := $(if $(DELVE_PORT),-p "$(DELVE_PORT)",)

-DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD)

+DOCKER_FLAGS := $(DOCKER) run --rm -i --privileged $(DOCKER_CONTAINER_NAME) $(DOCKER_ENVS) $(DOCKER_MOUNT) $(DOCKER_PORT_FORWARD) $(DELVE_PORT_FORWARD)

 BUILD_APT_MIRROR := $(if $(DOCKER_BUILD_APT_MIRROR),--build-arg APT_MIRROR=$(DOCKER_BUILD_APT_MIRROR))

 export BUILD_APT_MIRROR

@@ -77,6 +77,8 @@ if [ -z "$mtu" ]; then

 	mtu=1500

 fi

+dockerd="${DOCKERD:-dockerd}"

+

 if [ -z "$_DOCKERD_ROOTLESS_CHILD" ]; then

 	_DOCKERD_ROOTLESS_CHILD=1

 	export _DOCKERD_ROOTLESS_CHILD

@@ -128,5 +130,6 @@ else

 		mount --rbind ${realpath_etc_ssl} /etc/ssl

 	fi

-	exec dockerd $@

+	# shellcheck disable=SC2086

+	exec $dockerd "$@"

 fi

@@ -7,3 +7,4 @@

  * [Configure Git for contributing](set-up-git.md)

  * [Work with a development container](set-up-dev-env.md)

  * [Run tests and test documentation](test.md)

+ * [Debugging the daemon](debug.md)

new file mode 100644

@@ -0,0 +1,66 @@

+### Debugging the daemon

+

+The Docker daemon inside the development container can be debugged with [Delve](https://github.com/go-delve/delve).

+

+Delve debugger listens on a port, which has to be exposed outside the development container.

+Also, in order to be able to debug the daemon, it has to be compiled with the debugging symbols.

+This can be done by launching the development container with the following command:

+

+```bash

+$ make BIND_DIR=. DOCKER_DEBUG=1 DELVE_PORT=127.0.0.1:2345:2345 shell

+```

+

+The `DOCKER_DEBUG` variable disables build optimizations, allowing to debug the binary,

+while `DELVE_PORT` publishes the specified port for use with the debugger.

+

+The `DELVE_PORT` variable accepts the port in the same format as Docker CLI's `--publish` (`-p`) option.

+This means that the port can be published in multiple ways:

+

+1. `DELVE_PORT=127.0.0.1:2345:2345` - exposes debugger on port `2345` for local development only (recommended)

+2. `DELVE_PORT=2345:2345` - exposes debugger on port `2345` without binding to specific IP

+3. `DELVE_PORT=2345` - same as above

+

+**IMPORTANT:** Publishing the port without binding it to localhost (127.0.0.1) might expose the debugger

+outside the developer's machine and is not recommended.

+

+## Running Docker daemon with debugger attached

+

+1. Run development container with build optimizations disabled and Delve enabled:

+   ```bash

+   $ make BIND_DIR=. DOCKER_DEBUG=1 DELVE_PORT=127.0.0.1:2345:2345 shell

+   ```

+2. Inside the development container:

+   1. Build the Docker daemon:

+      ```bash

+      $ ./hack/make.sh binary

+      ```

+   2. Install the newly-built daemon:

+      ```bash

+      $ make install

+      ```

+   3. Run the daemon through the `make.sh` script:

+      ```bash

+      $ ./hack/make.sh run

+      ```

+      The execution will stop and wait for the IDE or Delve CLI to attach

+      to the port, specified with the `DELVE_PORT` variable.

+      Once the IDE or Delve CLI is attached, the execution will continue.

+   

+## Debugging from IDE (on example of GoLand 2021.3)

+

+1. Open the project in GoLand

+2. Click *Add Configuration* on the taskbar

+   ![GoLand - adding configuration](images/goland_add_config.png)

+3. Create the *Go Remote* configuration. 

+   No changes are necessary, unless a different port is to be used.

+   ![GoLand - adding remote configuration](images/goland_debug_config.png)

+4. Run the Docker binary in the development container, as described in the previous section.

+   Make sure that the port in the `DELVE_PORT` variable corresponds to one, used in the *Go Remote* configuration.

+5. Run the *Go Remote* configuration.

+   The Docker daemon will continue execution inside the container and debugger will stop it on the breakpoints.

+   ![GoLand - run Go Remote configuration](images/goland_run_debug_config.png)

+

+## Where to go next

+

+Congratulations, you have experienced how to use Delve to debug the Docker daemon

+and how to configure an IDE to make use of it.

\ No newline at end of file

new file mode 100644

Binary files /dev/null and b/docs/contributing/images/goland_add_config.png differ

new file mode 100644

Binary files /dev/null and b/docs/contributing/images/goland_debug_config.png differ

new file mode 100644

Binary files /dev/null and b/docs/contributing/images/goland_run_debug_config.png differ

@@ -250,3 +250,4 @@ jobs can be triggered and re-ran by the Moby maintainers

 Congratulations, you have successfully completed the basics you need to

 understand the Moby test framework.

+In the next section you'll [learn how to debug Docker daemon, running inside the development container](debug.md).

@@ -8,6 +8,8 @@ if ! command -v dockerd &> /dev/null; then

 	false

 fi

+DOCKER_COMMAND="$(command -v dockerd)"

+

 DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs}

 DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true}

@@ -23,8 +25,11 @@ fi

 listen_port=2375

 if [ -n "$DOCKER_PORT" ]; then

-	IFS=':' read -r -a ports <<< "$DOCKER_PORT"

-	listen_port="${ports[-1]}"

+	listen_port="${DOCKER_PORT##*:}"

+fi

+

+if [ -n "$DELVE_PORT" ]; then

+	delve_listen_port="${DELVE_PORT##*:}"

 fi

 extra_params="$DOCKERD_ARGS"

@@ -36,7 +41,6 @@ if [ -n "$DOCKER_EXPERIMENTAL" ]; then

 	extra_params="$extra_params --experimental"

 fi

-dockerd="dockerd"

 socket=/var/run/docker.sock

 if [ -n "$DOCKER_ROOTLESS" ]; then

 	user="unprivilegeduser"

@@ -44,17 +48,48 @@ if [ -n "$DOCKER_ROOTLESS" ]; then

 	# shellcheck disable=SC2174

 	mkdir -p -m 700 "/tmp/docker-${uid}"

 	chown $user "/tmp/docker-${uid}"

-	dockerd="sudo -u $user -E XDG_RUNTIME_DIR=/tmp/docker-${uid} -E HOME=/home/${user} -- dockerd-rootless.sh"

 	socket=/tmp/docker-${uid}/docker.sock

 fi

-args="--debug \

-	--host "tcp://0.0.0.0:${listen_port}" --host "unix://${socket}" \

-	--storage-driver "${DOCKER_GRAPHDRIVER}" \

-	--userland-proxy="${DOCKER_USERLANDPROXY}" \

-	$storage_params \

-	$extra_params"

+# shellcheck disable=SC2206

+args=(

+	--debug

+	--host="tcp://0.0.0.0:${listen_port}"

+	--host="unix://${socket}"

+	--storage-driver="${DOCKER_GRAPHDRIVER}"

+	--userland-proxy="${DOCKER_USERLANDPROXY}"

+	$storage_params

+	$extra_params

+)

+

+dockerd=("$DOCKER_COMMAND")

+

+if [ -n "$DELVE_PORT" ]; then

+	dockerd=(

+		dlv

+		--listen="0.0.0.0:$delve_listen_port"

+		--headless=true

+		--log

+		--api-version=2

+		--only-same-user=false

+		--check-go-version=false

+		--accept-multiclient

+		exec "${dockerd[@]}" --

+	)

+fi

+

+if [ -n "$DOCKER_ROOTLESS" ]; then

+	dockerd=(

+		sudo -u "$user"

+		-E DOCKERD="${dockerd[*]}"

+		-E XDG_RUNTIME_DIR="/tmp/docker-${uid}"

+		-E XDG_CONFIG_HOME="/home/${user}/.config"

+		-E HOME="/home/${user}"

+		--

+		dockerd-rootless.sh

+	)

+fi

-echo "${dockerd} ${args}"

+set -x

 # shellcheck disable=SC2086

-exec "${dockerd}" ${args}

+exec "${dockerd[@]}" "${args[@]}"