@@ -232,6 +232,7 @@

         '{{ devstack_log_dir }}/dstat-csv.log': logs

         '{{ devstack_log_dir }}/devstacklog.txt': logs

         '{{ devstack_log_dir }}/devstacklog.txt.summary': logs

+        '{{ devstack_log_dir }}/tcpdump.pcap': logs

         '{{ devstack_full_log}}': logs

         '{{ stage_dir }}/verify_tempest_conf.log': logs

         '{{ stage_dir }}/apache': logs

new file mode 100644

@@ -0,0 +1,46 @@

+=====================

+System-wide debugging

+=====================

+

+A lot can go wrong during a devstack run, and there are a few inbuilt

+tools to help you.

+

+dstat

+-----

+

+Enable the ``dstat`` service to produce performance logs during the

+devstack run.  These will be logged to the journal and also as a CSV

+file.

+

+memory_tracker

+--------------

+

+The ``memory_tracker`` service periodically monitors RAM usage and

+provides consumption output when available memory is seen to be

+falling (i.e. processes are consuming memory).  It also provides

+output showing locked (unswappable) memory.

+

+tcpdump

+-------

+

+Enable the ``tcpdump`` service to run a background tcpdump.  You must

+set the ``TCPDUMP_ARGS`` variable to something suitable (there is no

+default).  For example, to trace iSCSI communication during a job in

+the OpenStack gate and copy the result into the log output, you might

+use:

+

+.. code-block:: yaml

+

+   job:

+     name: devstack-job

+     parent: devstack

+     vars:

+       devstack_services:

+         tcpdump: true

+       devstack_localrc:

+         TCPDUMP_ARGS: "-i any tcp port 3260"

+       zuul_copy_output:

+         '{{ devstack_log_dir }}/tcpdump.pcap': logs

+

+

+

new file mode 100644

@@ -0,0 +1,43 @@

+#!/bin/bash

+#

+# lib/tcpdump

+# Functions to start and stop a tcpdump

+

+# Dependencies:

+#

+# - ``functions`` file

+

+# ``stack.sh`` calls the entry points in this order:

+#

+# - start_tcpdump

+# - stop_tcpdump

+

+# Save trace setting

+_XTRACE_TCPDUMP=$(set +o | grep xtrace)

+set +o xtrace

+

+TCPDUMP_OUTPUT=${TCPDUMP_OUTPUT:-$LOGDIR/tcpdump.pcap}

+

+# e.g. for iscsi

+#  "-i any tcp port 3260"

+TCPDUMP_ARGS=${TCPDUMP_ARGS:-""}

+

+# start_tcpdump() - Start running processes

+function start_tcpdump {

+    # Run a tcpdump with given arguments and save the packet capture

+    if is_service_enabled tcpdump; then

+        if [[ -z "${TCPDUMP_ARGS}" ]]; then

+            die $LINENO "The tcpdump service requires TCPDUMP_ARGS to be set"

+        fi

+        touch ${TCPDUMP_OUTPUT}

+        run_process tcpdump "/usr/sbin/tcpdump -w $TCPDUMP_OUTPUT $TCPDUMP_ARGS" root root

+    fi

+}

+

+# stop_tcpdump() stop tcpdump process

+function stop_tcpdump {

+    stop_process tcpdump

+}

+

+# Restore xtrace

+$_XTRACE_TCPDUMP

@@ -614,6 +614,7 @@ source $TOP_DIR/lib/swift

 source $TOP_DIR/lib/neutron

 source $TOP_DIR/lib/ldap

 source $TOP_DIR/lib/dstat

+source $TOP_DIR/lib/tcpdump

 source $TOP_DIR/lib/etcd3

 # Extras Source

@@ -1053,6 +1054,12 @@ fi

 # A better kind of sysstat, with the top process per time slice

 start_dstat

+# Run a background tcpdump for debugging

+# Note: must set TCPDUMP_ARGS with the enabled service

+if is_service_enabled tcpdump; then

+    start_tcpdump

+fi

+

 # Etcd

 # -----