tools/worlddump.py
97fcc7b2
 #!/usr/bin/env python
 #
 # Copyright 2014 Hewlett-Packard Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
 
e7361775
 
97fcc7b2
 """Dump the state of the world for post mortem."""
 
09949e0d
 from __future__ import print_function
 
97fcc7b2
 import argparse
 import datetime
57d79673
 from distutils import spawn
737e9420
 import fnmatch
97fcc7b2
 import os
 import os.path
99440f9d
 import subprocess
97fcc7b2
 import sys
 
 
ef219bfc
 GMR_PROCESSES = (
     'nova-compute',
     'neutron-dhcp-agent',
     'neutron-l3-agent',
     'neutron-linuxbridge-agent',
     'neutron-metadata-agent',
     'neutron-openvswitch-agent',
d8682dbd
     'cinder-volume',
ef219bfc
 )
 
 
97fcc7b2
 def get_options():
     parser = argparse.ArgumentParser(
         description='Dump world state for debugging')
     parser.add_argument('-d', '--dir',
                         default='.',
                         help='Output directory for worlddump')
ac9313e5
     parser.add_argument('-n', '--name',
                         default='',
                         help='Additional name to tag into file')
97fcc7b2
     return parser.parse_args()
 
 
ac9313e5
 def filename(dirname, name=""):
97fcc7b2
     now = datetime.datetime.utcnow()
ac9313e5
     fmt = "worlddump-%Y-%m-%d-%H%M%S"
     if name:
         fmt += "-" + name
     fmt += ".txt"
     return os.path.join(dirname, now.strftime(fmt))
97fcc7b2
 
 
 def warn(msg):
e7361775
     print("WARN: %s" % msg)
97fcc7b2
 
 
60a14057
 def _dump_cmd(cmd):
e7361775
     print(cmd)
     print("-" * len(cmd))
     print()
99440f9d
     try:
         subprocess.check_call(cmd, shell=True)
e7361775
         print()
7976aacd
     except subprocess.CalledProcessError as e:
e7361775
         print("*** Failed to run '%(cmd)s': %(err)s" % {'cmd': cmd, 'err': e})
60a14057
 
 
57d79673
 def _find_cmd(cmd):
     if not spawn.find_executable(cmd):
e7361775
         print("*** %s not found: skipping" % cmd)
57d79673
         return False
     return True
 
 
60a14057
 def _header(name):
e7361775
     print()
     print(name)
     print("=" * len(name))
     print()
60a14057
 
 
578459f8
 def _bridge_list():
6ba17f7d
     process = subprocess.Popen(['sudo', 'ovs-vsctl', 'list-br'],
                                stdout=subprocess.PIPE)
578459f8
     stdout, _ = process.communicate()
     return stdout.split()
 
 
6099401e
 # This method gets a max openflow version supported by openvswitch.
 # For example 'ovs-ofctl --version' displays the following:
 #
 #     ovs-ofctl (Open vSwitch) 2.0.2
 #     Compiled Dec  9 2015 14:08:08
 #     OpenFlow versions 0x1:0x4
 #
2bd2568b
 # The above shows that openvswitch supports from OpenFlow10 to OpenFlow13.
6099401e
 # This method gets max version searching 'OpenFlow versions 0x1:0x'.
 # And return a version value converted to an integer type.
 def _get_ofp_version():
     process = subprocess.Popen(['ovs-ofctl', '--version'], stdout=subprocess.PIPE)
     stdout, _ = process.communicate()
     find_str = 'OpenFlow versions 0x1:0x'
     offset = stdout.find(find_str)
     return int(stdout[offset + len(find_str):-1]) - 1
 
 
97fcc7b2
 def disk_space():
     # the df output
60a14057
     _header("File System Summary")
 
97fcc7b2
     dfraw = os.popen("df -Ph").read()
     df = [s.split() for s in dfraw.splitlines()]
     for fs in df:
         try:
             if int(fs[4][:-1]) > 95:
                 warn("Device %s (%s) is %s full, might be an issue" % (
                     fs[0], fs[5], fs[4]))
         except ValueError:
             # if it doesn't look like an int, that's fine
             pass
 
e7361775
     print(dfraw)
97fcc7b2
 
 
2da606da
 def ebtables_dump():
5c5e0866
     tables = ['filter', 'nat', 'broute']
2da606da
     _header("EB Tables Dump")
57d79673
     if not _find_cmd('ebtables'):
         return
5c5e0866
     for table in tables:
         _dump_cmd("sudo ebtables -t %s -L" % table)
2da606da
 
 
168b7c22
 def iptables_dump():
     tables = ['filter', 'nat', 'mangle']
60a14057
     _header("IP Tables Dump")
 
168b7c22
     for table in tables:
60a14057
         _dump_cmd("sudo iptables --line-numbers -L -nv -t %s" % table)
 
 
72c34ee1
 def _netns_list():
     process = subprocess.Popen(['ip', 'netns'], stdout=subprocess.PIPE)
     stdout, _ = process.communicate()
c6e6939e
     # NOTE(jlvillal): Sometimes 'ip netns list' can return output like:
     #   qrouter-0805fd7d-c493-4fa6-82ca-1c6c9b23cd9e (id: 1)
     #   qdhcp-bb2cc6ae-2ae8-474f-adda-a94059b872b5 (id: 0)
     output = [x.split()[0] for x in stdout.splitlines()]
     return output
72c34ee1
 
 
60a14057
 def network_dump():
     _header("Network Dump")
 
     _dump_cmd("brctl show")
c396355b
     ip_cmds = ["neigh", "addr", "link", "route"]
72c34ee1
     for cmd in ip_cmds + ['netns']:
         _dump_cmd("ip %s" % cmd)
     for netns_ in _netns_list():
         for cmd in ip_cmds:
             args = {'netns': netns_, 'cmd': cmd}
             _dump_cmd('sudo ip netns exec %(netns)s ip %(cmd)s' % args)
168b7c22
 
 
c1b7cb11
 def ovs_dump():
     _header("Open vSwitch Dump")
 
57d79673
     # NOTE(cdent): If we're not using neutron + ovs these commands
     # will not be present so
     if not _find_cmd('ovs-vsctl'):
         return
 
578459f8
     bridges = _bridge_list()
6099401e
     ofctl_cmds = ('show', 'dump-ports-desc', 'dump-ports', 'dump-flows')
     ofp_max = _get_ofp_version()
     vers = 'OpenFlow10'
578459f8
     for i in range(1, ofp_max + 1):
6099401e
         vers += ',OpenFlow1' + str(i)
c1b7cb11
     _dump_cmd("sudo ovs-vsctl show")
6099401e
     for ofctl_cmd in ofctl_cmds:
         for bridge in bridges:
             args = {'vers': vers, 'cmd': ofctl_cmd, 'bridge': bridge}
             _dump_cmd("sudo ovs-ofctl --protocols=%(vers)s %(cmd)s %(bridge)s" % args)
c1b7cb11
 
 
97fcc7b2
 def process_list():
60a14057
     _header("Process Listing")
     _dump_cmd("ps axo "
               "user,ppid,pid,pcpu,pmem,vsz,rss,tty,stat,start,time,args")
97fcc7b2
 
 
737e9420
 def compute_consoles():
     _header("Compute consoles")
     for root, dirnames, filenames in os.walk('/opt/stack'):
         for filename in fnmatch.filter(filenames, 'console.log'):
             fullpath = os.path.join(root, filename)
             _dump_cmd("sudo cat %s" % fullpath)
 
 
ef219bfc
 def guru_meditation_reports():
     for service in GMR_PROCESSES:
         _header("%s Guru Meditation Report" % service)
3a9df1da
 
ef219bfc
         try:
             subprocess.check_call(['pgrep', '-f', service])
         except subprocess.CalledProcessError:
             print("Skipping as %s does not appear to be running" % service)
             continue
3a9df1da
 
ef219bfc
         _dump_cmd("killall -e -USR2 %s" % service)
         print("guru meditation report in %s log" % service)
2ebe993b
 
 
bfcc760b
 def var_core():
     if os.path.exists('/var/core'):
         _header("/var/core dumps")
         # NOTE(ianw) : see DEBUG_LIBVIRT_COREDUMPS.  We could think
         # about getting backtraces out of these.  There are other
         # tools out there that can do that sort of thing though.
         _dump_cmd("ls -ltrah /var/core")
 
97fcc7b2
 def main():
     opts = get_options()
ac9313e5
     fname = filename(opts.dir, opts.name)
e7361775
     print("World dumping... see %s for details" % fname)
97fcc7b2
     sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
     with open(fname, 'w') as f:
         os.dup2(f.fileno(), sys.stdout.fileno())
         disk_space()
         process_list()
60a14057
         network_dump()
c1b7cb11
         ovs_dump()
168b7c22
         iptables_dump()
2da606da
         ebtables_dump()
737e9420
         compute_consoles()
ef219bfc
         guru_meditation_reports()
bfcc760b
         var_core()
97fcc7b2
 
 
 if __name__ == '__main__':
     try:
         sys.exit(main())
     except KeyboardInterrupt:
         sys.exit(1)