97fcc7b2 |
#!/usr/bin/env python
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
|
e7361775 |
|
97fcc7b2 |
"""Dump the state of the world for post mortem."""
|
09949e0d |
from __future__ import print_function
|
97fcc7b2 |
import argparse
import datetime |
57d79673 |
from distutils import spawn |
737e9420 |
import fnmatch |
97fcc7b2 |
import os
import os.path |
99440f9d |
import subprocess |
97fcc7b2 |
import sys
|
ef219bfc |
GMR_PROCESSES = (
'nova-compute',
'neutron-dhcp-agent',
'neutron-l3-agent',
'neutron-linuxbridge-agent',
'neutron-metadata-agent',
'neutron-openvswitch-agent', |
d8682dbd |
'cinder-volume', |
ef219bfc |
)
|
97fcc7b2 |
def get_options():
parser = argparse.ArgumentParser(
description='Dump world state for debugging')
parser.add_argument('-d', '--dir',
default='.',
help='Output directory for worlddump') |
ac9313e5 |
parser.add_argument('-n', '--name',
default='',
help='Additional name to tag into file') |
97fcc7b2 |
return parser.parse_args()
|
ac9313e5 |
def filename(dirname, name=""): |
97fcc7b2 |
now = datetime.datetime.utcnow() |
ac9313e5 |
fmt = "worlddump-%Y-%m-%d-%H%M%S"
if name:
fmt += "-" + name
fmt += ".txt"
return os.path.join(dirname, now.strftime(fmt)) |
97fcc7b2 |
def warn(msg): |
e7361775 |
print("WARN: %s" % msg) |
97fcc7b2 |
|
60a14057 |
def _dump_cmd(cmd): |
e7361775 |
print(cmd)
print("-" * len(cmd))
print() |
99440f9d |
try:
subprocess.check_call(cmd, shell=True) |
e7361775 |
print() |
7976aacd |
except subprocess.CalledProcessError as e: |
e7361775 |
print("*** Failed to run '%(cmd)s': %(err)s" % {'cmd': cmd, 'err': e}) |
60a14057 |
|
57d79673 |
def _find_cmd(cmd):
if not spawn.find_executable(cmd): |
e7361775 |
print("*** %s not found: skipping" % cmd) |
57d79673 |
return False
return True
|
60a14057 |
def _header(name): |
e7361775 |
print()
print(name)
print("=" * len(name))
print() |
60a14057 |
|
578459f8 |
def _bridge_list(): |
6ba17f7d |
process = subprocess.Popen(['sudo', 'ovs-vsctl', 'list-br'],
stdout=subprocess.PIPE) |
578459f8 |
stdout, _ = process.communicate()
return stdout.split()
|
6099401e |
# This method gets a max openflow version supported by openvswitch.
# For example 'ovs-ofctl --version' displays the following:
#
# ovs-ofctl (Open vSwitch) 2.0.2
# Compiled Dec 9 2015 14:08:08
# OpenFlow versions 0x1:0x4
# |
2bd2568b |
# The above shows that openvswitch supports from OpenFlow10 to OpenFlow13. |
6099401e |
# This method gets max version searching 'OpenFlow versions 0x1:0x'.
# And return a version value converted to an integer type.
def _get_ofp_version():
process = subprocess.Popen(['ovs-ofctl', '--version'], stdout=subprocess.PIPE)
stdout, _ = process.communicate()
find_str = 'OpenFlow versions 0x1:0x'
offset = stdout.find(find_str)
return int(stdout[offset + len(find_str):-1]) - 1
|
97fcc7b2 |
def disk_space():
# the df output |
60a14057 |
_header("File System Summary")
|
97fcc7b2 |
dfraw = os.popen("df -Ph").read()
df = [s.split() for s in dfraw.splitlines()]
for fs in df:
try:
if int(fs[4][:-1]) > 95:
warn("Device %s (%s) is %s full, might be an issue" % (
fs[0], fs[5], fs[4]))
except ValueError:
# if it doesn't look like an int, that's fine
pass
|
e7361775 |
print(dfraw) |
97fcc7b2 |
|
2da606da |
def ebtables_dump(): |
5c5e0866 |
tables = ['filter', 'nat', 'broute'] |
2da606da |
_header("EB Tables Dump") |
57d79673 |
if not _find_cmd('ebtables'):
return |
5c5e0866 |
for table in tables:
_dump_cmd("sudo ebtables -t %s -L" % table) |
2da606da |
|
168b7c22 |
def iptables_dump():
tables = ['filter', 'nat', 'mangle'] |
60a14057 |
_header("IP Tables Dump")
|
168b7c22 |
for table in tables: |
60a14057 |
_dump_cmd("sudo iptables --line-numbers -L -nv -t %s" % table)
|
72c34ee1 |
def _netns_list():
process = subprocess.Popen(['ip', 'netns'], stdout=subprocess.PIPE)
stdout, _ = process.communicate() |
c6e6939e |
# NOTE(jlvillal): Sometimes 'ip netns list' can return output like:
# qrouter-0805fd7d-c493-4fa6-82ca-1c6c9b23cd9e (id: 1)
# qdhcp-bb2cc6ae-2ae8-474f-adda-a94059b872b5 (id: 0)
output = [x.split()[0] for x in stdout.splitlines()]
return output |
72c34ee1 |
|
60a14057 |
def network_dump():
_header("Network Dump")
_dump_cmd("brctl show") |
c396355b |
ip_cmds = ["neigh", "addr", "link", "route"] |
72c34ee1 |
for cmd in ip_cmds + ['netns']:
_dump_cmd("ip %s" % cmd)
for netns_ in _netns_list():
for cmd in ip_cmds:
args = {'netns': netns_, 'cmd': cmd}
_dump_cmd('sudo ip netns exec %(netns)s ip %(cmd)s' % args) |
168b7c22 |
|
c1b7cb11 |
def ovs_dump():
_header("Open vSwitch Dump")
|
57d79673 |
# NOTE(cdent): If we're not using neutron + ovs these commands
# will not be present so
if not _find_cmd('ovs-vsctl'):
return
|
578459f8 |
bridges = _bridge_list() |
6099401e |
ofctl_cmds = ('show', 'dump-ports-desc', 'dump-ports', 'dump-flows')
ofp_max = _get_ofp_version()
vers = 'OpenFlow10' |
578459f8 |
for i in range(1, ofp_max + 1): |
6099401e |
vers += ',OpenFlow1' + str(i) |
c1b7cb11 |
_dump_cmd("sudo ovs-vsctl show") |
6099401e |
for ofctl_cmd in ofctl_cmds:
for bridge in bridges:
args = {'vers': vers, 'cmd': ofctl_cmd, 'bridge': bridge}
_dump_cmd("sudo ovs-ofctl --protocols=%(vers)s %(cmd)s %(bridge)s" % args) |
c1b7cb11 |
|
97fcc7b2 |
def process_list(): |
60a14057 |
_header("Process Listing")
_dump_cmd("ps axo "
"user,ppid,pid,pcpu,pmem,vsz,rss,tty,stat,start,time,args") |
97fcc7b2 |
|
737e9420 |
def compute_consoles():
_header("Compute consoles")
for root, dirnames, filenames in os.walk('/opt/stack'):
for filename in fnmatch.filter(filenames, 'console.log'):
fullpath = os.path.join(root, filename)
_dump_cmd("sudo cat %s" % fullpath)
|
ef219bfc |
def guru_meditation_reports():
for service in GMR_PROCESSES:
_header("%s Guru Meditation Report" % service) |
3a9df1da |
|
ef219bfc |
try:
subprocess.check_call(['pgrep', '-f', service])
except subprocess.CalledProcessError:
print("Skipping as %s does not appear to be running" % service)
continue |
3a9df1da |
|
ef219bfc |
_dump_cmd("killall -e -USR2 %s" % service)
print("guru meditation report in %s log" % service) |
2ebe993b |
|
bfcc760b |
def var_core():
if os.path.exists('/var/core'):
_header("/var/core dumps")
# NOTE(ianw) : see DEBUG_LIBVIRT_COREDUMPS. We could think
# about getting backtraces out of these. There are other
# tools out there that can do that sort of thing though.
_dump_cmd("ls -ltrah /var/core")
|
97fcc7b2 |
def main():
opts = get_options() |
ac9313e5 |
fname = filename(opts.dir, opts.name) |
e7361775 |
print("World dumping... see %s for details" % fname) |
97fcc7b2 |
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
with open(fname, 'w') as f:
os.dup2(f.fileno(), sys.stdout.fileno())
disk_space()
process_list() |
60a14057 |
network_dump() |
c1b7cb11 |
ovs_dump() |
168b7c22 |
iptables_dump() |
2da606da |
ebtables_dump() |
737e9420 |
compute_consoles() |
ef219bfc |
guru_meditation_reports() |
bfcc760b |
var_core() |
97fcc7b2 |
if __name__ == '__main__':
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1) |