#!/usr/bin/python2
# The MIT License (MIT)
#
# Copyright (c) 2015 Lance W. Shelton
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
A better way to watch /proc/interrupts, especially on large NUMA machines with
so many CPUs that /proc/interrupts is wider than the screen. Press '0'-'9'
for node views, 't' for node totals
"""
__version__ = '1.0.1-pre'
import os
import sys
import tty
import termios
import time
from time import sleep
import subprocess
from optparse import OptionParser
import thread
import threading
KEYEVENT = threading.Event()
def gen_numa(numafile):
"""Generate NUMA info"""
cpunodes = {}
numacores = {}
err_str = ""
try:
if not numafile:
temp = subprocess.Popen(['numactl', '--hardware'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(output, error) = temp.communicate()
temp.wait()
if error:
print("NUMACTL ERROR:")
print(error)
exit(1)
else:
numa_file = open(numafile, 'r')
output = numa_file.read()
numa_file.close()
output = output.split("\n")
for line in output:
arr = line.split()
if len(arr) < 3:
continue
if arr[0] == "node" and arr[2] == "cpus:":
node = arr[1]
numacores[node] = arr[3:]
for core in arr[3:]:
cpunodes[core] = node
continue
return numacores, cpunodes
except (OSError, IOError) as err:
if err.errno == 2: # No such file or directory
if numafile:
err_str = " (does '" + numafile + "' exist?)"
else:
err_str = err.strerror + " (is numactl installed?)"
print("ERROR: " + err.strerror + err_str)
exit(1)
# input character, passed between threads
INCHAR = ''
def wait_for_input():
"""Get a single character of input, validate"""
global INCHAR
acceptable_keys = ['0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', '0', 't']
while True:
key = sys.stdin.read(1)
# simple just to exit on any invalid input
if not key in acceptable_keys:
thread.interrupt_main()
# set the new input and notify the main thread
INCHAR = key
KEYEVENT.set()
def filter_found(name, filter_list):
"""Check if IRQ name matches anything in the filter list"""
for filt in filter_list:
if filt in name:
return True
return False
def display_itop(batch, seconds, rowcnt, iterations, sort, totals, dispnode,
zero, filters, file1, file2, overall, numafile):
"""Main I/O loop"""
irqs = {}
cpunodes = {}
numacores = {}
loops = 0
width = len('NODEXX')
if not file1 and batch:
print("Running in batch mode")
elif not file1:
print("interactive commands -- "
"t: view totals, 0-9: view node, any other key: quit\r")
if file1:
intr_filename = file1
else:
intr_filename = '/proc/interrupts'
while True:
# Grab the new display type at a time when nothing is in flux
if KEYEVENT.isSet():
KEYEVENT.clear()
dispnode = INCHAR if INCHAR in numacores else '-1'
with open(intr_filename, 'r') as intr_file:
header = intr_file.readline()
cpus = []
for name in header.split():
num = name[3:]
cpus.append(num)
# Only query the numa information when something is missing.
# This is effectively the first time and when any disabled CPUs
# are enabled
if not num in cpunodes:
numacores, cpunodes = gen_numa(numafile)
for line in intr_file.readlines():
vals = line.split()
irqnum = vals[0].rstrip(':')
# Optionally exclude rows that are not an IRQ number
if totals is None:
try:
num = int(irqnum)
except ValueError:
continue
irq = {}
irq['cpus'] = [int(x) for x in vals[1:len(cpus)+1]]
irq['oldcpus'] = (irqs[irqnum]['cpus'] if irqnum in irqs
else [0] * len(cpus))
irq['name'] = ' '.join(vals[len(cpus)+1:])
irq['oldsum'] = irqs[irqnum]['sum'] if irqnum in irqs else 0
irq['sum'] = sum(irq['cpus'])
irq['num'] = irqnum
for node in numacores:
oldkey = 'oldsum' + node
key = 'sum' + node
irq[oldkey] = (irqs[irqnum][key] if irqnum in irqs
and key in irqs[irqnum] else 0)
irq[key] = 0
for idx, val in enumerate(irq['cpus']):
key = 'sum' + cpunodes[cpus[idx]]
irq[key] = irq[key] + val if key in irq else val
# save old
irqs[irqnum] = irq
def sort_func(val):
"""Sort output"""
sortnum = -1
try:
sortnum = int(sort)
except ValueError:
pass
if sortnum >= 0:
for node in numacores:
if sortnum == int(node):
return val['sum' + node] - val['oldsum' + node]
if sort == 't':
return val['sum'] - val['oldsum']
if sort == 'i':
if val['num'].isdigit():
return int(val['num'])
return sys.maxsize
if sort == 'n':
return val['name']
raise Exception('Invalid sort type {}'.format(sort))
# reverse sort all IRQ count sorts
rev = sort not in ['i', 'n']
rows = sorted(irqs.values(), key=sort_func, reverse=rev)
# determine the width required for the count field
for idx, irq in enumerate(rows):
width = max(width, len(str(irq['sum'] - irq['oldsum'])))
if overall and loops > 0:
print("" + '\r')
if not file1 and (overall or loops > 0):
print(time.ctime() + '\r')
print("IRQs / " + str(seconds) + " second(s)" + '\r')
fmtstr = ('IRQ# %' + str(width) + 's') % 'TOTAL'
# node view header
if int(dispnode) >= 0:
node = 'NODE%s' % dispnode
fmtstr += (' %' + str(width) + 's ') % node
for idx, val in enumerate(irq['cpus']):
if cpunodes[cpus[idx]] == dispnode:
cpu = 'CPU%s' % cpus[idx]
fmtstr += (' %' + str(width) + 's ') % cpu
# top view header
else:
for node in sorted(numacores):
node = 'NODE%s' % node
fmtstr += (' %' + str(width) + 's ') % node
fmtstr += ' NAME'
if overall or loops > 0:
print(fmtstr + '\r')
displayed_rows = 0
for idx, irq in enumerate(rows):
if filters and not filter_found(irq['name'], filters):
continue
total = irq['sum'] - irq['oldsum']
if zero and not total:
continue
# IRQ# TOTAL
fmtstr = ('%4s %' + str(width) + 'd') % (irq['num'], total)
# node view
if int(dispnode) >= 0:
oldnodesum = 'oldsum' + dispnode
nodesum = 'sum' + dispnode
nodecnt = irq[nodesum] - irq[oldnodesum]
if zero and not nodecnt:
continue
fmtstr += (' %' + str(width) + 's ') % str(nodecnt)
for cpu, val in enumerate(irq['cpus']):
if cpunodes[cpus[cpu]] == dispnode:
fmtstr += ((' %' + str(width) + 's ') %
str(irq['cpus'][cpu] - irq['oldcpus'][cpu]))
# top view
else:
for node in sorted(numacores):
oldnodesum = 'oldsum' + node
nodesum = 'sum' + node
nodecnt = irq[nodesum] - irq[oldnodesum]
fmtstr += ((' %' + str(width) + 's ') % str(nodecnt))
fmtstr += ' ' + irq['name']
if overall or loops > 0:
print(fmtstr + '\r')
displayed_rows += 1
if displayed_rows == rowcnt:
break
# Update field widths after the first iteration. Data changes
# significantly between the all-time stats and the interval stats, so
# this compresses the fields quite a bit. Updating every iteration
# is too jumpy.
if loops == 0:
width = len('NODEXX')
loops += 1
if loops == iterations:
break
if file2 and loops == 1:
intr_filename = file2
continue
# thread.interrupt_main() does not seem to interrupt a sleep, so break
# it into tenth-of-a-second sleeps to improve user response time on exit
for _ in range(0, seconds * 10):
sleep(.1)
def main(args):
"""Parse arguments, call main loop"""
parser = OptionParser(description=__doc__)
parser.add_option("-b", "--batch", action="store_true",
help="run under batch mode")
parser.add_option("-i", "--iterations", default='-1',
help="iterations to run")
parser.add_option("-n", "--node", default='-1',
help="view a single node")
parser.add_option("-r", "--rows", default='10',
help="rows to display (default 10)")
parser.add_option("-s", "--sort", default='t',
help="column to sort on ('t':total, 'n': name, "
"'i':IRQ number, '1':node1, etc) (default: 't')")
parser.add_option("-t", "--time", default='5',
help="update interval in seconds")
parser.add_option("-z", "--zero", action="store_true",
help="exclude inactive IRQs")
parser.add_option("-v", "--version", action="store_true",
help="get version")
parser.add_option("--filter", default="",
help="filter IRQs based on name matching comma "
"separated filters")
parser.add_option("--totals", action="store_true",
help="include total rows")
parser.add_option("-f", "--file1", default="",
help="read a file instead of /proc/interrupts")
parser.add_option("-F", "--file2", default="",
help="no monitoring. Compare the samples from two files "
"instead")
parser.add_option("-O", "--overall", action="store_true",
help="print all-time stats at the beginning")
parser.add_option("-N", "--numafile", default="",
help="read the NUMA info from a file, instead of "
"calling numactl")
options = parser.parse_args(args)[0]
if options.version:
print __version__
return 0
if options.filter:
options.filter = options.filter.split(',')
else:
options.filter = []
# If file is specified, no iterations
if options.file1:
options.iterations = 1
options.batch = True
if options.file2:
if not options.file1:
print("ERROR: --file2 requires --file1")
return -1
options.iterations = 2
if options.file1 and not options.file2:
options.overall = True
# Set the terminal to unbuffered, to catch a single keypress
if not options.batch:
out = sys.stdin.fileno()
old_settings = termios.tcgetattr(out)
tty.setraw(sys.stdin.fileno())
# input thread
thread.start_new_thread(wait_for_input, tuple())
else:
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
try:
display_itop(options.batch, int(options.time), int(options.rows),
int(options.iterations), options.sort, options.totals,
options.node, options.zero, options.filter, options.file1,
options.file2, options.overall, options.numafile)
except (KeyboardInterrupt, SystemExit):
pass
finally:
if not options.batch:
termios.tcsetattr(out, termios.TCSADRAIN, old_settings)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
|