Deftly bridging the gap between the technical and the spiritual for over 16 years
parse_pl
#!/usr/bin/perl
use RRDs;
# 03/22/04 - David Krider
# This program will parse a report of sar stats generated like this:
#
# for i in /var/log/sa/<some glob to just get the binary logs>
# do
# sar -f $i -A -h >> report
# done
#
# Then just call this program as:
#
# ./stats.pl report
#
# It will generate a rrdtool (q.v.) database and a graph for most of
# the interesting stats that sar will create, including a one for
# each network interface and each disk device. (These may or may not
# be easily recognizable as their physical components, but that's an
# exercise left to the reader.) It also creates a simple web page
# that "wraps" the graphs to make them easy to view.
#
# This program is designed to complement something like MRTG or cacti,
# not replace it. It is my hope that should it need to be expanded,
# it would be easy to do considering the tool (rrdtool) it's based on.
#
# I've tried to keep this program generic because it seems that most
# implementations of ``sar'' can produce the sort of output like
# "-h" will produce on Linux. It is my hope to modify this program
# as needed to account for every *nix I encounter.
# Customize as desired
$height = 150;
$width = 600;
$input_file = $ARGV[0];
# May need a OS-based case statement here to capture the correct
# counters, though future use of these depends on the graphs that
# one would consider useful. Mainly, these are here for getting
# "overview" graphs, where you can loop over each counter in a
# category and see which ones are interesting.
@cpu_counters = ("pct_user", "pct_nice", "pct_system");
@io_counters = ("tps", "rtps", "wtps", "breadps", "bwrtnps");
@paging_counters = ("pgpginps", "pgpgoutps", "activepg", "inadtypg",
"inaclnpg", "inatarpg");
@process_stats = ("procps");
@disk_counters = ("tps", "sectps");
@network_counters = ("rxpckps", "txpckps", "rxbytps", "txbytps", "rxcmpps",
"txcmpps", "rxmcstps", "rxerrps", "txerrps", "collps", "rxdropps",
"txdropps", "txcarrps", "rxframps", "rxfifops", "txfifops", "totsck",
"tcpdck", "udpsck", "rawsck", "ip_frag");
@load_counters = ("runq_sz", "plist_sz", "ldavg_1", "ldavg_5");
#@swap_counters = ("kbmemfree", "kbmemused", "pct_memused", "kbmemshrd",
# "kbbuffers", "kbcached", "kbswpfree", "kbswpused", "pct_swpused");
@swap_counters = ("kbmemfree", "kbmemused", "kbmemshrd",
"kbbuffers", "kbcached", "kbswpfree", "kbswpused");
@memory_counters = ("frmpgps", "shmpgps", "bufpgps", "campgps");
# Again, customize as desired
@colors = ("ff0000", "00ff00", "0000ff", "ff00ff", "ffff00", "00ffff",
"0f0f0f", "f00000", "00f000", "0000f0");
# I tried making a hash of hashes out of this data, and it worked
# really well, until I saw that some "devices" didn't exist at some
# timestamps. Looping over a HoH would then produce fewer than
# expected values for loading in the rrd. I switched to using ordered
# arrays to loop over the hash, rather than using the keys of the
# hash by itself, and I found a bonus: I don't have to handle an
# exception! My code outputs nothing (e.g. #:#::::#) for the rrdupdate
# command, which is just fine by it.
open (REPORT, "<$input_file");
while (<REPORT>) {
chomp ($_);
($hostname, $interval, $timestamp, $device, $field, $value) =
split (/\s+/, $_);
# SuSE...
unless ($interval > 6100) {
# rrdtool can't deal with %'s in the field name
if ($field =~ /.*%.*/) {
$field =~ s/%/pct_/;
}
# or /'s, which stand for "per"
if ($field =~ /.*\/.*/) {
$field =~ s/\//p/;
}
# or -'s
if ($field =~ /.*-.*/) {
$field =~ s/-/_/;
}
if ($device eq "-") {
$device = "all";
}
if ($device =~ /.*-.*/) {
$device =~ s/-/_/;
}
$counter = $field . "_" . $device;
push (@timestamps, $timestamp);
push (@counters, $counter);
push (@devices, $device);
$data{$timestamp}{$counter} = $value;
$found = 0;
foreach $item (@network_counters) {
if ($item eq $field) {
push (@network_devices, $device);
$found = 1;
last;
}
}
$found = 0;
foreach $item (@disk_counters) {
if ($item eq $field) {
push (@disk_devices, $device);
$found = 1;
last;
}
}
} # unless
} # line
close REPORT;
@timestamps = mysort (@timestamps);
@counters = mysort (@counters);
@network_devices = mysort (@network_devices);
@disk_devices = mysort (@disk_devices);
$rrd_file = $hostname . "_stats.rrd";
$html_file = $hostname . "_stats.html";
# Create the database to hold every sort of field in the data.
# The step size is 10 minutes, which is the default for sar's
# logs (on Linux, at least), and the "heartbeat" is 3 times this,
# with one entry which just keeps the data point-for-point for
# up to 1 week.
$start = $timestamps[0] - 1;
$end = $timestamps[$#timestamps];
print localtime($start) . " - " . localtime($end) . "\n";
# Old way...
#$command = "rrdtool create --start $start --step 600 $rrd_file";
#foreach $counter (@counters) {
# $command = $command . " DS:$counter:GAUGE:1800:U:U";
#}
#$command = $command . " RRA:LAST:0.5:1:12072";
#print "$command\n\n";
#$result = system ($command);
# Harsh lesson: don't use spacing in the "@parameters" pushing for
# fancy debugging printing. Doing so will confuse RRDs::create...
# AND NOT THROW AN EXCEPTION!
my @parameters;
foreach $counter (@counters) {
push (@parameters, "DS:$counter:GAUGE:1800:U:U");
}
push (@parameters, "RRA:LAST:0.5:1:12072");
RRDs::create ($rrd_file, "--start=$start", "--step=600", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
# Note that both levels are sorted. I don't know an *easy* way of
# lining up the fields in the way that they come out of sar, which
# would be nice. I'm simply going to have to do the sort here,
# because this is the only way to get the results out of the hash
# repeatably.
foreach $timestamp (@timestamps) {
$record = "$timestamp";
foreach $counter (@counters) {
$record .= ":" . $data{$timestamp}{$counter};
}
# Old way...
#$command = "rrdtool update $rrd_file $record";
#$result = system ($command);
RRDs::update ($rrd_file, $record);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
}
# Output the HTML file to "wrap" all of these graphs
open (HTMLFILE, ">$html_file");
print HTMLFILE "<head>\n";
print HTMLFILE "<title>Stats for $hostname</title>\n";
print HTMLFILE "</head>\n";
print HTMLFILE "<html>\n";
print HTMLFILE "<div style='text-align:center'>\n";
print HTMLFILE "<h1>Stats for $hostname</h1>\n";
print HTMLFILE "<h2>" . localtime($start) . " - " . localtime($end) . "</h2>\n";
# I seem to keep wanting the same sort of average, minimum, and
# maximum values for each counter I'm graphing. Those GPRINT's
# probably ought to be subroutine'd...
# Create CPU graph
#$png_file = $hostname . "_stats-cpu.png";
#$command = "rrdtool graph $png_file -s $start -e $end \\
# -h $height -w $width -t \"CPU Stats\" \\
# DEF:pct_user=$rrd_file:pct_user_all:LAST \\
# AREA:pct_user#0000ff:\"User\" \\
# GPRINT:pct_user:MIN:\" Minimum\\:%10.2lf\" \\
# GPRINT:pct_user:MAX:\"Maximum\\:%10.2lf\" \\
# GPRINT:pct_user:AVERAGE:\"Average\\:%10.2lf\\j\" \\
# DEF:pct_system=$rrd_file:pct_system_all:LAST \\
# AREA:pct_system#ff0000:\"System\" \\
# GPRINT:pct_system:MIN:\" Minimum\\:%10.2lf\" \\
# GPRINT:pct_system:MAX:\"Maximum\\:%10.2lf\" \\
# GPRINT:pct_system:AVERAGE:\"Average\\:%10.2lf\\j\" \\
# DEF:pct_nice=$rrd_file:pct_nice_all:LAST \\
# AREA:pct_nice#00ff00:\"Nice\" \\
# GPRINT:pct_nice:MIN:\" Minimum\\:%10.2lf\" \\
# GPRINT:pct_nice:MAX:\"Maximum\\:%10.2lf\" \\
# GPRINT:pct_nice:AVERAGE:\"Average\\:%10.2lf\\j\" \\
# CDEF:total=pct_system,pct_user,pct_nice,+,+ \\
# LINE1:total#444444:\"Total\" \\
# GPRINT:total:MIN:\" Minimum\\:%10.2lf\" \\
# GPRINT:total:MAX:\"Maximum\\:%10.2lf\" \\
# GPRINT:total:AVERAGE:\"Average\\:%10.2lf\\j\"";
#$result = system ($command);
#print HTMLFILE "<p><img src='$png_file'>\n";
$png_file = $hostname . "_stats-cpu.png";
@parameters = (
"DEF:pct_system=$rrd_file:pct_system_all:LAST",
"AREA:pct_system#ff0000:System",
"GPRINT:pct_system:MIN: Minimum\\:%10.2lf",
"GPRINT:pct_system:MAX:Maximum\\:%10.2lf",
"GPRINT:pct_system:AVERAGE:Average\\:%10.2lf\\j",
"DEF:pct_user=$rrd_file:pct_user_all:LAST",
"STACK:pct_user#0000ff:User",
"GPRINT:pct_user:MIN: Minimum\\:%10.2lf",
"GPRINT:pct_user:MAX:Maximum\\:%10.2lf",
"GPRINT:pct_user:AVERAGE:Average\\:%10.2lf\\j",
"DEF:pct_nice=$rrd_file:pct_nice_all:LAST",
"STACK:pct_nice#00ff00:Nice",
"GPRINT:pct_nice:MIN: Minimum\\:%10.2lf",
"GPRINT:pct_nice:MAX:Maximum\\:%10.2lf",
"GPRINT:pct_nice:AVERAGE:Average\\:%10.2lf\\j",
"CDEF:total=pct_system,pct_user,pct_nice,+,+",
"LINE1:total#444444:Total",
"GPRINT:total:MIN: Minimum\\:%10.2lf",
"GPRINT:total:MAX:Maximum\\:%10.2lf",
"GPRINT:total:AVERAGE:Average\\:%10.2lf\\j"
);
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "CPU Stats", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
# Create load graph
$png_file = $hostname . "_stats-load.png";
@parameters = (
"DEF:ldavg_1=$rrd_file:ldavg_1_all:LAST",
"LINE1:ldavg_1#ff0000:1-min Load Avg",
"GPRINT:ldavg_1:MIN: Minimum\\:%10.2lf",
"GPRINT:ldavg_1:MAX:Maximum\\:%10.2lf",
"GPRINT:ldavg_1:AVERAGE:Average\\:%10.2lf\\j",
"DEF:ldavg_5=$rrd_file:ldavg_5_all:LAST",
"LINE2:ldavg_5#ffff00:5-min Load Avg",
"GPRINT:ldavg_5:MIN: Minimum\\:%10.2lf",
"GPRINT:ldavg_5:MAX:Maximum\\:%10.2lf",
"GPRINT:ldavg_5:AVERAGE:Average\\:%10.2lf\\j"
);
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Load Stats", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
# Create run_queue graph
$png_file = $hostname . "_stats-run_queue.png";
@parameters = (
"DEF:runq_sz=$rrd_file:runq_sz_all:LAST",
"AREA:runq_sz#444444:Run Queue",
"GPRINT:runq_sz:MIN: Minimum\\:%10.2lf",
"GPRINT:runq_sz:MAX:Maximum\\:%10.2lf",
"GPRINT:runq_sz:AVERAGE:Average\\:%10.2lf\\j"
);
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Run Queue", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
## Create disk graph(s), one for each device
#foreach $device (@disk_devices) {
# $png_file = $hostname . "_stats-disk_" . $device . ".png";
# unless ($device eq "all") {
# $command = "rrdtool graph $png_file \\
# -s $start -e $end \\
# -h $height -w $width -t \"Disk Stats for Disk: $device\" \\
# DEF:tps=$rrd_file:tps_$device:LAST \\
# AREA:tps#ff0000:\"$device tps\" \\
# GPRINT:tps:AVERAGE:%lf \\
# DEF:sectps=$rrd_file:sectps_$device:LAST \\
# AREA:sectps#00ff00:\"$device sectps\" \\
# GPRINT:sectps:AVERAGE:%lf";
# $result = system ($command);
# print HTMLFILE "<p><img src='$png_file'>\n";
# }
#}
if ($#disk_devices > 0) {
$png_file = $hostname . "_stats-disk_tps.png";
$index = 0;
@parameters = ();
foreach $device (@disk_devices) {
$spacer = " ";
for ($i = 12 - length ($device); $i > 0; $i--) {
$spacer = $spacer . " ";
}
unless ($device eq "all") {
push (@parameters, (
"DEF:tps_$device=$rrd_file:tps_$device:LAST",
"LINE1:tps_$device#" . $colors[$index++] . ":$device",
"GPRINT:tps_$device:MIN:" . $spacer . "Minimum\\:%10.2lf",
"GPRINT:tps_$device:MAX:Maximum\\:%10.2lf",
"GPRINT:tps_$device:AVERAGE:Average\\:%10.2lf\\j",
)
);
}
}
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Disk Stats (transactions per second)",
@parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
$png_file = $hostname . "_stats-disk_sectps.png";
$index = 0;
@parameters = ();
foreach $device (@disk_devices) {
$spacer = " ";
for ($i = 12 - length ($device); $i > 0; $i--) {
$spacer = $spacer . " ";
}
unless ($device eq "all") {
push (@parameters, (
"DEF:sectps_$device=$rrd_file:sectps_$device:LAST",
"LINE1:sectps_$device#" . $colors[$index++] . ":$device",
"GPRINT:sectps_$device:MIN:" . $spacer . "Minimum\\:%10.2lf %s",
"GPRINT:sectps_$device:MAX:Maximum\\:%10.2lf %s",
"GPRINT:sectps_$device:AVERAGE:Average\\:%10.2lf %s\\j"
)
);
}
}
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Disk Stats (sectors per second)",
@parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
}
# Create network graph(s), one per interface
foreach $device (@network_devices) {
$png_file = $hostname . "_stats-interface_" . $device . ".png";
unless (($device eq "all") or ($device eq "lo") or ($device =~ "sit?")) {
@parameters = (
"DEF:rxbytps=$rrd_file:rxbytps_$device:LAST",
"AREA:rxbytps#0000ff:Received Bytes/s",
"GPRINT:rxbytps:MIN: Minimum\\:%10.2lf %s",
"GPRINT:rxbytps:MAX:Maximum\\:%10.2lf %s",
"GPRINT:rxbytps:AVERAGE:Average\\:%10.2lf %s\\j",
"DEF:txbytps=$rrd_file:txbytps_$device:LAST",
"LINE1:txbytps#00ff00:Transmitted Bytes/s",
"GPRINT:txbytps:MIN:Minimum\\:%10.2lf %s",
"GPRINT:txbytps:MAX:Maximum\\:%10.2lf %s",
"GPRINT:txbytps:AVERAGE:Average\\:%10.2lf %s\\j",
);
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Network Stats for $device", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
}
}
# Create paging graph
$png_file = $hostname . "_stats-paging.png";
$index = 0;
@parameters = ();
foreach $counter (@paging_counters) {
$spacer = " ";
for ($i = 12 - length ($counter); $i > 0; $i--) {
$spacer = $spacer . " ";
}
push (@parameters, (
"DEF:$counter=$rrd_file:" . $counter . "_all:LAST",
"LINE1:$counter#" . $colors[$index++] . ":\"$counter",
"GPRINT:$counter:MIN:" . $spacer . "Minimum\\:%10.2lf",
"GPRINT:$counter:MAX:Maximum\\:%10.2lf",
"GPRINT:$counter:AVERAGE:Average\\:%10.2lf\\j"
)
);
}
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Paging Stats", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
# Create I/O graph
$png_file = $hostname . "_stats-io.png";
$index = 0;
@parameters = ();
foreach $counter (@io_counters) {
$spacer = " ";
for ($i = 12 - length ($counter); $i > 0; $i--) {
$spacer = $spacer . " ";
}
push (@parameters, (
"DEF:$counter=$rrd_file:" . $counter . "_all:LAST",
"LINE1:$counter#" . $colors[$index++] . ":$counter",
"GPRINT:$counter:MIN:" . $spacer . "Minimum\\:%10.2lf",
"GPRINT:$counter:MAX:Maximum\\:%10.2lf",
"GPRINT:$counter:AVERAGE:Average\\:%10.2lf\\j"
)
);
}
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "IO Stats", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
# Create process creation graph
$png_file = $hostname . "_stats-proc_creation.png";
@parameters = (
"DEF:procps=$rrd_file:procps_all:LAST",
"AREA:procps#444444:Processes per Sec",
"GPRINT:procps:MIN:" . $spacer . "Minimum\\:%10.2lf",
"GPRINT:procps:MAX:Maximum\\:%10.2lf",
"GPRINT:procps:AVERAGE:Average\\:%10.2lf\\j"
);
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Process Creation Stats", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
# Create swap graph
$png_file = $hostname . "_stats-swap.png";
$index = 0;
@parameters = ();
foreach $counter (@swap_counters) {
$spacer = " ";
for ($i = 12 - length ($counter); $i > 0; $i--) {
$spacer = $spacer . " ";
}
push (@parameters, (
"DEF:$counter=$rrd_file:" . $counter . "_all:LAST",
"LINE1:$counter#" . $colors[$index++] . ":$counter",
"GPRINT:$counter:MIN:" . $spacer . "Minimum\\:%10.2lf",
"GPRINT:$counter:MAX:Maximum\\:%10.2lf",
"GPRINT:$counter:AVERAGE:Average\\:%10.2lf\\j"
)
);
}
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Swap Stats", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
# Create memory graph
$png_file = $hostname . "_stats-memory.png";
$index = 0;
@parameters = ();
foreach $counter (@memory_counters) {
$spacer = " ";
for ($i = 12 - length ($counter); $i > 0; $i--) {
$spacer = $spacer . " ";
}
push (@parameters, (
"DEF:$counter=$rrd_file:" . $counter . "_all:LAST",
"LINE1:$counter#" . $colors[$index++] . ":$counter",
"GPRINT:$counter:MIN:" . $spacer . "Minimum\\:%10.2lf",
"GPRINT:$counter:MAX:Maximum\\:%10.2lf",
"GPRINT:$counter:AVERAGE:Average\\:%10.2lf\\j"
)
);
}
RRDs::graph ($png_file, "-s", $start, "-e", $end, "-h", $height,
"-w", $width, "-t", "Memory Stats", @parameters);
$error = RRDs::error;
warn "ERROR: $error\n" if $error;
print HTMLFILE "<p><img src='$png_file'>\n";
print HTMLFILE "</div>\n";
print HTMLFILE "</html>\n";
close HTMLFILE;
# Custom routine, mostly from a cookbook
sub mysort {
# This is the strange part. Since Perl passes arguments in an
# array, this has the effect of making the array passed to the
# subroutine in effect *becoming* that array. This doesn't
# seem wise to me, but then I've spent way too much time
# on this code, since everything I try takes 42 efforts
# to understand the proper syntax for what I want to do. Perl
# is great and all, but the loose syntax drives me nuts!
my @list = @_;
%seen = ();
foreach $item (@list) {
$seen{$item}++;
}
@uniq = sort (keys %seen);
return @uniq
}