Show cars.pl syntax highlighted
# ex: set tabstop=2 expandtab shiftwidth=2 softtabstop=2:
#!/usr/bin/perl
#
# sars - Sympathy Automated Regression System
use strict;
# global variables, change this to accomodate your own environments
my(%G);
#default variable settings
my $EMSTAR_HOME="$ENV{HOME}/cvs/emstar";
my @NODESKIP = (0,0,0,3,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,19,20,0,22,23,0,0,26,0,28,29,30);
my $NODES = 30;
my $SIM_FILE="../devel/confidence/simulation/confidence_test.sim";
my $SIMULATION_FILE_DIR="$ENV{HOME}/.cars";
my $SINKNODE='002'; # the heading '00' in '002' is VERY IMPORTANT
my $epoch = 3;
my $METRICS_PERIOD=180;
my $TRACK_FAIL_PERIOD = $METRICS_PERIOD * $epoch;
my $INITIAL_DELAY = 15 * 60;
my $JITTER_SCRIPT="$ENV{HOME}/bin/inject_jitter";
if (!-e $JITTER_SCRIPT) {
print "Script to inject jitter ($JITTER_SCRIPT) not found\n";
exit 1;
}
my $ITERATIONS=1;
my $SIMULATION = 1;
my $QUIT_AFTER_SUCC=1; # if we successfully found a fault/failure, then quit
my $Inject_fault = 0;
select(STDOUT); $|=1;
use POSIX;
use Fcntl;
use FileHandle;
# ----------------- define the instance object --------------------
package Instance;
use vars qw($AUTOLOAD);
my %vars = (
log_dir => undef,
sim_group => 'GROUP_UNDEF',
iter => 1,
C_summ_fail_handle => '',
S_summ_fail_handle => '',
#sigsegv_handle => '',
s_iter => 1, # current iteration
s_node => undef,
c_node => undef,
C_succ => -1, # highest success status
S_succ => -1, # highest success status
quit_timer => 30,
);
sub new {
my $me = shift;
my $class = ref($me) || $me;
my $self = {%vars};
bless $self, $class;
return $self;
}
sub keys {
my $self = shift;
return (keys %$self);
}
sub AUTOLOAD {
my $self = shift;
my $type = ref($self) || die "$self is not an object";
my $name = $AUTOLOAD;
$name =~ s/.*://;
if (@_) {
return $self->{$name} = shift;
} elsif (exists $self->{$name}) {
return $self->{$name};
} elsif ($name eq 'DESTROY') {
return undef;
} else {
print "ERROR: entry object->$name() not found\n";
exit 1;
#return undef;
}
}
# -----------------------------------------------------------------
package main;
my @INSTANCES;
my @FILEHANDLES;
my %BUFFER;
my $INSTANCES=1; # default num of instances run
my %MESSAGES; # mapping between STF_OK and "OK"
my %NODE; # success/fail info (SRC_NO_DATA, SRC_NODE_FAILED, etc)
my %HANDLE2FILE; # associate handle with command
my $LOG_SUFFIX=''; # extra information
my $EXIT_TIME=60*60*2; # kill EVERYTHING in 2 hours for safety
my $STDOUT_NO_CR=0; # for pretty print purpose
my $BZIP2=0;
# exception handling
$SIG{INT}=\&handler;
$SIG{HUP}=\&handler;
$SIG{STOP}=\&handler;
my @PID;
sub killProcesses {
if ($#PID!=-1) {
printw("Cleaning processes ",join(', ',@PID),"...");
my $pids=join(' ', @PID);
# Do the following instead of Perl's kill because Perl's kill hangs
system("(kill $pids > /dev/null 2>&1) &");
system("(sleep 10; kill -9 $pids > /dev/null 2>&1) &");
printw("done\n");
}
@PID=@FILEHANDLES=@INSTANCES=%BUFFER=%HANDLE2FILE=();
}
sub handler {
my $pids=join(' ', @PID);
killProcesses();
system("(kill -9 $pids > /dev/null 2>&1) &");
exit 1;
}
sub hasLine {
my($handle)=@_;
if ($BUFFER{$handle}=~/\n/) {return 1;}
}
sub getLine {
my($handle)=@_;
# Get rid of new-lines!
while ($BUFFER{$handle}=~/^\n/)
{
$BUFFER{$handle}=~s/(^\n)(.*)/$2/;
}
if ($BUFFER{$handle}=~s/^([^\n]+\n)(.*)/$2/) {return $1;}
}
sub processFileHandles {
#fcntl($handle, F_SETFL(), O_NONBLOCK());
my $buf;
foreach my $handle (@FILEHANDLES) {
my $bytes_read=1024;
my $bound=100; # safety for switching tasks
while ($bytes_read && $bound-->0) {
# IMPORTANT, below is a non-blocking call
$bytes_read = sysread($handle, $buf, 1024);
if (defined($bytes_read)) {
if ($bytes_read == 0) {
# Remote socket closed connection
warn "Handle closed!\n";
close($handle);
last;
} else {
$BUFFER{$handle} .= $buf;
}
} else {
my $stat = $!;
if ($stat == EAGAIN()) {
# Can return to select. Here we choose to
# spin around waiting for something to read.
} else {
last;
}
#print "Stat:$stat\n";
}
}
}
}
sub openAsynchCommand {
my($command)=@_;
my $handle = new FileHandle;
$HANDLE2FILE{$handle}=$command;
push @FILEHANDLES, $handle;
#open($handle, "$ENV{HOME}/bin/loop 2>&1 |");
open($handle, "$command 2>&1 |") || die "Can't execute $command";
fcntl($handle, F_SETFL, O_NONBLOCK | O_RDWR | O_NDELAY)
or die "Couldn't set flags for HANDLE: $!\n";
return $handle;
}
sub printw {
if ($G{verbose} && $STDOUT_NO_CR) {
$STDOUT_NO_CR=0;
print "\n";
}
print @_;
}
# Se open and then close to make sure we flush all the logs.
# Slightly inefficient but simple.
sub logw {
my($obj, $line)=@_;
$line=~s/\s+$//; # strip trailing space
my $file="$ENV{LOG_DIR}/sars.log";
open(WR, ">>$file") || die "Can't write to $file";
print(WR $line."\n");
close(WR);
printw "> $line\n";
}
sub S_processSummFail {
my($obj, $timepassed)=@_;
my $handle=$obj->S_summ_fail_handle;
my($s_node, $s_iter);
if (!$handle) {return 1}
while (hasLine($handle)) {
my $line=getLine($handle);
if ($line=~/\*\*\*\*\*\*\*\*/) {
next;
} elsif ($line=~/Node\s+(\d+).*Metric Pd:\s+(\d+)/) {
$s_node=$1; $s_iter=$2;
if ($s_iter != $obj->s_iter) {
$obj->s_iter($s_iter);
}
$obj->s_node($s_node);
} else {
foreach my $status (split(/,/,$line)) {
$status=~s/^\s+//;
$status=~s/\s+$//;
if ($status=~/^(.*)\s*(Failure).*Root-Cause: (.+)\(/) {
my $cat=$1;
my $ff=$2;
my $msg=$3;
my $node=$obj->s_node;
$msg=~s/\s+$//;
$msg=$MESSAGES{$msg};
my $mp = $NODE{$node."_$msg"};
if (!($msg eq 'SRC_OK')) {
my $success=0;
my $success_code = '2';
if ($cat=~/Root/) { $success_code = '3'; }
$success=($NODE{$node."_$msg"}?$success_code:
$NODE{$node}?'1':'0');
# FALSE report! Wrong node!
if ($success==0 && $cat=~/Root/) {$success=-1}
# record the highest succ level
if (($Inject_fault == 1) && ($obj->S_succ < $success)) {
$obj->S_succ($success);
}
#iter time s_iter node category type correctness
logw($obj,
sprintf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s S",
$obj->iter, $timepassed,
$obj->s_iter, $node,
$cat, $ff, $msg, $success));
}
}
}
}
}
return 1;
}
sub C_processSummFail {
my($obj, $timepassed)=@_;
my $handle=$obj->C_summ_fail_handle;
my($c_node, $s_iter);
if (!$handle) {return 1}
while (hasLine($handle)) {
my $line=getLine($handle);
if ($line=~/\*\*\*\*\*\*\*\*/) {
next;
} elsif ($line=~/^(\d+),(.*)$/) {
my $status = $2;
$c_node=$1;
$obj->c_node($c_node);
if (($status=~/(LOCALIZED)/) || ($status=~/(NO ACTION NEEDED)/)) {
my $cat = $1;
my $success = 0;
logw($obj,
sprintf("%s\t%s\t%s\t%s\t%s\t%s C",
$obj->iter, $timepassed,
$obj->s_iter, $c_node,
$cat, $success));
}
} else {
foreach my $status (split(/,/,$line)) {
$status=~s/^\s+//;
$status=~s/\s+$//;
if ($status=~/(check.*)$/) {
my $cat=$1;
my $node=$obj->c_node;
my $success=1;
if ($NODE{$node}) { $success = 3; }
else { $success = 0; }
# record the highest succ level
if (($Inject_fault == 1) && ($obj->C_succ < $success)) {
$obj->C_succ($success);
}
#iter time s_iter node category type correctness
logw($obj,
sprintf("%s\t%s\t%s\t%s\t%s\t%s C",
$obj->iter, $timepassed,
$obj->s_iter, $node,
$cat, $success));
}
}
}
}
return 1;
}
sub genNode {
my($node)=@_;
return sprintf("node%.3d", $node);
}
sub launch {
my($cmd, $logfile)=@_;
if ($logfile) {
$cmd.=" > $ENV{LOG_DIR}/$logfile";
} else {
my @file=split(/\//, $cmd);
my $file=$file[$#file];
$cmd.=" > $ENV{LOG_DIR}/._$file"; # make it hidden
}
# EXECUTE below
my $pid=`$cmd 2>&1 & wmpid=\$!; echo \$wmpid`; chomp($pid);
push(@PID, $pid);
printw("> Launching command: $cmd ($pid)\n");
return $pid
}
sub log_script {
my $sinknode=genNode($SINKNODE);
my $linkdump="$EMSTAR_HOME/obj.i686-linux/bin/linkdump";
my $echocat="$EMSTAR_HOME/obj.i686-linux/bin/echocat -w";
my @pids;
push(@pids, launch("cat /dev/sim/group$ENV{SIM_GROUP}/$sinknode/emlog/all/all-f", "log2"));
push(@pids, launch("cat /dev/sim/group$ENV{SIM_GROUP}/$sinknode/tos/logs/error-f", "log_error2"));
push(@pids,
launch("$linkdump -m -f -l -N 2 -G $ENV{SIM_GROUP} -r -U mote0 -T", "link2"),
launch("$echocat /dev/node002/sympathy/metrics", "all_metrics"),
launch("$echocat /dev/node002/confidence/nw_stream", "log_conf_nw"),
launch("$echocat -C --refresh 180000 /dev/node002/confidence/nw_clusters", "log_conf_nw_clusters"),
launch("$echocat /dev/node002/dse/stream", "log_dse_data"),
launch("$echocat /dev/emrun/last_msg", "last_message"),
launch("$echocat /dev/node002/sympathy/summary", "all_summ_fail"));
my $pids=join(' ', @pids);
my $time=$EXIT_TIME+5;
my $killpid=`(sleep $time; kill -9 $pids) > /dev/null 2>&1 & wmpid=\$!; echo \$wmpid`; chomp($killpid);
push(@PID, $killpid);
printw "After $time seconds, kill $pids\n";
}
# Central place to put in actions
sub processCommand {
my($obj, $cmd, $timepassed)=@_;
my($time,$cmd,$action)=split(/\s+/, $cmd);
$ENV{SIM_GROUP}=$obj->sim_group;
logw($obj,
sprintf("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s",
$obj->iter, $timepassed,
$obj->s_iter, $action,
"Inject", "Command", "$cmd".($action?",$action":""), '0'));
if ($G{verbose}) {printw "*Time:$time executing:$cmd $action (group $ENV{SIM_GROUP})\n"}
if ($cmd eq 'exit') {
return 'exit';
} elsif ($cmd eq 'monitor') {
# start reading summ_fail
if ($obj->C_summ_fail_handle) {
warn "Already parsing summ_fail! Command ignored\n";
return;
}
my $sinknode=genNode($SINKNODE);
log_script();
my $handler = openAsynchCommand("$EMSTAR_HOME/obj.i686-linux/bin/echocat -w /dev/$sinknode/confidence/nw_stream");
$obj->C_summ_fail_handle($handler);
$handler = openAsynchCommand("$EMSTAR_HOME/obj.i686-linux/bin/echocat -w /dev/$sinknode/sympathy/summary");
$obj->S_summ_fail_handle($handler);
# echo non-sense into nodes (try to kick them start)
printw "Checking if all the nodes are up and running...\n";
foreach my $i (1..$NODES) {
if ($i==int($SINKNODE) || $NODESKIP[$i]) {next;}
my $node=genNode($i);
system("echo WAKE > /dev/sim/group$ENV{SIM_GROUP}/$node/sympathy/fault_inject");
if ($?) {printw "\tWARNING: Node $i is not ready\n";}
}
} elsif ($cmd eq 'jitter') {
$cmd="$JITTER_SCRIPT";
printw "Injecting jitter with command:\n$cmd\n";
system($cmd);
if ($?) {printw("Problem injecting 'jitter' \n");}
}
else {
$Inject_fault = 1;
if ($cmd eq 'reboot') {
# write into /dev/.../fault_inject
my $node=genNode($action); # $action is the node number here
system("echo reboot > /dev/sim/group$ENV{SIM_GROUP}/$node/sympathy/fault_inject");
if ($?) {printw("Problem injecting 'reboot' into /dev/sim/group$ENV{SIM_GROUP}/$node/sympathy/fault_inject\n");}
} elsif ($cmd eq 'traffic') {
$action*=10; # Convert sec to 100ms units
$cmd="echo 'id=1:sensors=33:period=$action' > /dev/sim/group$ENV{SIM_GROUP}/node$SINKNODE/dse/query";
printw "Injecting traffic with command:\n$cmd\n";
system($cmd);
if ($?) {printw("Problem injecting 'traffic' \n");}
} elsif ($cmd eq 'die') {
# write into /dev/.../fault_inject
my $node=genNode($action);
system("echo die > /dev/sim/group$ENV{SIM_GROUP}/$node/sympathy/fault_inject");
if ($?) {printw("Problem injecting 'die' into /dev/sim/group$ENV{SIM_GROUP}/$node/sympathy/fault_inject\n");}
} elsif ($cmd eq 'halt') {
my $node=genNode($action);
print "GOING TO HALT NODE!\n";
system("echo halt > /dev/sim/group$ENV{SIM_GROUP}/$node/emrun/command");
if ($?) {printw("Problem injecting 'halt' into $node\n");}
} elsif ($cmd eq 'send' || $cmd eq 'recv') {
my($node,$prob)=(split(/,/, $action));
$node=genNode($action);
system("echo $cmd:node=$node:prob=$prob > /dev/sim/group$ENV{SIM_GROUP}/$node/sympathy/fault_inject");
}
else {
warn "Command ($cmd, $action) not understood\n";
}
}
}
# ------------------ program initialization --------------------
system("mkdir -p /tmp/$ENV{USER}");
if (!$ENV{HOME}) {die "You need to setup the HOME environment\n"; }
if (!-e $SIMULATION_FILE_DIR) {system("mkdir -p $SIMULATION_FILE_DIR");}
if (!-e $EMSTAR_HOME) {
if (!$ENV{EMSTAR_HOME}) {die "Can't find $EMSTAR_HOME, please set EMSTAR_HOME\n";}
if (!-e $ENV{EMSTAR_HOME}) {die "Can't find $ENV{EMSTAR_HOME}, please re-set EMSTAR_HOME\n"; }
$EMSTAR_HOME = $ENV{EMSTAR_HOME};
}
my $line=`grep METRICS_PERIOD $EMSTAR_HOME/tos-contrib/sympathy/tos/lib/Sympathy.h` || die "Can't open $EMSTAR_HOME/tos-contrib/sympathy/tos/lib/Sympathy.h";
if ($line=~/METRICS_PERIOD_MSEC\s+(\d+)/) {$METRICS_PERIOD=$1/1000;} else {die "Can't find METRICS_PERIOD in Sympathy.h";}
$TRACK_FAIL_PERIOD = $METRICS_PERIOD * $epoch;
print "CHECK metrics-period: $METRICS_PERIOD, track-fail: $TRACK_FAIL_PERIOD\n";
# process arguments
if ($#ARGV==-1) {
printw <<EOL;
Sympathy Automated Regression System (sars) Usage Guide:
> sars.pl [options below]
-instances <num> Number of EmStars to run simultaneously. [default=1]
-iter <num> Number of times to run/generate data [default=1]
-monitor <sec> Start reading /dev/*/files after <sec>
-die <node_num> After random time, node <node_num> radio dies
-halt <node_num> After random time, node <node_num> *HALTS* completely
-reboot <node_num> After random time, reboot node <node_num>
-jitter <sec1> After <sec1>, disable jitter on nodes (to force congestion on
Sympathy traffic
-epoch #metrics-pd per failure-epoch
-traffic <sec2> After random time, inject dse-traffic w/ period sec2
-keepgoing If this is specified, simulation will keep running
until time is up. If this is not specified, then
simulation will quit 30 seconds after it has
successfully detected fault/failure.
-exit <sec> After <sec> exit. By default exit after $METRICS_PERIOD*10 seconds
-verbose Print time
-kill Kill previous runs
-bzip2 Use bzip2 to compress logs
Example: sars.pl -sim_group 95 -jitter 150 -monitor 151 -die 500:7 -exit 2000 -verbose
Example: sars.pl -sim_group 95 -jitter 150 -monitor 151 -send 500:node=3,prob=70 -recv \\
500:node=3,prob=70 500:7 -exit 2000 -verbose
EOL
exit 0;
}
# parse Arguments
my @COMMAND_QUEUE;
my $_prevsec=0;
for (my $i=0; $i<=$#ARGV; $i++) {
my $arg=$ARGV[$i];
if ($arg!~s/^\-//) {die "Argument $arg not recognized";}
if ($arg eq 'kill') {
system("ps auxwww|grep $ENV{USER}|egrep '(emrun|emsim|sympathy_sink|sympathy_app|wmpid)' | awk '{print \$2}'|xargs kill -9");
printw "All Sympathy processes should be killed\n";
exit 0;
} elsif ($arg eq 'bzip2') {
$BZIP2=1;
} elsif ($arg eq 'sim_group') {
my $sim_group=$ARGV[++$i];
if ($sim_group!~/^\d+$/) {die "SIM_GROUP must be between 0-99";}
$ENV{SIM_GROUP}=$sim_group;
} elsif ($arg eq 'iter') {
$ITERATIONS=$ARGV[++$i];
} elsif ($arg eq 'instances') {
$INSTANCES=$ARGV[++$i];
} elsif ($arg eq 'keepgoing') {
$QUIT_AFTER_SUCC=0;
} elsif ($arg eq 'epoch') {
$epoch =$ARGV[++$i];
#Re-calculate the track-fail-period!
$TRACK_FAIL_PERIOD = $METRICS_PERIOD * $epoch;
} elsif ($arg =~/jitter|traffic|monitor|die|halt|recv|send|reboot|exit/) {
my $cmd=$arg;
my($sec,$arg)=split(':',$ARGV[++$i]);
#Randomize time failure is injected!
if ($cmd =~/die|halt|recv|send|reboot/) {
$arg = $sec;
# rand within stats period cuz we check every stats pd for failure
$sec = int(rand($METRICS_PERIOD));
#This is done to get rid of fact that sympathy waits for
# epoch*stats-pd in the beginning. And that the network
# takes some time to form.
$sec += $INITIAL_DELAY;
}
$_prevsec = $sec;
if ($cmd eq 'monitor') {
push(@COMMAND_QUEUE, "$sec $cmd");
} elsif ($cmd eq 'exit') {
push(@COMMAND_QUEUE, "$sec $cmd");
$EXIT_TIME=$sec;
} elsif ($cmd eq 'traffic') {
# arg is the period
if ($arg!~/^\d+/) {
printw "$cmd: Second argument period ($arg) must be a number\n";
exit 1;
}
push(@COMMAND_QUEUE, "$sec $cmd $arg");
$LOG_SUFFIX.="$sec$cmd$arg.";
} elsif ($cmd eq 'jitter') {
push(@COMMAND_QUEUE, "$sec $cmd");
$LOG_SUFFIX.="$sec$cmd.";
} elsif ($cmd eq 'die' || $cmd eq 'halt') {
# arg is node number
if ($arg!~/^\d+/) {
printw "Second argument node ($arg) must be a number\n";
exit 1;
}
push(@COMMAND_QUEUE, "$sec $cmd $arg");
$NODE{$arg."_SRC_NODE_FAILED"}=1;
$NODE{$arg}=1;
my $tmp = $NODE{$arg."_SRC_NODE_FAILED"};
print "for arg: $arg, msg: _SRC_NODE_FAILED: $tmp\n";
$LOG_SUFFIX.="$sec$cmd$arg.";
} elsif ($cmd eq 'send' || $cmd eq 'recv') {
my($node,$prob);
if ($arg=~/node=(\d+),prob=(\d+)/i) {
$node=$1;
$prob=$2;
push(@COMMAND_QUEUE, "$sec $cmd $node,$prob");
} else {
die "Sorry, send/recv accepts in the format of -send 500:node=5,prob=80";
}
$NODE{$node."_SRC_INSUFFICIENT_DATA"}=1;
$NODE{$node."_SRC_NODE_FAILED"}=1;
my $tmp = $NODE{$arg."_SRC_NODE_FAILED"};
print "for arg: $arg, msg: _SRC_NODE_FAILED: $tmp\n";
$NODE{$node}=1;
$LOG_SUFFIX.="$sec$cmd$arg.";
} elsif ($cmd eq 'reboot') {
# arg is node number
if ($arg!~/^\d+/) {
printw "Second argument node ($arg) must be a number\n";
exit 1;
}
push(@COMMAND_QUEUE, "$sec $cmd $arg");
# setup the node failure
$NODE{$arg."_SRC_NODE_REBOOTED"}=1;
$NODE{$arg}=1;
$LOG_SUFFIX.="$sec$cmd$arg.";
}
} elsif ($arg eq 'verbose') {
$G{verbose}=1;
} else {
warn "Argument '$arg' not understood.";
exit 1;
}
}
$SIM_FILE.=" $epoch";
$LOG_SUFFIX.="epoch$epoch.";
# put in an exit command if the last one isn't already exit
if ($COMMAND_QUEUE[$#COMMAND_QUEUE] !~ /exit/) {
push(@COMMAND_QUEUE, ($METRICS_PERIOD*10)." exit");
$EXIT_TIME=($METRICS_PERIOD*10);
}
if ($EXIT_TIME==0) {die "Exit time not specified!\n";}
if (!$ENV{SIM_GROUP}) {die "You need to set SIM_GROUP (or specify -sim_group between 0 to 99)"; }
printw(("="x75)."\n");
printw "Command review:\n";
foreach my $cmd (@COMMAND_QUEUE) {
printw "\t$cmd\n";
}
foreach my $node (sort (keys %NODE)) {
if ($node =~ s/^(\d+)_//) {
printw "Expecting fault $node for node $1\n";
} else {
printw "Expecting 'something' for node $node\n";
}
}
# Get a text to ID translation
$MESSAGES{"Unrecognized Fault"}="UNRECOGNIZED";
open(FD, "$EMSTAR_HOME/devel/sympathy/libsympathy/sympathy_decode.c") ||
die "Can't find $EMSTAR_HOME/devel/sympathy/libsympathy/sympathy_decode.c";
while(my $line=<FD>) {
if ($line=~/(SR._\w+)\s*\)\s+return\s+\"([^\"]+)\"/) {
$MESSAGES{$2}=$1;
}
}
# Start running the test here!
my @_CMD=@COMMAND_QUEUE;
my($_from, $_to)=($ENV{SIM_GROUP}, $ENV{SIM_GROUP}+$INSTANCES-1);
foreach my $iter (1..$ITERATIONS) {
@COMMAND_QUEUE=@_CMD;
printw "\n";
printw "Running iteration $iter\n";
# execute all the emrun command/instances asynchronously
foreach my $group ($_from..$_to) {
$ENV{SIM_GROUP}=$group;
my $type=$SIMULATION?'sim':'emu';
my $suffix=($LOG_SUFFIX?$LOG_SUFFIX.'.':'');
foreach my $count ('a'..'z') {
if (!-e ($ENV{LOG_DIR}="$SIMULATION_FILE_DIR/group$ENV{SIM_GROUP}.${LOG_SUFFIX}iter$iter$count.$type")) {
last;
}
}
if (-e $ENV{LOG_DIR}) {
print "WARNING: Directory $ENV{LOG_DIR} already exists! Over-writing...\n";
}
system("mkdir -p $ENV{LOG_DIR}");
unlink("$ENV{LOG_DIR}/sars.log");
printw "Log dir: $ENV{LOG_DIR}\n";
my $origdir=`pwd`; chomp($origdir);
if (!-e "./emrun/emsim") {die "Can't find ./emrun/emsim";}
my $empid=launch("./emrun/emsim $SIM_FILE", "emrun.log");
my $time=$EXIT_TIME+5;
my $pid2=`(sleep $time; kill -9 $empid) > /dev/null 2>&1 & wmpid=\$!; echo \$wmpid`; chomp($pid2);
push(@PID, $empid, $pid2);
my $obj = Instance->new();
$obj->log_dir($ENV{LOG_DIR});
$obj->sim_group($group);
$obj->iter($iter);
push(@INSTANCES, $obj);
logw($obj, sprintf("#h Run-Iter Timepassed TestIter Node-id Component Type Failure Correct"));
logw($obj, "# ($ENV{LOG_DIR}) ".`date`);
}
# loop here
my $starttime=time;
my $repeat=1;
while ($repeat) {
processFileHandles();
my $timepassed=time-$starttime;
if ($G{verbose}) {
if ($timepassed%60==0) {print(sprintf("%d min",int($timepassed/60)));}
$STDOUT_NO_CR=1;
print ".";
}
# process commands after time has passed
while ($#COMMAND_QUEUE!=-1 &&
(split(/\s+/, $COMMAND_QUEUE[0]))[0] <= $timepassed) {
my $cmd=shift(@COMMAND_QUEUE);
foreach my $obj (@INSTANCES) {
$ENV{LOG_DIR}=$obj->log_dir;
$ENV{SIM_GROUP}=$obj->sim_group;
if (processCommand($obj, $cmd, $timepassed) eq 'exit') {
$repeat=0;
}
}
}
# parse the lines
my $quits=$#INSTANCES+1;
foreach my $obj (@INSTANCES) {
$ENV{LOG_DIR}=$obj->log_dir;
$ENV{SIM_GROUP}=$obj->sim_group;
if (!C_processSummFail($obj, $timepassed)) {
printw "FATAL ERROR DETECTED, resetting...\n";
$repeat=0;
last;
}
if (!S_processSummFail($obj, $timepassed)) {
printw "FATAL ERROR DETECTED, resetting...\n";
$repeat=0;
last;
}
if ($QUIT_AFTER_SUCC) {
if ($obj->C_succ>=3 && $obj->S_succ >= 3) {
$obj->quit_timer($obj->quit_timer - 1);
system("touch $ENV{LOG_DIR}/SUCC");
}
if ($obj->quit_timer<=0) {
$quits--;
# one time user notification
if ($obj->quit_timer==0) {
printw("SUCC: group ".$obj->sim_group." successfully detected failure for a while.\n");
}
}
}
}
if ($quits<=0) {
printw "SUCC: Every instance has detected failure, next run...\n";
$repeat=0;
}
sleep 1;
}
# get network summary
foreach my $obj (@INSTANCES) {
$ENV{SIM_GROUP}=$obj->sim_group;
foreach my $i (1..$NODES) {
if ($i==int($SINKNODE) || $NODESKIP[$i]) {next;}
my $node=genNode($i);
my @summary=`cat /dev/sim/group$ENV{SIM_GROUP}/$node/sympathy/fault_inject`;
my($a,$b,$c,$d);
if ($summary[0]=~/Sympathy\D+(\d+)\D+(\d+)\D+(\d+)\D+(\d+)/) {
$a=$1; $b=$2; $c=$3; $d=$4;
} else {
logw ($obj, sprintf("Error, can't get packet statistics from %d %d '$summary[0]'\n",
$ENV{SIM_GROUP},$node));
}
if ($summary[1]=~/Sympathy\D+(\d+)\D+(\d+)\D+(\d+)\D+(\d+)/) {
logw ($obj,
sprintf("# Packet %d %d (send/recv) Sympathy:%d/%d B %d/%d # Other:%d/%d B %d/%d #\n",
$ENV{SIM_GROUP},$node, $a, $b, $1,$2, $c,$d, $3,$4));
}
}
}
killProcesses();
if ($BZIP2) {
system("bzip2 $ENV{LOG_DIR}/log? $ENV{LOG_DIR}/link? $ENV{LOG_DIR}/log?? $ENV{LOG_DIR}/emrun.log");
}
#if ($iter!=$ITERATIONS) {
printw "Waiting for processes to really end...";
sleep 12;
printw "done\n";
#}
}
printw "Sars all done, exiting.\n";
exit 0;
See more files for this project here