Monitoring PSSP

[screwloose]

I don’t use Perspectives and do all of my work from the command line and since there are so many predefined events that can be monitored with ‘haemqvar’ I decided to put the most commonly used events that I check into a script. Then I decided that I would make it so that I could monitor any of the 400+ events and the script has grown, albeit, rather awkwardly. Therefore it doesn’t look good, but it performs as it is intended. I believe I have added searches for all resource ID’s that are available, but if any have been missed they could be added via a function.

Given that, there are 3 scripts ‘psspchk.sh’, ‘search.f’, and ‘findnode.awk’ as well as a file that contains all of the haemqvar events that are defined fully in ‘vardefs.out’. I have placed all of these in /tmp on my Control Workstation. The ‘vardefs.out’ file can be created by running: `/usr/sbin/rsct/bin/haemqvar > /tmp/vardefs.out`.

Of course you will have to change your Node Numbers and frames, etc.

Here are the files:
# cat findnode.awk
$0 ~ first {
if ( index( $0, sought ) )
{ getline;
print
}
}

# cat search.f
awk -f findnode.awk -v first='(Resource variable|(Resource Variable Name|(Resource Variable|(Resource varible' -v sought="$cond" /tmp/vardefs.out

# cat psspchk.sh
#!/usr/dt/bin/dtksh

__GetDisks() {
x=$(grep hdisk /tmp/work.txt)
if [[ $? -eq 0 ]]
then
R=${x#*=}; L=${R%%;*}; yogi=${L##*k}
if [[ $yogi = "*" ]]; then
NodeNum="1 5 7 9 11 13"
for NODE in $NodeNum
do
dsh -w c2f1n${NODE}ex lspv | grep power | grep -iv none > /tmp/disk.work
if [[ -f /tmp/disk.work ]]
then
disks=$(cat /tmp/disk.work | grep "c2f1n${NODE}ex" | awk -F' ' '{print $2}')
for power in $disks
do
sed "s/NodeNum\=\*/NodeNum\=$NODE/" /tmp/work.txt > /tmp/work.txt.tmp
mv /tmp/work.txt.tmp /tmp/work.txt; chop=${power##*k}
sed "s/$yogi/$chop/" /tmp/work.txt > /tmp/work.txt.disk
vars=$(cat /tmp/work.txt.disk)
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "$vars"
done
else
print "BREAK"
break
fi
done
fi
else
return
fi
goob=1
return
}

__GetAdapter() {
NodeNum="1 5 7 9 11 13"
for NODE in $NodeNum
do
rm /tmp/adapter.work 2>/dev/null; rm /tmp/parse.work 2>/dev/null; rm /tmp/parse.use* 2>/dev/null
dsh -w c2f1n${NODE}ex lsdev -Cc adapter | grep $TypeX | awk '{print $2}' > /tmp/adapter.work
if [[ -f /tmp/adapter.work ]]; then
grep ent /tmp/adapter.work > /tmp/adapter.work.1; mv /tmp/adapter.work.1 /tmp/adapter.work
for inf in $(cat /tmp/adapter.work)
do
xnum=$(ls /tmp/parse*.txt | wc -l | awk '{print $1}'); xn=0
while (( xn < xnum )); do
grep ent /tmp/parse*.txt > /tmp/grep.work
pn=$(cat /tmp/grep.work | cut -c11)
((xn+=1))
done
b=0
while (( b < xnum )); do
fn=$(ls /tmp/parse$b.txt | cut -c11)
if [[ $fn -ne $pn ]]; then
cat /tmp/parse$fn.txt >> /tmp/parse.use
else
echo "Adapter=$inf" >> /tmp/parse.use
fi
((b+=1))
done
done
awk '{getline a; print $0";" a}' /tmp/parse.use > /tmp/parse.use1
sed "s/\*/$NODE/g" /tmp/parse.use1 > /tmp/parse.work
for string in $(cat /tmp/parse.work); do
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "$string"
done
fi
done
goob=1
return
}

__PrintDot() {
while :; do
sleep 1
echo ".\c"
#__twirl
done &
PID=$!
}

__twirl() {
indicators[0]="|"
indicators[1]="/"
indicators[2]="-"
indicators[3]="\\"

while true; do
tput cup 45 0; print ${indicators[0]}
tput cup 45 0; print ${indicators[1]}
tput cup 45 0; print ${indicators[2]}
tput cup 45 0; print ${indicators[3]}
done
}

__vg() {
IFS=";"
set -A mash $(cat /tmp/vars.tmp)
mashNum=${#mash[*]}
if [[ $mashNum -eq 3 ]]; then
for NODE in 1 5 7 9 11 13
do
dsh -w c2f1n${NODE}ex lsvg -o | awk '{print $2}' > /tmp/avg.tmp
sed '/paging/d' /tmp/avg.tmp > /tmp/avg.tmp1; mv /tmp/avg.tmp1 /tmp/avg.tmp
while read actvg; do
dsh -w c2f1n${NODE}ex lsvg -l $actvg | grep -v "N/A" | grep -v closed | tail +3l | awk '{print $2}' > /tmp/lv.tmp
while read logvols; do
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "VG=$actvg;LV=$logvols;NodeNum=$NODE"
done < /tmp/lv.tmp
done < /tmp/avg.tmp
done
goob=1
return
else
for NODE in 1 5 7 9 11 13
do
dsh -w c2f1n${NODE}ex lsvg -o | awk '{print $2}' > /tmp/avg.tmp
sed '/paging/d' /tmp/avg.tmp > /tmp/avg.tmp1; mv /tmp/avg.tmp1 /tmp/avg.tmp
while read actvg; do
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "VG=$actvg;NodeNum=$NODE"
done < /tmp/avg.tmp
done
goob=1
return
fi
}

__cpu() {
IFS=";"
set -A gomez $(cat /tmp/vars.tmp)
gomezNum=${#gomez[*]}
if [[ $gomezNum -eq 2 ]]; then
for NODE in 1 5 7 9 11 13
do
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "CPU=*;NodeNum=$NODE"
done
fi
goob=1
return
}

__FStotal() {
print
for LogVol in $(lsvg -l rootvg | grep -v "N/A" | tail +3l | awk '{print $1}')
do
FileSys=$(df | grep $LogVol | awk '{print $7}')
print "$LogVol ($FileSys)"
/usr/sbin/rsct/bin/haemqvar -c "" "IBM.PSSP.aixos.FS.%totused" "VG=rootvg;LV=$LogVol;NodeNum=*"
print
done
__Menu
}

__PagSpTotal() {
print
print "Paging Space"
/usr/sbin/rsct/bin/haemqvar -c "" "IBM.PSSP.aixos.PagSp.%totalused" NodeNum=*
__Menu
}

__KMBF() {
print
#Nodes="1 5 7 9 11 13"
Types="mbuf socket protcb otherip mblk streams other"
print "Kernel Memory Buffer Failures"
for a in $Types
do
#Types="mbuf socket protcb otherip mblk streams other"
Nodes="1 5 7 9 11 13"
#for c in $Types
for c in $Nodes
do
/usr/sbin/rsct/bin/haemqvar -c "" "IBM.PSSP.aixos.Mem.Kmem.failures" "Type=$a;NodeNum=$c"
#/usr/sbin/rsct/bin/haemqvar -c "" "IBM.PSSP.aixos.Mem.Kmem.failures" "Type=$c;NodeNum=$a"
done
print
done
__Menu
}

__Switch_IO_Errs() {
print
Nodes="1 5 7 9 11 13"
print "Switch Input and Output Errors"
for i in $Nodes
do
Resource="ibadpackets ipackets_drop ierrors opackets_drop oerrors xmitque_ovf"
for b in $Resource
do
/usr/sbin/rsct/bin/haemqvar -c "" "IBM.PSSP.CSS.$b" NodeNum=$i
done
print
done
__Menu
}

__Startd() {
print "LoadLeveler is not in use. Variable instance not available."
print
goob=1
return
varx=$(cat /tmp/vars.tmp)
for Nodes in 1 5 7 9 11 13
do
#/usr/sbin/rsct/bin/haemqvar -c "" "$i" "$varx"
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "NodeNum=*;STARTD=startd"
done
goob=1
return
}

__Paging() {
for Node in 1 5 7 9 11 13
do
dsh -w c2f1n${Node}ex lsps -a | tail +2l | awk '{print $2}' > /tmp/paging.work
for pag in $(cat /tmp/paging.work); do
sed "s/\paging\*/${pag}/g" /tmp/vars.tmp > /tmp/paging.txt
gx=$(cat /tmp/vars.tmp)
qx=$(cat /tmp/paging.txt)
print ${qx%%;*} > /tmp/pagnode1.tmp
print ${gx##*;} > /tmp/pagnode.tmp
sed "s/NodeNum=\*/NodeNum\=${Node}/g" /tmp/pagnode.tmp > /tmp/pagnode.tmp1
one=$(cat /tmp/pagnode1.tmp)
two=$(cat /tmp/pagnode.tmp1)
print "$one;$two" > /tmp/pag.use
varx=$(cat /tmp/pag.use)
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "$varx"
done
done
goob=1
return
}

__Schedd() {
print "LoadLeveler is not in use. Variable instance not available."
print
goob=1
return
varx=$(cat /tmp/vars.tmp)
for Nodes in 1 5 7 9 11 13
do
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "$varx"
done
goob=1
return
}

__DaemonActivity() {
print
Nodes="1 5 7 9 11 13"
print "Daemon Activity"
for y in $Nodes
do
DaemonProg="inetd srcmstr"
for d in $DaemonProg
do
/usr/sbin/rsct/bin/haemqvar -c "" "IBM.PSSP.Prog.xpcount" "ProgName=$d;UserName=root;NodeNum=$y"
done
print
done
__Menu
}

__DaemonActivityOther() {
print
Nodes="1 5 7 9 11 13"
print "Daemon Activity"
for z in $Nodes
do
DaemonProgs="biod portmap xntpd kerberos sdrd"
for m in $DaemonProgs
do
/usr/sbin/rsct/bin/haemqvar -c "" "IBM.PSSP.Prog.pcount" "ProgName=$m;UserName=root;NodeNum=$z"
done
print
done
__Menu
}

__Parse() {
awk -F';' '{print $1}' /tmp/vars.tmp > /tmp/parse0.txt
awk -F';' '{print $2}' /tmp/vars.tmp > /tmp/parse1.txt
awk -F';' '{print $3}' /tmp/vars.tmp > /tmp/parse2.txt
p[0]=/tmp/parse0.txt; p[1]=/tmp/parse1.txt; p[2]=/tmp/parse2.txt; s=0
while (( s < 3 )); do
size=$(ls -l ${p[$s]} | awk '{print $5}')
if [[ $size -le 1 ]]; then
rm ${p[$s]} 2>/dev/null
fi
((s+=1))
done
num=$(ls /tmp/parse*.txt | wc -l | awk '{print $1}')
j=0
while (( j < num )); do
type=$(cat /tmp/parse$j.txt)
Type=${type%%=*}
Typex=${type##*=}
strlen=${#Typex}
((TypexLen=strlen-1))
typeset -L$TypexLen TypeX=$Typex
case $Typex in
startd) __Startd;;
schedd) __Schedd;;
esac
case $Type in
Adapter) export j;
__GetAdapter;;
hdi) return;;
VG) __vg;;
CPU) __cpu;;
Name) typeset -R7 descr=$Typex; typeset -L6 descrx=$descr
if [[ $descrx = "paging" ]]; then
__Paging
else
__GetDisks
fi;;
esac
((j+=1))
done
return
}

__RunIt() {
i=$cond; export i; print
if [[ $i = "IBM.PSSP.aixos.Proc.swpque" ]]; then
echo "NodeNum=*" > /tmp/vars.tmp
vars=$(cat /tmp/vars.tmp)
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "$vars"
__Menu
fi
/tmp/search.f > /tmp/work.txt
tr -d ' ' < /tmp/work.txt > /tmp/work.txt1
mv /tmp/work.txt1 /tmp/work.txt
x=$(cat /tmp/work.txt)
variables=${x##*:}
echo $variables > /tmp/work.txt
sed 's/[0-9]/\*/g' /tmp/work.txt > /tmp/work.txt1
mv /tmp/work.txt1 /tmp/work.txt
vars=$(cat /tmp/work.txt | sed -e 's/\*\*/\*/')
echo $vars > /tmp/vars.tmp
#if [[ $gotit = aixos ]]; then
if [[ $gotit != "" ]]; then
__Parse
fi
if [[ $goob -eq 1 ]]; then
__Menu
fi
if [[ $gotit = disk ]]; then
__GetDisks
fi
if [[ $goob -eq 1 ]]; then
__Menu
fi
print "Checking condition '$i'"
__PrintDot
/usr/sbin/rsct/bin/haemqvar -c "" "$i" "$vars"
kill -1 $PID; print; return
}

__Search() {
print; print -n "Enter a word to search (or enter Q! to exit): " ; read gotit
if [[ $gotit = "Q!" ]]; then
__Cleanup; exit 0
fi
/usr/sbin/rsct/bin/haemqvar -d | grep "$gotit" > /tmp/workfile
filesize=$(ls -l /tmp/workfile | awk '{print $5}')
if [[ $filesize -eq 0 ]]; then
print
print "There is no event that matches your search. Please search on another description."
#__Menu
__Search
fi
more -e workfile
print -u2 "\n"
print -n "Copy & paste or type the Resource Variable Name you want to check for conditions (or enter Q! to exit): "
read cond
if [[ $cond = "Q!" ]]; then
__Cleanup; exit 0
else
export cond
fi
__RunIt
__Menu
}

__Cleanup() {
rm /tmp/parse.work /tmp/parse.use /tmp/parse.use1 /tmp/cpu.num /tmp/vars.tmp /tmp/workfile /tmp/work.txt /tmp/work.txt.disk /tmp/grep.work /tmp/a
dapter.work /tmp/disk.work /tmp/parse*.txt /tmp/avg.tmp /tmp/lv.tmp /tmp/paging.work /tmp/paging.txt /tmp/pag.use /tmp/pagnode.tmp /tmp/pagnode1.
tmp /tmp/pagnode.tmp1 2>/dev/null
echo
}

__Menu() {
print 'Select the SP conditions to monitor:'
PS3='condition? '
print
select list in 'File System becoming full' 'Kernel Memory Buffer Failures' 'Paging Space Low' 'Switch Input Output Errors' 'inetd srcmstr -- Daem
on Activity' 'biod portmap xntpd kerberos sdrd -- Daemon Activity' 'Search for an event' 'Quit'
do
case $REPLY in
1) __FStotal ;;
2) __KMBF ;;
3) __PagSpTotal ;;
4) __Switch_IO_Errs ;;
5) __DaemonActivity ;;
6) __DaemonActivityOther ;;
7) __Search ;;
8) __Cleanup; exit 0 ;;
*) if [[ ${#REPLY} -ne 1 ]]; then
clear; __Menu
else
clear; __Menu
fi ;;

esac
done
}

goob=0
__Menu