#!/usr/bin/perl

$logmpd="/var/log/mpd";
$bundle='KS\d*|LIFE\d*|RL\d*';
#$sys='huawei_ks';

$SIG{CHLD}=IGNORE;
open(LOGmpd, "tail -11F $logmpd|") || die "cant open mpd log $logmpd\n";

my $bomb=0;		# child pid
my $noremorse=0;	# kill
my $mpdpid;
my $first=1;
my $link='';

sub mpd_pid($){
	my $link=shift;
	my $sys='never';
	if   ($link=~/KS/)   { $sys='huawei_ks' }
	if   ($link=~/RL0/)  { $sys='pppoe_sistec' }
	elsif($link=~/LIFE/) { $sys='zte_life' }

	$mpdpid=`pgrep -f '/sbin/mpd5 .* $sys'`; chomp $mpdpid;
	$mpdpid=0 if $mpdpid<0;
	return $mpdpid;
}

while(<LOGmpd>) {
	if(/mpd: *\[($bundle)\] .*(No such file or directory|: File exists)/) {
		print ".   $_\n";
	};
	
	if(/mpd: *\[($bundle)\] +(?:LCP: state change Closed --> Initial|LCP: LayerFinish|Bundle.+not found|.*did not validate in bundle|.*No such file or directory|.*: File exists)/) {
		$link=$1.$2;
		mpd_pid($link);
		print "$0: $link mpd[$mpdpid] hung?? fork bomb: $_ \n";
		$bomb=fork();
		if(defined $bomb) {
			if ($bomb) {
				$BOMB{$link}=$bomb;
				print "$0: killing $link mpd[$mpdpid] soon\n";
				
			}
			else {
				if(/Bundle.+not found|No such file or directory|: File exists/) { $noremorse=1; print "NO REMORSE $mpdpid\n"; kill 9,$mpdpid } else { print "... $_\n"; };
				sleep 10;
				print "BOOM($noremorse) $link $mpdpid\n";
				kill 2,$mpdpid;
				sleep 15;
				if ( kill( 9,$mpdpid) ) {
					print "BOOM-KILL $link $mpdpid\n";
				}
				exit 0;
			}
		} else {
			print "$0: failed to bomb $link mpd[$mpdpid]: $!\n";
		}
		
	}
	
	else {	# any new messages
		if(0 && $first){
			mpd_pid($link);
			print "$0: start monitoring mpd[$mpdpid]\n";
			$first=0;
		};

		if(/mpd: Multi-link PPP daemon for FreeBSD/) {
		#	mpd_pid($link);
			print "WILL MONITOR mpd[$mpdpid]: $_ \n";
		}
		elsif(/mpd: \[($bundle)\] LCP: Layer/ ) {
			$link=$1;
			mpd_pid($link);
			print "$link mpd[$mpdpid]: $_";
		}
		elsif(/mpd: process .+ terminated/) {
			print "$_\n";
		}

		if(!$noremorse && /mpd: \[($bundle)\] / && $BOMB{$1}>0) {
			kill 2,$BOMB{$1};
			print "STOPPING $link bomb $bomb: $_\n";
			$BOMB{$link}=0;
		}
	}
	last if eof;
}
	

__END__

su - root -c "cd /; env SCREEN=KS screen -d -m -h 2000 -e ^F^F -S KS /root/bin/mon-mpd"


Jun 17 13:31:22 <daemon.info> cel mpd: [KS] LCP: state change Closing --> Closed
Jun 17 13:31:22 <daemon.info> cel mpd: [KS] LCP: LayerFinish
Jun 17 13:31:22 <daemon.info> cel mpd: [KS] Link: DOWN event
Jun 17 13:31:22 <daemon.info> cel mpd: [KS] LCP: Down event
Jun 17 13:31:22 <daemon.info> cel mpd: [KS] LCP: state change Closed --> Initial
Jun 17 13:57:15 <daemon.info> cel mpd: caught fatal signal TERM
Jun 17 13:57:15 <daemon.info> cel mpd: [KS] IFACE: Close event
Jun 17 13:57:15 <daemon.info> cel mpd: [KS] IPCP: Close event
^^^ bad

Jun 21 07:15:01 <daemon.info> cel mpd: [KS] LCP: Down event
Jun 21 07:15:01 <daemon.info> cel mpd: [KS] LCP: state change Closed --> Initial
Jun 21 07:15:03 <daemon.info> cel mpd: [KS] Bundle: Shutdown
Jun 21 07:15:03 <daemon.info> cel mpd: [KS] Link: Shutdown
Jun 21 07:15:03 <daemon.info> cel mpd: One-shot mode enabled and no links found. Terminating daemon.
Jun 21 07:15:03 <daemon.info> cel mpd: process 43722 terminated
^^^ good
Jun 21 07:15:13 <daemon.info> cel mpd: Multi-link PPP daemon for FreeBSD


Jun 21 22:40:18 <daemon.info> cel mpd: [KS] IPCP: state change Req-Sent --> Stopped
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] IPCP: LayerFinish
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] Bundle: No NCPs left. Closing links...
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] Bundle: closing link "KS"...
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] Link: CLOSE event
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] LCP: Close event
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] LCP: state change Opened --> Closing
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] Link: Leave bundle "KS"
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] Bundle: Status update: up 0 links, total bandwidth 9600 bps
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] IPCP: Close event
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] IPCP: state change Stopped --> Closed
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] IPCP: Down event
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] IPCP: state change Closed --> Initial
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] LCP: SendTerminateReq #4
Jun 21 22:40:18 <daemon.info> cel mpd: [KS] LCP: LayerDown
Jun 21 22:40:20 <daemon.info> cel mpd: [KS] LCP: SendTerminateReq #5
Jun 21 22:40:22 <daemon.info> cel mpd: [KS] LCP: state change Closing --> Closed
Jun 21 22:40:22 <daemon.info> cel mpd: [KS] LCP: LayerFinish
Jun 21 22:40:22 <daemon.info> cel mpd: [KS] Link: DOWN event
Jun 21 22:40:22 <daemon.info> cel mpd: [KS] LCP: Down event
Jun 21 22:40:22 <daemon.info> cel mpd: [KS] LCP: state change Closed --> Initial
^^^ bad, hang