#! /bin/perl --    # -*-Perl-*-
#
# outflow-stat - display outgoing feed statistics (#articles)
#                this works for innfeed and nntpsend/innxmit only
#
# History: This program is based on some ideas (and source code lines) of
#          Felix Kugler, SWITCH, inflow-package and could be part of it.
#
# 961029 V1.0   released
# 970109 V1.1   change ls to readdir for sitelist
#               fix strip path statement
# 970113 V1.2   aligned some paths, renumbered version (FK)
# 970128 V1.2.1	added config of innfeeds checkpointing (GW)
# 970220 V1.2.2	enriched outfile naming (FK)
# 970224 V1.2.3	innfeed.0.10 support added (FK)
# 970305 V1.2.4	read sitelist from conffile instead lockfiles (GW)
# 970310 V1.2.5	minor bug fixes concerning innfeed.conf parsing  
# 970705 V1.2.6 multiple innfeed support (FK)
# 971012 V1.3.0 nntpsend support, several minor modifications (FK)
# 971016 V1.3.1 improved handling of html-includes (FK)
# 971027 V1.3.2 added opt. external config file (FK)
# 971029 V1.4   added approximate line count (GW)
#               interactive calculation for one site
# 971109 V1.4.2 minor cosmetical modifications
# 971112 V1.5.0 version number aligned
# 980117 V1.5.3 show index of innfeed/nntpsend if multiple instances exist (FK)
# 980312 V1.5.4 added status file (backlog evolution) + innfeed fix for implicit ip-name (MR)
# 980406 V1.6.0 incorporated URLs to neighbour servers with monitoring
# 980514 V2.0.0 version number aligned
# 980525 V2.0.2 minor fix in html-header/footer handling
# 980609 V2.0.3 added warning if ext config file could not be found
# 980618 V2.2.0 inflow.pm, restricted hostlists, table appearance
# 980822 V2.3.0 table appearance
# 980830 V2.3.1 bugs fixed in color handling and URL lookup

$Copy="(c) 1996 Gerhard.Winkler\@univie.ac.at";
$RELDATE = "Sun Aug 30 22:35:56 MET DST 1998";
$RELEASE = "V2.3.1";

## local settings ----------------------------------------------------
#
$INFLOWCONF = "/home/news/config/inflow.conf"; # ext. local configs (optional)
#
## the following settings may be modified here or alternativly using
#  an external config file; the latter method makes updates more comfortable...
#
# innfeed stuff
$IFSPOOLDIR   = "/var/spool/news/out.going";
@INNFEEDCONF  = ( '/usr/local/news/inn/innfeed.conf',
		 '/usr/local/news/inn/innfeed.conf-1',
		 '/usr/local/news/inn/innfeed.conf-2',
		 '/usr/local/news/inn/innfeed.conf-3');
$OLDCONFSTYLE = 0;       # pre-0.10 innfeed uses old format for innfeed.conf
$CHKPT_FL     = 0;       # some innfeed versions use .checkpoint files
                         # some use .input for storing checkpoint info (>0.9.3)
#
$STATUSFILE="/home/news/inflow/outflow.status";

#
# approximate line count stuff
# please check this value; it is site dependant
$MAGIC = 65;           # approximate line length in spool file
$THRES_MAGIC = 1300000;  # do exact line count if filesize is smaller than
                         # threshold else do approximate line count
#
# nntpsend stuff
$NSSPOOLDIR   = "/newsspool/out.going";
@NNTPSENDCONF = ( '/newslib/etc/nntpsend.ctl',);
@NSPOSTFIXES  = ( '', '!n', '.work', '.nntp' );
#
# naming stuff
%ABBREV     = ( 'innfeed', 'innfeed',
		'nntpsend', 'nntpsend',
	       );
$OUTFILE      = "/usr/local/etc/httpd/htdocs/stats/news/outflow";  # generic outfile name
$WANTHOSTNAME  = 0;       # add hostname to outfile name ( -host) ?
$GENERICTITLE  = "Queues of outgoing feeds";
$RGENERICTITLE = "Queues of selected outgoing feeds";
$BGCOLDARK  = "#6090C0";        # cell background dark (column titles, total)
$BGCOLLIGHT = "#80B0E0";        # cell background light (diffs)
$BGCOLLGREY = "#E0E0E0";        # cell background grey for better readability
$BGCOLWHITE = "#FFFFFF";        # cell background white for better readability
#
# external programs & config files
@ADDTOPATH = ( '/usr/local/bin', '/newslib/bin' );
#
$DATE         = "date";
$WC           = "wc";
$GREP         = "grep";
$UNAME        = "uname";
$RESOLVCONF   = "/etc/resolv.conf";
#
# where to get this software
$INFLOWURL    = "ftp://ftp.switch.ch/info_service/netnews/wg/tools/inflow.tar.gz";
#
$URLLOOKUP = "/home/news/config/olm-lookup";       # URLs to other News servers
$HOSTFILTER= "/home/news/config/inflow.hostlist";  # hosts to display or hide
#
# local HTML stuff: you can put custom header & footer information into 
#                   local files; optional...
#
$HTMLHEADER = "/misc/inflow/config/header.html";
$HTMLFOOTER = "/misc/inflow/config/footer.html";
# 
# end local settings ------------------------------------------------

# initialisation ----------------------------------------------------
#

($path,$0) = ($0 =~ /^(.*)\/([^\/]+)$/);  # strip path...
push(@INC,$path);

umask 022;                                # file mask

require "getopts.pl";
require "inflow.pm";

&Getopts('adf:hi:rwF');

&modify_config;
&update_PATH;
&gethostandfqdn;
&get_urllookup;
&get_hostfilter if $opt_r;
&get_env;

$Usage="$0    -  $Copy

Release $RELEASE  of $RELDATE

Process information from innfeed spoolfiles in $IFSPOOLDIR
and nntpsend batches in $NSSPOOLDIR. Show 
how many articles are currently waiting be sent to the remote site.
Output available either in Text or HTML format.

Usage: 	$0 [-adhirwF] [-f<configfile>]  

Parameters:

   -a:            approximate line count for batches >$THRES_MAGIC byte,
		  taking an average line lenght of $MAGIC byte as basis. 
   -d:            turn on verbosity; for debugging only
   -f<configfile>:load external configuration file 
                  default: $inflowconf
   -h:            This help.
   -i <sitename>: calculate this site only
   -r:            restrict results according to host filter list
                  $HOSTFILTER
   -w:            WWW support: show results in HTML table format
   -F:            write to file (defined in script header)

external config file: $inflowconfinfo

Optional HTML-stuff:

header file: $HTMLHEADER
footer file: $HTMLFOOTER

If several instances of innfeed/nntpsend are running, $0 shows 
to which one a feed is assigned. The config files have to honor one of the
following conventions (explained with an example each):

  innfeed:      innfeed1.conf, innfeed-1.conf innfeed.conf1 innfeed.conf-1
  nntpsend.ctl: accordingly...

Environment details: $envstr
\n";

$date = `$DATE`;
if ($WANTHOSTNAME) { $OUTFILE .= "-$hostname"; }
$titleline = $opt_r ? "$RGENERICTITLE on $fqdn" : "$GENERICTITLE on $fqdn";

if ($opt_h) { print "$Usage"; exit 0; }

$bgcoldark  = " bgcolor=$BGCOLDARK";
$bgcollight = " bgcolor=$BGCOLLIGHT";

if ($opt_F) {
    if ($opt_r) { $OUTFILE .= "-r"; };           # restricted output 
    if ($opt_w) { $OUTFILE .= ".html"; }
    open(OUT,">$OUTFILE.tmp") || die "can't open $OUTFILE\n";
    warn "writing to $OUTFILE\n" if $opt_d;
} else {
    open(OUT,">-");
}

#  check for custom header & footer files
if ($opt_w) {
    if (open(IN,$HTMLHEADER)) {  
	while (<IN>) {
	    $line = &interpret;
	    $htmlheader .= $line;
	}
	close(IN); 
    } else {
	$htmlheader = "<html><head>
          <!--Content-type: text/html\nRefresh: 60-->\n
          <title>$titleline</title></head><body><h2>$titleline</h2>\n";
    }
    if (open(IN,$HTMLFOOTER)) { 
	while (<IN>) {
	    $line = &interpret; 
	    $htmlfooter .= $line; 
	}
	close(IN);
    } else {
	$htmlfooter = "</body></html>\n";
    }
}

if ($opt_d) {
    warn "time: $date\n";
    warn "debug info: $envstr\n";
    warn "outfile: $OUTFILE\n";
    warn "use_links=$use_links\n";
}

&readconf;
&onesite if ($opt_i);
&readstatus;
&calculate;
&writestatus;
&initpage;
&writedata;
&closepage;

close(OUT);
if ($opt_F) {
	rename $OUTFILE, $OUTFILE.".old";
	rename $OUTFILE.".tmp", $OUTFILE;
}


# readstatus
# -----------------------------------------------------------------------------
# read past queue lengths to allow computation of evolution
#
sub readstatus {
    $now=time;
    if ( -f $STATUSFILE) { 
	open(S, $STATUSFILE);
	$then=<S>;
	while(<S>) {
	    ($thesite, $thecount)=split;
	    $backlog{$thesite}=$thecount;
	}
	close(S);
	$deltat=$now-$then;
    } else {
	$deltat=0;
    }
}

# writestatus
# -----------------------------------------------------------------------------
# write current queue lengths to allow future comparison
#
sub writestatus {
    open(S, "> $STATUSFILE.tmp");
    print S "$now\n";
    for $site  (keys(%thecount)) {
	print S "$site $thecount{$site}\n";
    }
    close(S);
    rename $STATUSFILE, $STATUSFILE.".old";
    rename $STATUSFILE.".tmp", $STATUSFILE;
}

# calculate
# -----------------------------------------------------------------------------
# main calculation part
# we go through the sitelist once for every feedtype supported
# and sort the results at the end...
sub calculate {
    # innfeed
    if (chdir $IFSPOOLDIR) {
	foreach $site (sort keys %sitelist) {
	    if ($havehostlist) {       # check with host filter list
		$res = &takesite($conflist{$site});
		if ($res > 0) {
		    warn "printnodetable: $site accepted\n" if $opt_d;
		} else {
		    warn "printnodetable: $site skipped\n" if $opt_d;
		    next;
		}
	    }
            $appr = 0;
	    next if ($feedprog{$site} ne 'innfeed');
            chdir $backlogdir{$site};
	    $file = $site . ".lock";
	    unless (-e $file) {
		print "no lock file found: $site\n" if ($opt_d);
# is there a particular problem calculating backlog if innfeed
# is not running ? MR
#		&printsite("no data");
#		next;
	    }
	    print "check site: $site\n" if ($opt_d);
	    
	    $scount = $icount = $ocount = $offset = 0;
	    $file = $site;
	    if (-e $file) {
		$scount = &countfile($file);
		print "found articles in $file: $scount\n" if ($opt_d);
	    } else {
		print "no $file\n" if ($opt_d);
	    }
	    
	    $file = $site . ".input";
	    if (-e $file) {
		$icount = &countfile($file);
		print "found articles in $file: $icount\n" if ($opt_d);
		if ($CHKPT_FL) { $check = $site . ".checkpoint"; }
		else { $check = $site . ".input"; }
		$offset = 0;
		if (open(CFL,"<$check")) {
		    $offset = <CFL>;
		    chop($offset);
		    close(CFL);
		}
		print "found offset: $offset\n" if ($opt_d);
		@status = stat($file);
		$size = $status[7];
		$icount = $icount - ($icount * $offset / $size);
		$icount = int($icount);
		print "found articles in $file minus offset: $icount\n" if ($opt_d);
	    } else {
		print "no $file\n" if ($opt_d);
	    }
	    
	    $file = $site . ".output";
	    if (-e $file) {
		$ocount = &countfile($file);
		print "found articles in $file: $ocount\n" if ($opt_d);
	    } else {
		print "no $file\n" if ($opt_d);
	    }
	    
	    $count = $scount + $icount + $ocount;
	    $thecount{$site}=$count;
#	    &printsite($count);
	}
    }

    # nntpsend
    if (chdir $NSSPOOLDIR) {
	foreach $site (sort keys %sitelist) {
	    if ($havehostlist) {   # check with host filter list
		$res = &takesite($site);
		if ($res > 0) {
		    warn "printnodetable: $site accepted\n" if $opt_d;
		} else {
		    warn "printnodetable: $site skipped\n" if $opt_d;
		    next;
		}
	    }
            $appr = 0;
	    next if ($feedprog{$site} ne 'nntpsend');
	    warn "check site $site\n" if $opt_d;
	    #
	    $count = $c = 0;
	    #
	    foreach $postfix (@NSPOSTFIXES) {
		$file = $site . $postfix;
		if (-e $file) {
		    $c = &countfile($file);
		    warn "nntpsend batch $file contains $c articles\n" if $opt_d;
		    $count += $c;
		} else {
		    warn "no file $file found in $NSSPOOLDIR\n" if $opt_d; 
		}
	    }
	    #
	    $thecount{$site}=$count;
#	    &printsite($count);
	}
    }
}


# countfile
# -----------------------------------------------------------------------------
# count number of articles (lines) in file
# maybe there will be some size (byte) of articles in it
sub countfile {
   local($f) = @_;
   local($ret,$c,$s);
   if ($opt_a) {
      @status = stat($f);
      $s = $status[7];
      if ($s > $THRES_MAGIC) {
         print "using approximate line count for $f\n" if ($opt_d);
         $appr = 1;
         $c = $s / $MAGIC;
         $c = int($c);
      }
      else {
         $ret = `$WC -l $f`;
         ($c) = ($ret =~ /^\s*(\d+)/);
      }
   }
   else {
      $ret = `$WC -l $f`;
      ($c) = ($ret =~ /^\s*(\d+)/);
   }
   return($c);
}


# readconf
# -----------------------------------------------------------------------------
# read innfeed/nntpsend configuration files and extract site names
# support for multiple conf files
# innfeed part requires version 0.10 and newer
sub readconf {
    # innfeed
    foreach $conffile (@INNFEEDCONF) {
	unless (open(F,"<$conffile")) {
	    warn "can\'t open innfeed config file $conffile\n" if $opt_d;
	    next;
	}
	warn "loading conf file: $conffile\n" if $opt_d;
	$conffilefound = 1;
	if ($conffile =~ /innfeed-?(\d+)\.conf$|innfeed\.conf-?(\d+)$/) {
	    $hasnumberedconfigs = 1;
	    $feednumber = $1;
	    warn "this is the config for innfeed $feednumber\n" if $opt_d; 
	}
	while(<F>) {
	    next if (/^\#/);
	    next if (/^$/);
	    $backlogdir=$1 if (/^\s*backlog-directory:\s+(\S+)/);

	    if (/^\s*peer\s+([^{}\s]+)/) { 
		$peer = $1; 
		$sitelist{$peer}++;
		$backlogdir{$peer}=$backlogdir;
		$feedprog{$peer} = 'innfeed';
		$feedid{$peer} = $feednumber;
		$conflist{$peer} = $peer;
	    }
	    if (/ip-name:\s+(\S+)/ && $peer) {
		$ipname = $1;
		$conflist{$peer} = $ipname;
		print "conflist{$peer} = $ipname\n" if $opt_d;
		$peer = "";
	    }
	}
	close(F);
    }

    # nntpsend
    foreach $conffile (@NNTPSENDCONF) {
	unless (open(F,"<$conffile")) {
	    warn "can\'t open nntpsend config file $conffile\n" if $opt_d;
	    next;
	}
	$conffilefound = 1;
	if ($conffile =~ /nntpsend-?(\d+)\.conf$|nntpsend\.conf-?(\d+)$/) {
	    $hasnumberedconfigs = 1;
	    $feednumber = $1;
	    warn "this is the config for nntpsend $feednumber\n" if $opt_d;
	}
	while(<F>) {
	    next if (/^\#/);
	    next if (/^$/);
	    next if (/^default:/);
	    ($peer,$ipname) = split(/:/);
	    $sitelist{$peer}++;
	    $conflist{$peer} = $ipname;
	    $feedprog{$peer} = 'nntpsend';
	    $feedid{$peer} = $nntpsendnumber;
	}
    }
    die "couldn\'t find any config files - aborting\n" unless $conffilefound;
    if ($opt_d) {
       warn "\nfound following sites:\n";
       foreach $peer (sort keys %conflist) {
	   warn "$peer: $conflist{$peer}, $feedprog{$peer}:$feedid{$peer}\n";
       }
       warn "\n\n";
    }
}



# onesite
# -----------------------------------------------------------------------------
# calculate for one site

sub onesite {
   local($i,$s);
   $s = $opt_i;

   print "\ncalculate onesite: $s\n" if ($opt_d);
   foreach $i (keys %sitelist) {
      if ($i ne $s) { delete $sitelist{$i}; }
   }
   if ((keys %sitelist) == 0) {
      print "site $s not found in config --> abort\n";
      exit;
   }

}


# printsite
# -----------------------------------------------------------------------------
# assemble result line for one site, do NOT print yet !!

sub printsite {
   local($site) = @_;
   local($rowbgcolor) = ($line%2 == 0) ? " bgcolor=$BGCOLLGREY" : " bgcolor=$BGCOLWHITE";

   $theconflist{$site}=$conflist{$site};
   $thefeedprog{$site}=$ABBREV{$feedprog{$site}};
   $thefeedid{$site}=$feedid{$site};

   if ($deltat>300) {
	   $backlog=int(3600*($thecount{$site}-$backlog{$site})/$deltat);
   } else {
	   $backlog="&nbsp;";	
   }
   $backlog="=" if ($backlog==0);
   $backlog="+".$backlog if ($backlog>0);

   if ($opt_w) {
      $thecount{$site} = "~ " . $thecount{$site} if ($appr);
      if ($hasnumberedconfigs && $feedid{$site} eq "") { # html cosmetics
	  $feedid{$site} = "&nbsp;"; 
      }
      $siteentry = $conflist{$site};  # set default
      if ($use_links) {               # add URL for html-output if available
	  if ($srvrname{$conflist{$site}}) {
	      $siteentry = "<A HREF=\"$moni_url{$srvrname{$conflist{$site}}}\">$conflist{$site}</A>";
	      warn "URL ($conflist{$site}): $moni_url{$srvrname{$conflist{$site}}}\n" if $opt_d;
	  } else {
	      ($truncated) = ($conflist{$site} =~ /^[^.]+\.([^.]+\..*)$/);
	      if ($truncated ne "" && $srvrname{$truncated}) {
		  warn "URL (truncated: $truncated): $moni_url{$srvrname{$truncated}}\n" if $opt_d;
		  $siteentry = "<A HREF=\"$moni_url{$srvrname{$truncated}}\">$conflist{$site}</A>";
	      } else { warn "no URL available for $conflist{$site}...\n" if $opt_d; }
	  }
      }
      $result{$site} = "<TR$rowbgcolor>
                 <TD> $site </TD>
                 <TD> $siteentry </TD>
                 <TD align=left> $ABBREV{$feedprog{$site}} </TD>
                 <TD align=right> $feedid{$site} </TD>
                 <TD align=right> $thecount{$site} </TD>
                 <TD align=right> $backlog </TD>
                 </TR>\n";
   } else {
      $thecount{$site} = "~ " . $thecount{$site} if ($appr);
      $result{$site} = sprintf("%15s %35s %8s %2s %10s\n",
		       $site,$conflist{$site},$ABBREV{$feedprog{$site}},
		       $feedid{$site},$c);
   }
}


# initpage
# -----------------------------------------------------------------------------
# Print Page Headers 
#
sub initpage {
    local($feedtypecaption) = $hasnumberedconfigs ? "type + idx" : "type";
    if ($opt_w) { 
	print OUT "$htmlheader
                   <p><b>calculated at:</b> $date</p>\n";
        print OUT "<center><TABLE BORDER>
                   <TR$bgcoldark><TH align=left>feedname</TH>
                   <TH align=left>sitename</TH>
                   <TH colspan=2 align=center>$feedtypecaption</TH>
                   <TH>Articles<br>to transfer</TH>
                   <TH>Evolution<br>art/h</TH>
                   </TR>\n";
    } else { 
        print OUT "$titleline\n\n";
        print OUT "calculated at: $date\n";
	printf(OUT "%15s %35s %11s %10s\n\n",
	       "Feedname","Sitename",$feedtypecaption,"Articles to transfer");
    }
}


# writedata
# -----------------------------------------------------------------------------
# sort and write results
#
sub writedata {
    foreach $site (sort {$thecount{$b} <=> $thecount{$a}} keys (%thecount)) {
	&printsite($site);
	print OUT "$result{$site}\n";
	$line++;
    }
}

