#!/usr/bin/perl -w

# $Id: check-active-file-vs-isc-list,v 1.5 2006/11/20 03:00:34 dougmc Exp $
#
# check-active-file-vs-isc-list
# by Doug McLaren
# dougmc+b8mb [a@t] frenzied [dot] us <- email address, obfuscated.
# 2006-11-19
#
# This script is designed to compare the active file on your NNTP
# server vs. the master list kept at ftp.isc.org to see which groups
# should (could) be added, removed or changed.  It does not require
# any special privileges to run beyond access to a NNTP server and to
# the Internet, but of course you will need admin access to the NNTP
# server itself to make any changes.
#
# It will not actually make any changes by itself, but can instead
# give you a list of commands that can be run (if you're running innd)
# to make your server match the master list.  (If you're not running
# innd, send me the equivilent commands for your server and I'll add
# support for it.)
#
# I don't suggest blinding running the commands that this script emits
# -- instead, read them over and make sure they make sense before
# applying any of them, especially the commands involing the removal
# of existing groups (because your local users may be using these
# groups!)
#
# Your news server is your business, and I'm not telling you how to
# run it.  Instead, this is just a tool that may be of some assistance
# in making the list of newsgroups your server carries be more
# `standard', and if run occasionally may reduce/remove the need to
# enable PGP verification and application of control messages.
# (Basically what you'd be doing is trusting them to properly manage
# the creation and removal of groups instead of doing it yourself.)
#
# This script was written to run on your typical *nix system, but
# should also work fine under cygwin.  You may need to install the
# Net::NNTP and Net::FTP modules if not already installed.  It also
# relies on having gunzip available, but this requirement can be
# removed if you remove the .gz from the files it downloads from the
# isc.org site.  (Of course, it will then use more of their
# bandwidth.)
#
# Available options are listed in the script itself shortly after
# this text.
#
# This script is hereby released into the public domain, though I'd
# appreciate my name/email address not being removed, and being sent
# back any useful improvements to the script (also put into the public
# domain) so I can incorporate them into the script.  (Of course,
# being that I released it into the public domain, I cannot demand
# these things!)

use warnings ;
use strict ;
use Net::NNTP;
use Net::FTP ;
use Getopt::Std ;

# Where do we get our master file?
my $remote_host     = "ftp.isc.org" ;
my $active_file     = "/pub/usenet/CONFIG/active.gz" ;
my $newsgroups_file = "/pub/usenet/CONFIG/newsgroups.gz" ;

# Here's a list of the options we support :
#
# -n {} specifies the NNTP server to use.  If not set, we'll
#       use the NNTPSERVER variable.
# -u {} the username to use to talk to the NNTP server
# -p {} the password to use to talk to the NNTP server
# -i {} groups to include in our scan, regular expression
#       the default is: ^(comp|news|sci|humanities|rec|soc|talk|misc)\.
# -x {} groups to exclude in our scan, regular expression
#       Default is: \.binaries\.
#       To disable this entirely, use a string that won't match
#       any groups, like `-x dsagy54ghrghdf'
# -t {} tmp directory to work in.  Defaults to /tmp/check-active-$$
#       Note that the default value can be a security hazard on a multi-user
#       system!
# -n    don't get the newsgroups description file from the ISC.
# -k    keep downloaded files.
# -s {} save a list of commands to execute into this file.
# -h    very quick usage summary.

use vars qw($opt_n $opt_u $opt_p $opt_i $opt_x $opt_t $opt_d $opt_k $opt_s $opt_h) ;
getopts("n:u:p:i:x:t:dks:h") ;

if ($opt_h) {
   print "\n" ;
   print "Usage summary :\n" ;
   print "\n" ;
   print "$0 \\\n" ;
   print "   [-n NNTPSERVER] [-u USER] [-p PASSWORD] [-t tmpdir] \\\n" ;
   print "   [-i regex] [-x regex] [-s script-to-save] [-nkh]\n" ;
   print "\n" ;
   print "Read the script itself for a better description of these.\n" ;
   print "options.\n" ;
   exit 0 ;
}

my $nntpserver ;
$nntpserver = $ENV{"NNTPSERVER"} if ($ENV{"NNTPSERVER"}) ;
$nntpserver = $opt_n if ($opt_n) ;

die "Cannot determine your nntpserver.  Set \$NNTPSERVER or use the -n flag.\n"
   if (! $nntpserver) ;

my $group_include = '^(comp|news|sci|humanities|rec|soc|talk|misc)\.' ;
my $group_exclude = '\.binaries\.' ;
$group_include = $opt_i if ($opt_i) ;
$group_exclude = $opt_x if ($opt_x) ;

# Commands for adding/creating/modifying groups.
# Of course, if your software doesn't understand the
# newgroup/changegroup syntax of specifying the
# moderated status after the group name, you'll
# have to change more than just these lines.
my $server_software_name   = "inn" ;
my $server_software_create = "ctlinnd newgroup" ;
my $server_software_remove = "ctlinnd rmgroup" ;
my $server_software_modify = "ctlinnd changegroup" ;

my $tmp_directory = "/tmp/check-active-$$" ;
if ($opt_t) { $tmp_directory = $opt_t } ;

$| = 1 ;

print "Connecting to $nntpserver ..." ;
my $nntp = Net::NNTP->new($nntpserver) or
   die "Cannot connect to $nntpserver.\n" ;
print " done\n" ;

if ($opt_u or $opt_p) {
   print "Logging in as user $opt_u ..." ;
   $nntp->authinfo($opt_u, $opt_p) or die "auth failed: $!\n" ;
   print " done\n" ;
}

print "Getting newsgroup list from $nntpserver ..." ;

my %nntp_groups         = () ;
my %nntp_count          = () ; # this may not be accurate, of course ...
my %master_groups       = () ;
my %master_descriptions = () ;

my $we_created_tmp_directory = 0 ;
if (! -d $tmp_directory) {
   mkdir $tmp_directory or die "mkdir $tmp_directory failed: $!\n" ;
   $we_created_tmp_directory++ ;
}

{
   my $list          = $nntp->list ;
   my $count         = 0 ;
   my $count_we_care = 0 ;

   my $file_to_save_in = "$tmp_directory/$nntpserver-active" ;
   if ($opt_k) {
      open O, "> $file_to_save_in" or die "$!" ;
   }

   foreach my $group (sort keys %$list) {
      $count++ ;
      print O join (" ", $group, @{$$list{$group}}), "\n" if ($opt_k) ;
      if (($group =~ /$group_include/) and
         ($group !~ /$group_exclude/)) {
         my ($last, $first, $status) = (@{$$list{$group}})[0,1,2] ;
         $nntp_groups{$group} = lc($status) ; # should always be lc
         $nntp_count{$group} = $last - $first ;
         $nntp_count{$group} = "no" if ($nntp_count{$group} < 1) ;
         $count_we_care++
      }
   }
   $nntp->quit ;
   print " done.\n" ;
   print "... $count groups found, $count_we_care we care about.\n" ;
   if ($opt_k) {
      close O ;
      print "... saving $file_to_save_in\n" ;
   }
}

my $ftp = Net::FTP->new($remote_host) or
   die "Cannot connect to $remote_host: $@\n" ;

{
   print "Getting newsgroup list from $remote_host ..." ;

   $ftp->login ("anonymous", "anonymous@") or
      die "Cannot log into $remote_host: $@\n" ;
   $ftp->binary ;

   my $dest = $active_file ;
   $dest =~ s!.*/!! ;
   $dest = join "/", $tmp_directory, join ("-", $remote_host, $dest) ;
   $ftp->get ($active_file, $dest) or
      die "get $active_file from $remote_host failed: $@\n" ;
   if ($dest =~ /\.gz/) {
      open I, "gunzip -c $dest |" or die "$!" ;
   } else {
      open I, $dest or die "$!" ;
   }
   my $count         = 0 ;
   my $count_we_care = 0 ;

   while (<I>) {
      chomp ;
      my ($group, $last, $first, $status) = split / / ;
      $count++ ;
      if (($group =~ /$group_include/) and
         ($group !~ /$group_exclude/)) {
         $master_groups{$group} = lc($status) ;
         $count_we_care++
      }
   }
   close I ;
   print " done.\n" ;
   print "... $count groups found, $count_we_care we care about.\n" ;
   if ($opt_k) {
      close O ;
      print "... saving $dest\n" ;
   } else {
      unlink $dest or warn "unlink $dest failed: $!\n" ;
   }
}

if (! $opt_d) {

   print "Getting newsgroup descriptions from $remote_host ..." ;

   my $dest = $newsgroups_file ;
   $dest =~ s!.*/!! ;
   $dest = join "/", $tmp_directory, join ("-", $remote_host, $dest) ;
   $ftp->get ($newsgroups_file, $dest) or
      die "get $active_file from $remote_host failed: $@\n" ;
   if ($dest =~ /\.gz/) {
      open I, "gunzip -c $dest |" or die "$!" ;
   } else {
      open I, $dest or die "$!" ;
   }
   my $count         = 0 ;
   my $count_we_care = 0 ;
   while (<I>) {
      chomp ;
      if (/^(\S+)\s+(\S.+)$/) {
         my $group = $1 ;
         my $description = $2 ;
         $count++ ;
         if (($group =~ /$group_include/) and
            ($group !~ /$group_exclude/)) {
            $master_descriptions{$group} = $description ;
            $count_we_care++
         }
      }
   }
   close I ;
   print " done.\n" ;
   print "... $count group descriptions found, $count_we_care we care about.\n" ;

   if ($opt_k) {
      close O ;
      print "... saving $dest\n" ;
   } else {
      unlink $dest or warn "unlink $dest failed: $!\n" ;
   }
}
$ftp->quit ;

rmdir $tmp_directory if ($we_created_tmp_directory and (! $opt_k)) ;

# And now that we have our data, we can process it.

print "\n" ;
print "Looking for differences between $remote_host and $nntpserver ...\n" ;
print "\n" ;

my @commands = () ;

foreach my $group (sort keys %master_groups) {
   if (! $nntp_groups{$group}) {
      my $description = $master_descriptions{$group} ;

      if ($master_groups{$group} eq "m") {
         print "* $group (moderated) should be created.\n" ;
      } else {
         print "* $group should be created.\n" ;
      }
      print "  $description\n" if ($description) ;

      push @commands, "# $group = $master_descriptions{$group}" if ($description) ;
      push @commands, "$server_software_create $group $master_groups{$group}" ;
   }
}

foreach my $group (sort keys %nntp_groups) {
   if (! $master_groups{$group}) {
      my $count = $nntp_count{$group} ;
      print "* $group should be removed (currently has $count posts.)\n" ;
      push @commands, "# $group currently has $count posts in queue" ;
      push @commands, "$server_software_remove $group" ;
   }
}

foreach my $group (sort keys %nntp_groups) {
   if ($master_groups{$group}) {
      my $rest1 = $nntp_groups{$group} ;
      my $rest2 = $master_groups{$group} ;
      if ($rest1 ne $rest2) {
         if ($rest1 eq "m") {
            print "* $group is listed as moderated but should not be.\n" ;
         }
         if ($rest2 eq "m") {
            print "* $group is listed as not moderated but should be.\n" ;
         }
         push @commands, "$server_software_modify $group $rest2" ;

      }
   }
}

print "\n" ;

if (@commands) {
   if ($opt_s) {
      open O, "> $opt_s" or die "$!" ;
      print O "#!/bin/sh -x\n" ;
      print O "\n" ;
      print O "# This list of commands will bring your active file in sync\n" ;
      print O "# with the master list at $remote_host as of ",
            scalar localtime(time), ".\n" ;
      print O "# for users of the $server_software_name news server. Other news servers may require\n" ;
      print O "# some modifications to the commands given.\n" ;
      print O "\n" ;
      foreach my $command (@commands) {
         print O "$command\n" ;
      }
      close O ;
      print "List of commands saved in `$opt_s' for perusal/execution.\n" ;

   } else {

      print "By giving this script the `-s <file>' flag, it will save a list\n" ;
      print "of commands (for $server_software_name) to execute against your news server to bring things\n" ;
      print "in line with the master list at $remote_host.\n" ;
   }


} else {
   print "Congratulations!  Your active file matches the master active file.\n" ;
   print "(at least for the groups that we're looking at.)\n" ;
}