[usenet/newsstats.git] / gatherstats.pl

#! /usr/bin/perl -W
#
# gatherstats.pl
#
# This script will gather statistical information from a database
# containing headers and other information from a INN feed.
# 
# It is part of the NewsStats package.
#
# Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
#
# It can be redistributed and/or modified under the same terms under 
# which Perl itself is published.

BEGIN {
  our $VERSION = "0.01";
  use File::Basename;
  push(@INC, dirname($0));
}
use strict;

use NewsStats qw(:DEFAULT :TimePeriods ListNewsgroups ReadGroupList);

use DBI;
use Getopt::Long qw(GetOptions);
Getopt::Long::config ('bundling');

################################# Definitions ##################################

# define types of information that can be gathered
# all / groups (/ clients / hosts)
my %LegalStats;
@LegalStats{('all','groups')} = ();

################################# Main program #################################

### read commandline options
my ($OptCheckgroupsFile,$OptClientsDB,$OptDebug,$OptGroupsDB,$OptTLH,
    $OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest);
GetOptions ('c|checkgroups=s' => \$OptCheckgroupsFile,
            'clientsdb=s'     => \$OptClientsDB,
            'd|debug!'        => \$OptDebug,
            'groupsdb=s'      => \$OptGroupsDB,
            'hierarchy=s'     => \$OptTLH,
            'hostsdb=s'       => \$OptHostsDB,
            'm|month=s'       => \$OptMonth,
            'rawdb=s'         => \$OptRawDB,
            's|stats=s'       => \$OptStatsType,
            't|test!'         => \$OptTest,
            'h|help'          => \&ShowPOD,
            'V|version'       => \&ShowVersion) or exit 1;

### read configuration
my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};

### override configuration via commandline options
my %ConfOverride;
$ConfOverride{'DBTableRaw'}   = $OptRawDB if $OptRawDB;
$ConfOverride{'DBTableGrps'}  = $OptGroupsDB if $OptGroupsDB;
$ConfOverride{'DBTableClnts'} = $OptClientsDB if $OptClientsDB;
$ConfOverride{'DBTableHosts'} = $OptHostsDB if $OptHostsDB;
$ConfOverride{'TLH'} = $OptTLH if $OptTLH;
&OverrideConfig(\%Conf,\%ConfOverride);

### get type of information to gather, defaulting to 'all'
$OptStatsType = 'all' if !$OptStatsType;
&Bleat(2, sprintf("Unknown type '%s'!", $OptStatsType))
  if !exists($LegalStats{$OptStatsType});

### get time period from --month
# get verbal description of time period, drop SQL code
my ($Period) = &GetTimePeriod($OptMonth);
&Bleat(2,"--month option has an invalid format - please use 'YYYY-MM' or ".
         "'YYYY-MM:YYYY-MM'!") if (!$Period or $Period eq 'all time');

### reformat $Conf{'TLH'}
my $TLH;
if ($Conf{'TLH'}) {
  # $Conf{'TLH'} is parsed as an array by Config::Auto;
  # make a flat list again, separated by :
  if (ref($Conf{'TLH'}) eq 'ARRAY') {
    $TLH = join(':',@{$Conf{'TLH'}});
  } else {
    $TLH  = $Conf{'TLH'};
  }
  # strip whitespace
  $TLH =~ s/\s//g;
  # add trailing dots if none are present yet
  # (using negative look-behind assertions)
  $TLH =~ s/(?<!\.):/.:/g;
  $TLH =~ s/(?<!\.)$/./;
  # check for illegal characters
  &Bleat(2,'Config error - illegal characters in TLH definition!')
    if ($TLH !~ /^[a-zA-Z0-9:+.-]+$/);
  # escape dots
  $TLH =~ s/\./\\./g;
  if ($TLH =~ /:/) {
    # reformat $TLH from a:b to (a)|(b),
    # e.g. replace ':' by ')|('
    $TLH =~ s/:/)|(/g;
    $TLH = '(' . $TLH . ')';
  };
};

### init database
my $DBHandle = InitDB(\%Conf,1);

### get data for each month
&Bleat(1,'Test mode. Database is not updated.') if $OptTest;
foreach my $Month (&ListMonth($Period)) {

  print "---------- $Month ----------\n" if $OptDebug;

  if ($OptStatsType eq 'all' or $OptStatsType eq 'groups') {
    # read list of newsgroups from --checkgroups
    # into a hash
    my %ValidGroups = %{ReadGroupList(sprintf('%s-%s',$OptCheckgroupsFile,$Month))}
      if $OptCheckgroupsFile;

    ### ----------------------------------------------
    ### get groups data (number of postings per group)
    # get groups data from raw table for given month
    my $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups FROM %s.%s ".
                                             "WHERE day LIKE ? AND NOT disregard",
                                             $Conf{'DBDatabase'},
                                             $Conf{'DBTableRaw'}));
    $DBQuery->execute($Month.'-%')
      or &Bleat(2,sprintf("Can't get groups data for %s from %s.%s: ".
                          "$DBI::errstr\n",$Month,
                          $Conf{'DBDatabase'},$Conf{'DBTableRaw'}));

    # count postings per group
    my %Postings;
    while (($_) = $DBQuery->fetchrow_array) {
      # get list of newsgroups and hierarchies from Newsgroups:
      my %Newsgroups = ListNewsgroups($_,$TLH,
                                      $OptCheckgroupsFile ? \%ValidGroups : '');
      # count each newsgroup and hierarchy once
      foreach (sort keys %Newsgroups) {
        $Postings{$_}++;
      };
    };

    # add valid but empty groups if --checkgroups is set
    if (%ValidGroups) {
      foreach (sort keys %ValidGroups) {
        if (!defined($Postings{$_})) {
          # expand newsgroup with hierarchies
          my @Newsgroups = ParseHierarchies($_);
          # add each empty newsgroup and empty hierarchies, too, as needed
          foreach (@Newsgroups) {
            if (!defined($Postings{$_})) {
              $Postings{$_} = 0;
              warn (sprintf("ADDED: %s as empty group\n",$_));
            };
          };
        }
      };
    };
    
    # delete old data for that month
    if (!$OptTest) {
      $DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s WHERE month = ?",
                                       $Conf{'DBDatabase'},$Conf{'DBTableGrps'}),
                                       undef,$Month)
        or &Bleat(2,sprintf("Can't delete old groups data for %s from %s.%s: ".
                            "$DBI::errstr\n",$Month,
                            $Conf{'DBDatabase'},$Conf{'DBTableGrps'}));
    };

    print "----- GroupStats -----\n" if $OptDebug;
    foreach my $Newsgroup (sort keys %Postings) {
      print "$Newsgroup => $Postings{$Newsgroup}\n" if $OptDebug;
      if (!$OptTest) {
        # write to database
        $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s ".
                                              "(month,newsgroup,postings) ".
                                              "VALUES (?, ?, ?)",
                                              $Conf{'DBDatabase'},
                                              $Conf{'DBTableGrps'}));
        $DBQuery->execute($Month, $Newsgroup, $Postings{$Newsgroup})
          or &Bleat(2,sprintf("Can't write groups data for %s/%s to %s.%s: ".
                              "$DBI::errstr\n",$Month,$Newsgroup,
                              $Conf{'DBDatabase'},$Conf{'DBTableGrps'}));
        $DBQuery->finish;
      };
    };
  } else {
    # other types of information go here - later on
  };
};

### close handles
$DBHandle->disconnect;

__END__

################################ Documentation #################################

=head1 NAME

gatherstats - process statistical data from a raw source

=head1 SYNOPSIS

B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> | I<YYYY-MM:YYYY-MM>] [B<-s> I<stats] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>]

=head1 REQUIREMENTS

See L<doc/README>.

=head1 DESCRIPTION

This script will extract and process statistical information from a
database table which is fed from F<feedlog.pl> for a given time period
and write its results to (an)other database table(s). Entries marked
with I<'disregard'> in the database will be ignored; currently, you
have to set this flag yourself, using your database management tools.
You can exclude erroneous entries that way (e.g. automatic reposts
(think of cancels flood and resurrectors); spam; ...).

The time period to act on defaults to last month; you can assign
another time period or a single month via the B<--month> option (see
below).

By default B<gatherstats> will process all types of information; you
can change that using the B<--stats> option and assigning the type of
information to process. Currently that doesn't matter yet as only
processing of the number of postings per group per month is
implemented anyway.

Possible information types include:

=over 3

=item B<groups> (postings per group per month)

B<gatherstats> will examine Newsgroups: headers. Crosspostings will be
counted for each single group they appear in. Groups not in I<TLH>
will be ignored.

B<gatherstats> will also add up the number of postings for each
hierarchy level, but only count each posting once. A posting to
de.alt.test will be counted for de.alt.test, de.alt.ALL and de.ALL,
respectively. A crossposting to de.alt.test and de.alt.admin, on the
other hand, will be counted for de.alt.test and de.alt.admin each, but
only once for de.alt.ALL and de.ALL.

Data is written to I<DBTableGrps> (see L<doc/INSTALL>); you can
override that default through the B<--groupsdb> option.

=back

=head2 Configuration

B<gatherstats> will read its configuration from F<newsstats.conf>
which should be present in the same directory via Config::Auto.

See L<doc/INSTALL> for an overview of possible configuration options.

You can override configuration options via the B<--hierarchy>,
B<--rawdb>, B<--groupsdb>, B<--clientsdb> and B<--hostsdb> options,
respectively.

=head1 OPTIONS

=over 3

=item B<-V>, B<--version>

Print out version and copyright information and exit.

=item B<-h>, B<--help>

Print this man page and exit.

=item B<-d>, B<--debug>

Output debugging information to STDOUT while processing (number of
postings per group).

=item B<-t>, B<--test>

Do not write results to database. You should use B<--debug> in
conjunction with B<--test> ... everything else seems a bit pointless.

=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]>

Set processing period to a single month in YYYY-MM format or to a time
period between two month in YYYY-MM:YYYY-MM format (two month, separated
by a colon). 


=item B<-s>, B<--stats> I<type>

Set processing type to one of I<all> and I<groups>. Defaults to all
(and is currently rather pointless as only I<groups> has been
implemented).

=item B<-c>, B<--checkgroups> I<filename template>

Check each group against a list of valid newsgroups read from a file,
one group on each line and ignoring everything after the first
whitespace (so you can use a file in checkgroups format or (part of)
your INN active file).

The filename is taken from I<filename template>, amended by each B<--
month> B<gatherstats> is processing, so that

    gatherstats -m 2010-01:2010-12 -c checkgroups

will check against F<checkgroups-2010-01> for January 2010, against
F<checkgroups-2010-02> for February 2010 and so on.

Newsgroups not found in the checkgroups file will be dropped (and
logged to STDERR), and newsgroups found there but having no postings
will be added with a count of 0 (and logged to STDERR).

=item B<--hierarchy> I<TLH> (newsgroup hierarchy)

Override I<TLH> from F<newsstats.conf>.

=item B<--rawdb> I<table> (raw data table)

Override I<DBTableRaw> from F<newsstats.conf>.

=item B<--groupsdb> I<table> (postings per group table)

Override I<DBTableGrps> from F<newsstats.conf>.

=item B<--clientsdb> I<table> (client data table)

Override I<DBTableClnts> from F<newsstats.conf>.

=item B<--hostsdb> I<table> (host data table)

Override I<DBTableHosts> from F<newsstats.conf>.

=back

=head1 INSTALLATION

See L<doc/INSTALL>.

=head1 EXAMPLES

Process all types of information for lasth month:

    gatherstats

Do a dry run, showing results of processing:

    gatherstats --debug --test

Process all types of information for January of 2010:

    gatherstats --month 2010-01

Process only number of postings for the year of 2010,
checking against checkgroups-*:

    gatherstats -m 2010-01:2010-12 -s groups -c checkgroups

=head1 FILES

=over 4

=item F<gatherstats.pl>

The script itself.

=item F<NewsStats.pm>

Library functions for the NewsStats package.

=item F<newsstats.conf>

Runtime configuration file.

=back

=head1 BUGS

Please report any bugs or feature requests to the author or use the
bug tracker at L<http://bugs.th-h.de/>!

=head1 SEE ALSO

=over 2

=item -

L<doc/README>

=item -

L<doc/INSTALL>

=back

This script is part of the B<NewsStats> package.

=head1 AUTHOR

Thomas Hochstein <thh@inter.net>

=head1 COPYRIGHT AND LICENSE

Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>

This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

=cut
Commit	Line	Data
2832c235 TH	1	#! /usr/bin/perl -W
	2	#
	3	# gatherstats.pl
	4	#
	5	# This script will gather statistical information from a database
	6	# containing headers and other information from a INN feed.
	7	#
	8	# It is part of the NewsStats package.
	9	#
880c3eb2	10	# Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
2832c235 TH	11	#
	12	# It can be redistributed and/or modified under the same terms under
	13	# which Perl itself is published.
	14
	15	BEGIN {
	16	our $VERSION = "0.01";
	17	use File::Basename;
	18	push(@INC, dirname($0));
	19	}
	20	use strict;
	21
ad609792	22	use NewsStats qw(:DEFAULT :TimePeriods ListNewsgroups ReadGroupList);
2832c235 TH	23
2832c235 TH	24	use DBI;
880c3eb2 TH	25	use Getopt::Long qw(GetOptions);
880c3eb2 TH	26	Getopt::Long::config ('bundling');
2832c235 TH	27
	28	################################# Definitions ##################################
	29
	30	# define types of information that can be gathered
	31	# all / groups (/ clients / hosts)
880c3eb2 TH	32	my %LegalStats;
880c3eb2 TH	33	@LegalStats{('all','groups')} = ();
2832c235 TH	34
	35	################################# Main program #################################
	36
	37	### read commandline options
880c3eb2 TH	38	my ($OptCheckgroupsFile,$OptClientsDB,$OptDebug,$OptGroupsDB,$OptTLH,
	39	$OptHostsDB,$OptMonth,$OptRawDB,$OptStatsType,$OptTest);
	40	GetOptions ('c\|checkgroups=s' => \$OptCheckgroupsFile,
	41	'clientsdb=s' => \$OptClientsDB,
	42	'd\|debug!' => \$OptDebug,
	43	'groupsdb=s' => \$OptGroupsDB,
	44	'hierarchy=s' => \$OptTLH,
	45	'hostsdb=s' => \$OptHostsDB,
	46	'm\|month=s' => \$OptMonth,
	47	'rawdb=s' => \$OptRawDB,
	48	's\|stats=s' => \$OptStatsType,
	49	't\|test!' => \$OptTest,
	50	'h\|help' => \&ShowPOD,
	51	'V\|version' => \&ShowVersion) or exit 1;
2832c235 TH	52
2832c235 TH	53	### read configuration
880c3eb2	54	my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
2832c235 TH	55
	56	### override configuration via commandline options
	57	my %ConfOverride;
880c3eb2 TH	58	$ConfOverride{'DBTableRaw'} = $OptRawDB if $OptRawDB;
	59	$ConfOverride{'DBTableGrps'} = $OptGroupsDB if $OptGroupsDB;
	60	$ConfOverride{'DBTableClnts'} = $OptClientsDB if $OptClientsDB;
	61	$ConfOverride{'DBTableHosts'} = $OptHostsDB if $OptHostsDB;
	62	$ConfOverride{'TLH'} = $OptTLH if $OptTLH;
2832c235 TH	63	&OverrideConfig(\%Conf,\%ConfOverride);
	64
	65	### get type of information to gather, defaulting to 'all'
880c3eb2 TH	66	$OptStatsType = 'all' if !$OptStatsType;
	67	&Bleat(2, sprintf("Unknown type '%s'!", $OptStatsType))
	68	if !exists($LegalStats{$OptStatsType});
2832c235	69
880c3eb2 TH	70	### get time period from --month
	71	# get verbal description of time period, drop SQL code
	72	my ($Period) = &GetTimePeriod($OptMonth);
	73	&Bleat(2,"--month option has an invalid format - please use 'YYYY-MM' or ".
	74	"'YYYY-MM:YYYY-MM'!") if (!$Period or $Period eq 'all time');
2832c235	75
17ffbeba TH	76	### reformat $Conf{'TLH'}
	77	my $TLH;
	78	if ($Conf{'TLH'}) {
	79	# $Conf{'TLH'} is parsed as an array by Config::Auto;
	80	# make a flat list again, separated by :
43a0fc77	81	if (ref($Conf{'TLH'}) eq 'ARRAY') {
17ffbeba TH	82	$TLH = join(':',@{$Conf{'TLH'}});
	83	} else {
	84	$TLH = $Conf{'TLH'};
	85	}
	86	# strip whitespace
	87	$TLH =~ s/\s//g;
7773fb6d TH	88	# add trailing dots if none are present yet
	89	# (using negative look-behind assertions)
	90	$TLH =~ s/(?<!\.):/.:/g;
	91	$TLH =~ s/(?<!\.)$/./;
17ffbeba	92	# check for illegal characters
880c3eb2	93	&Bleat(2,'Config error - illegal characters in TLH definition!')
314e31aa	94	if ($TLH !~ /^[a-zA-Z0-9:+.-]+$/);
7773fb6d TH	95	# escape dots
7773fb6d TH	96	$TLH =~ s/\./\\./g;
17ffbeba	97	if ($TLH =~ /:/) {
880c3eb2	98	# reformat $TLH from a:b to (a)\|(b),
43a0fc77	99	# e.g. replace ':' by ')\|('
17ffbeba TH	100	$TLH =~ s/:/)\|(/g;
	101	$TLH = '(' . $TLH . ')';
	102	};
	103	};
	104
2832c235 TH	105	### init database
	106	my $DBHandle = InitDB(\%Conf,1);
	107
	108	### get data for each month
880c3eb2 TH	109	&Bleat(1,'Test mode. Database is not updated.') if $OptTest;
880c3eb2 TH	110	foreach my $Month (&ListMonth($Period)) {
2832c235	111
880c3eb2	112	print "---------- $Month ----------\n" if $OptDebug;
2832c235	113
880c3eb2	114	if ($OptStatsType eq 'all' or $OptStatsType eq 'groups') {
93c8eae2 TH	115	# read list of newsgroups from --checkgroups
	116	# into a hash
	117	my %ValidGroups = %{ReadGroupList(sprintf('%s-%s',$OptCheckgroupsFile,$Month))}
	118	if $OptCheckgroupsFile;
	119
2832c235 TH	120	### ----------------------------------------------
	121	### get groups data (number of postings per group)
	122	# get groups data from raw table for given month
880c3eb2 TH	123	my $DBQuery = $DBHandle->prepare(sprintf("SELECT newsgroups FROM %s.%s ".
	124	"WHERE day LIKE ? AND NOT disregard",
	125	$Conf{'DBDatabase'},
	126	$Conf{'DBTableRaw'}));
	127	$DBQuery->execute($Month.'-%')
	128	or &Bleat(2,sprintf("Can't get groups data for %s from %s.%s: ".
	129	"$DBI::errstr\n",$Month,
	130	$Conf{'DBDatabase'},$Conf{'DBTableRaw'}));
2832c235 TH	131
	132	# count postings per group
	133	my %Postings;
2832c235	134	while (($_) = $DBQuery->fetchrow_array) {
b5125b10	135	# get list of newsgroups and hierarchies from Newsgroups:
880c3eb2 TH	136	my %Newsgroups = ListNewsgroups($_,$TLH,
880c3eb2 TH	137	$OptCheckgroupsFile ? \%ValidGroups : '');
2832c235 TH	138	# count each newsgroup and hierarchy once
2832c235 TH	139	foreach (sort keys %Newsgroups) {
2832c235 TH	140	$Postings{$_}++;
	141	};
	142	};
	143
880c3eb2	144	# add valid but empty groups if --checkgroups is set
ad609792 TH	145	if (%ValidGroups) {
	146	foreach (sort keys %ValidGroups) {
	147	if (!defined($Postings{$_})) {
b5125b10 TH	148	# expand newsgroup with hierarchies
	149	my @Newsgroups = ParseHierarchies($_);
	150	# add each empty newsgroup and empty hierarchies, too, as needed
	151	foreach (@Newsgroups) {
	152	if (!defined($Postings{$_})) {
	153	$Postings{$_} = 0;
	154	warn (sprintf("ADDED: %s as empty group\n",$_));
	155	};
	156	};
ad609792 TH	157	}
	158	};
	159	};
b5125b10	160
71f0178b	161	# delete old data for that month
880c3eb2 TH	162	if (!$OptTest) {
	163	$DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s WHERE month = ?",
	164	$Conf{'DBDatabase'},$Conf{'DBTableGrps'}),
	165	undef,$Month)
	166	or &Bleat(2,sprintf("Can't delete old groups data for %s from %s.%s: ".
	167	"$DBI::errstr\n",$Month,
	168	$Conf{'DBDatabase'},$Conf{'DBTableGrps'}));
71f0178b TH	169	};
71f0178b TH	170
880c3eb2	171	print "----- GroupStats -----\n" if $OptDebug;
2832c235	172	foreach my $Newsgroup (sort keys %Postings) {
880c3eb2 TH	173	print "$Newsgroup => $Postings{$Newsgroup}\n" if $OptDebug;
880c3eb2 TH	174	if (!$OptTest) {
2832c235	175	# write to database
880c3eb2 TH	176	$DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s ".
	177	"(month,newsgroup,postings) ".
	178	"VALUES (?, ?, ?)",
	179	$Conf{'DBDatabase'},
	180	$Conf{'DBTableGrps'}));
	181	$DBQuery->execute($Month, $Newsgroup, $Postings{$Newsgroup})
	182	or &Bleat(2,sprintf("Can't write groups data for %s/%s to %s.%s: ".
	183	"$DBI::errstr\n",$Month,$Newsgroup,
	184	$Conf{'DBDatabase'},$Conf{'DBTableGrps'}));
2832c235 TH	185	$DBQuery->finish;
	186	};
	187	};
	188	} else {
	189	# other types of information go here - later on
	190	};
	191	};
	192
	193	### close handles
	194	$DBHandle->disconnect;
	195
	196	__END__
	197
	198	################################ Documentation #################################
	199
	200	=head1 NAME
	201
	202	gatherstats - process statistical data from a raw source
	203
	204	=head1 SYNOPSIS
	205
93c8eae2	206	B<gatherstats> [B<-Vhdt>] [B<-m> I<YYYY-MM> \| I<YYYY-MM:YYYY-MM>] [B<-s> I<stats] [B<-c> I<filename template>]] [B<--hierarchy> I<TLH>] [B<--rawdb> I<database table>] [B<-groupsdb> I<database table>] [B<--clientsdb> I<database table>] [B<--hostsdb> I<database table>]
2832c235 TH	207
	208	=head1 REQUIREMENTS
	209
880c3eb2	210	See L<doc/README>.
2832c235 TH	211
	212	=head1 DESCRIPTION
	213
	214	This script will extract and process statistical information from a
	215	database table which is fed from F<feedlog.pl> for a given time period
313610f6	216	and write its results to (an)other database table(s). Entries marked
880c3eb2 TH	217	with I<'disregard'> in the database will be ignored; currently, you
	218	have to set this flag yourself, using your database management tools.
	219	You can exclude erroneous entries that way (e.g. automatic reposts
	220	(think of cancels flood and resurrectors); spam; ...).
2832c235 TH	221
2832c235 TH	222	The time period to act on defaults to last month; you can assign
880c3eb2 TH	223	another time period or a single month via the B<--month> option (see
880c3eb2 TH	224	below).
2832c235 TH	225
2832c235 TH	226	By default B<gatherstats> will process all types of information; you
880c3eb2 TH	227	can change that using the B<--stats> option and assigning the type of
	228	information to process. Currently that doesn't matter yet as only
	229	processing of the number of postings per group per month is
	230	implemented anyway.
2832c235 TH	231
	232	Possible information types include:
	233
	234	=over 3
	235
	236	=item B<groups> (postings per group per month)
	237
	238	B<gatherstats> will examine Newsgroups: headers. Crosspostings will be
	239	counted for each single group they appear in. Groups not in I<TLH>
	240	will be ignored.
	241
	242	B<gatherstats> will also add up the number of postings for each
	243	hierarchy level, but only count each posting once. A posting to
	244	de.alt.test will be counted for de.alt.test, de.alt.ALL and de.ALL,
	245	respectively. A crossposting to de.alt.test and de.alt.admin, on the
	246	other hand, will be counted for de.alt.test and de.alt.admin each, but
	247	only once for de.alt.ALL and de.ALL.
	248
880c3eb2 TH	249	Data is written to I<DBTableGrps> (see L<doc/INSTALL>); you can
880c3eb2 TH	250	override that default through the B<--groupsdb> option.
2832c235 TH	251
	252	=back
	253
	254	=head2 Configuration
	255
880c3eb2	256	B<gatherstats> will read its configuration from F<newsstats.conf>
2832c235 TH	257	which should be present in the same directory via Config::Auto.
2832c235 TH	258
880c3eb2	259	See L<doc/INSTALL> for an overview of possible configuration options.
2832c235	260
880c3eb2 TH	261	You can override configuration options via the B<--hierarchy>,
	262	B<--rawdb>, B<--groupsdb>, B<--clientsdb> and B<--hostsdb> options,
	263	respectively.
2832c235 TH	264
	265	=head1 OPTIONS
	266
	267	=over 3
	268
880c3eb2	269	=item B<-V>, B<--version>
2832c235	270
880c3eb2	271	Print out version and copyright information and exit.
2832c235	272
880c3eb2	273	=item B<-h>, B<--help>
2832c235 TH	274
	275	Print this man page and exit.
	276
880c3eb2	277	=item B<-d>, B<--debug>
2832c235 TH	278
	279	Output debugging information to STDOUT while processing (number of
	280	postings per group).
	281
880c3eb2	282	=item B<-t>, B<--test>
2832c235	283
880c3eb2 TH	284	Do not write results to database. You should use B<--debug> in
880c3eb2 TH	285	conjunction with B<--test> ... everything else seems a bit pointless.
2832c235	286
880c3eb2	287	=item B<-m>, B<--month> I<YYYY-MM[:YYYY-MM]>
2832c235	288
880c3eb2 TH	289	Set processing period to a single month in YYYY-MM format or to a time
	290	period between two month in YYYY-MM:YYYY-MM format (two month, separated
	291	by a colon).
2832c235	292
2832c235	293
880c3eb2	294	=item B<-s>, B<--stats> I<type>
2832c235 TH	295
	296	Set processing type to one of I<all> and I<groups>. Defaults to all
	297	(and is currently rather pointless as only I<groups> has been
	298	implemented).
	299
93c8eae2 TH	300	=item B<-c>, B<--checkgroups> I<filename template>
	301
	302	Check each group against a list of valid newsgroups read from a file,
	303	one group on each line and ignoring everything after the first
	304	whitespace (so you can use a file in checkgroups format or (part of)
	305	your INN active file).
	306
	307	The filename is taken from I<filename template>, amended by each B<--
	308	month> B<gatherstats> is processing, so that
	309
	310	gatherstats -m 2010-01:2010-12 -c checkgroups
ad609792	311
93c8eae2 TH	312	will check against F<checkgroups-2010-01> for January 2010, against
93c8eae2 TH	313	F<checkgroups-2010-02> for February 2010 and so on.
ad609792	314
93c8eae2 TH	315	Newsgroups not found in the checkgroups file will be dropped (and
93c8eae2 TH	316	logged to STDERR), and newsgroups found there but having no postings
ad609792 TH	317	will be added with a count of 0 (and logged to STDERR).
ad609792 TH	318
880c3eb2	319	=item B<--hierarchy> I<TLH> (newsgroup hierarchy)
2832c235 TH	320
	321	Override I<TLH> from F<newsstats.conf>.
	322
880c3eb2	323	=item B<--rawdb> I<table> (raw data table)
2832c235 TH	324
	325	Override I<DBTableRaw> from F<newsstats.conf>.
	326
880c3eb2	327	=item B<--groupsdb> I<table> (postings per group table)
2832c235 TH	328
	329	Override I<DBTableGrps> from F<newsstats.conf>.
	330
880c3eb2	331	=item B<--clientsdb> I<table> (client data table)
2832c235 TH	332
	333	Override I<DBTableClnts> from F<newsstats.conf>.
	334
880c3eb2	335	=item B<--hostsdb> I<table> (host data table)
2832c235 TH	336
	337	Override I<DBTableHosts> from F<newsstats.conf>.
	338
	339	=back
	340
	341	=head1 INSTALLATION
	342
880c3eb2	343	See L<doc/INSTALL>.
2832c235 TH	344
	345	=head1 EXAMPLES
	346
	347	Process all types of information for lasth month:
	348
	349	gatherstats
	350
	351	Do a dry run, showing results of processing:
	352
880c3eb2	353	gatherstats --debug --test
2832c235 TH	354
	355	Process all types of information for January of 2010:
	356
880c3eb2	357	gatherstats --month 2010-01
2832c235	358
ad609792	359	Process only number of postings for the year of 2010,
93c8eae2	360	checking against checkgroups-*:
2832c235	361
93c8eae2	362	gatherstats -m 2010-01:2010-12 -s groups -c checkgroups
2832c235 TH	363
	364	=head1 FILES
	365
	366	=over 4
	367
	368	=item F<gatherstats.pl>
	369
	370	The script itself.
	371
	372	=item F<NewsStats.pm>
	373
	374	Library functions for the NewsStats package.
	375
	376	=item F<newsstats.conf>
	377
880c3eb2	378	Runtime configuration file.
2832c235 TH	379
	380	=back
	381
	382	=head1 BUGS
	383
	384	Please report any bugs or feature requests to the author or use the
	385	bug tracker at L<http://bugs.th-h.de/>!
	386
	387	=head1 SEE ALSO
	388
	389	=over 2
	390
	391	=item -
	392
880c3eb2	393	L<doc/README>
2832c235 TH	394
	395	=item -
	396
880c3eb2	397	L<doc/INSTALL>
2832c235 TH	398
	399	=back
	400
	401	This script is part of the B<NewsStats> package.
	402
	403	=head1 AUTHOR
	404
	405	Thomas Hochstein <thh@inter.net>
	406
	407	=head1 COPYRIGHT AND LICENSE
	408
880c3eb2	409	Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
2832c235 TH	410
	411	This program is free software; you may redistribute it and/or modify it
	412	under the same terms as Perl itself.
	413
	414	=cut