Add comments and POD.

[usenet/newsstats.git] / feedlog.pl

#! /usr/bin/perl -W\r
#\r
# feedlog.pl\r
#\r
# This script will log headers and other data to a database\r
# for further analysis by parsing a feed from INN.\r
# \r
# It is part of the NewsStats package.\r
#\r
# Copyright (c) 2010 Thomas Hochstein <thh@inter.net>\r
#\r
# It can be redistributed and/or modified under the same terms under \r
# which Perl itself is published.\r
\r
BEGIN {\r
  our $VERSION = "0.01";\r
  use File::Basename;\r
  push(@INC, dirname($0));\r
}\r
use strict;\r
\r
use NewsStats;\r
\r
use Sys::Syslog qw(:standard :macros);\r
\r
use Date::Format;\r
use DBI;\r
\r
################################# Main program #################################\r
\r
### read commandline options\r
my %Options = &ReadOptions('qd');\r
\r
### read configuration\r
my %Conf = %{ReadConfig('newsstats.conf')};\r
\r
### init syslog\r
openlog($MySelf, 'nofatal,pid', LOG_NEWS);\r
syslog(LOG_NOTICE, "$MyVersion starting up.") if !$Options{'q'};\r
\r
### init database\r
my $DBHandle = InitDB(\%Conf,0);\r
if (!$DBHandle) {\r
  syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);\r
  while (1) {}; # go into endless loop to suppress further errors and respawning\r
};\r
my $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,timestamp,token,size,peer,path,newsgroups,headers) VALUES (?,?,?,?,?,?,?,?,?,?)",$Conf{'DBDatabase'},$Conf{'DBTableRaw'}));\r
\r
### main loop\r
while (<>) {\r
  chomp;\r
  # catch empty lines trailing or leading\r
  if ($_ eq '') {\r
    next;\r
  }\r
  # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups\r
  my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;\r
  # remaining lines contain headers\r
  my $Headers = "";\r
  while (<>) {\r
    chomp;\r
    # empty line terminates this article\r
    if ($_ eq '') {\r
      last;\r
    }\r
    # collect headers\r
    $Headers .= $_."\n" ;\r
  }\r
\r
  # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp\r
  my $Day  = time2str("%Y-%m-%d", $Timestamp);\r
  my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);\r
\r
  # write to database\r
  if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups, $Headers)) {\r
    syslog(LOG_ERR, 'Database error: %s', $DBI::errstr);\r
  };\r
  $DBQuery->finish;\r
  \r
  warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\nSize: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",$Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups, $Headers) if !$Options{'d'};\r
}\r
\r
### close handles\r
$DBHandle->disconnect;\r
syslog(LOG_NOTICE, "$MySelf closing down.") if !$Options{'q'};\r
closelog();\r
\r
__END__\r
\r
################################ Documentation #################################\r
\r
=head1 NAME\r
\r
feedlog - log data from an INN feed to a database\r
\r
=head1 SYNOPSIS\r
\r
B<feedlog> [B<-Vhdq>]\r
\r
=head1 REQUIREMENTS\r
\r
See doc/README: Perl 5.8.x itself and the following modules from CPAN:\r
\r
=over 2\r
\r
=item -\r
\r
Config::Auto\r
\r
=item -\r
\r
Date::Format\r
\r
=item -\r
\r
DBI\r
\r
=back\r
\r
=head1 DESCRIPTION\r
\r
This script will log overview data and complete headers to a database\r
table for further examination by parsing a feed from INN. It will\r
parse that information and write it to a mysql database table in real\r
time.\r
\r
All reporting is done to I<syslog> via I<news> facility. If B<feedlog>\r
fails to initiate a database connection at startup, it will log to\r
I<syslog> with I<CRIT> priority and go in an endless loop, as\r
terminating would only result in a rapid respawn.\r
\r
=head2 Configuration\r
\r
F<feedlog.pl> will read its configuration from F<newsstats.conf> which\r
should be present in the same directory via Config::Auto.\r
\r
See doc/INSTALL for an overview of possible configuration options.\r
\r
=head1 OPTIONS\r
\r
=over 3\r
\r
=item B<-V> (version)\r
\r
Print out version and copyright information on B<yapfaq> and exit.\r
\r
=item B<-h> (help)\r
\r
Print this man page and exit.\r
\r
=item B<-d> (debug)\r
\r
Output debugging information to STDERR while parsing STDIN. You'll\r
find that information most probably in your B<INN> F<errlog> file.\r
\r
=item B<-q> (quiet)\r
\r
Suppress logging to syslog.\r
\r
=back\r
\r
=head1 INSTALLATION\r
\r
See doc/INSTALL.\r
\r
=head1 EXAMPLES\r
\r
Set up a feed like that in your B<INN> F<newsfeeds> file:\r
\r
    ## gather statistics for NewsStats\r
    newsstats!\r
            :!*,de.*\r
            :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl\r
\r
See doc/INSTALL for further information.\r
\r
=head1 FILES\r
\r
=over 4\r
\r
=item F<feedlog.pl>\r
\r
The script itself.\r
\r
=item F<NewsStats.pm>\r
\r
Library functions for the NewsStats package.\r
\r
=item F<newsstats.conf>\r
\r
Runtime configuration file for B<yapfaq>.\r
\r
=back\r
\r
=head1 BUGS\r
\r
Please report any bugs or feature requests to the author or use the\r
bug tracker at L<http://bugs.th-h.de/>!\r
\r
=head1 SEE ALSO\r
\r
=over 2\r
\r
=item -\r
\r
doc/README\r
\r
=item -\r
\r
doc/INSTALL\r
\r
=back\r
\r
This script is part of the B<NewsStats> package.\r
\r
=head1 AUTHOR\r
\r
Thomas Hochstein <thh@inter.net>\r
\r
=head1 COPYRIGHT AND LICENSE\r
\r
Copyright (c) 2010 Thomas Hochstein <thh@inter.net>\r
\r
This program is free software; you may redistribute it and/or modify it\r
under the same terms as Perl itself.\r
\r
=cut\r