[usenet/newsstats.git] / bin / feedlog.pl

#! /usr/bin/perl
#
# feedlog.pl
#
# This script will log headers and other data to a database
# for further analysis by parsing a feed from INN.
#
# It is part of the NewsStats package.
#
# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
#
# It can be redistributed and/or modified under the same terms under
# which Perl itself is published.

BEGIN {
  our $VERSION = "0.02";
  use File::Basename;
  # we're in .../bin, so our module is in ../lib
  push(@INC, dirname($0).'/../lib');
}
use strict;
use warnings;

use NewsStats;

use Sys::Syslog qw(:standard :macros);

use Date::Format;
use DBI;
use Getopt::Long qw(GetOptions);
Getopt::Long::config ('bundling');

################################# Subroutines ##################################

sub PrepareDB {
### initialise database connection, prepare statement
### and catch errors
### IN : \%Conf   : reference to configuration hash
### OUT: $DBHandle: database handle
###      $DBQuery : prepared statement
  our ($DBHandle, $DBQuery, $OptQuiet);
  my ($ConfigR) = @_;
  my %Conf = %$ConfigR;
  # drop current database connection - hard, if necessary
  if ($DBHandle) {
    $DBHandle->disconnect;
    undef $DBHandle;
  };
  # connect to database; try again every 5 seconds
  while (!$DBHandle) {
    $DBHandle = InitDB($ConfigR,0);
    if (!$DBHandle) {
      syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
      sleep(5);
    } else {;
      syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
    }
  };
  $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
                                         timestamp,token,size,peer,path,
                                         newsgroups,headers)
                                         VALUES (?,?,?,?,?,?,?,?,?,?)",
                                         $Conf{'DBDatabase'},
                                         $Conf{'DBTableRaw'}));
  return ($DBHandle,$DBQuery);
}


################################# Main program #################################

### read commandline options
my ($OptDebug,$OptQuiet,$OptConfFile);
GetOptions ('d|debug!'        => \$OptDebug,
            'q|test!'         => \$OptQuiet,
            'conffile=s'      => \$OptConfFile,
            'h|help'          => \&ShowPOD,
            'V|version'       => \&ShowVersion) or exit 1;

### read configuration
my %Conf = %{ReadConfig($OptConfFile)};

### init syslog
openlog($0, 'nofatal,pid', LOG_NEWS);
syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;

### init database
my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);

### main loop
while (<>) {
  chomp;
  # catch empty lines trailing or leading
  if ($_ eq '') {
    next;
  }
  # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
  my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
  # remaining lines contain headers
  my $Headers = "";
  while (<>) {
    chomp;
    # empty line terminates this article
    if ($_ eq '') {
      last;
    }
    # collect headers
    $Headers .= $_."\n" ;
  }

  # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
  my $Day  = time2str("%Y-%m-%d", $Timestamp);
  my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);

  # write to database
  if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
                         $Path, $Newsgroups, $Headers)) {
    syslog(LOG_ERR, 'Database error %s while processing %s: %s',
           $DBI::err, $Mid, $DBI::errstr);
    # if "MySQL server has gone away", try to recover
    if ($DBI::err == 2006) {
      # try to reconnect to database
      ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
      # try to repeat the write attempt as before
      if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
                             $Path, $Newsgroups, $Headers)) {
        syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
      };
    # otherwise log missing posting
    } else {
      syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
    };
  };
  $DBQuery->finish;

  warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
               "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
               $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
               $Newsgroups, $Headers) if $OptDebug;
}

### close handles
$DBHandle->disconnect;
syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
closelog();

__END__

################################ Documentation #################################

=head1 NAME

feedlog - log data from an INN feed to a database

=head1 SYNOPSIS

B<feedlog> [B<-Vhdq>] [B<--conffile> I<filename>]

=head1 REQUIREMENTS

See L<doc/README>.

=head1 DESCRIPTION

This script will log overview data and complete headers to a database
table for further examination by parsing a feed from INN. It will
parse that information and write it to a mysql database table in real
time.

All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
fails to initiate a database connection at startup, it will log to
I<syslog> with I<CRIT> priority and go in an endless loop, as
terminating would only result in a rapid respawn.

=head2 Configuration

B<feedlog> will read its configuration from F<newsstats.conf> which
should be present in etc/ via Config::Auto or from a configuration file
submitted by the B<--conffile> option.

See L<doc/INSTALL> for an overview of possible configuration options.

=head1 OPTIONS

=over 3

=item B<-V>, B<--version>

Print out version and copyright information and exit.

=item B<-h>, B<--help>

Print this man page and exit.

=item B<-d>, B<--debug>

Output debugging information to STDERR while parsing STDIN. You'll
find that information most probably in your B<INN> F<errlog> file.

=item B<-q>, B<--quiet>

Suppress logging to syslog.

=item B<--conffile> I<filename>

Load configuration from I<filename> instead of F<newsstats.conf>.

=back

=head1 INSTALLATION

See L<doc/INSTALL>.

=head1 EXAMPLES

Set up a feed like that in your B<INN> F<newsfeeds> file:

    ## gather statistics for NewsStats
    newsstats!
            :!*,de.*
            :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl

See L<doc/INSTALL> for further information.

=head1 FILES

=over 4

=item F<bin/feedlog.pl>

The script itself.

=item F<lib/NewsStats.pm>

Library functions for the NewsStats package.

=item F<etc/newsstats.conf>

Runtime configuration file.

=back

=head1 BUGS

Please report any bugs or feature requests to the author or use the
bug tracker at L<http://bugs.th-h.de/>!

=head1 SEE ALSO

=over 2

=item -

L<doc/README>

=item -

L<doc/INSTALL>

=back

This script is part of the B<NewsStats> package.

=head1 AUTHOR

Thomas Hochstein <thh@inter.net>

=head1 COPYRIGHT AND LICENSE

Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>

This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

=cut
Commit	Line	Data
	1	#! /usr/bin/perl
	2	#
	3	# feedlog.pl
	4	#
	5	# This script will log headers and other data to a database
	6	# for further analysis by parsing a feed from INN.
	7	#
	8	# It is part of the NewsStats package.
	9	#
	10	# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
	11	#
	12	# It can be redistributed and/or modified under the same terms under
	13	# which Perl itself is published.
	14
	15	BEGIN {
	16	our $VERSION = "0.02";
	17	use File::Basename;
	18	# we're in .../bin, so our module is in ../lib
	19	push(@INC, dirname($0).'/../lib');
	20	}
	21	use strict;
	22	use warnings;
	23
	24	use NewsStats;
	25
	26	use Sys::Syslog qw(:standard :macros);
	27
	28	use Date::Format;
	29	use DBI;
	30	use Getopt::Long qw(GetOptions);
	31	Getopt::Long::config ('bundling');
	32
	33	################################# Subroutines ##################################
	34
	35	sub PrepareDB {
	36	### initialise database connection, prepare statement
	37	### and catch errors
	38	### IN : \%Conf : reference to configuration hash
	39	### OUT: $DBHandle: database handle
	40	### $DBQuery : prepared statement
	41	our ($DBHandle, $DBQuery, $OptQuiet);
	42	my ($ConfigR) = @_;
	43	my %Conf = %$ConfigR;
	44	# drop current database connection - hard, if necessary
	45	if ($DBHandle) {
	46	$DBHandle->disconnect;
	47	undef $DBHandle;
	48	};
	49	# connect to database; try again every 5 seconds
	50	while (!$DBHandle) {
	51	$DBHandle = InitDB($ConfigR,0);
	52	if (!$DBHandle) {
	53	syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
	54	sleep(5);
	55	} else {;
	56	syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
	57	}
	58	};
	59	$DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
	60	timestamp,token,size,peer,path,
	61	newsgroups,headers)
	62	VALUES (?,?,?,?,?,?,?,?,?,?)",
	63	$Conf{'DBDatabase'},
	64	$Conf{'DBTableRaw'}));
	65	return ($DBHandle,$DBQuery);
	66	}
	67
	68
	69	################################# Main program #################################
	70
	71	### read commandline options
	72	my ($OptDebug,$OptQuiet,$OptConfFile);
	73	GetOptions ('d\|debug!' => \$OptDebug,
	74	'q\|test!' => \$OptQuiet,
	75	'conffile=s' => \$OptConfFile,
	76	'h\|help' => \&ShowPOD,
	77	'V\|version' => \&ShowVersion) or exit 1;
	78
	79	### read configuration
	80	my %Conf = %{ReadConfig($OptConfFile)};
	81
	82	### init syslog
	83	openlog($0, 'nofatal,pid', LOG_NEWS);
	84	syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
	85
	86	### init database
	87	my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
	88
	89	### main loop
	90	while (<>) {
	91	chomp;
	92	# catch empty lines trailing or leading
	93	if ($_ eq '') {
	94	next;
	95	}
	96	# first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
	97	my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
	98	# remaining lines contain headers
	99	my $Headers = "";
	100	while (<>) {
	101	chomp;
	102	# empty line terminates this article
	103	if ($_ eq '') {
	104	last;
	105	}
	106	# collect headers
	107	$Headers .= $_."\n" ;
	108	}
	109
	110	# parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
	111	my $Day = time2str("%Y-%m-%d", $Timestamp);
	112	my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
	113
	114	# write to database
	115	if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
	116	$Path, $Newsgroups, $Headers)) {
	117	syslog(LOG_ERR, 'Database error %s while processing %s: %s',
	118	$DBI::err, $Mid, $DBI::errstr);
	119	# if "MySQL server has gone away", try to recover
	120	if ($DBI::err == 2006) {
	121	# try to reconnect to database
	122	($DBHandle,$DBQuery) = PrepareDB(\%Conf);
	123	# try to repeat the write attempt as before
	124	if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
	125	$Path, $Newsgroups, $Headers)) {
	126	syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
	127	};
	128	# otherwise log missing posting
	129	} else {
	130	syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
	131	};
	132	};
	133	$DBQuery->finish;
	134
	135	warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
	136	"Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
	137	$Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
	138	$Newsgroups, $Headers) if $OptDebug;
	139	}
	140
	141	### close handles
	142	$DBHandle->disconnect;
	143	syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
	144	closelog();
	145
	146	__END__
	147
	148	################################ Documentation #################################
	149
	150	=head1 NAME
	151
	152	feedlog - log data from an INN feed to a database
	153
	154	=head1 SYNOPSIS
	155
	156	B<feedlog> [B<-Vhdq>] [B<--conffile> I<filename>]
	157
	158	=head1 REQUIREMENTS
	159
	160	See L<doc/README>.
	161
	162	=head1 DESCRIPTION
	163
	164	This script will log overview data and complete headers to a database
	165	table for further examination by parsing a feed from INN. It will
	166	parse that information and write it to a mysql database table in real
	167	time.
	168
	169	All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
	170	fails to initiate a database connection at startup, it will log to
	171	I<syslog> with I<CRIT> priority and go in an endless loop, as
	172	terminating would only result in a rapid respawn.
	173
	174	=head2 Configuration
	175
	176	B<feedlog> will read its configuration from F<newsstats.conf> which
	177	should be present in etc/ via Config::Auto or from a configuration file
	178	submitted by the B<--conffile> option.
	179
	180	See L<doc/INSTALL> for an overview of possible configuration options.
	181
	182	=head1 OPTIONS
	183
	184	=over 3
	185
	186	=item B<-V>, B<--version>
	187
	188	Print out version and copyright information and exit.
	189
	190	=item B<-h>, B<--help>
	191
	192	Print this man page and exit.
	193
	194	=item B<-d>, B<--debug>
	195
	196	Output debugging information to STDERR while parsing STDIN. You'll
	197	find that information most probably in your B<INN> F<errlog> file.
	198
	199	=item B<-q>, B<--quiet>
	200
	201	Suppress logging to syslog.
	202
	203	=item B<--conffile> I<filename>
	204
	205	Load configuration from I<filename> instead of F<newsstats.conf>.
	206
	207	=back
	208
	209	=head1 INSTALLATION
	210
	211	See L<doc/INSTALL>.
	212
	213	=head1 EXAMPLES
	214
	215	Set up a feed like that in your B<INN> F<newsfeeds> file:
	216
	217	## gather statistics for NewsStats
	218	newsstats!
	219	:!,de.
	220	:Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
	221
	222	See L<doc/INSTALL> for further information.
	223
	224	=head1 FILES
	225
	226	=over 4
	227
	228	=item F<bin/feedlog.pl>
	229
	230	The script itself.
	231
	232	=item F<lib/NewsStats.pm>
	233
	234	Library functions for the NewsStats package.
	235
	236	=item F<etc/newsstats.conf>
	237
	238	Runtime configuration file.
	239
	240	=back
	241
	242	=head1 BUGS
	243
	244	Please report any bugs or feature requests to the author or use the
	245	bug tracker at L<http://bugs.th-h.de/>!
	246
	247	=head1 SEE ALSO
	248
	249	=over 2
	250
	251	=item -
	252
	253	L<doc/README>
	254
	255	=item -
	256
	257	L<doc/INSTALL>
	258
	259	=back
	260
	261	This script is part of the B<NewsStats> package.
	262
	263	=head1 AUTHOR
	264
	265	Thomas Hochstein <thh@inter.net>
	266
	267	=head1 COPYRIGHT AND LICENSE
	268
	269	Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
	270
	271	This program is free software; you may redistribute it and/or modify it
	272	under the same terms as Perl itself.
	273
	274	=cut