5 # This script will log headers and other data to a database
6 # for further analysis by parsing a feed from INN.
8 # It is part of the NewsStats package.
10 # Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
12 # It can be redistributed and/or modified under the same terms under
13 # which Perl itself is published.
16 our $VERSION = "0.01";
18 # we're in .../bin, so our module is in ../lib
19 push(@INC, dirname($0).'/../lib');
26 use Sys::Syslog qw(:standard :macros);
30 use Getopt::Long qw(GetOptions);
31 Getopt::Long::config ('bundling');
33 ################################# Subroutines ##################################
36 ### initialise database connection, prepare statement
38 ### IN : \%Conf : reference to configuration hash
39 ### OUT: $DBHandle: database handle
40 ### $DBQuery : prepared statement
41 our ($DBHandle, $DBQuery, $OptQuiet);
44 # drop current database connection - hard, if necessary
46 $DBHandle->disconnect;
49 # connect to database; try again every 5 seconds
51 $DBHandle = InitDB($ConfigR,0);
53 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
56 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
59 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
60 timestamp,token,size,peer,path,
62 VALUES (?,?,?,?,?,?,?,?,?,?)",
64 $Conf{'DBTableRaw'}));
65 return ($DBHandle,$DBQuery);
69 ################################# Main program #################################
71 ### read commandline options
72 my ($OptDebug,$OptQuiet);
73 GetOptions ('d|debug!' => \$OptDebug,
74 'q|test!' => \$OptQuiet,
75 'h|help' => \&ShowPOD,
76 'V|version' => \&ShowVersion) or exit 1;
78 ### read configuration
79 my %Conf = %{ReadConfig('')};
82 openlog($0, 'nofatal,pid', LOG_NEWS);
83 syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
86 my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
91 # catch empty lines trailing or leading
95 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
96 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
97 # remaining lines contain headers
101 # empty line terminates this article
106 $Headers .= $_."\n" ;
109 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
110 my $Day = time2str("%Y-%m-%d", $Timestamp);
111 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
114 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
115 $Path, $Newsgroups, $Headers)) {
116 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
117 $DBI::err, $Mid, $DBI::errstr);
118 # if "MySQL server has gone away", try to recover
119 if ($DBI::err == 2006) {
120 # try to reconnect to database
121 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
122 # try to repeat the write attempt as before
123 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
124 $Path, $Newsgroups, $Headers)) {
125 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
127 # otherwise log missing posting
129 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
134 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
135 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
136 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
137 $Newsgroups, $Headers) if $OptDebug;
141 $DBHandle->disconnect;
142 syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
147 ################################ Documentation #################################
151 feedlog - log data from an INN feed to a database
155 B<feedlog> [B<-Vhdq>]
163 This script will log overview data and complete headers to a database
164 table for further examination by parsing a feed from INN. It will
165 parse that information and write it to a mysql database table in real
168 All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
169 fails to initiate a database connection at startup, it will log to
170 I<syslog> with I<CRIT> priority and go in an endless loop, as
171 terminating would only result in a rapid respawn.
175 B<feedlog> will read its configuration from F<newsstats.conf> which
176 should be present in the same directory via Config::Auto.
178 See L<doc/INSTALL> for an overview of possible configuration options.
184 =item B<-V>, B<--version>
186 Print out version and copyright information and exit.
188 =item B<-h>, B<--help>
190 Print this man page and exit.
192 =item B<-d>, B<--debug>
194 Output debugging information to STDERR while parsing STDIN. You'll
195 find that information most probably in your B<INN> F<errlog> file.
197 =item B<-q>, B<--quiet>
199 Suppress logging to syslog.
209 Set up a feed like that in your B<INN> F<newsfeeds> file:
211 ## gather statistics for NewsStats
214 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
216 See L<doc/INSTALL> for further information.
222 =item F<bin/feedlog.pl>
226 =item F<lib/NewsStats.pm>
228 Library functions for the NewsStats package.
230 =item F<etc/newsstats.conf>
232 Runtime configuration file.
238 Please report any bugs or feature requests to the author or use the
239 bug tracker at L<http://bugs.th-h.de/>!
255 This script is part of the B<NewsStats> package.
259 Thomas Hochstein <thh@inter.net>
261 =head1 COPYRIGHT AND LICENSE
263 Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
265 This program is free software; you may redistribute it and/or modify it
266 under the same terms as Perl itself.