5 # This script will log headers and other data to a database
6 # for further analysis by parsing a feed from INN.
8 # It is part of the NewsStats package.
10 # Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
12 # It can be redistributed and/or modified under the same terms under
13 # which Perl itself is published.
16 our $VERSION = "0.02";
18 # we're in .../bin, so our module is in ../lib
19 push(@INC, dirname($0).'/../lib');
26 use Sys::Syslog qw(:standard :macros);
30 use Getopt::Long qw(GetOptions);
31 Getopt::Long::config ('bundling');
33 ################################# Subroutines ##################################
36 ### initialise database connection, prepare statement
38 ### IN : \%Conf : reference to configuration hash
39 ### OUT: $DBHandle: database handle
40 ### $DBQuery : prepared statement
41 our ($DBHandle, $DBQuery, $OptQuiet);
44 # drop current database connection - hard, if necessary
46 $DBHandle->disconnect;
49 # connect to database; try again every 5 seconds
51 $DBHandle = InitDB($ConfigR,0);
53 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
56 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
59 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
60 timestamp,token,size,peer,path,
62 VALUES (?,?,?,?,?,?,?,?,?,?)",
64 $Conf{'DBTableRaw'}));
65 return ($DBHandle,$DBQuery);
69 ################################# Main program #################################
71 ### read commandline options
72 my ($OptDebug,$OptQuiet,$OptConfFile);
73 GetOptions ('d|debug!' => \$OptDebug,
74 'q|test!' => \$OptQuiet,
75 'conffile=s' => \$OptConfFile,
76 'h|help' => \&ShowPOD,
77 'V|version' => \&ShowVersion) or exit 1;
79 ### read configuration
80 my %Conf = %{ReadConfig($OptConfFile)};
83 openlog($0, 'nofatal,pid', LOG_NEWS);
84 syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
87 my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
92 # catch empty lines trailing or leading
96 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
97 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
98 # remaining lines contain headers
102 # empty line terminates this article
107 $Headers .= $_."\n" ;
110 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
111 my $Day = time2str("%Y-%m-%d", $Timestamp);
112 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
115 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
116 $Path, $Newsgroups, $Headers)) {
117 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
118 $DBI::err, $Mid, $DBI::errstr);
119 # if "MySQL server has gone away", try to recover
120 if ($DBI::err == 2006) {
121 # try to reconnect to database
122 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
123 # try to repeat the write attempt as before
124 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
125 $Path, $Newsgroups, $Headers)) {
126 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
128 # otherwise log missing posting
130 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
135 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
136 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
137 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
138 $Newsgroups, $Headers) if $OptDebug;
142 $DBHandle->disconnect;
143 syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
148 ################################ Documentation #################################
152 feedlog - log data from an INN feed to a database
156 B<feedlog> [B<-Vhdq>] [--conffile I<filename>]
164 This script will log overview data and complete headers to a database
165 table for further examination by parsing a feed from INN. It will
166 parse that information and write it to a mysql database table in real
169 All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
170 fails to initiate a database connection at startup, it will log to
171 I<syslog> with I<CRIT> priority and go in an endless loop, as
172 terminating would only result in a rapid respawn.
176 B<feedlog> will read its configuration from F<newsstats.conf> which
177 should be present in the same directory via Config::Auto.
179 See L<doc/INSTALL> for an overview of possible configuration options.
185 =item B<-V>, B<--version>
187 Print out version and copyright information and exit.
189 =item B<-h>, B<--help>
191 Print this man page and exit.
193 =item B<-d>, B<--debug>
195 Output debugging information to STDERR while parsing STDIN. You'll
196 find that information most probably in your B<INN> F<errlog> file.
198 =item B<-q>, B<--quiet>
200 Suppress logging to syslog.
202 =item B<--conffile> I<filename>
204 Load configuration from I<filename> instead of F<newsstats.conf>.
214 Set up a feed like that in your B<INN> F<newsfeeds> file:
216 ## gather statistics for NewsStats
219 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
221 See L<doc/INSTALL> for further information.
227 =item F<bin/feedlog.pl>
231 =item F<lib/NewsStats.pm>
233 Library functions for the NewsStats package.
235 =item F<etc/newsstats.conf>
237 Runtime configuration file.
243 Please report any bugs or feature requests to the author or use the
244 bug tracker at L<http://bugs.th-h.de/>!
260 This script is part of the B<NewsStats> package.
264 Thomas Hochstein <thh@inter.net>
266 =head1 COPYRIGHT AND LICENSE
268 Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
270 This program is free software; you may redistribute it and/or modify it
271 under the same terms as Perl itself.