5 # This script will log headers and other data to a database
6 # for further analysis by parsing a feed from INN.
8 # It is part of the NewsStats package.
10 # Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
12 # It can be redistributed and/or modified under the same terms under
13 # which Perl itself is published.
16 our $VERSION = "0.01";
18 push(@INC, dirname($0));
24 use Sys::Syslog qw(:standard :macros);
28 use Getopt::Long qw(GetOptions);
29 Getopt::Long::config ('bundling');
31 ################################# Subroutines ##################################
34 ### initialise database connection, prepare statement
36 ### IN : \%Conf : reference to configuration hash
37 ### OUT: $DBHandle: database handle
38 ### $DBQuery : prepared statement
39 our ($DBHandle, $DBQuery, $OptQuiet);
42 # drop current database connection - hard, if necessary
44 $DBHandle->disconnect;
47 # connect to database; try again every 5 seconds
49 $DBHandle = InitDB($ConfigR,0);
51 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
54 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
57 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
58 timestamp,token,size,peer,path,
60 VALUES (?,?,?,?,?,?,?,?,?,?)",
62 $Conf{'DBTableRaw'}));
63 return ($DBHandle,$DBQuery);
67 ################################# Main program #################################
69 ### read commandline options
70 my ($OptDebug,$OptQuiet);
71 GetOptions ('d|debug!' => \$OptDebug,
72 'q|test!' => \$OptQuiet,
73 'h|help' => \&ShowPOD,
74 'V|version' => \&ShowVersion) or exit 1;
76 ### read configuration
77 my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
80 openlog($0, 'nofatal,pid', LOG_NEWS);
81 syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
84 my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
89 # catch empty lines trailing or leading
93 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
94 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
95 # remaining lines contain headers
99 # empty line terminates this article
104 $Headers .= $_."\n" ;
107 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
108 my $Day = time2str("%Y-%m-%d", $Timestamp);
109 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
112 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
113 $Path, $Newsgroups, $Headers)) {
114 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
115 $DBI::err, $Mid, $DBI::errstr);
116 # if "MySQL server has gone away", try to recover
117 if ($DBI::err == 2006) {
118 # try to reconnect to database
119 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
120 # try to repeat the write attempt as before
121 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
122 $Path, $Newsgroups, $Headers)) {
123 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
125 # otherwise log missing posting
127 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
132 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
133 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
134 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
135 $Newsgroups, $Headers) if $OptDebug;
139 $DBHandle->disconnect;
140 syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
145 ################################ Documentation #################################
149 feedlog - log data from an INN feed to a database
153 B<feedlog> [B<-Vhdq>]
161 This script will log overview data and complete headers to a database
162 table for further examination by parsing a feed from INN. It will
163 parse that information and write it to a mysql database table in real
166 All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
167 fails to initiate a database connection at startup, it will log to
168 I<syslog> with I<CRIT> priority and go in an endless loop, as
169 terminating would only result in a rapid respawn.
173 B<feedlog> will read its configuration from F<newsstats.conf> which
174 should be present in the same directory via Config::Auto.
176 See L<doc/INSTALL> for an overview of possible configuration options.
182 =item B<-V>, B<--version>
184 Print out version and copyright information and exit.
186 =item B<-h>, B<--help>
188 Print this man page and exit.
190 =item B<-d>, B<--debug>
192 Output debugging information to STDERR while parsing STDIN. You'll
193 find that information most probably in your B<INN> F<errlog> file.
195 =item B<-q>, B<--quiet>
197 Suppress logging to syslog.
207 Set up a feed like that in your B<INN> F<newsfeeds> file:
209 ## gather statistics for NewsStats
212 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
214 See L<doc/INSTALL> for further information.
224 =item F<NewsStats.pm>
226 Library functions for the NewsStats package.
228 =item F<newsstats.conf>
230 Runtime configuration file.
236 Please report any bugs or feature requests to the author or use the
237 bug tracker at L<http://bugs.th-h.de/>!
253 This script is part of the B<NewsStats> package.
257 Thomas Hochstein <thh@inter.net>
259 =head1 COPYRIGHT AND LICENSE
261 Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
263 This program is free software; you may redistribute it and/or modify it
264 under the same terms as Perl itself.