5 # This script will log headers and other data to a database
6 # for further analysis by parsing a feed from INN.
8 # It is part of the NewsStats package.
10 # Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
12 # It can be redistributed and/or modified under the same terms under
13 # which Perl itself is published.
16 our $VERSION = "0.01";
18 push(@INC, dirname($0));
25 use Sys::Syslog qw(:standard :macros);
29 use Getopt::Long qw(GetOptions);
30 Getopt::Long::config ('bundling');
32 ################################# Subroutines ##################################
35 ### initialise database connection, prepare statement
37 ### IN : \%Conf : reference to configuration hash
38 ### OUT: $DBHandle: database handle
39 ### $DBQuery : prepared statement
40 our ($DBHandle, $DBQuery, $OptQuiet);
43 # drop current database connection - hard, if necessary
45 $DBHandle->disconnect;
48 # connect to database; try again every 5 seconds
50 $DBHandle = InitDB($ConfigR,0);
52 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
55 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
58 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
59 timestamp,token,size,peer,path,
61 VALUES (?,?,?,?,?,?,?,?,?,?)",
63 $Conf{'DBTableRaw'}));
64 return ($DBHandle,$DBQuery);
68 ################################# Main program #################################
70 ### read commandline options
71 my ($OptDebug,$OptQuiet);
72 GetOptions ('d|debug!' => \$OptDebug,
73 'q|test!' => \$OptQuiet,
74 'h|help' => \&ShowPOD,
75 'V|version' => \&ShowVersion) or exit 1;
77 ### read configuration
78 my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
81 openlog($0, 'nofatal,pid', LOG_NEWS);
82 syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
85 my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
90 # catch empty lines trailing or leading
94 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
95 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
96 # remaining lines contain headers
100 # empty line terminates this article
105 $Headers .= $_."\n" ;
108 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
109 my $Day = time2str("%Y-%m-%d", $Timestamp);
110 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
113 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
114 $Path, $Newsgroups, $Headers)) {
115 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
116 $DBI::err, $Mid, $DBI::errstr);
117 # if "MySQL server has gone away", try to recover
118 if ($DBI::err == 2006) {
119 # try to reconnect to database
120 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
121 # try to repeat the write attempt as before
122 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
123 $Path, $Newsgroups, $Headers)) {
124 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
126 # otherwise log missing posting
128 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
133 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
134 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
135 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
136 $Newsgroups, $Headers) if $OptDebug;
140 $DBHandle->disconnect;
141 syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
146 ################################ Documentation #################################
150 feedlog - log data from an INN feed to a database
154 B<feedlog> [B<-Vhdq>]
162 This script will log overview data and complete headers to a database
163 table for further examination by parsing a feed from INN. It will
164 parse that information and write it to a mysql database table in real
167 All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
168 fails to initiate a database connection at startup, it will log to
169 I<syslog> with I<CRIT> priority and go in an endless loop, as
170 terminating would only result in a rapid respawn.
174 B<feedlog> will read its configuration from F<newsstats.conf> which
175 should be present in the same directory via Config::Auto.
177 See L<doc/INSTALL> for an overview of possible configuration options.
183 =item B<-V>, B<--version>
185 Print out version and copyright information and exit.
187 =item B<-h>, B<--help>
189 Print this man page and exit.
191 =item B<-d>, B<--debug>
193 Output debugging information to STDERR while parsing STDIN. You'll
194 find that information most probably in your B<INN> F<errlog> file.
196 =item B<-q>, B<--quiet>
198 Suppress logging to syslog.
208 Set up a feed like that in your B<INN> F<newsfeeds> file:
210 ## gather statistics for NewsStats
213 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
215 See L<doc/INSTALL> for further information.
225 =item F<NewsStats.pm>
227 Library functions for the NewsStats package.
229 =item F<newsstats.conf>
231 Runtime configuration file.
237 Please report any bugs or feature requests to the author or use the
238 bug tracker at L<http://bugs.th-h.de/>!
254 This script is part of the B<NewsStats> package.
258 Thomas Hochstein <thh@inter.net>
260 =head1 COPYRIGHT AND LICENSE
262 Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
264 This program is free software; you may redistribute it and/or modify it
265 under the same terms as Perl itself.