Merge branch 'thh-bug53' into next
[usenet/newsstats.git] / feedlog.pl
... / ...
CommitLineData
1#! /usr/bin/perl
2#
3# feedlog.pl
4#
5# This script will log headers and other data to a database
6# for further analysis by parsing a feed from INN.
7#
8# It is part of the NewsStats package.
9#
10# Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
11#
12# It can be redistributed and/or modified under the same terms under
13# which Perl itself is published.
14
15BEGIN {
16 our $VERSION = "0.01";
17 use File::Basename;
18 push(@INC, dirname($0));
19}
20use strict;
21use warnings;
22
23use NewsStats;
24
25use Sys::Syslog qw(:standard :macros);
26
27use Date::Format;
28use DBI;
29use Getopt::Long qw(GetOptions);
30Getopt::Long::config ('bundling');
31
32################################# Subroutines ##################################
33
34sub PrepareDB {
35### initialise database connection, prepare statement
36### and catch errors
37### IN : \%Conf : reference to configuration hash
38### OUT: $DBHandle: database handle
39### $DBQuery : prepared statement
40 our ($DBHandle, $DBQuery, $OptQuiet);
41 my ($ConfigR) = @_;
42 my %Conf = %$ConfigR;
43 # drop current database connection - hard, if necessary
44 if ($DBHandle) {
45 $DBHandle->disconnect;
46 undef $DBHandle;
47 };
48 # connect to database; try again every 5 seconds
49 while (!$DBHandle) {
50 $DBHandle = InitDB($ConfigR,0);
51 if (!$DBHandle) {
52 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
53 sleep(5);
54 } else {;
55 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
56 }
57 };
58 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
59 timestamp,token,size,peer,path,
60 newsgroups,headers)
61 VALUES (?,?,?,?,?,?,?,?,?,?)",
62 $Conf{'DBDatabase'},
63 $Conf{'DBTableRaw'}));
64 return ($DBHandle,$DBQuery);
65}
66
67
68################################# Main program #################################
69
70### read commandline options
71my ($OptDebug,$OptQuiet);
72GetOptions ('d|debug!' => \$OptDebug,
73 'q|test!' => \$OptQuiet,
74 'h|help' => \&ShowPOD,
75 'V|version' => \&ShowVersion) or exit 1;
76
77### read configuration
78my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
79
80### init syslog
81openlog($0, 'nofatal,pid', LOG_NEWS);
82syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
83
84### init database
85my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
86
87### main loop
88while (<>) {
89 chomp;
90 # catch empty lines trailing or leading
91 if ($_ eq '') {
92 next;
93 }
94 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
95 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
96 # remaining lines contain headers
97 my $Headers = "";
98 while (<>) {
99 chomp;
100 # empty line terminates this article
101 if ($_ eq '') {
102 last;
103 }
104 # collect headers
105 $Headers .= $_."\n" ;
106 }
107
108 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
109 my $Day = time2str("%Y-%m-%d", $Timestamp);
110 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
111
112 # write to database
113 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
114 $Path, $Newsgroups, $Headers)) {
115 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
116 $DBI::err, $Mid, $DBI::errstr);
117 # if "MySQL server has gone away", try to recover
118 if ($DBI::err == 2006) {
119 # try to reconnect to database
120 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
121 # try to repeat the write attempt as before
122 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
123 $Path, $Newsgroups, $Headers)) {
124 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
125 };
126 # otherwise log missing posting
127 } else {
128 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
129 };
130 };
131 $DBQuery->finish;
132
133 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
134 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
135 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
136 $Newsgroups, $Headers) if $OptDebug;
137}
138
139### close handles
140$DBHandle->disconnect;
141syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
142closelog();
143
144__END__
145
146################################ Documentation #################################
147
148=head1 NAME
149
150feedlog - log data from an INN feed to a database
151
152=head1 SYNOPSIS
153
154B<feedlog> [B<-Vhdq>]
155
156=head1 REQUIREMENTS
157
158See L<doc/README>.
159
160=head1 DESCRIPTION
161
162This script will log overview data and complete headers to a database
163table for further examination by parsing a feed from INN. It will
164parse that information and write it to a mysql database table in real
165time.
166
167All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
168fails to initiate a database connection at startup, it will log to
169I<syslog> with I<CRIT> priority and go in an endless loop, as
170terminating would only result in a rapid respawn.
171
172=head2 Configuration
173
174B<feedlog> will read its configuration from F<newsstats.conf> which
175should be present in the same directory via Config::Auto.
176
177See L<doc/INSTALL> for an overview of possible configuration options.
178
179=head1 OPTIONS
180
181=over 3
182
183=item B<-V>, B<--version>
184
185Print out version and copyright information and exit.
186
187=item B<-h>, B<--help>
188
189Print this man page and exit.
190
191=item B<-d>, B<--debug>
192
193Output debugging information to STDERR while parsing STDIN. You'll
194find that information most probably in your B<INN> F<errlog> file.
195
196=item B<-q>, B<--quiet>
197
198Suppress logging to syslog.
199
200=back
201
202=head1 INSTALLATION
203
204See L<doc/INSTALL.>
205
206=head1 EXAMPLES
207
208Set up a feed like that in your B<INN> F<newsfeeds> file:
209
210 ## gather statistics for NewsStats
211 newsstats!
212 :!*,de.*
213 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
214
215See L<doc/INSTALL> for further information.
216
217=head1 FILES
218
219=over 4
220
221=item F<feedlog.pl>
222
223The script itself.
224
225=item F<NewsStats.pm>
226
227Library functions for the NewsStats package.
228
229=item F<newsstats.conf>
230
231Runtime configuration file.
232
233=back
234
235=head1 BUGS
236
237Please report any bugs or feature requests to the author or use the
238bug tracker at L<http://bugs.th-h.de/>!
239
240=head1 SEE ALSO
241
242=over 2
243
244=item -
245
246L<doc/README>
247
248=item -
249
250L<doc/INSTALL>
251
252=back
253
254This script is part of the B<NewsStats> package.
255
256=head1 AUTHOR
257
258Thomas Hochstein <thh@inter.net>
259
260=head1 COPYRIGHT AND LICENSE
261
262Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
263
264This program is free software; you may redistribute it and/or modify it
265under the same terms as Perl itself.
266
267=cut
This page took 0.009913 seconds and 4 git commands to generate.