Release 0.01
[usenet/newsstats.git] / feedlog.pl
CommitLineData
3f817eb4 1#! /usr/bin/perl
2832c235
TH
2#
3# feedlog.pl
4#
5# This script will log headers and other data to a database
6# for further analysis by parsing a feed from INN.
7#
8# It is part of the NewsStats package.
9#
07c0b258 10# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
2832c235
TH
11#
12# It can be redistributed and/or modified under the same terms under
13# which Perl itself is published.
14
15BEGIN {
16 our $VERSION = "0.01";
17 use File::Basename;
18 push(@INC, dirname($0));
19}
20use strict;
3f817eb4 21use warnings;
2832c235
TH
22
23use NewsStats;
24
25use Sys::Syslog qw(:standard :macros);
26
27use Date::Format;
28use DBI;
f05ec4ae
TH
29use Getopt::Long qw(GetOptions);
30Getopt::Long::config ('bundling');
2832c235 31
98563c61
TH
32################################# Subroutines ##################################
33
34sub PrepareDB {
35### initialise database connection, prepare statement
36### and catch errors
37### IN : \%Conf : reference to configuration hash
38### OUT: $DBHandle: database handle
39### $DBQuery : prepared statement
40 our ($DBHandle, $DBQuery, $OptQuiet);
41 my ($ConfigR) = @_;
42 my %Conf = %$ConfigR;
43 # drop current database connection - hard, if necessary
44 if ($DBHandle) {
45 $DBHandle->disconnect;
46 undef $DBHandle;
47 };
48 # connect to database; try again every 5 seconds
49 while (!$DBHandle) {
50 $DBHandle = InitDB($ConfigR,0);
51 if (!$DBHandle) {
52 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
53 sleep(5);
54 } else {;
55 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
56 }
57 };
58 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
59 timestamp,token,size,peer,path,
60 newsgroups,headers)
61 VALUES (?,?,?,?,?,?,?,?,?,?)",
62 $Conf{'DBDatabase'},
63 $Conf{'DBTableRaw'}));
64 return ($DBHandle,$DBQuery);
65}
66
67
2832c235
TH
68################################# Main program #################################
69
70### read commandline options
f05ec4ae
TH
71my ($OptDebug,$OptQuiet);
72GetOptions ('d|debug!' => \$OptDebug,
73 'q|test!' => \$OptQuiet,
74 'h|help' => \&ShowPOD,
75 'V|version' => \&ShowVersion) or exit 1;
2832c235
TH
76
77### read configuration
f05ec4ae 78my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
2832c235
TH
79
80### init syslog
f05ec4ae
TH
81openlog($0, 'nofatal,pid', LOG_NEWS);
82syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
2832c235
TH
83
84### init database
98563c61 85my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
2832c235
TH
86
87### main loop
88while (<>) {
89 chomp;
90 # catch empty lines trailing or leading
91 if ($_ eq '') {
92 next;
93 }
94 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
95 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
96 # remaining lines contain headers
97 my $Headers = "";
98 while (<>) {
99 chomp;
100 # empty line terminates this article
101 if ($_ eq '') {
102 last;
103 }
104 # collect headers
105 $Headers .= $_."\n" ;
106 }
107
108 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
109 my $Day = time2str("%Y-%m-%d", $Timestamp);
110 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
111
112 # write to database
f05ec4ae
TH
113 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
114 $Path, $Newsgroups, $Headers)) {
98563c61
TH
115 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
116 $DBI::err, $Mid, $DBI::errstr);
117 # if "MySQL server has gone away", try to recover
118 if ($DBI::err == 2006) {
119 # try to reconnect to database
120 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
121 # try to repeat the write attempt as before
122 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
123 $Path, $Newsgroups, $Headers)) {
124 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
125 };
126 # otherwise log missing posting
127 } else {
128 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
129 };
2832c235
TH
130 };
131 $DBQuery->finish;
132
f05ec4ae
TH
133 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
134 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
135 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
136 $Newsgroups, $Headers) if $OptDebug;
2832c235
TH
137}
138
139### close handles
140$DBHandle->disconnect;
f05ec4ae 141syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
2832c235
TH
142closelog();
143
144__END__
145
146################################ Documentation #################################
147
148=head1 NAME
149
150feedlog - log data from an INN feed to a database
151
152=head1 SYNOPSIS
153
154B<feedlog> [B<-Vhdq>]
155
156=head1 REQUIREMENTS
157
f05ec4ae 158See L<doc/README>.
2832c235
TH
159
160=head1 DESCRIPTION
161
162This script will log overview data and complete headers to a database
163table for further examination by parsing a feed from INN. It will
164parse that information and write it to a mysql database table in real
165time.
166
167All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
168fails to initiate a database connection at startup, it will log to
169I<syslog> with I<CRIT> priority and go in an endless loop, as
170terminating would only result in a rapid respawn.
171
172=head2 Configuration
173
f05ec4ae 174B<feedlog> will read its configuration from F<newsstats.conf> which
2832c235
TH
175should be present in the same directory via Config::Auto.
176
f05ec4ae 177See L<doc/INSTALL> for an overview of possible configuration options.
2832c235
TH
178
179=head1 OPTIONS
180
181=over 3
182
f05ec4ae 183=item B<-V>, B<--version>
2832c235 184
f05ec4ae 185Print out version and copyright information and exit.
2832c235 186
f05ec4ae 187=item B<-h>, B<--help>
2832c235
TH
188
189Print this man page and exit.
190
f05ec4ae 191=item B<-d>, B<--debug>
2832c235
TH
192
193Output debugging information to STDERR while parsing STDIN. You'll
194find that information most probably in your B<INN> F<errlog> file.
195
f05ec4ae 196=item B<-q>, B<--quiet>
2832c235
TH
197
198Suppress logging to syslog.
199
200=back
201
202=head1 INSTALLATION
203
95d9fe2c 204See L<doc/INSTALL>.
2832c235
TH
205
206=head1 EXAMPLES
207
208Set up a feed like that in your B<INN> F<newsfeeds> file:
209
210 ## gather statistics for NewsStats
211 newsstats!
212 :!*,de.*
213 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
214
f05ec4ae 215See L<doc/INSTALL> for further information.
2832c235
TH
216
217=head1 FILES
218
219=over 4
220
221=item F<feedlog.pl>
222
223The script itself.
224
225=item F<NewsStats.pm>
226
227Library functions for the NewsStats package.
228
229=item F<newsstats.conf>
230
f05ec4ae 231Runtime configuration file.
2832c235
TH
232
233=back
234
235=head1 BUGS
236
237Please report any bugs or feature requests to the author or use the
238bug tracker at L<http://bugs.th-h.de/>!
239
240=head1 SEE ALSO
241
242=over 2
243
244=item -
245
f05ec4ae 246L<doc/README>
2832c235
TH
247
248=item -
249
f05ec4ae 250L<doc/INSTALL>
2832c235
TH
251
252=back
253
254This script is part of the B<NewsStats> package.
255
256=head1 AUTHOR
257
258Thomas Hochstein <thh@inter.net>
259
260=head1 COPYRIGHT AND LICENSE
261
f05ec4ae 262Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
2832c235
TH
263
264This program is free software; you may redistribute it and/or modify it
265under the same terms as Perl itself.
266
267=cut
This page took 0.023485 seconds and 4 git commands to generate.