Refactor database initialisation in feedlog.pl.
[usenet/newsstats.git] / feedlog.pl
CommitLineData
2832c235
TH
1#! /usr/bin/perl -W
2#
3# feedlog.pl
4#
5# This script will log headers and other data to a database
6# for further analysis by parsing a feed from INN.
7#
8# It is part of the NewsStats package.
9#
f05ec4ae 10# Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
2832c235
TH
11#
12# It can be redistributed and/or modified under the same terms under
13# which Perl itself is published.
14
15BEGIN {
16 our $VERSION = "0.01";
17 use File::Basename;
18 push(@INC, dirname($0));
19}
20use strict;
21
22use NewsStats;
23
24use Sys::Syslog qw(:standard :macros);
25
26use Date::Format;
27use DBI;
f05ec4ae
TH
28use Getopt::Long qw(GetOptions);
29Getopt::Long::config ('bundling');
2832c235 30
98563c61
TH
31################################# Subroutines ##################################
32
33sub PrepareDB {
34### initialise database connection, prepare statement
35### and catch errors
36### IN : \%Conf : reference to configuration hash
37### OUT: $DBHandle: database handle
38### $DBQuery : prepared statement
39 our ($DBHandle, $DBQuery, $OptQuiet);
40 my ($ConfigR) = @_;
41 my %Conf = %$ConfigR;
42 # drop current database connection - hard, if necessary
43 if ($DBHandle) {
44 $DBHandle->disconnect;
45 undef $DBHandle;
46 };
47 # connect to database; try again every 5 seconds
48 while (!$DBHandle) {
49 $DBHandle = InitDB($ConfigR,0);
50 if (!$DBHandle) {
51 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
52 sleep(5);
53 } else {;
54 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
55 }
56 };
57 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
58 timestamp,token,size,peer,path,
59 newsgroups,headers)
60 VALUES (?,?,?,?,?,?,?,?,?,?)",
61 $Conf{'DBDatabase'},
62 $Conf{'DBTableRaw'}));
63 return ($DBHandle,$DBQuery);
64}
65
66
2832c235
TH
67################################# Main program #################################
68
69### read commandline options
f05ec4ae
TH
70my ($OptDebug,$OptQuiet);
71GetOptions ('d|debug!' => \$OptDebug,
72 'q|test!' => \$OptQuiet,
73 'h|help' => \&ShowPOD,
74 'V|version' => \&ShowVersion) or exit 1;
2832c235
TH
75
76### read configuration
f05ec4ae 77my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
2832c235
TH
78
79### init syslog
f05ec4ae
TH
80openlog($0, 'nofatal,pid', LOG_NEWS);
81syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
2832c235
TH
82
83### init database
98563c61 84my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
2832c235
TH
85
86### main loop
87while (<>) {
88 chomp;
89 # catch empty lines trailing or leading
90 if ($_ eq '') {
91 next;
92 }
93 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
94 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
95 # remaining lines contain headers
96 my $Headers = "";
97 while (<>) {
98 chomp;
99 # empty line terminates this article
100 if ($_ eq '') {
101 last;
102 }
103 # collect headers
104 $Headers .= $_."\n" ;
105 }
106
107 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
108 my $Day = time2str("%Y-%m-%d", $Timestamp);
109 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
110
111 # write to database
f05ec4ae
TH
112 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
113 $Path, $Newsgroups, $Headers)) {
98563c61
TH
114 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
115 $DBI::err, $Mid, $DBI::errstr);
116 # if "MySQL server has gone away", try to recover
117 if ($DBI::err == 2006) {
118 # try to reconnect to database
119 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
120 # try to repeat the write attempt as before
121 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
122 $Path, $Newsgroups, $Headers)) {
123 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
124 };
125 # otherwise log missing posting
126 } else {
127 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
128 };
2832c235
TH
129 };
130 $DBQuery->finish;
131
f05ec4ae
TH
132 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
133 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
134 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
135 $Newsgroups, $Headers) if $OptDebug;
2832c235
TH
136}
137
138### close handles
139$DBHandle->disconnect;
f05ec4ae 140syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
2832c235
TH
141closelog();
142
143__END__
144
145################################ Documentation #################################
146
147=head1 NAME
148
149feedlog - log data from an INN feed to a database
150
151=head1 SYNOPSIS
152
153B<feedlog> [B<-Vhdq>]
154
155=head1 REQUIREMENTS
156
f05ec4ae 157See L<doc/README>.
2832c235
TH
158
159=head1 DESCRIPTION
160
161This script will log overview data and complete headers to a database
162table for further examination by parsing a feed from INN. It will
163parse that information and write it to a mysql database table in real
164time.
165
166All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
167fails to initiate a database connection at startup, it will log to
168I<syslog> with I<CRIT> priority and go in an endless loop, as
169terminating would only result in a rapid respawn.
170
171=head2 Configuration
172
f05ec4ae 173B<feedlog> will read its configuration from F<newsstats.conf> which
2832c235
TH
174should be present in the same directory via Config::Auto.
175
f05ec4ae 176See L<doc/INSTALL> for an overview of possible configuration options.
2832c235
TH
177
178=head1 OPTIONS
179
180=over 3
181
f05ec4ae 182=item B<-V>, B<--version>
2832c235 183
f05ec4ae 184Print out version and copyright information and exit.
2832c235 185
f05ec4ae 186=item B<-h>, B<--help>
2832c235
TH
187
188Print this man page and exit.
189
f05ec4ae 190=item B<-d>, B<--debug>
2832c235
TH
191
192Output debugging information to STDERR while parsing STDIN. You'll
193find that information most probably in your B<INN> F<errlog> file.
194
f05ec4ae 195=item B<-q>, B<--quiet>
2832c235
TH
196
197Suppress logging to syslog.
198
199=back
200
201=head1 INSTALLATION
202
f05ec4ae 203See L<doc/INSTALL.>
2832c235
TH
204
205=head1 EXAMPLES
206
207Set up a feed like that in your B<INN> F<newsfeeds> file:
208
209 ## gather statistics for NewsStats
210 newsstats!
211 :!*,de.*
212 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
213
f05ec4ae 214See L<doc/INSTALL> for further information.
2832c235
TH
215
216=head1 FILES
217
218=over 4
219
220=item F<feedlog.pl>
221
222The script itself.
223
224=item F<NewsStats.pm>
225
226Library functions for the NewsStats package.
227
228=item F<newsstats.conf>
229
f05ec4ae 230Runtime configuration file.
2832c235
TH
231
232=back
233
234=head1 BUGS
235
236Please report any bugs or feature requests to the author or use the
237bug tracker at L<http://bugs.th-h.de/>!
238
239=head1 SEE ALSO
240
241=over 2
242
243=item -
244
f05ec4ae 245L<doc/README>
2832c235
TH
246
247=item -
248
f05ec4ae 249L<doc/INSTALL>
2832c235
TH
250
251=back
252
253This script is part of the B<NewsStats> package.
254
255=head1 AUTHOR
256
257Thomas Hochstein <thh@inter.net>
258
259=head1 COPYRIGHT AND LICENSE
260
f05ec4ae 261Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
2832c235
TH
262
263This program is free software; you may redistribute it and/or modify it
264under the same terms as Perl itself.
265
266=cut
This page took 0.023695 seconds and 4 git commands to generate.