Fix some whitespace.
[usenet/newsstats.git] / bin / feedlog.pl
CommitLineData
3f817eb4 1#! /usr/bin/perl
2832c235
TH
2#
3# feedlog.pl
4#
5# This script will log headers and other data to a database
6# for further analysis by parsing a feed from INN.
dfc2b81c 7#
2832c235
TH
8# It is part of the NewsStats package.
9#
07c0b258 10# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
2832c235 11#
dfc2b81c 12# It can be redistributed and/or modified under the same terms under
2832c235
TH
13# which Perl itself is published.
14
15BEGIN {
16 our $VERSION = "0.01";
17 use File::Basename;
2ad99c20
TH
18 # we're in .../bin, so our module is in ../lib
19 push(@INC, dirname($0).'/../lib');
2832c235
TH
20}
21use strict;
3f817eb4 22use warnings;
2832c235
TH
23
24use NewsStats;
25
26use Sys::Syslog qw(:standard :macros);
27
28use Date::Format;
29use DBI;
f05ec4ae
TH
30use Getopt::Long qw(GetOptions);
31Getopt::Long::config ('bundling');
2832c235 32
98563c61
TH
33################################# Subroutines ##################################
34
35sub PrepareDB {
36### initialise database connection, prepare statement
37### and catch errors
38### IN : \%Conf : reference to configuration hash
39### OUT: $DBHandle: database handle
40### $DBQuery : prepared statement
41 our ($DBHandle, $DBQuery, $OptQuiet);
42 my ($ConfigR) = @_;
43 my %Conf = %$ConfigR;
44 # drop current database connection - hard, if necessary
45 if ($DBHandle) {
46 $DBHandle->disconnect;
47 undef $DBHandle;
48 };
49 # connect to database; try again every 5 seconds
50 while (!$DBHandle) {
51 $DBHandle = InitDB($ConfigR,0);
52 if (!$DBHandle) {
53 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
54 sleep(5);
55 } else {;
56 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
57 }
58 };
59 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
60 timestamp,token,size,peer,path,
61 newsgroups,headers)
62 VALUES (?,?,?,?,?,?,?,?,?,?)",
63 $Conf{'DBDatabase'},
64 $Conf{'DBTableRaw'}));
65 return ($DBHandle,$DBQuery);
66}
67
68
2832c235
TH
69################################# Main program #################################
70
71### read commandline options
f05ec4ae
TH
72my ($OptDebug,$OptQuiet);
73GetOptions ('d|debug!' => \$OptDebug,
74 'q|test!' => \$OptQuiet,
75 'h|help' => \&ShowPOD,
76 'V|version' => \&ShowVersion) or exit 1;
2832c235
TH
77
78### read configuration
2ad99c20 79my %Conf = %{ReadConfig('')};
2832c235
TH
80
81### init syslog
f05ec4ae
TH
82openlog($0, 'nofatal,pid', LOG_NEWS);
83syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
2832c235
TH
84
85### init database
98563c61 86my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
2832c235
TH
87
88### main loop
89while (<>) {
90 chomp;
91 # catch empty lines trailing or leading
92 if ($_ eq '') {
93 next;
94 }
95 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
96 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
97 # remaining lines contain headers
98 my $Headers = "";
99 while (<>) {
100 chomp;
101 # empty line terminates this article
102 if ($_ eq '') {
103 last;
104 }
105 # collect headers
106 $Headers .= $_."\n" ;
107 }
108
109 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
110 my $Day = time2str("%Y-%m-%d", $Timestamp);
111 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
112
113 # write to database
f05ec4ae
TH
114 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
115 $Path, $Newsgroups, $Headers)) {
98563c61
TH
116 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
117 $DBI::err, $Mid, $DBI::errstr);
118 # if "MySQL server has gone away", try to recover
119 if ($DBI::err == 2006) {
120 # try to reconnect to database
121 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
122 # try to repeat the write attempt as before
123 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
124 $Path, $Newsgroups, $Headers)) {
125 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
126 };
127 # otherwise log missing posting
128 } else {
129 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
130 };
2832c235
TH
131 };
132 $DBQuery->finish;
133
f05ec4ae
TH
134 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
135 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
136 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
137 $Newsgroups, $Headers) if $OptDebug;
2832c235
TH
138}
139
140### close handles
141$DBHandle->disconnect;
f05ec4ae 142syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
2832c235
TH
143closelog();
144
145__END__
146
147################################ Documentation #################################
148
149=head1 NAME
150
151feedlog - log data from an INN feed to a database
152
153=head1 SYNOPSIS
154
155B<feedlog> [B<-Vhdq>]
156
157=head1 REQUIREMENTS
158
f05ec4ae 159See L<doc/README>.
2832c235
TH
160
161=head1 DESCRIPTION
162
163This script will log overview data and complete headers to a database
164table for further examination by parsing a feed from INN. It will
165parse that information and write it to a mysql database table in real
166time.
167
168All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
169fails to initiate a database connection at startup, it will log to
170I<syslog> with I<CRIT> priority and go in an endless loop, as
171terminating would only result in a rapid respawn.
172
173=head2 Configuration
174
f05ec4ae 175B<feedlog> will read its configuration from F<newsstats.conf> which
2832c235
TH
176should be present in the same directory via Config::Auto.
177
f05ec4ae 178See L<doc/INSTALL> for an overview of possible configuration options.
2832c235
TH
179
180=head1 OPTIONS
181
182=over 3
183
f05ec4ae 184=item B<-V>, B<--version>
2832c235 185
f05ec4ae 186Print out version and copyright information and exit.
2832c235 187
f05ec4ae 188=item B<-h>, B<--help>
2832c235
TH
189
190Print this man page and exit.
191
f05ec4ae 192=item B<-d>, B<--debug>
2832c235
TH
193
194Output debugging information to STDERR while parsing STDIN. You'll
195find that information most probably in your B<INN> F<errlog> file.
196
f05ec4ae 197=item B<-q>, B<--quiet>
2832c235
TH
198
199Suppress logging to syslog.
200
201=back
202
203=head1 INSTALLATION
204
95d9fe2c 205See L<doc/INSTALL>.
2832c235
TH
206
207=head1 EXAMPLES
208
209Set up a feed like that in your B<INN> F<newsfeeds> file:
210
211 ## gather statistics for NewsStats
212 newsstats!
213 :!*,de.*
214 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
215
f05ec4ae 216See L<doc/INSTALL> for further information.
2832c235
TH
217
218=head1 FILES
219
220=over 4
221
2ad99c20 222=item F<bin/feedlog.pl>
2832c235
TH
223
224The script itself.
225
2ad99c20 226=item F<lib/NewsStats.pm>
2832c235
TH
227
228Library functions for the NewsStats package.
229
2ad99c20 230=item F<etc/newsstats.conf>
2832c235 231
f05ec4ae 232Runtime configuration file.
2832c235
TH
233
234=back
235
236=head1 BUGS
237
238Please report any bugs or feature requests to the author or use the
239bug tracker at L<http://bugs.th-h.de/>!
240
241=head1 SEE ALSO
242
243=over 2
244
245=item -
246
f05ec4ae 247L<doc/README>
2832c235
TH
248
249=item -
250
f05ec4ae 251L<doc/INSTALL>
2832c235
TH
252
253=back
254
255This script is part of the B<NewsStats> package.
256
257=head1 AUTHOR
258
259Thomas Hochstein <thh@inter.net>
260
261=head1 COPYRIGHT AND LICENSE
262
f05ec4ae 263Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
2832c235
TH
264
265This program is free software; you may redistribute it and/or modify it
266under the same terms as Perl itself.
267
268=cut
This page took 0.023638 seconds and 4 git commands to generate.