Fix --conffile in POD.
[usenet/newsstats.git] / bin / feedlog.pl
CommitLineData
3f817eb4 1#! /usr/bin/perl
2832c235
TH
2#
3# feedlog.pl
4#
5# This script will log headers and other data to a database
6# for further analysis by parsing a feed from INN.
dfc2b81c 7#
2832c235
TH
8# It is part of the NewsStats package.
9#
07c0b258 10# Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
2832c235 11#
dfc2b81c 12# It can be redistributed and/or modified under the same terms under
2832c235
TH
13# which Perl itself is published.
14
15BEGIN {
24d2011f 16 our $VERSION = "0.02";
2832c235 17 use File::Basename;
2ad99c20
TH
18 # we're in .../bin, so our module is in ../lib
19 push(@INC, dirname($0).'/../lib');
2832c235
TH
20}
21use strict;
3f817eb4 22use warnings;
2832c235
TH
23
24use NewsStats;
25
26use Sys::Syslog qw(:standard :macros);
27
28use Date::Format;
29use DBI;
f05ec4ae
TH
30use Getopt::Long qw(GetOptions);
31Getopt::Long::config ('bundling');
2832c235 32
98563c61
TH
33################################# Subroutines ##################################
34
35sub PrepareDB {
36### initialise database connection, prepare statement
37### and catch errors
38### IN : \%Conf : reference to configuration hash
39### OUT: $DBHandle: database handle
40### $DBQuery : prepared statement
41 our ($DBHandle, $DBQuery, $OptQuiet);
42 my ($ConfigR) = @_;
43 my %Conf = %$ConfigR;
44 # drop current database connection - hard, if necessary
45 if ($DBHandle) {
46 $DBHandle->disconnect;
47 undef $DBHandle;
48 };
49 # connect to database; try again every 5 seconds
50 while (!$DBHandle) {
51 $DBHandle = InitDB($ConfigR,0);
52 if (!$DBHandle) {
53 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
54 sleep(5);
55 } else {;
56 syslog(LOG_NOTICE, "Database connection (re-)established successfully.") if !$OptQuiet;
57 }
58 };
59 $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
60 timestamp,token,size,peer,path,
61 newsgroups,headers)
62 VALUES (?,?,?,?,?,?,?,?,?,?)",
63 $Conf{'DBDatabase'},
64 $Conf{'DBTableRaw'}));
65 return ($DBHandle,$DBQuery);
66}
67
68
2832c235
TH
69################################# Main program #################################
70
71### read commandline options
23ab67a0 72my ($OptDebug,$OptQuiet,$OptConfFile);
f05ec4ae
TH
73GetOptions ('d|debug!' => \$OptDebug,
74 'q|test!' => \$OptQuiet,
23ab67a0 75 'conffile=s' => \$OptConfFile,
f05ec4ae
TH
76 'h|help' => \&ShowPOD,
77 'V|version' => \&ShowVersion) or exit 1;
2832c235
TH
78
79### read configuration
23ab67a0 80my %Conf = %{ReadConfig($OptConfFile)};
2832c235
TH
81
82### init syslog
f05ec4ae
TH
83openlog($0, 'nofatal,pid', LOG_NEWS);
84syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
2832c235
TH
85
86### init database
98563c61 87my ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
2832c235
TH
88
89### main loop
90while (<>) {
91 chomp;
92 # catch empty lines trailing or leading
93 if ($_ eq '') {
94 next;
95 }
96 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
97 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
98 # remaining lines contain headers
99 my $Headers = "";
100 while (<>) {
101 chomp;
102 # empty line terminates this article
103 if ($_ eq '') {
104 last;
105 }
106 # collect headers
107 $Headers .= $_."\n" ;
108 }
109
110 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
111 my $Day = time2str("%Y-%m-%d", $Timestamp);
112 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
113
114 # write to database
f05ec4ae
TH
115 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
116 $Path, $Newsgroups, $Headers)) {
98563c61
TH
117 syslog(LOG_ERR, 'Database error %s while processing %s: %s',
118 $DBI::err, $Mid, $DBI::errstr);
119 # if "MySQL server has gone away", try to recover
120 if ($DBI::err == 2006) {
121 # try to reconnect to database
122 ($DBHandle,$DBQuery) = PrepareDB(\%Conf);
123 # try to repeat the write attempt as before
124 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
125 $Path, $Newsgroups, $Headers)) {
126 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
127 };
128 # otherwise log missing posting
129 } else {
130 syslog(LOG_ERR, '%s was dropped and lost.',$Mid);
131 };
2832c235
TH
132 };
133 $DBQuery->finish;
23ab67a0 134
f05ec4ae
TH
135 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
136 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
137 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
138 $Newsgroups, $Headers) if $OptDebug;
2832c235
TH
139}
140
141### close handles
142$DBHandle->disconnect;
f05ec4ae 143syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
2832c235
TH
144closelog();
145
146__END__
147
148################################ Documentation #################################
149
150=head1 NAME
151
152feedlog - log data from an INN feed to a database
153
154=head1 SYNOPSIS
155
e39d4207 156B<feedlog> [B<-Vhdq>] [B<--conffile> I<filename>]
2832c235
TH
157
158=head1 REQUIREMENTS
159
f05ec4ae 160See L<doc/README>.
2832c235
TH
161
162=head1 DESCRIPTION
163
164This script will log overview data and complete headers to a database
165table for further examination by parsing a feed from INN. It will
166parse that information and write it to a mysql database table in real
167time.
168
169All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
170fails to initiate a database connection at startup, it will log to
171I<syslog> with I<CRIT> priority and go in an endless loop, as
172terminating would only result in a rapid respawn.
173
174=head2 Configuration
175
f05ec4ae 176B<feedlog> will read its configuration from F<newsstats.conf> which
2832c235
TH
177should be present in the same directory via Config::Auto.
178
f05ec4ae 179See L<doc/INSTALL> for an overview of possible configuration options.
2832c235
TH
180
181=head1 OPTIONS
182
183=over 3
184
f05ec4ae 185=item B<-V>, B<--version>
2832c235 186
f05ec4ae 187Print out version and copyright information and exit.
2832c235 188
f05ec4ae 189=item B<-h>, B<--help>
2832c235
TH
190
191Print this man page and exit.
192
f05ec4ae 193=item B<-d>, B<--debug>
2832c235
TH
194
195Output debugging information to STDERR while parsing STDIN. You'll
196find that information most probably in your B<INN> F<errlog> file.
197
f05ec4ae 198=item B<-q>, B<--quiet>
2832c235
TH
199
200Suppress logging to syslog.
201
23ab67a0
TH
202=item B<--conffile> I<filename>
203
204Load configuration from I<filename> instead of F<newsstats.conf>.
205
2832c235
TH
206=back
207
208=head1 INSTALLATION
209
95d9fe2c 210See L<doc/INSTALL>.
2832c235
TH
211
212=head1 EXAMPLES
213
214Set up a feed like that in your B<INN> F<newsfeeds> file:
215
216 ## gather statistics for NewsStats
217 newsstats!
218 :!*,de.*
219 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
220
f05ec4ae 221See L<doc/INSTALL> for further information.
2832c235
TH
222
223=head1 FILES
224
225=over 4
226
2ad99c20 227=item F<bin/feedlog.pl>
2832c235
TH
228
229The script itself.
230
2ad99c20 231=item F<lib/NewsStats.pm>
2832c235
TH
232
233Library functions for the NewsStats package.
234
2ad99c20 235=item F<etc/newsstats.conf>
2832c235 236
f05ec4ae 237Runtime configuration file.
2832c235
TH
238
239=back
240
241=head1 BUGS
242
243Please report any bugs or feature requests to the author or use the
244bug tracker at L<http://bugs.th-h.de/>!
245
246=head1 SEE ALSO
247
248=over 2
249
250=item -
251
f05ec4ae 252L<doc/README>
2832c235
TH
253
254=item -
255
f05ec4ae 256L<doc/INSTALL>
2832c235
TH
257
258=back
259
260This script is part of the B<NewsStats> package.
261
262=head1 AUTHOR
263
264Thomas Hochstein <thh@inter.net>
265
266=head1 COPYRIGHT AND LICENSE
267
28717921 268Copyright (c) 2010-2013 Thomas Hochstein <thh@inter.net>
2832c235
TH
269
270This program is free software; you may redistribute it and/or modify it
271under the same terms as Perl itself.
272
273=cut
This page took 0.026689 seconds and 4 git commands to generate.