Be more fault-tolerant when reading checkgroups.
[usenet/newsstats.git] / feedlog.pl
CommitLineData
2832c235
TH
1#! /usr/bin/perl -W
2#
3# feedlog.pl
4#
5# This script will log headers and other data to a database
6# for further analysis by parsing a feed from INN.
7#
8# It is part of the NewsStats package.
9#
f05ec4ae 10# Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
2832c235
TH
11#
12# It can be redistributed and/or modified under the same terms under
13# which Perl itself is published.
14
15BEGIN {
16 our $VERSION = "0.01";
17 use File::Basename;
18 push(@INC, dirname($0));
19}
20use strict;
21
22use NewsStats;
23
24use Sys::Syslog qw(:standard :macros);
25
26use Date::Format;
27use DBI;
f05ec4ae
TH
28use Getopt::Long qw(GetOptions);
29Getopt::Long::config ('bundling');
2832c235
TH
30
31################################# Main program #################################
32
33### read commandline options
f05ec4ae
TH
34my ($OptDebug,$OptQuiet);
35GetOptions ('d|debug!' => \$OptDebug,
36 'q|test!' => \$OptQuiet,
37 'h|help' => \&ShowPOD,
38 'V|version' => \&ShowVersion) or exit 1;
2832c235
TH
39
40### read configuration
f05ec4ae 41my %Conf = %{ReadConfig($HomePath.'/newsstats.conf')};
2832c235
TH
42
43### init syslog
f05ec4ae
TH
44openlog($0, 'nofatal,pid', LOG_NEWS);
45syslog(LOG_NOTICE, "$MyVersion starting up.") if !$OptQuiet;
2832c235
TH
46
47### init database
48my $DBHandle = InitDB(\%Conf,0);
49if (!$DBHandle) {
50 syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);
51 while (1) {}; # go into endless loop to suppress further errors and respawning
52};
f05ec4ae
TH
53my $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,
54 timestamp,token,size,peer,path,
55 newsgroups,headers)
56 VALUES (?,?,?,?,?,?,?,?,?,?)",
57 $Conf{'DBDatabase'},$Conf{'DBTableRaw'}));
2832c235
TH
58
59### main loop
60while (<>) {
61 chomp;
62 # catch empty lines trailing or leading
63 if ($_ eq '') {
64 next;
65 }
66 # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups
67 my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;
68 # remaining lines contain headers
69 my $Headers = "";
70 while (<>) {
71 chomp;
72 # empty line terminates this article
73 if ($_ eq '') {
74 last;
75 }
76 # collect headers
77 $Headers .= $_."\n" ;
78 }
79
80 # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp
81 my $Day = time2str("%Y-%m-%d", $Timestamp);
82 my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);
83
84 # write to database
f05ec4ae
TH
85 if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer,
86 $Path, $Newsgroups, $Headers)) {
2832c235
TH
87 syslog(LOG_ERR, 'Database error: %s', $DBI::errstr);
88 };
89 $DBQuery->finish;
90
f05ec4ae
TH
91 warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\n".
92 "Size: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",
93 $Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path,
94 $Newsgroups, $Headers) if $OptDebug;
2832c235
TH
95}
96
97### close handles
98$DBHandle->disconnect;
f05ec4ae 99syslog(LOG_NOTICE, "$0 closing down.") if !$OptQuiet;
2832c235
TH
100closelog();
101
102__END__
103
104################################ Documentation #################################
105
106=head1 NAME
107
108feedlog - log data from an INN feed to a database
109
110=head1 SYNOPSIS
111
112B<feedlog> [B<-Vhdq>]
113
114=head1 REQUIREMENTS
115
f05ec4ae 116See L<doc/README>.
2832c235
TH
117
118=head1 DESCRIPTION
119
120This script will log overview data and complete headers to a database
121table for further examination by parsing a feed from INN. It will
122parse that information and write it to a mysql database table in real
123time.
124
125All reporting is done to I<syslog> via I<news> facility. If B<feedlog>
126fails to initiate a database connection at startup, it will log to
127I<syslog> with I<CRIT> priority and go in an endless loop, as
128terminating would only result in a rapid respawn.
129
130=head2 Configuration
131
f05ec4ae 132B<feedlog> will read its configuration from F<newsstats.conf> which
2832c235
TH
133should be present in the same directory via Config::Auto.
134
f05ec4ae 135See L<doc/INSTALL> for an overview of possible configuration options.
2832c235
TH
136
137=head1 OPTIONS
138
139=over 3
140
f05ec4ae 141=item B<-V>, B<--version>
2832c235 142
f05ec4ae 143Print out version and copyright information and exit.
2832c235 144
f05ec4ae 145=item B<-h>, B<--help>
2832c235
TH
146
147Print this man page and exit.
148
f05ec4ae 149=item B<-d>, B<--debug>
2832c235
TH
150
151Output debugging information to STDERR while parsing STDIN. You'll
152find that information most probably in your B<INN> F<errlog> file.
153
f05ec4ae 154=item B<-q>, B<--quiet>
2832c235
TH
155
156Suppress logging to syslog.
157
158=back
159
160=head1 INSTALLATION
161
f05ec4ae 162See L<doc/INSTALL.>
2832c235
TH
163
164=head1 EXAMPLES
165
166Set up a feed like that in your B<INN> F<newsfeeds> file:
167
168 ## gather statistics for NewsStats
169 newsstats!
170 :!*,de.*
171 :Tc,WmtfbsPNH,Ac:/path/to/feedlog.pl
172
f05ec4ae 173See L<doc/INSTALL> for further information.
2832c235
TH
174
175=head1 FILES
176
177=over 4
178
179=item F<feedlog.pl>
180
181The script itself.
182
183=item F<NewsStats.pm>
184
185Library functions for the NewsStats package.
186
187=item F<newsstats.conf>
188
f05ec4ae 189Runtime configuration file.
2832c235
TH
190
191=back
192
193=head1 BUGS
194
195Please report any bugs or feature requests to the author or use the
196bug tracker at L<http://bugs.th-h.de/>!
197
198=head1 SEE ALSO
199
200=over 2
201
202=item -
203
f05ec4ae 204L<doc/README>
2832c235
TH
205
206=item -
207
f05ec4ae 208L<doc/INSTALL>
2832c235
TH
209
210=back
211
212This script is part of the B<NewsStats> package.
213
214=head1 AUTHOR
215
216Thomas Hochstein <thh@inter.net>
217
218=head1 COPYRIGHT AND LICENSE
219
f05ec4ae 220Copyright (c) 2010-2012 Thomas Hochstein <thh@inter.net>
2832c235
TH
221
222This program is free software; you may redistribute it and/or modify it
223under the same terms as Perl itself.
224
225=cut
This page took 0.021376 seconds and 4 git commands to generate.