Add decoding and parsing of From: etc.
[usenet/newsstats.git] / bin / parsedb.pl
CommitLineData
6d72dad2
TH
1#! /usr/bin/perl
2#
3# parsedb.pl
4#
5# This script will parse a database with raw header information
6# from a INN feed to a structured database.
7#
8# It is part of the NewsStats package.
9#
10# Copyright (c) 2013 Thomas Hochstein <thh@inter.net>
11#
12# It can be redistributed and/or modified under the same terms under
13# which Perl itself is published.
14
15BEGIN {
16 our $VERSION = "0.01";
17 use File::Basename;
18 # we're in .../bin, so our module is in ../lib
19 push(@INC, dirname($0).'/../lib');
20}
21use strict;
22use warnings;
23
24use NewsStats qw(:DEFAULT :TimePeriods :SQLHelper);
25
26use DBI;
27use Getopt::Long qw(GetOptions);
28Getopt::Long::config ('bundling');
29
9630376c
TH
30use Encode qw/decode/;
31use Mail::Address;
32
6d72dad2
TH
33################################# Definitions ##################################
34
35# define header names with separate database fields
36my %DBFields = ('date' => 'date',
37 'references' => 'refs',
38 'followup-to' => 'fupto',
39 'from' => 'from_',
40 'sender' => 'sender',
41 'reply-to' => 'replyto',
42 'subject' => 'subject',
43 'organization' => 'organization',
44 'lines' => 'linecount',
45 'approved' => 'approved',
46 'supersedes' => 'supersedes',
47 'expires' => 'expires',
48 'user-agent' => 'useragent',
49 'x-newsreader' => 'xnewsreader',
50 'x-mailer' => 'xmailer',
51 'x-no-archive' => 'xnoarchive',
52 'content-type' => 'contenttype',
53 'content-transfer-encoding' => 'contentencoding',
54 'cancel-lock' => 'cancellock',
55 'injection-info' => 'injectioninfo',
56 'x-trace' => 'xtrace',
57 'nntp-posting-host' => 'postinghost');
58
59# define field list for database
60my @DBFields = qw/day mid refs date path newsgroups fupto from_ from_parsed
61 from_name from_address sender sender_parsed sender_name
62 sender_address replyto replyto_parsed replyto_name
63 replyto_address subject subject_parsed organization linecount
64 approved supersedes expires useragent xnewsreader xmailer
65 xnoarchive contenttype contentencoding cancellock injectioninfo
66 xtrace postinghost headers disregard/;
67
68################################# Main program #################################
69
70### read commandline options
71my ($OptDay,$OptDebug,$OptParseDB,$OptRawDB,$OptTest,$OptConfFile);
72GetOptions ('d|day=s' => \$OptDay,
73 'debug!' => \$OptDebug,
74 'parsedb=s' => \$OptParseDB,
75 'rawdb=s' => \$OptRawDB,
76 't|test!' => \$OptTest,
77 'conffile=s' => \$OptConfFile,
78 'h|help' => \&ShowPOD,
79 'V|version' => \&ShowVersion) or exit 1;
80
81### read configuration
82my %Conf = %{ReadConfig($OptConfFile)};
83
84### override configuration via commandline options
85my %ConfOverride;
86$ConfOverride{'DBTableRaw'} = $OptRawDB if $OptRawDB;
87$ConfOverride{'DBTableParse'} = $OptParseDB if $OptParseDB;
88&OverrideConfig(\%Conf,\%ConfOverride);
89
90### get time period
91### and set $Period for output and expression for SQL 'WHERE' clause
92my ($Period,$SQLWherePeriod) = &GetTimePeriod($OptDay,'day');
93# bail out if --month is invalid or "all"
94&Bleat(2,"--day option has an invalid format - please use 'YYYY-MM-DD' or ".
95 "'YYYY-MM-DD:YYYY-MM-DD'!") if (!$Period or $Period eq 'all time');
96
97### init database
98my $DBHandle = InitDB(\%Conf,1);
99
100### get & write data
101&Bleat(1,'Test mode. Database is not updated.') if $OptTest;
102
103# create $SQLWhereClause
104my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,'NOT disregard');
105
106# delete old data for current period
107if (!$OptTest) {
108 print "----------- Deleting old data ... -----------\n" if $OptDebug;
109 my $DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s %s",
110 $Conf{'DBDatabase'},$Conf{'DBTableParse'},
111 $SQLWhereClause))
112 or &Bleat(2,sprintf("Can't delete old parsed data for %s from %s.%s: ".
113 "$DBI::errstr\n",$Period,
114 $Conf{'DBDatabase'},$Conf{'DBTableParse'}));
115};
116
117# read from DBTableRaw
118print "-------------- Reading data ... -------------\n" if $OptDebug;
119my $DBQuery = $DBHandle->prepare(sprintf("SELECT id, day, mid, peer, path, ".
120 "newsgroups, headers, disregard ".
121 "FROM %s.%s %s", $Conf{'DBDatabase'},
122 $Conf{'DBTableRaw'}, $SQLWhereClause));
123$DBQuery->execute()
124 or &Bleat(2,sprintf("Can't get data for %s from %s.%s: ".
125 "$DBI::errstr\n",$Period,
126 $Conf{'DBDatabase'},$Conf{'DBTableRaw'}));
127
128# set output and database connection to UTF-8
129# as we're going to write decoded header contents containing UTF-8 chars
130binmode(STDOUT, ":utf8");
131$DBHandle->do("SET NAMES 'utf8'");
132
133# parse data in a loop and write it out
134print "-------------- Parsing data ... -------------\n" if $OptDebug;
135while (my $HeadersR = $DBQuery->fetchrow_hashref) {
136 my %Headers = %{$HeadersR};
137
138 # parse $Headers{'headers'} ('headers' from DBTableRaw)
139 # merge continuation lines
140 # from Perl Cookbook, 1st German ed. 1999, pg. 91
141 $Headers{'headers'} =~ s/\n\s+/ /g;
142 # split headers in single lines
143 my $OtherHeaders;
144 for (split(/\n/,$Headers{'headers'})) {
145 # split header lines in header name and header content
146 my ($key,$value) = split(/:/,$_,2);
147 $key =~ s/\s*//;
148 $value =~ s/^\s*(.+)\s*$/$1/;
149 # save each header, separate database fields in %Headers,
150 # the rest in $OtherHeaders (but not Message-ID, Path, Peer
151 # and Newsgroups as those do already exist)
152 if (defined($DBFields{lc($key)})) {
153 $Headers{$DBFields{lc($key)}} = $value;
154 } else {
155 $OtherHeaders .= sprintf("%s: %s\n",$key,$value)
156 if lc($key) !~ /^(message-id|path|peer|newsgroups)$/;
157 }
158 }
159 # replace old (now parsed) $Headers{'headers'} with remanining $OtherHeaders
160 chomp($OtherHeaders);
161 $Headers{'headers'} = $OtherHeaders;
162
9630376c
TH
163 foreach ('from_','sender', 'replyto', 'subject') {
164 if ($Headers{$_}) {
165 my $HeaderName = $_;
166 $HeaderName =~ s/_$//;
167 # decode From: / Sender: / Reply-To: / Subject:
168 if ($Headers{$_} =~ /\?(B|Q)\?/) {
169 $Headers{$HeaderName.'_parsed'} = decode('MIME-Header',$Headers{$_});
170 }
171 # extract name(s) and mail(s) from From: / Sender: / Reply-To:
172 # in parsed form, if available
173 if ($_ ne 'subject') {
174 my @Address;
175 # start parser on header or parsed header
176 # @Address will have an array of Mail::Address objects, one for
177 # each name/mail (you can have more than one person in From:!)
178 if (defined($Headers{$HeaderName.'_parsed'})) {
179 @Address = Mail::Address->parse($Headers{$HeaderName.'_parsed'});
180 } else {
181 @Address = Mail::Address->parse($Headers{$_});
182 }
183 # split each Mail::Address object
184 foreach (@Address) {
185 # take address part
186 $Headers{$HeaderName.'_address'} = $_->address();
187 # take name part form "phrase", if there is one:
188 # From: My Name <addr@ess> (Comment)
189 # otherwise, take it from "comment":
190 # From: addr@ess (Comment)
191 $Headers{$HeaderName.'_name'} = $_->comment()
192 unless $Headers{$HeaderName.'_name'}= $_->phrase;
193 $Headers{$HeaderName.'_name'} =~ s/^\((.+)\)$/$1/;
194 # FIMXE - handle more than one Mail::Address object!
195 }
196 }
197 }
198 }
199
6d72dad2
TH
200 # order output for database entry: fill @SQLBindVars
201 print "-------------- Next entry:\n" if $OptDebug;
202 my @SQLBindVars;
203 foreach (@DBFields) {
204 if (defined($Headers{$_}) and $Headers{$_} ne '') {
205 push (@SQLBindVars,$Headers{$_});
206 printf ("FOUND: %s -> %s\n",$_,$Headers{$_}) if $OptDebug;
207 } else {
208 push (@SQLBindVars,undef);
209 }
210 }
211
212 # write data to DBTableParse
213 if (!$OptTest) {
214 print "-------------- Writing data ... -------------\n" if $OptDebug;
215 my $DBWrite =
216 $DBHandle->prepare(sprintf("INSERT INTO %s.%s (%s) VALUES (%s)",
217 $Conf{'DBDatabase'},
218 $Conf{'DBTableParse'},
219 # get field names from @DBFields
220 join(', ',@DBFields),
221 # create a list of '?' for each DBField
222 join(', ',
223 split(/ /,'? ' x scalar(@DBFields)))
224 ));
225 $DBWrite->execute(@SQLBindVars)
226 or &Bleat(2,sprintf("Can't write parsed data for %s to %s.%s: ".
227 "$DBI::errstr\n",$Period,
228 $Conf{'DBDatabase'},$Conf{'DBTableParse'}));
229 $DBWrite->finish;
230 }
231};
232$DBQuery->finish;
233
234### close handles
235$DBHandle->disconnect;
236
237print "------------------- DONE! -------------------\n" if $OptDebug;
238__END__
239
240################################ Documentation #################################
241
242=head1 NAME
243
244parsedb - parse raw data and save it to a database
245
246=head1 SYNOPSIS
247
248B<parsedb> [B<-Vht>] [B<--day> I<YYYY-MM-DD> | I<YYYY-MM-DD:YYYY-MM-DD>] [B<--rawdb> I<database table>] [B<--parsedb> I<database table>] [B<--conffile> I<filename>] [B<--debug>]
249
250=head1 REQUIREMENTS
251
252See L<doc/README>.
253
254=head1 DESCRIPTION
255
256...
257
258=head2 Configuration
259
260...
261
262=head1 OPTIONS
263
264=over 3
265
266=item B<-V>, B<--version>
267
268Print out version and copyright information and exit.
269
270=item B<-h>, B<--help>
271
272Print this man page and exit.
273
274=item B<--debug>
275
276Output (rather much) debugging information to STDOUT while processing.
277
278=item B<-t>, B<--test>
279
280Do not write results to database. You should use B<--debug> in
281conjunction with B<--test> ... everything else seems a bit pointless.
282
283=item B<-d>, B<--day> I<YYYY-MM-DD[:YYYY-MM-DD]>
284
285Set processing period to a single day in YYYY-MM-DD format or to a time
286period between two days in YYYY-MM-DD:YYYY-MM-DD format (two days, separated
287by a colon).
288
289Defaults to yesterday.
290
291=item B<--rawdb> I<table> (raw data table)
292
293Override I<DBTableRaw> from F<newsstats.conf>.
294
295=item B<--parsedb> I<table> (parsed data table)
296
297Override I<DBTableParse> from F<newsstats.conf>.
298
299=item B<--conffile> I<filename>
300
301Load configuration from I<filename> instead of F<newsstats.conf>.
302
303=back
304
305=head1 INSTALLATION
306
307See L<doc/INSTALL>.
308
309=head1 EXAMPLES
310
311...
312
313=head1 FILES
314
315=over 4
316
317=item F<bin/parsedb.pl>
318
319The script itself.
320
321=item F<lib/NewsStats.pm>
322
323Library functions for the NewsStats package.
324
325=item F<etc/newsstats.conf>
326
327Runtime configuration file.
328
329=back
330
331=head1 BUGS
332
333Please report any bugs or feature requests to the author or use the
334bug tracker at L<http://bugs.th-h.de/>!
335
336=head1 SEE ALSO
337
338=over 2
339
340=item -
341
342L<doc/README>
343
344=item -
345
346L<doc/INSTALL>
347
348=back
349
350This script is part of the B<NewsStats> package.
351
352=head1 AUTHOR
353
354Thomas Hochstein <thh@inter.net>
355
356=head1 COPYRIGHT AND LICENSE
357
358Copyright (c) 2013 Thomas Hochstein <thh@inter.net>
359
360This program is free software; you may redistribute it and/or modify it
361under the same terms as Perl itself.
362
363=cut
This page took 0.026523 seconds and 4 git commands to generate.