[usenet/newsstats.git] / bin / parsedb.pl

#! /usr/bin/perl
#
# parsedb.pl
#
# This script will parse a database with raw header information
# from a INN feed to a structured database.
#
# It is part of the NewsStats package.
#
# Copyright (c) 2013 Thomas Hochstein <thh@inter.net>
#
# It can be redistributed and/or modified under the same terms under
# which Perl itself is published.

BEGIN {
  our $VERSION = "0.01";
  use File::Basename;
  # we're in .../bin, so our module is in ../lib
  push(@INC, dirname($0).'/../lib');
}
use strict;
use warnings;

use NewsStats qw(:DEFAULT :TimePeriods :SQLHelper);

use DBI;
use Getopt::Long qw(GetOptions);
Getopt::Long::config ('bundling');

use Encode qw/decode/;
use Mail::Address;

################################# Definitions ##################################

# define header names with separate database fields
my %DBFields = ('date'                      => 'date',
                'references'                => 'refs',
                'followup-to'               => 'fupto',
                'from'                      => 'from_',
                'sender'                    => 'sender',
                'reply-to'                  => 'replyto',
                'subject'                   => 'subject',
                'organization'              => 'organization',
                'lines'                     => 'linecount',
                'approved'                  => 'approved',
                'supersedes'                => 'supersedes',
                'expires'                   => 'expires',
                'user-agent'                => 'useragent',
                'x-newsreader'              => 'xnewsreader',
                'x-mailer'                  => 'xmailer',
                'x-no-archive'              => 'xnoarchive',
                'content-type'              => 'contenttype',
                'content-transfer-encoding' => 'contentencoding',
                'cancel-lock'               => 'cancellock',
                'injection-info'            => 'injectioninfo',
                'x-trace'                   => 'xtrace',
                'nntp-posting-host'         => 'postinghost');

# define field list for database
my @DBFields = qw/day mid refs date path newsgroups fupto from_ from_parsed
                 from_name from_address sender sender_parsed sender_name
                 sender_address replyto replyto_parsed replyto_name
                 replyto_address subject subject_parsed organization linecount
                 approved supersedes expires useragent xnewsreader xmailer
                 xnoarchive contenttype contentencoding cancellock injectioninfo
                 xtrace postinghost headers disregard/;

################################# Main program #################################

### read commandline options
my ($OptDay,$OptDebug,$OptParseDB,$OptRawDB,$OptTest,$OptConfFile);
GetOptions ('d|day=s'         => \$OptDay,
            'debug!'          => \$OptDebug,
            'parsedb=s'       => \$OptParseDB,
            'rawdb=s'         => \$OptRawDB,
            't|test!'         => \$OptTest,
            'conffile=s'      => \$OptConfFile,
            'h|help'          => \&ShowPOD,
            'V|version'       => \&ShowVersion) or exit 1;

### read configuration
my %Conf = %{ReadConfig($OptConfFile)};

### override configuration via commandline options
my %ConfOverride;
$ConfOverride{'DBTableRaw'}   = $OptRawDB if $OptRawDB;
$ConfOverride{'DBTableParse'} = $OptParseDB if $OptParseDB;
&OverrideConfig(\%Conf,\%ConfOverride);

### get time period
### and set $Period for output and expression for SQL 'WHERE' clause
my ($Period,$SQLWherePeriod) = &GetTimePeriod($OptDay,'day');
# bail out if --month is invalid or "all"
&Bleat(2,"--day option has an invalid format - please use 'YYYY-MM-DD' or ".
         "'YYYY-MM-DD:YYYY-MM-DD'!") if (!$Period or $Period eq 'all time');

### init database
my $DBHandle = InitDB(\%Conf,1);

### get & write data
&Bleat(1,'Test mode. Database is not updated.') if $OptTest;

# create $SQLWhereClause
my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,'NOT disregard');

# delete old data for current period
if (!$OptTest) {
  print "----------- Deleting old data ... -----------\n" if $OptDebug;
  my $DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s %s",
                                     $Conf{'DBDatabase'},$Conf{'DBTableParse'},
                                     $SQLWhereClause))
      or &Bleat(2,sprintf("Can't delete old parsed data for %s from %s.%s: ".
                          "$DBI::errstr\n",$Period,
                          $Conf{'DBDatabase'},$Conf{'DBTableParse'}));
};

# read from DBTableRaw
print "-------------- Reading data ... -------------\n" if $OptDebug;
my $DBQuery = $DBHandle->prepare(sprintf("SELECT id, day, mid, peer, path, ".
                                         "newsgroups, headers, disregard ".
                                         "FROM %s.%s %s", $Conf{'DBDatabase'},
                                         $Conf{'DBTableRaw'}, $SQLWhereClause));
$DBQuery->execute()
  or &Bleat(2,sprintf("Can't get data for %s from %s.%s: ".
                      "$DBI::errstr\n",$Period,
                      $Conf{'DBDatabase'},$Conf{'DBTableRaw'}));

# set output and database connection to UTF-8
# as we're going to write decoded header contents containing UTF-8 chars
binmode(STDOUT, ":utf8");
$DBHandle->do("SET NAMES 'utf8'");

# parse data in a loop and write it out
print "-------------- Parsing data ... -------------\n" if $OptDebug;
while (my $HeadersR = $DBQuery->fetchrow_hashref) {
  my %Headers = %{$HeadersR};

  # parse $Headers{'headers'} ('headers' from DBTableRaw)
  # merge continuation lines
  # from Perl Cookbook, 1st German ed. 1999, pg. 91
  $Headers{'headers'} =~ s/\n\s+/ /g;
  # split headers in single lines
  my $OtherHeaders;
  for (split(/\n/,$Headers{'headers'})) {
    # split header lines in header name and header content
    my ($key,$value) = split(/:/,$_,2);
    $key =~ s/\s*//;
    $value =~ s/^\s*(.+)\s*$/$1/;
    # save each header, separate database fields in %Headers,
    # the rest in $OtherHeaders (but not Message-ID, Path, Peer
    # and Newsgroups as those do already exist)
    if (defined($DBFields{lc($key)})) {
      $Headers{$DBFields{lc($key)}} = $value;
    } else {
      $OtherHeaders .= sprintf("%s: %s\n",$key,$value)
        if lc($key) !~ /^(message-id|path|peer|newsgroups)$/;
    }
  }
  # replace old (now parsed) $Headers{'headers'} with remanining $OtherHeaders
  chomp($OtherHeaders);
  $Headers{'headers'} = $OtherHeaders;

  foreach ('from_','sender', 'replyto', 'subject') {
    if ($Headers{$_}) {
      my $HeaderName = $_;
      $HeaderName  =~ s/_$//;
      # decode From: / Sender: / Reply-To: / Subject:
      if ($Headers{$_} =~ /\?(B|Q)\?/) {
        $Headers{$HeaderName.'_parsed'} = decode('MIME-Header',$Headers{$_});
      }
      # extract name(s) and mail(s) from From: / Sender: / Reply-To:
      # in parsed form, if available
      if ($_ ne 'subject') {
        my @Address;
        # start parser on header or parsed header
        # @Address will have an array of Mail::Address objects, one for
        # each name/mail (you can have more than one person in From:!)
        if (defined($Headers{$HeaderName.'_parsed'})) {
          @Address = Mail::Address->parse($Headers{$HeaderName.'_parsed'});
        } else {
          @Address = Mail::Address->parse($Headers{$_});
        }
        # split each Mail::Address object to @Names and @Adresses
        my (@Names,@Adresses);
        foreach (@Address) {
          # take address part in @Addresses
          push (@Adresses, $_->address());
          # take name part form "phrase", if there is one:
          # From: My Name <addr@ess> (Comment)
          # otherwise, take it from "comment":
          # From: addr@ess (Comment)
          # and push it in @Names
          my ($Name);
          $Name = $_->comment() unless $Name = $_->phrase;
          $Name =~ s/^\((.+)\)$/$1/;
          push (@Names, $Name);
        }
        # put all @Adresses and all @Names in %Headers as comma separated lists
        $Headers{$HeaderName.'_address'} = join(', ',@Adresses);
        $Headers{$HeaderName.'_name'}    = join(', ',@Names);
      }
    }
  }

  # order output for database entry: fill @SQLBindVars
  print "-------------- Next entry:\n" if $OptDebug;
  my @SQLBindVars;
  foreach (@DBFields) {
    if (defined($Headers{$_}) and $Headers{$_} ne '') {
      push (@SQLBindVars,$Headers{$_});
      printf ("FOUND: %s -> %s\n",$_,$Headers{$_}) if $OptDebug;
    } else {
      push (@SQLBindVars,undef);
    }
  }

  # write data to DBTableParse
  if (!$OptTest) {
    print "-------------- Writing data ... -------------\n" if $OptDebug;
    my $DBWrite =
       $DBHandle->prepare(sprintf("INSERT INTO %s.%s (%s) VALUES (%s)",
                                  $Conf{'DBDatabase'},
                                  $Conf{'DBTableParse'},
                                  # get field names from @DBFields
                                  join(', ',@DBFields),
                                  # create a list of '?' for each DBField
                                  join(', ',
                                       split(/ /,'? ' x scalar(@DBFields)))
                                ));
  $DBWrite->execute(@SQLBindVars)
      or &Bleat(2,sprintf("Can't write parsed data for %s to %s.%s: ".
                          "$DBI::errstr\n",$Period,
                          $Conf{'DBDatabase'},$Conf{'DBTableParse'}));
    $DBWrite->finish;
  }
};
$DBQuery->finish;

### close handles
$DBHandle->disconnect;

print "------------------- DONE! -------------------\n" if $OptDebug;
__END__

################################ Documentation #################################

=head1 NAME

parsedb - parse raw data and save it to a database

=head1 SYNOPSIS

B<parsedb> [B<-Vht>] [B<--day> I<YYYY-MM-DD> | I<YYYY-MM-DD:YYYY-MM-DD>] [B<--rawdb> I<database table>] [B<--parsedb> I<database table>] [B<--conffile> I<filename>] [B<--debug>]

=head1 REQUIREMENTS

See L<doc/README>.

=head1 DESCRIPTION

This script will parse raw, unstructured headers from a database table which is
fed from F<feedlog.pl> for a given time period and write its results to
nother database table with separate fields (columns) for most (or even all)
relevant headers.

I<Subject:>, I<From:>, I<Sender:> and I<Reply-To:> will be parsed from MIME
encoded words to UTF-8 as needed while the unparsed copy is kept. From that
parsed copy, I<From:>, I<Sender:> and I<Reply-To:> will also be split into
separate name(s) and address(es) fields while the un-splitted copy is kept,
too.

B<parsedb> should be run nightly from cron for yesterdays data so all
other scripts get current information. The time period to act on defaults to
yesterday, accordingly; you can assign another time period or a single day via
the B<--day> option (see below).

=head2 Configuration

B<parsedb> will read its configuration from F<newsstats.conf>
should be present in etc/ via Config::Auto or from a configuration file
submitted by the B<--conffile> option.

See L<doc/INSTALL> for an overview of possible configuration options.

You can override configuration options via the B<--rawdb> and
B<--parsedb> options, respectively.

=head1 OPTIONS

=over 3

=item B<-V>, B<--version>

Print out version and copyright information and exit.

=item B<-h>, B<--help>

Print this man page and exit.

=item B<--debug>

Output (rather much) debugging information to STDOUT while processing.

=item B<-t>, B<--test>

Do not write results to database. You should use B<--debug> in
conjunction with B<--test> ... everything else seems a bit pointless.

=item B<-d>, B<--day> I<YYYY-MM-DD[:YYYY-MM-DD]>

Set processing period to a single day in YYYY-MM-DD format or to a time
period between two days in YYYY-MM-DD:YYYY-MM-DD format (two days, separated
by a colon).

Defaults to yesterday.

=item B<--rawdb> I<table> (raw data table)

Override I<DBTableRaw> from F<newsstats.conf>.

=item B<--parsedb> I<table> (parsed data table)

Override I<DBTableParse> from F<newsstats.conf>.

=item B<--conffile> I<filename>

Load configuration from I<filename> instead of F<newsstats.conf>.

=back

=head1 INSTALLATION

See L<doc/INSTALL>.

=head1 EXAMPLES

An example crontab entry:

    0 1 * * * /path/to/bin/parsedb.pl

Do a dry run for yesterday's data, showing results of processing:

    parsedb --debug --test | less

=head1 FILES

=over 4

=item F<bin/parsedb.pl>

The script itself.

=item F<lib/NewsStats.pm>

Library functions for the NewsStats package.

=item F<etc/newsstats.conf>

Runtime configuration file.

=back

=head1 BUGS

Please report any bugs or feature requests to the author or use the
bug tracker at L<http://bugs.th-h.de/>!

=head1 SEE ALSO

=over 2

=item -

L<doc/README>

=item -

L<doc/INSTALL>

=back

This script is part of the B<NewsStats> package.

=head1 AUTHOR

Thomas Hochstein <thh@inter.net>

=head1 COPYRIGHT AND LICENSE

Copyright (c) 2013 Thomas Hochstein <thh@inter.net>

This program is free software; you may redistribute it and/or modify it
under the same terms as Perl itself.

=cut
Commit	Line	Data
6d72dad2 TH	1	#! /usr/bin/perl
	2	#
	3	# parsedb.pl
	4	#
	5	# This script will parse a database with raw header information
	6	# from a INN feed to a structured database.
	7	#
	8	# It is part of the NewsStats package.
	9	#
	10	# Copyright (c) 2013 Thomas Hochstein <thh@inter.net>
	11	#
	12	# It can be redistributed and/or modified under the same terms under
	13	# which Perl itself is published.
	14
	15	BEGIN {
	16	our $VERSION = "0.01";
	17	use File::Basename;
	18	# we're in .../bin, so our module is in ../lib
	19	push(@INC, dirname($0).'/../lib');
	20	}
	21	use strict;
	22	use warnings;
	23
	24	use NewsStats qw(:DEFAULT :TimePeriods :SQLHelper);
	25
	26	use DBI;
	27	use Getopt::Long qw(GetOptions);
	28	Getopt::Long::config ('bundling');
	29
9630376c TH	30	use Encode qw/decode/;
	31	use Mail::Address;
	32
6d72dad2 TH	33	################################# Definitions ##################################
	34
	35	# define header names with separate database fields
	36	my %DBFields = ('date' => 'date',
	37	'references' => 'refs',
	38	'followup-to' => 'fupto',
	39	'from' => 'from_',
	40	'sender' => 'sender',
	41	'reply-to' => 'replyto',
	42	'subject' => 'subject',
	43	'organization' => 'organization',
	44	'lines' => 'linecount',
	45	'approved' => 'approved',
	46	'supersedes' => 'supersedes',
	47	'expires' => 'expires',
	48	'user-agent' => 'useragent',
	49	'x-newsreader' => 'xnewsreader',
	50	'x-mailer' => 'xmailer',
	51	'x-no-archive' => 'xnoarchive',
	52	'content-type' => 'contenttype',
	53	'content-transfer-encoding' => 'contentencoding',
	54	'cancel-lock' => 'cancellock',
	55	'injection-info' => 'injectioninfo',
	56	'x-trace' => 'xtrace',
	57	'nntp-posting-host' => 'postinghost');
	58
	59	# define field list for database
	60	my @DBFields = qw/day mid refs date path newsgroups fupto from_ from_parsed
	61	from_name from_address sender sender_parsed sender_name
	62	sender_address replyto replyto_parsed replyto_name
	63	replyto_address subject subject_parsed organization linecount
	64	approved supersedes expires useragent xnewsreader xmailer
	65	xnoarchive contenttype contentencoding cancellock injectioninfo
	66	xtrace postinghost headers disregard/;
	67
	68	################################# Main program #################################
	69
	70	### read commandline options
	71	my ($OptDay,$OptDebug,$OptParseDB,$OptRawDB,$OptTest,$OptConfFile);
	72	GetOptions ('d\|day=s' => \$OptDay,
	73	'debug!' => \$OptDebug,
	74	'parsedb=s' => \$OptParseDB,
	75	'rawdb=s' => \$OptRawDB,
	76	't\|test!' => \$OptTest,
	77	'conffile=s' => \$OptConfFile,
	78	'h\|help' => \&ShowPOD,
	79	'V\|version' => \&ShowVersion) or exit 1;
	80
	81	### read configuration
	82	my %Conf = %{ReadConfig($OptConfFile)};
	83
	84	### override configuration via commandline options
	85	my %ConfOverride;
	86	$ConfOverride{'DBTableRaw'} = $OptRawDB if $OptRawDB;
	87	$ConfOverride{'DBTableParse'} = $OptParseDB if $OptParseDB;
	88	&OverrideConfig(\%Conf,\%ConfOverride);
	89
	90	### get time period
	91	### and set $Period for output and expression for SQL 'WHERE' clause
	92	my ($Period,$SQLWherePeriod) = &GetTimePeriod($OptDay,'day');
	93	# bail out if --month is invalid or "all"
	94	&Bleat(2,"--day option has an invalid format - please use 'YYYY-MM-DD' or ".
	95	"'YYYY-MM-DD:YYYY-MM-DD'!") if (!$Period or $Period eq 'all time');
	96
97	### init database
98	my $DBHandle = InitDB(\%Conf,1);
99
100	### get & write data
101	&Bleat(1,'Test mode. Database is not updated.') if $OptTest;
102
103	# create $SQLWhereClause
104	my $SQLWhereClause = SQLBuildClause('where',$SQLWherePeriod,'NOT disregard');
105
106	# delete old data for current period
107	if (!$OptTest) {
108	print "----------- Deleting old data ... -----------\n" if $OptDebug;
109	my $DBQuery = $DBHandle->do(sprintf("DELETE FROM %s.%s %s",
110	$Conf{'DBDatabase'},$Conf{'DBTableParse'},
111	$SQLWhereClause))
112	or &Bleat(2,sprintf("Can't delete old parsed data for %s from %s.%s: ".
113	"$DBI::errstr\n",$Period,
114	$Conf{'DBDatabase'},$Conf{'DBTableParse'}));
115	};
116
117	# read from DBTableRaw
118	print "-------------- Reading data ... -------------\n" if $OptDebug;
119	my $DBQuery = $DBHandle->prepare(sprintf("SELECT id, day, mid, peer, path, ".
120	"newsgroups, headers, disregard ".
121	"FROM %s.%s %s", $Conf{'DBDatabase'},
122	$Conf{'DBTableRaw'}, $SQLWhereClause));
123	$DBQuery->execute()
124	or &Bleat(2,sprintf("Can't get data for %s from %s.%s: ".
125	"$DBI::errstr\n",$Period,
126	$Conf{'DBDatabase'},$Conf{'DBTableRaw'}));
127
128	# set output and database connection to UTF-8
129	# as we're going to write decoded header contents containing UTF-8 chars
130	binmode(STDOUT, ":utf8");
131	$DBHandle->do("SET NAMES 'utf8'");
132
133	# parse data in a loop and write it out
134	print "-------------- Parsing data ... -------------\n" if $OptDebug;
135	while (my $HeadersR = $DBQuery->fetchrow_hashref) {
136	my %Headers = %{$HeadersR};
137
138	# parse $Headers{'headers'} ('headers' from DBTableRaw)
139	# merge continuation lines
140	# from Perl Cookbook, 1st German ed. 1999, pg. 91
141	$Headers{'headers'} =~ s/\n\s+/ /g;
142	# split headers in single lines
143	my $OtherHeaders;
144	for (split(/\n/,$Headers{'headers'})) {
145	# split header lines in header name and header content
146	my ($key,$value) = split(/:/,$_,2);
147	$key =~ s/\s*//;
148	$value =~ s/^\s(.+)\s$/$1/;
149	# save each header, separate database fields in %Headers,
150	# the rest in $OtherHeaders (but not Message-ID, Path, Peer
151	# and Newsgroups as those do already exist)
152	if (defined($DBFields{lc($key)})) {
153	$Headers{$DBFields{lc($key)}} = $value;
154	} else {
155	$OtherHeaders .= sprintf("%s: %s\n",$key,$value)
156	if lc($key) !~ /^(message-id\|path\|peer\|newsgroups)$/;
157	}
158	}
159	# replace old (now parsed) $Headers{'headers'} with remanining $OtherHeaders
160	chomp($OtherHeaders);
161	$Headers{'headers'} = $OtherHeaders;
162
9630376c TH	163	foreach ('from_','sender', 'replyto', 'subject') {
	164	if ($Headers{$_}) {
	165	my $HeaderName = $_;
	166	$HeaderName =~ s/_$//;
	167	# decode From: / Sender: / Reply-To: / Subject:
	168	if ($Headers{$_} =~ /\?(B\|Q)\?/) {
	169	$Headers{$HeaderName.'_parsed'} = decode('MIME-Header',$Headers{$_});
	170	}
	171	# extract name(s) and mail(s) from From: / Sender: / Reply-To:
	172	# in parsed form, if available
	173	if ($_ ne 'subject') {
	174	my @Address;
	175	# start parser on header or parsed header
	176	# @Address will have an array of Mail::Address objects, one for
	177	# each name/mail (you can have more than one person in From:!)
	178	if (defined($Headers{$HeaderName.'_parsed'})) {
	179	@Address = Mail::Address->parse($Headers{$HeaderName.'_parsed'});
	180	} else {
	181	@Address = Mail::Address->parse($Headers{$_});
	182	}
aef5467b TH	183	# split each Mail::Address object to @Names and @Adresses
aef5467b TH	184	my (@Names,@Adresses);
9630376c	185	foreach (@Address) {
aef5467b TH	186	# take address part in @Addresses
aef5467b TH	187	push (@Adresses, $_->address());
9630376c TH	188	# take name part form "phrase", if there is one:
	189	# From: My Name <addr@ess> (Comment)
	190	# otherwise, take it from "comment":
	191	# From: addr@ess (Comment)
aef5467b TH	192	# and push it in @Names
	193	my ($Name);
	194	$Name = $_->comment() unless $Name = $_->phrase;
	195	$Name =~ s/^\((.+)\)$/$1/;
	196	push (@Names, $Name);
9630376c	197	}
aef5467b TH	198	# put all @Adresses and all @Names in %Headers as comma separated lists
	199	$Headers{$HeaderName.'_address'} = join(', ',@Adresses);
	200	$Headers{$HeaderName.'_name'} = join(', ',@Names);
9630376c TH	201	}
	202	}
	203	}
	204
6d72dad2 TH	205	# order output for database entry: fill @SQLBindVars
	206	print "-------------- Next entry:\n" if $OptDebug;
	207	my @SQLBindVars;
	208	foreach (@DBFields) {
	209	if (defined($Headers{$_}) and $Headers{$_} ne '') {
	210	push (@SQLBindVars,$Headers{$_});
	211	printf ("FOUND: %s -> %s\n",$_,$Headers{$_}) if $OptDebug;
	212	} else {
	213	push (@SQLBindVars,undef);
	214	}
	215	}
	216
	217	# write data to DBTableParse
	218	if (!$OptTest) {
	219	print "-------------- Writing data ... -------------\n" if $OptDebug;
	220	my $DBWrite =
	221	$DBHandle->prepare(sprintf("INSERT INTO %s.%s (%s) VALUES (%s)",
	222	$Conf{'DBDatabase'},
	223	$Conf{'DBTableParse'},
	224	# get field names from @DBFields
	225	join(', ',@DBFields),
	226	# create a list of '?' for each DBField
	227	join(', ',
	228	split(/ /,'? ' x scalar(@DBFields)))
	229	));
	230	$DBWrite->execute(@SQLBindVars)
	231	or &Bleat(2,sprintf("Can't write parsed data for %s to %s.%s: ".
	232	"$DBI::errstr\n",$Period,
	233	$Conf{'DBDatabase'},$Conf{'DBTableParse'}));
	234	$DBWrite->finish;
	235	}
	236	};
	237	$DBQuery->finish;
	238
	239	### close handles
	240	$DBHandle->disconnect;
	241
	242	print "------------------- DONE! -------------------\n" if $OptDebug;
	243	__END__
	244
	245	################################ Documentation #################################
	246
	247	=head1 NAME
	248
	249	parsedb - parse raw data and save it to a database
	250
	251	=head1 SYNOPSIS
	252
	253	B<parsedb> [B<-Vht>] [B<--day> I<YYYY-MM-DD> \| I<YYYY-MM-DD:YYYY-MM-DD>] [B<--rawdb> I<database table>] [B<--parsedb> I<database table>] [B<--conffile> I<filename>] [B<--debug>]
	254
	255	=head1 REQUIREMENTS
	256
	257	See L<doc/README>.
	258
	259	=head1 DESCRIPTION
	260
13e00610 TH	261	This script will parse raw, unstructured headers from a database table which is
	262	fed from F<feedlog.pl> for a given time period and write its results to
	263	nother database table with separate fields (columns) for most (or even all)
	264	relevant headers.
	265
	266	I<Subject:>, I<From:>, I<Sender:> and I<Reply-To:> will be parsed from MIME
	267	encoded words to UTF-8 as needed while the unparsed copy is kept. From that
	268	parsed copy, I<From:>, I<Sender:> and I<Reply-To:> will also be split into
	269	separate name(s) and address(es) fields while the un-splitted copy is kept,
	270	too.
	271
	272	B<parsedb> should be run nightly from cron for yesterdays data so all
	273	other scripts get current information. The time period to act on defaults to
	274	yesterday, accordingly; you can assign another time period or a single day via
	275	the B<--day> option (see below).
6d72dad2 TH	276
	277	=head2 Configuration
	278
13e00610 TH	279	B<parsedb> will read its configuration from F<newsstats.conf>
	280	should be present in etc/ via Config::Auto or from a configuration file
	281	submitted by the B<--conffile> option.
	282
	283	See L<doc/INSTALL> for an overview of possible configuration options.
	284
	285	You can override configuration options via the B<--rawdb> and
	286	B<--parsedb> options, respectively.
6d72dad2 TH	287
	288	=head1 OPTIONS
	289
	290	=over 3
	291
	292	=item B<-V>, B<--version>
	293
	294	Print out version and copyright information and exit.
	295
	296	=item B<-h>, B<--help>
	297
	298	Print this man page and exit.
	299
	300	=item B<--debug>
	301
	302	Output (rather much) debugging information to STDOUT while processing.
	303
	304	=item B<-t>, B<--test>
	305
	306	Do not write results to database. You should use B<--debug> in
	307	conjunction with B<--test> ... everything else seems a bit pointless.
	308
	309	=item B<-d>, B<--day> I<YYYY-MM-DD[:YYYY-MM-DD]>
	310
	311	Set processing period to a single day in YYYY-MM-DD format or to a time
	312	period between two days in YYYY-MM-DD:YYYY-MM-DD format (two days, separated
	313	by a colon).
	314
	315	Defaults to yesterday.
	316
	317	=item B<--rawdb> I<table> (raw data table)
	318
	319	Override I<DBTableRaw> from F<newsstats.conf>.
	320
	321	=item B<--parsedb> I<table> (parsed data table)
	322
	323	Override I<DBTableParse> from F<newsstats.conf>.
	324
	325	=item B<--conffile> I<filename>
	326
	327	Load configuration from I<filename> instead of F<newsstats.conf>.
	328
	329	=back
	330
	331	=head1 INSTALLATION
	332
	333	See L<doc/INSTALL>.
	334
	335	=head1 EXAMPLES
	336
13e00610 TH	337	An example crontab entry:
	338
	339	0 1 * * * /path/to/bin/parsedb.pl
	340
	341	Do a dry run for yesterday's data, showing results of processing:
	342
	343	parsedb --debug --test \| less
6d72dad2 TH	344
	345	=head1 FILES
	346
	347	=over 4
	348
	349	=item F<bin/parsedb.pl>
	350
	351	The script itself.
	352
	353	=item F<lib/NewsStats.pm>
	354
	355	Library functions for the NewsStats package.
	356
	357	=item F<etc/newsstats.conf>
	358
	359	Runtime configuration file.
	360
	361	=back
	362
	363	=head1 BUGS
	364
	365	Please report any bugs or feature requests to the author or use the
	366	bug tracker at L<http://bugs.th-h.de/>!
	367
	368	=head1 SEE ALSO
	369
	370	=over 2
	371
	372	=item -
	373
	374	L<doc/README>
	375
	376	=item -
	377
	378	L<doc/INSTALL>
	379
	380	=back
	381
	382	This script is part of the B<NewsStats> package.
	383
	384	=head1 AUTHOR
	385
	386	Thomas Hochstein <thh@inter.net>
	387
	388	=head1 COPYRIGHT AND LICENSE
	389
	390	Copyright (c) 2013 Thomas Hochstein <thh@inter.net>
	391
	392	This program is free software; you may redistribute it and/or modify it
	393	under the same terms as Perl itself.
	394
	395	=cut