Initial checkin of working branch.

[usenet/newsstats.git] / feedlog.pl

diff --git a/feedlog.pl b/feedlog.pl
new file mode 100755 (executable)
index 0000000..a68b833
--- /dev/null
+++ b/feedlog.pl
@@ -0,0 +1,87 @@
+#! /usr/bin/perl -W\r
+#\r
+# feedlog.pl\r
+#\r
+# This script will log headers and other data to a database\r
+# for further analysis by parsing a feed from INN.\r
+# \r
+# It is part of the NewsStats package.\r
+#\r
+# Copyright (c) 2010 Thomas Hochstein <thh@inter.net>\r
+#\r
+# It can be redistributed and/or modified under the same terms under \r
+# which Perl itself is published.\r
+\r
+BEGIN {\r
+  our $VERSION = "0.01";\r
+  use File::Basename;\r
+  push(@INC, dirname($0));\r
+}\r
+use strict;\r
+\r
+use NewsStats;\r
+\r
+use Sys::Syslog qw(:standard :macros);\r
+\r
+use Date::Format;\r
+use DBI;\r
+\r
+################################# Main program #################################\r
+\r
+### read commandline options\r
+my %Options = &ReadOptions('qd');\r
+\r
+### read configuration\r
+my %Conf = %{ReadConfig('newsstats.conf')};\r
+\r
+### init syslog\r
+openlog($MySelf, 'nofatal,pid', LOG_NEWS);\r
+syslog(LOG_NOTICE, "$MyVersion starting up.") if !$Options{'q'};\r
+\r
+### init database\r
+my $DBHandle = InitDB(\%Conf,0);\r
+if (!$DBHandle) {\r
+  syslog(LOG_CRIT, 'Database connection failed: %s', $DBI::errstr);\r
+  while (1) {}; # go into endless loop to suppress further errors and respawning\r
+};\r
+my $DBQuery = $DBHandle->prepare(sprintf("INSERT INTO %s.%s (day,date,mid,timestamp,token,size,peer,path,newsgroups,headers) VALUES (?,?,?,?,?,?,?,?,?,?)",$Conf{'DBDatabase'},$Conf{'DBTableRaw'}));\r
+\r
+### main loop\r
+while (<>) {\r
+  chomp;\r
+  # catch empty lines trailing or leading\r
+  if ($_ eq '') {\r
+    next;\r
+  }\r
+  # first line contains: mid, timestamp, token, size, peer, Path, Newsgroups\r
+  my ($Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups) = split;\r
+  # remaining lines contain headers\r
+  my $Headers = "";\r
+  while (<>) {\r
+    chomp;\r
+    # empty line terminates this article\r
+    if ($_ eq '') {\r
+      last;\r
+    }\r
+    # collect headers\r
+    $Headers .= $_."\n" ;\r
+  }\r
+\r
+  # parse timestamp to day (YYYY-MM-DD) and to MySQL timestamp\r
+  my $Day  = time2str("%Y-%m-%d", $Timestamp);\r
+  my $Date = time2str("%Y-%m-%d %H:%M:%S", $Timestamp);\r
+\r
+  # write to database\r
+  if (!$DBQuery->execute($Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups, $Headers)) {\r
+    syslog(LOG_ERR, 'Database error: %s', $DBI::errstr);\r
+  };\r
+  $DBQuery->finish;\r
+  \r
+  warn sprintf("-----\nDay: %s\nDate: %s\nMID: %s\nTS: %s\nToken: %s\nSize: %s\nPeer: %s\nPath: %s\nNewsgroups: %s\nHeaders: %s\n",$Day, $Date, $Mid, $Timestamp, $Token, $Size, $Peer, $Path, $Newsgroups, $Headers) if !$Options{'d'};\r
+}\r
+\r
+### close handles\r
+$DBHandle->disconnect;\r
+syslog(LOG_NOTICE, "$MySelf closing down.") if !$Options{'q'};\r
+closelog();\r
+\r