summaryrefslogtreecommitdiffstats
path: root/contrib
diff options
context:
space:
mode:
Diffstat (limited to 'contrib')
-rw-r--r--contrib/Makefile.in5
-rw-r--r--contrib/README39
-rw-r--r--contrib/create_tables.sql51
-rw-r--r--contrib/make_combined_log.pl139
-rw-r--r--contrib/mysql_import_combined_log.pl61
5 files changed, 31 insertions, 264 deletions
diff --git a/contrib/Makefile.in b/contrib/Makefile.in
index b384fd0..c40a711 100644
--- a/contrib/Makefile.in
+++ b/contrib/Makefile.in
@@ -3,10 +3,7 @@
3# Modify these top variables. 3# Modify these top variables.
4SUBDIRS = 4SUBDIRS =
5 5
6EXTRA_DIST = README \ 6EXTRA_DIST = README mysql_import_combined_log.pl
7 create_tables.sql \
8 make_combined_log.pl \
9 mysql_import_combined_log.pl
10 7
11#Don't modify anything below here 8#Don't modify anything below here
12 9
diff --git a/contrib/README b/contrib/README
index bf0e65f..7dac946 100644
--- a/contrib/README
+++ b/contrib/README
@@ -1,38 +1 @@
1This directory contains contributed scripts/programs/utilites for mod_log_sql. This directory contains contributed scripts/programs/utilites related to mod_log_sql.
2
3* create_tables.sql
4
5This is the create table SQL commands to create the access, headers_in,
6headers_out, cookies, and notes tables in the MySQL database.
7Use it like this.
8mysql -u user -h host -p apachelogdatabase < create_tables.sql
9Where:
10 user is the username to log in as,
11 host is the hostname the server is on,
12 apachelogdatabase is the database to put the tables into
13 -p will have mysql ask you for a password for the user
14
15*make_combined_log.pl
16
17This perl script will extract the data from mod_log_sql's tables in the
18database and export a standard Apache combined log file. Use this to run
19logs through a program like webalizer.
20
21You must edit the perl script to configure variables before you run it.
22Usage:
23./make_combined_log.pl days virtualhost
24Where:
25 days is the number of days to fetch (starting from now and going back
26 in time)
27 virtualhost is the name of the virtualhost to retrieve
28
29Example:
30 ./make_combined_log.pl 2 example.com
31
32*mysql_import_combined_log.pl
33
34This is a perl script written by Aaron Jenson that imports a combined log file
35from apache into a SQL database table.. You can use this script to import logs
36from a webserver you are converting over from the standard Apache log system to
37mod_log_sql. A Usage statement can be fetch by running the program with no
38parameters or with --help or -?.
diff --git a/contrib/create_tables.sql b/contrib/create_tables.sql
deleted file mode 100644
index 0fe0f4b..0000000
--- a/contrib/create_tables.sql
+++ /dev/null
@@ -1,51 +0,0 @@
1create table access_log (
2 id char(19) ,
3 agent varchar(255) ,
4 bytes_sent int unsigned ,
5 child_pid smallint unsigned,
6 cookie varchar(255),
7 machine_id varchar(25),
8 request_file varchar(255),
9 referer varchar(255) ,
10 remote_host varchar(50) ,
11 remote_logname varchar(50) ,
12 remote_user varchar(50) ,
13 request_duration smallint unsigned ,
14 request_line varchar(255),
15 request_method varchar(10) ,
16 request_protocol varchar(10) ,
17 request_time char(28),
18 request_uri varchar(255),
19 request_args varchar(255),
20 server_port smallint unsigned,
21 ssl_cipher varchar(25),
22 ssl_keysize smallint unsigned,
23 ssl_maxkeysize smallint unsigned,
24 status smallint unsigned ,
25 time_stamp int unsigned ,
26 virtual_host varchar(255)
27);
28
29create table notes (
30 id char(19),
31 item varchar(80),
32 val varchar(80)
33);
34
35create table headers_in (
36 id char(19),
37 item varchar(80),
38 val varchar(80)
39);
40
41create table headers_out (
42 id char(19),
43 item varchar(80),
44 val varchar(80)
45);
46
47create table cookies (
48 id char(19),
49 item varchar(80),
50 val varchar(80)
51);
diff --git a/contrib/make_combined_log.pl b/contrib/make_combined_log.pl
deleted file mode 100644
index 2c55271..0000000
--- a/contrib/make_combined_log.pl
+++ /dev/null
@@ -1,139 +0,0 @@
1#!/usr/bin/perl
2
3# $Id: make_combined_log.pl,v 1.2 2004/02/12 23:32:55 urkle Exp $
4#
5# make_combined_log.pl
6#
7# Usage: make_combined_log <days> <virtual host>
8#
9# This perl script extracts the httpd access data from a MySQL database
10# and formats it properly for parsing by 3rd-party log analysis tools.
11#
12# The script is intended to be run out by cron. Its commandline arguments tell
13# it how many days' worth of access records to extract, and which virtual_host
14# you are interested in (because many people log several virthosts to one MySQL
15# db.) This permits you to run it daily, weekly, every 9 days -- whatever you
16# decide.
17#
18# Note: By "days" I mean "chunks of 24 hours prior to the moment this script is
19# run." So if you run it at 4:34 p.m. on the 12th, it will go back through 4:34
20# p.m. on the 11th.
21#
22# Known issues:
23# * Because GET and POST are not discriminated in the MySQL log, we'll just
24# assume that all requests are GETs. This should have negligible effect
25# on any analysis software. This could be remedied IF you stored the full
26# HTTP request in your database instead of just the URI, but that's going to
27# cost you a LOT of space really quickly...
28#
29# * Because this is somewhat of a quick hack it doesn't do the most robust
30# error checking in the world. Run it by hand to confirm your usage before
31# putting it in crontab.
32
33$| = 1;
34
35use DBI;
36
37# Remove the # in front of this line when you have
38# edited the variables below.
39#$has_edited_source = 1;
40#
41# Set up the proper variables to permit database access
42#
43$serverName = "your.dbhost.com";
44$serverPort = "3306";
45$serverUser = "someuser";
46$serverPass = "somepass";
47$serverTbl = "acc_log_tbl";
48$serverDb = "apache";
49
50if (!defined($has_edited_source)) {
51 print "Please edit this file and configure it first.\n";
52 print "This program is $0\n";
53 exit 1;
54}
55# Remember, $#ARGV is parameters minus one...
56if ($#ARGV != 1) {
57 print "Usage $0 days virtualhost\n";
58 exit 1;
59}
60
61$days = $ARGV[0];
62$virthost = $ARGV[1];
63
64#
65# Other constants
66#
67$st_tz = "-0800";
68$dt_tz = "-0700";
69
70$now = time();
71$start = $now - (86400 * $days);
72
73#
74# Connect and fetch the records
75#
76$dbh = DBI->connect("DBI:mysql:database=$serverDb;host=$serverName;port=$serverPort",$serverUser,$serverPass);
77if (not $dbh) {
78 die "Unable to connect to the database. Please check your connection variables. (Bad password? Incorrect perms?)";
79}
80
81$records = $dbh->prepare("select remote_host,remote_user,request_uri,time_stamp,status,bytes_sent,referer,agent,request_method,request_protocol from `$serverTbl` where virtual_host='$virthost' and time_stamp >= $start order by time_stamp");
82$records->execute;
83if (not $records) {
84 die "No such table or the select returned no records."
85}
86
87#Right
88#ariston.netcraft.com - - [14/Nov/2001:05:13:39 -0800] "GET / HTTP/1.0" 200 502 "-" "Mozilla/4.08 [en] (Win98; I)"
89#ariston.netcraft.com - - [14/Nov/2001:05:13:39 -0800] "GET / HTTP/1.0" 200 502 "-" "Mozilla/4.08 [en] (Win98; I)"
90
91#Bad
92#ariston.netcraft.com - - [2001-11-14 05:13:39 -0800] "GET / HTTP/1.1" 200 502 "-" "Mozilla/4.08 [en] (Win98; I)"
93#ariston.netcraft.com - - [2001-11-14 05:13:39 -0800] "GET / HTTP/1.1" 200 502 "-" "Mozilla/4.08 [en] (Win98; I)"
94
95
96#
97# Pull out the data row by row and format it
98#
99while (@data = $records->fetchrow_array) {
100 ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($data[3]);
101 $year=$year+1900;
102
103 # Create format for leading-zero formatting
104 if ($mday < 10) { $mday = "0$mday"; }
105 if ($mon < 10) { $mon = "0$mon"; }
106 if ($hour < 10) { $hour = "0$hour"; }
107 if ($min < 10) { $min = "0$min"; }
108 if ($sec < 10) { $sec = "0$sec"; }
109
110 # Convert numeric month to string month
111 for ($mon) {
112 if (/00/) { $mon = "Jan";}
113 elsif (/01/) { $mon = "Feb";}
114 elsif (/02/) { $mon = "Mar";}
115 elsif (/03/) { $mon = "Apr";}
116 elsif (/04/) { $mon = "May";}
117 elsif (/05/) { $mon = "Jun";}
118 elsif (/06/) { $mon = "Jul";}
119 elsif (/07/) { $mon = "Aug";}
120 elsif (/08/) { $mon = "Sep";}
121 elsif (/09/) { $mon = "Oct";}
122 elsif (/10/) { $mon = "Nov";}
123 elsif (/11/) { $mon = "Dec";}
124 }
125
126 # Create the output
127 print "$data[0] $data[1] - [$mday/$mon/$year:$hour:$min:$sec ";
128 if ($isdst) {
129 print "$dt_tz\] ";
130 } else {
131 print "$st_tz\] ";
132 }
133 print "\"$data[8] $data[2] $data[9]\" $data[4] $data[5] \"$data[6]\" \"$data[7]\"\n";
134}
135
136#
137# Done
138#
139$records->finish;
diff --git a/contrib/mysql_import_combined_log.pl b/contrib/mysql_import_combined_log.pl
index bffec20..1c801f2 100644
--- a/contrib/mysql_import_combined_log.pl
+++ b/contrib/mysql_import_combined_log.pl
@@ -1,8 +1,4 @@
1#!/usr/bin/perl -w 1#!/usr/bin/perl -w
2# $Id: mysql_import_combined_log.pl,v 1.4 2004/02/21 18:09:50 urkle Exp $
3# Written by Aaron Jenson.
4# Original source: http://www.visualprose.com/software.php
5# Updated to work under Perl 5.6.1 by Edward Rudd
6use strict; 2use strict;
7use Getopt::Long qw(:config bundling); 3use Getopt::Long qw(:config bundling);
8use DBI; 4use DBI;
@@ -37,44 +33,47 @@ my @cols = (
37); 33);
38my $col = ''; 34my $col = '';
39 35
36%options = (
37 "version" => sub { VERSION_MESSAGE(); exit 0; },
38 "help|?" => sub { HELP_MESSAGE(); exit 0; },
39 );
40
40GetOptions (\%options, 41GetOptions (\%options,
41 "version" => sub { VERSION_MESSAGE(); exit 0; }, 42 "h|host=s",
42 "help|?" => sub { HELP_MESSAGE(); exit 0; }, 43 "d|database=s",
43 "host|h=s", 44 "t|table=s",
44 "database|d=s", 45 "u|username=s",
45 "table|t=s", 46 "p|password=s",
46 "username|u=s", 47 "f|logfile=s");
47 "password|p=s", 48
48 "logfile|f=s"); 49$options{h} ||= 'localhost';
49 50$options{d} ||= '';
50$options{host} ||= 'localhost'; 51$options{u} ||= '';
51$options{database} ||= ''; 52$options{p} ||= '';
52$options{username} ||= ''; 53$options{f} ||= '';
53$options{password} ||= ''; 54
54$options{logfile} ||= ''; 55if( ! $options{d} )
55
56if( ! $options{database} )
57{ 56{
58 HELP_MESSAGE(); 57 HELP_MESSAGE();
59 print "Must supply a database to connect to.\n"; 58 print "Must supply a database to connect to.\n";
60 exit 1; 59 exit 1;
61} 60}
62 61
63if( ! $options{table} ) 62if( ! $options{t} )
64{ 63{
65 HELP_MESSAGE(); 64 HELP_MESSAGE();
66 print "Must supply table name.\n"; 65 print "Must supply table name.\n";
67 exit 1; 66 exit 1;
68} 67}
69 68
70if( $options{logfile} ) 69if( $options{f} )
71{ 70{
72 if( ! -e $options{logfile} ) 71 if( ! -e $options{f} )
73 { 72 {
74 print "File '$options{logfile}' doesn't exist.\n"; 73 print "File '$options{f}' doesn't exist.\n";
75 exit 1; 74 exit 1;
76 } 75 }
77 open(STDIN, "<$options{logfile}") || die "Can't open $options{logfile} for reading."; 76 open(STDIN, "<$options{f}") || die "Can't open $options{f} for reading.";
78} 77}
79 78
80$dbh = Connect(); 79$dbh = Connect();
@@ -82,7 +81,7 @@ if (! $dbh) {
82 exit 1; 81 exit 1;
83} 82}
84 83
85$sql = "INSERT INTO $options{table} ("; 84$sql = "INSERT INTO $options{t} (";
86foreach $col (@cols) 85foreach $col (@cols)
87{ 86{
88 $sql .= "$col," if( $col ); 87 $sql .= "$col," if( $col );
@@ -115,15 +114,15 @@ while($line = <STDIN>)
115} 114}
116print "Parsed $linecount Log lines\n"; 115print "Parsed $linecount Log lines\n";
117print "Inserted $insertcount records\n"; 116print "Inserted $insertcount records\n";
118print "to table '$options{table}' in database '$options{database}' on '$options{host}'\n"; 117print "to table '$options{t}' in database '$options{d}' on '$options{h}'\n";
119 118
120# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 119# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
121# Connects to a MySQL database and returns the connection. 120# Connects to a MySQL database and returns the connection.
122# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 121# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
123sub Connect 122sub Connect
124{ 123{
125 my $dsn = "DBI:mysql:$options{database};hostname=$options{host}"; 124 my $dsn = "DBI:mysql:$options{d};hostname=$options{h}";
126 return DBI->connect( $dsn, $options{username}, $options{password} ); 125 return DBI->connect( $dsn, $options{u}, $options{p} );
127} 126}
128 127
129 128
@@ -216,9 +215,7 @@ EOF
216# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 215# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
217sub VERSION_MESSAGE 216sub VERSION_MESSAGE
218{ 217{
219 print "mysql_import_combined_log.pl version 1.2\n"; 218 print "mysql_import_combined_log.pl version 1.1\n";
220 print "Version 1.0 Written by Aaron Jenson.\n";
221 print "Update to work with perl 5.6.1 by Edward Rudd\n";
222} 219}
223 220
2241; 2211;