diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/Makefile.in | 5 | ||||
| -rw-r--r-- | contrib/README | 39 | ||||
| -rw-r--r-- | contrib/create_tables.sql | 51 | ||||
| -rw-r--r-- | contrib/make_combined_log.pl | 139 | ||||
| -rw-r--r-- | contrib/mysql_import_combined_log.pl | 61 |
5 files changed, 31 insertions, 264 deletions
diff --git a/contrib/Makefile.in b/contrib/Makefile.in index b384fd0..c40a711 100644 --- a/contrib/Makefile.in +++ b/contrib/Makefile.in | |||
| @@ -3,10 +3,7 @@ | |||
| 3 | # Modify these top variables. | 3 | # Modify these top variables. |
| 4 | SUBDIRS = | 4 | SUBDIRS = |
| 5 | 5 | ||
| 6 | EXTRA_DIST = README \ | 6 | EXTRA_DIST = README mysql_import_combined_log.pl |
| 7 | create_tables.sql \ | ||
| 8 | make_combined_log.pl \ | ||
| 9 | mysql_import_combined_log.pl | ||
| 10 | 7 | ||
| 11 | #Don't modify anything below here | 8 | #Don't modify anything below here |
| 12 | 9 | ||
diff --git a/contrib/README b/contrib/README index bf0e65f..7dac946 100644 --- a/contrib/README +++ b/contrib/README | |||
| @@ -1,38 +1 @@ | |||
| 1 | This directory contains contributed scripts/programs/utilites for mod_log_sql. | This directory contains contributed scripts/programs/utilites related to mod_log_sql. | |
| 2 | |||
| 3 | * create_tables.sql | ||
| 4 | |||
| 5 | This is the create table SQL commands to create the access, headers_in, | ||
| 6 | headers_out, cookies, and notes tables in the MySQL database. | ||
| 7 | Use it like this. | ||
| 8 | mysql -u user -h host -p apachelogdatabase < create_tables.sql | ||
| 9 | Where: | ||
| 10 | user is the username to log in as, | ||
| 11 | host is the hostname the server is on, | ||
| 12 | apachelogdatabase is the database to put the tables into | ||
| 13 | -p will have mysql ask you for a password for the user | ||
| 14 | |||
| 15 | *make_combined_log.pl | ||
| 16 | |||
| 17 | This perl script will extract the data from mod_log_sql's tables in the | ||
| 18 | database and export a standard Apache combined log file. Use this to run | ||
| 19 | logs through a program like webalizer. | ||
| 20 | |||
| 21 | You must edit the perl script to configure variables before you run it. | ||
| 22 | Usage: | ||
| 23 | ./make_combined_log.pl days virtualhost | ||
| 24 | Where: | ||
| 25 | days is the number of days to fetch (starting from now and going back | ||
| 26 | in time) | ||
| 27 | virtualhost is the name of the virtualhost to retrieve | ||
| 28 | |||
| 29 | Example: | ||
| 30 | ./make_combined_log.pl 2 example.com | ||
| 31 | |||
| 32 | *mysql_import_combined_log.pl | ||
| 33 | |||
| 34 | This is a perl script written by Aaron Jenson that imports a combined log file | ||
| 35 | from apache into a SQL database table.. You can use this script to import logs | ||
| 36 | from a webserver you are converting over from the standard Apache log system to | ||
| 37 | mod_log_sql. A Usage statement can be fetch by running the program with no | ||
| 38 | parameters or with --help or -?. | ||
diff --git a/contrib/create_tables.sql b/contrib/create_tables.sql deleted file mode 100644 index 0fe0f4b..0000000 --- a/contrib/create_tables.sql +++ /dev/null | |||
| @@ -1,51 +0,0 @@ | |||
| 1 | create table access_log ( | ||
| 2 | id char(19) , | ||
| 3 | agent varchar(255) , | ||
| 4 | bytes_sent int unsigned , | ||
| 5 | child_pid smallint unsigned, | ||
| 6 | cookie varchar(255), | ||
| 7 | machine_id varchar(25), | ||
| 8 | request_file varchar(255), | ||
| 9 | referer varchar(255) , | ||
| 10 | remote_host varchar(50) , | ||
| 11 | remote_logname varchar(50) , | ||
| 12 | remote_user varchar(50) , | ||
| 13 | request_duration smallint unsigned , | ||
| 14 | request_line varchar(255), | ||
| 15 | request_method varchar(10) , | ||
| 16 | request_protocol varchar(10) , | ||
| 17 | request_time char(28), | ||
| 18 | request_uri varchar(255), | ||
| 19 | request_args varchar(255), | ||
| 20 | server_port smallint unsigned, | ||
| 21 | ssl_cipher varchar(25), | ||
| 22 | ssl_keysize smallint unsigned, | ||
| 23 | ssl_maxkeysize smallint unsigned, | ||
| 24 | status smallint unsigned , | ||
| 25 | time_stamp int unsigned , | ||
| 26 | virtual_host varchar(255) | ||
| 27 | ); | ||
| 28 | |||
| 29 | create table notes ( | ||
| 30 | id char(19), | ||
| 31 | item varchar(80), | ||
| 32 | val varchar(80) | ||
| 33 | ); | ||
| 34 | |||
| 35 | create table headers_in ( | ||
| 36 | id char(19), | ||
| 37 | item varchar(80), | ||
| 38 | val varchar(80) | ||
| 39 | ); | ||
| 40 | |||
| 41 | create table headers_out ( | ||
| 42 | id char(19), | ||
| 43 | item varchar(80), | ||
| 44 | val varchar(80) | ||
| 45 | ); | ||
| 46 | |||
| 47 | create table cookies ( | ||
| 48 | id char(19), | ||
| 49 | item varchar(80), | ||
| 50 | val varchar(80) | ||
| 51 | ); | ||
diff --git a/contrib/make_combined_log.pl b/contrib/make_combined_log.pl deleted file mode 100644 index 2c55271..0000000 --- a/contrib/make_combined_log.pl +++ /dev/null | |||
| @@ -1,139 +0,0 @@ | |||
| 1 | #!/usr/bin/perl | ||
| 2 | |||
| 3 | # $Id: make_combined_log.pl,v 1.2 2004/02/12 23:32:55 urkle Exp $ | ||
| 4 | # | ||
| 5 | # make_combined_log.pl | ||
| 6 | # | ||
| 7 | # Usage: make_combined_log <days> <virtual host> | ||
| 8 | # | ||
| 9 | # This perl script extracts the httpd access data from a MySQL database | ||
| 10 | # and formats it properly for parsing by 3rd-party log analysis tools. | ||
| 11 | # | ||
| 12 | # The script is intended to be run out by cron. Its commandline arguments tell | ||
| 13 | # it how many days' worth of access records to extract, and which virtual_host | ||
| 14 | # you are interested in (because many people log several virthosts to one MySQL | ||
| 15 | # db.) This permits you to run it daily, weekly, every 9 days -- whatever you | ||
| 16 | # decide. | ||
| 17 | # | ||
| 18 | # Note: By "days" I mean "chunks of 24 hours prior to the moment this script is | ||
| 19 | # run." So if you run it at 4:34 p.m. on the 12th, it will go back through 4:34 | ||
| 20 | # p.m. on the 11th. | ||
| 21 | # | ||
| 22 | # Known issues: | ||
| 23 | # * Because GET and POST are not discriminated in the MySQL log, we'll just | ||
| 24 | # assume that all requests are GETs. This should have negligible effect | ||
| 25 | # on any analysis software. This could be remedied IF you stored the full | ||
| 26 | # HTTP request in your database instead of just the URI, but that's going to | ||
| 27 | # cost you a LOT of space really quickly... | ||
| 28 | # | ||
| 29 | # * Because this is somewhat of a quick hack it doesn't do the most robust | ||
| 30 | # error checking in the world. Run it by hand to confirm your usage before | ||
| 31 | # putting it in crontab. | ||
| 32 | |||
| 33 | $| = 1; | ||
| 34 | |||
| 35 | use DBI; | ||
| 36 | |||
| 37 | # Remove the # in front of this line when you have | ||
| 38 | # edited the variables below. | ||
| 39 | #$has_edited_source = 1; | ||
| 40 | # | ||
| 41 | # Set up the proper variables to permit database access | ||
| 42 | # | ||
| 43 | $serverName = "your.dbhost.com"; | ||
| 44 | $serverPort = "3306"; | ||
| 45 | $serverUser = "someuser"; | ||
| 46 | $serverPass = "somepass"; | ||
| 47 | $serverTbl = "acc_log_tbl"; | ||
| 48 | $serverDb = "apache"; | ||
| 49 | |||
| 50 | if (!defined($has_edited_source)) { | ||
| 51 | print "Please edit this file and configure it first.\n"; | ||
| 52 | print "This program is $0\n"; | ||
| 53 | exit 1; | ||
| 54 | } | ||
| 55 | # Remember, $#ARGV is parameters minus one... | ||
| 56 | if ($#ARGV != 1) { | ||
| 57 | print "Usage $0 days virtualhost\n"; | ||
| 58 | exit 1; | ||
| 59 | } | ||
| 60 | |||
| 61 | $days = $ARGV[0]; | ||
| 62 | $virthost = $ARGV[1]; | ||
| 63 | |||
| 64 | # | ||
| 65 | # Other constants | ||
| 66 | # | ||
| 67 | $st_tz = "-0800"; | ||
| 68 | $dt_tz = "-0700"; | ||
| 69 | |||
| 70 | $now = time(); | ||
| 71 | $start = $now - (86400 * $days); | ||
| 72 | |||
| 73 | # | ||
| 74 | # Connect and fetch the records | ||
| 75 | # | ||
| 76 | $dbh = DBI->connect("DBI:mysql:database=$serverDb;host=$serverName;port=$serverPort",$serverUser,$serverPass); | ||
| 77 | if (not $dbh) { | ||
| 78 | die "Unable to connect to the database. Please check your connection variables. (Bad password? Incorrect perms?)"; | ||
| 79 | } | ||
| 80 | |||
| 81 | $records = $dbh->prepare("select remote_host,remote_user,request_uri,time_stamp,status,bytes_sent,referer,agent,request_method,request_protocol from `$serverTbl` where virtual_host='$virthost' and time_stamp >= $start order by time_stamp"); | ||
| 82 | $records->execute; | ||
| 83 | if (not $records) { | ||
| 84 | die "No such table or the select returned no records." | ||
| 85 | } | ||
| 86 | |||
| 87 | #Right | ||
| 88 | #ariston.netcraft.com - - [14/Nov/2001:05:13:39 -0800] "GET / HTTP/1.0" 200 502 "-" "Mozilla/4.08 [en] (Win98; I)" | ||
| 89 | #ariston.netcraft.com - - [14/Nov/2001:05:13:39 -0800] "GET / HTTP/1.0" 200 502 "-" "Mozilla/4.08 [en] (Win98; I)" | ||
| 90 | |||
| 91 | #Bad | ||
| 92 | #ariston.netcraft.com - - [2001-11-14 05:13:39 -0800] "GET / HTTP/1.1" 200 502 "-" "Mozilla/4.08 [en] (Win98; I)" | ||
| 93 | #ariston.netcraft.com - - [2001-11-14 05:13:39 -0800] "GET / HTTP/1.1" 200 502 "-" "Mozilla/4.08 [en] (Win98; I)" | ||
| 94 | |||
| 95 | |||
| 96 | # | ||
| 97 | # Pull out the data row by row and format it | ||
| 98 | # | ||
| 99 | while (@data = $records->fetchrow_array) { | ||
| 100 | ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($data[3]); | ||
| 101 | $year=$year+1900; | ||
| 102 | |||
| 103 | # Create format for leading-zero formatting | ||
| 104 | if ($mday < 10) { $mday = "0$mday"; } | ||
| 105 | if ($mon < 10) { $mon = "0$mon"; } | ||
| 106 | if ($hour < 10) { $hour = "0$hour"; } | ||
| 107 | if ($min < 10) { $min = "0$min"; } | ||
| 108 | if ($sec < 10) { $sec = "0$sec"; } | ||
| 109 | |||
| 110 | # Convert numeric month to string month | ||
| 111 | for ($mon) { | ||
| 112 | if (/00/) { $mon = "Jan";} | ||
| 113 | elsif (/01/) { $mon = "Feb";} | ||
| 114 | elsif (/02/) { $mon = "Mar";} | ||
| 115 | elsif (/03/) { $mon = "Apr";} | ||
| 116 | elsif (/04/) { $mon = "May";} | ||
| 117 | elsif (/05/) { $mon = "Jun";} | ||
| 118 | elsif (/06/) { $mon = "Jul";} | ||
| 119 | elsif (/07/) { $mon = "Aug";} | ||
| 120 | elsif (/08/) { $mon = "Sep";} | ||
| 121 | elsif (/09/) { $mon = "Oct";} | ||
| 122 | elsif (/10/) { $mon = "Nov";} | ||
| 123 | elsif (/11/) { $mon = "Dec";} | ||
| 124 | } | ||
| 125 | |||
| 126 | # Create the output | ||
| 127 | print "$data[0] $data[1] - [$mday/$mon/$year:$hour:$min:$sec "; | ||
| 128 | if ($isdst) { | ||
| 129 | print "$dt_tz\] "; | ||
| 130 | } else { | ||
| 131 | print "$st_tz\] "; | ||
| 132 | } | ||
| 133 | print "\"$data[8] $data[2] $data[9]\" $data[4] $data[5] \"$data[6]\" \"$data[7]\"\n"; | ||
| 134 | } | ||
| 135 | |||
| 136 | # | ||
| 137 | # Done | ||
| 138 | # | ||
| 139 | $records->finish; | ||
diff --git a/contrib/mysql_import_combined_log.pl b/contrib/mysql_import_combined_log.pl index bffec20..1c801f2 100644 --- a/contrib/mysql_import_combined_log.pl +++ b/contrib/mysql_import_combined_log.pl | |||
| @@ -1,8 +1,4 @@ | |||
| 1 | #!/usr/bin/perl -w | 1 | #!/usr/bin/perl -w |
| 2 | # $Id: mysql_import_combined_log.pl,v 1.4 2004/02/21 18:09:50 urkle Exp $ | ||
| 3 | # Written by Aaron Jenson. | ||
| 4 | # Original source: http://www.visualprose.com/software.php | ||
| 5 | # Updated to work under Perl 5.6.1 by Edward Rudd | ||
| 6 | use strict; | 2 | use strict; |
| 7 | use Getopt::Long qw(:config bundling); | 3 | use Getopt::Long qw(:config bundling); |
| 8 | use DBI; | 4 | use DBI; |
| @@ -37,44 +33,47 @@ my @cols = ( | |||
| 37 | ); | 33 | ); |
| 38 | my $col = ''; | 34 | my $col = ''; |
| 39 | 35 | ||
| 36 | %options = ( | ||
| 37 | "version" => sub { VERSION_MESSAGE(); exit 0; }, | ||
| 38 | "help|?" => sub { HELP_MESSAGE(); exit 0; }, | ||
| 39 | ); | ||
| 40 | |||
| 40 | GetOptions (\%options, | 41 | GetOptions (\%options, |
| 41 | "version" => sub { VERSION_MESSAGE(); exit 0; }, | 42 | "h|host=s", |
| 42 | "help|?" => sub { HELP_MESSAGE(); exit 0; }, | 43 | "d|database=s", |
| 43 | "host|h=s", | 44 | "t|table=s", |
| 44 | "database|d=s", | 45 | "u|username=s", |
| 45 | "table|t=s", | 46 | "p|password=s", |
| 46 | "username|u=s", | 47 | "f|logfile=s"); |
| 47 | "password|p=s", | 48 | |
| 48 | "logfile|f=s"); | 49 | $options{h} ||= 'localhost'; |
| 49 | 50 | $options{d} ||= ''; | |
| 50 | $options{host} ||= 'localhost'; | 51 | $options{u} ||= ''; |
| 51 | $options{database} ||= ''; | 52 | $options{p} ||= ''; |
| 52 | $options{username} ||= ''; | 53 | $options{f} ||= ''; |
| 53 | $options{password} ||= ''; | 54 | |
| 54 | $options{logfile} ||= ''; | 55 | if( ! $options{d} ) |
| 55 | |||
| 56 | if( ! $options{database} ) | ||
| 57 | { | 56 | { |
| 58 | HELP_MESSAGE(); | 57 | HELP_MESSAGE(); |
| 59 | print "Must supply a database to connect to.\n"; | 58 | print "Must supply a database to connect to.\n"; |
| 60 | exit 1; | 59 | exit 1; |
| 61 | } | 60 | } |
| 62 | 61 | ||
| 63 | if( ! $options{table} ) | 62 | if( ! $options{t} ) |
| 64 | { | 63 | { |
| 65 | HELP_MESSAGE(); | 64 | HELP_MESSAGE(); |
| 66 | print "Must supply table name.\n"; | 65 | print "Must supply table name.\n"; |
| 67 | exit 1; | 66 | exit 1; |
| 68 | } | 67 | } |
| 69 | 68 | ||
| 70 | if( $options{logfile} ) | 69 | if( $options{f} ) |
| 71 | { | 70 | { |
| 72 | if( ! -e $options{logfile} ) | 71 | if( ! -e $options{f} ) |
| 73 | { | 72 | { |
| 74 | print "File '$options{logfile}' doesn't exist.\n"; | 73 | print "File '$options{f}' doesn't exist.\n"; |
| 75 | exit 1; | 74 | exit 1; |
| 76 | } | 75 | } |
| 77 | open(STDIN, "<$options{logfile}") || die "Can't open $options{logfile} for reading."; | 76 | open(STDIN, "<$options{f}") || die "Can't open $options{f} for reading."; |
| 78 | } | 77 | } |
| 79 | 78 | ||
| 80 | $dbh = Connect(); | 79 | $dbh = Connect(); |
| @@ -82,7 +81,7 @@ if (! $dbh) { | |||
| 82 | exit 1; | 81 | exit 1; |
| 83 | } | 82 | } |
| 84 | 83 | ||
| 85 | $sql = "INSERT INTO $options{table} ("; | 84 | $sql = "INSERT INTO $options{t} ("; |
| 86 | foreach $col (@cols) | 85 | foreach $col (@cols) |
| 87 | { | 86 | { |
| 88 | $sql .= "$col," if( $col ); | 87 | $sql .= "$col," if( $col ); |
| @@ -115,15 +114,15 @@ while($line = <STDIN>) | |||
| 115 | } | 114 | } |
| 116 | print "Parsed $linecount Log lines\n"; | 115 | print "Parsed $linecount Log lines\n"; |
| 117 | print "Inserted $insertcount records\n"; | 116 | print "Inserted $insertcount records\n"; |
| 118 | print "to table '$options{table}' in database '$options{database}' on '$options{host}'\n"; | 117 | print "to table '$options{t}' in database '$options{d}' on '$options{h}'\n"; |
| 119 | 118 | ||
| 120 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # | 119 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # |
| 121 | # Connects to a MySQL database and returns the connection. | 120 | # Connects to a MySQL database and returns the connection. |
| 122 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # | 121 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # |
| 123 | sub Connect | 122 | sub Connect |
| 124 | { | 123 | { |
| 125 | my $dsn = "DBI:mysql:$options{database};hostname=$options{host}"; | 124 | my $dsn = "DBI:mysql:$options{d};hostname=$options{h}"; |
| 126 | return DBI->connect( $dsn, $options{username}, $options{password} ); | 125 | return DBI->connect( $dsn, $options{u}, $options{p} ); |
| 127 | } | 126 | } |
| 128 | 127 | ||
| 129 | 128 | ||
| @@ -216,9 +215,7 @@ EOF | |||
| 216 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # | 215 | # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # |
| 217 | sub VERSION_MESSAGE | 216 | sub VERSION_MESSAGE |
| 218 | { | 217 | { |
| 219 | print "mysql_import_combined_log.pl version 1.2\n"; | 218 | print "mysql_import_combined_log.pl version 1.1\n"; |
| 220 | print "Version 1.0 Written by Aaron Jenson.\n"; | ||
| 221 | print "Update to work with perl 5.6.1 by Edward Rudd\n"; | ||
| 222 | } | 219 | } |
| 223 | 220 | ||
| 224 | 1; | 221 | 1; |
