From 9e046d5c9d8fedb8cf7612d08a7d07c7a3f0322b Mon Sep 17 00:00:00 2001 From: Edward Rudd Date: Tue, 30 Dec 2003 23:26:04 +0000 Subject: added mysql import script by Aaron Jenson. --- contrib/Makefile.in | 86 +++++++++++++++ contrib/mysql_import_combined_log.pl | 203 +++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+) create mode 100644 contrib/Makefile.in create mode 100644 contrib/mysql_import_combined_log.pl diff --git a/contrib/Makefile.in b/contrib/Makefile.in new file mode 100644 index 0000000..ca70e2f --- /dev/null +++ b/contrib/Makefile.in @@ -0,0 +1,86 @@ +# @configure_input@ + +# Modify these top variables. +SUBDIRS = + +EXTRA_DIST = README documentation.lyx + +#Don't modify anything below here + +srcdir = @abs_srcdir@ +builddir = @abs_builddir@ + +STD_DIST = Makefile.in + +DISTFILES = $(STD_DIST) $(EXTRA_DIST) + +all: $(TARGET) all-subdirs + +all-subdirs install-subdirs update-subdirs clean-subdirs distclean-subdirs: + @otarget=`echo $@|sed s/-subdirs//`; \ + list=' $(SUBDIRS)'; \ + for i in $$list; do \ + if test -d "$$i"; then \ + target="$$otarget"; \ + echo "Making $$target in $$i"; \ + if test "$$i" = "."; then \ + made_local=yes; \ + target="local-$$target"; \ + fi; \ + (cd $$i && $(MAKE) $$target) || exit 1; \ + fi; \ + done; \ + +include: + rm -rf include + ln -s @APACHE_INCDIR@ include + +install: install-subdirs + @APXS_BIN@ -i -a -n $(subst mod_,,@PACKAGE_NAME@) $(TARGET) + +update: update-subdirs + +clean: clean-subdirs + +distclean: clean distclean-subdirs + $(RM) Makefile + +DESTDIR = @PACKAGE_TARNAME@-@PACKAGE_VERSION@ +DESTTGZ = $(DESTDIR).tar.gz +dist: + @rm -rf $(DESTDIR); \ + list=' $(SUBDIRS)'; \ + for i in $$list; do \ + if test -d "$$i"; then \ + target=local-dist; \ + echo "Making $$target in $$i"; \ + if test "$$i" = "."; then \ + made_local=yes; \ + target="local-dist"; \ + fi; \ + NEWDESTDIR=$(builddir)/$(DESTDIR)/$$i; \ + echo $(NEWDESTDIR); \ + (cd $$i && $(MAKE) DESTDIR=$(builddir)/$(DESTDIR)/$$i $$target) || exit 1; \ + fi; \ + done; + if test "$$made_local" != "yes"; then \ + $(MAKE) "local-dist" || exit 1; \ + fi + tar -zcf $(DESTTGZ) $(DESTDIR) + rm -rf $(DESTDIR) + +local-dist: $(DISTFILES) + mkdir -p $(DESTDIR) + cp -dp --parents $(DISTFILES) $(DESTDIR) + +$(builddir)/.deps: depend + +depend: $(SOURCES) $(HEADERS) + if test -n "`ls $(srcdir)/*.c 2> /dev/null`"; then \ + $(CC) -MM -MT $(TARGET) -I@APACHE_INCDIR@ $(INCLUDES) $(srcdir)/*.c > $(builddir)/.deps || true; \ + fi + +include $(builddir)/.deps + +.PHONY: include all-subdirs update-subdirs install-subdirs \ + clean-subdirs distclean-subdirs dist diff --git a/contrib/mysql_import_combined_log.pl b/contrib/mysql_import_combined_log.pl new file mode 100644 index 0000000..3384b77 --- /dev/null +++ b/contrib/mysql_import_combined_log.pl @@ -0,0 +1,203 @@ +#!/usr/bin/perl +use strict; +use Getopt::Std; +use DBI; +use Time::ParseDate; + +my %options = (); +my $i = 0; +my $sql = ''; +my $valuesSql = ''; +my $line = ''; +my $dbh = 0; +my $sth = 0; +my @parts = (); +my $part; +my $TIMESTAMP = 3; +my $REQUEST_LINE = 4; +my @cols = ( + 'remote_host', ## 0 + 'remote_user', ## 1 + '', ## 2 + 'time_stamp', ## 4 + 'request_line', ## 5 + 'request_method', ## 6 + 'request_uri', ## 7 + 'request_args', ## 8 + 'request_protocol', ## 9 + 'status', ## 10 + 'bytes_sent', ## 11 + 'referer', ## 12 + 'agent' ## 13 +); +my $col = ''; + +$Getopt::Std::STANDARD_HELP_VERSION = 1; ## if we show the help, exit afterwards. +getopts('h:u:p:d:t:f:', \%options); + + +$options{h} ||= 'localhost'; +$options{d} ||= ''; +$options{u} ||= ''; +$options{p} ||= ''; +$options{f} ||= ''; + +if( ! $options{d} ) +{ + print "Must supply a database to connect to.\n"; + exit 1; +} + +if( ! $options{t} ) +{ + print "Must supply table name.\n"; + exit 1; +} + +if( $options{f} ) +{ + if( ! -e $options{f} ) + { + print "File '$options{f}' doesn't exist.\n"; + exit 1; + } + open(STDIN, "<$options{f}") || die "Can't open $options{f} for reading."; +} + +$dbh = Connect(); + +$sql = "INSERT INTO $options{t} ("; +foreach $col (@cols) +{ + $sql .= "$col," if( $col ); +} +chop($sql); +$sql .= ') VALUES ('; + +while($line = ) +{ + @parts = SplitLogLine( $line ); + next if( $parts[$TIMESTAMP] == 0 ); + $valuesSql = ''; + for( $i = 0; $i < @cols; ++$i ) + { + $parts[$i] =~ s/\\/\\\\/g; + $parts[$i] =~ s/'/\\'/g; + $valuesSql .= "'$parts[$i]'," if( $cols[$i] ); + } + chop($valuesSql); + + $sth = $dbh->prepare("$sql$valuesSql)"); + if( ! $sth->execute() ) + { + print "Unable to perform specified query.\n$sql$valuesSql\n" . $sth->errstr() . "\n"; + } + $sth->finish(); +} + + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# Connects to a MySQL database and returns the connection. +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +sub Connect +{ + my $dsn = "DBI:mysql:$options{d};hostname=$options{h}"; + return DBI->connect( $dsn, $options{u}, $options{p} ); +} + + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# Splits up a log line into its parts. +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +sub SplitLogLine +{ + my $line = shift; + my $i = 0; + my $inQuote = 0; + my $char = ''; + my $part = ''; + my @parts = (); + my $count; + chomp($line); + for( $i = 0; $i < length($line); ++$i ) + { + $char = substr($line, $i, 1); + if( $char eq ' ' && ! $inQuote ) + { + ## print "Found part $part.\n"; + if( $count == $TIMESTAMP ) + { + $part = parsedate($part, WHOLE => 1, DATE_REQUIRED => 1, TIME_REQUIRED => 2); + } + push(@parts, $part); + if( $count == $REQUEST_LINE ) + { + my @request = split(/[ ?]/, $part); + push(@parts, $request[0]); + push(@parts, $request[1]); + if( $request[3] ) + { + push(@parts, $request[2]); + push(@parts, $request[3]); + } + else + { + push(@parts, ''); + push(@parts, $request[2]); + } + $count += 5; + } + else + { + ++$count; + } + $part = ''; + } + elsif( $char eq '"' || $char eq '[' || $char eq ']' ) + { + $inQuote = !$inQuote; + } + else + { + $part .= $char; + } + } + push(@parts,$part) if $part; + + return @parts; +} + + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# Prints the usage/help message for this program. +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +sub HELP_MESSAGE +{ + print< -t [-h ] [-u ] [-p ] [-f The host to connect to. Default is localhost. + -d The database to use. Required. + -u The user to connect as. + -p The user's password. + -t
The name of the table in which to insert data. + -f The file to read from. If not given, data is read from stdin. + --help Print out this help message. + --version Print out the version of this software. +EOF +} + + + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# Prints the version information for this program +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +sub VERSION_MESSAGE +{ + print "mysql_import_combined_log.pl version 1.0\n"; +} + +1; + +1; + -- cgit