Initial revision of command line importer.

8 files changed, 840 insertions, 0 deletions

diff --git a/utility/Makefile.in b/utility/Makefile.in
new file mode 100644
index 0000000..d397b96
--- /dev/null
+++ b/utility/Makefile.in
@@ -0,0 +1,38 @@
+# @configure_input@
+
+top_srcdir = @top_srcdir@
+srcdir = @abs_srcdir@
+builddir = @abs_builddir@
+
+CFLAGS = -g -Wall -fno-strict-aliasing @APR_CFLAGS@ @APR_INCLUDES@ @APU_INCLUDES@
+CPPFLAGS = @APR_CPPFLAGS@
+LDFLAGS = @APR_LDFLAGS@ @APU_LDFLAGS@
+
+ifeq (@OOO_MAINTAIN@,1)
+CFLAGS += -Werror
+endif
+
+STD_DIST = Makefile.in
+
+DISTFILES = $(STD_DIST) $(EXTRA_DIST) $(SOURCES) $(HEADERS)
+
+SOURCES = shell.c config.c logparse.c
+HEADERS = shell.h config.h logparse.h
+OBJECTS = $(patsubst %.c,%.o,$(SOURCES))
+
+TARGETS = mod_log_sql
+
+all: $(TARGETS)
+
+mod_log_sql: $(OBJECTS) $(HEADERS)
+        $(CC) -o $@ $(OBJECTS) $(LDFLAGS)
+
+clean:
+        $(RM) $(OBJECTS) $(TARGETS)
+
+local-dist: $(DISTFILES)
+        mkdir -p $(DESTDIR)
+        cp -dp --parents $(DISTFILES) $(DESTDIR)
+
+Makefile: Makefile.in ../config.status
+        cd .. && ./config.status
diff --git a/utility/config.c b/utility/config.c
new file mode 100644
index 0000000..6b3dce1
--- /dev/null
+++ b/utility/config.c
@@ -0,0 +1,305 @@
+#include "apr.h"
+#include "apr_file_info.h"
+#include "apr_file_io.h"
+#include "apr_strings.h"
+#include "apr_hash.h"
+#include "apr_lib.h"
+#include "shell.h"
+#include "config.h"
+
+apr_hash_t *g_config_opts;
+
+apr_status_t config_set_string(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    int offset = (int)(long)opt->data;
+    char **data = (char **)((void *)cfg + offset);
+    if (argc != 2) return APR_EINVAL;
+    *data = apr_pstrdup(cfg->pool, argv[1]);
+    return APR_SUCCESS;
+}
+
+apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    int offset = (int)(long)opt->data;
+    int *data = (int *)((void *)cfg + offset);
+    if (argc != 2) return APR_EINVAL;
+    *data = apr_atoi64(argv[1]);
+    return APR_SUCCESS;
+}
+
+apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    int offset = (int)(long)opt->data;
+    int *data = (int *)((void *)cfg + offset);
+    if (argc != 2) return APR_EINVAL;
+    *data = CHECK_YESNO(argv[1]);
+    return APR_SUCCESS;
+}
+
+apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    if (argc != 2) return APR_EINVAL;
+    if (!strcasecmp(argv[1],"error")) {
+        cfg->loglevel = LOGLEVEL_ERROR;
+    } else if (!strcasecmp(argv[1],"warn")) {
+        cfg->loglevel = LOGLEVEL_WARN;
+    } else if (!strcasecmp(argv[1],"debug")) {
+        cfg->loglevel = LOGLEVEL_DEBUG;
+    } else if (!strcasecmp(argv[1],"quiet")) {
+        cfg->loglevel = LOGLEVEL_QUIET;
+    } else {
+        cfg->loglevel = LOGLEVEL_ERROR;
+    }
+    return APR_SUCCESS;
+}
+
+apr_status_t config_set_dbconnect(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    return APR_SUCCESS;
+}
+
+apr_status_t config_set_dbparam(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    return APR_SUCCESS;
+}
+
+apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    char **newp;
+    if (argc != 2) return APR_EINVAL;
+    newp = (char **)apr_array_push(cfg->input_files);
+    *newp = apr_pstrdup(cfg->pool, argv[1]);
+    return APR_SUCCESS;
+}
+
+apr_status_t config_set_dummy(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    return APR_SUCCESS;
+}
+
+apr_status_t config_set_logformat(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    config_logformat_t *format;
+    config_logformat_field_t *field;
+
+    if (argc != 4) return APR_EINVAL;
+
+    format = apr_hash_get(cfg->log_formats,argv[1],APR_HASH_KEY_STRING);
+    if (!format) {
+        format = apr_palloc(cfg->pool, sizeof(config_logformat_t));
+        format->name = apr_pstrdup(cfg->pool, argv[1]);
+        format->fields = apr_array_make(cfg->pool, 5,
+                sizeof(config_logformat_field_t));
+        apr_hash_set(cfg->log_formats, argv[1], APR_HASH_KEY_STRING, format);
+    }
+    field = (config_logformat_field_t *)apr_array_push(format->fields);
+    field->name = apr_pstrdup(cfg->pool, argv[2]);
+    field->datatype = apr_pstrdup(cfg->pool, argv[3]);
+    return APR_SUCCESS;
+}
+
+void config_dump(config_t *cfg)
+{
+    apr_hash_index_t *hi;
+
+    printf("ErrorLog: %s\n",cfg->errorlog);
+    printf("LogLevel: %d\n",cfg->loglevel);
+
+    printf("InputDir: %s\n",cfg->input_dir);
+
+    printf("Table: %s\n",cfg->table);
+    printf("Transactions: %d\n",cfg->transactions);
+    printf("MachineID: %s\n",cfg->machineid);
+
+    printf("Log formats:\n");
+    for (hi = apr_hash_first(cfg->pool, cfg->log_formats); hi; hi
+            = apr_hash_next(hi)) {
+        config_logformat_t *format;
+        config_logformat_field_t *fields;
+        int i;
+
+        apr_hash_this(hi, NULL, NULL, (void **)&format);
+        printf(">> %s\n",format->name);
+        fields = (config_logformat_field_t *)format->fields->elts;
+        for (i=0; i<format->fields->nelts; i++) {
+            printf(">>>> %s:%s\n", fields[i].name, fields[i].datatype);
+        }
+    }
+    printf("Log Format: %s\n",cfg->logformat);
+
+    printf("DryRun: %d\n",cfg->dryrun);
+    printf("Summary: %d\n",cfg->summary);
+}
+
+static char *lowerstr(apr_pool_t *pool, const char *input) {
+        char *temp;
+        char *itr;
+        temp = apr_pstrdup(pool, input);
+        for (itr=temp; *itr!='\0'; itr++) {
+                *itr = apr_tolower(*itr);
+        }
+        return temp;
+}
+
+#define config_get_option(name) apr_hash_get(g_config_opts, name, APR_HASH_KEY_STRING)
+
+void config_add_option(apr_pool_t *p, const char *const name,
+        const char *const help, config_func_t func, void *data)
+{
+    config_opt_t *opt;
+    if (!g_config_opts) {
+        g_config_opts = apr_hash_make(p);
+    }
+    opt = apr_palloc(p, sizeof(config_opt_t));
+    opt->name = name;
+    opt->help = help;
+    opt->func = func;
+    opt->data = data;
+    apr_hash_set(g_config_opts, lowerstr(p,name), APR_HASH_KEY_STRING, opt);
+}
+
+void config_init(apr_pool_t *p)
+{
+    config_add_option(p, "ErrorLog", "File to log errors",
+            config_set_string, (void *)APR_OFFSETOF(config_t, errorlog));
+    config_add_option(p, "LogLevel", "Set Log Level (error, warn, debug, quiet)",
+            config_set_loglevel, NULL);
+
+    config_add_option(p, "InputDirectory", "Directory to scan for log files",
+            config_set_string, (void *)APR_OFFSETOF(config_t, input_dir));
+    config_add_option(p, "InputFile", "Parse only this file",
+            config_set_inputfile, NULL);
+
+    config_add_option(p, "DBConnect", "DB Connection information  type://user:pass@hostname/database",
+            config_set_dbconnect, NULL);
+    config_add_option(p, "DBParam", "DB Connection Parameter",
+            config_set_dbparam, NULL);
+    config_add_option(p, "Table", "Table to import the log to",
+            config_set_string, (void *)APR_OFFSETOF(config_t, table));
+    config_add_option(p, "UseTransactions", "Enable Transactions?",
+            config_set_flag, (void *)APR_OFFSETOF(config_t, transactions));
+    config_add_option(p, "MachineID", "Machine ID to set",
+            config_set_string, (void *)APR_OFFSETOF(config_t, machineid));
+
+    config_add_option(p, "LogFormatConfig", "Define input log formats",
+            config_set_logformat, NULL);
+    config_add_option(p, "LogFormat", "Use this logformat when parsing files",
+            config_set_string, (void *)APR_OFFSETOF(config_t, logformat));
+
+    config_add_option(p, "DryRun", "Don't perform any actual database changes",
+            config_set_flag, (void *)APR_OFFSETOF(config_t, dryrun));
+    config_add_option(p, "Config", "Dummy to handle config directive",
+            config_set_dummy, NULL);
+    config_add_option(p, "Summary", "Show the summary before exit?",
+            config_set_flag, (void *)APR_OFFSETOF(config_t, summary));
+}
+
+config_t *config_create(apr_pool_t *p)
+{
+    config_t *cfg;
+    apr_pool_t *sp;
+    apr_pool_create(&sp, p);
+    cfg = apr_pcalloc(sp, sizeof(config_t));
+    cfg->pool = sp;
+    cfg->loglevel = LOGLEVEL_WARN;
+    cfg->summary = 1;
+    cfg->transactions = 1;
+    cfg->input_files = apr_array_make(cfg->pool, 10, sizeof(char *));
+    cfg->dbconfig = apr_table_make(cfg->pool, 5);
+    cfg->log_formats = apr_hash_make(cfg->pool);
+
+    return cfg;
+}
+
+int config_merge(void *rec, const char *key, const char *value) {
+    config_t *cfg = (config_t *)rec;
+
+    config_opt_t *opt = config_get_option(key);
+    if (opt) {
+        const char *args[] = {key, value};
+        opt->func(cfg, opt, 2, args);
+    } else {
+        printf("Unhandled: %s\n", key);
+    }
+    return 1;
+}
+
+apr_status_t config_read(config_t *cfg, const char *filename,
+        apr_table_t *merge)
+{
+    apr_finfo_t finfo;
+    apr_file_t *file;
+    apr_status_t rv, ret = APR_SUCCESS;
+    apr_pool_t *tp, *targp;
+    config_opt_t *opt;
+    char buff[1024];
+    char *ptr, *ptr2;
+    char **targv;
+    int targc;
+    int line;
+
+    apr_pool_create(&tp, cfg->pool);
+    apr_pool_create(&targp, tp);
+
+    if (apr_stat(&finfo, filename, APR_FINFO_MIN, tp) != APR_SUCCESS) {
+        return APR_ENOENT;
+    }
+    rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED,
+            APR_OS_DEFAULT, tp);
+    if (rv != APR_SUCCESS)
+        return rv;
+
+    line = 0;
+    do {
+        rv = apr_file_gets(buff, 1024, file);
+        if (rv == APR_SUCCESS) { // we read data
+            line++;
+
+            // skip leading white space
+            for (ptr = buff; *ptr == ' ' || *ptr == '\t'; ptr++)
+                ;
+            // chomp off newline
+            for (ptr2 = ptr + strlen(ptr); *ptr2 != '\r' && *ptr2 != '\n'; ptr2--)
+                ;
+            *ptr2 = '\0';
+
+            // skip comments
+            if (*ptr == '#')
+                continue;
+            if (*ptr == '\0')
+                continue;
+            apr_pool_clear(targp);
+            apr_tokenize_to_argv(buff, &targv, targp);
+            targc = 0;
+            while (targv[targc]) targc++;
+            opt = config_get_option(lowerstr(targp,targv[0]));
+            if (opt) {
+                rv = opt->func(cfg, opt, targc, (const char **)targv);
+                if (APR_STATUS_IS_EINVAL(rv)) {
+                    printf("Config Error: Invalid Arguments for %s\n\t%s\n",
+                            opt->name, opt->help);
+                    ret = rv;
+                }
+            } else {
+                printf("Unhandled: %s\n", targv[0]);
+            }
+        }
+    } while (rv == APR_SUCCESS);
+
+    // Apply merges
+    apr_table_do(config_merge,(void *)cfg,merge,NULL);
+
+    apr_file_close(file);
+    apr_pool_destroy(tp);
+    return ret;
+}
+
diff --git a/utility/config.h b/utility/config.h
new file mode 100644
index 0000000..e1827fe
--- /dev/null
+++ b/utility/config.h
@@ -0,0 +1,104 @@
+#ifndef CONFIG_H_
+#define CONFIG_H_
+
+#include "apr_tables.h"
+#include "apr_hash.h"
+#include "apr_file_io.h"
+
+typedef enum {
+    LOGLEVEL_QUIET = 0,
+    LOGLEVEL_ERROR = 1,
+    LOGLEVEL_WARN = 2,
+    LOGLEVEL_DEBUG = 3,
+} loglevel_e;
+
+typedef struct {
+    /** the structures pool (to ease function arguments) */
+    apr_pool_t *pool;
+
+    /** error log file */
+    const char *errorlog;
+    /** log level */
+    loglevel_e loglevel;
+    /** error_log */
+    apr_file_t *errorlog_fp;
+
+    /** input directory of log files */
+    const char *input_dir;
+    /** list of files to scan */
+    apr_array_header_t *input_files;
+
+    /** db connection configuration */
+    apr_table_t *dbconfig;
+    /** Logging table */
+    const char *table;
+    /** Use transactons */
+    int transactions;
+    /** Machine ID */
+    const char *machineid;
+
+    /** Log file formats */
+    apr_hash_t *log_formats;
+    /** format to use to parse files */
+    const char *logformat;
+
+    /** output fields */
+    apr_array_header_t *output_fields;
+
+    /** filter configuration */
+    apr_array_header_t *filters;
+
+    /** Dry Run */
+    int dryrun;
+
+    /* Show the summary */
+    int summary;
+} config_t;
+
+typedef struct {
+    const char *name;
+    apr_array_header_t *fields;
+} config_logformat_t;
+
+typedef struct {
+    const char *name;
+    const char *datatype;
+} config_logformat_field_t;
+
+typedef struct config_opt_t config_opt_t;
+typedef apr_status_t (*config_func_t)(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv);
+
+struct config_opt_t {
+    const char *name;
+    const char *help;
+    config_func_t func;
+    void *data;
+};
+
+#define CHECK_YESNO(c) ((!strcasecmp(c,"on") || !strcasecmp(c,"yes")) ? 1 : 0)
+
+/**
+ * Initialize the config parser
+ */
+void config_init(apr_pool_t *p);
+
+/**
+ * Dump the configuration to stdout
+ */
+void config_dump(config_t *cfg);
+
+/**
+ * Creates the default configuration
+ */
+config_t *config_create(apr_pool_t *p);
+
+/**
+ * Read in a configuration file
+ */
+apr_status_t config_read(config_t *cfg, const char *filename,
+        apr_table_t *merge);
+
+void config_generate(const char *filename);
+
+#endif /*CONFIG_H_*/
diff --git a/utility/logparse.c b/utility/logparse.c
new file mode 100644
index 0000000..2940534
--- /dev/null
+++ b/utility/logparse.c
@@ -0,0 +1,205 @@
+#include "logparse.h"
+#include "apr_file_info.h"
+#include "apr_file_io.h"
+#include "apr_strings.h"
+
+void find_log_files(config_t *cfg)
+{
+    apr_pool_t *tp;
+    apr_dir_t *dir;
+    apr_finfo_t finfo;
+    char **newp;
+
+    if (!cfg->input_dir)
+        return;
+    apr_pool_create(&tp, cfg->pool);
+    if (apr_dir_open(&dir, cfg->input_dir, tp)==APR_SUCCESS) {
+        while (apr_dir_read(&finfo, APR_FINFO_NAME | APR_FINFO_TYPE, dir)
+                == APR_SUCCESS) {
+            if (finfo.filetype == APR_DIR)
+                continue;
+            newp = (char **)apr_array_push(cfg->input_files);
+            apr_filepath_merge(newp, cfg->input_dir, finfo.name,
+                    APR_FILEPATH_TRUENAME, cfg->pool);
+        }
+        apr_dir_close(dir);
+    }
+    apr_pool_destroy(tp);
+}
+
+/*
+ * Modified version of apr_tokenize_to_argv to add [] as quoting characters
+ *
+ *    token_context: Context from which pool allocations will occur.
+ *    arg_str:       Input string for conversion to argv[].
+ *    argv_out:      Output location. This is a pointer to an array
+ *                   of pointers to strings (ie. &(char *argv[]).
+ *                   This value will be allocated from the contexts
+ *                   pool and filled in with copies of the tokens
+ *                   found during parsing of the arg_str.
+ *    keepquotes:    Keep the quotes instead of stripping them
+ */
+apr_status_t tokenize_logline(const char *arg_str, char ***argv_out,
+        apr_pool_t *token_context, int keepquotes)
+{
+    const char *cp;
+    const char *ct;
+    char *cleaned, *dirty;
+    int escaped;
+    int isquoted, numargs = 0, argnum;
+
+#define SKIP_WHITESPACE(cp) \
+    for ( ; *cp == ' ' || *cp == '\t'; ) { \
+        cp++; \
+    };
+
+#define CHECK_QUOTATION(cp,isquoted) \
+    isquoted = 0; \
+    if (*cp == '"') { \
+        isquoted = 1; \
+        cp++; \
+    } \
+    else if (*cp == '\'') { \
+        isquoted = 2; \
+        cp++; \
+    } \
+    else if (*cp == '[') { \
+        isquoted = 3; \
+        cp++; \
+    }
+
+    /* DETERMINE_NEXTSTRING:
+     * At exit, cp will point to one of the following:  NULL, SPACE, TAB or QUOTE.
+     * NULL implies the argument string has been fully traversed.
+     */
+#define DETERMINE_NEXTSTRING(cp,isquoted) \
+    for ( ; *cp != '\0'; cp++) { \
+        if (   (isquoted    && (*cp     == ' ' || *cp     == '\t')) \
+            || (*cp == '\\' && (*(cp+1) == ' ' || *(cp+1) == '\t' || \
+                                *(cp+1) == '"' || *(cp+1) == '\'' || \
+                                *(cp+1) == '[' || *(cp+1) == ']'))) { \
+            cp++; \
+            continue; \
+        } \
+        if (   (!isquoted && (*cp == ' ' || *cp == '\t')) \
+            || (isquoted == 1 && *cp == '"') \
+            || (isquoted == 2 && *cp == '\'') \
+            || (isquoted == 3 && *cp == ']') \
+            ) { \
+            break; \
+        } \
+    }
+
+    /* REMOVE_ESCAPE_CHARS:
+     * Compresses the arg string to remove all of the '\' escape chars.
+     * The final argv strings should not have any extra escape chars in it.
+     */
+#define REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped) \
+    escaped = 0; \
+    while(*dirty) { \
+        if (!escaped && *dirty == '\\') { \
+            escaped = 1; \
+        } \
+        else { \
+            escaped = 0; \
+            *cleaned++ = *dirty; \
+        } \
+        ++dirty; \
+    } \
+    *cleaned = 0;        /* last line of macro... */
+
+    cp = arg_str;
+    SKIP_WHITESPACE(cp);
+    ct = cp;
+
+    /* This is ugly and expensive, but if anyone wants to figure a
+     * way to support any number of args without counting and
+     * allocating, please go ahead and change the code.
+     *
+     * Must account for the trailing NULL arg.
+     */
+    numargs = 1;
+    while (*ct != '\0') {
+        CHECK_QUOTATION(ct, isquoted)
+        ;
+        DETERMINE_NEXTSTRING(ct, isquoted);
+        if (*ct != '\0') {
+            ct++;
+        }
+        numargs++;
+        SKIP_WHITESPACE(ct);
+    }
+    *argv_out = apr_palloc(token_context, numargs * sizeof(char*));
+
+    /*  determine first argument */
+    for (argnum = 0; argnum < (numargs-1); argnum++) {
+        SKIP_WHITESPACE(cp);
+        CHECK_QUOTATION(cp, isquoted)
+        ;
+        ct = cp;
+        DETERMINE_NEXTSTRING(cp, isquoted);
+        cp++;
+        if (isquoted && keepquotes) {
+            (*argv_out)[argnum] = apr_palloc(token_context, cp - ct + 2);
+            apr_cpystrn((*argv_out)[argnum], ct -1, cp - ct + 2);
+        } else {
+            (*argv_out)[argnum] = apr_palloc(token_context, cp - ct);
+            apr_cpystrn((*argv_out)[argnum], ct, cp - ct);
+        }
+        cleaned = dirty = (*argv_out)[argnum];
+        REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped)
+        ;
+    }
+    (*argv_out)[argnum] = NULL;
+
+    return APR_SUCCESS;
+}
+
+apr_status_t parse_logfile(config_t *cfg, const char *filename)
+{
+    apr_pool_t *tp, *argp;
+    apr_file_t *file;
+    apr_status_t rv;
+    char buff[2048];
+    char **targv;
+    int targc;
+    int line;
+
+    apr_pool_create(&tp, cfg->pool);
+    apr_pool_create(&argp, tp);
+
+    rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED,
+            APR_OS_DEFAULT, tp);
+    if (rv != APR_SUCCESS) {
+        printf("Could not open %s\n", filename);
+        return rv;
+    }
+
+    line = 0;
+    do {
+        rv = apr_file_gets(buff, 1024, file);
+        if (rv == APR_SUCCESS) {
+            line++;
+            char *ptr;
+            // chomp off newline
+            for (ptr = buff + strlen(buff); *ptr != '\r' && *ptr != '\n'; ptr--)
+                ;
+            *ptr = '\0';
+            apr_pool_clear(argp);
+            tokenize_logline(buff, &targv, argp, 1);
+            targc = 0;
+            while (targv[targc]) targc++;
+            if (targc != 9) {
+                int i;
+                printf("Line %d(%d): %s\n",line, targc, buff);
+                for (i = 0; targv[i]; i++) {
+                    printf("Arg (%d): '%s'\n", i, targv[i]);
+                }
+            }
+        }
+    } while (rv == APR_SUCCESS);
+    printf("Total Lines: %d\n", line);
+    apr_file_close(file);
+    apr_pool_destroy(tp);
+    return APR_SUCCESS;
+}
diff --git a/utility/logparse.h b/utility/logparse.h
new file mode 100644
index 0000000..540a9e0
--- /dev/null
+++ b/utility/logparse.h
@@ -0,0 +1,10 @@
+#ifndef LOGPARSE_H_
+#define LOGPARSE_H_
+
+#include "config.h"
+
+void find_log_files(config_t *cfg);
+
+apr_status_t parse_logfile(config_t *cfg, const char *filename);
+
+#endif /*LOGPARSE_H_*/
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
new file mode 100644
index 0000000..6cfae61
--- /dev/null
+++ b/utility/mod_log_sql.conf
@@ -0,0 +1,30 @@
+InputDirectory ./logs
+ErrorLog ./error_log
+DBConnect mysql://username:host@server/database
+DBParam socketfile /tmp/mysql.sock
+Table apache_logs
+MachineID 7of9
+UseTransactions on
+LogLevel notice
+DryRun on
+Summary on
+
+LogFormatConfig CLF host String
+LogFormatConfig CLF ident String
+LogFormatConfig CLF user String
+LogFormatConfig CLF date Date
+LogFormatConfig CLF request String
+LogFormatConfig CLF status Number
+LogFormatConfig CLF bytes_sent Number
+
+LogFormatConfig Combined host String
+LogFormatConfig Combined ident String
+LogFormatConfig Combined user String
+LogFormatConfig Combined date Date
+LogFormatConfig Combined request String
+LogFormatConfig Combined status Number
+LogFormatConfig Combined bytes_sent Number
+LogFormatConfig Combined referer String
+LogFormatConfig Combined agent String
+
+LogFormat Combined
diff --git a/utility/shell.c b/utility/shell.c
new file mode 100644
index 0000000..07d9da1
--- /dev/null
+++ b/utility/shell.c
@@ -0,0 +1,148 @@
+#include "apr.h"
+#include "apr_getopt.h"
+#include "apr_tables.h"
+
+#define APR_WANT_STDIO
+#include "apr_want.h"
+#include "stdlib.h"
+
+#include "shell.h"
+#include "config.h"
+#include "logparse.h"
+
+const apr_getopt_option_t _opt_config[]   = {
+    {"machineid",   'm',    1,  "Machine ID for the log file"},
+    {"transaction", 't',    1,  "Use a Transaction (yes,no)"},
+    {"logformat",   'r',    1,  "Use this logformat to parse files"},
+    {"file",        'f',    1,  "Parse this single log file (input dir is NOT scanned)"},
+    {"inputdir",    'd',    1,  "Input Directory to look for log files"},
+    {"config",      'c',    1,  "Configuration file to use (default mod_log_sql.conf)"},
+    {"dryrun",      'n',    0,  "Perform a dry run (do not actually alter the databse)"},
+    {"loglevel",    'l',    1,  "Log Level (deubg, warn, error)"},
+    {"summary",     's',    1,  "Summary (yes,no)"},
+    {"help",        'h',    0,  "Show Help"},
+    {NULL}
+};
+
+void show_help(const char *prog, const apr_getopt_option_t *opts, FILE *output)
+{
+    int ptr = 0;
+    fprintf(output, "Usage: %s [OPTIONS] [files...]\n\n", prog);
+    while (opts[ptr].optch != 0) {
+        if (opts[ptr].optch > 255) {
+            if (opts[ptr].name) {
+                fprintf(output, "    --%-10s", opts[ptr].name);
+            } else {
+                fprintf(output, "                ");
+            }
+        } else {
+            if (opts[ptr].name) {
+                fprintf(output, " -%c --%-10s", opts[ptr].optch, opts[ptr].name);
+            } else {
+                fprintf(output, " -%c             ", opts[ptr].optch);
+            }
+        }
+        if (opts[ptr].has_arg) {
+            fprintf(output, " (arg)");
+        } else {
+            fprintf(output, "      ");
+        }
+        fprintf(output, " %s\n", opts[ptr].description);
+        ptr++;
+    }
+}
+
+int main(int argc, const char *const argv[])
+{
+    apr_pool_t *pool;
+    apr_getopt_t *opts;
+    int opt;
+    const char *opt_arg;
+    apr_status_t rv;
+    apr_table_t *args;
+    config_t *base;
+
+    apr_app_initialize(&argc, &argv, NULL);
+    atexit(apr_terminate);
+
+    if (apr_pool_create(&pool, NULL) != APR_SUCCESS) {
+        fprintf(stderr, "Failed to create memory pool!\n");
+        exit(1);
+    }
+
+    /** Iterate over command line arguments
+     * shoving args in a apr_table for processing later*/
+    args = apr_table_make(pool, 5);
+    apr_table_setn(args, "config", "mod_log_sql.conf");
+    apr_getopt_init(&opts, pool, argc, argv);
+    while ((rv = apr_getopt_long(opts, _opt_config, &opt, &opt_arg)) == APR_SUCCESS) {
+        switch (opt) {
+        case 'c':
+            apr_table_setn(args,"config",opt_arg);
+            break;
+        case 'd':
+            apr_table_setn(args,"inputdirectory",opt_arg);
+            break;
+        case 'f':
+            apr_table_setn(args,"inputfile",opt_arg);
+            break;
+        case 'h':
+            show_help(argv[0], _opt_config, stdout);
+            exit(1);
+            break;
+        case 'l':
+            apr_table_setn(args,"loglevel",opt_arg);
+            break;
+        case 'm':
+            apr_table_setn(args,"machineid",opt_arg);
+            break;
+        case 'n':
+            apr_table_setn(args,"dryrun","yes");
+            break;
+        case 'r':
+            apr_table_setn(args,"logformat",opt_arg);
+            break;
+        case 's':
+            apr_table_setn(args,"summary",opt_arg);
+            break;
+        case 't':
+            apr_table_setn(args,"usetransactions",opt_arg);
+            break;
+        }
+    }
+    if (rv != APR_EOF) {
+        show_help(argv[0], _opt_config, stderr);
+        exit(1);
+    }
+    // Check if no extra args were passed
+    if (opts->ind != opts->argc) {
+        show_help(argv[0], _opt_config, stderr);
+        fprintf(stderr, "\n%s: Extra unknown arguments passed\n\n",argv[0]);
+        exit(1);
+    }
+
+    // Process configuration file
+    config_init(pool);
+    base = config_create(pool);
+    rv = config_read(base, apr_table_get(args,"Config"), args);
+    if (APR_STATUS_IS_ENOENT(rv)) {
+        fprintf(stderr,"Could not load configuration file: %s\n",apr_table_get(args,"config"));
+    } else if (rv) {
+        exit(1);
+    }
+    config_dump(base);
+    // Apply overrides from command line
+    find_log_files(base);
+    if (!apr_is_empty_array(base->input_files)) {
+        char **filelist;
+        int f, l;
+        filelist = (char **)base->input_files->elts;
+        for (f=0, l=base->input_files->nelts; f < l; f++) {
+            printf("Scanning %s\n",filelist[f]);
+            parse_logfile(base, filelist[f]);
+        }
+    } else {
+        printf("No input files\n");
+    }
+    return 0;
+}
diff --git a/utility/shell.h b/utility/shell.h
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/utility/shell.h