1 files changed, 768 insertions, 0 deletions

diff --git a/utility/logparse.c b/utility/logparse.c
new file mode 100644
index 0000000..1b3cc97
--- /dev/null
+++ b/utility/logparse.c
@@ -0,0 +1,768 @@
+#include "logparse.h"
+#include "apr_file_info.h"
+#include "apr_file_io.h"
+#include "apr_strings.h"
+#include "apr_time.h"
+
+#include "time.h"
+#include "stdlib.h"
+
+#include "util.h"
+#include "ap_pcre.h"
+#include "database.h"
+
+apr_hash_t *g_parser_funcs;
+void **g_parser_linedata;
+
+static apr_status_t parser_func_wrap(apr_pool_t *p, config_t *cfg,
+        config_output_field_t *field, const char *value, const char **ret)
+{
+    if (field->args[0] && field->args[1]) {
+        struct iovec vec[3];
+        apr_size_t len;
+
+        vec[0].iov_base = (void *)field->args[0];
+        vec[0].iov_len = strlen(field->args[0]);
+        vec[1].iov_base = (void *)value;
+        vec[1].iov_len = strlen(value);
+        vec[2].iov_base = (void *)field->args[1];
+        vec[2].iov_len = strlen(field->args[1]);
+
+        *ret = apr_pstrcatv(p, vec, 3, &len);
+    } else {
+        logging_log(cfg, LOGLEVEL_NOISE, "wrap requires before and after strings");
+        return APR_EINVAL;
+    }
+    return APR_SUCCESS;
+}
+
+static apr_status_t parser_func_regexmatch(apr_pool_t *p, config_t *cfg,
+        config_output_field_t *field, const char *value, const char **ret)
+{
+    struct {
+        ap_regex_t *rx;
+        const char *substr;
+    }*_data;
+    ap_regmatch_t regm[AP_MAX_REG_MATCH];
+    // Check if a regular expression configured
+    if (!field->args[0])
+        return APR_EINVAL;
+    if (!field->data) {
+        // pre compile the regex
+        _data = apr_palloc(cfg->pool, sizeof(ap_regex_t)+sizeof(const char *));
+        _data->rx = ap_pregcomp(cfg->pool, field->args[0],
+                AP_REG_EXTENDED|AP_REG_ICASE);
+        if (field->args[1]) {
+            _data->substr = field->args[1];
+        } else {
+            _data->substr = "$1";
+        }
+        if (!_data->rx) {
+            logging_log(cfg, LOGLEVEL_NOISE, "Failed to compile regular expression");
+            return APR_EINVAL;
+        }
+        field->data = _data;
+    } else
+        _data = field->data;
+
+    if (!ap_regexec(_data->rx, value, AP_MAX_REG_MATCH, regm, 0)) {
+        *ret = ap_pregsub(p, _data->substr, value, AP_MAX_REG_MATCH, regm);
+    }
+    logging_log(cfg, LOGLEVEL_DEBUG, "REGEX: matched %s against %s to %s", value,
+            field->args[0], *ret);
+    return APR_SUCCESS;
+}
+
+static apr_status_t parser_func_totimestamp(apr_pool_t *p, config_t *cfg,
+        config_output_field_t *field, const char *value, const char **ret)
+{
+    time_t time;
+    struct tm ts;
+
+    //memset(&ts,0,sizeof(struct tm));
+
+    strptime(value, "%d/%b/%Y:%H:%M:%S %z", &ts);
+    time = mktime(&ts);
+
+    *ret = apr_itoa(p, time);
+    return APR_SUCCESS;
+}
+
+static apr_status_t parser_func_machineid(apr_pool_t *p, config_t *cfg,
+        config_output_field_t *field, const char *value, const char **ret)
+{
+    if (cfg->machineid) {
+        *ret = apr_pstrdup(p, cfg->machineid);
+    }
+    return APR_SUCCESS;
+}
+
+static apr_status_t parser_func_queryarg(apr_pool_t *p, config_t *cfg,
+        config_output_field_t *field, const char *value, const char **ret)
+{
+    apr_table_t *query = parser_get_linedata(field->func);
+
+    if (!field->args[0]) {
+        logging_log(cfg, LOGLEVEL_NOISE, "queryarg requires name of query arg");
+        return APR_EINVAL;
+    }
+
+    if (!query) {
+        char *query_beg;
+
+        query = apr_table_make(p,3);
+
+        query_beg = strchr(value, '?');
+        // if we have a query string, rip it apart
+        if (query_beg) {
+            char *key;
+            char *value;
+            const char *delim = "&";
+            char *query_string;
+            char *strtok_state;
+            char *query_end = strrchr(++query_beg,' ');
+
+            query_string = apr_pstrndup(p, query_beg, query_end-query_beg);
+            logging_log(cfg, LOGLEVEL_DEBUG, "QUERY: Found String %pp, %pp, %s",
+                    query_beg, query_end, query_string);
+            if (field->args[1]) {
+                delim = field->args[1];
+            }
+            key = apr_strtok(query_string, delim, &strtok_state);
+            while (key) {
+                value = strchr(key, '=');
+                if (value) {
+                    *value = '\0';      /* Split the string in two */
+                    value++;            /* Skip past the = */
+                }
+                else {
+                    value = "1";
+                }
+                ap_unescape_url(key);
+                ap_unescape_url(value);
+                apr_table_set(query, key, value);
+
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                    "QUERY: Found arg: %s = %s", key, value);
+
+                key = apr_strtok(NULL, delim, &strtok_state);
+            }
+        }
+        parser_set_linedata(field->func,query);
+    }
+    *ret = apr_table_get(query, field->args[0]);
+    return APR_SUCCESS;
+}
+
+parser_func_t *parser_get_func(const char *name)
+{
+    return apr_hash_get(g_parser_funcs, name, APR_HASH_KEY_STRING);
+}
+
+static void parser_add_func(apr_pool_t *p, const char *const name,
+        parser_func_f func, int id)
+{
+    parser_func_t *s;
+    if (!g_parser_funcs) {
+        g_parser_funcs = apr_hash_make(p);
+    }
+    s = apr_palloc(p, sizeof(parser_func_t));
+    s->func = func;
+    s->pos = id;
+    s->data = NULL;
+    s->linedata = &g_parser_linedata;
+    apr_hash_set(g_parser_funcs, lowerstr(p, name), APR_HASH_KEY_STRING, s);
+}
+
+void parser_init(apr_pool_t *p)
+{
+    int i = 0;
+    parser_add_func(p, "regexmatch", parser_func_regexmatch, ++i);
+    parser_add_func(p, "totimestamp", parser_func_totimestamp, ++i);
+    parser_add_func(p, "machineid", parser_func_machineid, ++i);
+    parser_add_func(p, "queryarg", parser_func_queryarg, ++i);
+    parser_add_func(p, "wrap", parser_func_wrap, ++i);
+    g_parser_linedata = apr_pcalloc(p, sizeof(void *) * (i+1));
+    g_parser_linedata[0] = (void *)i;
+}
+
+void parser_find_logs(config_t *cfg)
+{
+    apr_pool_t *tp;
+    apr_dir_t *dir;
+    apr_finfo_t finfo;
+    config_filestat_t *newp;
+
+    logging_log(cfg, LOGLEVEL_NOTICE, "Find Log files");
+    if (!cfg->input_dir)
+        return;
+    apr_pool_create(&tp, cfg->pool);
+    if (apr_dir_open(&dir, cfg->input_dir, tp)==APR_SUCCESS) {
+        while (apr_dir_read(&finfo, APR_FINFO_NAME | APR_FINFO_TYPE, dir)
+                == APR_SUCCESS) {
+            char *temp;
+            if (finfo.filetype == APR_DIR)
+                continue;
+            newp = (config_filestat_t *)apr_array_push(cfg->input_files);
+            newp->result = "Not Parsed";
+            apr_filepath_merge(&temp, cfg->input_dir, finfo.name,
+                        APR_FILEPATH_TRUENAME, cfg->pool);
+            newp->fname = temp;
+        }
+        apr_dir_close(dir);
+    }
+    apr_pool_destroy(tp);
+}
+
+#define BUFFER_SIZE (16 * 1024)
+
+void parser_split_logs(config_t *cfg)
+{
+    apr_pool_t *tp, *tfp;
+    apr_array_header_t *foundfiles;
+    config_filestat_t *filelist;
+    config_filestat_t *newfile;
+    apr_file_t *infile;
+    int f, l;
+    apr_status_t rv;
+    apr_finfo_t finfo;
+    char buff[BUFFER_SIZE];
+    int linecount;
+    int piecesize;
+
+    if (!cfg->split_enabled) return;
+    if (!cfg->split_dir) {
+        logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Missing Split Output directory");
+        return;
+    }
+    apr_pool_create(&tp, cfg->pool);
+    apr_pool_create(&tfp, tp);
+
+    if (APR_SUCCESS != apr_stat(&finfo, cfg->split_dir, APR_FINFO_MIN, tp)) {
+       logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Directory %s does not exist", cfg->split_dir);
+       return;
+    }
+    foundfiles = apr_array_copy(tp, cfg->input_files);
+    apr_array_clear(cfg->input_files);
+
+    filelist = (config_filestat_t *)foundfiles->elts;
+    for (f=0, l=foundfiles->nelts; f < l; f++) {
+        apr_pool_clear(tfp);
+        logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Begin Splitting Log File '%s'", filelist[f].fname);
+        rv = apr_file_open(&infile, filelist[f].fname, APR_FOPEN_READ, APR_OS_DEFAULT, tfp);
+
+        if (rv != APR_SUCCESS) {
+            logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s", filelist[f].fname);
+            return;
+        }
+        linecount = 0;
+        while (apr_file_eof(infile) == APR_SUCCESS) {
+            apr_size_t read = BUFFER_SIZE;
+            char *p;
+            apr_file_read(infile, buff, &read);
+            p = buff;
+            while ((p = memchr(p, '\n', (buff + read) - p))) {
+                ++p;
+                ++linecount;
+            }
+        }
+        // now we know how long it is.  Lets split up the file
+        piecesize = linecount / cfg->split_count;
+        if (piecesize < cfg->split_minimum)
+            piecesize = cfg->split_minimum;
+        if (piecesize > cfg->split_maximum && cfg->split_maximum > 0)
+            piecesize = cfg->split_maximum;
+        if (piecesize > linecount) {
+            // File is smaller than piece size just add it back in as is
+            newfile = (config_filestat_t *)apr_array_push(cfg->input_files);
+            newfile->result = "Not Parsed";
+            newfile->fname = filelist[f].fname;
+        } else {
+            //split apart the files
+            int cur_line = 0;
+            int file_count = 1;
+            int out_lines = 0;
+            const char *basefile, *file;
+            apr_file_t *outfile;
+            char trail[2048];
+            apr_size_t trail_size = 0;
+            apr_size_t write;
+            apr_off_t off = 0;
+
+            apr_file_seek(infile, APR_SET, &off);
+
+            basefile = apr_pstrdup(tfp, basename(apr_pstrdup(tfp, filelist[f].fname)));
+
+            file = apr_psprintf(tfp, "%s/%s-%d", cfg->split_dir, basefile, file_count++);
+            logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Creating output file %s", file);
+            rv = apr_file_open(&outfile, file, APR_FOPEN_WRITE | APR_FOPEN_CREATE | APR_FOPEN_TRUNCATE, APR_OS_DEFAULT, tfp);
+            if (rv != APR_SUCCESS) {
+                logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s (%d)", file, rv);
+                return;
+            }
+            newfile = (config_filestat_t *)apr_array_push(cfg->input_files);
+            newfile->result = "Not Parsed";
+            newfile->fname = apr_pstrdup(cfg->pool, file);
+
+            while (apr_file_eof(infile) == APR_SUCCESS) {
+                apr_size_t read = BUFFER_SIZE;
+                char *p, *pp, *buff_start;
+                apr_file_read(infile, buff, &read);
+                buff_start = p = pp = buff;
+                if (trail_size) {
+                    p = memchr(p, '\n', (buff + read) - p);
+                    if (p) {
+                        //printf("Trail Line: %p, %p, %d\n", pp, p, (p - pp) + trail_size);
+                        ++p;
+                        pp = p;
+                        ++cur_line;
+                        ++out_lines;
+                        // write out to file
+                        apr_file_write(outfile, trail, &trail_size);
+                        trail_size = 0;
+                    } else {
+                        if ((read + trail_size) > 2048) {
+                            logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Excessively long line %d in file %s", cur_line, filelist[f].fname);
+                            exit(1);
+                        } else {
+                            memcpy(trail+trail_size, buff, read);
+                            trail_size += read;
+                        }
+                    }
+                }
+                while ((p = memchr(p, '\n', (buff + read) - p))) {
+                    //printf("Line: %p, %p, %d\n", pp, p, (p - pp));
+                    if (out_lines == piecesize) {
+                        // Write out to file
+                        write = pp - buff_start;
+                        apr_file_write(outfile, buff_start, &write);
+                        buff_start = pp;
+                        out_lines = 0;
+                        // Open new file
+                        file = apr_psprintf(tfp, "%s/%s-%d", cfg->split_dir, basefile, file_count++);
+                        logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Creating output file %s", file);
+                        rv = apr_file_open(&outfile, file, APR_FOPEN_WRITE | APR_FOPEN_CREATE | APR_FOPEN_TRUNCATE, APR_OS_DEFAULT, tfp);
+                        if (rv != APR_SUCCESS) {
+                            logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s (%d)", file, rv);
+                            return;
+                        }
+                        newfile = (config_filestat_t *)apr_array_push(cfg->input_files);
+                        newfile->result = "Not Parsed";
+                        newfile->fname = apr_pstrdup(cfg->pool, file);
+                    }
+                    ++p;
+                    pp = p;
+                    ++cur_line;
+                    ++out_lines;
+                }
+                // Write out to file
+                write = pp - buff_start;
+                apr_file_write(outfile, buff_start, &write);
+
+                trail_size = (buff+read) - pp;
+                if (trail_size) {
+                    memcpy(trail, pp, trail_size);
+                }
+            }
+        }
+    }
+    apr_pool_destroy(tfp);
+    apr_pool_destroy(tp);
+}
+
+apr_status_t parser_logbadline(config_t *cfg, const char *filename,
+        const char *badline)
+{
+    apr_status_t rv = APR_SUCCESS;
+    apr_size_t len;
+    struct iovec vec[5];
+
+    if (cfg->badlinefile) {
+        if (!cfg->badline_fp) {
+            rv = apr_file_open(&cfg->badline_fp, cfg->badlinefile,
+                    APR_FOPEN_CREATE | APR_FOPEN_WRITE | APR_FOPEN_APPEND,
+                    APR_OS_DEFAULT, cfg->pool);
+            if (rv) {
+                logging_log(cfg, LOGLEVEL_NOISE,
+                        "Error opening badline file %s\n", cfg->badlinefile);
+                cfg->badlinefile = NULL;
+            }
+        }
+        if (!rv) {
+            if (filename != cfg->badlastfile){
+                char date[APR_RFC822_DATE_LEN];
+                vec[0].iov_base = "Starting BadLines for \"";
+                vec[0].iov_len = sizeof("Starting BadLines for \"")-1;
+                vec[1].iov_base = (void *)filename;
+                vec[1].iov_len = strlen(filename);
+                vec[2].iov_base = "\" on ";
+                vec[2].iov_len = sizeof("\" on ")-1;
+                apr_rfc822_date(date, apr_time_now());
+                vec[3].iov_base = date;
+                vec[3].iov_len = APR_RFC822_DATE_LEN-1;
+                vec[4].iov_base = "\n";
+                vec[4].iov_len = 1;
+                apr_file_writev(cfg->badline_fp, vec,5, &len);
+                cfg->badlastfile = filename;
+            }
+
+            if ((++cfg->badline_count) > cfg->badlinemax) {
+                logging_log(cfg, LOGLEVEL_NOISE,
+                        "Found more than %d bad lines (found %d)",
+                        cfg->badlinemax, cfg->badline_count);
+                rv = APR_EINVAL;
+            } else {
+                vec[0].iov_base = (void *)badline;
+                vec[0].iov_len = strlen(badline);
+                vec[1].iov_base = "\n";
+                vec[1].iov_len = 1;
+                apr_file_writev(cfg->badline_fp, vec,2, &len);
+            }
+        }
+    }
+    return rv;
+}
+
+/*
+ * Modified version of apr_tokenize_to_argv to add [] as quoting characters
+ *
+ *    token_context: Context from which pool allocations will occur.
+ *    arg_str:       Input string for conversion to argv[].
+ *    argv_out:      Output location. This is a pointer to an array
+ *                   of pointers to strings (ie. &(char *argv[]).
+ *                   This value will be allocated from the contexts
+ *                   pool and filled in with copies of the tokens
+ *                   found during parsing of the arg_str.
+ */
+apr_status_t parser_tokenize_line(const char *arg_str, char ***argv_out,
+        apr_pool_t *token_context)
+{
+    const char *cp;
+    const char *ct;
+    char *cleaned, *dirty;
+    int escaped;
+    int isquoted, numargs = 0, argnum;
+
+#define SKIP_WHITESPACE(cp) \
+    for ( ; *cp == ' ' || *cp == '\t'; ) { \
+        cp++; \
+    };
+
+#define CHECK_QUOTATION(cp,isquoted) \
+    isquoted = 0; \
+    if (*cp == '"') { \
+        isquoted = 1; \
+        cp++; \
+    } \
+    else if (*cp == '\'') { \
+        isquoted = 2; \
+        cp++; \
+    } \
+    else if (*cp == '[') { \
+        isquoted = 3; \
+        cp++; \
+    }
+
+    /* DETERMINE_NEXTSTRING:
+     * At exit, cp will point to one of the following:  NULL, SPACE, TAB or QUOTE.
+     * NULL implies the argument string has been fully traversed.
+     */
+#define DETERMINE_NEXTSTRING(cp,isquoted) \
+    for ( ; *cp != '\0'; cp++) { \
+        if (   (*cp == '\\' && (*(cp+1) == ' ' || *(cp+1) == '\t' || \
+                                *(cp+1) == '"' || *(cp+1) == '\'' || \
+                                *(cp+1) == '[' || *(cp+1) == ']'))) { \
+            cp++; \
+            continue; \
+        } \
+        if (   (!isquoted && (*cp == ' ' || *cp == '\t')) \
+            || (isquoted == 1 && *cp == '"') \
+            || (isquoted == 2 && *cp == '\'') \
+            || (isquoted == 3 && *cp == ']') \
+            ) { \
+            break; \
+        } \
+    }
+
+    /* REMOVE_ESCAPE_CHARS:
+     * Compresses the arg string to remove all of the '\' escape chars.
+     * The final argv strings should not have any extra escape chars in it.
+     */
+#define REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped) \
+    escaped = 0; \
+    while(*dirty) { \
+        if (!escaped && *dirty == '\\') { \
+            escaped = 1; \
+        } \
+        else { \
+            escaped = 0; \
+            *cleaned++ = *dirty; \
+        } \
+        ++dirty; \
+    } \
+    *cleaned = 0;        /* last line of macro... */
+
+    cp = arg_str;
+    SKIP_WHITESPACE(cp);
+    ct = cp;
+
+    /* This is ugly and expensive, but if anyone wants to figure a
+     * way to support any number of args without counting and
+     * allocating, please go ahead and change the code.
+     *
+     * Must account for the trailing NULL arg.
+     */
+    numargs = 1;
+    while (*ct != '\0') {
+        CHECK_QUOTATION(ct, isquoted)
+        ;
+        DETERMINE_NEXTSTRING(ct, isquoted);
+        if (*ct != '\0') {
+            ct++;
+        }
+        numargs++;
+        SKIP_WHITESPACE(ct);
+    }
+    *argv_out = apr_palloc(token_context, numargs * sizeof(char*));
+
+    /*  determine first argument */
+    for (argnum = 0; argnum < (numargs-1); argnum++) {
+        SKIP_WHITESPACE(cp);
+        CHECK_QUOTATION(cp, isquoted)
+        ;
+        ct = cp;
+        DETERMINE_NEXTSTRING(cp, isquoted);
+        cp++;
+        (*argv_out)[argnum] = apr_palloc(token_context, cp - ct);
+        apr_cpystrn((*argv_out)[argnum], ct, cp - ct);
+        cleaned = dirty = (*argv_out)[argnum];
+        REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped)
+        ;
+    }
+    (*argv_out)[argnum] = NULL;
+
+    return APR_SUCCESS;
+}
+
+apr_status_t parser_parsefile(config_t *cfg, config_dbd_t *dbconn,
+        config_filestat_t *fstat)
+{
+    apr_pool_t *tp, *targp;
+    apr_file_t *file;
+    apr_status_t rv;
+    char buff[2048];
+    char readbuff[BUFFER_SIZE];
+    char **targv;
+    int targc;
+
+    apr_pool_create(&tp, cfg->pool);
+    apr_pool_create(&targp, tp);
+
+    logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", fstat->fname);
+
+    rv = apr_file_open(&file, fstat->fname, APR_FOPEN_READ, APR_OS_DEFAULT, tp);
+    apr_file_buffer_set(file, readbuff, BUFFER_SIZE);
+    if (rv != APR_SUCCESS) {
+        logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", fstat->fname);
+        return rv;
+    }
+
+    fstat->linesparsed = 0;
+    // Start Transaction
+    fstat->start = apr_time_now();
+    if (!cfg->dryrun && database_trans_start(cfg, dbconn, tp)) {
+        fstat->result = "Database Transaction Error";
+        fstat->stop = apr_time_now();
+        return rv;
+    }
+
+    do {
+        rv = apr_file_gets(buff, 2048, file);
+        if (rv == APR_SUCCESS) {
+            int i,m, cont = 0;
+            config_filter_t *filters;
+
+            fstat->linesparsed++;
+            // chomp off newline
+            line_chomp(buff);
+            // Run line filters
+            for (i=0, m=cfg->linefilters->nelts,
+                    filters = (config_filter_t *)cfg->linefilters->elts;
+                    i<m; i++) {
+                if (!filters[i].regex || ap_regexec(filters[i].regex, buff, 0, NULL,0)==0) {
+                    if (filters[i].negative) {
+                        logging_log(cfg, LOGLEVEL_DEBUG,
+                                "PARSER: LINEFILTER: Skipping Line %d due to Filter (%d)%s",
+                                fstat->linesparsed, i, filters[i].filter);
+                        fstat->lineskipped++;
+                        cont = 1;
+                    } else {
+                        logging_log(cfg, LOGLEVEL_DEBUG,
+                                "PARSER: LINEFILTER: Force Parsing Line %d due to Filter (%d)%s",
+                                fstat->linesparsed, i, filters[i].filter);
+                    }
+                    break;
+                }
+            }
+            if (cont) continue;
+
+            apr_pool_clear(targp);
+            parser_tokenize_line(buff, &targv, targp);
+            targc = 0;
+            while (targv[targc])
+                targc++;
+            rv = parser_processline(targp, cfg, dbconn, fstat, targv, targc);
+            if (rv != APR_SUCCESS) {
+                int i;
+
+                fstat->linesbad++;
+                rv = parser_logbadline(cfg, fstat->fname, buff);
+                if (rv) {
+                    if (!cfg->dryrun) database_trans_abort(cfg, dbconn);
+                    logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed,
+                            targc, buff);
+                    for (i = 0; targv[i]; i++) {
+                        logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i,
+                                targv[i]);
+                    }
+                }
+            }
+        } else {
+            rv = APR_SUCCESS;
+            break;
+        }
+    } while (rv == APR_SUCCESS);
+    apr_file_close(file);
+    // Finish Transaction
+    if (!cfg->dryrun && database_trans_stop(cfg, dbconn, tp)) {
+        fstat->result = apr_psprintf(cfg->pool,
+                "Input line %d, Database Transaction Error",
+                fstat->linesparsed);
+    }
+
+    apr_pool_destroy(tp);
+    logging_log(cfg, LOGLEVEL_NOTICE,
+            "PARSER: Finish Parsing Log File '%s'. Lines: (%d/%d)",
+            fstat->fname, fstat->linesparsed - fstat->lineskipped, fstat->linesparsed);
+    if (!rv) {
+        fstat->result = "File Parsed Succesfully";
+    }
+    fstat->stop = apr_time_now();
+    return rv;
+}
+
+apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg,
+        config_dbd_t *dbconn, config_filestat_t *fstat, char **argv, int argc)
+{
+    config_logformat_t *fmt;
+    config_logformat_field_t *ifields;
+    config_output_field_t *ofields;
+    config_filter_t *filters;
+    apr_table_t *datain;
+    apr_table_t *dataout;
+    apr_status_t rv= APR_SUCCESS;
+    int i,m;
+
+    fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING);
+    if (!fmt) {
+        logging_log(cfg, LOGLEVEL_NOISE, "PARSER: No Input Log format");
+        return APR_EINVAL;
+    }
+    if (fmt->fields->nelts != argc) {
+        logging_log(cfg, LOGLEVEL_NOISE,
+                "PARSER: Input line field number differs from expected. Expected %d got %d.",
+                fmt->fields->nelts, argc);
+        fstat->result = apr_psprintf(cfg->pool,
+                "Input line %d is badly formatted (wrong number of fields)",
+                fstat->linesparsed);
+        return APR_EINVAL;
+    }
+
+    datain = apr_table_make(ptemp, fmt->fields->nelts);
+    dataout = apr_table_make(ptemp, cfg->output_fields->nelts);
+
+    ifields = (config_logformat_field_t *)fmt->fields->elts;
+    for (i=0; i<fmt->fields->nelts; i++) {
+        apr_table_setn(datain, ifields[i].name, argv[i]);
+    }
+    // Run Pre Filters
+    for (i=0, m=cfg->prefilters->nelts,
+            filters = (config_filter_t *)cfg->prefilters->elts;
+            i<m; i++) {
+        const char *temp = apr_table_get(datain, filters[i].field);
+        if (temp && (!filters[i].regex || ap_regexec(filters[i].regex, temp, 0, NULL,0)==0)) {
+            if (filters[i].negative) {
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                        "PARSER: PREFILTER: Skipping Line %d due to Filter (%d)%s",
+                        fstat->linesparsed, i, filters[i].filter);
+                fstat->lineskipped++;
+                return APR_SUCCESS;
+            } else {
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                        "PARSER: PREFILTER: Force Parsing Line %d due to Filter (%d)%s",
+                        fstat->linesparsed, i, filters[i].filter);
+            }
+            break;
+        }
+    }
+
+    ofields = (config_output_field_t *)cfg->output_fields->elts;
+    // clear out ofield function per-line data
+    memset(&g_parser_linedata[1],0,sizeof(void *)*(int)g_parser_linedata[0]);
+    // Convert input fields to output fields
+    for (i=0,m=cfg->output_fields->nelts; i<m; i++) {
+        const char *val;
+        val = apr_table_get(datain, ofields[i].source);
+        // If we can't find the source field just continue
+        if (!val && !(ofields[i].source[0]=='\0' && ofields[i].func)) {
+            apr_table_setn(dataout, ofields[i].field, ofields[i].def);
+            continue;
+        }
+        if (!ofields[i].func) {
+            apr_table_setn(dataout, ofields[i].field, val);
+        } else {
+            const char *ret= NULL;
+            rv = ((parser_func_t *)ofields[i].func)->func(ptemp, cfg,
+                    &ofields[i], val, &ret);
+            if (rv) {
+                fstat->result = apr_psprintf(cfg->pool,
+                        "Input line %d, Parser function %s returned error (%d)%s",
+                        fstat->linesparsed, ofields[i].fname, rv, logging_strerror(rv));
+                return rv;
+            }
+            apr_table_setn(dataout, ofields[i].field, ret ? ret : ofields[i].def);
+        }
+    }
+
+    // Run Post filters
+    for (i=0, m=cfg->postfilters->nelts,
+            filters = (config_filter_t *)cfg->postfilters->elts;
+            i<m; i++) {
+        const char *temp = apr_table_get(dataout, filters[i].field);
+        if (temp && (!filters[i].regex || ap_regexec(filters[i].regex, temp, 0, NULL,0)==0)) {
+            if (filters[i].negative) {
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                        "PARSER: POSTFILTER: Skipping Line %d due to Filter (%d)%s",
+                        fstat->linesparsed, i, filters[i].filter);
+                fstat->lineskipped++;
+                return APR_SUCCESS;
+            } else {
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                        "PARSER: POSTFILTER: Force Parsing Line %d due to Filter (%d)%s",
+                        fstat->linesparsed, i, filters[i].filter);
+            }
+            break;
+        }
+    }
+
+    // Process DB Query
+    if (!cfg->dryrun) {
+        rv = database_insert(cfg, dbconn, ptemp, dataout);
+        if (rv) {
+            fstat->result = apr_psprintf(cfg->pool,
+                    "Input line %d, Database Error",
+                    fstat->linesparsed);
+        }
+    }
+    return rv;
+}