implement filter support

add in check for Apache 1.3 apr_dbd_transaction_mode_set method
add transaction stubs

10 files changed, 177 insertions, 33 deletions

diff --git a/utility/Makefile.in b/utility/Makefile.in
index eeb26b8..c7a5e3d 100644
--- a/utility/Makefile.in
+++ b/utility/Makefile.in
@@ -6,7 +6,8 @@ builddir = @abs_builddir@
 
 #@APR_CFLAGS@
 CFLAGS = -g3 -Wall -fno-strict-aliasing \
-         @APR_INCLUDES@ @APU_INCLUDES@ @PCRE_CFLAGS@
+         @APR_INCLUDES@ @APU_INCLUDES@ @PCRE_CFLAGS@ \
+         -I$(top_srcdir)/include
 CPPFLAGS = @APR_CPPFLAGS@
 LDFLAGS = @APR_LDFLAGS@ @APU_LDFLAGS@ @PCRE_LIBS@
 
diff --git a/utility/config.c b/utility/config.c
index fca2f77..28bb5cd 100644
--- a/utility/config.c
+++ b/utility/config.c
@@ -9,6 +9,7 @@
 #include "config.h"
 #include "util.h"
 #include "logparse.h"
+#include "autoconfig.h"
 
 apr_hash_t *g_config_opts;
 
@@ -148,9 +149,19 @@ static apr_status_t config_set_filter(config_t *cfg, config_opt_t *opt,
 {
     int argn = 1;
     config_filter_t *filter;
-    filter = apr_pcalloc(cfg->pool, sizeof(config_filter_t));
+    switch (opt->name[1]) {
+    case 'i': //line
+        filter = apr_array_push(cfg->linefilters);
+        break;
+    case 'r': //pre
+        filter = apr_array_push(cfg->prefilters);
+        break;
+    case 'o': //post
+        filter = apr_array_push(cfg->postfilters);
+        break;
+    }
 
-    if (opt->name[0]!='L') { // Pre or post 2-3 args
+    if (opt->name[0]=='P') { // Pre or post 2-3 args
         if (argc == 1)
             return APR_EINVAL;
         filter->field = apr_pstrdup(cfg->pool, argv[1]);
@@ -158,16 +169,20 @@ static apr_status_t config_set_filter(config_t *cfg, config_opt_t *opt,
     } // Otherwise Line based only 1-2 args (no field)
     if (argc <= argn)
         return APR_EINVAL;
-    if (argv[argn][0] == '+')
+    if (*argv[argn] == '+')
         argn++;
-    if (argv[argn][0] == '-') {
+    if (*argv[argn] == '-') {
         filter->negative = 1;
         argn++;
     }
+    if (filter->negative && argc == argn) {
+        // if no filter for negative.. that's ok.. Assume ALL
+        return APR_SUCCESS;
+    }
     if (argc <= argn)
         return APR_EINVAL;
     filter->filter = apr_pstrdup(cfg->pool, argv[argn]);
-    filter->regex = ap_pregcomp(cfg->pool, filter->filter, AP_REG_ICASE);
+    filter->regex = ap_pregcomp(cfg->pool, filter->filter, AP_REG_EXTENDED|AP_REG_ICASE);
     return APR_SUCCESS;
 }
 
@@ -176,6 +191,7 @@ void config_dump(config_t *cfg)
     apr_hash_index_t *hi;
     int i;
     config_output_field_t *fields;
+    config_filter_t *filters;
 
     printf("ErrorLog: %s\n", cfg->errorlog);
     printf("LogLevel: %d\n", cfg->loglevel);
@@ -224,6 +240,26 @@ void config_dump(config_t *cfg)
         }
         printf("\n");
     }
+    printf("Filters:\n>> Line:\n");
+    filters = cfg->linefilters->elts;
+    for (i=0; i<cfg->linefilters->nelts; i++) {
+        printf(">>>> %c \"%s\" (%pp)\n",filters[i].negative ? '-':'+',
+                filters[i].filter,  filters[i].regex);
+    }
+    printf(">> Pre:\n");
+    filters = cfg->prefilters->elts;
+    for (i=0; i<cfg->prefilters->nelts; i++) {
+        printf(">>>> %s %c \"%s\" (%pp)\n",
+                filters[i].field, filters[i].negative ? '-':'+',
+                filters[i].filter,  filters[i].regex);
+    }
+    printf(">> Post:\n");
+    filters = cfg->postfilters->elts;
+    for (i=0; i<cfg->postfilters->nelts; i++) {
+        printf(">>>> %s %c \"%s\" (%pp)\n",
+                filters[i].field, filters[i].negative ? '-':'+',
+                filters[i].filter,  filters[i].regex);
+    }
 
     printf("DryRun: %d\n", cfg->dryrun);
     printf("Summary: %d\n", cfg->summary);
@@ -309,10 +345,13 @@ config_t *config_create(apr_pool_t *p)
     cfg->loglevel = LOGLEVEL_ERROR;
     cfg->summary = 1;
     cfg->transactions = 1;
-    cfg->input_files = apr_array_make(cfg->pool, 10, sizeof(char *));
+    cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(char *));
     cfg->log_formats = apr_hash_make(cfg->pool);
     cfg->output_fields = apr_array_make(cfg->pool, 10,
             sizeof(config_output_field_t));
+    cfg->linefilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t));
+    cfg->prefilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t));
+    cfg->postfilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t));
     return cfg;
 }
 
@@ -335,6 +374,12 @@ apr_status_t config_check(config_t *cfg)
         logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: No Input Log Formats Defined");
         ret = APR_EINVAL;
     }
+#if !defined(HAVE_APR_DBD_TRANSACTION_MODE_GET)
+    if (cfg->transactions) {
+        logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: Disabling Transaction Support.  Requires apr-util 1.3.0 or higher");
+        cfg->transactions = 0;
+    }
+#endif
     return ret;
 }
 
diff --git a/utility/database.c b/utility/database.c
index c4e4bc9..0d1d4f7 100644
--- a/utility/database.c
+++ b/utility/database.c
@@ -4,7 +4,7 @@
 #include "apr_strings.h"
 
 #include "util.h"
-#include "mysql/mysql.h"
+#include "autoconfig.h"
 
 struct config_dbd_t {
     const apr_dbd_driver_t *driver;
@@ -30,7 +30,7 @@ apr_status_t database_connect(config_t *cfg)
     if (rv) {
 
         logging_log(cfg, LOGLEVEL_ERROR,
-                "Could not load database driver %s. Error %s", cfg->dbdriver,
+                "DB: Could not load database driver %s. Error %s", cfg->dbdriver,
                 logging_strerror(rv));
         return rv;
     }
@@ -39,7 +39,7 @@ apr_status_t database_connect(config_t *cfg)
             &(cfg->dbconn->dbd));
     if (rv) {
         logging_log(cfg, LOGLEVEL_ERROR,
-                "Could not connect to database. Error %s", logging_strerror(rv));
+                "DB: Could not connect to database. Error %s", logging_strerror(rv));
         return rv;
     }
 
@@ -89,13 +89,13 @@ static apr_dbd_prepared_t *database_prepare_insert(config_t *cfg, apr_pool_t *p)
 
     sql = apr_pstrcatv(p, vec, i+2, NULL);
 
-    logging_log(cfg, LOGLEVEL_DEBUG, "Generated SQL: %s", sql);
+    logging_log(cfg, LOGLEVEL_DEBUG, "DB: Generated SQL: %s", sql);
 
     rv = apr_dbd_prepare(cfg->dbconn->driver, cfg->pool, cfg->dbconn->dbd, sql,
             "INSERT", &stmt);
 
     if (rv) {
-        logging_log(cfg, LOGLEVEL_ERROR, "Unable to Prepare SQL insert: %s",
+        logging_log(cfg, LOGLEVEL_NOISE, "DB: Unable to Prepare SQL insert: %s",
                 apr_dbd_error(cfg->dbconn->driver, cfg->dbconn->dbd, rv));
         return NULL;
     }
@@ -112,7 +112,6 @@ apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data)
     if (!cfg->dbconn->stmt) {
         cfg->dbconn->stmt = database_prepare_insert(cfg, p);
         if (!cfg->dbconn->stmt) {
-            logging_log(cfg, LOGLEVEL_NOISE, "Unable to prepare SQL statement");
             return APR_EINVAL;
         }
         cfg->dbconn->args = apr_palloc(cfg->pool, nfs * sizeof(char *));
@@ -123,7 +122,7 @@ apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data)
     rv = apr_dbd_pquery(cfg->dbconn->driver, p, cfg->dbconn->dbd, &f,
             cfg->dbconn->stmt, nfs, cfg->dbconn->args);
     if (rv) {
-        logging_log(cfg, LOGLEVEL_ERROR, "Unable to Insert SQL: %s",
+        logging_log(cfg, LOGLEVEL_ERROR, "DB: Unable to Insert SQL: %s",
                 apr_dbd_error(cfg->dbconn->driver, cfg->dbconn->dbd, rv));
         return rv;
     }
@@ -131,3 +130,26 @@ apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data)
 }
 
 /** @todo implement transactions */
+apr_status_t database_trans_start(config_t *cfg, apr_pool_t *p)
+{
+#if HAVE_APR_DBD_TRANSACTION_MODE_GET
+#else
+    return APR_SUCCESS;
+#endif
+}
+
+apr_status_t database_trans_stop(config_t *cfg, apr_pool_t *p)
+{
+#if HAVE_APR_DBD_TRANSACTION_MODE_GET
+#else
+    return APR_SUCCESS;
+#endif
+}
+
+apr_status_t database_trans_abort(config_t *cfg)
+{
+#if HAVE_APR_DBD_TRANSACTION_MODE_GET
+#else
+    return APR_SUCCESS;
+#endif
+}
diff --git a/utility/database.h b/utility/database.h
index fd24994..eed8898 100644
--- a/utility/database.h
+++ b/utility/database.h
@@ -13,4 +13,10 @@ apr_status_t database_disconnect(config_t *cfg);
 
 apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data);
 
+apr_status_t database_trans_start(config_t *cfg, apr_pool_t *p);
+
+apr_status_t database_trans_stop(config_t *cfg, apr_pool_t *p);
+
+apr_status_t database_trans_abort(config_t *cfg);
+
 #endif /*DATABASE_H_*/
diff --git a/utility/logparse.c b/utility/logparse.c
index 6a21964..ac70ee2 100644
--- a/utility/logparse.c
+++ b/utility/logparse.c
@@ -349,12 +349,12 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
     apr_pool_create(&tp, cfg->pool);
     apr_pool_create(&targp, tp);
 
-    logging_log(cfg, LOGLEVEL_NOTICE, "Begin Parsing Log File '%s'", filename);
+    logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", filename);
 
     rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED,
     APR_OS_DEFAULT, tp);
     if (rv != APR_SUCCESS) {
-        logging_log(cfg, LOGLEVEL_NOISE, "Could not open %s", filename);
+        logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", filename);
         return rv;
     }
 
@@ -362,17 +362,38 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
     do {
         rv = apr_file_gets(buff, 1024, file);
         if (rv == APR_SUCCESS) {
+            int i,m, cont = 0;
+            config_filter_t *filters;
+
             line++;
             // chomp off newline
             line_chomp(buff);
+            // Run line filters
+            for (i=0, m=cfg->linefilters->nelts,
+                    filters = (config_filter_t *)cfg->linefilters->elts;
+                    i<m; i++) {
+                if (!filters[i].regex || ap_regexec(filters[i].regex, buff, 0, NULL,0)==0) {
+                    if (filters[i].negative) {
+                        logging_log(cfg, LOGLEVEL_DEBUG,
+                                "PARSER: LINEFILTER: Skipping Line %d due to Filter (%d)%s",
+                                line, i, filters[i].filter);
+                        cont = 1;
+                    } else {
+                        logging_log(cfg, LOGLEVEL_DEBUG,
+                                "PARSER: LINEFILTER: Force Parsing Line %d due to Filter (%d)%s",
+                                line, i, filters[i].filter);
+                    }
+                    break;
+                }
+            }
+            if (cont) continue;
 
             apr_pool_clear(targp);
             tokenize_logline(buff, &targv, targp, 0);
             targc = 0;
             while (targv[targc])
                 targc++;
-            /** @todo Run Line Filters here */
-            rv = parse_processline(targp, cfg, targv, targc);
+            rv = parse_processline(targp, cfg, line, targv, targc);
             if (rv != APR_SUCCESS) {
                 int i;
                 logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", line,
@@ -387,21 +408,22 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
     apr_file_close(file);
     apr_pool_destroy(tp);
     logging_log(cfg, LOGLEVEL_NOTICE,
-            "Finish Parsing Log File '%s'. Lines: %d", filename, line);
+            "PARSER: Finish Parsing Log File '%s'. Lines: %d", filename, line);
 
     return APR_SUCCESS;
 }
 
-apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv,
-        int argc)
+apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
+        char **argv, int argc)
 {
     config_logformat_t *fmt;
     config_logformat_field_t *ifields;
     config_output_field_t *ofields;
+    config_filter_t *filters;
     apr_table_t *datain;
     apr_table_t *dataout;
     apr_status_t rv= APR_SUCCESS;
-    int i;
+    int i,m;
 
     fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING);
     if (!fmt)
@@ -416,13 +438,31 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv,
     for (i=0; i<fmt->fields->nelts; i++) {
         apr_table_setn(datain, ifields[i].name, argv[i]);
     }
-    /** @todo Run Pre Filters here */
+    // Run Pre Filters
+    for (i=0, m=cfg->prefilters->nelts,
+            filters = (config_filter_t *)cfg->prefilters->elts;
+            i<m; i++) {
+        const char *temp = apr_table_get(datain, filters[i].field);
+        if (temp && (!filters[i].regex || ap_regexec(filters[i].regex, temp, 0, NULL,0)==0)) {
+            if (filters[i].negative) {
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                        "PARSER: PREFILTER: Skipping Line %d due to Filter (%d)%s",
+                        line, i, filters[i].filter);
+                return APR_SUCCESS;
+            } else {
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                        "PARSER: PREFILTER: Force Parsing Line %d due to Filter (%d)%s",
+                        line, i, filters[i].filter);
+            }
+            break;
+        }
+    }
 
     ofields = (config_output_field_t *)cfg->output_fields->elts;
     // clear out ofield function per-line data
     memset(&g_parser_linedata[1],0,sizeof(void *)*(int)g_parser_linedata[0]);
     // Convert input fields to output fields
-    for (i=0; i<cfg->output_fields->nelts; i++) {
+    for (i=0,m=cfg->output_fields->nelts; i<m; i++) {
         const char *val;
         val = apr_table_get(datain, ofields[i].source);
         // If we can't find the source field just continue
@@ -442,7 +482,25 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv,
         }
     }
 
-    /** @todo Run Post Filters here */
+    // Run Post filters
+    for (i=0, m=cfg->postfilters->nelts,
+            filters = (config_filter_t *)cfg->postfilters->elts;
+            i<m; i++) {
+        const char *temp = apr_table_get(dataout, filters[i].field);
+        if (temp && (!filters[i].regex || ap_regexec(filters[i].regex, temp, 0, NULL,0)==0)) {
+            if (filters[i].negative) {
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                        "PARSER: POSTFILTER: Skipping Line %d due to Filter (%d)%s",
+                        line, i, filters[i].filter);
+                return APR_SUCCESS;
+            } else {
+                logging_log(cfg, LOGLEVEL_DEBUG,
+                        "PARSER: POSTFILTER: Force Parsing Line %d due to Filter (%d)%s",
+                        line, i, filters[i].filter);
+            }
+            break;
+        }
+    }
 
     // Process DB Query
     if (!cfg->dryrun) {
diff --git a/utility/logparse.h b/utility/logparse.h
index bc39cb1..0b8607b 100644
--- a/utility/logparse.h
+++ b/utility/logparse.h
@@ -25,7 +25,7 @@ void parser_find_logs(config_t *cfg);
 
 apr_status_t parse_logfile(config_t *cfg, const char *filename);
 
-apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv,
-        int argc);
+apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
+        char **argv, int argc);
 
 #endif /*LOGPARSE_H_*/
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
index ee0cb07..fde7cc6 100644
--- a/utility/mod_log_sql.conf
+++ b/utility/mod_log_sql.conf
@@ -29,9 +29,12 @@ LogFormatConfig Combined agent String
 
 LogFormat Combined
 
-# not yet implemented
 Linefilter - "BAD"
-PreFilter request - "GET \/images"
+LineFilter "GOOD"
+LineFilter + "BETTER"
+# the next filter ignores ALL lines
+LineFilter -
+PreFilter request + "Rebuild"
 PostFilter request_method "GET"
 
 # Usage     field            datatype(size) default source      [function [param]...]
@@ -41,7 +44,8 @@ OutputField remote_host      varchar(50) ""         remhost
 OutputField request_method   varchar(25) ""         request     regexmatch "^(\\w+)"
 OutputField time_stamp       int 0                  date        totimestamp
 OutputField status           smallint 0             status
-OutputField request_uri      varchar(255) ""        request     regexmatch "^\\w+ (.+) \\w+/[\\d\\.]+$"
+OutputField request_line     varchar(255) ""        request
+#OutputField request_uri      varchar(255) ""        request     regexmatch "^\\w+ (.+) \\w+/[\\d\\.]+$"
 OutputField remote_user      varchar(50) ""         user
 OutputField remote_logname   varchar(50) ""         ident
 OutputField request_time     char(28) ""            date        wrap "[" "]"
diff --git a/utility/shell.c b/utility/shell.c
index e521916..3de65b3 100644
--- a/utility/shell.c
+++ b/utility/shell.c
@@ -134,6 +134,8 @@ int main(int argc, const char *const argv[])
     database_init(pool);
     // Process configuration file
     cfg = config_create(pool);
+    // initialize STD out error log
+    logging_preinit(cfg);
     rv = config_read(cfg, apr_table_get(args,"Config"), args);
     apr_pool_destroy(ptemp);
 
diff --git a/utility/util.c b/utility/util.c
index 7ecb902..45b95ba 100644
--- a/utility/util.c
+++ b/utility/util.c
@@ -124,11 +124,15 @@ int ap_unescape_url(char *url)
 #endif
 }
 
-void logging_init(config_t *cfg)
+void logging_preinit(config_t *cfg)
 {
-    apr_status_t rv;
     apr_pool_create(&cfg->errorlog_p, cfg->pool);
     apr_file_open_stderr(&cfg->errorlog_fperr, cfg->pool);
+}
+
+void logging_init(config_t *cfg)
+{
+    apr_status_t rv;
     if (cfg->errorlog) {
         rv = apr_file_open(&cfg->errorlog_fp, cfg->errorlog,
                 APR_FOPEN_CREATE | APR_FOPEN_WRITE | APR_FOPEN_APPEND,
@@ -170,7 +174,7 @@ void logging_log(config_t *cfg, loglevel_e level, const char *fmt, ...)
     if (level == LOGLEVEL_NOISE) {
         apr_file_writev(cfg->errorlog_fperr,vec,2,&blen);
     }
-    if (cfg->loglevel > LOGLEVEL_NONE) {
+    if (cfg->loglevel > LOGLEVEL_NONE && cfg->errorlog_fp) {
         apr_file_writev(cfg->errorlog_fp,vec,2,&blen);
     }
 
diff --git a/utility/util.h b/utility/util.h
index 99f93aa..a5a7f7e 100644
--- a/utility/util.h
+++ b/utility/util.h
@@ -14,6 +14,8 @@ void line_chomp(char *str);
 
 int ap_unescape_url(char *url);
 
+void logging_preinit(config_t *cfg);
+
 void logging_init(config_t *cfg);
 
 void logging_log(config_t *cfg, loglevel_e level, const char *fmt, ...)