From 017c8fdf246df82385fb0222890be85c461d0275 Mon Sep 17 00:00:00 2001 From: Edward Rudd Date: Sun, 26 Oct 2008 00:11:22 +0000 Subject: implement filter support add in check for Apache 1.3 apr_dbd_transaction_mode_set method add transaction stubs --- utility/Makefile.in | 3 +- utility/config.c | 57 ++++++++++++++++++++++++++++++---- utility/database.c | 36 +++++++++++++++++----- utility/database.h | 6 ++++ utility/logparse.c | 80 +++++++++++++++++++++++++++++++++++++++++------- utility/logparse.h | 4 +-- utility/mod_log_sql.conf | 10 ++++-- utility/shell.c | 2 ++ utility/util.c | 10 ++++-- utility/util.h | 2 ++ 10 files changed, 177 insertions(+), 33 deletions(-) diff --git a/utility/Makefile.in b/utility/Makefile.in index eeb26b8..c7a5e3d 100644 --- a/utility/Makefile.in +++ b/utility/Makefile.in @@ -6,7 +6,8 @@ builddir = @abs_builddir@ #@APR_CFLAGS@ CFLAGS = -g3 -Wall -fno-strict-aliasing \ - @APR_INCLUDES@ @APU_INCLUDES@ @PCRE_CFLAGS@ + @APR_INCLUDES@ @APU_INCLUDES@ @PCRE_CFLAGS@ \ + -I$(top_srcdir)/include CPPFLAGS = @APR_CPPFLAGS@ LDFLAGS = @APR_LDFLAGS@ @APU_LDFLAGS@ @PCRE_LIBS@ diff --git a/utility/config.c b/utility/config.c index fca2f77..28bb5cd 100644 --- a/utility/config.c +++ b/utility/config.c @@ -9,6 +9,7 @@ #include "config.h" #include "util.h" #include "logparse.h" +#include "autoconfig.h" apr_hash_t *g_config_opts; @@ -148,9 +149,19 @@ static apr_status_t config_set_filter(config_t *cfg, config_opt_t *opt, { int argn = 1; config_filter_t *filter; - filter = apr_pcalloc(cfg->pool, sizeof(config_filter_t)); + switch (opt->name[1]) { + case 'i': //line + filter = apr_array_push(cfg->linefilters); + break; + case 'r': //pre + filter = apr_array_push(cfg->prefilters); + break; + case 'o': //post + filter = apr_array_push(cfg->postfilters); + break; + } - if (opt->name[0]!='L') { // Pre or post 2-3 args + if (opt->name[0]=='P') { // Pre or post 2-3 args if (argc == 1) return APR_EINVAL; filter->field = apr_pstrdup(cfg->pool, argv[1]); @@ -158,16 +169,20 @@ static apr_status_t config_set_filter(config_t *cfg, config_opt_t *opt, } // Otherwise Line based only 1-2 args (no field) if (argc <= argn) return APR_EINVAL; - if (argv[argn][0] == '+') + if (*argv[argn] == '+') argn++; - if (argv[argn][0] == '-') { + if (*argv[argn] == '-') { filter->negative = 1; argn++; } + if (filter->negative && argc == argn) { + // if no filter for negative.. that's ok.. Assume ALL + return APR_SUCCESS; + } if (argc <= argn) return APR_EINVAL; filter->filter = apr_pstrdup(cfg->pool, argv[argn]); - filter->regex = ap_pregcomp(cfg->pool, filter->filter, AP_REG_ICASE); + filter->regex = ap_pregcomp(cfg->pool, filter->filter, AP_REG_EXTENDED|AP_REG_ICASE); return APR_SUCCESS; } @@ -176,6 +191,7 @@ void config_dump(config_t *cfg) apr_hash_index_t *hi; int i; config_output_field_t *fields; + config_filter_t *filters; printf("ErrorLog: %s\n", cfg->errorlog); printf("LogLevel: %d\n", cfg->loglevel); @@ -224,6 +240,26 @@ void config_dump(config_t *cfg) } printf("\n"); } + printf("Filters:\n>> Line:\n"); + filters = cfg->linefilters->elts; + for (i=0; ilinefilters->nelts; i++) { + printf(">>>> %c \"%s\" (%pp)\n",filters[i].negative ? '-':'+', + filters[i].filter, filters[i].regex); + } + printf(">> Pre:\n"); + filters = cfg->prefilters->elts; + for (i=0; iprefilters->nelts; i++) { + printf(">>>> %s %c \"%s\" (%pp)\n", + filters[i].field, filters[i].negative ? '-':'+', + filters[i].filter, filters[i].regex); + } + printf(">> Post:\n"); + filters = cfg->postfilters->elts; + for (i=0; ipostfilters->nelts; i++) { + printf(">>>> %s %c \"%s\" (%pp)\n", + filters[i].field, filters[i].negative ? '-':'+', + filters[i].filter, filters[i].regex); + } printf("DryRun: %d\n", cfg->dryrun); printf("Summary: %d\n", cfg->summary); @@ -309,10 +345,13 @@ config_t *config_create(apr_pool_t *p) cfg->loglevel = LOGLEVEL_ERROR; cfg->summary = 1; cfg->transactions = 1; - cfg->input_files = apr_array_make(cfg->pool, 10, sizeof(char *)); + cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(char *)); cfg->log_formats = apr_hash_make(cfg->pool); cfg->output_fields = apr_array_make(cfg->pool, 10, sizeof(config_output_field_t)); + cfg->linefilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t)); + cfg->prefilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t)); + cfg->postfilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t)); return cfg; } @@ -335,6 +374,12 @@ apr_status_t config_check(config_t *cfg) logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: No Input Log Formats Defined"); ret = APR_EINVAL; } +#if !defined(HAVE_APR_DBD_TRANSACTION_MODE_GET) + if (cfg->transactions) { + logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: Disabling Transaction Support. Requires apr-util 1.3.0 or higher"); + cfg->transactions = 0; + } +#endif return ret; } diff --git a/utility/database.c b/utility/database.c index c4e4bc9..0d1d4f7 100644 --- a/utility/database.c +++ b/utility/database.c @@ -4,7 +4,7 @@ #include "apr_strings.h" #include "util.h" -#include "mysql/mysql.h" +#include "autoconfig.h" struct config_dbd_t { const apr_dbd_driver_t *driver; @@ -30,7 +30,7 @@ apr_status_t database_connect(config_t *cfg) if (rv) { logging_log(cfg, LOGLEVEL_ERROR, - "Could not load database driver %s. Error %s", cfg->dbdriver, + "DB: Could not load database driver %s. Error %s", cfg->dbdriver, logging_strerror(rv)); return rv; } @@ -39,7 +39,7 @@ apr_status_t database_connect(config_t *cfg) &(cfg->dbconn->dbd)); if (rv) { logging_log(cfg, LOGLEVEL_ERROR, - "Could not connect to database. Error %s", logging_strerror(rv)); + "DB: Could not connect to database. Error %s", logging_strerror(rv)); return rv; } @@ -89,13 +89,13 @@ static apr_dbd_prepared_t *database_prepare_insert(config_t *cfg, apr_pool_t *p) sql = apr_pstrcatv(p, vec, i+2, NULL); - logging_log(cfg, LOGLEVEL_DEBUG, "Generated SQL: %s", sql); + logging_log(cfg, LOGLEVEL_DEBUG, "DB: Generated SQL: %s", sql); rv = apr_dbd_prepare(cfg->dbconn->driver, cfg->pool, cfg->dbconn->dbd, sql, "INSERT", &stmt); if (rv) { - logging_log(cfg, LOGLEVEL_ERROR, "Unable to Prepare SQL insert: %s", + logging_log(cfg, LOGLEVEL_NOISE, "DB: Unable to Prepare SQL insert: %s", apr_dbd_error(cfg->dbconn->driver, cfg->dbconn->dbd, rv)); return NULL; } @@ -112,7 +112,6 @@ apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data) if (!cfg->dbconn->stmt) { cfg->dbconn->stmt = database_prepare_insert(cfg, p); if (!cfg->dbconn->stmt) { - logging_log(cfg, LOGLEVEL_NOISE, "Unable to prepare SQL statement"); return APR_EINVAL; } cfg->dbconn->args = apr_palloc(cfg->pool, nfs * sizeof(char *)); @@ -123,7 +122,7 @@ apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data) rv = apr_dbd_pquery(cfg->dbconn->driver, p, cfg->dbconn->dbd, &f, cfg->dbconn->stmt, nfs, cfg->dbconn->args); if (rv) { - logging_log(cfg, LOGLEVEL_ERROR, "Unable to Insert SQL: %s", + logging_log(cfg, LOGLEVEL_ERROR, "DB: Unable to Insert SQL: %s", apr_dbd_error(cfg->dbconn->driver, cfg->dbconn->dbd, rv)); return rv; } @@ -131,3 +130,26 @@ apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data) } /** @todo implement transactions */ +apr_status_t database_trans_start(config_t *cfg, apr_pool_t *p) +{ +#if HAVE_APR_DBD_TRANSACTION_MODE_GET +#else + return APR_SUCCESS; +#endif +} + +apr_status_t database_trans_stop(config_t *cfg, apr_pool_t *p) +{ +#if HAVE_APR_DBD_TRANSACTION_MODE_GET +#else + return APR_SUCCESS; +#endif +} + +apr_status_t database_trans_abort(config_t *cfg) +{ +#if HAVE_APR_DBD_TRANSACTION_MODE_GET +#else + return APR_SUCCESS; +#endif +} diff --git a/utility/database.h b/utility/database.h index fd24994..eed8898 100644 --- a/utility/database.h +++ b/utility/database.h @@ -13,4 +13,10 @@ apr_status_t database_disconnect(config_t *cfg); apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data); +apr_status_t database_trans_start(config_t *cfg, apr_pool_t *p); + +apr_status_t database_trans_stop(config_t *cfg, apr_pool_t *p); + +apr_status_t database_trans_abort(config_t *cfg); + #endif /*DATABASE_H_*/ diff --git a/utility/logparse.c b/utility/logparse.c index 6a21964..ac70ee2 100644 --- a/utility/logparse.c +++ b/utility/logparse.c @@ -349,12 +349,12 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename) apr_pool_create(&tp, cfg->pool); apr_pool_create(&targp, tp); - logging_log(cfg, LOGLEVEL_NOTICE, "Begin Parsing Log File '%s'", filename); + logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", filename); rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED, APR_OS_DEFAULT, tp); if (rv != APR_SUCCESS) { - logging_log(cfg, LOGLEVEL_NOISE, "Could not open %s", filename); + logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", filename); return rv; } @@ -362,17 +362,38 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename) do { rv = apr_file_gets(buff, 1024, file); if (rv == APR_SUCCESS) { + int i,m, cont = 0; + config_filter_t *filters; + line++; // chomp off newline line_chomp(buff); + // Run line filters + for (i=0, m=cfg->linefilters->nelts, + filters = (config_filter_t *)cfg->linefilters->elts; + ilog_formats, cfg->logformat, APR_HASH_KEY_STRING); if (!fmt) @@ -416,13 +438,31 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv, for (i=0; ifields->nelts; i++) { apr_table_setn(datain, ifields[i].name, argv[i]); } - /** @todo Run Pre Filters here */ + // Run Pre Filters + for (i=0, m=cfg->prefilters->nelts, + filters = (config_filter_t *)cfg->prefilters->elts; + ioutput_fields->elts; // clear out ofield function per-line data memset(&g_parser_linedata[1],0,sizeof(void *)*(int)g_parser_linedata[0]); // Convert input fields to output fields - for (i=0; ioutput_fields->nelts; i++) { + for (i=0,m=cfg->output_fields->nelts; ipostfilters->nelts, + filters = (config_filter_t *)cfg->postfilters->elts; + idryrun) { diff --git a/utility/logparse.h b/utility/logparse.h index bc39cb1..0b8607b 100644 --- a/utility/logparse.h +++ b/utility/logparse.h @@ -25,7 +25,7 @@ void parser_find_logs(config_t *cfg); apr_status_t parse_logfile(config_t *cfg, const char *filename); -apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv, - int argc); +apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, + char **argv, int argc); #endif /*LOGPARSE_H_*/ diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf index ee0cb07..fde7cc6 100644 --- a/utility/mod_log_sql.conf +++ b/utility/mod_log_sql.conf @@ -29,9 +29,12 @@ LogFormatConfig Combined agent String LogFormat Combined -# not yet implemented Linefilter - "BAD" -PreFilter request - "GET \/images" +LineFilter "GOOD" +LineFilter + "BETTER" +# the next filter ignores ALL lines +LineFilter - +PreFilter request + "Rebuild" PostFilter request_method "GET" # Usage field datatype(size) default source [function [param]...] @@ -41,7 +44,8 @@ OutputField remote_host varchar(50) "" remhost OutputField request_method varchar(25) "" request regexmatch "^(\\w+)" OutputField time_stamp int 0 date totimestamp OutputField status smallint 0 status -OutputField request_uri varchar(255) "" request regexmatch "^\\w+ (.+) \\w+/[\\d\\.]+$" +OutputField request_line varchar(255) "" request +#OutputField request_uri varchar(255) "" request regexmatch "^\\w+ (.+) \\w+/[\\d\\.]+$" OutputField remote_user varchar(50) "" user OutputField remote_logname varchar(50) "" ident OutputField request_time char(28) "" date wrap "[" "]" diff --git a/utility/shell.c b/utility/shell.c index e521916..3de65b3 100644 --- a/utility/shell.c +++ b/utility/shell.c @@ -134,6 +134,8 @@ int main(int argc, const char *const argv[]) database_init(pool); // Process configuration file cfg = config_create(pool); + // initialize STD out error log + logging_preinit(cfg); rv = config_read(cfg, apr_table_get(args,"Config"), args); apr_pool_destroy(ptemp); diff --git a/utility/util.c b/utility/util.c index 7ecb902..45b95ba 100644 --- a/utility/util.c +++ b/utility/util.c @@ -124,11 +124,15 @@ int ap_unescape_url(char *url) #endif } -void logging_init(config_t *cfg) +void logging_preinit(config_t *cfg) { - apr_status_t rv; apr_pool_create(&cfg->errorlog_p, cfg->pool); apr_file_open_stderr(&cfg->errorlog_fperr, cfg->pool); +} + +void logging_init(config_t *cfg) +{ + apr_status_t rv; if (cfg->errorlog) { rv = apr_file_open(&cfg->errorlog_fp, cfg->errorlog, APR_FOPEN_CREATE | APR_FOPEN_WRITE | APR_FOPEN_APPEND, @@ -170,7 +174,7 @@ void logging_log(config_t *cfg, loglevel_e level, const char *fmt, ...) if (level == LOGLEVEL_NOISE) { apr_file_writev(cfg->errorlog_fperr,vec,2,&blen); } - if (cfg->loglevel > LOGLEVEL_NONE) { + if (cfg->loglevel > LOGLEVEL_NONE && cfg->errorlog_fp) { apr_file_writev(cfg->errorlog_fp,vec,2,&blen); } diff --git a/utility/util.h b/utility/util.h index 99f93aa..a5a7f7e 100644 --- a/utility/util.h +++ b/utility/util.h @@ -14,6 +14,8 @@ void line_chomp(char *str); int ap_unescape_url(char *url); +void logging_preinit(config_t *cfg); + void logging_init(config_t *cfg); void logging_log(config_t *cfg, loglevel_e level, const char *fmt, ...) -- cgit