From e940dd14426c5a725fda70a154fd2bf4bb91ba44 Mon Sep 17 00:00:00 2001 From: Edward Rudd Date: Thu, 30 Oct 2008 23:03:13 +0000 Subject: add ability to skip N number of bad lines before throwing an error. Will log the bad lines to a log file so they can be preserved --- utility/config.c | 21 +++++++++++++++ utility/config.h | 7 +++++ utility/logparse.c | 67 +++++++++++++++++++++++++++++++++++++++++++----- utility/mod_log_sql.conf | 4 +++ utility/shell.c | 6 ++--- 5 files changed, 96 insertions(+), 9 deletions(-) diff --git a/utility/config.c b/utility/config.c index 064a5e2..6d3f61e 100644 --- a/utility/config.c +++ b/utility/config.c @@ -24,6 +24,17 @@ static apr_status_t config_set_string(config_t *cfg, config_opt_t *opt, return APR_SUCCESS; } +static apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + int offset = (int)(long)opt->data; + int *data = (int *)((void *)cfg + offset); + if (argc != 2) + return APR_EINVAL; + *data = apr_atoi64(argv[1]); + return APR_SUCCESS; +} + static apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc, const char **argv) { @@ -197,6 +208,9 @@ void config_dump(config_t *cfg) printf("ErrorLog: %s\n", cfg->errorlog); printf("LogLevel: %d\n", cfg->loglevel); + printf("BadLineFile: %s\n", cfg->badlinefile); + printf("BadLineMax: %d\n", cfg->badlinemax); + printf("InputDir: %s\n", cfg->input_dir); printf("DB Driver: %s\n", cfg->dbdriver); @@ -291,6 +305,13 @@ void config_init(apr_pool_t *p) "Set Log Level (error, warn, debug, quiet)", config_set_loglevel, NULL); + config_add_option(p, "BadLineFile", "File to log bad log lines", config_set_string, + (void *)APR_OFFSETOF(config_t, badlinefile)); + config_add_option(p, "BadLineMax", + "Max number of bad lines before aborting", config_set_int, + (void *)APR_OFFSETOF(config_t, badlinemax)); + + config_add_option(p, "InputDirectory", "Directory to scan for log files", config_set_string, (void *)APR_OFFSETOF(config_t, input_dir)); config_add_option(p, "InputFile", "Parse only this file", diff --git a/utility/config.h b/utility/config.h index d4dde77..c627662 100644 --- a/utility/config.h +++ b/utility/config.h @@ -29,6 +29,12 @@ struct config_t { apr_file_t *errorlog_fperr; apr_pool_t *errorlog_p; + const char *badlinefile; + apr_file_t *badline_fp; + int badline_count; + int badlinemax; + + /** input directory of log files */ const char *input_dir; /** list of files to scan */ @@ -73,6 +79,7 @@ struct config_filestat_t { char *fname; apr_size_t linesparsed; apr_size_t lineskipped; + apr_size_t linesbad; const char *result; apr_time_t start; apr_time_t stop; diff --git a/utility/logparse.c b/utility/logparse.c index 097e66d..ec587a8 100644 --- a/utility/logparse.c +++ b/utility/logparse.c @@ -212,6 +212,56 @@ void parser_find_logs(config_t *cfg) apr_pool_destroy(tp); } +apr_status_t parser_logbadline(config_t *cfg, const char *filename, + const char *badline) +{ + apr_status_t rv = APR_SUCCESS; + apr_size_t len; + struct iovec vec[5]; + + if (cfg->badlinefile) { + if (!cfg->badline_fp) { + rv = apr_file_open(&cfg->badline_fp, cfg->badlinefile, + APR_FOPEN_CREATE | APR_FOPEN_WRITE | APR_FOPEN_APPEND, + APR_OS_DEFAULT, cfg->pool); + if (rv) { + logging_log(cfg, LOGLEVEL_NOISE, + "Error opening badline file %s\n", cfg->badlinefile); + cfg->badlinefile = NULL; + } else { + char date[APR_RFC822_DATE_LEN]; + vec[0].iov_base = "Starting BadLines for \""; + vec[0].iov_len = sizeof("Starting BadLines for \"")-1; + vec[1].iov_base = (void *)filename; + vec[1].iov_len = strlen(filename); + vec[2].iov_base = "\" on "; + vec[2].iov_len = sizeof("\" on ")-1; + apr_rfc822_date(date, apr_time_now()); + vec[3].iov_base = date; + vec[3].iov_len = APR_RFC822_DATE_LEN-1; + vec[4].iov_base = "\n"; + vec[4].iov_len = 1; + apr_file_writev(cfg->badline_fp, vec,5, &len); + } + } + if (!rv) { + if ((++cfg->badline_count) > cfg->badlinemax) { + logging_log(cfg, LOGLEVEL_NOISE, + "Found more than %d bad lines (found %d)", + cfg->badlinemax, cfg->badline_count); + rv = APR_EINVAL; + } else { + vec[0].iov_base = (void *)badline; + vec[0].iov_len = strlen(badline); + vec[1].iov_base = "\n"; + vec[1].iov_len = 1; + apr_file_writev(cfg->badline_fp, vec,2, &len); + } + } + } + return rv; +} + /* * Modified version of apr_tokenize_to_argv to add [] as quoting characters * @@ -406,12 +456,17 @@ apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat) rv = parser_processline(targp, cfg, fstat, targv, targc); if (rv != APR_SUCCESS) { int i; - if (!cfg->dryrun) database_trans_abort(cfg); - logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed, - targc, buff); - for (i = 0; targv[i]; i++) { - logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i, - targv[i]); + + fstat->linesbad++; + rv = parser_logbadline(cfg, fstat->fname, buff); + if (rv) { + if (!cfg->dryrun) database_trans_abort(cfg); + logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed, + targc, buff); + for (i = 0; targv[i]; i++) { + logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i, + targv[i]); + } } } } else { diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf index f26f113..5b2c0f9 100644 --- a/utility/mod_log_sql.conf +++ b/utility/mod_log_sql.conf @@ -1,5 +1,6 @@ InputDirectory ./logs ErrorLog ./error_log + DBDDriver mysql DBDParams "host=localhost;user=root;dbname=apache_log" Table access_log @@ -9,6 +10,9 @@ LogLevel debug DryRun off Summary on +BadLineFile ./badlines.log +BadLineMax 10 + LogFormatConfig CLF remhost String LogFormatConfig CLF ident String LogFormatConfig CLF user String diff --git a/utility/shell.c b/utility/shell.c index 2c19bb4..0e9d646 100644 --- a/utility/shell.c +++ b/utility/shell.c @@ -64,13 +64,13 @@ void print_summary(config_t *cfg) { printf("Execution Summary\n"); for (i=0, m=cfg->input_files->nelts; i