From f9f8794a67561c243823cac0a45d8906e90ee7bb Mon Sep 17 00:00:00 2001 From: Edward Rudd Date: Mon, 27 Oct 2008 02:10:15 +0000 Subject: Added summary page including line count, duration, and final status of file --- utility/config.c | 15 ++++---- utility/config.h | 9 +++++ utility/logparse.c | 90 +++++++++++++++++++++++++++++++++--------------- utility/logparse.h | 6 ++-- utility/mod_log_sql.conf | 2 +- utility/shell.c | 35 ++++++++++++++++--- 6 files changed, 114 insertions(+), 43 deletions(-) diff --git a/utility/config.c b/utility/config.c index 28bb5cd..3b6e946 100644 --- a/utility/config.c +++ b/utility/config.c @@ -55,11 +55,12 @@ static apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt, static apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt, int argc, const char **argv) { - char **newp; + config_filestat_t *newp; if (argc != 2) return APR_EINVAL; - newp = (char **)apr_array_push(cfg->input_files); - *newp = apr_pstrdup(cfg->pool, argv[1]); + newp = (config_filestat_t *)apr_array_push(cfg->input_files); + newp->fname = apr_pstrdup(cfg->pool, argv[1]); + newp->result = "Not Parsed"; return APR_SUCCESS; } @@ -241,20 +242,20 @@ void config_dump(config_t *cfg) printf("\n"); } printf("Filters:\n>> Line:\n"); - filters = cfg->linefilters->elts; + filters = (config_filter_t *)cfg->linefilters->elts; for (i=0; ilinefilters->nelts; i++) { printf(">>>> %c \"%s\" (%pp)\n",filters[i].negative ? '-':'+', filters[i].filter, filters[i].regex); } printf(">> Pre:\n"); - filters = cfg->prefilters->elts; + filters = (config_filter_t *)cfg->prefilters->elts; for (i=0; iprefilters->nelts; i++) { printf(">>>> %s %c \"%s\" (%pp)\n", filters[i].field, filters[i].negative ? '-':'+', filters[i].filter, filters[i].regex); } printf(">> Post:\n"); - filters = cfg->postfilters->elts; + filters = (config_filter_t *)cfg->postfilters->elts; for (i=0; ipostfilters->nelts; i++) { printf(">>>> %s %c \"%s\" (%pp)\n", filters[i].field, filters[i].negative ? '-':'+', @@ -345,7 +346,7 @@ config_t *config_create(apr_pool_t *p) cfg->loglevel = LOGLEVEL_ERROR; cfg->summary = 1; cfg->transactions = 1; - cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(char *)); + cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(config_filestat_t)); cfg->log_formats = apr_hash_make(cfg->pool); cfg->output_fields = apr_array_make(cfg->pool, 10, sizeof(config_output_field_t)); diff --git a/utility/config.h b/utility/config.h index 26a4e99..d4dde77 100644 --- a/utility/config.h +++ b/utility/config.h @@ -68,6 +68,15 @@ struct config_t { int summary; }; +typedef struct config_filestat_t config_filestat_t; +struct config_filestat_t { + char *fname; + apr_size_t linesparsed; + apr_size_t lineskipped; + const char *result; + apr_time_t start; + apr_time_t stop; +}; typedef struct config_logformat_t config_logformat_t; struct config_logformat_t { diff --git a/utility/logparse.c b/utility/logparse.c index 923b581..afb1824 100644 --- a/utility/logparse.c +++ b/utility/logparse.c @@ -188,7 +188,7 @@ void parser_find_logs(config_t *cfg) apr_pool_t *tp; apr_dir_t *dir; apr_finfo_t finfo; - char **newp; + config_filestat_t *newp; logging_log(cfg, LOGLEVEL_NOTICE, "Find Log files"); if (!cfg->input_dir) @@ -199,9 +199,10 @@ void parser_find_logs(config_t *cfg) == APR_SUCCESS) { if (finfo.filetype == APR_DIR) continue; - newp = (char **)apr_array_push(cfg->input_files); - apr_filepath_merge(newp, cfg->input_dir, finfo.name, - APR_FILEPATH_TRUENAME, cfg->pool); + newp = (config_filestat_t *)apr_array_push(cfg->input_files); + newp->result = "Not Parsed"; + apr_filepath_merge(&(newp->fname), cfg->input_dir, finfo.name, + APR_FILEPATH_TRUENAME, cfg->pool); } apr_dir_close(dir); } @@ -336,7 +337,7 @@ static apr_status_t tokenize_logline(const char *arg_str, char ***argv_out, return APR_SUCCESS; } -apr_status_t parse_logfile(config_t *cfg, const char *filename) +apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat) { apr_pool_t *tp, *targp; apr_file_t *file; @@ -344,23 +345,27 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename) char buff[2048]; char **targv; int targc; - int line; apr_pool_create(&tp, cfg->pool); apr_pool_create(&targp, tp); - logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", filename); + logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", fstat->fname); - rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED, + rv = apr_file_open(&file, fstat->fname, APR_FOPEN_READ | APR_BUFFERED, APR_OS_DEFAULT, tp); if (rv != APR_SUCCESS) { - logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", filename); + logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", fstat->fname); return rv; } - line = 0; + fstat->linesparsed = 0; // Start Transaction - database_trans_start(cfg,tp); + fstat->start = apr_time_now(); + if (database_trans_start(cfg,tp)) { + fstat->result = "Database Transaction Error"; + fstat->stop = apr_time_now(); + return rv; + } do { rv = apr_file_gets(buff, 1024, file); @@ -368,7 +373,7 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename) int i,m, cont = 0; config_filter_t *filters; - line++; + fstat->linesparsed++; // chomp off newline line_chomp(buff); // Run line filters @@ -379,12 +384,13 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename) if (filters[i].negative) { logging_log(cfg, LOGLEVEL_DEBUG, "PARSER: LINEFILTER: Skipping Line %d due to Filter (%d)%s", - line, i, filters[i].filter); + fstat->linesparsed, i, filters[i].filter); + fstat->lineskipped++; cont = 1; } else { logging_log(cfg, LOGLEVEL_DEBUG, "PARSER: LINEFILTER: Force Parsing Line %d due to Filter (%d)%s", - line, i, filters[i].filter); + fstat->linesparsed, i, filters[i].filter); } break; } @@ -396,11 +402,11 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename) targc = 0; while (targv[targc]) targc++; - rv = parse_processline(targp, cfg, line, targv, targc); + rv = parser_processline(targp, cfg, fstat, targv, targc); if (rv != APR_SUCCESS) { int i; database_trans_abort(cfg); - logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", line, + logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed, targc, buff); for (i = 0; targv[i]; i++) { logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i, @@ -414,17 +420,25 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename) } while (rv == APR_SUCCESS); apr_file_close(file); // Finish Transaction - database_trans_stop(cfg,tp); + if (database_trans_stop(cfg,tp)) { + fstat->result = apr_psprintf(cfg->pool, + "Input line %d, Database Transaction Error", + fstat->linesparsed); + } apr_pool_destroy(tp); logging_log(cfg, LOGLEVEL_NOTICE, - "PARSER: Finish Parsing Log File '%s'. Lines: %d", filename, line); - + "PARSER: Finish Parsing Log File '%s'. Lines: (%d/%d)", + fstat->fname, fstat->linesparsed - fstat->lineskipped, fstat->linesparsed); + if (!rv) { + fstat->result = "File Parsed Succesfully"; + } + fstat->stop = apr_time_now(); return rv; } -apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, - char **argv, int argc) +apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg, + config_filestat_t *fstat, char **argv, int argc) { config_logformat_t *fmt; config_logformat_field_t *ifields; @@ -436,10 +450,19 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, int i,m; fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING); - if (!fmt) + if (!fmt) { + logging_log(cfg, LOGLEVEL_NOISE, "PARSER: No Input Log format"); return APR_EINVAL; - if (fmt->fields->nelts != argc) + } + if (fmt->fields->nelts != argc) { + logging_log(cfg, LOGLEVEL_NOISE, + "PARSER: Input line field number differs from expected. Expected %d got %d.", + fmt->fields->nelts, argc); + fstat->result = apr_psprintf(cfg->pool, + "Input line %d is badly formatted (wrong number of fields)", + fstat->linesparsed); return APR_EINVAL; + } datain = apr_table_make(ptemp, fmt->fields->nelts); dataout = apr_table_make(ptemp, cfg->output_fields->nelts); @@ -457,12 +480,13 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, if (filters[i].negative) { logging_log(cfg, LOGLEVEL_DEBUG, "PARSER: PREFILTER: Skipping Line %d due to Filter (%d)%s", - line, i, filters[i].filter); + fstat->linesparsed, i, filters[i].filter); + fstat->lineskipped++; return APR_SUCCESS; } else { logging_log(cfg, LOGLEVEL_DEBUG, "PARSER: PREFILTER: Force Parsing Line %d due to Filter (%d)%s", - line, i, filters[i].filter); + fstat->linesparsed, i, filters[i].filter); } break; } @@ -486,8 +510,12 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, const char *ret= NULL; rv = ((parser_func_t *)ofields[i].func)->func(ptemp, cfg, &ofields[i], val, &ret); - if (rv) + if (rv) { + fstat->result = apr_psprintf(cfg->pool, + "Input line %d, Parser function %s returned error (%d)%s", + fstat->linesparsed, ofields[i].fname, rv, logging_strerror(rv)); return rv; + } apr_table_setn(dataout, ofields[i].field, ret ? ret : ofields[i].def); } } @@ -501,12 +529,13 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, if (filters[i].negative) { logging_log(cfg, LOGLEVEL_DEBUG, "PARSER: POSTFILTER: Skipping Line %d due to Filter (%d)%s", - line, i, filters[i].filter); + fstat->linesparsed, i, filters[i].filter); + fstat->lineskipped++; return APR_SUCCESS; } else { logging_log(cfg, LOGLEVEL_DEBUG, "PARSER: POSTFILTER: Force Parsing Line %d due to Filter (%d)%s", - line, i, filters[i].filter); + fstat->linesparsed, i, filters[i].filter); } break; } @@ -515,6 +544,11 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, // Process DB Query if (!cfg->dryrun) { rv = database_insert(cfg, ptemp, dataout); + if (rv) { + fstat->result = apr_psprintf(cfg->pool, + "Input line %d, Database Error", + fstat->linesparsed); + } } return rv; } diff --git a/utility/logparse.h b/utility/logparse.h index 0b8607b..fe708a5 100644 --- a/utility/logparse.h +++ b/utility/logparse.h @@ -23,9 +23,9 @@ void parser_init(apr_pool_t *p); void parser_find_logs(config_t *cfg); -apr_status_t parse_logfile(config_t *cfg, const char *filename); +apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat); -apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, - char **argv, int argc); +apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg, + config_filestat_t *line, char **argv, int argc); #endif /*LOGPARSE_H_*/ diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf index 54ac85d..de9cd28 100644 --- a/utility/mod_log_sql.conf +++ b/utility/mod_log_sql.conf @@ -34,7 +34,7 @@ LogFormat Combined #LineFilter + "BETTER" # the next filter ignores ALL lines #LineFilter - -#PreFilter request + "Rebuild" +#PreFilter request - "Rebuild" #PostFilter request_method "GET" # Usage field datatype(size) default source [function [param]...] diff --git a/utility/shell.c b/utility/shell.c index 5f011a6..6f98055 100644 --- a/utility/shell.c +++ b/utility/shell.c @@ -55,6 +55,30 @@ void show_help(const char *prog, const apr_getopt_option_t *opts, FILE *output) } } +void print_summary(config_t *cfg) { + config_filestat_t *fstat; + int i,m; + + fstat = (config_filestat_t *)cfg->input_files->elts; + + printf("Execution Summary\n"); + for (i=0, m=cfg->input_files->nelts; iinput_files)) { - char **filelist; + config_filestat_t *filelist; int f, l; - filelist = (char **)cfg->input_files->elts; + filelist = (config_filestat_t *)cfg->input_files->elts; for (f=0, l=cfg->input_files->nelts; f < l; f++) { - rv = parse_logfile(cfg, filelist[f]); + rv = parser_parsefile(cfg, &filelist[f]); if (rv) { logging_log(cfg, LOGLEVEL_NOISE, "Error occured parsing log files. Aborting"); @@ -183,6 +207,9 @@ int main(int argc, const char *const argv[]) if (!cfg->dryrun) { database_disconnect(cfg); } - /** @todo summary goes here */ + + if (cfg->summary) { + print_summary(cfg); + } return 0; } -- cgit