Added summary page including line count, duration, and final status of file

6 files changed, 114 insertions, 43 deletions

diff --git a/utility/config.c b/utility/config.c
index 28bb5cd..3b6e946 100644
--- a/utility/config.c
+++ b/utility/config.c
@@ -55,11 +55,12 @@ static apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt,
 static apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt,
         int argc, const char **argv)
 {
-    char **newp;
+    config_filestat_t *newp;
     if (argc != 2)
         return APR_EINVAL;
-    newp = (char **)apr_array_push(cfg->input_files);
-    *newp = apr_pstrdup(cfg->pool, argv[1]);
+    newp = (config_filestat_t *)apr_array_push(cfg->input_files);
+    newp->fname = apr_pstrdup(cfg->pool, argv[1]);
+    newp->result = "Not Parsed";
     return APR_SUCCESS;
 }
 
@@ -241,20 +242,20 @@ void config_dump(config_t *cfg)
         printf("\n");
     }
     printf("Filters:\n>> Line:\n");
-    filters = cfg->linefilters->elts;
+    filters = (config_filter_t *)cfg->linefilters->elts;
     for (i=0; i<cfg->linefilters->nelts; i++) {
         printf(">>>> %c \"%s\" (%pp)\n",filters[i].negative ? '-':'+',
                 filters[i].filter,  filters[i].regex);
     }
     printf(">> Pre:\n");
-    filters = cfg->prefilters->elts;
+    filters = (config_filter_t *)cfg->prefilters->elts;
     for (i=0; i<cfg->prefilters->nelts; i++) {
         printf(">>>> %s %c \"%s\" (%pp)\n",
                 filters[i].field, filters[i].negative ? '-':'+',
                 filters[i].filter,  filters[i].regex);
     }
     printf(">> Post:\n");
-    filters = cfg->postfilters->elts;
+    filters = (config_filter_t *)cfg->postfilters->elts;
     for (i=0; i<cfg->postfilters->nelts; i++) {
         printf(">>>> %s %c \"%s\" (%pp)\n",
                 filters[i].field, filters[i].negative ? '-':'+',
@@ -345,7 +346,7 @@ config_t *config_create(apr_pool_t *p)
     cfg->loglevel = LOGLEVEL_ERROR;
     cfg->summary = 1;
     cfg->transactions = 1;
-    cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(char *));
+    cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(config_filestat_t));
     cfg->log_formats = apr_hash_make(cfg->pool);
     cfg->output_fields = apr_array_make(cfg->pool, 10,
             sizeof(config_output_field_t));
diff --git a/utility/config.h b/utility/config.h
index 26a4e99..d4dde77 100644
--- a/utility/config.h
+++ b/utility/config.h
@@ -68,6 +68,15 @@ struct config_t {
     int summary;
 };
 
+typedef struct config_filestat_t config_filestat_t;
+struct config_filestat_t {
+    char *fname;
+    apr_size_t linesparsed;
+    apr_size_t lineskipped;
+    const char *result;
+    apr_time_t start;
+    apr_time_t stop;
+};
 
 typedef struct config_logformat_t config_logformat_t;
 struct config_logformat_t {
diff --git a/utility/logparse.c b/utility/logparse.c
index 923b581..afb1824 100644
--- a/utility/logparse.c
+++ b/utility/logparse.c
@@ -188,7 +188,7 @@ void parser_find_logs(config_t *cfg)
     apr_pool_t *tp;
     apr_dir_t *dir;
     apr_finfo_t finfo;
-    char **newp;
+    config_filestat_t *newp;
 
     logging_log(cfg, LOGLEVEL_NOTICE, "Find Log files");
     if (!cfg->input_dir)
@@ -199,9 +199,10 @@ void parser_find_logs(config_t *cfg)
                 == APR_SUCCESS) {
             if (finfo.filetype == APR_DIR)
                 continue;
-            newp = (char **)apr_array_push(cfg->input_files);
-            apr_filepath_merge(newp, cfg->input_dir, finfo.name,
-            APR_FILEPATH_TRUENAME, cfg->pool);
+            newp = (config_filestat_t *)apr_array_push(cfg->input_files);
+            newp->result = "Not Parsed";
+            apr_filepath_merge(&(newp->fname), cfg->input_dir, finfo.name,
+                        APR_FILEPATH_TRUENAME, cfg->pool);
         }
         apr_dir_close(dir);
     }
@@ -336,7 +337,7 @@ static apr_status_t tokenize_logline(const char *arg_str, char ***argv_out,
     return APR_SUCCESS;
 }
 
-apr_status_t parse_logfile(config_t *cfg, const char *filename)
+apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat)
 {
     apr_pool_t *tp, *targp;
     apr_file_t *file;
@@ -344,23 +345,27 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
     char buff[2048];
     char **targv;
     int targc;
-    int line;
 
     apr_pool_create(&tp, cfg->pool);
     apr_pool_create(&targp, tp);
 
-    logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", filename);
+    logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", fstat->fname);
 
-    rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED,
+    rv = apr_file_open(&file, fstat->fname, APR_FOPEN_READ | APR_BUFFERED,
     APR_OS_DEFAULT, tp);
     if (rv != APR_SUCCESS) {
-        logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", filename);
+        logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", fstat->fname);
         return rv;
     }
 
-    line = 0;
+    fstat->linesparsed = 0;
     // Start Transaction
-    database_trans_start(cfg,tp);
+    fstat->start = apr_time_now();
+    if (database_trans_start(cfg,tp)) {
+        fstat->result = "Database Transaction Error";
+        fstat->stop = apr_time_now();
+        return rv;
+    }
 
     do {
         rv = apr_file_gets(buff, 1024, file);
@@ -368,7 +373,7 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
             int i,m, cont = 0;
             config_filter_t *filters;
 
-            line++;
+            fstat->linesparsed++;
             // chomp off newline
             line_chomp(buff);
             // Run line filters
@@ -379,12 +384,13 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
                     if (filters[i].negative) {
                         logging_log(cfg, LOGLEVEL_DEBUG,
                                 "PARSER: LINEFILTER: Skipping Line %d due to Filter (%d)%s",
-                                line, i, filters[i].filter);
+                                fstat->linesparsed, i, filters[i].filter);
+                        fstat->lineskipped++;
                         cont = 1;
                     } else {
                         logging_log(cfg, LOGLEVEL_DEBUG,
                                 "PARSER: LINEFILTER: Force Parsing Line %d due to Filter (%d)%s",
-                                line, i, filters[i].filter);
+                                fstat->linesparsed, i, filters[i].filter);
                     }
                     break;
                 }
@@ -396,11 +402,11 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
             targc = 0;
             while (targv[targc])
                 targc++;
-            rv = parse_processline(targp, cfg, line, targv, targc);
+            rv = parser_processline(targp, cfg, fstat, targv, targc);
             if (rv != APR_SUCCESS) {
                 int i;
                 database_trans_abort(cfg);
-                logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", line,
+                logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed,
                         targc, buff);
                 for (i = 0; targv[i]; i++) {
                     logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i,
@@ -414,17 +420,25 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
     } while (rv == APR_SUCCESS);
     apr_file_close(file);
     // Finish Transaction
-    database_trans_stop(cfg,tp);
+    if (database_trans_stop(cfg,tp)) {
+        fstat->result = apr_psprintf(cfg->pool,
+                "Input line %d, Database Transaction Error",
+                fstat->linesparsed);
+    }
 
     apr_pool_destroy(tp);
     logging_log(cfg, LOGLEVEL_NOTICE,
-            "PARSER: Finish Parsing Log File '%s'. Lines: %d", filename, line);
-
+            "PARSER: Finish Parsing Log File '%s'. Lines: (%d/%d)",
+            fstat->fname, fstat->linesparsed - fstat->lineskipped, fstat->linesparsed);
+    if (!rv) {
+        fstat->result = "File Parsed Succesfully";
+    }
+    fstat->stop = apr_time_now();
     return rv;
 }
 
-apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
-        char **argv, int argc)
+apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg,
+        config_filestat_t *fstat, char **argv, int argc)
 {
     config_logformat_t *fmt;
     config_logformat_field_t *ifields;
@@ -436,10 +450,19 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
     int i,m;
 
     fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING);
-    if (!fmt)
+    if (!fmt) {
+        logging_log(cfg, LOGLEVEL_NOISE, "PARSER: No Input Log format");
         return APR_EINVAL;
-    if (fmt->fields->nelts != argc)
+    }
+    if (fmt->fields->nelts != argc) {
+        logging_log(cfg, LOGLEVEL_NOISE,
+                "PARSER: Input line field number differs from expected. Expected %d got %d.",
+                fmt->fields->nelts, argc);
+        fstat->result = apr_psprintf(cfg->pool,
+                "Input line %d is badly formatted (wrong number of fields)",
+                fstat->linesparsed);
         return APR_EINVAL;
+    }
 
     datain = apr_table_make(ptemp, fmt->fields->nelts);
     dataout = apr_table_make(ptemp, cfg->output_fields->nelts);
@@ -457,12 +480,13 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
             if (filters[i].negative) {
                 logging_log(cfg, LOGLEVEL_DEBUG,
                         "PARSER: PREFILTER: Skipping Line %d due to Filter (%d)%s",
-                        line, i, filters[i].filter);
+                        fstat->linesparsed, i, filters[i].filter);
+                fstat->lineskipped++;
                 return APR_SUCCESS;
             } else {
                 logging_log(cfg, LOGLEVEL_DEBUG,
                         "PARSER: PREFILTER: Force Parsing Line %d due to Filter (%d)%s",
-                        line, i, filters[i].filter);
+                        fstat->linesparsed, i, filters[i].filter);
             }
             break;
         }
@@ -486,8 +510,12 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
             const char *ret= NULL;
             rv = ((parser_func_t *)ofields[i].func)->func(ptemp, cfg,
                     &ofields[i], val, &ret);
-            if (rv)
+            if (rv) {
+                fstat->result = apr_psprintf(cfg->pool,
+                        "Input line %d, Parser function %s returned error (%d)%s",
+                        fstat->linesparsed, ofields[i].fname, rv, logging_strerror(rv));
                 return rv;
+            }
             apr_table_setn(dataout, ofields[i].field, ret ? ret : ofields[i].def);
         }
     }
@@ -501,12 +529,13 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
             if (filters[i].negative) {
                 logging_log(cfg, LOGLEVEL_DEBUG,
                         "PARSER: POSTFILTER: Skipping Line %d due to Filter (%d)%s",
-                        line, i, filters[i].filter);
+                        fstat->linesparsed, i, filters[i].filter);
+                fstat->lineskipped++;
                 return APR_SUCCESS;
             } else {
                 logging_log(cfg, LOGLEVEL_DEBUG,
                         "PARSER: POSTFILTER: Force Parsing Line %d due to Filter (%d)%s",
-                        line, i, filters[i].filter);
+                        fstat->linesparsed, i, filters[i].filter);
             }
             break;
         }
@@ -515,6 +544,11 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
     // Process DB Query
     if (!cfg->dryrun) {
         rv = database_insert(cfg, ptemp, dataout);
+        if (rv) {
+            fstat->result = apr_psprintf(cfg->pool,
+                    "Input line %d, Database Error",
+                    fstat->linesparsed);
+        }
     }
     return rv;
 }
diff --git a/utility/logparse.h b/utility/logparse.h
index 0b8607b..fe708a5 100644
--- a/utility/logparse.h
+++ b/utility/logparse.h
@@ -23,9 +23,9 @@ void parser_init(apr_pool_t *p);
 
 void parser_find_logs(config_t *cfg);
 
-apr_status_t parse_logfile(config_t *cfg, const char *filename);
+apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat);
 
-apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
-        char **argv, int argc);
+apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg,
+        config_filestat_t *line, char **argv, int argc);
 
 #endif /*LOGPARSE_H_*/
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
index 54ac85d..de9cd28 100644
--- a/utility/mod_log_sql.conf
+++ b/utility/mod_log_sql.conf
@@ -34,7 +34,7 @@ LogFormat Combined
 #LineFilter + "BETTER"
 # the next filter ignores ALL lines
 #LineFilter -
-#PreFilter request + "Rebuild"
+#PreFilter request - "Rebuild"
 #PostFilter request_method "GET"
 
 # Usage     field            datatype(size) default source      [function [param]...]
diff --git a/utility/shell.c b/utility/shell.c
index 5f011a6..6f98055 100644
--- a/utility/shell.c
+++ b/utility/shell.c
@@ -55,6 +55,30 @@ void show_help(const char *prog, const apr_getopt_option_t *opts, FILE *output)
     }
 }
 
+void print_summary(config_t *cfg) {
+    config_filestat_t *fstat;
+    int i,m;
+
+    fstat = (config_filestat_t *)cfg->input_files->elts;
+
+    printf("Execution Summary\n");
+    for (i=0, m=cfg->input_files->nelts; i<m; i++) {
+        printf(" File: %s\n"
+                "  Lines Parsed %d out of %d (Skipped %d)\n"
+                "  Status: %s\n"
+                "  Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n"
+                "\n",
+               fstat[i].fname,
+               fstat[i].linesparsed - fstat[i].lineskipped,
+               fstat[i].linesparsed, fstat[i].lineskipped,
+               fstat[i].result,
+               apr_time_sec(fstat[i].stop - fstat[i].start)/60,
+               apr_time_sec(fstat[i].stop - fstat[i].start),
+               apr_time_msec(fstat[i].stop - fstat[i].start)
+               );
+    }
+}
+
 int main(int argc, const char *const argv[])
 {
     apr_pool_t *pool, *ptemp;
@@ -166,11 +190,11 @@ int main(int argc, const char *const argv[])
         }
     }
     if (!apr_is_empty_array(cfg->input_files)) {
-        char **filelist;
+        config_filestat_t *filelist;
         int f, l;
-        filelist = (char **)cfg->input_files->elts;
+        filelist = (config_filestat_t *)cfg->input_files->elts;
         for (f=0, l=cfg->input_files->nelts; f < l; f++) {
-            rv = parse_logfile(cfg, filelist[f]);
+            rv = parser_parsefile(cfg, &filelist[f]);
             if (rv) {
                 logging_log(cfg, LOGLEVEL_NOISE,
                         "Error occured parsing log files. Aborting");
@@ -183,6 +207,9 @@ int main(int argc, const char *const argv[])
     if (!cfg->dryrun) {
         database_disconnect(cfg);
     }
-    /** @todo summary goes here */
+
+    if (cfg->summary) {
+        print_summary(cfg);
+    }
     return 0;
 }