add ability to skip N number of bad lines before throwing an error.

Will log the bad lines to a log file so they can be preserved

5 files changed, 96 insertions, 9 deletions

diff --git a/utility/config.c b/utility/config.c
index 064a5e2..6d3f61e 100644
--- a/utility/config.c
+++ b/utility/config.c
@@ -24,6 +24,17 @@ static apr_status_t config_set_string(config_t *cfg, config_opt_t *opt,
     return APR_SUCCESS;
 }
 
+static apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv)
+{
+    int offset = (int)(long)opt->data;
+    int *data = (int *)((void *)cfg + offset);
+    if (argc != 2)
+        return APR_EINVAL;
+    *data = apr_atoi64(argv[1]);
+    return APR_SUCCESS;
+}
+
 static apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc,
         const char **argv)
 {
@@ -197,6 +208,9 @@ void config_dump(config_t *cfg)
     printf("ErrorLog: %s\n", cfg->errorlog);
     printf("LogLevel: %d\n", cfg->loglevel);
 
+    printf("BadLineFile: %s\n", cfg->badlinefile);
+    printf("BadLineMax: %d\n", cfg->badlinemax);
+
     printf("InputDir: %s\n", cfg->input_dir);
 
     printf("DB Driver: %s\n", cfg->dbdriver);
@@ -291,6 +305,13 @@ void config_init(apr_pool_t *p)
             "Set Log Level (error, warn, debug, quiet)", config_set_loglevel,
             NULL);
 
+    config_add_option(p, "BadLineFile", "File to log bad log lines", config_set_string,
+            (void *)APR_OFFSETOF(config_t, badlinefile));
+    config_add_option(p, "BadLineMax",
+            "Max number of bad lines before aborting", config_set_int,
+            (void *)APR_OFFSETOF(config_t, badlinemax));
+
+
     config_add_option(p, "InputDirectory", "Directory to scan for log files",
             config_set_string, (void *)APR_OFFSETOF(config_t, input_dir));
     config_add_option(p, "InputFile", "Parse only this file",
diff --git a/utility/config.h b/utility/config.h
index d4dde77..c627662 100644
--- a/utility/config.h
+++ b/utility/config.h
@@ -29,6 +29,12 @@ struct config_t {
     apr_file_t *errorlog_fperr;
     apr_pool_t *errorlog_p;
 
+    const char *badlinefile;
+    apr_file_t *badline_fp;
+    int badline_count;
+    int badlinemax;
+
+
     /** input directory of log files */
     const char *input_dir;
     /** list of files to scan */
@@ -73,6 +79,7 @@ struct config_filestat_t {
     char *fname;
     apr_size_t linesparsed;
     apr_size_t lineskipped;
+    apr_size_t linesbad;
     const char *result;
     apr_time_t start;
     apr_time_t stop;
diff --git a/utility/logparse.c b/utility/logparse.c
index 097e66d..ec587a8 100644
--- a/utility/logparse.c
+++ b/utility/logparse.c
@@ -212,6 +212,56 @@ void parser_find_logs(config_t *cfg)
     apr_pool_destroy(tp);
 }
 
+apr_status_t parser_logbadline(config_t *cfg, const char *filename,
+        const char *badline)
+{
+    apr_status_t rv = APR_SUCCESS;
+    apr_size_t len;
+    struct iovec vec[5];
+
+    if (cfg->badlinefile) {
+        if (!cfg->badline_fp) {
+            rv = apr_file_open(&cfg->badline_fp, cfg->badlinefile,
+                    APR_FOPEN_CREATE | APR_FOPEN_WRITE | APR_FOPEN_APPEND,
+                    APR_OS_DEFAULT, cfg->pool);
+            if (rv) {
+                logging_log(cfg, LOGLEVEL_NOISE,
+                        "Error opening badline file %s\n", cfg->badlinefile);
+                cfg->badlinefile = NULL;
+            } else {
+                char date[APR_RFC822_DATE_LEN];
+                vec[0].iov_base = "Starting BadLines for \"";
+                vec[0].iov_len = sizeof("Starting BadLines for \"")-1;
+                vec[1].iov_base = (void *)filename;
+                vec[1].iov_len = strlen(filename);
+                vec[2].iov_base = "\" on ";
+                vec[2].iov_len = sizeof("\" on ")-1;
+                apr_rfc822_date(date, apr_time_now());
+                vec[3].iov_base = date;
+                vec[3].iov_len = APR_RFC822_DATE_LEN-1;
+                vec[4].iov_base = "\n";
+                vec[4].iov_len = 1;
+                apr_file_writev(cfg->badline_fp, vec,5, &len);
+            }
+        }
+        if (!rv) {
+            if ((++cfg->badline_count) > cfg->badlinemax) {
+                logging_log(cfg, LOGLEVEL_NOISE,
+                        "Found more than %d bad lines (found %d)",
+                        cfg->badlinemax, cfg->badline_count);
+                rv = APR_EINVAL;
+            } else {
+                vec[0].iov_base = (void *)badline;
+                vec[0].iov_len = strlen(badline);
+                vec[1].iov_base = "\n";
+                vec[1].iov_len = 1;
+                apr_file_writev(cfg->badline_fp, vec,2, &len);
+            }
+        }
+    }
+    return rv;
+}
+
 /*
  * Modified version of apr_tokenize_to_argv to add [] as quoting characters
  *
@@ -406,12 +456,17 @@ apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat)
             rv = parser_processline(targp, cfg, fstat, targv, targc);
             if (rv != APR_SUCCESS) {
                 int i;
-                if (!cfg->dryrun) database_trans_abort(cfg);
-                logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed,
-                        targc, buff);
-                for (i = 0; targv[i]; i++) {
-                    logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i,
-                            targv[i]);
+
+                fstat->linesbad++;
+                rv = parser_logbadline(cfg, fstat->fname, buff);
+                if (rv) {
+                    if (!cfg->dryrun) database_trans_abort(cfg);
+                    logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed,
+                            targc, buff);
+                    for (i = 0; targv[i]; i++) {
+                        logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i,
+                                targv[i]);
+                    }
                 }
             }
         } else {
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
index f26f113..5b2c0f9 100644
--- a/utility/mod_log_sql.conf
+++ b/utility/mod_log_sql.conf
@@ -1,5 +1,6 @@
 InputDirectory ./logs
 ErrorLog ./error_log
+
 DBDDriver mysql
 DBDParams "host=localhost;user=root;dbname=apache_log"
 Table access_log
@@ -9,6 +10,9 @@ LogLevel debug
 DryRun off
 Summary on
 
+BadLineFile ./badlines.log
+BadLineMax 10
+
 LogFormatConfig CLF remhost String
 LogFormatConfig CLF ident String
 LogFormatConfig CLF user String
diff --git a/utility/shell.c b/utility/shell.c
index 2c19bb4..0e9d646 100644
--- a/utility/shell.c
+++ b/utility/shell.c
@@ -64,13 +64,13 @@ void print_summary(config_t *cfg) {
     printf("Execution Summary\n");
     for (i=0, m=cfg->input_files->nelts; i<m; i++) {
         printf(" File: %s\n"
-                "  Lines Parsed %d out of %d (Skipped %d)\n"
+                "  Lines Parsed %d out of %d (Skipped %d, Bad %d)\n"
                 "  Status: %s\n"
                 "  Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n"
                 "\n",
                fstat[i].fname,
-               fstat[i].linesparsed - fstat[i].lineskipped,
-               fstat[i].linesparsed, fstat[i].lineskipped,
+               fstat[i].linesparsed - fstat[i].lineskipped - fstat[i].linesbad,
+               fstat[i].linesparsed, fstat[i].lineskipped, fstat[i].linesbad,
                fstat[i].result,
                apr_time_sec(fstat[i].stop - fstat[i].start)/60,
                apr_time_sec(fstat[i].stop - fstat[i].start),