summaryrefslogtreecommitdiffstatsabout
diff options
context:
space:
mode:
-rw-r--r--utility/config.c21
-rw-r--r--utility/config.h7
-rw-r--r--utility/logparse.c67
-rw-r--r--utility/mod_log_sql.conf4
-rw-r--r--utility/shell.c6
5 files changed, 96 insertions, 9 deletions
diff --git a/utility/config.c b/utility/config.c
index 064a5e2..6d3f61e 100644
--- a/utility/config.c
+++ b/utility/config.c
@@ -24,6 +24,17 @@ static apr_status_t config_set_string(config_t *cfg, config_opt_t *opt,
24 return APR_SUCCESS; 24 return APR_SUCCESS;
25} 25}
26 26
27static apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc,
28 const char **argv)
29{
30 int offset = (int)(long)opt->data;
31 int *data = (int *)((void *)cfg + offset);
32 if (argc != 2)
33 return APR_EINVAL;
34 *data = apr_atoi64(argv[1]);
35 return APR_SUCCESS;
36}
37
27static apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc, 38static apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc,
28 const char **argv) 39 const char **argv)
29{ 40{
@@ -197,6 +208,9 @@ void config_dump(config_t *cfg)
197 printf("ErrorLog: %s\n", cfg->errorlog); 208 printf("ErrorLog: %s\n", cfg->errorlog);
198 printf("LogLevel: %d\n", cfg->loglevel); 209 printf("LogLevel: %d\n", cfg->loglevel);
199 210
211 printf("BadLineFile: %s\n", cfg->badlinefile);
212 printf("BadLineMax: %d\n", cfg->badlinemax);
213
200 printf("InputDir: %s\n", cfg->input_dir); 214 printf("InputDir: %s\n", cfg->input_dir);
201 215
202 printf("DB Driver: %s\n", cfg->dbdriver); 216 printf("DB Driver: %s\n", cfg->dbdriver);
@@ -291,6 +305,13 @@ void config_init(apr_pool_t *p)
291 "Set Log Level (error, warn, debug, quiet)", config_set_loglevel, 305 "Set Log Level (error, warn, debug, quiet)", config_set_loglevel,
292 NULL); 306 NULL);
293 307
308 config_add_option(p, "BadLineFile", "File to log bad log lines", config_set_string,
309 (void *)APR_OFFSETOF(config_t, badlinefile));
310 config_add_option(p, "BadLineMax",
311 "Max number of bad lines before aborting", config_set_int,
312 (void *)APR_OFFSETOF(config_t, badlinemax));
313
314
294 config_add_option(p, "InputDirectory", "Directory to scan for log files", 315 config_add_option(p, "InputDirectory", "Directory to scan for log files",
295 config_set_string, (void *)APR_OFFSETOF(config_t, input_dir)); 316 config_set_string, (void *)APR_OFFSETOF(config_t, input_dir));
296 config_add_option(p, "InputFile", "Parse only this file", 317 config_add_option(p, "InputFile", "Parse only this file",
diff --git a/utility/config.h b/utility/config.h
index d4dde77..c627662 100644
--- a/utility/config.h
+++ b/utility/config.h
@@ -29,6 +29,12 @@ struct config_t {
29 apr_file_t *errorlog_fperr; 29 apr_file_t *errorlog_fperr;
30 apr_pool_t *errorlog_p; 30 apr_pool_t *errorlog_p;
31 31
32 const char *badlinefile;
33 apr_file_t *badline_fp;
34 int badline_count;
35 int badlinemax;
36
37
32 /** input directory of log files */ 38 /** input directory of log files */
33 const char *input_dir; 39 const char *input_dir;
34 /** list of files to scan */ 40 /** list of files to scan */
@@ -73,6 +79,7 @@ struct config_filestat_t {
73 char *fname; 79 char *fname;
74 apr_size_t linesparsed; 80 apr_size_t linesparsed;
75 apr_size_t lineskipped; 81 apr_size_t lineskipped;
82 apr_size_t linesbad;
76 const char *result; 83 const char *result;
77 apr_time_t start; 84 apr_time_t start;
78 apr_time_t stop; 85 apr_time_t stop;
diff --git a/utility/logparse.c b/utility/logparse.c
index 097e66d..ec587a8 100644
--- a/utility/logparse.c
+++ b/utility/logparse.c
@@ -212,6 +212,56 @@ void parser_find_logs(config_t *cfg)
212 apr_pool_destroy(tp); 212 apr_pool_destroy(tp);
213} 213}
214 214
215apr_status_t parser_logbadline(config_t *cfg, const char *filename,
216 const char *badline)
217{
218 apr_status_t rv = APR_SUCCESS;
219 apr_size_t len;
220 struct iovec vec[5];
221
222 if (cfg->badlinefile) {
223 if (!cfg->badline_fp) {
224 rv = apr_file_open(&cfg->badline_fp, cfg->badlinefile,
225 APR_FOPEN_CREATE | APR_FOPEN_WRITE | APR_FOPEN_APPEND,
226 APR_OS_DEFAULT, cfg->pool);
227 if (rv) {
228 logging_log(cfg, LOGLEVEL_NOISE,
229 "Error opening badline file %s\n", cfg->badlinefile);
230 cfg->badlinefile = NULL;
231 } else {
232 char date[APR_RFC822_DATE_LEN];
233 vec[0].iov_base = "Starting BadLines for \"";
234 vec[0].iov_len = sizeof("Starting BadLines for \"")-1;
235 vec[1].iov_base = (void *)filename;
236 vec[1].iov_len = strlen(filename);
237 vec[2].iov_base = "\" on ";
238 vec[2].iov_len = sizeof("\" on ")-1;
239 apr_rfc822_date(date, apr_time_now());
240 vec[3].iov_base = date;
241 vec[3].iov_len = APR_RFC822_DATE_LEN-1;
242 vec[4].iov_base = "\n";
243 vec[4].iov_len = 1;
244 apr_file_writev(cfg->badline_fp, vec,5, &len);
245 }
246 }
247 if (!rv) {
248 if ((++cfg->badline_count) > cfg->badlinemax) {
249 logging_log(cfg, LOGLEVEL_NOISE,
250 "Found more than %d bad lines (found %d)",
251 cfg->badlinemax, cfg->badline_count);
252 rv = APR_EINVAL;
253 } else {
254 vec[0].iov_base = (void *)badline;
255 vec[0].iov_len = strlen(badline);
256 vec[1].iov_base = "\n";
257 vec[1].iov_len = 1;
258 apr_file_writev(cfg->badline_fp, vec,2, &len);
259 }
260 }
261 }
262 return rv;
263}
264
215/* 265/*
216 * Modified version of apr_tokenize_to_argv to add [] as quoting characters 266 * Modified version of apr_tokenize_to_argv to add [] as quoting characters
217 * 267 *
@@ -406,12 +456,17 @@ apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat)
406 rv = parser_processline(targp, cfg, fstat, targv, targc); 456 rv = parser_processline(targp, cfg, fstat, targv, targc);
407 if (rv != APR_SUCCESS) { 457 if (rv != APR_SUCCESS) {
408 int i; 458 int i;
409 if (!cfg->dryrun) database_trans_abort(cfg); 459
410 logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed, 460 fstat->linesbad++;
411 targc, buff); 461 rv = parser_logbadline(cfg, fstat->fname, buff);
412 for (i = 0; targv[i]; i++) { 462 if (rv) {
413 logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i, 463 if (!cfg->dryrun) database_trans_abort(cfg);
414 targv[i]); 464 logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed,
465 targc, buff);
466 for (i = 0; targv[i]; i++) {
467 logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i,
468 targv[i]);
469 }
415 } 470 }
416 } 471 }
417 } else { 472 } else {
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
index f26f113..5b2c0f9 100644
--- a/utility/mod_log_sql.conf
+++ b/utility/mod_log_sql.conf
@@ -1,5 +1,6 @@
1InputDirectory ./logs 1InputDirectory ./logs
2ErrorLog ./error_log 2ErrorLog ./error_log
3
3DBDDriver mysql 4DBDDriver mysql
4DBDParams "host=localhost;user=root;dbname=apache_log" 5DBDParams "host=localhost;user=root;dbname=apache_log"
5Table access_log 6Table access_log
@@ -9,6 +10,9 @@ LogLevel debug
9DryRun off 10DryRun off
10Summary on 11Summary on
11 12
13BadLineFile ./badlines.log
14BadLineMax 10
15
12LogFormatConfig CLF remhost String 16LogFormatConfig CLF remhost String
13LogFormatConfig CLF ident String 17LogFormatConfig CLF ident String
14LogFormatConfig CLF user String 18LogFormatConfig CLF user String
diff --git a/utility/shell.c b/utility/shell.c
index 2c19bb4..0e9d646 100644
--- a/utility/shell.c
+++ b/utility/shell.c
@@ -64,13 +64,13 @@ void print_summary(config_t *cfg) {
64 printf("Execution Summary\n"); 64 printf("Execution Summary\n");
65 for (i=0, m=cfg->input_files->nelts; i<m; i++) { 65 for (i=0, m=cfg->input_files->nelts; i<m; i++) {
66 printf(" File: %s\n" 66 printf(" File: %s\n"
67 " Lines Parsed %d out of %d (Skipped %d)\n" 67 " Lines Parsed %d out of %d (Skipped %d, Bad %d)\n"
68 " Status: %s\n" 68 " Status: %s\n"
69 " Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n" 69 " Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n"
70 "\n", 70 "\n",
71 fstat[i].fname, 71 fstat[i].fname,
72 fstat[i].linesparsed - fstat[i].lineskipped, 72 fstat[i].linesparsed - fstat[i].lineskipped - fstat[i].linesbad,
73 fstat[i].linesparsed, fstat[i].lineskipped, 73 fstat[i].linesparsed, fstat[i].lineskipped, fstat[i].linesbad,
74 fstat[i].result, 74 fstat[i].result,
75 apr_time_sec(fstat[i].stop - fstat[i].start)/60, 75 apr_time_sec(fstat[i].stop - fstat[i].start)/60,
76 apr_time_sec(fstat[i].stop - fstat[i].start), 76 apr_time_sec(fstat[i].stop - fstat[i].start),