summaryrefslogtreecommitdiffstatsabout
diff options
context:
space:
mode:
-rw-r--r--utility/config.c15
-rw-r--r--utility/config.h9
-rw-r--r--utility/logparse.c90
-rw-r--r--utility/logparse.h6
-rw-r--r--utility/mod_log_sql.conf2
-rw-r--r--utility/shell.c35
6 files changed, 114 insertions, 43 deletions
diff --git a/utility/config.c b/utility/config.c
index 28bb5cd..3b6e946 100644
--- a/utility/config.c
+++ b/utility/config.c
@@ -55,11 +55,12 @@ static apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt,
55static apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt, 55static apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt,
56 int argc, const char **argv) 56 int argc, const char **argv)
57{ 57{
58 char **newp; 58 config_filestat_t *newp;
59 if (argc != 2) 59 if (argc != 2)
60 return APR_EINVAL; 60 return APR_EINVAL;
61 newp = (char **)apr_array_push(cfg->input_files); 61 newp = (config_filestat_t *)apr_array_push(cfg->input_files);
62 *newp = apr_pstrdup(cfg->pool, argv[1]); 62 newp->fname = apr_pstrdup(cfg->pool, argv[1]);
63 newp->result = "Not Parsed";
63 return APR_SUCCESS; 64 return APR_SUCCESS;
64} 65}
65 66
@@ -241,20 +242,20 @@ void config_dump(config_t *cfg)
241 printf("\n"); 242 printf("\n");
242 } 243 }
243 printf("Filters:\n>> Line:\n"); 244 printf("Filters:\n>> Line:\n");
244 filters = cfg->linefilters->elts; 245 filters = (config_filter_t *)cfg->linefilters->elts;
245 for (i=0; i<cfg->linefilters->nelts; i++) { 246 for (i=0; i<cfg->linefilters->nelts; i++) {
246 printf(">>>> %c \"%s\" (%pp)\n",filters[i].negative ? '-':'+', 247 printf(">>>> %c \"%s\" (%pp)\n",filters[i].negative ? '-':'+',
247 filters[i].filter, filters[i].regex); 248 filters[i].filter, filters[i].regex);
248 } 249 }
249 printf(">> Pre:\n"); 250 printf(">> Pre:\n");
250 filters = cfg->prefilters->elts; 251 filters = (config_filter_t *)cfg->prefilters->elts;
251 for (i=0; i<cfg->prefilters->nelts; i++) { 252 for (i=0; i<cfg->prefilters->nelts; i++) {
252 printf(">>>> %s %c \"%s\" (%pp)\n", 253 printf(">>>> %s %c \"%s\" (%pp)\n",
253 filters[i].field, filters[i].negative ? '-':'+', 254 filters[i].field, filters[i].negative ? '-':'+',
254 filters[i].filter, filters[i].regex); 255 filters[i].filter, filters[i].regex);
255 } 256 }
256 printf(">> Post:\n"); 257 printf(">> Post:\n");
257 filters = cfg->postfilters->elts; 258 filters = (config_filter_t *)cfg->postfilters->elts;
258 for (i=0; i<cfg->postfilters->nelts; i++) { 259 for (i=0; i<cfg->postfilters->nelts; i++) {
259 printf(">>>> %s %c \"%s\" (%pp)\n", 260 printf(">>>> %s %c \"%s\" (%pp)\n",
260 filters[i].field, filters[i].negative ? '-':'+', 261 filters[i].field, filters[i].negative ? '-':'+',
@@ -345,7 +346,7 @@ config_t *config_create(apr_pool_t *p)
345 cfg->loglevel = LOGLEVEL_ERROR; 346 cfg->loglevel = LOGLEVEL_ERROR;
346 cfg->summary = 1; 347 cfg->summary = 1;
347 cfg->transactions = 1; 348 cfg->transactions = 1;
348 cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(char *)); 349 cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(config_filestat_t));
349 cfg->log_formats = apr_hash_make(cfg->pool); 350 cfg->log_formats = apr_hash_make(cfg->pool);
350 cfg->output_fields = apr_array_make(cfg->pool, 10, 351 cfg->output_fields = apr_array_make(cfg->pool, 10,
351 sizeof(config_output_field_t)); 352 sizeof(config_output_field_t));
diff --git a/utility/config.h b/utility/config.h
index 26a4e99..d4dde77 100644
--- a/utility/config.h
+++ b/utility/config.h
@@ -68,6 +68,15 @@ struct config_t {
68 int summary; 68 int summary;
69}; 69};
70 70
71typedef struct config_filestat_t config_filestat_t;
72struct config_filestat_t {
73 char *fname;
74 apr_size_t linesparsed;
75 apr_size_t lineskipped;
76 const char *result;
77 apr_time_t start;
78 apr_time_t stop;
79};
71 80
72typedef struct config_logformat_t config_logformat_t; 81typedef struct config_logformat_t config_logformat_t;
73struct config_logformat_t { 82struct config_logformat_t {
diff --git a/utility/logparse.c b/utility/logparse.c
index 923b581..afb1824 100644
--- a/utility/logparse.c
+++ b/utility/logparse.c
@@ -188,7 +188,7 @@ void parser_find_logs(config_t *cfg)
188 apr_pool_t *tp; 188 apr_pool_t *tp;
189 apr_dir_t *dir; 189 apr_dir_t *dir;
190 apr_finfo_t finfo; 190 apr_finfo_t finfo;
191 char **newp; 191 config_filestat_t *newp;
192 192
193 logging_log(cfg, LOGLEVEL_NOTICE, "Find Log files"); 193 logging_log(cfg, LOGLEVEL_NOTICE, "Find Log files");
194 if (!cfg->input_dir) 194 if (!cfg->input_dir)
@@ -199,9 +199,10 @@ void parser_find_logs(config_t *cfg)
199 == APR_SUCCESS) { 199 == APR_SUCCESS) {
200 if (finfo.filetype == APR_DIR) 200 if (finfo.filetype == APR_DIR)
201 continue; 201 continue;
202 newp = (char **)apr_array_push(cfg->input_files); 202 newp = (config_filestat_t *)apr_array_push(cfg->input_files);
203 apr_filepath_merge(newp, cfg->input_dir, finfo.name, 203 newp->result = "Not Parsed";
204 APR_FILEPATH_TRUENAME, cfg->pool); 204 apr_filepath_merge(&(newp->fname), cfg->input_dir, finfo.name,
205 APR_FILEPATH_TRUENAME, cfg->pool);
205 } 206 }
206 apr_dir_close(dir); 207 apr_dir_close(dir);
207 } 208 }
@@ -336,7 +337,7 @@ static apr_status_t tokenize_logline(const char *arg_str, char ***argv_out,
336 return APR_SUCCESS; 337 return APR_SUCCESS;
337} 338}
338 339
339apr_status_t parse_logfile(config_t *cfg, const char *filename) 340apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat)
340{ 341{
341 apr_pool_t *tp, *targp; 342 apr_pool_t *tp, *targp;
342 apr_file_t *file; 343 apr_file_t *file;
@@ -344,23 +345,27 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
344 char buff[2048]; 345 char buff[2048];
345 char **targv; 346 char **targv;
346 int targc; 347 int targc;
347 int line;
348 348
349 apr_pool_create(&tp, cfg->pool); 349 apr_pool_create(&tp, cfg->pool);
350 apr_pool_create(&targp, tp); 350 apr_pool_create(&targp, tp);
351 351
352 logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", filename); 352 logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", fstat->fname);
353 353
354 rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED, 354 rv = apr_file_open(&file, fstat->fname, APR_FOPEN_READ | APR_BUFFERED,
355 APR_OS_DEFAULT, tp); 355 APR_OS_DEFAULT, tp);
356 if (rv != APR_SUCCESS) { 356 if (rv != APR_SUCCESS) {
357 logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", filename); 357 logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", fstat->fname);
358 return rv; 358 return rv;
359 } 359 }
360 360
361 line = 0; 361 fstat->linesparsed = 0;
362 // Start Transaction 362 // Start Transaction
363 database_trans_start(cfg,tp); 363 fstat->start = apr_time_now();
364 if (database_trans_start(cfg,tp)) {
365 fstat->result = "Database Transaction Error";
366 fstat->stop = apr_time_now();
367 return rv;
368 }
364 369
365 do { 370 do {
366 rv = apr_file_gets(buff, 1024, file); 371 rv = apr_file_gets(buff, 1024, file);
@@ -368,7 +373,7 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
368 int i,m, cont = 0; 373 int i,m, cont = 0;
369 config_filter_t *filters; 374 config_filter_t *filters;
370 375
371 line++; 376 fstat->linesparsed++;
372 // chomp off newline 377 // chomp off newline
373 line_chomp(buff); 378 line_chomp(buff);
374 // Run line filters 379 // Run line filters
@@ -379,12 +384,13 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
379 if (filters[i].negative) { 384 if (filters[i].negative) {
380 logging_log(cfg, LOGLEVEL_DEBUG, 385 logging_log(cfg, LOGLEVEL_DEBUG,
381 "PARSER: LINEFILTER: Skipping Line %d due to Filter (%d)%s", 386 "PARSER: LINEFILTER: Skipping Line %d due to Filter (%d)%s",
382 line, i, filters[i].filter); 387 fstat->linesparsed, i, filters[i].filter);
388 fstat->lineskipped++;
383 cont = 1; 389 cont = 1;
384 } else { 390 } else {
385 logging_log(cfg, LOGLEVEL_DEBUG, 391 logging_log(cfg, LOGLEVEL_DEBUG,
386 "PARSER: LINEFILTER: Force Parsing Line %d due to Filter (%d)%s", 392 "PARSER: LINEFILTER: Force Parsing Line %d due to Filter (%d)%s",
387 line, i, filters[i].filter); 393 fstat->linesparsed, i, filters[i].filter);
388 } 394 }
389 break; 395 break;
390 } 396 }
@@ -396,11 +402,11 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
396 targc = 0; 402 targc = 0;
397 while (targv[targc]) 403 while (targv[targc])
398 targc++; 404 targc++;
399 rv = parse_processline(targp, cfg, line, targv, targc); 405 rv = parser_processline(targp, cfg, fstat, targv, targc);
400 if (rv != APR_SUCCESS) { 406 if (rv != APR_SUCCESS) {
401 int i; 407 int i;
402 database_trans_abort(cfg); 408 database_trans_abort(cfg);
403 logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", line, 409 logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed,
404 targc, buff); 410 targc, buff);
405 for (i = 0; targv[i]; i++) { 411 for (i = 0; targv[i]; i++) {
406 logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i, 412 logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i,
@@ -414,17 +420,25 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
414 } while (rv == APR_SUCCESS); 420 } while (rv == APR_SUCCESS);
415 apr_file_close(file); 421 apr_file_close(file);
416 // Finish Transaction 422 // Finish Transaction
417 database_trans_stop(cfg,tp); 423 if (database_trans_stop(cfg,tp)) {
424 fstat->result = apr_psprintf(cfg->pool,
425 "Input line %d, Database Transaction Error",
426 fstat->linesparsed);
427 }
418 428
419 apr_pool_destroy(tp); 429 apr_pool_destroy(tp);
420 logging_log(cfg, LOGLEVEL_NOTICE, 430 logging_log(cfg, LOGLEVEL_NOTICE,
421 "PARSER: Finish Parsing Log File '%s'. Lines: %d", filename, line); 431 "PARSER: Finish Parsing Log File '%s'. Lines: (%d/%d)",
422 432 fstat->fname, fstat->linesparsed - fstat->lineskipped, fstat->linesparsed);
433 if (!rv) {
434 fstat->result = "File Parsed Succesfully";
435 }
436 fstat->stop = apr_time_now();
423 return rv; 437 return rv;
424} 438}
425 439
426apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, 440apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg,
427 char **argv, int argc) 441 config_filestat_t *fstat, char **argv, int argc)
428{ 442{
429 config_logformat_t *fmt; 443 config_logformat_t *fmt;
430 config_logformat_field_t *ifields; 444 config_logformat_field_t *ifields;
@@ -436,10 +450,19 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
436 int i,m; 450 int i,m;
437 451
438 fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING); 452 fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING);
439 if (!fmt) 453 if (!fmt) {
454 logging_log(cfg, LOGLEVEL_NOISE, "PARSER: No Input Log format");
440 return APR_EINVAL; 455 return APR_EINVAL;
441 if (fmt->fields->nelts != argc) 456 }
457 if (fmt->fields->nelts != argc) {
458 logging_log(cfg, LOGLEVEL_NOISE,
459 "PARSER: Input line field number differs from expected. Expected %d got %d.",
460 fmt->fields->nelts, argc);
461 fstat->result = apr_psprintf(cfg->pool,
462 "Input line %d is badly formatted (wrong number of fields)",
463 fstat->linesparsed);
442 return APR_EINVAL; 464 return APR_EINVAL;
465 }
443 466
444 datain = apr_table_make(ptemp, fmt->fields->nelts); 467 datain = apr_table_make(ptemp, fmt->fields->nelts);
445 dataout = apr_table_make(ptemp, cfg->output_fields->nelts); 468 dataout = apr_table_make(ptemp, cfg->output_fields->nelts);
@@ -457,12 +480,13 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
457 if (filters[i].negative) { 480 if (filters[i].negative) {
458 logging_log(cfg, LOGLEVEL_DEBUG, 481 logging_log(cfg, LOGLEVEL_DEBUG,
459 "PARSER: PREFILTER: Skipping Line %d due to Filter (%d)%s", 482 "PARSER: PREFILTER: Skipping Line %d due to Filter (%d)%s",
460 line, i, filters[i].filter); 483 fstat->linesparsed, i, filters[i].filter);
484 fstat->lineskipped++;
461 return APR_SUCCESS; 485 return APR_SUCCESS;
462 } else { 486 } else {
463 logging_log(cfg, LOGLEVEL_DEBUG, 487 logging_log(cfg, LOGLEVEL_DEBUG,
464 "PARSER: PREFILTER: Force Parsing Line %d due to Filter (%d)%s", 488 "PARSER: PREFILTER: Force Parsing Line %d due to Filter (%d)%s",
465 line, i, filters[i].filter); 489 fstat->linesparsed, i, filters[i].filter);
466 } 490 }
467 break; 491 break;
468 } 492 }
@@ -486,8 +510,12 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
486 const char *ret= NULL; 510 const char *ret= NULL;
487 rv = ((parser_func_t *)ofields[i].func)->func(ptemp, cfg, 511 rv = ((parser_func_t *)ofields[i].func)->func(ptemp, cfg,
488 &ofields[i], val, &ret); 512 &ofields[i], val, &ret);
489 if (rv) 513 if (rv) {
514 fstat->result = apr_psprintf(cfg->pool,
515 "Input line %d, Parser function %s returned error (%d)%s",
516 fstat->linesparsed, ofields[i].fname, rv, logging_strerror(rv));
490 return rv; 517 return rv;
518 }
491 apr_table_setn(dataout, ofields[i].field, ret ? ret : ofields[i].def); 519 apr_table_setn(dataout, ofields[i].field, ret ? ret : ofields[i].def);
492 } 520 }
493 } 521 }
@@ -501,12 +529,13 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
501 if (filters[i].negative) { 529 if (filters[i].negative) {
502 logging_log(cfg, LOGLEVEL_DEBUG, 530 logging_log(cfg, LOGLEVEL_DEBUG,
503 "PARSER: POSTFILTER: Skipping Line %d due to Filter (%d)%s", 531 "PARSER: POSTFILTER: Skipping Line %d due to Filter (%d)%s",
504 line, i, filters[i].filter); 532 fstat->linesparsed, i, filters[i].filter);
533 fstat->lineskipped++;
505 return APR_SUCCESS; 534 return APR_SUCCESS;
506 } else { 535 } else {
507 logging_log(cfg, LOGLEVEL_DEBUG, 536 logging_log(cfg, LOGLEVEL_DEBUG,
508 "PARSER: POSTFILTER: Force Parsing Line %d due to Filter (%d)%s", 537 "PARSER: POSTFILTER: Force Parsing Line %d due to Filter (%d)%s",
509 line, i, filters[i].filter); 538 fstat->linesparsed, i, filters[i].filter);
510 } 539 }
511 break; 540 break;
512 } 541 }
@@ -515,6 +544,11 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line,
515 // Process DB Query 544 // Process DB Query
516 if (!cfg->dryrun) { 545 if (!cfg->dryrun) {
517 rv = database_insert(cfg, ptemp, dataout); 546 rv = database_insert(cfg, ptemp, dataout);
547 if (rv) {
548 fstat->result = apr_psprintf(cfg->pool,
549 "Input line %d, Database Error",
550 fstat->linesparsed);
551 }
518 } 552 }
519 return rv; 553 return rv;
520} 554}
diff --git a/utility/logparse.h b/utility/logparse.h
index 0b8607b..fe708a5 100644
--- a/utility/logparse.h
+++ b/utility/logparse.h
@@ -23,9 +23,9 @@ void parser_init(apr_pool_t *p);
23 23
24void parser_find_logs(config_t *cfg); 24void parser_find_logs(config_t *cfg);
25 25
26apr_status_t parse_logfile(config_t *cfg, const char *filename); 26apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat);
27 27
28apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, int line, 28apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg,
29 char **argv, int argc); 29 config_filestat_t *line, char **argv, int argc);
30 30
31#endif /*LOGPARSE_H_*/ 31#endif /*LOGPARSE_H_*/
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
index 54ac85d..de9cd28 100644
--- a/utility/mod_log_sql.conf
+++ b/utility/mod_log_sql.conf
@@ -34,7 +34,7 @@ LogFormat Combined
34#LineFilter + "BETTER" 34#LineFilter + "BETTER"
35# the next filter ignores ALL lines 35# the next filter ignores ALL lines
36#LineFilter - 36#LineFilter -
37#PreFilter request + "Rebuild" 37#PreFilter request - "Rebuild"
38#PostFilter request_method "GET" 38#PostFilter request_method "GET"
39 39
40# Usage field datatype(size) default source [function [param]...] 40# Usage field datatype(size) default source [function [param]...]
diff --git a/utility/shell.c b/utility/shell.c
index 5f011a6..6f98055 100644
--- a/utility/shell.c
+++ b/utility/shell.c
@@ -55,6 +55,30 @@ void show_help(const char *prog, const apr_getopt_option_t *opts, FILE *output)
55 } 55 }
56} 56}
57 57
58void print_summary(config_t *cfg) {
59 config_filestat_t *fstat;
60 int i,m;
61
62 fstat = (config_filestat_t *)cfg->input_files->elts;
63
64 printf("Execution Summary\n");
65 for (i=0, m=cfg->input_files->nelts; i<m; i++) {
66 printf(" File: %s\n"
67 " Lines Parsed %d out of %d (Skipped %d)\n"
68 " Status: %s\n"
69 " Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n"
70 "\n",
71 fstat[i].fname,
72 fstat[i].linesparsed - fstat[i].lineskipped,
73 fstat[i].linesparsed, fstat[i].lineskipped,
74 fstat[i].result,
75 apr_time_sec(fstat[i].stop - fstat[i].start)/60,
76 apr_time_sec(fstat[i].stop - fstat[i].start),
77 apr_time_msec(fstat[i].stop - fstat[i].start)
78 );
79 }
80}
81
58int main(int argc, const char *const argv[]) 82int main(int argc, const char *const argv[])
59{ 83{
60 apr_pool_t *pool, *ptemp; 84 apr_pool_t *pool, *ptemp;
@@ -166,11 +190,11 @@ int main(int argc, const char *const argv[])
166 } 190 }
167 } 191 }
168 if (!apr_is_empty_array(cfg->input_files)) { 192 if (!apr_is_empty_array(cfg->input_files)) {
169 char **filelist; 193 config_filestat_t *filelist;
170 int f, l; 194 int f, l;
171 filelist = (char **)cfg->input_files->elts; 195 filelist = (config_filestat_t *)cfg->input_files->elts;
172 for (f=0, l=cfg->input_files->nelts; f < l; f++) { 196 for (f=0, l=cfg->input_files->nelts; f < l; f++) {
173 rv = parse_logfile(cfg, filelist[f]); 197 rv = parser_parsefile(cfg, &filelist[f]);
174 if (rv) { 198 if (rv) {
175 logging_log(cfg, LOGLEVEL_NOISE, 199 logging_log(cfg, LOGLEVEL_NOISE,
176 "Error occured parsing log files. Aborting"); 200 "Error occured parsing log files. Aborting");
@@ -183,6 +207,9 @@ int main(int argc, const char *const argv[])
183 if (!cfg->dryrun) { 207 if (!cfg->dryrun) {
184 database_disconnect(cfg); 208 database_disconnect(cfg);
185 } 209 }
186 /** @todo summary goes here */ 210
211 if (cfg->summary) {
212 print_summary(cfg);
213 }
187 return 0; 214 return 0;
188} 215}