From 99867e8a2eca4421075900e44f24cfd749db7dcb Mon Sep 17 00:00:00 2001 From: Edward Rudd Date: Wed, 11 Mar 2009 04:27:34 +0000 Subject: add in splitting code fixed time display (modulus of seconds) --- (limited to 'utility/logparse.c') diff --git a/utility/logparse.c b/utility/logparse.c index 534703d..7267682 100644 --- a/utility/logparse.c +++ b/utility/logparse.c @@ -200,18 +200,180 @@ void parser_find_logs(config_t *cfg) if (apr_dir_open(&dir, cfg->input_dir, tp)==APR_SUCCESS) { while (apr_dir_read(&finfo, APR_FINFO_NAME | APR_FINFO_TYPE, dir) == APR_SUCCESS) { + char *temp; if (finfo.filetype == APR_DIR) continue; newp = (config_filestat_t *)apr_array_push(cfg->input_files); newp->result = "Not Parsed"; - apr_filepath_merge(&(newp->fname), cfg->input_dir, finfo.name, + apr_filepath_merge(&temp, cfg->input_dir, finfo.name, APR_FILEPATH_TRUENAME, cfg->pool); + newp->fname = temp; } apr_dir_close(dir); } apr_pool_destroy(tp); } +#define BUFFER_SIZE (16 * 1024) + +void parser_split_logs(config_t *cfg) +{ + apr_pool_t *tp, *tfp; + apr_array_header_t *foundfiles; + config_filestat_t *filelist; + config_filestat_t *newfile; + apr_file_t *infile; + int f, l; + apr_status_t rv; + apr_finfo_t finfo; + char buff[BUFFER_SIZE]; + int linecount; + int piecesize; + + if (!cfg->split_enabled) return; + if (!cfg->split_dir) { + logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Missing Split Output directory"); + return; + } + apr_pool_create(&tp, cfg->pool); + apr_pool_create(&tfp, tp); + + if (APR_SUCCESS != apr_stat(&finfo, cfg->split_dir, APR_FINFO_MIN, tp)) { + logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Directory %s does not exist", cfg->split_dir); + return; + } + foundfiles = apr_array_copy(tp, cfg->input_files); + apr_array_clear(cfg->input_files); + + filelist = (config_filestat_t *)foundfiles->elts; + for (f=0, l=foundfiles->nelts; f < l; f++) { + apr_pool_clear(tfp); + logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Begin Splitting Log File '%s'", filelist[f].fname); + rv = apr_file_open(&infile, filelist[f].fname, APR_FOPEN_READ, APR_OS_DEFAULT, tfp); + + if (rv != APR_SUCCESS) { + logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s", filelist[f].fname); + return; + } + linecount = 0; + while (apr_file_eof(infile) == APR_SUCCESS) { + apr_size_t read = BUFFER_SIZE; + char *p; + apr_file_read(infile, buff, &read); + p = buff; + while ((p = memchr(p, '\n', (buff + read) - p))) { + ++p; + ++linecount; + } + } + printf("Lines %'d\n",linecount); + // now we know how long it is. Lets split up the file + piecesize = linecount / cfg->split_count; + if (piecesize < cfg->split_minimum) + piecesize = cfg->split_minimum; + if (piecesize > cfg->split_maximum && cfg->split_maximum > 0) + piecesize = cfg->split_maximum; + printf("Piece size %'d\n", piecesize); + if (piecesize > linecount) { + // File is smaller than piece size just add it back in as is + newfile = (config_filestat_t *)apr_array_push(cfg->input_files); + newfile->result = "Not Parsed"; + newfile->fname = filelist[f].fname; + } else { + //split apart the files + int cur_line = 0; + int file_count = 1; + int out_lines = 0; + const char *basefile, *file; + apr_file_t *outfile; + char trail[2048]; + apr_size_t trail_size = 0; + apr_size_t write; + apr_off_t off = 0; + + apr_file_seek(infile, APR_SET, &off); + + basefile = apr_pstrdup(tfp, basename(apr_pstrdup(tfp, filelist[f].fname))); + + file = apr_psprintf(tfp, "%s/%s-%d", cfg->split_dir, basefile, file_count++); + printf("Out file %s\n", file); + logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Creating output file %s", file); + rv = apr_file_open(&outfile, file, APR_FOPEN_WRITE | APR_FOPEN_CREATE | APR_FOPEN_TRUNCATE, APR_OS_DEFAULT, tfp); + if (rv != APR_SUCCESS) { + logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s (%d)", file, rv); + return; + } + newfile = (config_filestat_t *)apr_array_push(cfg->input_files); + newfile->result = "Not Parsed"; + newfile->fname = apr_pstrdup(cfg->pool, file); + + while (apr_file_eof(infile) == APR_SUCCESS) { + apr_size_t read = BUFFER_SIZE; + char *p, *pp, *buff_start; + apr_file_read(infile, buff, &read); + buff_start = p = pp = buff; + if (trail_size) { + p = memchr(p, '\n', (buff + read) - p); + if (p) { + //printf("Trail Line: %p, %p, %d\n", pp, p, (p - pp) + trail_size); + ++p; + pp = p; + ++cur_line; + ++out_lines; + // write out to file + apr_file_write(outfile, trail, &trail_size); + trail_size = 0; + } else { + if ((read + trail_size) > 2048) { + logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Excessively long line %d in file %s", cur_line, filelist[f].fname); + exit(1); + } else { + memcpy(trail+trail_size, buff, read); + trail_size += read; + } + } + } + while ((p = memchr(p, '\n', (buff + read) - p))) { + //printf("Line: %p, %p, %d\n", pp, p, (p - pp)); + if (out_lines == piecesize) { + // Write out to file + write = pp - buff_start; + apr_file_write(outfile, buff_start, &write); + buff_start = pp; + out_lines = 0; + // Open new file + file = apr_psprintf(tfp, "%s/%s-%d", cfg->split_dir, basefile, file_count++); + printf("Out file %s\n", file); + logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Creating output file %s", file); + rv = apr_file_open(&outfile, file, APR_FOPEN_WRITE | APR_FOPEN_CREATE | APR_FOPEN_TRUNCATE, APR_OS_DEFAULT, tfp); + if (rv != APR_SUCCESS) { + logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s (%d)", file, rv); + return; + } + newfile = (config_filestat_t *)apr_array_push(cfg->input_files); + newfile->result = "Not Parsed"; + newfile->fname = apr_pstrdup(cfg->pool, file); + } + ++p; + pp = p; + ++cur_line; + ++out_lines; + } + // Write out to file + write = pp - buff_start; + apr_file_write(outfile, buff_start, &write); + + trail_size = (buff+read) - pp; + if (trail_size) { + memcpy(trail, pp, trail_size); + } + } + } + } + apr_pool_destroy(tfp); + apr_pool_destroy(tp); +} + apr_status_t parser_logbadline(config_t *cfg, const char *filename, const char *badline) { @@ -392,6 +554,7 @@ apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat) apr_file_t *file; apr_status_t rv; char buff[2048]; + char readbuff[BUFFER_SIZE]; char **targv; int targc; @@ -400,8 +563,8 @@ apr_status_t parser_parsefile(config_t *cfg, config_filestat_t *fstat) logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", fstat->fname); - rv = apr_file_open(&file, fstat->fname, APR_FOPEN_READ | APR_BUFFERED, - APR_OS_DEFAULT, tp); + rv = apr_file_open(&file, fstat->fname, APR_FOPEN_READ, APR_OS_DEFAULT, tp); + apr_file_buffer_set(file, readbuff, BUFFER_SIZE); if (rv != APR_SUCCESS) { logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", fstat->fname); return rv; -- cgit v0.9.2