From cc75ebf7e8560a69a6847f0260cce4772fff440a Mon Sep 17 00:00:00 2001 From: Edward Rudd Date: Wed, 1 Oct 2008 03:57:58 +0000 Subject: Initial revision of command line importer. --- .cproject | 21 ++-- Makefile.in | 2 +- configure.ac | 1 + src/Makefile.in | 3 + utility/Makefile.in | 38 ++++++ utility/config.c | 305 +++++++++++++++++++++++++++++++++++++++++++++++ utility/config.h | 104 ++++++++++++++++ utility/logparse.c | 205 +++++++++++++++++++++++++++++++ utility/logparse.h | 10 ++ utility/mod_log_sql.conf | 30 +++++ utility/shell.c | 148 +++++++++++++++++++++++ utility/shell.h | 0 12 files changed, 858 insertions(+), 9 deletions(-) create mode 100644 utility/Makefile.in create mode 100644 utility/config.c create mode 100644 utility/config.h create mode 100644 utility/logparse.c create mode 100644 utility/logparse.h create mode 100644 utility/mod_log_sql.conf create mode 100644 utility/shell.c create mode 100644 utility/shell.h diff --git a/.cproject b/.cproject index 3ef91b9..ce89a76 100644 --- a/.cproject +++ b/.cproject @@ -11,6 +11,7 @@ + @@ -25,16 +26,15 @@ - - - - - + + + + + + + + - - @@ -100,6 +104,7 @@ + diff --git a/Makefile.in b/Makefile.in index fb4ea42..7d71505 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1,7 +1,7 @@ # @configure_input@ # Modify these top variables. -SUBDIRS = docs contrib src +SUBDIRS = docs contrib src parser utility EXTRA_DIST = AUTHORS INSTALL TODO LICENSE CHANGELOG \ build-apache13.bat build-apache2.bat \ diff --git a/configure.ac b/configure.ac index 07ac486..43633e3 100644 --- a/configure.ac +++ b/configure.ac @@ -85,6 +85,7 @@ dnl Write config.status and the Makefile AC_CONFIG_FILES([Makefile src/Makefile docs/Makefile +utility/Makefile contrib/Makefile]) AC_OUTPUT diff --git a/src/Makefile.in b/src/Makefile.in index 558b621..5a5dd6a 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -163,3 +163,6 @@ clean: local-dist: $(DISTFILES) mkdir -p $(DESTDIR) cp -dp --parents $(DISTFILES) $(DESTDIR) + +Makefile: Makefile.in ../config.status + cd .. && ./config.status diff --git a/utility/Makefile.in b/utility/Makefile.in new file mode 100644 index 0000000..d397b96 --- /dev/null +++ b/utility/Makefile.in @@ -0,0 +1,38 @@ +# @configure_input@ + +top_srcdir = @top_srcdir@ +srcdir = @abs_srcdir@ +builddir = @abs_builddir@ + +CFLAGS = -g -Wall -fno-strict-aliasing @APR_CFLAGS@ @APR_INCLUDES@ @APU_INCLUDES@ +CPPFLAGS = @APR_CPPFLAGS@ +LDFLAGS = @APR_LDFLAGS@ @APU_LDFLAGS@ + +ifeq (@OOO_MAINTAIN@,1) +CFLAGS += -Werror +endif + +STD_DIST = Makefile.in + +DISTFILES = $(STD_DIST) $(EXTRA_DIST) $(SOURCES) $(HEADERS) + +SOURCES = shell.c config.c logparse.c +HEADERS = shell.h config.h logparse.h +OBJECTS = $(patsubst %.c,%.o,$(SOURCES)) + +TARGETS = mod_log_sql + +all: $(TARGETS) + +mod_log_sql: $(OBJECTS) $(HEADERS) + $(CC) -o $@ $(OBJECTS) $(LDFLAGS) + +clean: + $(RM) $(OBJECTS) $(TARGETS) + +local-dist: $(DISTFILES) + mkdir -p $(DESTDIR) + cp -dp --parents $(DISTFILES) $(DESTDIR) + +Makefile: Makefile.in ../config.status + cd .. && ./config.status diff --git a/utility/config.c b/utility/config.c new file mode 100644 index 0000000..6b3dce1 --- /dev/null +++ b/utility/config.c @@ -0,0 +1,305 @@ +#include "apr.h" +#include "apr_file_info.h" +#include "apr_file_io.h" +#include "apr_strings.h" +#include "apr_hash.h" +#include "apr_lib.h" +#include "shell.h" +#include "config.h" + +apr_hash_t *g_config_opts; + +apr_status_t config_set_string(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + int offset = (int)(long)opt->data; + char **data = (char **)((void *)cfg + offset); + if (argc != 2) return APR_EINVAL; + *data = apr_pstrdup(cfg->pool, argv[1]); + return APR_SUCCESS; +} + +apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + int offset = (int)(long)opt->data; + int *data = (int *)((void *)cfg + offset); + if (argc != 2) return APR_EINVAL; + *data = apr_atoi64(argv[1]); + return APR_SUCCESS; +} + +apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + int offset = (int)(long)opt->data; + int *data = (int *)((void *)cfg + offset); + if (argc != 2) return APR_EINVAL; + *data = CHECK_YESNO(argv[1]); + return APR_SUCCESS; +} + +apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + if (argc != 2) return APR_EINVAL; + if (!strcasecmp(argv[1],"error")) { + cfg->loglevel = LOGLEVEL_ERROR; + } else if (!strcasecmp(argv[1],"warn")) { + cfg->loglevel = LOGLEVEL_WARN; + } else if (!strcasecmp(argv[1],"debug")) { + cfg->loglevel = LOGLEVEL_DEBUG; + } else if (!strcasecmp(argv[1],"quiet")) { + cfg->loglevel = LOGLEVEL_QUIET; + } else { + cfg->loglevel = LOGLEVEL_ERROR; + } + return APR_SUCCESS; +} + +apr_status_t config_set_dbconnect(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + return APR_SUCCESS; +} + +apr_status_t config_set_dbparam(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + return APR_SUCCESS; +} + +apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + char **newp; + if (argc != 2) return APR_EINVAL; + newp = (char **)apr_array_push(cfg->input_files); + *newp = apr_pstrdup(cfg->pool, argv[1]); + return APR_SUCCESS; +} + +apr_status_t config_set_dummy(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + return APR_SUCCESS; +} + +apr_status_t config_set_logformat(config_t *cfg, config_opt_t *opt, int argc, + const char **argv) +{ + config_logformat_t *format; + config_logformat_field_t *field; + + if (argc != 4) return APR_EINVAL; + + format = apr_hash_get(cfg->log_formats,argv[1],APR_HASH_KEY_STRING); + if (!format) { + format = apr_palloc(cfg->pool, sizeof(config_logformat_t)); + format->name = apr_pstrdup(cfg->pool, argv[1]); + format->fields = apr_array_make(cfg->pool, 5, + sizeof(config_logformat_field_t)); + apr_hash_set(cfg->log_formats, argv[1], APR_HASH_KEY_STRING, format); + } + field = (config_logformat_field_t *)apr_array_push(format->fields); + field->name = apr_pstrdup(cfg->pool, argv[2]); + field->datatype = apr_pstrdup(cfg->pool, argv[3]); + return APR_SUCCESS; +} + +void config_dump(config_t *cfg) +{ + apr_hash_index_t *hi; + + printf("ErrorLog: %s\n",cfg->errorlog); + printf("LogLevel: %d\n",cfg->loglevel); + + printf("InputDir: %s\n",cfg->input_dir); + + printf("Table: %s\n",cfg->table); + printf("Transactions: %d\n",cfg->transactions); + printf("MachineID: %s\n",cfg->machineid); + + printf("Log formats:\n"); + for (hi = apr_hash_first(cfg->pool, cfg->log_formats); hi; hi + = apr_hash_next(hi)) { + config_logformat_t *format; + config_logformat_field_t *fields; + int i; + + apr_hash_this(hi, NULL, NULL, (void **)&format); + printf(">> %s\n",format->name); + fields = (config_logformat_field_t *)format->fields->elts; + for (i=0; ifields->nelts; i++) { + printf(">>>> %s:%s\n", fields[i].name, fields[i].datatype); + } + } + printf("Log Format: %s\n",cfg->logformat); + + printf("DryRun: %d\n",cfg->dryrun); + printf("Summary: %d\n",cfg->summary); +} + +static char *lowerstr(apr_pool_t *pool, const char *input) { + char *temp; + char *itr; + temp = apr_pstrdup(pool, input); + for (itr=temp; *itr!='\0'; itr++) { + *itr = apr_tolower(*itr); + } + return temp; +} + +#define config_get_option(name) apr_hash_get(g_config_opts, name, APR_HASH_KEY_STRING) + +void config_add_option(apr_pool_t *p, const char *const name, + const char *const help, config_func_t func, void *data) +{ + config_opt_t *opt; + if (!g_config_opts) { + g_config_opts = apr_hash_make(p); + } + opt = apr_palloc(p, sizeof(config_opt_t)); + opt->name = name; + opt->help = help; + opt->func = func; + opt->data = data; + apr_hash_set(g_config_opts, lowerstr(p,name), APR_HASH_KEY_STRING, opt); +} + +void config_init(apr_pool_t *p) +{ + config_add_option(p, "ErrorLog", "File to log errors", + config_set_string, (void *)APR_OFFSETOF(config_t, errorlog)); + config_add_option(p, "LogLevel", "Set Log Level (error, warn, debug, quiet)", + config_set_loglevel, NULL); + + config_add_option(p, "InputDirectory", "Directory to scan for log files", + config_set_string, (void *)APR_OFFSETOF(config_t, input_dir)); + config_add_option(p, "InputFile", "Parse only this file", + config_set_inputfile, NULL); + + config_add_option(p, "DBConnect", "DB Connection information type://user:pass@hostname/database", + config_set_dbconnect, NULL); + config_add_option(p, "DBParam", "DB Connection Parameter", + config_set_dbparam, NULL); + config_add_option(p, "Table", "Table to import the log to", + config_set_string, (void *)APR_OFFSETOF(config_t, table)); + config_add_option(p, "UseTransactions", "Enable Transactions?", + config_set_flag, (void *)APR_OFFSETOF(config_t, transactions)); + config_add_option(p, "MachineID", "Machine ID to set", + config_set_string, (void *)APR_OFFSETOF(config_t, machineid)); + + config_add_option(p, "LogFormatConfig", "Define input log formats", + config_set_logformat, NULL); + config_add_option(p, "LogFormat", "Use this logformat when parsing files", + config_set_string, (void *)APR_OFFSETOF(config_t, logformat)); + + config_add_option(p, "DryRun", "Don't perform any actual database changes", + config_set_flag, (void *)APR_OFFSETOF(config_t, dryrun)); + config_add_option(p, "Config", "Dummy to handle config directive", + config_set_dummy, NULL); + config_add_option(p, "Summary", "Show the summary before exit?", + config_set_flag, (void *)APR_OFFSETOF(config_t, summary)); +} + +config_t *config_create(apr_pool_t *p) +{ + config_t *cfg; + apr_pool_t *sp; + apr_pool_create(&sp, p); + cfg = apr_pcalloc(sp, sizeof(config_t)); + cfg->pool = sp; + cfg->loglevel = LOGLEVEL_WARN; + cfg->summary = 1; + cfg->transactions = 1; + cfg->input_files = apr_array_make(cfg->pool, 10, sizeof(char *)); + cfg->dbconfig = apr_table_make(cfg->pool, 5); + cfg->log_formats = apr_hash_make(cfg->pool); + + return cfg; +} + +int config_merge(void *rec, const char *key, const char *value) { + config_t *cfg = (config_t *)rec; + + config_opt_t *opt = config_get_option(key); + if (opt) { + const char *args[] = {key, value}; + opt->func(cfg, opt, 2, args); + } else { + printf("Unhandled: %s\n", key); + } + return 1; +} + +apr_status_t config_read(config_t *cfg, const char *filename, + apr_table_t *merge) +{ + apr_finfo_t finfo; + apr_file_t *file; + apr_status_t rv, ret = APR_SUCCESS; + apr_pool_t *tp, *targp; + config_opt_t *opt; + char buff[1024]; + char *ptr, *ptr2; + char **targv; + int targc; + int line; + + apr_pool_create(&tp, cfg->pool); + apr_pool_create(&targp, tp); + + if (apr_stat(&finfo, filename, APR_FINFO_MIN, tp) != APR_SUCCESS) { + return APR_ENOENT; + } + rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED, + APR_OS_DEFAULT, tp); + if (rv != APR_SUCCESS) + return rv; + + line = 0; + do { + rv = apr_file_gets(buff, 1024, file); + if (rv == APR_SUCCESS) { // we read data + line++; + + // skip leading white space + for (ptr = buff; *ptr == ' ' || *ptr == '\t'; ptr++) + ; + // chomp off newline + for (ptr2 = ptr + strlen(ptr); *ptr2 != '\r' && *ptr2 != '\n'; ptr2--) + ; + *ptr2 = '\0'; + + // skip comments + if (*ptr == '#') + continue; + if (*ptr == '\0') + continue; + apr_pool_clear(targp); + apr_tokenize_to_argv(buff, &targv, targp); + targc = 0; + while (targv[targc]) targc++; + opt = config_get_option(lowerstr(targp,targv[0])); + if (opt) { + rv = opt->func(cfg, opt, targc, (const char **)targv); + if (APR_STATUS_IS_EINVAL(rv)) { + printf("Config Error: Invalid Arguments for %s\n\t%s\n", + opt->name, opt->help); + ret = rv; + } + } else { + printf("Unhandled: %s\n", targv[0]); + } + } + } while (rv == APR_SUCCESS); + + // Apply merges + apr_table_do(config_merge,(void *)cfg,merge,NULL); + + apr_file_close(file); + apr_pool_destroy(tp); + return ret; +} + diff --git a/utility/config.h b/utility/config.h new file mode 100644 index 0000000..e1827fe --- /dev/null +++ b/utility/config.h @@ -0,0 +1,104 @@ +#ifndef CONFIG_H_ +#define CONFIG_H_ + +#include "apr_tables.h" +#include "apr_hash.h" +#include "apr_file_io.h" + +typedef enum { + LOGLEVEL_QUIET = 0, + LOGLEVEL_ERROR = 1, + LOGLEVEL_WARN = 2, + LOGLEVEL_DEBUG = 3, +} loglevel_e; + +typedef struct { + /** the structures pool (to ease function arguments) */ + apr_pool_t *pool; + + /** error log file */ + const char *errorlog; + /** log level */ + loglevel_e loglevel; + /** error_log */ + apr_file_t *errorlog_fp; + + /** input directory of log files */ + const char *input_dir; + /** list of files to scan */ + apr_array_header_t *input_files; + + /** db connection configuration */ + apr_table_t *dbconfig; + /** Logging table */ + const char *table; + /** Use transactons */ + int transactions; + /** Machine ID */ + const char *machineid; + + /** Log file formats */ + apr_hash_t *log_formats; + /** format to use to parse files */ + const char *logformat; + + /** output fields */ + apr_array_header_t *output_fields; + + /** filter configuration */ + apr_array_header_t *filters; + + /** Dry Run */ + int dryrun; + + /* Show the summary */ + int summary; +} config_t; + +typedef struct { + const char *name; + apr_array_header_t *fields; +} config_logformat_t; + +typedef struct { + const char *name; + const char *datatype; +} config_logformat_field_t; + +typedef struct config_opt_t config_opt_t; +typedef apr_status_t (*config_func_t)(config_t *cfg, config_opt_t *opt, + int argc, const char **argv); + +struct config_opt_t { + const char *name; + const char *help; + config_func_t func; + void *data; +}; + +#define CHECK_YESNO(c) ((!strcasecmp(c,"on") || !strcasecmp(c,"yes")) ? 1 : 0) + +/** + * Initialize the config parser + */ +void config_init(apr_pool_t *p); + +/** + * Dump the configuration to stdout + */ +void config_dump(config_t *cfg); + +/** + * Creates the default configuration + */ +config_t *config_create(apr_pool_t *p); + +/** + * Read in a configuration file + */ +apr_status_t config_read(config_t *cfg, const char *filename, + apr_table_t *merge); + +void config_generate(const char *filename); + +#endif /*CONFIG_H_*/ diff --git a/utility/logparse.c b/utility/logparse.c new file mode 100644 index 0000000..2940534 --- /dev/null +++ b/utility/logparse.c @@ -0,0 +1,205 @@ +#include "logparse.h" +#include "apr_file_info.h" +#include "apr_file_io.h" +#include "apr_strings.h" + +void find_log_files(config_t *cfg) +{ + apr_pool_t *tp; + apr_dir_t *dir; + apr_finfo_t finfo; + char **newp; + + if (!cfg->input_dir) + return; + apr_pool_create(&tp, cfg->pool); + if (apr_dir_open(&dir, cfg->input_dir, tp)==APR_SUCCESS) { + while (apr_dir_read(&finfo, APR_FINFO_NAME | APR_FINFO_TYPE, dir) + == APR_SUCCESS) { + if (finfo.filetype == APR_DIR) + continue; + newp = (char **)apr_array_push(cfg->input_files); + apr_filepath_merge(newp, cfg->input_dir, finfo.name, + APR_FILEPATH_TRUENAME, cfg->pool); + } + apr_dir_close(dir); + } + apr_pool_destroy(tp); +} + +/* + * Modified version of apr_tokenize_to_argv to add [] as quoting characters + * + * token_context: Context from which pool allocations will occur. + * arg_str: Input string for conversion to argv[]. + * argv_out: Output location. This is a pointer to an array + * of pointers to strings (ie. &(char *argv[]). + * This value will be allocated from the contexts + * pool and filled in with copies of the tokens + * found during parsing of the arg_str. + * keepquotes: Keep the quotes instead of stripping them + */ +apr_status_t tokenize_logline(const char *arg_str, char ***argv_out, + apr_pool_t *token_context, int keepquotes) +{ + const char *cp; + const char *ct; + char *cleaned, *dirty; + int escaped; + int isquoted, numargs = 0, argnum; + +#define SKIP_WHITESPACE(cp) \ + for ( ; *cp == ' ' || *cp == '\t'; ) { \ + cp++; \ + }; + +#define CHECK_QUOTATION(cp,isquoted) \ + isquoted = 0; \ + if (*cp == '"') { \ + isquoted = 1; \ + cp++; \ + } \ + else if (*cp == '\'') { \ + isquoted = 2; \ + cp++; \ + } \ + else if (*cp == '[') { \ + isquoted = 3; \ + cp++; \ + } + + /* DETERMINE_NEXTSTRING: + * At exit, cp will point to one of the following: NULL, SPACE, TAB or QUOTE. + * NULL implies the argument string has been fully traversed. + */ +#define DETERMINE_NEXTSTRING(cp,isquoted) \ + for ( ; *cp != '\0'; cp++) { \ + if ( (isquoted && (*cp == ' ' || *cp == '\t')) \ + || (*cp == '\\' && (*(cp+1) == ' ' || *(cp+1) == '\t' || \ + *(cp+1) == '"' || *(cp+1) == '\'' || \ + *(cp+1) == '[' || *(cp+1) == ']'))) { \ + cp++; \ + continue; \ + } \ + if ( (!isquoted && (*cp == ' ' || *cp == '\t')) \ + || (isquoted == 1 && *cp == '"') \ + || (isquoted == 2 && *cp == '\'') \ + || (isquoted == 3 && *cp == ']') \ + ) { \ + break; \ + } \ + } + + /* REMOVE_ESCAPE_CHARS: + * Compresses the arg string to remove all of the '\' escape chars. + * The final argv strings should not have any extra escape chars in it. + */ +#define REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped) \ + escaped = 0; \ + while(*dirty) { \ + if (!escaped && *dirty == '\\') { \ + escaped = 1; \ + } \ + else { \ + escaped = 0; \ + *cleaned++ = *dirty; \ + } \ + ++dirty; \ + } \ + *cleaned = 0; /* last line of macro... */ + + cp = arg_str; + SKIP_WHITESPACE(cp); + ct = cp; + + /* This is ugly and expensive, but if anyone wants to figure a + * way to support any number of args without counting and + * allocating, please go ahead and change the code. + * + * Must account for the trailing NULL arg. + */ + numargs = 1; + while (*ct != '\0') { + CHECK_QUOTATION(ct, isquoted) + ; + DETERMINE_NEXTSTRING(ct, isquoted); + if (*ct != '\0') { + ct++; + } + numargs++; + SKIP_WHITESPACE(ct); + } + *argv_out = apr_palloc(token_context, numargs * sizeof(char*)); + + /* determine first argument */ + for (argnum = 0; argnum < (numargs-1); argnum++) { + SKIP_WHITESPACE(cp); + CHECK_QUOTATION(cp, isquoted) + ; + ct = cp; + DETERMINE_NEXTSTRING(cp, isquoted); + cp++; + if (isquoted && keepquotes) { + (*argv_out)[argnum] = apr_palloc(token_context, cp - ct + 2); + apr_cpystrn((*argv_out)[argnum], ct -1, cp - ct + 2); + } else { + (*argv_out)[argnum] = apr_palloc(token_context, cp - ct); + apr_cpystrn((*argv_out)[argnum], ct, cp - ct); + } + cleaned = dirty = (*argv_out)[argnum]; + REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped) + ; + } + (*argv_out)[argnum] = NULL; + + return APR_SUCCESS; +} + +apr_status_t parse_logfile(config_t *cfg, const char *filename) +{ + apr_pool_t *tp, *argp; + apr_file_t *file; + apr_status_t rv; + char buff[2048]; + char **targv; + int targc; + int line; + + apr_pool_create(&tp, cfg->pool); + apr_pool_create(&argp, tp); + + rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED, + APR_OS_DEFAULT, tp); + if (rv != APR_SUCCESS) { + printf("Could not open %s\n", filename); + return rv; + } + + line = 0; + do { + rv = apr_file_gets(buff, 1024, file); + if (rv == APR_SUCCESS) { + line++; + char *ptr; + // chomp off newline + for (ptr = buff + strlen(buff); *ptr != '\r' && *ptr != '\n'; ptr--) + ; + *ptr = '\0'; + apr_pool_clear(argp); + tokenize_logline(buff, &targv, argp, 1); + targc = 0; + while (targv[targc]) targc++; + if (targc != 9) { + int i; + printf("Line %d(%d): %s\n",line, targc, buff); + for (i = 0; targv[i]; i++) { + printf("Arg (%d): '%s'\n", i, targv[i]); + } + } + } + } while (rv == APR_SUCCESS); + printf("Total Lines: %d\n", line); + apr_file_close(file); + apr_pool_destroy(tp); + return APR_SUCCESS; +} diff --git a/utility/logparse.h b/utility/logparse.h new file mode 100644 index 0000000..540a9e0 --- /dev/null +++ b/utility/logparse.h @@ -0,0 +1,10 @@ +#ifndef LOGPARSE_H_ +#define LOGPARSE_H_ + +#include "config.h" + +void find_log_files(config_t *cfg); + +apr_status_t parse_logfile(config_t *cfg, const char *filename); + +#endif /*LOGPARSE_H_*/ diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf new file mode 100644 index 0000000..6cfae61 --- /dev/null +++ b/utility/mod_log_sql.conf @@ -0,0 +1,30 @@ +InputDirectory ./logs +ErrorLog ./error_log +DBConnect mysql://username:host@server/database +DBParam socketfile /tmp/mysql.sock +Table apache_logs +MachineID 7of9 +UseTransactions on +LogLevel notice +DryRun on +Summary on + +LogFormatConfig CLF host String +LogFormatConfig CLF ident String +LogFormatConfig CLF user String +LogFormatConfig CLF date Date +LogFormatConfig CLF request String +LogFormatConfig CLF status Number +LogFormatConfig CLF bytes_sent Number + +LogFormatConfig Combined host String +LogFormatConfig Combined ident String +LogFormatConfig Combined user String +LogFormatConfig Combined date Date +LogFormatConfig Combined request String +LogFormatConfig Combined status Number +LogFormatConfig Combined bytes_sent Number +LogFormatConfig Combined referer String +LogFormatConfig Combined agent String + +LogFormat Combined diff --git a/utility/shell.c b/utility/shell.c new file mode 100644 index 0000000..07d9da1 --- /dev/null +++ b/utility/shell.c @@ -0,0 +1,148 @@ +#include "apr.h" +#include "apr_getopt.h" +#include "apr_tables.h" + +#define APR_WANT_STDIO +#include "apr_want.h" +#include "stdlib.h" + +#include "shell.h" +#include "config.h" +#include "logparse.h" + +const apr_getopt_option_t _opt_config[] = { + {"machineid", 'm', 1, "Machine ID for the log file"}, + {"transaction", 't', 1, "Use a Transaction (yes,no)"}, + {"logformat", 'r', 1, "Use this logformat to parse files"}, + {"file", 'f', 1, "Parse this single log file (input dir is NOT scanned)"}, + {"inputdir", 'd', 1, "Input Directory to look for log files"}, + {"config", 'c', 1, "Configuration file to use (default mod_log_sql.conf)"}, + {"dryrun", 'n', 0, "Perform a dry run (do not actually alter the databse)"}, + {"loglevel", 'l', 1, "Log Level (deubg, warn, error)"}, + {"summary", 's', 1, "Summary (yes,no)"}, + {"help", 'h', 0, "Show Help"}, + {NULL} +}; + +void show_help(const char *prog, const apr_getopt_option_t *opts, FILE *output) +{ + int ptr = 0; + fprintf(output, "Usage: %s [OPTIONS] [files...]\n\n", prog); + while (opts[ptr].optch != 0) { + if (opts[ptr].optch > 255) { + if (opts[ptr].name) { + fprintf(output, " --%-10s", opts[ptr].name); + } else { + fprintf(output, " "); + } + } else { + if (opts[ptr].name) { + fprintf(output, " -%c --%-10s", opts[ptr].optch, opts[ptr].name); + } else { + fprintf(output, " -%c ", opts[ptr].optch); + } + } + if (opts[ptr].has_arg) { + fprintf(output, " (arg)"); + } else { + fprintf(output, " "); + } + fprintf(output, " %s\n", opts[ptr].description); + ptr++; + } +} + +int main(int argc, const char *const argv[]) +{ + apr_pool_t *pool; + apr_getopt_t *opts; + int opt; + const char *opt_arg; + apr_status_t rv; + apr_table_t *args; + config_t *base; + + apr_app_initialize(&argc, &argv, NULL); + atexit(apr_terminate); + + if (apr_pool_create(&pool, NULL) != APR_SUCCESS) { + fprintf(stderr, "Failed to create memory pool!\n"); + exit(1); + } + + /** Iterate over command line arguments + * shoving args in a apr_table for processing later*/ + args = apr_table_make(pool, 5); + apr_table_setn(args, "config", "mod_log_sql.conf"); + apr_getopt_init(&opts, pool, argc, argv); + while ((rv = apr_getopt_long(opts, _opt_config, &opt, &opt_arg)) == APR_SUCCESS) { + switch (opt) { + case 'c': + apr_table_setn(args,"config",opt_arg); + break; + case 'd': + apr_table_setn(args,"inputdirectory",opt_arg); + break; + case 'f': + apr_table_setn(args,"inputfile",opt_arg); + break; + case 'h': + show_help(argv[0], _opt_config, stdout); + exit(1); + break; + case 'l': + apr_table_setn(args,"loglevel",opt_arg); + break; + case 'm': + apr_table_setn(args,"machineid",opt_arg); + break; + case 'n': + apr_table_setn(args,"dryrun","yes"); + break; + case 'r': + apr_table_setn(args,"logformat",opt_arg); + break; + case 's': + apr_table_setn(args,"summary",opt_arg); + break; + case 't': + apr_table_setn(args,"usetransactions",opt_arg); + break; + } + } + if (rv != APR_EOF) { + show_help(argv[0], _opt_config, stderr); + exit(1); + } + // Check if no extra args were passed + if (opts->ind != opts->argc) { + show_help(argv[0], _opt_config, stderr); + fprintf(stderr, "\n%s: Extra unknown arguments passed\n\n",argv[0]); + exit(1); + } + + // Process configuration file + config_init(pool); + base = config_create(pool); + rv = config_read(base, apr_table_get(args,"Config"), args); + if (APR_STATUS_IS_ENOENT(rv)) { + fprintf(stderr,"Could not load configuration file: %s\n",apr_table_get(args,"config")); + } else if (rv) { + exit(1); + } + config_dump(base); + // Apply overrides from command line + find_log_files(base); + if (!apr_is_empty_array(base->input_files)) { + char **filelist; + int f, l; + filelist = (char **)base->input_files->elts; + for (f=0, l=base->input_files->nelts; f < l; f++) { + printf("Scanning %s\n",filelist[f]); + parse_logfile(base, filelist[f]); + } + } else { + printf("No input files\n"); + } + return 0; +} diff --git a/utility/shell.h b/utility/shell.h new file mode 100644 index 0000000..e69de29 -- cgit