1 files changed, 190 insertions, 73 deletions

diff --git a/utility/config.c b/utility/config.c
index 6b3dce1..847d474 100644
--- a/utility/config.c
+++ b/utility/config.c
@@ -3,53 +3,61 @@
 #include "apr_file_io.h"
 #include "apr_strings.h"
 #include "apr_hash.h"
-#include "apr_lib.h"
+
 #include "shell.h"
 #include "config.h"
+#include "util.h"
+#include "logparse.h"
 
 apr_hash_t *g_config_opts;
 
-apr_status_t config_set_string(config_t *cfg, config_opt_t *opt, int argc,
-        const char **argv)
+static apr_status_t config_set_string(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
 {
     int offset = (int)(long)opt->data;
     char **data = (char **)((void *)cfg + offset);
-    if (argc != 2) return APR_EINVAL;
+    if (argc != 2)
+        return APR_EINVAL;
     *data = apr_pstrdup(cfg->pool, argv[1]);
     return APR_SUCCESS;
 }
 
-apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc,
+static apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc,
+        const char **argv) __attribute__ ((__unused__));
+static apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc,
         const char **argv)
 {
     int offset = (int)(long)opt->data;
     int *data = (int *)((void *)cfg + offset);
-    if (argc != 2) return APR_EINVAL;
+    if (argc != 2)
+        return APR_EINVAL;
     *data = apr_atoi64(argv[1]);
     return APR_SUCCESS;
 }
 
-apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc,
+static apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc,
         const char **argv)
 {
     int offset = (int)(long)opt->data;
     int *data = (int *)((void *)cfg + offset);
-    if (argc != 2) return APR_EINVAL;
+    if (argc != 2)
+        return APR_EINVAL;
     *data = CHECK_YESNO(argv[1]);
     return APR_SUCCESS;
 }
 
-apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt, int argc,
-        const char **argv)
+static apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
 {
-    if (argc != 2) return APR_EINVAL;
-    if (!strcasecmp(argv[1],"error")) {
+    if (argc != 2)
+        return APR_EINVAL;
+    if (!strcasecmp(argv[1], "error")) {
         cfg->loglevel = LOGLEVEL_ERROR;
-    } else if (!strcasecmp(argv[1],"warn")) {
+    } else if (!strcasecmp(argv[1], "warn")) {
         cfg->loglevel = LOGLEVEL_WARN;
-    } else if (!strcasecmp(argv[1],"debug")) {
+    } else if (!strcasecmp(argv[1], "debug")) {
         cfg->loglevel = LOGLEVEL_DEBUG;
-    } else if (!strcasecmp(argv[1],"quiet")) {
+    } else if (!strcasecmp(argv[1], "quiet")) {
         cfg->loglevel = LOGLEVEL_QUIET;
     } else {
         cfg->loglevel = LOGLEVEL_ERROR;
@@ -57,49 +65,51 @@ apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt, int argc,
     return APR_SUCCESS;
 }
 
-apr_status_t config_set_dbconnect(config_t *cfg, config_opt_t *opt, int argc,
-        const char **argv)
+static apr_status_t config_set_dbconnect(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
 {
     return APR_SUCCESS;
 }
 
-apr_status_t config_set_dbparam(config_t *cfg, config_opt_t *opt, int argc,
-        const char **argv)
+static apr_status_t config_set_dbparam(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
 {
     return APR_SUCCESS;
 }
 
-apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt, int argc,
-        const char **argv)
+static apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
 {
     char **newp;
-    if (argc != 2) return APR_EINVAL;
+    if (argc != 2)
+        return APR_EINVAL;
     newp = (char **)apr_array_push(cfg->input_files);
     *newp = apr_pstrdup(cfg->pool, argv[1]);
     return APR_SUCCESS;
 }
 
-apr_status_t config_set_dummy(config_t *cfg, config_opt_t *opt, int argc,
-        const char **argv)
+static apr_status_t config_set_dummy(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
 {
     return APR_SUCCESS;
 }
 
-apr_status_t config_set_logformat(config_t *cfg, config_opt_t *opt, int argc,
-        const char **argv)
+static apr_status_t config_set_logformat(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
 {
     config_logformat_t *format;
     config_logformat_field_t *field;
 
-    if (argc != 4) return APR_EINVAL;
+    if (argc != 4)
+        return APR_EINVAL;
 
-    format = apr_hash_get(cfg->log_formats,argv[1],APR_HASH_KEY_STRING);
+    format = apr_hash_get(cfg->log_formats, argv[1], APR_HASH_KEY_STRING);
     if (!format) {
         format = apr_palloc(cfg->pool, sizeof(config_logformat_t));
         format->name = apr_pstrdup(cfg->pool, argv[1]);
         format->fields = apr_array_make(cfg->pool, 5,
                 sizeof(config_logformat_field_t));
-        apr_hash_set(cfg->log_formats, argv[1], APR_HASH_KEY_STRING, format);
+        apr_hash_set(cfg->log_formats, apr_pstrdup(cfg->pool,argv[1]), APR_HASH_KEY_STRING, format);
     }
     field = (config_logformat_field_t *)apr_array_push(format->fields);
     field->name = apr_pstrdup(cfg->pool, argv[2]);
@@ -107,18 +117,100 @@ apr_status_t config_set_logformat(config_t *cfg, config_opt_t *opt, int argc,
     return APR_SUCCESS;
 }
 
+static apr_status_t config_set_output_field(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
+{
+    config_output_field_t *field;
+    char *type, *size, *temp;
+
+    if (argc < 4)
+        return APR_EINVAL;
+    field = (config_output_field_t *)apr_array_push(cfg->output_fields);
+    field->field = apr_pstrdup(cfg->pool, argv[1]);
+    field->source = apr_pstrdup(cfg->pool, argv[3]);
+
+    type = size = apr_pstrdup(cfg->pool, argv[2]);
+    while (*size!='\0' && *size!='(')
+        size++;
+    if (*size == '(') {
+        *size = '\0';
+        size++;
+        temp = size;
+        while (*temp != '\0' && *temp != ')')
+            temp++;
+        *temp = '\0';
+        field->size = apr_atoi64(size);
+    }
+    if (strcasecmp("VARCHAR", type)==0) {
+        field->datatype = LOGSQL_DATATYPE_VARCHAR;
+    } else if (strcasecmp("INT", type)==0) {
+        field->datatype = LOGSQL_DATATYPE_INT;
+    } else if (strcasecmp("CHAR", type)==0) {
+        field->datatype = LOGSQL_DATATYPE_CHAR;
+    } else if (strcasecmp("SMALLINT", type)==0) {
+        field->datatype = LOGSQL_DATATYPE_SMALLINT;
+    } else if (strcasecmp("BIGINT", type)==0) {
+        field->datatype = LOGSQL_DATATYPE_BIGINT;
+    } else {
+        return APR_EINVAL;
+    }
+
+    // Has a function
+    if (argc > 4) {
+        int i;
+        field->fname = apr_pstrdup(cfg->pool, argv[4]);
+        field->func = parser_get_func(field->fname);
+        field->args = apr_pcalloc(cfg->pool, sizeof(char *) * (argc-4+1));
+        for (i=5; i<=argc; i++) {
+            field->args[i-5] = apr_pstrdup(cfg->pool, argv[i]);
+        }
+    }
+
+    return APR_SUCCESS;
+}
+
+static apr_status_t config_set_filter(config_t *cfg, config_opt_t *opt,
+        int argc, const char **argv)
+{
+    int argn = 1;
+    config_filter_t *filter;
+    filter = apr_pcalloc(cfg->pool, sizeof(config_filter_t));
+
+    if (opt->name[0]!='L') { // Pre or post 2-3 args
+        if (argc == 1)
+            return APR_EINVAL;
+        filter->field = apr_pstrdup(cfg->pool, argv[1]);
+        argn++;
+    } // Otherwise Line based only 1-2 args (no field)
+    if (argc <= argn)
+        return APR_EINVAL;
+    if (argv[argn][0] == '+')
+        argn++;
+    if (argv[argn][0] == '-') {
+        filter->negative = 1;
+        argn++;
+    }
+    if (argc <= argn)
+        return APR_EINVAL;
+    filter->filter = apr_pstrdup(cfg->pool, argv[argn]);
+    filter->regex = ap_pregcomp(cfg->pool, filter->filter, AP_REG_ICASE);
+    return APR_SUCCESS;
+}
+
 void config_dump(config_t *cfg)
 {
     apr_hash_index_t *hi;
+    int i;
+    config_output_field_t *fields;
 
-    printf("ErrorLog: %s\n",cfg->errorlog);
-    printf("LogLevel: %d\n",cfg->loglevel);
+    printf("ErrorLog: %s\n", cfg->errorlog);
+    printf("LogLevel: %d\n", cfg->loglevel);
 
-    printf("InputDir: %s\n",cfg->input_dir);
+    printf("InputDir: %s\n", cfg->input_dir);
 
-    printf("Table: %s\n",cfg->table);
-    printf("Transactions: %d\n",cfg->transactions);
-    printf("MachineID: %s\n",cfg->machineid);
+    printf("Table: %s\n", cfg->table);
+    printf("Transactions: %d\n", cfg->transactions);
+    printf("MachineID: %s\n", cfg->machineid);
 
     printf("Log formats:\n");
     for (hi = apr_hash_first(cfg->pool, cfg->log_formats); hi; hi
@@ -128,31 +220,39 @@ void config_dump(config_t *cfg)
         int i;
 
         apr_hash_this(hi, NULL, NULL, (void **)&format);
-        printf(">> %s\n",format->name);
+        printf(">> '%s'\n", format->name);
         fields = (config_logformat_field_t *)format->fields->elts;
         for (i=0; i<format->fields->nelts; i++) {
             printf(">>>> %s:%s\n", fields[i].name, fields[i].datatype);
         }
     }
-    printf("Log Format: %s\n",cfg->logformat);
-
-    printf("DryRun: %d\n",cfg->dryrun);
-    printf("Summary: %d\n",cfg->summary);
-}
-
-static char *lowerstr(apr_pool_t *pool, const char *input) {
-        char *temp;
-        char *itr;
-        temp = apr_pstrdup(pool, input);
-        for (itr=temp; *itr!='\0'; itr++) {
-                *itr = apr_tolower(*itr);
+    printf("Log Format: '%s'\n", cfg->logformat);
+
+    printf("Output Fields:\n");
+    fields = (config_output_field_t *)cfg->output_fields->elts;
+    for (i=0; i<cfg->output_fields->nelts; i++) {
+        printf(">> %s %s(%d): %s", fields[i].field, logsql_field_datatyeName(fields[i].datatype), fields[i].size, fields[i].source);
+        if (fields[i].func) {
+            printf(" :: %s(", fields[i].fname);
+            if (fields[i].args) {
+                int a = 0;
+                while (fields[i].args[a]) {
+                    printf("%s,", fields[i].args[a]);
+                    a++;
+                }
+            }
+            printf(")");
         }
-        return temp;
+        printf("\n");
+    }
+
+    printf("DryRun: %d\n", cfg->dryrun);
+    printf("Summary: %d\n", cfg->summary);
 }
 
 #define config_get_option(name) apr_hash_get(g_config_opts, name, APR_HASH_KEY_STRING)
 
-void config_add_option(apr_pool_t *p, const char *const name,
+static void config_add_option(apr_pool_t *p, const char *const name,
         const char *const help, config_func_t func, void *data)
 {
     config_opt_t *opt;
@@ -164,22 +264,24 @@ void config_add_option(apr_pool_t *p, const char *const name,
     opt->help = help;
     opt->func = func;
     opt->data = data;
-    apr_hash_set(g_config_opts, lowerstr(p,name), APR_HASH_KEY_STRING, opt);
+    apr_hash_set(g_config_opts, lowerstr(p, name), APR_HASH_KEY_STRING, opt);
 }
 
 void config_init(apr_pool_t *p)
 {
-    config_add_option(p, "ErrorLog", "File to log errors",
-            config_set_string, (void *)APR_OFFSETOF(config_t, errorlog));
-    config_add_option(p, "LogLevel", "Set Log Level (error, warn, debug, quiet)",
-            config_set_loglevel, NULL);
+    config_add_option(p, "ErrorLog", "File to log errors", config_set_string,
+            (void *)APR_OFFSETOF(config_t, errorlog));
+    config_add_option(p, "LogLevel",
+            "Set Log Level (error, warn, debug, quiet)", config_set_loglevel,
+            NULL);
 
     config_add_option(p, "InputDirectory", "Directory to scan for log files",
             config_set_string, (void *)APR_OFFSETOF(config_t, input_dir));
     config_add_option(p, "InputFile", "Parse only this file",
             config_set_inputfile, NULL);
 
-    config_add_option(p, "DBConnect", "DB Connection information  type://user:pass@hostname/database",
+    config_add_option(p, "DBConnect",
+            "DB Connection information  type://user:pass@hostname/database",
             config_set_dbconnect, NULL);
     config_add_option(p, "DBParam", "DB Connection Parameter",
             config_set_dbparam, NULL);
@@ -187,14 +289,28 @@ void config_init(apr_pool_t *p)
             config_set_string, (void *)APR_OFFSETOF(config_t, table));
     config_add_option(p, "UseTransactions", "Enable Transactions?",
             config_set_flag, (void *)APR_OFFSETOF(config_t, transactions));
-    config_add_option(p, "MachineID", "Machine ID to set",
-            config_set_string, (void *)APR_OFFSETOF(config_t, machineid));
+    config_add_option(p, "MachineID", "Machine ID to set", config_set_string,
+            (void *)APR_OFFSETOF(config_t, machineid));
 
     config_add_option(p, "LogFormatConfig", "Define input log formats",
             config_set_logformat, NULL);
     config_add_option(p, "LogFormat", "Use this logformat when parsing files",
             config_set_string, (void *)APR_OFFSETOF(config_t, logformat));
 
+    config_add_option(p, "LineFilter",
+            "A regular expression to apply to the input line",
+            config_set_filter, (void *)APR_OFFSETOF(config_t, linefilters));
+    config_add_option(p, "PreFilter",
+            "A regular expression to apply to a specific input field",
+            config_set_filter, (void *)APR_OFFSETOF(config_t, prefilters));
+    config_add_option(p, "PostFilter",
+            "A regular expression to apply to a specific SQL output field",
+            config_set_filter, (void *)APR_OFFSETOF(config_t, postfilters));
+
+    config_add_option(p, "OutputField",
+            "Define output fields: field datatype source optfunc optarg...",
+            config_set_output_field, NULL);
+
     config_add_option(p, "DryRun", "Don't perform any actual database changes",
             config_set_flag, (void *)APR_OFFSETOF(config_t, dryrun));
     config_add_option(p, "Config", "Dummy to handle config directive",
@@ -216,16 +332,20 @@ config_t *config_create(apr_pool_t *p)
     cfg->input_files = apr_array_make(cfg->pool, 10, sizeof(char *));
     cfg->dbconfig = apr_table_make(cfg->pool, 5);
     cfg->log_formats = apr_hash_make(cfg->pool);
-
+    cfg->output_fields = apr_array_make(cfg->pool, 10,
+            sizeof(config_output_field_t));
     return cfg;
 }
 
-int config_merge(void *rec, const char *key, const char *value) {
+static int config_merge(void *rec, const char *key, const char *value)
+{
     config_t *cfg = (config_t *)rec;
 
-    config_opt_t *opt = config_get_option(key);
+    config_opt_t *opt= config_get_option(key);
     if (opt) {
-        const char *args[] = {key, value};
+        const char *args[] = {
+            key,
+            value };
         opt->func(cfg, opt, 2, args);
     } else {
         printf("Unhandled: %s\n", key);
@@ -238,11 +358,11 @@ apr_status_t config_read(config_t *cfg, const char *filename,
 {
     apr_finfo_t finfo;
     apr_file_t *file;
-    apr_status_t rv, ret = APR_SUCCESS;
+    apr_status_t rv, ret= APR_SUCCESS;
     apr_pool_t *tp, *targp;
     config_opt_t *opt;
     char buff[1024];
-    char *ptr, *ptr2;
+    char *ptr;
     char **targv;
     int targc;
     int line;
@@ -254,7 +374,7 @@ apr_status_t config_read(config_t *cfg, const char *filename,
         return APR_ENOENT;
     }
     rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED,
-            APR_OS_DEFAULT, tp);
+    APR_OS_DEFAULT, tp);
     if (rv != APR_SUCCESS)
         return rv;
 
@@ -267,10 +387,7 @@ apr_status_t config_read(config_t *cfg, const char *filename,
             // skip leading white space
             for (ptr = buff; *ptr == ' ' || *ptr == '\t'; ptr++)
                 ;
-            // chomp off newline
-            for (ptr2 = ptr + strlen(ptr); *ptr2 != '\r' && *ptr2 != '\n'; ptr2--)
-                ;
-            *ptr2 = '\0';
+            line_chomp(ptr);
 
             // skip comments
             if (*ptr == '#')
@@ -278,9 +395,10 @@ apr_status_t config_read(config_t *cfg, const char *filename,
             if (*ptr == '\0')
                 continue;
             apr_pool_clear(targp);
-            apr_tokenize_to_argv(buff, &targv, targp);
+            apr_tokenize_to_argv(ptr, &targv, targp);
             targc = 0;
-            while (targv[targc]) targc++;
+            while (targv[targc])
+                targc++;
             opt = config_get_option(lowerstr(targp,targv[0]));
             if (opt) {
                 rv = opt->func(cfg, opt, targc, (const char **)targv);
@@ -296,10 +414,9 @@ apr_status_t config_read(config_t *cfg, const char *filename,
     } while (rv == APR_SUCCESS);
 
     // Apply merges
-    apr_table_do(config_merge,(void *)cfg,merge,NULL);
+    apr_table_do(config_merge, (void *)cfg, merge, NULL);
 
     apr_file_close(file);
     apr_pool_destroy(tp);
     return ret;
 }
-