summaryrefslogtreecommitdiffstatsabout
diff options
context:
space:
mode:
authorEdward Rudd <urkle@outoforder.cc>2008-10-23 06:29:36 (GMT)
committer Edward Rudd <urkle@outoforder.cc>2008-10-23 06:29:36 (GMT)
commitaed9f10440d8789f99919c3a6f91e8d76bbc44dc (patch)
tree2560bad5b4959cddc090916e8e522b37ce74df75
parent0ddd719a72469f732a881c93d4c804e9aca787fe (diff)
more parsing implementation (Custom functions)
initial DB inserting (w/ prepared query)
-rw-r--r--utility/Makefile.in4
-rw-r--r--utility/ap_pcre.h2
-rw-r--r--utility/config.c46
-rw-r--r--utility/config.h9
-rw-r--r--utility/database.c95
-rw-r--r--utility/database.h12
-rw-r--r--utility/logparse.c89
-rw-r--r--utility/logparse.h7
-rw-r--r--utility/mod_log_sql.conf32
-rw-r--r--utility/shell.c10
10 files changed, 245 insertions, 61 deletions
diff --git a/utility/Makefile.in b/utility/Makefile.in
index 69a746c..eeb26b8 100644
--- a/utility/Makefile.in
+++ b/utility/Makefile.in
@@ -18,8 +18,8 @@ STD_DIST = Makefile.in
18 18
19DISTFILES = $(STD_DIST) $(EXTRA_DIST) $(SOURCES) $(HEADERS) 19DISTFILES = $(STD_DIST) $(EXTRA_DIST) $(SOURCES) $(HEADERS)
20 20
21SOURCES = shell.c config.c logparse.c ap_pcre.c util.c 21SOURCES = shell.c config.c logparse.c ap_pcre.c util.c database.c
22HEADERS = shell.h config.h logparse.h ap_pcre.h util.h 22HEADERS = shell.h config.h logparse.h ap_pcre.h util.h database.h
23OBJECTS = $(SOURCES:.c=.o) 23OBJECTS = $(SOURCES:.c=.o)
24DEPS = $(SOURCES:.c=.d) 24DEPS = $(SOURCES:.c=.d)
25TARGETS = mod_log_sql 25TARGETS = mod_log_sql
diff --git a/utility/ap_pcre.h b/utility/ap_pcre.h
index a851d29..e817dc2 100644
--- a/utility/ap_pcre.h
+++ b/utility/ap_pcre.h
@@ -73,6 +73,8 @@ extern "C" {
73#define AP_REG_EXTENDED (0) /** unused */ 73#define AP_REG_EXTENDED (0) /** unused */
74#define AP_REG_NOSUB (0) /** unused */ 74#define AP_REG_NOSUB (0) /** unused */
75 75
76#define AP_MAX_REG_MATCH 10
77
76/* Error values: */ 78/* Error values: */
77enum { 79enum {
78 AP_REG_ASSERT = 1, /** internal error ? */ 80 AP_REG_ASSERT = 1, /** internal error ? */
diff --git a/utility/config.c b/utility/config.c
index 847d474..d3a6ca5 100644
--- a/utility/config.c
+++ b/utility/config.c
@@ -3,6 +3,7 @@
3#include "apr_file_io.h" 3#include "apr_file_io.h"
4#include "apr_strings.h" 4#include "apr_strings.h"
5#include "apr_hash.h" 5#include "apr_hash.h"
6#include "apr_uri.h"
6 7
7#include "shell.h" 8#include "shell.h"
8#include "config.h" 9#include "config.h"
@@ -65,18 +66,6 @@ static apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt,
65 return APR_SUCCESS; 66 return APR_SUCCESS;
66} 67}
67 68
68static apr_status_t config_set_dbconnect(config_t *cfg, config_opt_t *opt,
69 int argc, const char **argv)
70{
71 return APR_SUCCESS;
72}
73
74static apr_status_t config_set_dbparam(config_t *cfg, config_opt_t *opt,
75 int argc, const char **argv)
76{
77 return APR_SUCCESS;
78}
79
80static apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt, 69static apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt,
81 int argc, const char **argv) 70 int argc, const char **argv)
82{ 71{
@@ -123,12 +112,12 @@ static apr_status_t config_set_output_field(config_t *cfg, config_opt_t *opt,
123 config_output_field_t *field; 112 config_output_field_t *field;
124 char *type, *size, *temp; 113 char *type, *size, *temp;
125 114
126 if (argc < 4) 115 if (argc < 5)
127 return APR_EINVAL; 116 return APR_EINVAL;
128 field = (config_output_field_t *)apr_array_push(cfg->output_fields); 117 field = (config_output_field_t *)apr_array_push(cfg->output_fields);
129 field->field = apr_pstrdup(cfg->pool, argv[1]); 118 field->field = apr_pstrdup(cfg->pool, argv[1]);
130 field->source = apr_pstrdup(cfg->pool, argv[3]); 119 field->source = apr_pstrdup(cfg->pool, argv[4]);
131 120 field->def = apr_pstrdup(cfg->pool, argv[3]);
132 type = size = apr_pstrdup(cfg->pool, argv[2]); 121 type = size = apr_pstrdup(cfg->pool, argv[2]);
133 while (*size!='\0' && *size!='(') 122 while (*size!='\0' && *size!='(')
134 size++; 123 size++;
@@ -156,13 +145,13 @@ static apr_status_t config_set_output_field(config_t *cfg, config_opt_t *opt,
156 } 145 }
157 146
158 // Has a function 147 // Has a function
159 if (argc > 4) { 148 if (argc > 5) {
160 int i; 149 int i;
161 field->fname = apr_pstrdup(cfg->pool, argv[4]); 150 field->fname = apr_pstrdup(cfg->pool, argv[5]);
162 field->func = parser_get_func(field->fname); 151 field->func = parser_get_func(field->fname);
163 field->args = apr_pcalloc(cfg->pool, sizeof(char *) * (argc-4+1)); 152 field->args = apr_pcalloc(cfg->pool, sizeof(char *) * (argc-5+1));
164 for (i=5; i<=argc; i++) { 153 for (i=6; i<=argc; i++) {
165 field->args[i-5] = apr_pstrdup(cfg->pool, argv[i]); 154 field->args[i-6] = apr_pstrdup(cfg->pool, argv[i]);
166 } 155 }
167 } 156 }
168 157
@@ -208,6 +197,9 @@ void config_dump(config_t *cfg)
208 197
209 printf("InputDir: %s\n", cfg->input_dir); 198 printf("InputDir: %s\n", cfg->input_dir);
210 199
200 printf("DB Driver: %s\n", cfg->dbdriver);
201 printf("DB Params: %s\n", cfg->dbparams);
202
211 printf("Table: %s\n", cfg->table); 203 printf("Table: %s\n", cfg->table);
212 printf("Transactions: %d\n", cfg->transactions); 204 printf("Transactions: %d\n", cfg->transactions);
213 printf("MachineID: %s\n", cfg->machineid); 205 printf("MachineID: %s\n", cfg->machineid);
@@ -231,7 +223,9 @@ void config_dump(config_t *cfg)
231 printf("Output Fields:\n"); 223 printf("Output Fields:\n");
232 fields = (config_output_field_t *)cfg->output_fields->elts; 224 fields = (config_output_field_t *)cfg->output_fields->elts;
233 for (i=0; i<cfg->output_fields->nelts; i++) { 225 for (i=0; i<cfg->output_fields->nelts; i++) {
234 printf(">> %s %s(%d): %s", fields[i].field, logsql_field_datatyeName(fields[i].datatype), fields[i].size, fields[i].source); 226 printf(">> %s %s(%d) DEFAULT '%s': %s", fields[i].field,
227 logsql_field_datatyeName(fields[i].datatype),
228 fields[i].size, fields[i].def, fields[i].source);
235 if (fields[i].func) { 229 if (fields[i].func) {
236 printf(" :: %s(", fields[i].fname); 230 printf(" :: %s(", fields[i].fname);
237 if (fields[i].args) { 231 if (fields[i].args) {
@@ -280,11 +274,10 @@ void config_init(apr_pool_t *p)
280 config_add_option(p, "InputFile", "Parse only this file", 274 config_add_option(p, "InputFile", "Parse only this file",
281 config_set_inputfile, NULL); 275 config_set_inputfile, NULL);
282 276
283 config_add_option(p, "DBConnect", 277 config_add_option(p, "DBDDriver", "DBD Driver to use",
284 "DB Connection information type://user:pass@hostname/database", 278 config_set_string, (void *)APR_OFFSETOF(config_t, dbdriver));
285 config_set_dbconnect, NULL); 279 config_add_option(p, "DBDParams", "DBD Connection Parameters",
286 config_add_option(p, "DBParam", "DB Connection Parameter", 280 config_set_string, (void *)APR_OFFSETOF(config_t, dbparams));
287 config_set_dbparam, NULL);
288 config_add_option(p, "Table", "Table to import the log to", 281 config_add_option(p, "Table", "Table to import the log to",
289 config_set_string, (void *)APR_OFFSETOF(config_t, table)); 282 config_set_string, (void *)APR_OFFSETOF(config_t, table));
290 config_add_option(p, "UseTransactions", "Enable Transactions?", 283 config_add_option(p, "UseTransactions", "Enable Transactions?",
@@ -330,7 +323,6 @@ config_t *config_create(apr_pool_t *p)
330 cfg->summary = 1; 323 cfg->summary = 1;
331 cfg->transactions = 1; 324 cfg->transactions = 1;
332 cfg->input_files = apr_array_make(cfg->pool, 10, sizeof(char *)); 325 cfg->input_files = apr_array_make(cfg->pool, 10, sizeof(char *));
333 cfg->dbconfig = apr_table_make(cfg->pool, 5);
334 cfg->log_formats = apr_hash_make(cfg->pool); 326 cfg->log_formats = apr_hash_make(cfg->pool);
335 cfg->output_fields = apr_array_make(cfg->pool, 10, 327 cfg->output_fields = apr_array_make(cfg->pool, 10,
336 sizeof(config_output_field_t)); 328 sizeof(config_output_field_t));
diff --git a/utility/config.h b/utility/config.h
index 67f8ea5..0e80856 100644
--- a/utility/config.h
+++ b/utility/config.h
@@ -4,7 +4,6 @@
4#include "apr_tables.h" 4#include "apr_tables.h"
5#include "apr_hash.h" 5#include "apr_hash.h"
6#include "apr_file_io.h" 6#include "apr_file_io.h"
7
8#include "ap_pcre.h" 7#include "ap_pcre.h"
9 8
10typedef enum { 9typedef enum {
@@ -14,6 +13,7 @@ typedef enum {
14 LOGLEVEL_DEBUG = 3, 13 LOGLEVEL_DEBUG = 3,
15} loglevel_e; 14} loglevel_e;
16 15
16typedef struct config_dbd_t config_dbd_t;
17typedef struct config_t config_t; 17typedef struct config_t config_t;
18struct config_t { 18struct config_t {
19 /** the structures pool (to ease function arguments) */ 19 /** the structures pool (to ease function arguments) */
@@ -32,7 +32,10 @@ struct config_t {
32 apr_array_header_t *input_files; 32 apr_array_header_t *input_files;
33 33
34 /** db connection configuration */ 34 /** db connection configuration */
35 apr_table_t *dbconfig; 35 const char *dbdriver;
36 const char *dbparams;
37 config_dbd_t *dbconn;
38
36 /** Logging table */ 39 /** Logging table */
37 const char *table; 40 const char *table;
38 /** Use transactons */ 41 /** Use transactons */
@@ -111,10 +114,12 @@ struct config_output_field_t {
111 const char *field; 114 const char *field;
112 logsql_field_datatype datatype; 115 logsql_field_datatype datatype;
113 apr_size_t size; 116 apr_size_t size;
117 const char *def;
114 const char *source; 118 const char *source;
115 const char *fname; 119 const char *fname;
116 void *func; 120 void *func;
117 const char **args; 121 const char **args;
122 void *data;
118}; 123};
119 124
120#define CHECK_YESNO(c) ((!strcasecmp(c,"on") || !strcasecmp(c,"yes")) ? 1 : 0) 125#define CHECK_YESNO(c) ((!strcasecmp(c,"on") || !strcasecmp(c,"yes")) ? 1 : 0)
diff --git a/utility/database.c b/utility/database.c
index 5fece50..e7650aa 100644
--- a/utility/database.c
+++ b/utility/database.c
@@ -1,2 +1,97 @@
1#include "database.h" 1#include "database.h"
2#include "apu.h"
3#include "apr_dbd.h"
4#include "apr_strings.h"
2 5
6struct config_dbd_t {
7 const apr_dbd_driver_t *driver;
8 apr_dbd_t *dbd;
9 apr_dbd_prepared_t *stmt;
10 const char **args;
11};
12
13void database_init(apr_pool_t *p)
14{
15 apr_dbd_init(p);
16}
17
18apr_status_t database_connect(config_t *cfg)
19{
20 apr_status_t rv;
21 if (!cfg->dbconn) {
22 cfg->dbconn = apr_palloc(cfg->pool, sizeof(config_dbd_t));
23 }
24 rv = apr_dbd_get_driver(cfg->pool, cfg->dbdriver, &(cfg->dbconn->driver));
25 if (rv)
26 return rv;
27
28 rv = apr_dbd_open(cfg->dbconn->driver, cfg->pool, cfg->dbparams,
29 &(cfg->dbconn->dbd));
30 if (rv)
31 return rv;
32
33 return APR_SUCCESS;
34}
35
36apr_status_t database_disconnect(config_t *cfg)
37{
38 return apr_dbd_close(cfg->dbconn->driver, cfg->dbconn->dbd);
39}
40
41apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data)
42{
43 apr_status_t rv;
44 int f, nfs;
45 config_output_field_t *ofields;
46 ofields = (config_output_field_t *)cfg->output_fields->elts;
47 nfs = cfg->output_fields->nelts;
48 // Prepare statement
49 if (!cfg->dbconn->stmt) {
50 char *sql;
51 int i;
52 struct iovec *vec;
53 vec = apr_palloc(p, (nfs*2 + 5) * sizeof(struct iovec));
54 sql = apr_palloc(p, (nfs*3));
55 vec[0].iov_base = "INSERT INTO ";
56 vec[0].iov_len = 12;
57 vec[1].iov_base = (void *)cfg->table;
58 vec[1].iov_len = strlen(cfg->table);
59 vec[2].iov_base = " (";
60 vec[2].iov_len = 2;
61 for (i=3, f=0; f<nfs; f++, i+=2) {
62 vec[i].iov_base = (void *)ofields[f].field;
63 vec[i].iov_len = strlen(vec[i].iov_base);
64 vec[i+1].iov_base = ",";
65 vec[i+1].iov_len = 1;
66 memcpy(&sql[f*3], "%s,", 3);
67 }
68 sql[nfs*3-1] = '\0';
69 vec[i-1].iov_base = ") VALUES (";
70 vec[i-1].iov_len = 10;
71 vec[i].iov_base = sql;
72 vec[i].iov_len = nfs*3-1;
73 vec[i+1].iov_base = ")";
74 vec[i+1].iov_len = 1;
75 sql = apr_pstrcatv(p, vec, i+2, NULL);
76 printf("SQL: %s\n", sql);
77 cfg->dbconn->args = apr_palloc(cfg->pool, nfs * sizeof(char *));
78 rv = apr_dbd_prepare(cfg->dbconn->driver, cfg->pool, cfg->dbconn->dbd,
79 sql, "INSERT", &(cfg->dbconn->stmt));
80 if (rv) {
81 printf("DB Error: %s\n", apr_dbd_error(cfg->dbconn->driver,
82 cfg->dbconn->dbd, rv));
83 return rv;
84 }
85 }
86 for (f=0; f<nfs; f++) {
87 cfg->dbconn->args[f] = apr_table_get(data, ofields[f].field);
88 }
89 rv = apr_dbd_pquery(cfg->dbconn->driver, p, cfg->dbconn->dbd, &f,
90 cfg->dbconn->stmt, nfs, cfg->dbconn->args);
91 if (rv) {
92 printf("DB Error: %s\n", apr_dbd_error(cfg->dbconn->driver,
93 cfg->dbconn->dbd, rv));
94 return rv;
95 }
96 return APR_SUCCESS;
97}
diff --git a/utility/database.h b/utility/database.h
index 9fc4844..fd24994 100644
--- a/utility/database.h
+++ b/utility/database.h
@@ -1,4 +1,16 @@
1#ifndef DATABASE_H_ 1#ifndef DATABASE_H_
2#define DATABASE_H_ 2#define DATABASE_H_
3 3
4#include "apr_pools.h"
5
6#include "config.h"
7
8void database_init(apr_pool_t *p);
9
10apr_status_t database_connect(config_t *cfg);
11
12apr_status_t database_disconnect(config_t *cfg);
13
14apr_status_t database_insert(config_t *cfg, apr_pool_t *p, apr_table_t *data);
15
4#endif /*DATABASE_H_*/ 16#endif /*DATABASE_H_*/
diff --git a/utility/logparse.c b/utility/logparse.c
index 4d823ce..e9ca340 100644
--- a/utility/logparse.c
+++ b/utility/logparse.c
@@ -2,16 +2,70 @@
2#include "apr_file_info.h" 2#include "apr_file_info.h"
3#include "apr_file_io.h" 3#include "apr_file_io.h"
4#include "apr_strings.h" 4#include "apr_strings.h"
5#include "apr_time.h"
6
7#include "time.h"
8#include "stdlib.h"
5 9
6#include "util.h" 10#include "util.h"
11#include "ap_pcre.h"
12#include "database.h"
13
7 14
8apr_hash_t *g_parser_funcs; 15apr_hash_t *g_parser_funcs;
9 16
10static apr_status_t parser_func_regexmatch(config_t *cfg, const char *data, 17static apr_status_t parser_func_regexmatch(apr_pool_t *p, config_t *cfg,
11 int argc, const char **argv) 18 config_output_field_t *field, const char *value, char **ret)
12{ 19{
20 struct {
21 ap_regex_t *rx;
22 const char *substr;
23 } *data;
24 ap_regmatch_t regm[AP_MAX_REG_MATCH];
25 // Check if a regular expression configured
26 if (!field->args[0]) return APR_EINVAL;
27 if (!field->data) {
28 // pre compile the regex
29 data = apr_palloc(cfg->pool, sizeof(ap_regex_t)+sizeof(const char *));
30 data->rx = ap_pregcomp(cfg->pool, field->args[0],
31 AP_REG_EXTENDED|AP_REG_ICASE);
32 if (field->args[1]) {
33 data->substr = field->args[1];
34 } else {
35 data->substr = "$1";
36 }
37 if (!data->rx) return APR_EINVAL;
38 field->data = data;
39 } else data = field->data;
40
41 if (!ap_regexec(data->rx, value, AP_MAX_REG_MATCH, regm, 0)) {
42 *ret = ap_pregsub(p, data->substr, value, AP_MAX_REG_MATCH, regm);
43 }
44 //printf("We matched %s against %s to %s\n",value, field->args[0], *ret);
13 return APR_SUCCESS; 45 return APR_SUCCESS;
14} 46}
47
48static apr_status_t parser_func_totimestamp(apr_pool_t *p, config_t *cfg,
49 config_output_field_t *field, const char *value, char **ret)
50{
51 time_t time;
52 struct tm ts;
53
54
55 strptime(value, "%d/%b/%Y:%H:%M:%S %z", &ts);
56 time = mktime(&ts);
57
58 *ret = apr_itoa(p, time);
59 return APR_SUCCESS;
60}
61
62static apr_status_t parser_func_machineid(apr_pool_t *p, config_t *cfg,
63 config_output_field_t *field, const char *value, char **ret)
64{
65 *ret = apr_pstrdup(p,cfg->machineid);
66 return APR_SUCCESS;
67}
68
15parser_func_t parser_get_func(const char *name) 69parser_func_t parser_get_func(const char *name)
16{ 70{
17 return apr_hash_get(g_parser_funcs, name, APR_HASH_KEY_STRING); 71 return apr_hash_get(g_parser_funcs, name, APR_HASH_KEY_STRING);
@@ -29,6 +83,8 @@ static void parser_add_func(apr_pool_t *p, const char *const name,
29void parser_init(apr_pool_t *p) 83void parser_init(apr_pool_t *p)
30{ 84{
31 parser_add_func(p, "regexmatch", parser_func_regexmatch); 85 parser_add_func(p, "regexmatch", parser_func_regexmatch);
86 parser_add_func(p, "totimestamp", parser_func_totimestamp);
87 parser_add_func(p, "machineid", parser_func_machineid);
32} 88}
33 89
34void parser_find_logs(config_t *cfg) 90void parser_find_logs(config_t *cfg)
@@ -212,7 +268,7 @@ apr_status_t parse_logfile(config_t *cfg, const char *filename)
212 line_chomp(buff); 268 line_chomp(buff);
213 269
214 apr_pool_clear(targp); 270 apr_pool_clear(targp);
215 tokenize_logline(buff, &targv, targp, 1); 271 tokenize_logline(buff, &targv, targp, 0);
216 targc = 0; 272 targc = 0;
217 while (targv[targc]) targc++; 273 while (targv[targc]) targc++;
218 /** @todo Run Line Filters here */ 274 /** @todo Run Line Filters here */
@@ -239,6 +295,7 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv, in
239 config_output_field_t *ofields; 295 config_output_field_t *ofields;
240 apr_table_t *datain; 296 apr_table_t *datain;
241 apr_table_t *dataout; 297 apr_table_t *dataout;
298 apr_status_t rv;
242 int i; 299 int i;
243 300
244 fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING); 301 fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING);
@@ -257,20 +314,30 @@ apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv, in
257 // Convert input fields to output fields 314 // Convert input fields to output fields
258 ofields = (config_output_field_t *)cfg->output_fields->elts; 315 ofields = (config_output_field_t *)cfg->output_fields->elts;
259 for (i=0; i<cfg->output_fields->nelts; i++) { 316 for (i=0; i<cfg->output_fields->nelts; i++) {
260 const char *t; 317 const char *val;
318 val = apr_table_get(datain, ofields[i].source);
319 // If we can't find the source field just continue
320 if (!val && !(ofields[i].source[0]=='\0' && ofields[i].func)) {
321 apr_table_setn(dataout, ofields[i].field, ofields[i].def);
322 continue;
323 }
261 if (!ofields[i].func) { 324 if (!ofields[i].func) {
262 t = apr_table_get(datain, ofields[i].source); 325 apr_table_setn(dataout,ofields[i].field, val);
263 if (!t) { 326 //printf("S: %s = %s\n",ofields[i].field, val);
264 return APR_EINVAL;
265 }
266 apr_table_setn(dataout,ofields[i].field, t);
267 printf("S: %s = %s\n",ofields[i].source, t);
268 } else { 327 } else {
269 printf("S: %s, F: %p\n",ofields[i].source, ofields[i].func); 328 char *ret = NULL;
329 rv = ((parser_func_t)ofields[i].func)(ptemp, cfg, &ofields[i], val,
330 &ret);
331 if (rv) return rv;
332 apr_table_setn(dataout, ofields[i].field, ret);
333 //printf("S: %s = %s\n",ofields[i].field, ret);
270 } 334 }
271 } 335 }
272 336
273 /** @todo Run Post Filters here */ 337 /** @todo Run Post Filters here */
274 338
339 // Process DB Query
340 rv = database_insert(cfg, ptemp, dataout);
341 if (rv) return rv;
275 return APR_SUCCESS; 342 return APR_SUCCESS;
276} 343}
diff --git a/utility/logparse.h b/utility/logparse.h
index ebabf56..53b376b 100644
--- a/utility/logparse.h
+++ b/utility/logparse.h
@@ -3,8 +3,8 @@
3 3
4#include "config.h" 4#include "config.h"
5 5
6typedef apr_status_t (*parser_func_t)(config_t *cfg, const char *data, 6typedef apr_status_t (*parser_func_t)(apr_pool_t *p, config_t *cfg,
7 int argc, const char **argv); 7 config_output_field_t *field, const char *value, char **ret);
8 8
9parser_func_t parser_get_func(const char *name); 9parser_func_t parser_get_func(const char *name);
10 10
@@ -14,6 +14,7 @@ void parser_find_logs(config_t *cfg);
14 14
15apr_status_t parse_logfile(config_t *cfg, const char *filename); 15apr_status_t parse_logfile(config_t *cfg, const char *filename);
16 16
17apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv, int argc); 17apr_status_t parse_processline(apr_pool_t *ptemp, config_t *cfg, char **argv,
18 int argc);
18 19
19#endif /*LOGPARSE_H_*/ 20#endif /*LOGPARSE_H_*/
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
index 72f0205..9730983 100644
--- a/utility/mod_log_sql.conf
+++ b/utility/mod_log_sql.conf
@@ -1,8 +1,9 @@
1InputDirectory ./logs 1InputDirectory ./logs
2ErrorLog ./error_log 2ErrorLog ./error_log
3DBConnect mysql://username:host@server/database 3DBDDriver mysql
4DBDParams "host=localhost;user=root;dbname=apache_log"
4DBParam socketfile /tmp/mysql.sock 5DBParam socketfile /tmp/mysql.sock
5Table apache_logs 6Table access_log
6MachineID 7of9 7MachineID 7of9
7UseTransactions on 8UseTransactions on
8LogLevel notice 9LogLevel notice
@@ -34,17 +35,18 @@ Linefilter - "BAD"
34PreFilter request - "GET \/images" 35PreFilter request - "GET \/images"
35PostFilter request_method "GET" 36PostFilter request_method "GET"
36 37
37# Usage field datatype(size) source [function [param]...] 38# Usage field datatype(size) default source [function [param]...]
38OutputField bytes_sent int bytes_sent 39OutputField bytes_sent int 0 bytes_sent
39OutputField request_protocol varchar(10) request regexmatch "(HTTP\/[\d\.]+)$" 40OutputField request_protocol varchar(10) "" request regexmatch "(HTTP/[\\d\\.]+)$"
40OutputField remote_host varchar(50) remhost 41OutputField remote_host varchar(50) "" remhost
41OutputField request_method varchar(25) request regexmatch "^(\w+)" 42OutputField request_method varchar(25) "" request regexmatch "^(\\w+)"
42OutputField time_stamp int date totimestamp 43OutputField time_stamp int 0 date totimestamp
43OutputField status smallint status 44OutputField status smallint 0 status
44OutputField request_uri varchar(255) request regexmatch "^\w+ (.+) \w+\.[\d\.]+$" 45OutputField request_uri varchar(255) "" request regexmatch "^\\w+ (.+) \\w+/[\\d\\.]+$"
45OutputField remote_user varchar(50) user 46OutputField remote_user varchar(50) "" user
46OutputField remote_logname varchar(50) ident 47OutputField remote_logname varchar(50) "" ident
47OutputField remote_time char(28) date 48OutputField request_time char(28) "" date regexmatch ".+" "[$0]"
48#Only used for Combined log input, if standard CLF input, they are ignored 49#Only used for Combined log input, if standard CLF input, they are ignored
49OutputField agent varchar(255) agent 50OutputField agent varchar(255) "" agent
50OutputField referer varchar(255) referer 51OutputField referer varchar(255) "" referer
52OutputField machine_id varchar(25) "" "" machineid
diff --git a/utility/shell.c b/utility/shell.c
index eaa7098..94cca35 100644
--- a/utility/shell.c
+++ b/utility/shell.c
@@ -9,6 +9,7 @@
9#include "shell.h" 9#include "shell.h"
10#include "config.h" 10#include "config.h"
11#include "logparse.h" 11#include "logparse.h"
12#include "database.h"
12 13
13const apr_getopt_option_t _opt_config[] = { 14const apr_getopt_option_t _opt_config[] = {
14 {"machineid", 'm', 1, "Machine ID for the log file"}, 15 {"machineid", 'm', 1, "Machine ID for the log file"},
@@ -122,9 +123,11 @@ int main(int argc, const char *const argv[])
122 exit(1); 123 exit(1);
123 } 124 }
124 125
125 // Process configuration file 126 // Initialize sub systems
126 parser_init(pool); 127 parser_init(pool);
127 config_init(pool); 128 config_init(pool);
129 database_init(pool);
130 // Process configuration file
128 base = config_create(pool); 131 base = config_create(pool);
129 rv = config_read(base, apr_table_get(args,"Config"), args); 132 rv = config_read(base, apr_table_get(args,"Config"), args);
130 apr_pool_destroy(ptemp); 133 apr_pool_destroy(ptemp);
@@ -138,6 +141,10 @@ int main(int argc, const char *const argv[])
138 141
139 // Find files and parse 142 // Find files and parse
140 parser_find_logs(base); 143 parser_find_logs(base);
144 if ((rv = database_connect(base))) {
145 printf("Error Connecting to Database: %d\n",rv);
146 exit(1);
147 }
141 if (!apr_is_empty_array(base->input_files)) { 148 if (!apr_is_empty_array(base->input_files)) {
142 char **filelist; 149 char **filelist;
143 int f, l; 150 int f, l;
@@ -149,5 +156,6 @@ int main(int argc, const char *const argv[])
149 } else { 156 } else {
150 printf("No input files\n"); 157 printf("No input files\n");
151 } 158 }
159 database_disconnect(base);
152 return 0; 160 return 0;
153} 161}