summaryrefslogtreecommitdiffstats
path: root/utility
diff options
context:
space:
mode:
Diffstat (limited to 'utility')
-rw-r--r--utility/Makefile.in49
-rw-r--r--utility/ap_pcre.c344
-rw-r--r--utility/ap_pcre.h178
-rw-r--r--utility/config.c534
-rw-r--r--utility/config.h190
-rw-r--r--utility/database.c212
-rw-r--r--utility/database.h25
-rw-r--r--utility/logparse.c768
-rw-r--r--utility/logparse.h37
-rw-r--r--utility/mod_log_sql.conf70
-rw-r--r--utility/shell.c342
-rw-r--r--utility/shell.h0
-rw-r--r--utility/util.c189
-rw-r--r--utility/util.h26
14 files changed, 2964 insertions, 0 deletions
diff --git a/utility/Makefile.in b/utility/Makefile.in
new file mode 100644
index 0000000..fdc3846
--- /dev/null
+++ b/utility/Makefile.in
@@ -0,0 +1,49 @@
1# @configure_input@
2
3top_srcdir = @top_srcdir@
4srcdir = @abs_srcdir@
5top_builddir = @top_builddir@
6builddir = @abs_builddir@
7
8LIBTOOL=@LIBTOOL@
9CFLAGS = -g3 -Wall -fno-strict-aliasing \
10 @APR_INCLUDES@ @APU_INCLUDES@ @PCRE_CFLAGS@ \
11 -I$(top_srcdir)/include
12CPPFLAGS = @APR_CPPFLAGS@
13LDFLAGS = @APR_LIBTOOL@ @APU_LIBTOOL@ @PCRE_LIBS@
14
15ifeq (@OOO_MAINTAIN@,1)
16CFLAGS += -Werror
17endif
18
19STD_DIST = Makefile.in
20
21DISTFILES = $(STD_DIST) $(EXTRA_DIST) $(SOURCES) $(HEADERS)
22
23SOURCES = shell.c config.c logparse.c ap_pcre.c util.c database.c
24HEADERS = shell.h config.h logparse.h ap_pcre.h util.h database.h
25OBJECTS = $(SOURCES:.c=.o)
26DEPS = $(SOURCES:.c=.d)
27TARGETS = mod_log_sql
28
29all: $(TARGETS)
30
31mod_log_sql: $(OBJECTS) $(HEADERS)
32 $(LIBTOOL) --tag=CC --mode=link $(CC) -o $@ $(OBJECTS) $(LDFLAGS)
33
34%.o: %.c %.d
35 gcc -c $(CFLAGS) $(CPPFLAGS) $< -o $@
36%.d: %.c
37 gcc -MM $(CFLAGS) $(CPPFLAGS) $< -o $@
38
39-include $(DEPS)
40
41clean:
42 $(RM) $(OBJECTS) $(TARGETS) $(DEPS)
43
44local-dist: $(DISTFILES)
45 mkdir -p $(DESTDIR)
46 cp -dp --parents $(DISTFILES) $(DESTDIR)
47
48Makefile: Makefile.in ../config.status
49 cd .. && ./config.status
diff --git a/utility/ap_pcre.c b/utility/ap_pcre.c
new file mode 100644
index 0000000..b2b9767
--- /dev/null
+++ b/utility/ap_pcre.c
@@ -0,0 +1,344 @@
1/*************************************************
2 * Perl-Compatible Regular Expressions *
3 *************************************************/
4
5/*
6 This is a library of functions to support regular expressions whose syntax
7 and semantics are as close as possible to those of the Perl 5 language. See
8 the file Tech.Notes for some information on the internals.
9
10 This module is a wrapper that provides a POSIX API to the underlying PCRE
11 functions.
12
13 Written by: Philip Hazel <ph10@cam.ac.uk>
14
15 Copyright (c) 1997-2004 University of Cambridge
16
17 -----------------------------------------------------------------------------
18 Redistribution and use in source and binary forms, with or without
19 modification, are permitted provided that the following conditions are met:
20
21 * Redistributions of source code must retain the above copyright notice,
22 this list of conditions and the following disclaimer.
23
24 * Redistributions in binary form must reproduce the above copyright
25 notice, this list of conditions and the following disclaimer in the
26 documentation and/or other materials provided with the distribution.
27
28 * Neither the name of the University of Cambridge nor the names of its
29 contributors may be used to endorse or promote products derived from
30 this software without specific prior written permission.
31
32 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
33 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
34 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
35 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
36 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
37 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
38 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
39 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
40 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
41 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
42 POSSIBILITY OF SUCH DAMAGE.
43 -----------------------------------------------------------------------------
44 */
45
46#include "apr_lib.h"
47#include "apr_strings.h"
48#include "ap_pcre.h"
49#include "pcre.h"
50
51#define APR_WANT_STRFUNC
52#include "apr_want.h"
53
54#ifndef POSIX_MALLOC_THRESHOLD
55#define POSIX_MALLOC_THRESHOLD (10)
56#endif
57
58/* Table of error strings corresponding to POSIX error codes; must be
59 * kept in synch with include/ap_regex.h's AP_REG_E* definitions. */
60
61static const char *const pstring[] = {
62 "", /* Dummy for value 0 */
63 "internal error", /* AP_REG_ASSERT */
64 "failed to get memory", /* AP_REG_ESPACE */
65 "bad argument", /* AP_REG_INVARG */
66 "match failed" /* AP_REG_NOMATCH */
67};
68
69apr_size_t ap_regerror(int errcode, const ap_regex_t *preg, char *errbuf,
70 apr_size_t errbuf_size)
71{
72 const char *message, *addmessage;
73 apr_size_t length, addlength;
74
75 message
76 = (errcode >= (int)(sizeof(pstring)/sizeof(char *))) ? "unknown error code"
77 : pstring[errcode];
78 length = strlen(message) + 1;
79
80 addmessage = " at offset ";
81 addlength
82 = (preg != NULL && (int)preg->re_erroffset != -1) ? strlen(addmessage)
83 + 6
84 : 0;
85
86 if (errbuf_size > 0) {
87 if (addlength > 0 && errbuf_size >= length + addlength)
88 apr_snprintf(errbuf, sizeof errbuf, "%s%s%-6d", message,
89 addmessage, (int)preg->re_erroffset);
90 else {
91 strncpy(errbuf, message, errbuf_size - 1);
92 errbuf[errbuf_size-1] = 0;
93 }
94 }
95
96 return length + addlength;
97}
98
99/*************************************************
100 * Free store held by a regex *
101 *************************************************/
102
103void ap_regfree(ap_regex_t *preg)
104{
105 (pcre_free)(preg->re_pcre);
106}
107
108/*************************************************
109 * Compile a regular expression *
110 *************************************************/
111
112/*
113 Arguments:
114 preg points to a structure for recording the compiled expression
115 pattern the pattern to compile
116 cflags compilation flags
117
118 Returns: 0 on success
119 various non-zero codes on failure
120 */
121
122int ap_regcomp(ap_regex_t *preg, const char *pattern, int cflags)
123{
124 const char *errorptr;
125 int erroffset;
126 int options = 0;
127
128 if ((cflags & AP_REG_ICASE) != 0)
129 options |= PCRE_CASELESS;
130 if ((cflags & AP_REG_NEWLINE) != 0)
131 options |= PCRE_MULTILINE;
132
133 preg->re_pcre = pcre_compile(pattern, options, &errorptr, &erroffset, NULL);
134 preg->re_erroffset = erroffset;
135
136 if (preg->re_pcre == NULL)
137 return AP_REG_INVARG;
138
139 preg->re_nsub = pcre_info((const pcre *)preg->re_pcre, NULL, NULL);
140 return 0;
141}
142
143/*************************************************
144 * Match a regular expression *
145 *************************************************/
146
147/* Unfortunately, PCRE requires 3 ints of working space for each captured
148 substring, so we have to get and release working store instead of just using
149 the POSIX structures as was done in earlier releases when PCRE needed only 2
150 ints. However, if the number of possible capturing brackets is small, use a
151 block of store on the stack, to reduce the use of malloc/free. The threshold is
152 in a macro that can be changed at configure time. */
153
154int ap_regexec(const ap_regex_t *preg, const char *string, apr_size_t nmatch,
155 ap_regmatch_t pmatch[], int eflags)
156{
157 int rc;
158 int options = 0;
159 int *ovector= NULL;
160 int small_ovector[POSIX_MALLOC_THRESHOLD * 3];
161 int allocated_ovector = 0;
162
163 if ((eflags & AP_REG_NOTBOL) != 0)
164 options |= PCRE_NOTBOL;
165 if ((eflags & AP_REG_NOTEOL) != 0)
166 options |= PCRE_NOTEOL;
167
168 ((ap_regex_t *)preg)->re_erroffset = (apr_size_t)(-1); /* Only has meaning after compile */
169
170 if (nmatch > 0) {
171 if (nmatch <= POSIX_MALLOC_THRESHOLD) {
172 ovector = &(small_ovector[0]);
173 } else {
174 ovector = (int *)malloc(sizeof(int) * nmatch * 3);
175 if (ovector == NULL)
176 return AP_REG_ESPACE;
177 allocated_ovector = 1;
178 }
179 }
180
181 rc = pcre_exec((const pcre *)preg->re_pcre, NULL, string,
182 (int)strlen(string), 0, options, ovector, nmatch * 3);
183
184 if (rc == 0)
185 rc = nmatch; /* All captured slots were filled in */
186
187 if (rc >= 0) {
188 apr_size_t i;
189 for (i = 0; i < (apr_size_t)rc; i++) {
190 pmatch[i].rm_so = ovector[i*2];
191 pmatch[i].rm_eo = ovector[i*2+1];
192 }
193 if (allocated_ovector)
194 free(ovector);
195 for (; i < nmatch; i++)
196 pmatch[i].rm_so = pmatch[i].rm_eo = -1;
197 return 0;
198 }
199
200 else {
201 if (allocated_ovector)
202 free(ovector);
203 switch (rc) {
204 case PCRE_ERROR_NOMATCH:
205 return AP_REG_NOMATCH;
206 case PCRE_ERROR_NULL:
207 return AP_REG_INVARG;
208 case PCRE_ERROR_BADOPTION:
209 return AP_REG_INVARG;
210 case PCRE_ERROR_BADMAGIC:
211 return AP_REG_INVARG;
212 case PCRE_ERROR_UNKNOWN_NODE:
213 return AP_REG_ASSERT;
214 case PCRE_ERROR_NOMEMORY:
215 return AP_REG_ESPACE;
216#ifdef PCRE_ERROR_MATCHLIMIT
217 case PCRE_ERROR_MATCHLIMIT: return AP_REG_ESPACE;
218#endif
219#ifdef PCRE_ERROR_BADUTF8
220 case PCRE_ERROR_BADUTF8: return AP_REG_INVARG;
221#endif
222#ifdef PCRE_ERROR_BADUTF8_OFFSET
223 case PCRE_ERROR_BADUTF8_OFFSET: return AP_REG_INVARG;
224#endif
225 default:
226 return AP_REG_ASSERT;
227 }
228 }
229}
230
231/*
232 * Here's a pool-based interface to the POSIX-esque ap_regcomp().
233 * Note that we return ap_regex_t instead of being passed one.
234 * The reason is that if you use an already-used ap_regex_t structure,
235 * the memory that you've already allocated gets forgotten, and
236 * regfree() doesn't clear it. So we don't allow it.
237 */
238
239static apr_status_t regex_cleanup(void *preg)
240{
241 ap_regfree((ap_regex_t *) preg);
242 return APR_SUCCESS;
243}
244
245ap_regex_t *ap_pregcomp(apr_pool_t *p, const char *pattern, int cflags)
246{
247 ap_regex_t *preg = apr_palloc(p, sizeof *preg);
248
249 if (ap_regcomp(preg, pattern, cflags)) {
250 return NULL;
251 }
252
253 apr_pool_cleanup_register(p, (void *) preg, regex_cleanup,
254 apr_pool_cleanup_null);
255
256 return preg;
257}
258
259void ap_pregfree(apr_pool_t *p, ap_regex_t *reg)
260{
261 ap_regfree(reg);
262 apr_pool_cleanup_kill(p, (void *) reg, regex_cleanup);
263}
264
265/* This function substitutes for $0-$9, filling in regular expression
266 * submatches. Pass it the same nmatch and pmatch arguments that you
267 * passed ap_regexec(). pmatch should not be greater than the maximum number
268 * of subexpressions - i.e. one more than the re_nsub member of ap_regex_t.
269 *
270 * input should be the string with the $-expressions, source should be the
271 * string that was matched against.
272 *
273 * It returns the substituted string, or NULL on error.
274 *
275 * Parts of this code are based on Henry Spencer's regsub(), from his
276 * AT&T V8 regexp package.
277 */
278
279char * ap_pregsub(apr_pool_t *p, const char *input, const char *source,
280 size_t nmatch, ap_regmatch_t pmatch[])
281{
282 const char *src = input;
283 char *dest, *dst;
284 char c;
285 size_t no;
286 int len;
287
288 if (!source)
289 return NULL;
290 if (!nmatch)
291 return apr_pstrdup(p, src);
292
293 /* First pass, find the size */
294
295 len = 0;
296
297 while ((c = *src++) != '\0') {
298 if (c == '&')
299 no = 0;
300 else if (c == '$' && apr_isdigit(*src))
301 no = *src++ - '0';
302 else
303 no = 10;
304
305 if (no> 9) { /* Ordinary character. */
306 if (c == '\\' && (*src == '$' || *src == '&'))
307 c = *src++;
308 len++;
309 } else if (no < nmatch && pmatch[no].rm_so < pmatch[no].rm_eo) {
310 len += pmatch[no].rm_eo - pmatch[no].rm_so;
311 }
312
313 }
314
315 dest = dst = apr_pcalloc(p, len + 1);
316
317 /* Now actually fill in the string */
318
319 src = input;
320
321 while ((c = *src++) != '\0') {
322 if (c == '&')
323 no = 0;
324 else if (c == '$' && apr_isdigit(*src))
325 no = *src++ - '0';
326 else
327 no = 10;
328
329 if (no> 9) { /* Ordinary character. */
330 if (c == '\\' && (*src == '$' || *src == '&'))
331 c = *src++;
332 *dst++ = c;
333 } else if (no < nmatch && pmatch[no].rm_so < pmatch[no].rm_eo) {
334 len = pmatch[no].rm_eo - pmatch[no].rm_so;
335 memcpy(dst, source + pmatch[no].rm_so, len);
336 dst += len;
337 }
338
339 }
340 *dst = '\0';
341
342 return dest;
343}
344/* End of pcreposix.c */
diff --git a/utility/ap_pcre.h b/utility/ap_pcre.h
new file mode 100644
index 0000000..e817dc2
--- /dev/null
+++ b/utility/ap_pcre.h
@@ -0,0 +1,178 @@
1/* Licensed to the Apache Software Foundation (ASF) under one or more
2 * contributor license agreements. See the NOTICE file distributed with
3 * this work for additional information regarding copyright ownership.
4 * The ASF licenses this file to You under the Apache License, Version 2.0
5 * (the "License"); you may not use this file except in compliance with
6 * the License. You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/* Derived from PCRE's pcreposix.h.
18
19 Copyright (c) 1997-2004 University of Cambridge
20
21-----------------------------------------------------------------------------
22Redistribution and use in source and binary forms, with or without
23modification, are permitted provided that the following conditions are met:
24
25 * Redistributions of source code must retain the above copyright notice,
26 this list of conditions and the following disclaimer.
27
28 * Redistributions in binary form must reproduce the above copyright
29 notice, this list of conditions and the following disclaimer in the
30 documentation and/or other materials provided with the distribution.
31
32 * Neither the name of the University of Cambridge nor the names of its
33 contributors may be used to endorse or promote products derived from
34 this software without specific prior written permission.
35
36THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
37AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
38IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
39ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
40LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
41CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
42SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
43INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
44CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
45ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
46POSSIBILITY OF SUCH DAMAGE.
47-----------------------------------------------------------------------------
48*/
49
50/**
51 * @file ap_regex.h
52 * @brief Apache Regex defines
53 */
54
55#ifndef AP_REGEX_H
56#define AP_REGEX_H
57
58#include "apr.h"
59
60/* Allow for C++ users */
61
62#ifdef __cplusplus
63extern "C" {
64#endif
65
66/* Options for ap_regexec: */
67
68#define AP_REG_ICASE 0x01 /** use a case-insensitive match */
69#define AP_REG_NEWLINE 0x02 /** don't match newlines against '.' etc */
70#define AP_REG_NOTBOL 0x04 /** ^ will not match against start-of-string */
71#define AP_REG_NOTEOL 0x08 /** $ will not match against end-of-string */
72
73#define AP_REG_EXTENDED (0) /** unused */
74#define AP_REG_NOSUB (0) /** unused */
75
76#define AP_MAX_REG_MATCH 10
77
78/* Error values: */
79enum {
80 AP_REG_ASSERT = 1, /** internal error ? */
81 AP_REG_ESPACE, /** failed to get memory */
82 AP_REG_INVARG, /** invalid argument */
83 AP_REG_NOMATCH /** match failed */
84};
85
86/* The structure representing a compiled regular expression. */
87typedef struct {
88 void *re_pcre;
89 apr_size_t re_nsub;
90 apr_size_t re_erroffset;
91} ap_regex_t;
92
93/* The structure in which a captured offset is returned. */
94typedef struct {
95 int rm_so;
96 int rm_eo;
97} ap_regmatch_t;
98
99/* The functions */
100
101/**
102 * Compile a regular expression.
103 * @param preg Returned compiled regex
104 * @param regex The regular expression string
105 * @param cflags Must be zero (currently).
106 * @return Zero on success or non-zero on error
107 */
108int ap_regcomp(ap_regex_t *preg, const char *regex, int cflags);
109
110/**
111 * Match a NUL-terminated string against a pre-compiled regex.
112 * @param preg The pre-compiled regex
113 * @param string The string to match
114 * @param nmatch Provide information regarding the location of any matches
115 * @param pmatch Provide information regarding the location of any matches
116 * @param eflags Bitwise OR of any of AP_REG_* flags
117 * @return 0 for successful match, #REG_NOMATCH otherwise
118 */
119int ap_regexec(const ap_regex_t *preg, const char *string,
120 apr_size_t nmatch, ap_regmatch_t *pmatch, int eflags);
121
122/**
123 * Return the error code returned by regcomp or regexec into error messages
124 * @param errcode the error code returned by regexec or regcomp
125 * @param preg The precompiled regex
126 * @param errbuf A buffer to store the error in
127 * @param errbuf_size The size of the buffer
128 */
129apr_size_t ap_regerror(int errcode, const ap_regex_t *preg,
130 char *errbuf, apr_size_t errbuf_size);
131
132/** Destroy a pre-compiled regex.
133 * @param preg The pre-compiled regex to free.
134 */
135void ap_regfree(ap_regex_t *preg);
136
137/**
138 * Compile a regular expression to be used later
139 * @param p The pool to allocate from
140 * @param pattern the regular expression to compile
141 * @param cflags The bitwise or of one or more of the following:
142 * @li REG_EXTENDED - Use POSIX extended Regular Expressions
143 * @li REG_ICASE - Ignore case
144 * @li REG_NOSUB - Support for substring addressing of matches
145 * not required
146 * @li REG_NEWLINE - Match-any-character operators don't match new-line
147 * @return The compiled regular expression
148 */
149ap_regex_t * ap_pregcomp(apr_pool_t *p, const char *pattern,
150 int cflags);
151
152/**
153 * Free the memory associated with a compiled regular expression
154 * @param p The pool the regex was allocated from
155 * @param reg The regular expression to free
156 */
157void ap_pregfree(apr_pool_t *p, ap_regex_t *reg);
158
159/**
160 * After performing a successful regex match, you may use this function to
161 * perform a series of string substitutions based on subexpressions that were
162 * matched during the call to ap_regexec
163 * @param p The pool to allocate from
164 * @param input An arbitrary string containing $1 through $9. These are
165 * replaced with the corresponding matched sub-expressions
166 * @param source The string that was originally matched to the regex
167 * @param nmatch the nmatch returned from ap_pregex
168 * @param pmatch the pmatch array returned from ap_pregex
169 */
170char * ap_pregsub(apr_pool_t *p, const char *input, const char *source,
171 size_t nmatch, ap_regmatch_t pmatch[]);
172
173#ifdef __cplusplus
174} /* extern "C" */
175#endif
176
177#endif /* AP_REGEX_T */
178
diff --git a/utility/config.c b/utility/config.c
new file mode 100644
index 0000000..6867476
--- /dev/null
+++ b/utility/config.c
@@ -0,0 +1,534 @@
1#include "apr.h"
2#include "apr_file_info.h"
3#include "apr_file_io.h"
4#include "apr_strings.h"
5#include "apr_hash.h"
6#include "apr_uri.h"
7
8#include "shell.h"
9#include "config.h"
10#include "util.h"
11#include "logparse.h"
12#include "autoconfig.h"
13
14apr_hash_t *g_config_opts;
15
16static apr_status_t config_set_string(config_t *cfg, config_opt_t *opt,
17 int argc, const char **argv)
18{
19 int offset = (int)(long)opt->data;
20 char **data = (char **)((void *)cfg + offset);
21 if (argc != 2)
22 return APR_EINVAL;
23 *data = apr_pstrdup(cfg->pool, argv[1]);
24 return APR_SUCCESS;
25}
26
27static apr_status_t config_set_file(config_t *cfg, config_opt_t *opt,
28 int argc, const char **argv)
29{
30 int offset = (int)(long)opt->data;
31 char **data = (char **)((void *)cfg + offset);
32 if (argc != 2)
33 return APR_EINVAL;
34 apr_filepath_merge(data, NULL, argv[1],
35 APR_FILEPATH_TRUENAME, cfg->pool);
36 return APR_SUCCESS;
37}
38
39static apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc,
40 const char **argv)
41{
42 int offset = (int)(long)opt->data;
43 int *data = (int *)((void *)cfg + offset);
44 if (argc != 2)
45 return APR_EINVAL;
46 *data = apr_atoi64(argv[1]);
47 return APR_SUCCESS;
48}
49
50static apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc,
51 const char **argv)
52{
53 int offset = (int)(long)opt->data;
54 int *data = (int *)((void *)cfg + offset);
55 if (argc != 2)
56 return APR_EINVAL;
57 *data = CHECK_YESNO(argv[1]);
58 return APR_SUCCESS;
59}
60
61static apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt,
62 int argc, const char **argv)
63{
64 if (argc != 2)
65 return APR_EINVAL;
66 if (!strcasecmp(argv[1], "error")) {
67 cfg->loglevel = LOGLEVEL_ERROR;
68 } else if (!strcasecmp(argv[1], "notice")) {
69 cfg->loglevel = LOGLEVEL_NOTICE;
70 } else if (!strcasecmp(argv[1], "debug")) {
71 cfg->loglevel = LOGLEVEL_DEBUG;
72 } else {
73 cfg->loglevel = LOGLEVEL_ERROR;
74 }
75 return APR_SUCCESS;
76}
77
78static apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt,
79 int argc, const char **argv)
80{
81 config_filestat_t *newp;
82 if (argc != 2)
83 return APR_EINVAL;
84 newp = (config_filestat_t *)apr_array_push(cfg->input_files);
85 char *temp;
86 apr_filepath_merge(&temp, NULL, argv[1],
87 APR_FILEPATH_TRUENAME, cfg->pool);
88 newp->fname = temp;
89 newp->result = "Not Parsed";
90 return APR_SUCCESS;
91}
92
93static apr_status_t config_set_dummy(config_t *cfg, config_opt_t *opt,
94 int argc, const char **argv)
95{
96 return APR_SUCCESS;
97}
98
99static apr_status_t config_set_logformat(config_t *cfg, config_opt_t *opt,
100 int argc, const char **argv)
101{
102 config_logformat_t *format;
103 config_logformat_field_t *field;
104
105 if (argc != 4)
106 return APR_EINVAL;
107
108 format = apr_hash_get(cfg->log_formats, argv[1], APR_HASH_KEY_STRING);
109 if (!format) {
110 format = apr_palloc(cfg->pool, sizeof(config_logformat_t));
111 format->name = apr_pstrdup(cfg->pool, argv[1]);
112 format->fields = apr_array_make(cfg->pool, 5,
113 sizeof(config_logformat_field_t));
114 apr_hash_set(cfg->log_formats, apr_pstrdup(cfg->pool,argv[1]), APR_HASH_KEY_STRING, format);
115 }
116 field = (config_logformat_field_t *)apr_array_push(format->fields);
117 field->name = apr_pstrdup(cfg->pool, argv[2]);
118 field->datatype = apr_pstrdup(cfg->pool, argv[3]);
119 return APR_SUCCESS;
120}
121
122static apr_status_t config_set_output_field(config_t *cfg, config_opt_t *opt,
123 int argc, const char **argv)
124{
125 config_output_field_t *field;
126 char *type, *size, *temp;
127
128 if (argc < 5)
129 return APR_EINVAL;
130 field = (config_output_field_t *)apr_array_push(cfg->output_fields);
131 field->field = apr_pstrdup(cfg->pool, argv[1]);
132 field->source = apr_pstrdup(cfg->pool, argv[4]);
133 field->def = apr_pstrdup(cfg->pool, argv[3]);
134 type = size = apr_pstrdup(cfg->pool, argv[2]);
135 while (*size!='\0' && *size!='(')
136 size++;
137 if (*size == '(') {
138 *size = '\0';
139 size++;
140 temp = size;
141 while (*temp != '\0' && *temp != ')')
142 temp++;
143 *temp = '\0';
144 field->size = apr_atoi64(size);
145 }
146 if (strcasecmp("VARCHAR", type)==0) {
147 field->datatype = LOGSQL_DATATYPE_VARCHAR;
148 } else if (strcasecmp("INT", type)==0) {
149 field->datatype = LOGSQL_DATATYPE_INT;
150 } else if (strcasecmp("CHAR", type)==0) {
151 field->datatype = LOGSQL_DATATYPE_CHAR;
152 } else if (strcasecmp("SMALLINT", type)==0) {
153 field->datatype = LOGSQL_DATATYPE_SMALLINT;
154 } else if (strcasecmp("BIGINT", type)==0) {
155 field->datatype = LOGSQL_DATATYPE_BIGINT;
156 } else {
157 return APR_EINVAL;
158 }
159
160 // Has a function
161 if (argc > 5) {
162 int i;
163 field->fname = apr_pstrdup(cfg->pool, argv[5]);
164 field->func = parser_get_func(field->fname);
165 field->args = apr_pcalloc(cfg->pool, sizeof(char *) * (argc-5+1));
166 for (i=6; i<=argc; i++) {
167 field->args[i-6] = apr_pstrdup(cfg->pool, argv[i]);
168 }
169 }
170
171 return APR_SUCCESS;
172}
173
174static apr_status_t config_set_filter(config_t *cfg, config_opt_t *opt,
175 int argc, const char **argv)
176{
177 int argn = 1;
178 config_filter_t *filter;
179 switch (opt->name[1]) {
180 case 'i': //line
181 filter = apr_array_push(cfg->linefilters);
182 break;
183 case 'r': //pre
184 filter = apr_array_push(cfg->prefilters);
185 break;
186 case 'o': //post
187 filter = apr_array_push(cfg->postfilters);
188 break;
189 }
190
191 if (opt->name[0]=='P') { // Pre or post 2-3 args
192 if (argc == 1)
193 return APR_EINVAL;
194 filter->field = apr_pstrdup(cfg->pool, argv[1]);
195 argn++;
196 } // Otherwise Line based only 1-2 args (no field)
197 if (argc <= argn)
198 return APR_EINVAL;
199 if (*argv[argn] == '+')
200 argn++;
201 if (*argv[argn] == '-') {
202 filter->negative = 1;
203 argn++;
204 }
205 if (filter->negative && argc == argn) {
206 // if no filter for negative.. that's ok.. Assume ALL
207 return APR_SUCCESS;
208 }
209 if (argc <= argn)
210 return APR_EINVAL;
211 filter->filter = apr_pstrdup(cfg->pool, argv[argn]);
212 filter->regex = ap_pregcomp(cfg->pool, filter->filter, AP_REG_EXTENDED|AP_REG_ICASE);
213 return APR_SUCCESS;
214}
215
216void config_dump(config_t *cfg)
217{
218 apr_hash_index_t *hi;
219 int i;
220 config_output_field_t *fields;
221 config_filter_t *filters;
222
223 printf("ErrorLog: %s\n", cfg->errorlog);
224 printf("LogLevel: %d\n", cfg->loglevel);
225
226 printf("BadLineFile: %s\n", cfg->badlinefile);
227 printf("BadLineMax: %d\n", cfg->badlinemax);
228
229 printf("InputDir: %s\n", cfg->input_dir);
230
231 printf("Split input files: %d\n", cfg->split_enabled);
232 printf("Split output directory: %s\n", cfg->split_dir);
233 printf("Split file count: %d\n", cfg->split_count);
234 printf("Split min lines: %'d\n", cfg->split_minimum);
235 printf("Split max lines: %'d\n", cfg->split_maximum);
236
237 printf("DB Driver: %s\n", cfg->dbdriver);
238 printf("DB Params: %s\n", cfg->dbparams);
239
240 printf("Table: %s\n", cfg->table);
241 printf("Transactions: %d\n", cfg->transactions);
242 printf("MachineID: %s\n", cfg->machineid);
243
244 printf("Log formats:\n");
245 for (hi = apr_hash_first(cfg->pool, cfg->log_formats); hi; hi
246 = apr_hash_next(hi)) {
247 config_logformat_t *format;
248 config_logformat_field_t *fields;
249 int i;
250
251 apr_hash_this(hi, NULL, NULL, (void **)&format);
252 printf(">> '%s'\n", format->name);
253 fields = (config_logformat_field_t *)format->fields->elts;
254 for (i=0; i<format->fields->nelts; i++) {
255 printf(">>>> %s:%s\n", fields[i].name, fields[i].datatype);
256 }
257 }
258 printf("Log Format: '%s'\n", cfg->logformat);
259
260 printf("Output Fields:\n");
261 fields = (config_output_field_t *)cfg->output_fields->elts;
262 for (i=0; i<cfg->output_fields->nelts; i++) {
263 printf(">> %s %s(%d) DEFAULT '%s': %s", fields[i].field,
264 logsql_field_datatyeName(fields[i].datatype),
265 fields[i].size, fields[i].def, fields[i].source);
266 if (fields[i].func) {
267 printf(" :: %s(", fields[i].fname);
268 if (fields[i].args) {
269 int a = 0;
270 while (fields[i].args[a]) {
271 printf("%s,", fields[i].args[a]);
272 a++;
273 }
274 }
275 printf(")");
276 }
277 printf("\n");
278 }
279 printf("Filters:\n>> Line:\n");
280 filters = (config_filter_t *)cfg->linefilters->elts;
281 for (i=0; i<cfg->linefilters->nelts; i++) {
282 printf(">>>> %c \"%s\" (%pp)\n",filters[i].negative ? '-':'+',
283 filters[i].filter, filters[i].regex);
284 }
285 printf(">> Pre:\n");
286 filters = (config_filter_t *)cfg->prefilters->elts;
287 for (i=0; i<cfg->prefilters->nelts; i++) {
288 printf(">>>> %s %c \"%s\" (%pp)\n",
289 filters[i].field, filters[i].negative ? '-':'+',
290 filters[i].filter, filters[i].regex);
291 }
292 printf(">> Post:\n");
293 filters = (config_filter_t *)cfg->postfilters->elts;
294 for (i=0; i<cfg->postfilters->nelts; i++) {
295 printf(">>>> %s %c \"%s\" (%pp)\n",
296 filters[i].field, filters[i].negative ? '-':'+',
297 filters[i].filter, filters[i].regex);
298 }
299
300 printf("DryRun: %d\n", cfg->dryrun);
301 printf("Summary: %d\n", cfg->summary);
302}
303
304#define config_get_option(name) apr_hash_get(g_config_opts, name, APR_HASH_KEY_STRING)
305
306static void config_add_option(apr_pool_t *p, const char *const name,
307 const char *const help, config_func_t func, void *data)
308{
309 config_opt_t *opt;
310 if (!g_config_opts) {
311 g_config_opts = apr_hash_make(p);
312 }
313 opt = apr_palloc(p, sizeof(config_opt_t));
314 opt->name = name;
315 opt->help = help;
316 opt->func = func;
317 opt->data = data;
318 apr_hash_set(g_config_opts, lowerstr(p, name), APR_HASH_KEY_STRING, opt);
319}
320
321void config_init(apr_pool_t *p)
322{
323 config_add_option(p, "ErrorLog", "File to log errors", config_set_file,
324 (void *)APR_OFFSETOF(config_t, errorlog));
325 config_add_option(p, "LogLevel", "Set Log Level (error, warn, debug, quiet)",
326 config_set_loglevel, NULL);
327
328 config_add_option(p, "BadLineFile", "File to log bad log lines", config_set_file,
329 (void *)APR_OFFSETOF(config_t, badlinefile));
330 config_add_option(p, "BadLineMax", "Max number of bad lines before aborting",
331 config_set_int, (void *)APR_OFFSETOF(config_t, badlinemax));
332
333
334 config_add_option(p, "InputDirectory", "Directory to scan for log files",
335 config_set_file, (void *)APR_OFFSETOF(config_t, input_dir));
336 config_add_option(p, "InputFile", "Parse only this file",
337 config_set_inputfile, NULL);
338
339 config_add_option(p, "SplitInput",
340 "Split the file into pieces, then process",
341 config_set_flag, (void *)APR_OFFSETOF(config_t, split_enabled));
342 config_add_option(p, "SplitCount",
343 "Split the file into N number of pieces",
344 config_set_int, (void *)APR_OFFSETOF(config_t, split_count));
345 config_add_option(p, "SplitMinLines",
346 "Each split piece will have a minumum of N lines",
347 config_set_int, (void *)APR_OFFSETOF(config_t, split_minimum));
348 config_add_option(p, "SplitMaxLines",
349 "Each split piece will have a maximum of N lines",
350 config_set_int, (void *)APR_OFFSETOF(config_t, split_maximum));
351 config_add_option(p, "SplitDirectory",
352 "Output directory to put intermediate split files",
353 config_set_file, (void *)APR_OFFSETOF(config_t, split_dir));
354
355 config_add_option(p, "ThreadCount",
356 "Numer of threads to use for processing the input files",
357 config_set_int, (void *)APR_OFFSETOF(config_t, thread_count));
358
359 config_add_option(p, "DBDDriver", "DBD Driver to use",
360 config_set_string, (void *)APR_OFFSETOF(config_t, dbdriver));
361 config_add_option(p, "DBDParams", "DBD Connection Parameters",
362 config_set_string, (void *)APR_OFFSETOF(config_t, dbparams));
363 config_add_option(p, "Table", "Table to import the log to",
364 config_set_string, (void *)APR_OFFSETOF(config_t, table));
365 config_add_option(p, "UseTransactions", "Enable Transactions?",
366 config_set_flag, (void *)APR_OFFSETOF(config_t, transactions));
367 config_add_option(p, "MachineID", "Machine ID to set",
368 config_set_string, (void *)APR_OFFSETOF(config_t, machineid));
369
370 config_add_option(p, "LogFormatConfig", "Define input log formats",
371 config_set_logformat, NULL);
372 config_add_option(p, "LogFormat", "Use this logformat when parsing files",
373 config_set_string, (void *)APR_OFFSETOF(config_t, logformat));
374
375 config_add_option(p, "LineFilter",
376 "A regular expression to apply to the input line",
377 config_set_filter, (void *)APR_OFFSETOF(config_t, linefilters));
378 config_add_option(p, "PreFilter",
379 "A regular expression to apply to a specific input field",
380 config_set_filter, (void *)APR_OFFSETOF(config_t, prefilters));
381 config_add_option(p, "PostFilter",
382 "A regular expression to apply to a specific SQL output field",
383 config_set_filter, (void *)APR_OFFSETOF(config_t, postfilters));
384
385 config_add_option(p, "OutputField",
386 "Define output fields: field datatype source optfunc optarg...",
387 config_set_output_field, NULL);
388
389 config_add_option(p, "DryRun", "Don't perform any actual database changes",
390 config_set_flag, (void *)APR_OFFSETOF(config_t, dryrun));
391 config_add_option(p, "Dump", "Dump Configuration and quit",
392 config_set_flag, (void *)APR_OFFSETOF(config_t, dump));
393 config_add_option(p, "Config", "Dummy to handle config directive",
394 config_set_dummy, NULL);
395 config_add_option(p, "Summary", "Show the summary before exit?",
396 config_set_flag, (void *)APR_OFFSETOF(config_t, summary));
397}
398
399config_t *config_create(apr_pool_t *p)
400{
401 config_t *cfg;
402 apr_pool_t *sp;
403 apr_pool_create(&sp, p);
404 cfg = apr_pcalloc(sp, sizeof(config_t));
405 cfg->pool = sp;
406 cfg->loglevel = LOGLEVEL_ERROR;
407 cfg->summary = 1;
408 cfg->transactions = 1;
409 cfg->thread_count = 0; // default zero thread (aka non-threaded)
410 cfg->split_count = 4;
411 cfg->split_minimum = 10000;
412 cfg->split_maximum = 50000;
413 cfg->input_files = apr_array_make(cfg->pool, 2, sizeof(config_filestat_t));
414 cfg->log_formats = apr_hash_make(cfg->pool);
415 cfg->output_fields = apr_array_make(cfg->pool, 10,
416 sizeof(config_output_field_t));
417 cfg->linefilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t));
418 cfg->prefilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t));
419 cfg->postfilters = apr_array_make(cfg->pool, 2, sizeof(config_filter_t));
420 return cfg;
421}
422
423apr_status_t config_check(config_t *cfg)
424{
425 apr_status_t ret = APR_SUCCESS;
426 if (!cfg->dbdriver || !cfg->dbparams) {
427 logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: Database configuration is missing");
428 ret = APR_EINVAL;
429 }
430 if (!cfg->table) {
431 logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: No Log Table defined");
432 ret = APR_EINVAL;
433 }
434 if (apr_is_empty_array(cfg->output_fields)) {
435 logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: No Output Fields Defined");
436 ret = APR_EINVAL;
437 }
438 if (apr_hash_count(cfg->log_formats)==0) {
439 logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: No Input Log Formats Defined");
440 ret = APR_EINVAL;
441 }
442#if !defined(HAVE_APR_DBD_TRANSACTION_MODE_GET)
443 if (cfg->transactions) {
444 logging_log(cfg, LOGLEVEL_NOISE, "CONFIG: Disabling Transaction Support. Requires apr-util 1.3.0 or higher");
445 cfg->transactions = 0;
446 }
447#endif
448 return ret;
449}
450
451static int config_merge(void *rec, const char *key, const char *value)
452{
453 config_t *cfg = (config_t *)rec;
454
455 config_opt_t *opt= config_get_option(key);
456 if (opt) {
457 const char *args[] = {
458 key,
459 value };
460 opt->func(cfg, opt, 2, args);
461 } else {
462 logging_log(cfg, LOGLEVEL_NOISE, "Unhandled: %s\n", key);
463 }
464 return 1;
465}
466
467apr_status_t config_read(config_t *cfg, const char *filename,
468 apr_table_t *merge)
469{
470 apr_finfo_t finfo;
471 apr_file_t *file;
472 apr_status_t rv, ret= APR_SUCCESS;
473 apr_pool_t *tp, *targp;
474 config_opt_t *opt;
475 char buff[1024];
476 char *ptr;
477 char **targv;
478 int targc;
479 int line;
480
481 apr_pool_create(&tp, cfg->pool);
482 apr_pool_create(&targp, tp);
483
484 if (apr_stat(&finfo, filename, APR_FINFO_MIN, tp) != APR_SUCCESS) {
485 return APR_ENOENT;
486 }
487 rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED,
488 APR_OS_DEFAULT, tp);
489 if (rv != APR_SUCCESS)
490 return rv;
491
492 line = 0;
493 do {
494 rv = apr_file_gets(buff, 1024, file);
495 if (rv == APR_SUCCESS) { // we read data
496 line++;
497
498 // skip leading white space
499 for (ptr = buff; *ptr == ' ' || *ptr == '\t'; ptr++)
500 ;
501 line_chomp(ptr);
502
503 // skip comments
504 if (*ptr == '#')
505 continue;
506 if (*ptr == '\0')
507 continue;
508 apr_pool_clear(targp);
509 parser_tokenize_line(ptr, &targv, targp);
510 targc = 0;
511 while (targv[targc])
512 targc++;
513 opt = config_get_option(lowerstr(targp,targv[0]));
514 if (opt) {
515 rv = opt->func(cfg, opt, targc, (const char **)targv);
516 if (APR_STATUS_IS_EINVAL(rv)) {
517 logging_log(cfg, LOGLEVEL_NOISE,
518 "Config Error: Invalid Arguments for %s\n\t%s\n",
519 opt->name, opt->help);
520 ret = rv;
521 }
522 } else {
523 logging_log(cfg, LOGLEVEL_NOISE, "Unhandled: %s\n", targv[0]);
524 }
525 }
526 } while (rv == APR_SUCCESS);
527
528 // Apply merges
529 apr_table_do(config_merge, (void *)cfg, merge, NULL);
530
531 apr_file_close(file);
532 apr_pool_destroy(tp);
533 return ret;
534}
diff --git a/utility/config.h b/utility/config.h
new file mode 100644
index 0000000..a9bd1b5
--- /dev/null
+++ b/utility/config.h
@@ -0,0 +1,190 @@
1#ifndef CONFIG_H_
2#define CONFIG_H_
3
4#include "apr_tables.h"
5#include "apr_hash.h"
6#include "apr_file_io.h"
7#include "ap_pcre.h"
8
9typedef enum {
10 LOGLEVEL_NOISE = 0,
11 LOGLEVEL_NONE,
12 LOGLEVEL_ERROR,
13 LOGLEVEL_NOTICE,
14 LOGLEVEL_DEBUG,
15} loglevel_e;
16
17typedef struct config_dbd_t config_dbd_t;
18typedef struct config_t config_t;
19struct config_t {
20 /** the structures pool (to ease function arguments) */
21 apr_pool_t *pool;
22
23 /** error log file */
24 const char *errorlog;
25 /** log level */
26 loglevel_e loglevel;
27 /** error_log */
28 apr_file_t *errorlog_fp;
29 apr_file_t *errorlog_fperr;
30 apr_pool_t *errorlog_p;
31
32 const char *badlinefile;
33 const char *badlastfile;
34 apr_file_t *badline_fp;
35 int badline_count;
36 int badlinemax;
37
38
39 /** input directory of log files */
40 const char *input_dir;
41 /** list of files to scan */
42 apr_array_header_t *input_files;
43
44 /** split the input file before processing */
45 int split_enabled;
46 /** the number of files to split each input file into */
47 int split_count;
48 /** the minimum number of lines for each piece */
49 int split_minimum;
50 /** the maximum number of lines for each piece */
51 int split_maximum;
52 /** directory to put ouput split files */
53 const char *split_dir;
54
55 /** the number of threads to run the import in */
56 int thread_count;
57
58 /** db connection configuration */
59 const char *dbdriver;
60 const char *dbparams;
61
62 /** Logging table */
63 const char *table;
64 /** Use transactons */
65 int transactions;
66 /** Machine ID */
67 const char *machineid;
68
69 /** Log file formats */
70 apr_hash_t *log_formats;
71 /** format to use to parse files */
72 const char *logformat;
73
74 /** output fields */
75 apr_array_header_t *output_fields;
76
77 /** filter configuration */
78 apr_array_header_t *linefilters;
79 apr_array_header_t *prefilters;
80 apr_array_header_t *postfilters;
81
82 /** Dry Run */
83 int dryrun;
84 /** dump configuration only */
85 int dump;
86
87 /* Show the summary */
88 int summary;
89};
90
91typedef struct config_filestat_t config_filestat_t;
92struct config_filestat_t {
93 const char *fname;
94 apr_size_t linesparsed;
95 apr_size_t lineskipped;
96 apr_size_t linesbad;
97 const char *result;
98 apr_time_t start;
99 apr_time_t stop;
100};
101
102typedef struct config_logformat_t config_logformat_t;
103struct config_logformat_t {
104 const char *name;
105 apr_array_header_t *fields;
106};
107
108typedef struct config_logformat_field_t config_logformat_field_t;
109struct config_logformat_field_t {
110 const char *name;
111 const char *datatype;
112};
113
114typedef struct config_opt_t config_opt_t;
115typedef apr_status_t (*config_func_t)(config_t *cfg, config_opt_t *opt,
116 int argc, const char **argv);
117struct config_opt_t {
118 const char *name;
119 const char *help;
120 config_func_t func;
121 void *data;
122};
123
124typedef struct config_filter_t config_filter_t;
125struct config_filter_t {
126 const char *field;
127 const char *filter;
128 int negative;
129 ap_regex_t *regex;
130};
131
132typedef enum {
133 LOGSQL_DATATYPE_INT = 0,
134 LOGSQL_DATATYPE_SMALLINT,
135 LOGSQL_DATATYPE_VARCHAR,
136 LOGSQL_DATATYPE_CHAR,
137 LOGSQL_DATATYPE_BIGINT
138} logsql_field_datatype;
139#define logsql_field_datatyeName(x) \
140 (x == LOGSQL_DATATYPE_INT ? "INT" \
141 : (x == LOGSQL_DATATYPE_SMALLINT ? "SMALLINT" \
142 : (x == LOGSQL_DATATYPE_VARCHAR ? "VARCHAR" \
143 : (x == LOGSQL_DATATYPE_CHAR ? "CHAR" \
144 : (x == LOGSQL_DATATYPE_BIGINT ? "BIGINT" : "ERR")))))
145
146typedef struct config_output_field_t config_output_field_t;
147
148typedef struct parser_func_t parser_func_t;
149
150struct config_output_field_t {
151 const char *field;
152 logsql_field_datatype datatype;
153 apr_size_t size;
154 const char *def;
155 const char *source;
156 const char *fname;
157 parser_func_t *func;
158 const char **args;
159 void *data;
160};
161
162#define CHECK_YESNO(c) ((!strcasecmp(c,"on") || !strcasecmp(c,"yes")) ? 1 : 0)
163
164/**
165 * Initialize the config parser
166 */
167void config_init(apr_pool_t *p);
168
169/**
170 * Dump the configuration to stdout
171 */
172void config_dump(config_t *cfg);
173
174/**
175 * Checks the configuration to ensure all the required settings are set
176 */
177apr_status_t config_check(config_t *cfg);
178
179/**
180 * Creates the default configuration
181 */
182config_t *config_create(apr_pool_t *p);
183
184/**
185 * Read in a configuration file
186 */
187apr_status_t config_read(config_t *cfg, const char *filename,
188 apr_table_t *merge);
189
190#endif /*CONFIG_H_*/
diff --git a/utility/database.c b/utility/database.c
new file mode 100644
index 0000000..ff81caa
--- /dev/null
+++ b/utility/database.c
@@ -0,0 +1,212 @@
1#include "database.h"
2#include "apu.h"
3#include "apr_dbd.h"
4#include "apr_strings.h"
5
6#include "util.h"
7#include "autoconfig.h"
8
9struct config_dbd_t {
10 const apr_dbd_driver_t *driver;
11 apr_dbd_t *dbd;
12 apr_dbd_prepared_t *stmt;
13 apr_dbd_transaction_t *txn;
14 const char **args;
15};
16
17void database_init(apr_pool_t *p)
18{
19 apr_dbd_init(p);
20}
21
22/** @todo split this into load and connect */
23apr_status_t database_connect(config_t *cfg, config_dbd_t **dbconn)
24{
25 apr_status_t rv;
26 if (!cfg->dbdriver || !cfg->dbparams)
27 return APR_EINVAL;
28 if (!*dbconn) {
29 *dbconn = apr_pcalloc(cfg->pool, sizeof(config_dbd_t));
30 }
31 rv = apr_dbd_get_driver(cfg->pool, cfg->dbdriver, &((*dbconn)->driver));
32 if (rv) {
33
34 logging_log(cfg, LOGLEVEL_ERROR,
35 "DB: Could not load database driver %s. Error %s",
36 cfg->dbdriver, logging_strerror(rv));
37 return rv;
38 }
39
40 rv = apr_dbd_open((*dbconn)->driver, cfg->pool, cfg->dbparams,
41 &((*dbconn)->dbd));
42 if (rv) {
43 logging_log(cfg, LOGLEVEL_ERROR,
44 "DB: Could not connect to database. Error (%d)%s", rv,
45 logging_strerror(rv));
46 return rv;
47 }
48
49 return APR_SUCCESS;
50}
51
52apr_status_t database_disconnect(config_dbd_t *dbconn)
53{
54 return apr_dbd_close(dbconn->driver, dbconn->dbd);
55}
56
57static apr_dbd_prepared_t *database_prepare_insert(config_t *cfg,
58 config_dbd_t *dbconn, apr_pool_t *p)
59{
60 apr_status_t rv;
61 char *sql;
62 int i, f;
63 struct iovec *vec;
64 apr_dbd_prepared_t *stmt= NULL;
65 int nfs = cfg->output_fields->nelts;
66 config_output_field_t *ofields;
67
68 ofields = (config_output_field_t *)cfg->output_fields->elts;
69
70 vec = apr_palloc(p, (nfs*2 + 5) * sizeof(struct iovec));
71 sql = apr_palloc(p, (nfs*3));
72
73 vec[0].iov_base = "INSERT INTO ";
74 vec[0].iov_len = 12;
75 vec[1].iov_base = (void *)cfg->table;
76 vec[1].iov_len = strlen(cfg->table);
77 vec[2].iov_base = " (";
78 vec[2].iov_len = 2;
79 for (i=3, f=0; f<nfs; f++, i+=2) {
80 vec[i].iov_base = (void *)ofields[f].field;
81 vec[i].iov_len = strlen(vec[i].iov_base);
82 vec[i+1].iov_base = ",";
83 vec[i+1].iov_len = 1;
84 memcpy(&sql[f*3], "%s,", 3);
85 }
86 sql[nfs*3-1] = '\0';
87 vec[i-1].iov_base = ") VALUES (";
88 vec[i-1].iov_len = 10;
89 vec[i].iov_base = sql;
90 vec[i].iov_len = nfs*3-1;
91 vec[i+1].iov_base = ")";
92 vec[i+1].iov_len = 1;
93
94 sql = apr_pstrcatv(p, vec, i+2, NULL);
95
96 logging_log(cfg, LOGLEVEL_DEBUG, "DB: Generated SQL: %s", sql);
97
98 rv = apr_dbd_prepare(dbconn->driver, cfg->pool, dbconn->dbd, sql,
99 "INSERT", &stmt);
100
101 if (rv) {
102 logging_log(cfg, LOGLEVEL_NOISE,
103 "DB: Unable to Prepare SQL insert: %s", apr_dbd_error(
104 dbconn->driver, dbconn->dbd, rv));
105 return NULL;
106 }
107 return stmt;
108}
109
110apr_status_t database_insert(config_t *cfg, config_dbd_t *dbconn,
111 apr_pool_t *p, apr_table_t *data)
112{
113 apr_status_t rv;
114 int f, nfs;
115 config_output_field_t *ofields;
116 ofields = (config_output_field_t *)cfg->output_fields->elts;
117 nfs = cfg->output_fields->nelts;
118 // Prepare statement
119 if (!dbconn->stmt) {
120 dbconn->stmt = database_prepare_insert(cfg, dbconn, p);
121 if (!dbconn->stmt) {
122 return APR_EINVAL;
123 }
124 dbconn->args = apr_palloc(cfg->pool, nfs * sizeof(char *));
125 }
126 for (f=0; f<nfs; f++) {
127 dbconn->args[f] = apr_table_get(data, ofields[f].field);
128 }
129 rv = apr_dbd_pquery(dbconn->driver, p, dbconn->dbd, &f,
130 dbconn->stmt, nfs, dbconn->args);
131 if (rv) {
132 logging_log(cfg, LOGLEVEL_ERROR, "DB: Unable to Insert SQL: %s",
133 apr_dbd_error(dbconn->driver, dbconn->dbd, rv));
134 return rv;
135 }
136 return APR_SUCCESS;
137}
138
139apr_status_t database_trans_start(config_t *cfg, config_dbd_t *dbconn,
140 apr_pool_t *p)
141{
142#if HAVE_APR_DBD_TRANSACTION_MODE_GET
143 apr_status_t rv;
144 if (!cfg->transactions)
145 return APR_SUCCESS;
146 if (dbconn->txn) {
147 logging_log(cfg, LOGLEVEL_NOISE,
148 "Transaction Already Started. Something is BROKE");
149 return APR_EINVAL;
150 }
151 logging_log(cfg, LOGLEVEL_DEBUG, "DB: Starting Transaction");
152 rv = apr_dbd_transaction_start(dbconn->driver, p, dbconn->dbd,
153 &dbconn->txn);
154 if (rv)
155 logging_log(cfg, LOGLEVEL_NOISE,
156 "DB: Error Starting Transaction: (%d)%s", rv, apr_dbd_error(
157 dbconn->driver, dbconn->dbd, rv));
158 return rv;
159#else
160 return APR_SUCCESS;
161#endif
162}
163
164apr_status_t database_trans_stop(config_t *cfg, config_dbd_t *dbconn,
165 apr_pool_t *p)
166{
167#if HAVE_APR_DBD_TRANSACTION_MODE_GET
168 apr_status_t rv;
169 if (!cfg->transactions)
170 return APR_SUCCESS;
171 if (!dbconn->txn) {
172 logging_log(cfg, LOGLEVEL_NOISE,
173 "No Transaction Started. Something is BROKE");
174 return APR_EINVAL;
175 }
176 logging_log(cfg, LOGLEVEL_DEBUG, "DB: Stopping Transaction");
177 rv = apr_dbd_transaction_end(dbconn->driver, p, dbconn->txn);
178 if (rv)
179 logging_log(cfg, LOGLEVEL_NOISE,
180 "DB: Error Stopping Transaction: (%d)%s", rv, apr_dbd_error(
181 dbconn->driver, dbconn->dbd, rv));
182
183 dbconn->txn = NULL;
184 return rv;
185#else
186 return APR_SUCCESS;
187#endif
188}
189
190apr_status_t database_trans_abort(config_t *cfg, config_dbd_t *dbconn)
191{
192#if HAVE_APR_DBD_TRANSACTION_MODE_GET
193 apr_status_t rv;
194 if (!cfg->transactions)
195 return APR_SUCCESS;
196 if (!dbconn->txn) {
197 logging_log(cfg, LOGLEVEL_NOISE,
198 "No Transaction Started. Something is BROKE");
199 return APR_EINVAL;
200 }
201 logging_log(cfg, LOGLEVEL_NOTICE, "DB: Aborting Transaction");
202 rv = apr_dbd_transaction_mode_set(dbconn->driver, dbconn->txn,
203 APR_DBD_TRANSACTION_ROLLBACK);
204 if (rv)
205 logging_log(cfg, LOGLEVEL_NOISE,
206 "DB: Error Aborting Transaction: (%d)%s", rv, apr_dbd_error(
207 dbconn->driver, dbconn->dbd, rv));
208 return rv;
209#else
210 return APR_SUCCESS;
211#endif
212}
diff --git a/utility/database.h b/utility/database.h
new file mode 100644
index 0000000..9797d51
--- /dev/null
+++ b/utility/database.h
@@ -0,0 +1,25 @@
1#ifndef DATABASE_H_
2#define DATABASE_H_
3
4#include "apr_pools.h"
5
6#include "config.h"
7
8void database_init(apr_pool_t *p);
9
10apr_status_t database_connect(config_t *cfg, config_dbd_t **dbconn);
11
12apr_status_t database_disconnect(config_dbd_t *dbconn);
13
14apr_status_t database_insert(config_t *cfg, config_dbd_t *dbconn,
15 apr_pool_t *p, apr_table_t *data);
16
17apr_status_t database_trans_start(config_t *cfg, config_dbd_t *dbconn,
18 apr_pool_t *p);
19
20apr_status_t database_trans_stop(config_t *cfg, config_dbd_t *dbconn,
21 apr_pool_t *p);
22
23apr_status_t database_trans_abort(config_t *cfg, config_dbd_t *dbconn);
24
25#endif /*DATABASE_H_*/
diff --git a/utility/logparse.c b/utility/logparse.c
new file mode 100644
index 0000000..1b3cc97
--- /dev/null
+++ b/utility/logparse.c
@@ -0,0 +1,768 @@
1#include "logparse.h"
2#include "apr_file_info.h"
3#include "apr_file_io.h"
4#include "apr_strings.h"
5#include "apr_time.h"
6
7#include "time.h"
8#include "stdlib.h"
9
10#include "util.h"
11#include "ap_pcre.h"
12#include "database.h"
13
14apr_hash_t *g_parser_funcs;
15void **g_parser_linedata;
16
17static apr_status_t parser_func_wrap(apr_pool_t *p, config_t *cfg,
18 config_output_field_t *field, const char *value, const char **ret)
19{
20 if (field->args[0] && field->args[1]) {
21 struct iovec vec[3];
22 apr_size_t len;
23
24 vec[0].iov_base = (void *)field->args[0];
25 vec[0].iov_len = strlen(field->args[0]);
26 vec[1].iov_base = (void *)value;
27 vec[1].iov_len = strlen(value);
28 vec[2].iov_base = (void *)field->args[1];
29 vec[2].iov_len = strlen(field->args[1]);
30
31 *ret = apr_pstrcatv(p, vec, 3, &len);
32 } else {
33 logging_log(cfg, LOGLEVEL_NOISE, "wrap requires before and after strings");
34 return APR_EINVAL;
35 }
36 return APR_SUCCESS;
37}
38
39static apr_status_t parser_func_regexmatch(apr_pool_t *p, config_t *cfg,
40 config_output_field_t *field, const char *value, const char **ret)
41{
42 struct {
43 ap_regex_t *rx;
44 const char *substr;
45 }*_data;
46 ap_regmatch_t regm[AP_MAX_REG_MATCH];
47 // Check if a regular expression configured
48 if (!field->args[0])
49 return APR_EINVAL;
50 if (!field->data) {
51 // pre compile the regex
52 _data = apr_palloc(cfg->pool, sizeof(ap_regex_t)+sizeof(const char *));
53 _data->rx = ap_pregcomp(cfg->pool, field->args[0],
54 AP_REG_EXTENDED|AP_REG_ICASE);
55 if (field->args[1]) {
56 _data->substr = field->args[1];
57 } else {
58 _data->substr = "$1";
59 }
60 if (!_data->rx) {
61 logging_log(cfg, LOGLEVEL_NOISE, "Failed to compile regular expression");
62 return APR_EINVAL;
63 }
64 field->data = _data;
65 } else
66 _data = field->data;
67
68 if (!ap_regexec(_data->rx, value, AP_MAX_REG_MATCH, regm, 0)) {
69 *ret = ap_pregsub(p, _data->substr, value, AP_MAX_REG_MATCH, regm);
70 }
71 logging_log(cfg, LOGLEVEL_DEBUG, "REGEX: matched %s against %s to %s", value,
72 field->args[0], *ret);
73 return APR_SUCCESS;
74}
75
76static apr_status_t parser_func_totimestamp(apr_pool_t *p, config_t *cfg,
77 config_output_field_t *field, const char *value, const char **ret)
78{
79 time_t time;
80 struct tm ts;
81
82 //memset(&ts,0,sizeof(struct tm));
83
84 strptime(value, "%d/%b/%Y:%H:%M:%S %z", &ts);
85 time = mktime(&ts);
86
87 *ret = apr_itoa(p, time);
88 return APR_SUCCESS;
89}
90
91static apr_status_t parser_func_machineid(apr_pool_t *p, config_t *cfg,
92 config_output_field_t *field, const char *value, const char **ret)
93{
94 if (cfg->machineid) {
95 *ret = apr_pstrdup(p, cfg->machineid);
96 }
97 return APR_SUCCESS;
98}
99
100static apr_status_t parser_func_queryarg(apr_pool_t *p, config_t *cfg,
101 config_output_field_t *field, const char *value, const char **ret)
102{
103 apr_table_t *query = parser_get_linedata(field->func);
104
105 if (!field->args[0]) {
106 logging_log(cfg, LOGLEVEL_NOISE, "queryarg requires name of query arg");
107 return APR_EINVAL;
108 }
109
110 if (!query) {
111 char *query_beg;
112
113 query = apr_table_make(p,3);
114
115 query_beg = strchr(value, '?');
116 // if we have a query string, rip it apart
117 if (query_beg) {
118 char *key;
119 char *value;
120 const char *delim = "&";
121 char *query_string;
122 char *strtok_state;
123 char *query_end = strrchr(++query_beg,' ');
124
125 query_string = apr_pstrndup(p, query_beg, query_end-query_beg);
126 logging_log(cfg, LOGLEVEL_DEBUG, "QUERY: Found String %pp, %pp, %s",
127 query_beg, query_end, query_string);
128 if (field->args[1]) {
129 delim = field->args[1];
130 }
131 key = apr_strtok(query_string, delim, &strtok_state);
132 while (key) {
133 value = strchr(key, '=');
134 if (value) {
135 *value = '\0'; /* Split the string in two */
136 value++; /* Skip past the = */
137 }
138 else {
139 value = "1";
140 }
141 ap_unescape_url(key);
142 ap_unescape_url(value);
143 apr_table_set(query, key, value);
144
145 logging_log(cfg, LOGLEVEL_DEBUG,
146 "QUERY: Found arg: %s = %s", key, value);
147
148 key = apr_strtok(NULL, delim, &strtok_state);
149 }
150 }
151 parser_set_linedata(field->func,query);
152 }
153 *ret = apr_table_get(query, field->args[0]);
154 return APR_SUCCESS;
155}
156
157parser_func_t *parser_get_func(const char *name)
158{
159 return apr_hash_get(g_parser_funcs, name, APR_HASH_KEY_STRING);
160}
161
162static void parser_add_func(apr_pool_t *p, const char *const name,
163 parser_func_f func, int id)
164{
165 parser_func_t *s;
166 if (!g_parser_funcs) {
167 g_parser_funcs = apr_hash_make(p);
168 }
169 s = apr_palloc(p, sizeof(parser_func_t));
170 s->func = func;
171 s->pos = id;
172 s->data = NULL;
173 s->linedata = &g_parser_linedata;
174 apr_hash_set(g_parser_funcs, lowerstr(p, name), APR_HASH_KEY_STRING, s);
175}
176
177void parser_init(apr_pool_t *p)
178{
179 int i = 0;
180 parser_add_func(p, "regexmatch", parser_func_regexmatch, ++i);
181 parser_add_func(p, "totimestamp", parser_func_totimestamp, ++i);
182 parser_add_func(p, "machineid", parser_func_machineid, ++i);
183 parser_add_func(p, "queryarg", parser_func_queryarg, ++i);
184 parser_add_func(p, "wrap", parser_func_wrap, ++i);
185 g_parser_linedata = apr_pcalloc(p, sizeof(void *) * (i+1));
186 g_parser_linedata[0] = (void *)i;
187}
188
189void parser_find_logs(config_t *cfg)
190{
191 apr_pool_t *tp;
192 apr_dir_t *dir;
193 apr_finfo_t finfo;
194 config_filestat_t *newp;
195
196 logging_log(cfg, LOGLEVEL_NOTICE, "Find Log files");
197 if (!cfg->input_dir)
198 return;
199 apr_pool_create(&tp, cfg->pool);
200 if (apr_dir_open(&dir, cfg->input_dir, tp)==APR_SUCCESS) {
201 while (apr_dir_read(&finfo, APR_FINFO_NAME | APR_FINFO_TYPE, dir)
202 == APR_SUCCESS) {
203 char *temp;
204 if (finfo.filetype == APR_DIR)
205 continue;
206 newp = (config_filestat_t *)apr_array_push(cfg->input_files);
207 newp->result = "Not Parsed";
208 apr_filepath_merge(&temp, cfg->input_dir, finfo.name,
209 APR_FILEPATH_TRUENAME, cfg->pool);
210 newp->fname = temp;
211 }
212 apr_dir_close(dir);
213 }
214 apr_pool_destroy(tp);
215}
216
217#define BUFFER_SIZE (16 * 1024)
218
219void parser_split_logs(config_t *cfg)
220{
221 apr_pool_t *tp, *tfp;
222 apr_array_header_t *foundfiles;
223 config_filestat_t *filelist;
224 config_filestat_t *newfile;
225 apr_file_t *infile;
226 int f, l;
227 apr_status_t rv;
228 apr_finfo_t finfo;
229 char buff[BUFFER_SIZE];
230 int linecount;
231 int piecesize;
232
233 if (!cfg->split_enabled) return;
234 if (!cfg->split_dir) {
235 logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Missing Split Output directory");
236 return;
237 }
238 apr_pool_create(&tp, cfg->pool);
239 apr_pool_create(&tfp, tp);
240
241 if (APR_SUCCESS != apr_stat(&finfo, cfg->split_dir, APR_FINFO_MIN, tp)) {
242 logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Directory %s does not exist", cfg->split_dir);
243 return;
244 }
245 foundfiles = apr_array_copy(tp, cfg->input_files);
246 apr_array_clear(cfg->input_files);
247
248 filelist = (config_filestat_t *)foundfiles->elts;
249 for (f=0, l=foundfiles->nelts; f < l; f++) {
250 apr_pool_clear(tfp);
251 logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Begin Splitting Log File '%s'", filelist[f].fname);
252 rv = apr_file_open(&infile, filelist[f].fname, APR_FOPEN_READ, APR_OS_DEFAULT, tfp);
253
254 if (rv != APR_SUCCESS) {
255 logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s", filelist[f].fname);
256 return;
257 }
258 linecount = 0;
259 while (apr_file_eof(infile) == APR_SUCCESS) {
260 apr_size_t read = BUFFER_SIZE;
261 char *p;
262 apr_file_read(infile, buff, &read);
263 p = buff;
264 while ((p = memchr(p, '\n', (buff + read) - p))) {
265 ++p;
266 ++linecount;
267 }
268 }
269 // now we know how long it is. Lets split up the file
270 piecesize = linecount / cfg->split_count;
271 if (piecesize < cfg->split_minimum)
272 piecesize = cfg->split_minimum;
273 if (piecesize > cfg->split_maximum && cfg->split_maximum > 0)
274 piecesize = cfg->split_maximum;
275 if (piecesize > linecount) {
276 // File is smaller than piece size just add it back in as is
277 newfile = (config_filestat_t *)apr_array_push(cfg->input_files);
278 newfile->result = "Not Parsed";
279 newfile->fname = filelist[f].fname;
280 } else {
281 //split apart the files
282 int cur_line = 0;
283 int file_count = 1;
284 int out_lines = 0;
285 const char *basefile, *file;
286 apr_file_t *outfile;
287 char trail[2048];
288 apr_size_t trail_size = 0;
289 apr_size_t write;
290 apr_off_t off = 0;
291
292 apr_file_seek(infile, APR_SET, &off);
293
294 basefile = apr_pstrdup(tfp, basename(apr_pstrdup(tfp, filelist[f].fname)));
295
296 file = apr_psprintf(tfp, "%s/%s-%d", cfg->split_dir, basefile, file_count++);
297 logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Creating output file %s", file);
298 rv = apr_file_open(&outfile, file, APR_FOPEN_WRITE | APR_FOPEN_CREATE | APR_FOPEN_TRUNCATE, APR_OS_DEFAULT, tfp);
299 if (rv != APR_SUCCESS) {
300 logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s (%d)", file, rv);
301 return;
302 }
303 newfile = (config_filestat_t *)apr_array_push(cfg->input_files);
304 newfile->result = "Not Parsed";
305 newfile->fname = apr_pstrdup(cfg->pool, file);
306
307 while (apr_file_eof(infile) == APR_SUCCESS) {
308 apr_size_t read = BUFFER_SIZE;
309 char *p, *pp, *buff_start;
310 apr_file_read(infile, buff, &read);
311 buff_start = p = pp = buff;
312 if (trail_size) {
313 p = memchr(p, '\n', (buff + read) - p);
314 if (p) {
315 //printf("Trail Line: %p, %p, %d\n", pp, p, (p - pp) + trail_size);
316 ++p;
317 pp = p;
318 ++cur_line;
319 ++out_lines;
320 // write out to file
321 apr_file_write(outfile, trail, &trail_size);
322 trail_size = 0;
323 } else {
324 if ((read + trail_size) > 2048) {
325 logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Excessively long line %d in file %s", cur_line, filelist[f].fname);
326 exit(1);
327 } else {
328 memcpy(trail+trail_size, buff, read);
329 trail_size += read;
330 }
331 }
332 }
333 while ((p = memchr(p, '\n', (buff + read) - p))) {
334 //printf("Line: %p, %p, %d\n", pp, p, (p - pp));
335 if (out_lines == piecesize) {
336 // Write out to file
337 write = pp - buff_start;
338 apr_file_write(outfile, buff_start, &write);
339 buff_start = pp;
340 out_lines = 0;
341 // Open new file
342 file = apr_psprintf(tfp, "%s/%s-%d", cfg->split_dir, basefile, file_count++);
343 logging_log(cfg, LOGLEVEL_NOTICE, "SPLITTER: Creating output file %s", file);
344 rv = apr_file_open(&outfile, file, APR_FOPEN_WRITE | APR_FOPEN_CREATE | APR_FOPEN_TRUNCATE, APR_OS_DEFAULT, tfp);
345 if (rv != APR_SUCCESS) {
346 logging_log(cfg, LOGLEVEL_NOISE, "SPLITTER: Could not open %s (%d)", file, rv);
347 return;
348 }
349 newfile = (config_filestat_t *)apr_array_push(cfg->input_files);
350 newfile->result = "Not Parsed";
351 newfile->fname = apr_pstrdup(cfg->pool, file);
352 }
353 ++p;
354 pp = p;
355 ++cur_line;
356 ++out_lines;
357 }
358 // Write out to file
359 write = pp - buff_start;
360 apr_file_write(outfile, buff_start, &write);
361
362 trail_size = (buff+read) - pp;
363 if (trail_size) {
364 memcpy(trail, pp, trail_size);
365 }
366 }
367 }
368 }
369 apr_pool_destroy(tfp);
370 apr_pool_destroy(tp);
371}
372
373apr_status_t parser_logbadline(config_t *cfg, const char *filename,
374 const char *badline)
375{
376 apr_status_t rv = APR_SUCCESS;
377 apr_size_t len;
378 struct iovec vec[5];
379
380 if (cfg->badlinefile) {
381 if (!cfg->badline_fp) {
382 rv = apr_file_open(&cfg->badline_fp, cfg->badlinefile,
383 APR_FOPEN_CREATE | APR_FOPEN_WRITE | APR_FOPEN_APPEND,
384 APR_OS_DEFAULT, cfg->pool);
385 if (rv) {
386 logging_log(cfg, LOGLEVEL_NOISE,
387 "Error opening badline file %s\n", cfg->badlinefile);
388 cfg->badlinefile = NULL;
389 }
390 }
391 if (!rv) {
392 if (filename != cfg->badlastfile){
393 char date[APR_RFC822_DATE_LEN];
394 vec[0].iov_base = "Starting BadLines for \"";
395 vec[0].iov_len = sizeof("Starting BadLines for \"")-1;
396 vec[1].iov_base = (void *)filename;
397 vec[1].iov_len = strlen(filename);
398 vec[2].iov_base = "\" on ";
399 vec[2].iov_len = sizeof("\" on ")-1;
400 apr_rfc822_date(date, apr_time_now());
401 vec[3].iov_base = date;
402 vec[3].iov_len = APR_RFC822_DATE_LEN-1;
403 vec[4].iov_base = "\n";
404 vec[4].iov_len = 1;
405 apr_file_writev(cfg->badline_fp, vec,5, &len);
406 cfg->badlastfile = filename;
407 }
408
409 if ((++cfg->badline_count) > cfg->badlinemax) {
410 logging_log(cfg, LOGLEVEL_NOISE,
411 "Found more than %d bad lines (found %d)",
412 cfg->badlinemax, cfg->badline_count);
413 rv = APR_EINVAL;
414 } else {
415 vec[0].iov_base = (void *)badline;
416 vec[0].iov_len = strlen(badline);
417 vec[1].iov_base = "\n";
418 vec[1].iov_len = 1;
419 apr_file_writev(cfg->badline_fp, vec,2, &len);
420 }
421 }
422 }
423 return rv;
424}
425
426/*
427 * Modified version of apr_tokenize_to_argv to add [] as quoting characters
428 *
429 * token_context: Context from which pool allocations will occur.
430 * arg_str: Input string for conversion to argv[].
431 * argv_out: Output location. This is a pointer to an array
432 * of pointers to strings (ie. &(char *argv[]).
433 * This value will be allocated from the contexts
434 * pool and filled in with copies of the tokens
435 * found during parsing of the arg_str.
436 */
437apr_status_t parser_tokenize_line(const char *arg_str, char ***argv_out,
438 apr_pool_t *token_context)
439{
440 const char *cp;
441 const char *ct;
442 char *cleaned, *dirty;
443 int escaped;
444 int isquoted, numargs = 0, argnum;
445
446#define SKIP_WHITESPACE(cp) \
447 for ( ; *cp == ' ' || *cp == '\t'; ) { \
448 cp++; \
449 };
450
451#define CHECK_QUOTATION(cp,isquoted) \
452 isquoted = 0; \
453 if (*cp == '"') { \
454 isquoted = 1; \
455 cp++; \
456 } \
457 else if (*cp == '\'') { \
458 isquoted = 2; \
459 cp++; \
460 } \
461 else if (*cp == '[') { \
462 isquoted = 3; \
463 cp++; \
464 }
465
466 /* DETERMINE_NEXTSTRING:
467 * At exit, cp will point to one of the following: NULL, SPACE, TAB or QUOTE.
468 * NULL implies the argument string has been fully traversed.
469 */
470#define DETERMINE_NEXTSTRING(cp,isquoted) \
471 for ( ; *cp != '\0'; cp++) { \
472 if ( (*cp == '\\' && (*(cp+1) == ' ' || *(cp+1) == '\t' || \
473 *(cp+1) == '"' || *(cp+1) == '\'' || \
474 *(cp+1) == '[' || *(cp+1) == ']'))) { \
475 cp++; \
476 continue; \
477 } \
478 if ( (!isquoted && (*cp == ' ' || *cp == '\t')) \
479 || (isquoted == 1 && *cp == '"') \
480 || (isquoted == 2 && *cp == '\'') \
481 || (isquoted == 3 && *cp == ']') \
482 ) { \
483 break; \
484 } \
485 }
486
487 /* REMOVE_ESCAPE_CHARS:
488 * Compresses the arg string to remove all of the '\' escape chars.
489 * The final argv strings should not have any extra escape chars in it.
490 */
491#define REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped) \
492 escaped = 0; \
493 while(*dirty) { \
494 if (!escaped && *dirty == '\\') { \
495 escaped = 1; \
496 } \
497 else { \
498 escaped = 0; \
499 *cleaned++ = *dirty; \
500 } \
501 ++dirty; \
502 } \
503 *cleaned = 0; /* last line of macro... */
504
505 cp = arg_str;
506 SKIP_WHITESPACE(cp);
507 ct = cp;
508
509 /* This is ugly and expensive, but if anyone wants to figure a
510 * way to support any number of args without counting and
511 * allocating, please go ahead and change the code.
512 *
513 * Must account for the trailing NULL arg.
514 */
515 numargs = 1;
516 while (*ct != '\0') {
517 CHECK_QUOTATION(ct, isquoted)
518 ;
519 DETERMINE_NEXTSTRING(ct, isquoted);
520 if (*ct != '\0') {
521 ct++;
522 }
523 numargs++;
524 SKIP_WHITESPACE(ct);
525 }
526 *argv_out = apr_palloc(token_context, numargs * sizeof(char*));
527
528 /* determine first argument */
529 for (argnum = 0; argnum < (numargs-1); argnum++) {
530 SKIP_WHITESPACE(cp);
531 CHECK_QUOTATION(cp, isquoted)
532 ;
533 ct = cp;
534 DETERMINE_NEXTSTRING(cp, isquoted);
535 cp++;
536 (*argv_out)[argnum] = apr_palloc(token_context, cp - ct);
537 apr_cpystrn((*argv_out)[argnum], ct, cp - ct);
538 cleaned = dirty = (*argv_out)[argnum];
539 REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped)
540 ;
541 }
542 (*argv_out)[argnum] = NULL;
543
544 return APR_SUCCESS;
545}
546
547apr_status_t parser_parsefile(config_t *cfg, config_dbd_t *dbconn,
548 config_filestat_t *fstat)
549{
550 apr_pool_t *tp, *targp;
551 apr_file_t *file;
552 apr_status_t rv;
553 char buff[2048];
554 char readbuff[BUFFER_SIZE];
555 char **targv;
556 int targc;
557
558 apr_pool_create(&tp, cfg->pool);
559 apr_pool_create(&targp, tp);
560
561 logging_log(cfg, LOGLEVEL_NOTICE, "PARSER: Begin Parsing Log File '%s'", fstat->fname);
562
563 rv = apr_file_open(&file, fstat->fname, APR_FOPEN_READ, APR_OS_DEFAULT, tp);
564 apr_file_buffer_set(file, readbuff, BUFFER_SIZE);
565 if (rv != APR_SUCCESS) {
566 logging_log(cfg, LOGLEVEL_NOISE, "PARSER: Could not open %s", fstat->fname);
567 return rv;
568 }
569
570 fstat->linesparsed = 0;
571 // Start Transaction
572 fstat->start = apr_time_now();
573 if (!cfg->dryrun && database_trans_start(cfg, dbconn, tp)) {
574 fstat->result = "Database Transaction Error";
575 fstat->stop = apr_time_now();
576 return rv;
577 }
578
579 do {
580 rv = apr_file_gets(buff, 2048, file);
581 if (rv == APR_SUCCESS) {
582 int i,m, cont = 0;
583 config_filter_t *filters;
584
585 fstat->linesparsed++;
586 // chomp off newline
587 line_chomp(buff);
588 // Run line filters
589 for (i=0, m=cfg->linefilters->nelts,
590 filters = (config_filter_t *)cfg->linefilters->elts;
591 i<m; i++) {
592 if (!filters[i].regex || ap_regexec(filters[i].regex, buff, 0, NULL,0)==0) {
593 if (filters[i].negative) {
594 logging_log(cfg, LOGLEVEL_DEBUG,
595 "PARSER: LINEFILTER: Skipping Line %d due to Filter (%d)%s",
596 fstat->linesparsed, i, filters[i].filter);
597 fstat->lineskipped++;
598 cont = 1;
599 } else {
600 logging_log(cfg, LOGLEVEL_DEBUG,
601 "PARSER: LINEFILTER: Force Parsing Line %d due to Filter (%d)%s",
602 fstat->linesparsed, i, filters[i].filter);
603 }
604 break;
605 }
606 }
607 if (cont) continue;
608
609 apr_pool_clear(targp);
610 parser_tokenize_line(buff, &targv, targp);
611 targc = 0;
612 while (targv[targc])
613 targc++;
614 rv = parser_processline(targp, cfg, dbconn, fstat, targv, targc);
615 if (rv != APR_SUCCESS) {
616 int i;
617
618 fstat->linesbad++;
619 rv = parser_logbadline(cfg, fstat->fname, buff);
620 if (rv) {
621 if (!cfg->dryrun) database_trans_abort(cfg, dbconn);
622 logging_log(cfg, LOGLEVEL_ERROR, "Line %d(%d): %s", fstat->linesparsed,
623 targc, buff);
624 for (i = 0; targv[i]; i++) {
625 logging_log(cfg, LOGLEVEL_ERROR, "Arg (%d): '%s'", i,
626 targv[i]);
627 }
628 }
629 }
630 } else {
631 rv = APR_SUCCESS;
632 break;
633 }
634 } while (rv == APR_SUCCESS);
635 apr_file_close(file);
636 // Finish Transaction
637 if (!cfg->dryrun && database_trans_stop(cfg, dbconn, tp)) {
638 fstat->result = apr_psprintf(cfg->pool,
639 "Input line %d, Database Transaction Error",
640 fstat->linesparsed);
641 }
642
643 apr_pool_destroy(tp);
644 logging_log(cfg, LOGLEVEL_NOTICE,
645 "PARSER: Finish Parsing Log File '%s'. Lines: (%d/%d)",
646 fstat->fname, fstat->linesparsed - fstat->lineskipped, fstat->linesparsed);
647 if (!rv) {
648 fstat->result = "File Parsed Succesfully";
649 }
650 fstat->stop = apr_time_now();
651 return rv;
652}
653
654apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg,
655 config_dbd_t *dbconn, config_filestat_t *fstat, char **argv, int argc)
656{
657 config_logformat_t *fmt;
658 config_logformat_field_t *ifields;
659 config_output_field_t *ofields;
660 config_filter_t *filters;
661 apr_table_t *datain;
662 apr_table_t *dataout;
663 apr_status_t rv= APR_SUCCESS;
664 int i,m;
665
666 fmt = apr_hash_get(cfg->log_formats, cfg->logformat, APR_HASH_KEY_STRING);
667 if (!fmt) {
668 logging_log(cfg, LOGLEVEL_NOISE, "PARSER: No Input Log format");
669 return APR_EINVAL;
670 }
671 if (fmt->fields->nelts != argc) {
672 logging_log(cfg, LOGLEVEL_NOISE,
673 "PARSER: Input line field number differs from expected. Expected %d got %d.",
674 fmt->fields->nelts, argc);
675 fstat->result = apr_psprintf(cfg->pool,
676 "Input line %d is badly formatted (wrong number of fields)",
677 fstat->linesparsed);
678 return APR_EINVAL;
679 }
680
681 datain = apr_table_make(ptemp, fmt->fields->nelts);
682 dataout = apr_table_make(ptemp, cfg->output_fields->nelts);
683
684 ifields = (config_logformat_field_t *)fmt->fields->elts;
685 for (i=0; i<fmt->fields->nelts; i++) {
686 apr_table_setn(datain, ifields[i].name, argv[i]);
687 }
688 // Run Pre Filters
689 for (i=0, m=cfg->prefilters->nelts,
690 filters = (config_filter_t *)cfg->prefilters->elts;
691 i<m; i++) {
692 const char *temp = apr_table_get(datain, filters[i].field);
693 if (temp && (!filters[i].regex || ap_regexec(filters[i].regex, temp, 0, NULL,0)==0)) {
694 if (filters[i].negative) {
695 logging_log(cfg, LOGLEVEL_DEBUG,
696 "PARSER: PREFILTER: Skipping Line %d due to Filter (%d)%s",
697 fstat->linesparsed, i, filters[i].filter);
698 fstat->lineskipped++;
699 return APR_SUCCESS;
700 } else {
701 logging_log(cfg, LOGLEVEL_DEBUG,
702 "PARSER: PREFILTER: Force Parsing Line %d due to Filter (%d)%s",
703 fstat->linesparsed, i, filters[i].filter);
704 }
705 break;
706 }
707 }
708
709 ofields = (config_output_field_t *)cfg->output_fields->elts;
710 // clear out ofield function per-line data
711 memset(&g_parser_linedata[1],0,sizeof(void *)*(int)g_parser_linedata[0]);
712 // Convert input fields to output fields
713 for (i=0,m=cfg->output_fields->nelts; i<m; i++) {
714 const char *val;
715 val = apr_table_get(datain, ofields[i].source);
716 // If we can't find the source field just continue
717 if (!val && !(ofields[i].source[0]=='\0' && ofields[i].func)) {
718 apr_table_setn(dataout, ofields[i].field, ofields[i].def);
719 continue;
720 }
721 if (!ofields[i].func) {
722 apr_table_setn(dataout, ofields[i].field, val);
723 } else {
724 const char *ret= NULL;
725 rv = ((parser_func_t *)ofields[i].func)->func(ptemp, cfg,
726 &ofields[i], val, &ret);
727 if (rv) {
728 fstat->result = apr_psprintf(cfg->pool,
729 "Input line %d, Parser function %s returned error (%d)%s",
730 fstat->linesparsed, ofields[i].fname, rv, logging_strerror(rv));
731 return rv;
732 }
733 apr_table_setn(dataout, ofields[i].field, ret ? ret : ofields[i].def);
734 }
735 }
736
737 // Run Post filters
738 for (i=0, m=cfg->postfilters->nelts,
739 filters = (config_filter_t *)cfg->postfilters->elts;
740 i<m; i++) {
741 const char *temp = apr_table_get(dataout, filters[i].field);
742 if (temp && (!filters[i].regex || ap_regexec(filters[i].regex, temp, 0, NULL,0)==0)) {
743 if (filters[i].negative) {
744 logging_log(cfg, LOGLEVEL_DEBUG,
745 "PARSER: POSTFILTER: Skipping Line %d due to Filter (%d)%s",
746 fstat->linesparsed, i, filters[i].filter);
747 fstat->lineskipped++;
748 return APR_SUCCESS;
749 } else {
750 logging_log(cfg, LOGLEVEL_DEBUG,
751 "PARSER: POSTFILTER: Force Parsing Line %d due to Filter (%d)%s",
752 fstat->linesparsed, i, filters[i].filter);
753 }
754 break;
755 }
756 }
757
758 // Process DB Query
759 if (!cfg->dryrun) {
760 rv = database_insert(cfg, dbconn, ptemp, dataout);
761 if (rv) {
762 fstat->result = apr_psprintf(cfg->pool,
763 "Input line %d, Database Error",
764 fstat->linesparsed);
765 }
766 }
767 return rv;
768}
diff --git a/utility/logparse.h b/utility/logparse.h
new file mode 100644
index 0000000..7ca0958
--- /dev/null
+++ b/utility/logparse.h
@@ -0,0 +1,37 @@
1#ifndef LOGPARSE_H_
2#define LOGPARSE_H_
3
4#include "config.h"
5
6typedef apr_status_t (*parser_func_f)(apr_pool_t *p, config_t *cfg,
7 config_output_field_t *field, const char *value, const char **ret);
8
9struct parser_func_t {
10 parser_func_f func;
11 int pos;
12 void *data;
13 void ***linedata;
14};
15
16#define parser_get_linedata(f) (*f->linedata)[f->pos]
17
18#define parser_set_linedata(f, v) (*f->linedata)[f->pos] = v
19
20parser_func_t *parser_get_func(const char *name);
21
22void parser_init(apr_pool_t *p);
23
24void parser_find_logs(config_t *cfg);
25
26void parser_split_logs(config_t *cfg);
27
28apr_status_t parser_tokenize_line(const char *arg_str, char ***argv_out,
29 apr_pool_t *token_context);
30
31apr_status_t parser_parsefile(config_t *cfg, config_dbd_t *dbconn,
32 config_filestat_t *fstat);
33
34apr_status_t parser_processline(apr_pool_t *ptemp, config_t *cfg,
35 config_dbd_t *dbconn, config_filestat_t *line, char **argv, int argc);
36
37#endif /*LOGPARSE_H_*/
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
new file mode 100644
index 0000000..911fc18
--- /dev/null
+++ b/utility/mod_log_sql.conf
@@ -0,0 +1,70 @@
1InputDirectory ./logs
2ErrorLog ./error_log
3
4DBDDriver mysql
5DBDParams "host=localhost;user=root;dbname=apache_log"
6Table access_log
7MachineID 7of9
8UseTransactions on
9LogLevel notice
10DryRun off
11Summary on
12
13SplitInput on
14#SplitCount 4
15SplitMinLines 0
16SplitMaxLines 50000
17SplitDirectory ./split_temp
18
19#ThreadCount 0
20
21BadLineFile ./badlines.log
22BadLineMax 10
23
24LogFormatConfig CLF remhost String
25LogFormatConfig CLF ident String
26LogFormatConfig CLF user String
27LogFormatConfig CLF date Date
28LogFormatConfig CLF request String
29LogFormatConfig CLF status Number
30LogFormatConfig CLF bytes_sent Number
31
32LogFormatConfig Combined remhost String
33LogFormatConfig Combined ident String
34LogFormatConfig Combined user String
35LogFormatConfig Combined date Date
36LogFormatConfig Combined request String
37LogFormatConfig Combined status Number
38LogFormatConfig Combined bytes_sent Number
39LogFormatConfig Combined referer String
40LogFormatConfig Combined agent String
41
42LogFormat Combined
43
44#Linefilter - "BAD"
45#LineFilter "GOOD"
46#LineFilter + "BETTER"
47# the next filter ignores ALL lines
48#LineFilter -
49#PreFilter request - "Rebuild"
50#PostFilter request_method "GET"
51
52# Usage field datatype(size) default source [function [param]...]
53OutputField bytes_sent int 0 bytes_sent
54OutputField request_protocol varchar(10) "" request regexmatch "(HTTP/[\\d\\.]+)$"
55OutputField remote_host varchar(50) "" remhost
56OutputField request_method varchar(25) "" request regexmatch "^(\\w+)"
57OutputField time_stamp int 0 date totimestamp
58OutputField status smallint 0 status
59OutputField request_line varchar(255) "" request
60#OutputField request_uri varchar(255) "" request regexmatch "^\\w+ (.+) \\w+/[\\d\\.]+$"
61OutputField remote_user varchar(50) "" user
62OutputField remote_logname varchar(50) "" ident
63OutputField request_time char(28) "" date wrap "[" "]"
64#Only used for Combined log input, if standard CLF input, they are ignored
65OutputField agent varchar(255) "" agent
66OutputField referer varchar(255) "" referer
67OutputField machine_id varchar(25) "" "" machineid
68#VIZU Fields
69OutputField poll_id int 0 request queryarg n "&;"
70
diff --git a/utility/shell.c b/utility/shell.c
new file mode 100644
index 0000000..1b9e890
--- /dev/null
+++ b/utility/shell.c
@@ -0,0 +1,342 @@
1#include "apr.h"
2#include "apr_getopt.h"
3#include "apr_tables.h"
4
5#define APR_WANT_STDIO
6#include "apr_want.h"
7#include "stdlib.h"
8
9#include "shell.h"
10#include "config.h"
11#include "logparse.h"
12#include "database.h"
13#include "util.h"
14
15#if APR_HAS_THREADS
16#include "apr_queue.h"
17#include "apr_thread_pool.h"
18
19static apr_queue_t *queue;
20
21void run_multithreaded(config_t *cfg);
22#endif
23
24void run_singlethreaded(config_t *cfg);
25
26const apr_getopt_option_t _opt_config[] = {
27 {"machineid", 'm', 1, "Machine ID for the log file"},
28 {"transaction", 't', 1, "Use a Transaction (yes,no)"},
29 {"logformat", 'r', 1, "Use this logformat to parse files"},
30 {"file", 'f', 1, "Parse this single log file (input dir is NOT scanned)"},
31 {"inputdir", 'i', 1, "Input Directory to look for log files"},
32 {"config", 'c', 1, "Configuration file to use (default mod_log_sql.conf)"},
33 {"dryrun", 'n', 0, "Perform a dry run (do not actually alter the databse)"},
34 {"dump", 'd', 0, "Dump the configuration after parsing and quit"},
35 {"loglevel", 'l', 1, "Log Level (deubg, notice, error)"},
36 {"summary", 's', 1, "Summary (yes,no)"},
37 {"threadcount", 'p', 1, "Set thread count (a number greater than 0)"},
38 {"help", 'h', 0, "Show Help"},
39 {NULL}
40};
41
42void show_help(const char *prog, const apr_getopt_option_t *opts, FILE *output)
43{
44 int ptr = 0;
45 fprintf(output, "Usage: %s [OPTIONS] [files...]\n\n", prog);
46 while (opts[ptr].optch != 0) {
47 if (opts[ptr].optch > 255) {
48 if (opts[ptr].name) {
49 fprintf(output, " --%-10s", opts[ptr].name);
50 } else {
51 fprintf(output, " ");
52 }
53 } else {
54 if (opts[ptr].name) {
55 fprintf(output, " -%c --%-10s", opts[ptr].optch, opts[ptr].name);
56 } else {
57 fprintf(output, " -%c ", opts[ptr].optch);
58 }
59 }
60 if (opts[ptr].has_arg) {
61 fprintf(output, " (arg)");
62 } else {
63 fprintf(output, " ");
64 }
65 fprintf(output, " %s\n", opts[ptr].description);
66 ptr++;
67 }
68}
69
70void print_summary(config_t *cfg) {
71 config_filestat_t *fstat;
72 int i,m;
73 apr_time_t totaltime = 0;
74 apr_size_t totalparsed = 0, totalskipped = 0, totalbad = 0;
75
76 fstat = (config_filestat_t *)cfg->input_files->elts;
77
78 printf("Execution Summary\nParsed %d files\n", cfg->input_files->nelts);
79 for (i=0, m=cfg->input_files->nelts; i<m; i++) {
80 totaltime += fstat[i].stop - fstat[i].start;
81 totalparsed += fstat[i].linesparsed;
82 totalskipped += fstat[i].lineskipped;
83 totalbad += fstat[i].linesbad;
84 printf(" File: %s\n"
85 " Lines Added %'d out of %'d (Skipped %'d, Bad %'d)\n"
86 " Status: %s\n"
87 " Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n"
88 "\n",
89 fstat[i].fname,
90 fstat[i].linesparsed - fstat[i].lineskipped - fstat[i].linesbad,
91 fstat[i].linesparsed,
92 fstat[i].lineskipped,
93 fstat[i].linesbad,
94 fstat[i].result,
95 apr_time_sec(fstat[i].stop - fstat[i].start)/60,
96 apr_time_sec(fstat[i].stop - fstat[i].start) % 60,
97 apr_time_msec(fstat[i].stop - fstat[i].start)
98 );
99 }
100 printf("Totals\n"
101 " Lines Added %'d out of %'d (Skipped %'d, Bad %'d)\n"
102 " Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n"
103 "\n",
104 totalparsed - totalskipped - totalbad,
105 totalparsed,
106 totalskipped,
107 totalbad,
108 apr_time_sec(totaltime)/60,
109 apr_time_sec(totaltime) % 60,
110 apr_time_msec(totaltime)
111 );
112}
113
114int main(int argc, const char *const argv[])
115{
116 apr_pool_t *pool, *ptemp;
117 apr_getopt_t *opts;
118 int opt;
119 const char *opt_arg;
120 apr_status_t rv;
121 apr_table_t *args;
122 config_t *cfg;
123
124 apr_app_initialize(&argc, &argv, NULL);
125 atexit(apr_terminate);
126
127 if (apr_pool_create(&pool, NULL) != APR_SUCCESS) {
128 fprintf(stderr, "Failed to create memory pool!\n");
129 exit(1);
130 }
131 apr_pool_create(&ptemp, NULL);
132
133 /** Iterate over command line arguments
134 * shoving args in a apr_table for processing later*/
135 args = apr_table_make(ptemp, 5);
136 apr_table_setn(args, "config", "mod_log_sql.conf");
137 apr_getopt_init(&opts, ptemp, argc, argv);
138 while ((rv = apr_getopt_long(opts, _opt_config, &opt, &opt_arg)) == APR_SUCCESS) {
139 switch (opt) {
140 case 'c':
141 apr_table_setn(args,"config",opt_arg);
142 break;
143 case 'd':
144 apr_table_setn(args,"dump","yes");
145 break;
146 case 'f':
147 apr_table_setn(args,"inputfile",opt_arg);
148 break;
149 case 'h':
150 show_help(argv[0], _opt_config, stdout);
151 exit(1);
152 break;
153 case 'i':
154 apr_table_setn(args,"inputdirectory",opt_arg);
155 break;
156 case 'l':
157 apr_table_setn(args,"loglevel",opt_arg);
158 break;
159 case 'm':
160 apr_table_setn(args,"machineid",opt_arg);
161 break;
162 case 'n':
163 apr_table_setn(args,"dryrun","yes");
164 break;
165 case 'p':
166 apr_table_setn(args,"threadcount",opt_arg);
167 break;
168 case 'r':
169 apr_table_setn(args,"logformat",opt_arg);
170 break;
171 case 's':
172 apr_table_setn(args,"summary",opt_arg);
173 break;
174 case 't':
175 apr_table_setn(args,"usetransactions",opt_arg);
176 break;
177 }
178 }
179 if (rv != APR_EOF) {
180 show_help(argv[0], _opt_config, stderr);
181 exit(1);
182 }
183 // Check if no extra args were passed
184 if (opts->ind != opts->argc) {
185 show_help(argv[0], _opt_config, stderr);
186 fprintf(stderr, "\n%s: Extra unknown arguments passed\n\n",argv[0]);
187 exit(1);
188 }
189
190 // Initialize sub systems
191 parser_init(pool);
192 config_init(pool);
193 database_init(pool);
194 // Process configuration file
195 cfg = config_create(pool);
196 // initialize STD out error log
197 logging_preinit(cfg);
198 rv = config_read(cfg, apr_table_get(args,"Config"), args);
199 apr_pool_destroy(ptemp);
200
201 // Initialize Log system AFTER we parse the configuration
202 logging_init(cfg);
203
204 if (APR_STATUS_IS_ENOENT(rv)) {
205 logging_log(cfg,LOGLEVEL_NOISE,"Could not load configuration file: %s",apr_table_get(args,"config"));
206 } else if (rv) {
207 exit(1);
208 }
209 if (cfg->dump) {
210 config_dump(cfg);
211 exit(0);
212 }
213
214 if (config_check(cfg)) {
215 logging_log(cfg,LOGLEVEL_NOISE, "Please correct the configuration");
216 exit(1);
217 }
218
219 // Only Find files IF no filename was passed via the command line
220 if (apr_is_empty_array(cfg->input_files)) {
221 parser_find_logs(cfg);
222 }
223 if (!apr_is_empty_array(cfg->input_files)) {
224 parser_split_logs(cfg);
225#if APR_HAS_THREADS
226 if (cfg->thread_count > 0) {
227 run_multithreaded(cfg);
228 } else {
229#endif
230 run_singlethreaded(cfg);
231#if APR_HAS_THREADS
232 }
233#endif
234 } else {
235 logging_log(cfg,LOGLEVEL_NOISE,"No log files found to parse");
236 }
237
238 if (cfg->summary) {
239 print_summary(cfg);
240 }
241 return 0;
242}
243
244void run_singlethreaded(config_t *cfg)
245{
246 config_filestat_t *filelist;
247 config_dbd_t *dbconn = NULL;
248 int f, l;
249 apr_status_t rv;
250
251 if (!cfg->dryrun) {
252 if ((rv = database_connect(cfg, &dbconn))) {
253 logging_log(cfg,LOGLEVEL_NOISE, "Error Connecting to Database");
254 exit(1);
255 }
256 }
257
258 filelist = (config_filestat_t *)cfg->input_files->elts;
259 for (f=0, l=cfg->input_files->nelts; f < l; f++) {
260 rv = parser_parsefile(cfg, dbconn, &filelist[f]);
261 if (rv) {
262 logging_log(cfg, LOGLEVEL_NOISE,
263 "Error occured parsing log files. Aborting");
264 break;
265 }
266 }
267
268 if (!cfg->dryrun) {
269 database_disconnect(dbconn);
270 }
271}
272
273#if APR_HAS_THREADS
274void * APR_THREAD_FUNC run_filethread(apr_thread_t *thd, void *data)
275{
276 config_t *cfg = data;
277 config_dbd_t *dbconn = NULL;
278 config_filestat_t *fileentry;
279 apr_status_t rv;
280
281 if (!cfg->dryrun) {
282 if ((rv = database_connect(cfg, &dbconn))) {
283 logging_log(cfg,LOGLEVEL_NOISE, "Error Connecting to Database");
284 return NULL;
285 }
286 }
287
288 while (1) {
289 rv = apr_queue_pop(queue, (void **)&fileentry);
290 if (rv == APR_EINTR)
291 continue;
292 if (rv == APR_EOF)
293 break;
294 rv = parser_parsefile(cfg, dbconn, fileentry);
295 if (rv) {
296 logging_log(cfg, LOGLEVEL_NOISE,
297 "Error occured parsing log file %s", fileentry->fname);
298 }
299 }
300
301 if (!cfg->dryrun) {
302 database_disconnect(dbconn);
303 }
304 return NULL;
305}
306
307void run_multithreaded(config_t *cfg)
308{
309 logging_log(cfg, LOGLEVEL_NOISE, "Running Multithreaded");
310
311 config_filestat_t *filelist;
312 int f, l;
313 apr_status_t rv;
314 apr_pool_t *tp;
315 apr_thread_pool_t *thrp;
316 unsigned int count;
317
318 apr_pool_create(&tp, cfg->pool);
319 rv = apr_queue_create(&queue, cfg->input_files->nelts, tp);
320
321 rv = apr_thread_pool_create(&thrp, 0, cfg->thread_count, tp);
322
323 //populate queue
324 filelist = (config_filestat_t *)cfg->input_files->elts;
325 for (f=0, l=cfg->input_files->nelts; f < l; f++) {
326 rv = apr_queue_push(queue, &filelist[f]);
327 }
328 // populate the worker threads
329 for (f=0; f<cfg->thread_count; f++) {
330 rv = apr_thread_pool_push(thrp, run_filethread, cfg, 0, NULL);
331 }
332
333 do {
334 apr_sleep(apr_time_from_sec(1));
335 count = apr_queue_size(queue);
336 } while (count > 0);
337
338 rv = apr_queue_term(queue);
339
340 rv = apr_thread_pool_destroy(thrp);
341}
342#endif
diff --git a/utility/shell.h b/utility/shell.h
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/utility/shell.h
diff --git a/utility/util.c b/utility/util.c
new file mode 100644
index 0000000..dcfb028
--- /dev/null
+++ b/utility/util.c
@@ -0,0 +1,189 @@
1#include "util.h"
2#include "apr_strings.h"
3#include "apr_lib.h"
4#include "apr_file_io.h"
5#include "apr_time.h"
6
7#include "config.h"
8
9#include <stdarg.h>
10
11char *lowerstr(apr_pool_t *pool, const char *input)
12{
13 char *temp;
14 char *itr;
15 temp = apr_pstrdup(pool, input);
16 for (itr=temp; *itr!='\0'; itr++) {
17 *itr = apr_tolower(*itr);
18 }
19 return temp;
20}
21
22void line_chomp(char *str)
23{
24 int len;
25 // chomp off newline
26 len = strlen(str);
27 if (len) {
28 while (str[len-1] == '\r' || str[len-1] == '\n') {
29 str[len-1] = '\0';
30 len--;
31 }
32 }
33}
34
35/*
36 * *** Ripped from HTTPD util.c (why are so many PORTABLE things not in APR UTIL?)
37 */
38static char x2c(const char *what)
39{
40 register char digit;
41
42 digit = ((what[0] >= 'A') ? ((what[0] & 0xdf) - 'A') + 10
43 : (what[0] - '0'));
44 digit *= 16;
45 digit += (what[1] >= 'A' ? ((what[1] & 0xdf) - 'A') + 10
46 : (what[1] - '0'));
47 return (digit);
48}
49
50/*
51 * *** Ripped from HTTPD util.c (why are so many PORTABLE things not in APR UTIL?)
52 *
53 * Unescapes a URL, leaving reserved characters intact.
54 * Returns 0 on success, non-zero on error
55 * Failure is due to
56 * bad % escape returns HTTP_BAD_REQUEST
57 *
58 * decoding %00 or a forbidden character returns HTTP_NOT_FOUND
59 */
60static int unescape_url(char *url, const char *forbid, const char *reserved)
61{
62 register int badesc, badpath;
63 char *x, *y;
64
65 badesc = 0;
66 badpath = 0;
67 /* Initial scan for first '%'. Don't bother writing values before
68 * seeing a '%' */
69 y = strchr(url, '%');
70 if (y == NULL) {
71 return APR_SUCCESS;
72 }
73 for (x = y; *y; ++x, ++y) {
74 if (*y != '%') {
75 *x = *y;
76 }
77 else {
78 if (!apr_isxdigit(*(y + 1)) || !apr_isxdigit(*(y + 2))) {
79 badesc = 1;
80 *x = '%';
81 }
82 else {
83 char decoded;
84 decoded = x2c(y + 1);
85 if ((decoded == '\0')
86 || (forbid && strchr(forbid, decoded))) {
87 badpath = 1;
88 *x = decoded;
89 y += 2;
90 }
91 else if (reserved && strchr(reserved, decoded)) {
92 *x++ = *y++;
93 *x++ = *y++;
94 *x = *y;
95 }
96 else {
97 *x = decoded;
98 y += 2;
99 }
100 }
101 }
102 }
103 *x = '\0';
104 if (badesc) {
105 return APR_EINVAL;
106 }
107 else if (badpath) {
108 return APR_EINVAL;
109 }
110 else {
111 return APR_SUCCESS;
112 }
113}
114
115/*
116 * *** Ripped from HTTPD util.c (why are so many PORTABLE things not in APR UTIL?)
117 */
118int ap_unescape_url(char *url)
119{
120 /* Traditional */
121#ifdef CASE_BLIND_FILESYSTEM
122 return unescape_url(url, "/\\", NULL);
123#else
124 return unescape_url(url, "/", NULL);
125#endif
126}
127
128void logging_preinit(config_t *cfg)
129{
130 apr_pool_create(&cfg->errorlog_p, cfg->pool);
131 apr_file_open_stderr(&cfg->errorlog_fperr, cfg->pool);
132}
133
134void logging_init(config_t *cfg)
135{
136 apr_status_t rv;
137 if (cfg->errorlog) {
138 rv = apr_file_open(&cfg->errorlog_fp, cfg->errorlog,
139 APR_FOPEN_CREATE | APR_FOPEN_WRITE | APR_FOPEN_APPEND,
140 APR_OS_DEFAULT,
141 cfg->pool);
142 if (rv) {
143 printf("Error opening %s\n",cfg->errorlog);
144 cfg->loglevel = LOGLEVEL_NONE;
145 }
146 logging_log(cfg, LOGLEVEL_ERROR, "Log file Opened");
147 } else {
148 cfg->loglevel = LOGLEVEL_NONE;
149 logging_log(cfg, LOGLEVEL_NOISE, "No Log file specified, disabled logging");
150 }
151}
152
153const char *logging_strerror(apr_status_t rv)
154{
155 char buff[256];
156 return apr_strerror(rv, buff, 256);
157}
158
159void logging_log(config_t *cfg, loglevel_e level, const char *fmt, ...)
160{
161 va_list ap;
162 char date[APR_RFC822_DATE_LEN];
163 struct iovec vec[4];
164 apr_size_t blen;
165
166 if (cfg->loglevel < level) return;
167
168 va_start(ap, fmt);
169 apr_pool_clear(cfg->errorlog_p);
170
171 apr_rfc822_date(date, apr_time_now());
172 vec[0].iov_base = date;
173 vec[0].iov_len = APR_RFC822_DATE_LEN-1;
174 vec[1].iov_base = " ";
175 vec[1].iov_len = 2;
176 vec[2].iov_base = apr_pvsprintf(cfg->errorlog_p, fmt, ap);
177 vec[2].iov_len = strlen(vec[2].iov_base);
178 vec[3].iov_base = "\n";
179 vec[3].iov_len = 1;
180
181 if (level == LOGLEVEL_NOISE) {
182 apr_file_writev(cfg->errorlog_fperr,&vec[2],2,&blen);
183 }
184 if (cfg->loglevel > LOGLEVEL_NONE && cfg->errorlog_fp) {
185 apr_file_writev(cfg->errorlog_fp,vec,4,&blen);
186 }
187
188 va_end(ap);
189}
diff --git a/utility/util.h b/utility/util.h
new file mode 100644
index 0000000..a5a7f7e
--- /dev/null
+++ b/utility/util.h
@@ -0,0 +1,26 @@
1#ifndef UTIL_H_
2#define UTIL_H_
3
4#include "apr_pools.h"
5
6#include "config.h"
7
8char *lowerstr(apr_pool_t *pool, const char *input);
9
10/**
11 * Chomp new line characters off the end of the line
12 */
13void line_chomp(char *str);
14
15int ap_unescape_url(char *url);
16
17void logging_preinit(config_t *cfg);
18
19void logging_init(config_t *cfg);
20
21void logging_log(config_t *cfg, loglevel_e level, const char *fmt, ...)
22 __attribute__((format(printf, 3, 4)));
23
24const char *logging_strerror(apr_status_t rv);
25
26#endif /*UTIL_H_*/
generated by cgit (git 2.34.1) at 2024-11-21 21:46:08 +0000