summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.cproject21
-rw-r--r--Makefile.in2
-rw-r--r--configure.ac1
-rw-r--r--src/Makefile.in3
-rw-r--r--utility/Makefile.in38
-rw-r--r--utility/config.c305
-rw-r--r--utility/config.h104
-rw-r--r--utility/logparse.c205
-rw-r--r--utility/logparse.h10
-rw-r--r--utility/mod_log_sql.conf30
-rw-r--r--utility/shell.c148
-rw-r--r--utility/shell.h0
12 files changed, 858 insertions, 9 deletions
diff --git a/.cproject b/.cproject
index 3ef91b9..ce89a76 100644
--- a/.cproject
+++ b/.cproject
@@ -11,6 +11,7 @@
11<extensions> 11<extensions>
12<extension id="org.eclipse.cdt.core.domsourceindexer" point="org.eclipse.cdt.core.CIndexer"/> 12<extension id="org.eclipse.cdt.core.domsourceindexer" point="org.eclipse.cdt.core.CIndexer"/>
13<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/> 13<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
14<extension id="org.eclipse.cdt.core.GNU_ELF" point="org.eclipse.cdt.core.BinaryParser"/>
14<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/> 15<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
15<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/> 16<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
16<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/> 17<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
@@ -25,16 +26,15 @@
25<storageModule moduleId="cdtBuildSystem" version="4.0.0"> 26<storageModule moduleId="cdtBuildSystem" version="4.0.0">
26<configuration artifactExtension="" artifactName="mod_log_sql" buildProperties="" description="" errorParsers="" id="converted.config.350824298" name="Linux GCC" parent="org.eclipse.cdt.build.core.prefbase.cfg"> 27<configuration artifactExtension="" artifactName="mod_log_sql" buildProperties="" description="" errorParsers="" id="converted.config.350824298" name="Linux GCC" parent="org.eclipse.cdt.build.core.prefbase.cfg">
27<folderInfo id="converted.config.350824298." name="/" resourcePath=""> 28<folderInfo id="converted.config.350824298." name="/" resourcePath="">
28<toolChain id="cdt.managedbuild.toolchain.gnu.base.1351398655" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.base"> 29<toolChain id="cdt.managedbuild.toolchain.gnu.base.1351398655" name="Linux GCC" resourceTypeBasedDiscovery="true" superClass="cdt.managedbuild.toolchain.gnu.base">
29<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.target.gnu.platform.base.2070600171" name="Debug Platform" osList="linux,hpux,aix,qnx" superClass="cdt.managedbuild.target.gnu.platform.base"/> 30<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.GNU_ELF" id="cdt.managedbuild.target.gnu.platform.base.2070600171" name="Debug Platform" osList="linux,hpux,aix,qnx" superClass="cdt.managedbuild.target.gnu.platform.base"/>
30<builder appendEnvironment="true" arguments="${project_name} make" buildPath="${workspace_loc:/mod_log_sql/Linux GCC}" command="${HOME}/bin/sshmake" environment="" id="cdt.managedbuild.target.gnu.builder.base.1704124595" keepEnvironmentInBuildfile="false" managedBuildOn="false" name="Gnu Make Builder" stopOnErr="false" superClass="cdt.managedbuild.target.gnu.builder.base"/> 31<builder appendEnvironment="true" arguments="${project_name} make" autoBuildTarget="all" buildPath="${workspace_loc:/mod_log_sql/Linux GCC}" cleanBuildTarget="clean" command="${HOME}/bin/sshmake" enableAutoBuild="false" enableCleanBuild="true" enabledIncrementalBuild="true" environment="" id="cdt.managedbuild.target.gnu.builder.base.1704124595" incrementalBuildTarget="all" keepEnvironmentInBuildfile="false" managedBuildOn="false" name="Gnu Make Builder" stopOnErr="false" superClass="cdt.managedbuild.target.gnu.builder.base"/>
31<tool id="cdt.managedbuild.tool.gnu.archiver.base.1647937860" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
32<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.base.1017302318" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.base"/>
33<tool id="cdt.managedbuild.tool.gnu.c.compiler.base.264929639" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.base"> 32<tool id="cdt.managedbuild.tool.gnu.c.compiler.base.264929639" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.base">
34<option id="gnu.c.compiler.option.include.paths.277599024" name="Include paths (-I)" superClass="gnu.c.compiler.option.include.paths" valueType="includePath"> 33<option id="gnu.c.compiler.option.include.paths.277599024" name="Include paths (-I)" superClass="gnu.c.compiler.option.include.paths" valueType="includePath">
35<listOptionValue builtIn="false" value="&quot;${workspace_loc:/includes/2.2/apr-0}&quot;"/> 34<listOptionValue builtIn="false" value="&quot;${workspace_loc:/includes/2.2/apr-1}&quot;"/>
36<listOptionValue builtIn="false" value="&quot;${workspace_loc:/includes/2.2/httpd}&quot;"/> 35<listOptionValue builtIn="false" value="&quot;${workspace_loc:/includes/2.2/httpd}&quot;"/>
37<listOptionValue builtIn="false" value="&quot;${workspace_loc:/Includes/mysql}&quot;"/> 36<listOptionValue builtIn="false" value="&quot;${workspace_loc:/Includes/mysql}&quot;"/>
37<listOptionValue builtIn="false" value="/usr/include"/>
38</option> 38</option>
39<option id="gnu.c.compiler.option.preprocessor.def.symbols.1267853547" name="Defined symbols (-D)" superClass="gnu.c.compiler.option.preprocessor.def.symbols" valueType="definedSymbols"> 39<option id="gnu.c.compiler.option.preprocessor.def.symbols.1267853547" name="Defined symbols (-D)" superClass="gnu.c.compiler.option.preprocessor.def.symbols" valueType="definedSymbols">
40<listOptionValue builtIn="false" value="WITH_APACHE20=1"/> 40<listOptionValue builtIn="false" value="WITH_APACHE20=1"/>
@@ -60,6 +60,11 @@
60<option id="gnu.c.compiler.option.misc.ansi.1838461503" name="Support ANSI programs (-ansi)" superClass="gnu.c.compiler.option.misc.ansi"/> 60<option id="gnu.c.compiler.option.misc.ansi.1838461503" name="Support ANSI programs (-ansi)" superClass="gnu.c.compiler.option.misc.ansi"/>
61<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.616458355" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/> 61<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.616458355" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
62</tool> 62</tool>
63<tool id="cdt.managedbuild.tool.gnu.cpp.linker.base.1761639329" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.base"/>
64<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.base.1017302318" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.base"/>
65<tool id="com.redhat.eclipse.cdt.autotools.tool.autogen.131560145" name="autogen.sh" superClass="com.redhat.eclipse.cdt.autotools.tool.autogen"/>
66<tool id="com.redhat.eclipse.cdt.autotools.tool.configure.1351107831" name="configure" superClass="com.redhat.eclipse.cdt.autotools.tool.configure"/>
67<tool id="cdt.managedbuild.tool.gnu.archiver.base.1647937860" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
63<tool id="cdt.managedbuild.tool.gnu.c.linker.base.78392638" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.base"> 68<tool id="cdt.managedbuild.tool.gnu.c.linker.base.78392638" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.base">
64<option id="gnu.c.link.option.nostart.1696010990" name="Do not use standard start files (-nostartfiles)" superClass="gnu.c.link.option.nostart"/> 69<option id="gnu.c.link.option.nostart.1696010990" name="Do not use standard start files (-nostartfiles)" superClass="gnu.c.link.option.nostart"/>
65<option id="gnu.c.link.option.nodeflibs.149057311" name="Do not use default libraries (-nodefaultlibs)" superClass="gnu.c.link.option.nodeflibs"/> 70<option id="gnu.c.link.option.nodeflibs.149057311" name="Do not use default libraries (-nodefaultlibs)" superClass="gnu.c.link.option.nodeflibs"/>
@@ -80,9 +85,8 @@
80<additionalInput kind="additionalinput" paths="$(LIBS)"/> 85<additionalInput kind="additionalinput" paths="$(LIBS)"/>
81</inputType> 86</inputType>
82</tool> 87</tool>
83<tool id="cdt.managedbuild.tool.gnu.cpp.linker.base.1761639329" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.base"/>
84<tool id="cdt.managedbuild.tool.gnu.assembler.base.467752860" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.base"> 88<tool id="cdt.managedbuild.tool.gnu.assembler.base.467752860" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.base">
85<option id="gnu.both.asm.option.include.paths.73704182" name="Include paths (-I)" superClass="gnu.both.asm.option.include.paths" valueType="includePath"/> 89<option id="gnu.both.asm.option.include.paths.73704182" name="Include paths (-I)" superClass="gnu.both.asm.option.include.paths"/>
86<inputType id="cdt.managedbuild.tool.gnu.assembler.input.1742318494" superClass="cdt.managedbuild.tool.gnu.assembler.input"/> 90<inputType id="cdt.managedbuild.tool.gnu.assembler.input.1742318494" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
87</tool> 91</tool>
88</toolChain> 92</toolChain>
@@ -100,6 +104,7 @@
100</storageModule> 104</storageModule>
101<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets"/> 105<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets"/>
102 106
107
103<storageModule moduleId="scannerConfiguration"> 108<storageModule moduleId="scannerConfiguration">
104<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/> 109<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"/>
105<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile"> 110<profile id="org.eclipse.cdt.make.core.GCCStandardMakePerProjectProfile">
diff --git a/Makefile.in b/Makefile.in
index fb4ea42..7d71505 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1,7 +1,7 @@
1# @configure_input@ 1# @configure_input@
2 2
3# Modify these top variables. 3# Modify these top variables.
4SUBDIRS = docs contrib src 4SUBDIRS = docs contrib src parser utility
5 5
6EXTRA_DIST = AUTHORS INSTALL TODO LICENSE CHANGELOG \ 6EXTRA_DIST = AUTHORS INSTALL TODO LICENSE CHANGELOG \
7 build-apache13.bat build-apache2.bat \ 7 build-apache13.bat build-apache2.bat \
diff --git a/configure.ac b/configure.ac
index 07ac486..43633e3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -85,6 +85,7 @@ dnl Write config.status and the Makefile
85AC_CONFIG_FILES([Makefile 85AC_CONFIG_FILES([Makefile
86src/Makefile 86src/Makefile
87docs/Makefile 87docs/Makefile
88utility/Makefile
88contrib/Makefile]) 89contrib/Makefile])
89AC_OUTPUT 90AC_OUTPUT
90 91
diff --git a/src/Makefile.in b/src/Makefile.in
index 558b621..5a5dd6a 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -163,3 +163,6 @@ clean:
163local-dist: $(DISTFILES) 163local-dist: $(DISTFILES)
164 mkdir -p $(DESTDIR) 164 mkdir -p $(DESTDIR)
165 cp -dp --parents $(DISTFILES) $(DESTDIR) 165 cp -dp --parents $(DISTFILES) $(DESTDIR)
166
167Makefile: Makefile.in ../config.status
168 cd .. && ./config.status
diff --git a/utility/Makefile.in b/utility/Makefile.in
new file mode 100644
index 0000000..d397b96
--- /dev/null
+++ b/utility/Makefile.in
@@ -0,0 +1,38 @@
1# @configure_input@
2
3top_srcdir = @top_srcdir@
4srcdir = @abs_srcdir@
5builddir = @abs_builddir@
6
7CFLAGS = -g -Wall -fno-strict-aliasing @APR_CFLAGS@ @APR_INCLUDES@ @APU_INCLUDES@
8CPPFLAGS = @APR_CPPFLAGS@
9LDFLAGS = @APR_LDFLAGS@ @APU_LDFLAGS@
10
11ifeq (@OOO_MAINTAIN@,1)
12CFLAGS += -Werror
13endif
14
15STD_DIST = Makefile.in
16
17DISTFILES = $(STD_DIST) $(EXTRA_DIST) $(SOURCES) $(HEADERS)
18
19SOURCES = shell.c config.c logparse.c
20HEADERS = shell.h config.h logparse.h
21OBJECTS = $(patsubst %.c,%.o,$(SOURCES))
22
23TARGETS = mod_log_sql
24
25all: $(TARGETS)
26
27mod_log_sql: $(OBJECTS) $(HEADERS)
28 $(CC) -o $@ $(OBJECTS) $(LDFLAGS)
29
30clean:
31 $(RM) $(OBJECTS) $(TARGETS)
32
33local-dist: $(DISTFILES)
34 mkdir -p $(DESTDIR)
35 cp -dp --parents $(DISTFILES) $(DESTDIR)
36
37Makefile: Makefile.in ../config.status
38 cd .. && ./config.status
diff --git a/utility/config.c b/utility/config.c
new file mode 100644
index 0000000..6b3dce1
--- /dev/null
+++ b/utility/config.c
@@ -0,0 +1,305 @@
1#include "apr.h"
2#include "apr_file_info.h"
3#include "apr_file_io.h"
4#include "apr_strings.h"
5#include "apr_hash.h"
6#include "apr_lib.h"
7#include "shell.h"
8#include "config.h"
9
10apr_hash_t *g_config_opts;
11
12apr_status_t config_set_string(config_t *cfg, config_opt_t *opt, int argc,
13 const char **argv)
14{
15 int offset = (int)(long)opt->data;
16 char **data = (char **)((void *)cfg + offset);
17 if (argc != 2) return APR_EINVAL;
18 *data = apr_pstrdup(cfg->pool, argv[1]);
19 return APR_SUCCESS;
20}
21
22apr_status_t config_set_int(config_t *cfg, config_opt_t *opt, int argc,
23 const char **argv)
24{
25 int offset = (int)(long)opt->data;
26 int *data = (int *)((void *)cfg + offset);
27 if (argc != 2) return APR_EINVAL;
28 *data = apr_atoi64(argv[1]);
29 return APR_SUCCESS;
30}
31
32apr_status_t config_set_flag(config_t *cfg, config_opt_t *opt, int argc,
33 const char **argv)
34{
35 int offset = (int)(long)opt->data;
36 int *data = (int *)((void *)cfg + offset);
37 if (argc != 2) return APR_EINVAL;
38 *data = CHECK_YESNO(argv[1]);
39 return APR_SUCCESS;
40}
41
42apr_status_t config_set_loglevel(config_t *cfg, config_opt_t *opt, int argc,
43 const char **argv)
44{
45 if (argc != 2) return APR_EINVAL;
46 if (!strcasecmp(argv[1],"error")) {
47 cfg->loglevel = LOGLEVEL_ERROR;
48 } else if (!strcasecmp(argv[1],"warn")) {
49 cfg->loglevel = LOGLEVEL_WARN;
50 } else if (!strcasecmp(argv[1],"debug")) {
51 cfg->loglevel = LOGLEVEL_DEBUG;
52 } else if (!strcasecmp(argv[1],"quiet")) {
53 cfg->loglevel = LOGLEVEL_QUIET;
54 } else {
55 cfg->loglevel = LOGLEVEL_ERROR;
56 }
57 return APR_SUCCESS;
58}
59
60apr_status_t config_set_dbconnect(config_t *cfg, config_opt_t *opt, int argc,
61 const char **argv)
62{
63 return APR_SUCCESS;
64}
65
66apr_status_t config_set_dbparam(config_t *cfg, config_opt_t *opt, int argc,
67 const char **argv)
68{
69 return APR_SUCCESS;
70}
71
72apr_status_t config_set_inputfile(config_t *cfg, config_opt_t *opt, int argc,
73 const char **argv)
74{
75 char **newp;
76 if (argc != 2) return APR_EINVAL;
77 newp = (char **)apr_array_push(cfg->input_files);
78 *newp = apr_pstrdup(cfg->pool, argv[1]);
79 return APR_SUCCESS;
80}
81
82apr_status_t config_set_dummy(config_t *cfg, config_opt_t *opt, int argc,
83 const char **argv)
84{
85 return APR_SUCCESS;
86}
87
88apr_status_t config_set_logformat(config_t *cfg, config_opt_t *opt, int argc,
89 const char **argv)
90{
91 config_logformat_t *format;
92 config_logformat_field_t *field;
93
94 if (argc != 4) return APR_EINVAL;
95
96 format = apr_hash_get(cfg->log_formats,argv[1],APR_HASH_KEY_STRING);
97 if (!format) {
98 format = apr_palloc(cfg->pool, sizeof(config_logformat_t));
99 format->name = apr_pstrdup(cfg->pool, argv[1]);
100 format->fields = apr_array_make(cfg->pool, 5,
101 sizeof(config_logformat_field_t));
102 apr_hash_set(cfg->log_formats, argv[1], APR_HASH_KEY_STRING, format);
103 }
104 field = (config_logformat_field_t *)apr_array_push(format->fields);
105 field->name = apr_pstrdup(cfg->pool, argv[2]);
106 field->datatype = apr_pstrdup(cfg->pool, argv[3]);
107 return APR_SUCCESS;
108}
109
110void config_dump(config_t *cfg)
111{
112 apr_hash_index_t *hi;
113
114 printf("ErrorLog: %s\n",cfg->errorlog);
115 printf("LogLevel: %d\n",cfg->loglevel);
116
117 printf("InputDir: %s\n",cfg->input_dir);
118
119 printf("Table: %s\n",cfg->table);
120 printf("Transactions: %d\n",cfg->transactions);
121 printf("MachineID: %s\n",cfg->machineid);
122
123 printf("Log formats:\n");
124 for (hi = apr_hash_first(cfg->pool, cfg->log_formats); hi; hi
125 = apr_hash_next(hi)) {
126 config_logformat_t *format;
127 config_logformat_field_t *fields;
128 int i;
129
130 apr_hash_this(hi, NULL, NULL, (void **)&format);
131 printf(">> %s\n",format->name);
132 fields = (config_logformat_field_t *)format->fields->elts;
133 for (i=0; i<format->fields->nelts; i++) {
134 printf(">>>> %s:%s\n", fields[i].name, fields[i].datatype);
135 }
136 }
137 printf("Log Format: %s\n",cfg->logformat);
138
139 printf("DryRun: %d\n",cfg->dryrun);
140 printf("Summary: %d\n",cfg->summary);
141}
142
143static char *lowerstr(apr_pool_t *pool, const char *input) {
144 char *temp;
145 char *itr;
146 temp = apr_pstrdup(pool, input);
147 for (itr=temp; *itr!='\0'; itr++) {
148 *itr = apr_tolower(*itr);
149 }
150 return temp;
151}
152
153#define config_get_option(name) apr_hash_get(g_config_opts, name, APR_HASH_KEY_STRING)
154
155void config_add_option(apr_pool_t *p, const char *const name,
156 const char *const help, config_func_t func, void *data)
157{
158 config_opt_t *opt;
159 if (!g_config_opts) {
160 g_config_opts = apr_hash_make(p);
161 }
162 opt = apr_palloc(p, sizeof(config_opt_t));
163 opt->name = name;
164 opt->help = help;
165 opt->func = func;
166 opt->data = data;
167 apr_hash_set(g_config_opts, lowerstr(p,name), APR_HASH_KEY_STRING, opt);
168}
169
170void config_init(apr_pool_t *p)
171{
172 config_add_option(p, "ErrorLog", "File to log errors",
173 config_set_string, (void *)APR_OFFSETOF(config_t, errorlog));
174 config_add_option(p, "LogLevel", "Set Log Level (error, warn, debug, quiet)",
175 config_set_loglevel, NULL);
176
177 config_add_option(p, "InputDirectory", "Directory to scan for log files",
178 config_set_string, (void *)APR_OFFSETOF(config_t, input_dir));
179 config_add_option(p, "InputFile", "Parse only this file",
180 config_set_inputfile, NULL);
181
182 config_add_option(p, "DBConnect", "DB Connection information type://user:pass@hostname/database",
183 config_set_dbconnect, NULL);
184 config_add_option(p, "DBParam", "DB Connection Parameter",
185 config_set_dbparam, NULL);
186 config_add_option(p, "Table", "Table to import the log to",
187 config_set_string, (void *)APR_OFFSETOF(config_t, table));
188 config_add_option(p, "UseTransactions", "Enable Transactions?",
189 config_set_flag, (void *)APR_OFFSETOF(config_t, transactions));
190 config_add_option(p, "MachineID", "Machine ID to set",
191 config_set_string, (void *)APR_OFFSETOF(config_t, machineid));
192
193 config_add_option(p, "LogFormatConfig", "Define input log formats",
194 config_set_logformat, NULL);
195 config_add_option(p, "LogFormat", "Use this logformat when parsing files",
196 config_set_string, (void *)APR_OFFSETOF(config_t, logformat));
197
198 config_add_option(p, "DryRun", "Don't perform any actual database changes",
199 config_set_flag, (void *)APR_OFFSETOF(config_t, dryrun));
200 config_add_option(p, "Config", "Dummy to handle config directive",
201 config_set_dummy, NULL);
202 config_add_option(p, "Summary", "Show the summary before exit?",
203 config_set_flag, (void *)APR_OFFSETOF(config_t, summary));
204}
205
206config_t *config_create(apr_pool_t *p)
207{
208 config_t *cfg;
209 apr_pool_t *sp;
210 apr_pool_create(&sp, p);
211 cfg = apr_pcalloc(sp, sizeof(config_t));
212 cfg->pool = sp;
213 cfg->loglevel = LOGLEVEL_WARN;
214 cfg->summary = 1;
215 cfg->transactions = 1;
216 cfg->input_files = apr_array_make(cfg->pool, 10, sizeof(char *));
217 cfg->dbconfig = apr_table_make(cfg->pool, 5);
218 cfg->log_formats = apr_hash_make(cfg->pool);
219
220 return cfg;
221}
222
223int config_merge(void *rec, const char *key, const char *value) {
224 config_t *cfg = (config_t *)rec;
225
226 config_opt_t *opt = config_get_option(key);
227 if (opt) {
228 const char *args[] = {key, value};
229 opt->func(cfg, opt, 2, args);
230 } else {
231 printf("Unhandled: %s\n", key);
232 }
233 return 1;
234}
235
236apr_status_t config_read(config_t *cfg, const char *filename,
237 apr_table_t *merge)
238{
239 apr_finfo_t finfo;
240 apr_file_t *file;
241 apr_status_t rv, ret = APR_SUCCESS;
242 apr_pool_t *tp, *targp;
243 config_opt_t *opt;
244 char buff[1024];
245 char *ptr, *ptr2;
246 char **targv;
247 int targc;
248 int line;
249
250 apr_pool_create(&tp, cfg->pool);
251 apr_pool_create(&targp, tp);
252
253 if (apr_stat(&finfo, filename, APR_FINFO_MIN, tp) != APR_SUCCESS) {
254 return APR_ENOENT;
255 }
256 rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED,
257 APR_OS_DEFAULT, tp);
258 if (rv != APR_SUCCESS)
259 return rv;
260
261 line = 0;
262 do {
263 rv = apr_file_gets(buff, 1024, file);
264 if (rv == APR_SUCCESS) { // we read data
265 line++;
266
267 // skip leading white space
268 for (ptr = buff; *ptr == ' ' || *ptr == '\t'; ptr++)
269 ;
270 // chomp off newline
271 for (ptr2 = ptr + strlen(ptr); *ptr2 != '\r' && *ptr2 != '\n'; ptr2--)
272 ;
273 *ptr2 = '\0';
274
275 // skip comments
276 if (*ptr == '#')
277 continue;
278 if (*ptr == '\0')
279 continue;
280 apr_pool_clear(targp);
281 apr_tokenize_to_argv(buff, &targv, targp);
282 targc = 0;
283 while (targv[targc]) targc++;
284 opt = config_get_option(lowerstr(targp,targv[0]));
285 if (opt) {
286 rv = opt->func(cfg, opt, targc, (const char **)targv);
287 if (APR_STATUS_IS_EINVAL(rv)) {
288 printf("Config Error: Invalid Arguments for %s\n\t%s\n",
289 opt->name, opt->help);
290 ret = rv;
291 }
292 } else {
293 printf("Unhandled: %s\n", targv[0]);
294 }
295 }
296 } while (rv == APR_SUCCESS);
297
298 // Apply merges
299 apr_table_do(config_merge,(void *)cfg,merge,NULL);
300
301 apr_file_close(file);
302 apr_pool_destroy(tp);
303 return ret;
304}
305
diff --git a/utility/config.h b/utility/config.h
new file mode 100644
index 0000000..e1827fe
--- /dev/null
+++ b/utility/config.h
@@ -0,0 +1,104 @@
1#ifndef CONFIG_H_
2#define CONFIG_H_
3
4#include "apr_tables.h"
5#include "apr_hash.h"
6#include "apr_file_io.h"
7
8typedef enum {
9 LOGLEVEL_QUIET = 0,
10 LOGLEVEL_ERROR = 1,
11 LOGLEVEL_WARN = 2,
12 LOGLEVEL_DEBUG = 3,
13} loglevel_e;
14
15typedef struct {
16 /** the structures pool (to ease function arguments) */
17 apr_pool_t *pool;
18
19 /** error log file */
20 const char *errorlog;
21 /** log level */
22 loglevel_e loglevel;
23 /** error_log */
24 apr_file_t *errorlog_fp;
25
26 /** input directory of log files */
27 const char *input_dir;
28 /** list of files to scan */
29 apr_array_header_t *input_files;
30
31 /** db connection configuration */
32 apr_table_t *dbconfig;
33 /** Logging table */
34 const char *table;
35 /** Use transactons */
36 int transactions;
37 /** Machine ID */
38 const char *machineid;
39
40 /** Log file formats */
41 apr_hash_t *log_formats;
42 /** format to use to parse files */
43 const char *logformat;
44
45 /** output fields */
46 apr_array_header_t *output_fields;
47
48 /** filter configuration */
49 apr_array_header_t *filters;
50
51 /** Dry Run */
52 int dryrun;
53
54 /* Show the summary */
55 int summary;
56} config_t;
57
58typedef struct {
59 const char *name;
60 apr_array_header_t *fields;
61} config_logformat_t;
62
63typedef struct {
64 const char *name;
65 const char *datatype;
66} config_logformat_field_t;
67
68typedef struct config_opt_t config_opt_t;
69typedef apr_status_t (*config_func_t)(config_t *cfg, config_opt_t *opt,
70 int argc, const char **argv);
71
72struct config_opt_t {
73 const char *name;
74 const char *help;
75 config_func_t func;
76 void *data;
77};
78
79#define CHECK_YESNO(c) ((!strcasecmp(c,"on") || !strcasecmp(c,"yes")) ? 1 : 0)
80
81/**
82 * Initialize the config parser
83 */
84void config_init(apr_pool_t *p);
85
86/**
87 * Dump the configuration to stdout
88 */
89void config_dump(config_t *cfg);
90
91/**
92 * Creates the default configuration
93 */
94config_t *config_create(apr_pool_t *p);
95
96/**
97 * Read in a configuration file
98 */
99apr_status_t config_read(config_t *cfg, const char *filename,
100 apr_table_t *merge);
101
102void config_generate(const char *filename);
103
104#endif /*CONFIG_H_*/
diff --git a/utility/logparse.c b/utility/logparse.c
new file mode 100644
index 0000000..2940534
--- /dev/null
+++ b/utility/logparse.c
@@ -0,0 +1,205 @@
1#include "logparse.h"
2#include "apr_file_info.h"
3#include "apr_file_io.h"
4#include "apr_strings.h"
5
6void find_log_files(config_t *cfg)
7{
8 apr_pool_t *tp;
9 apr_dir_t *dir;
10 apr_finfo_t finfo;
11 char **newp;
12
13 if (!cfg->input_dir)
14 return;
15 apr_pool_create(&tp, cfg->pool);
16 if (apr_dir_open(&dir, cfg->input_dir, tp)==APR_SUCCESS) {
17 while (apr_dir_read(&finfo, APR_FINFO_NAME | APR_FINFO_TYPE, dir)
18 == APR_SUCCESS) {
19 if (finfo.filetype == APR_DIR)
20 continue;
21 newp = (char **)apr_array_push(cfg->input_files);
22 apr_filepath_merge(newp, cfg->input_dir, finfo.name,
23 APR_FILEPATH_TRUENAME, cfg->pool);
24 }
25 apr_dir_close(dir);
26 }
27 apr_pool_destroy(tp);
28}
29
30/*
31 * Modified version of apr_tokenize_to_argv to add [] as quoting characters
32 *
33 * token_context: Context from which pool allocations will occur.
34 * arg_str: Input string for conversion to argv[].
35 * argv_out: Output location. This is a pointer to an array
36 * of pointers to strings (ie. &(char *argv[]).
37 * This value will be allocated from the contexts
38 * pool and filled in with copies of the tokens
39 * found during parsing of the arg_str.
40 * keepquotes: Keep the quotes instead of stripping them
41 */
42apr_status_t tokenize_logline(const char *arg_str, char ***argv_out,
43 apr_pool_t *token_context, int keepquotes)
44{
45 const char *cp;
46 const char *ct;
47 char *cleaned, *dirty;
48 int escaped;
49 int isquoted, numargs = 0, argnum;
50
51#define SKIP_WHITESPACE(cp) \
52 for ( ; *cp == ' ' || *cp == '\t'; ) { \
53 cp++; \
54 };
55
56#define CHECK_QUOTATION(cp,isquoted) \
57 isquoted = 0; \
58 if (*cp == '"') { \
59 isquoted = 1; \
60 cp++; \
61 } \
62 else if (*cp == '\'') { \
63 isquoted = 2; \
64 cp++; \
65 } \
66 else if (*cp == '[') { \
67 isquoted = 3; \
68 cp++; \
69 }
70
71 /* DETERMINE_NEXTSTRING:
72 * At exit, cp will point to one of the following: NULL, SPACE, TAB or QUOTE.
73 * NULL implies the argument string has been fully traversed.
74 */
75#define DETERMINE_NEXTSTRING(cp,isquoted) \
76 for ( ; *cp != '\0'; cp++) { \
77 if ( (isquoted && (*cp == ' ' || *cp == '\t')) \
78 || (*cp == '\\' && (*(cp+1) == ' ' || *(cp+1) == '\t' || \
79 *(cp+1) == '"' || *(cp+1) == '\'' || \
80 *(cp+1) == '[' || *(cp+1) == ']'))) { \
81 cp++; \
82 continue; \
83 } \
84 if ( (!isquoted && (*cp == ' ' || *cp == '\t')) \
85 || (isquoted == 1 && *cp == '"') \
86 || (isquoted == 2 && *cp == '\'') \
87 || (isquoted == 3 && *cp == ']') \
88 ) { \
89 break; \
90 } \
91 }
92
93 /* REMOVE_ESCAPE_CHARS:
94 * Compresses the arg string to remove all of the '\' escape chars.
95 * The final argv strings should not have any extra escape chars in it.
96 */
97#define REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped) \
98 escaped = 0; \
99 while(*dirty) { \
100 if (!escaped && *dirty == '\\') { \
101 escaped = 1; \
102 } \
103 else { \
104 escaped = 0; \
105 *cleaned++ = *dirty; \
106 } \
107 ++dirty; \
108 } \
109 *cleaned = 0; /* last line of macro... */
110
111 cp = arg_str;
112 SKIP_WHITESPACE(cp);
113 ct = cp;
114
115 /* This is ugly and expensive, but if anyone wants to figure a
116 * way to support any number of args without counting and
117 * allocating, please go ahead and change the code.
118 *
119 * Must account for the trailing NULL arg.
120 */
121 numargs = 1;
122 while (*ct != '\0') {
123 CHECK_QUOTATION(ct, isquoted)
124 ;
125 DETERMINE_NEXTSTRING(ct, isquoted);
126 if (*ct != '\0') {
127 ct++;
128 }
129 numargs++;
130 SKIP_WHITESPACE(ct);
131 }
132 *argv_out = apr_palloc(token_context, numargs * sizeof(char*));
133
134 /* determine first argument */
135 for (argnum = 0; argnum < (numargs-1); argnum++) {
136 SKIP_WHITESPACE(cp);
137 CHECK_QUOTATION(cp, isquoted)
138 ;
139 ct = cp;
140 DETERMINE_NEXTSTRING(cp, isquoted);
141 cp++;
142 if (isquoted && keepquotes) {
143 (*argv_out)[argnum] = apr_palloc(token_context, cp - ct + 2);
144 apr_cpystrn((*argv_out)[argnum], ct -1, cp - ct + 2);
145 } else {
146 (*argv_out)[argnum] = apr_palloc(token_context, cp - ct);
147 apr_cpystrn((*argv_out)[argnum], ct, cp - ct);
148 }
149 cleaned = dirty = (*argv_out)[argnum];
150 REMOVE_ESCAPE_CHARS(cleaned, dirty, escaped)
151 ;
152 }
153 (*argv_out)[argnum] = NULL;
154
155 return APR_SUCCESS;
156}
157
158apr_status_t parse_logfile(config_t *cfg, const char *filename)
159{
160 apr_pool_t *tp, *argp;
161 apr_file_t *file;
162 apr_status_t rv;
163 char buff[2048];
164 char **targv;
165 int targc;
166 int line;
167
168 apr_pool_create(&tp, cfg->pool);
169 apr_pool_create(&argp, tp);
170
171 rv = apr_file_open(&file, filename, APR_FOPEN_READ | APR_BUFFERED,
172 APR_OS_DEFAULT, tp);
173 if (rv != APR_SUCCESS) {
174 printf("Could not open %s\n", filename);
175 return rv;
176 }
177
178 line = 0;
179 do {
180 rv = apr_file_gets(buff, 1024, file);
181 if (rv == APR_SUCCESS) {
182 line++;
183 char *ptr;
184 // chomp off newline
185 for (ptr = buff + strlen(buff); *ptr != '\r' && *ptr != '\n'; ptr--)
186 ;
187 *ptr = '\0';
188 apr_pool_clear(argp);
189 tokenize_logline(buff, &targv, argp, 1);
190 targc = 0;
191 while (targv[targc]) targc++;
192 if (targc != 9) {
193 int i;
194 printf("Line %d(%d): %s\n",line, targc, buff);
195 for (i = 0; targv[i]; i++) {
196 printf("Arg (%d): '%s'\n", i, targv[i]);
197 }
198 }
199 }
200 } while (rv == APR_SUCCESS);
201 printf("Total Lines: %d\n", line);
202 apr_file_close(file);
203 apr_pool_destroy(tp);
204 return APR_SUCCESS;
205}
diff --git a/utility/logparse.h b/utility/logparse.h
new file mode 100644
index 0000000..540a9e0
--- /dev/null
+++ b/utility/logparse.h
@@ -0,0 +1,10 @@
1#ifndef LOGPARSE_H_
2#define LOGPARSE_H_
3
4#include "config.h"
5
6void find_log_files(config_t *cfg);
7
8apr_status_t parse_logfile(config_t *cfg, const char *filename);
9
10#endif /*LOGPARSE_H_*/
diff --git a/utility/mod_log_sql.conf b/utility/mod_log_sql.conf
new file mode 100644
index 0000000..6cfae61
--- /dev/null
+++ b/utility/mod_log_sql.conf
@@ -0,0 +1,30 @@
1InputDirectory ./logs
2ErrorLog ./error_log
3DBConnect mysql://username:host@server/database
4DBParam socketfile /tmp/mysql.sock
5Table apache_logs
6MachineID 7of9
7UseTransactions on
8LogLevel notice
9DryRun on
10Summary on
11
12LogFormatConfig CLF host String
13LogFormatConfig CLF ident String
14LogFormatConfig CLF user String
15LogFormatConfig CLF date Date
16LogFormatConfig CLF request String
17LogFormatConfig CLF status Number
18LogFormatConfig CLF bytes_sent Number
19
20LogFormatConfig Combined host String
21LogFormatConfig Combined ident String
22LogFormatConfig Combined user String
23LogFormatConfig Combined date Date
24LogFormatConfig Combined request String
25LogFormatConfig Combined status Number
26LogFormatConfig Combined bytes_sent Number
27LogFormatConfig Combined referer String
28LogFormatConfig Combined agent String
29
30LogFormat Combined
diff --git a/utility/shell.c b/utility/shell.c
new file mode 100644
index 0000000..07d9da1
--- /dev/null
+++ b/utility/shell.c
@@ -0,0 +1,148 @@
1#include "apr.h"
2#include "apr_getopt.h"
3#include "apr_tables.h"
4
5#define APR_WANT_STDIO
6#include "apr_want.h"
7#include "stdlib.h"
8
9#include "shell.h"
10#include "config.h"
11#include "logparse.h"
12
13const apr_getopt_option_t _opt_config[] = {
14 {"machineid", 'm', 1, "Machine ID for the log file"},
15 {"transaction", 't', 1, "Use a Transaction (yes,no)"},
16 {"logformat", 'r', 1, "Use this logformat to parse files"},
17 {"file", 'f', 1, "Parse this single log file (input dir is NOT scanned)"},
18 {"inputdir", 'd', 1, "Input Directory to look for log files"},
19 {"config", 'c', 1, "Configuration file to use (default mod_log_sql.conf)"},
20 {"dryrun", 'n', 0, "Perform a dry run (do not actually alter the databse)"},
21 {"loglevel", 'l', 1, "Log Level (deubg, warn, error)"},
22 {"summary", 's', 1, "Summary (yes,no)"},
23 {"help", 'h', 0, "Show Help"},
24 {NULL}
25};
26
27void show_help(const char *prog, const apr_getopt_option_t *opts, FILE *output)
28{
29 int ptr = 0;
30 fprintf(output, "Usage: %s [OPTIONS] [files...]\n\n", prog);
31 while (opts[ptr].optch != 0) {
32 if (opts[ptr].optch > 255) {
33 if (opts[ptr].name) {
34 fprintf(output, " --%-10s", opts[ptr].name);
35 } else {
36 fprintf(output, " ");
37 }
38 } else {
39 if (opts[ptr].name) {
40 fprintf(output, " -%c --%-10s", opts[ptr].optch, opts[ptr].name);
41 } else {
42 fprintf(output, " -%c ", opts[ptr].optch);
43 }
44 }
45 if (opts[ptr].has_arg) {
46 fprintf(output, " (arg)");
47 } else {
48 fprintf(output, " ");
49 }
50 fprintf(output, " %s\n", opts[ptr].description);
51 ptr++;
52 }
53}
54
55int main(int argc, const char *const argv[])
56{
57 apr_pool_t *pool;
58 apr_getopt_t *opts;
59 int opt;
60 const char *opt_arg;
61 apr_status_t rv;
62 apr_table_t *args;
63 config_t *base;
64
65 apr_app_initialize(&argc, &argv, NULL);
66 atexit(apr_terminate);
67
68 if (apr_pool_create(&pool, NULL) != APR_SUCCESS) {
69 fprintf(stderr, "Failed to create memory pool!\n");
70 exit(1);
71 }
72
73 /** Iterate over command line arguments
74 * shoving args in a apr_table for processing later*/
75 args = apr_table_make(pool, 5);
76 apr_table_setn(args, "config", "mod_log_sql.conf");
77 apr_getopt_init(&opts, pool, argc, argv);
78 while ((rv = apr_getopt_long(opts, _opt_config, &opt, &opt_arg)) == APR_SUCCESS) {
79 switch (opt) {
80 case 'c':
81 apr_table_setn(args,"config",opt_arg);
82 break;
83 case 'd':
84 apr_table_setn(args,"inputdirectory",opt_arg);
85 break;
86 case 'f':
87 apr_table_setn(args,"inputfile",opt_arg);
88 break;
89 case 'h':
90 show_help(argv[0], _opt_config, stdout);
91 exit(1);
92 break;
93 case 'l':
94 apr_table_setn(args,"loglevel",opt_arg);
95 break;
96 case 'm':
97 apr_table_setn(args,"machineid",opt_arg);
98 break;
99 case 'n':
100 apr_table_setn(args,"dryrun","yes");
101 break;
102 case 'r':
103 apr_table_setn(args,"logformat",opt_arg);
104 break;
105 case 's':
106 apr_table_setn(args,"summary",opt_arg);
107 break;
108 case 't':
109 apr_table_setn(args,"usetransactions",opt_arg);
110 break;
111 }
112 }
113 if (rv != APR_EOF) {
114 show_help(argv[0], _opt_config, stderr);
115 exit(1);
116 }
117 // Check if no extra args were passed
118 if (opts->ind != opts->argc) {
119 show_help(argv[0], _opt_config, stderr);
120 fprintf(stderr, "\n%s: Extra unknown arguments passed\n\n",argv[0]);
121 exit(1);
122 }
123
124 // Process configuration file
125 config_init(pool);
126 base = config_create(pool);
127 rv = config_read(base, apr_table_get(args,"Config"), args);
128 if (APR_STATUS_IS_ENOENT(rv)) {
129 fprintf(stderr,"Could not load configuration file: %s\n",apr_table_get(args,"config"));
130 } else if (rv) {
131 exit(1);
132 }
133 config_dump(base);
134 // Apply overrides from command line
135 find_log_files(base);
136 if (!apr_is_empty_array(base->input_files)) {
137 char **filelist;
138 int f, l;
139 filelist = (char **)base->input_files->elts;
140 for (f=0, l=base->input_files->nelts; f < l; f++) {
141 printf("Scanning %s\n",filelist[f]);
142 parse_logfile(base, filelist[f]);
143 }
144 } else {
145 printf("No input files\n");
146 }
147 return 0;
148}
diff --git a/utility/shell.h b/utility/shell.h
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/utility/shell.h
generated by cgit (git 2.34.1) at 2024-11-21 23:00:58 +0000