diff options
Diffstat (limited to 'utility/shell.c')
-rw-r--r-- | utility/shell.c | 342 |
1 files changed, 342 insertions, 0 deletions
diff --git a/utility/shell.c b/utility/shell.c new file mode 100644 index 0000000..1b9e890 --- /dev/null +++ b/utility/shell.c | |||
@@ -0,0 +1,342 @@ | |||
1 | #include "apr.h" | ||
2 | #include "apr_getopt.h" | ||
3 | #include "apr_tables.h" | ||
4 | |||
5 | #define APR_WANT_STDIO | ||
6 | #include "apr_want.h" | ||
7 | #include "stdlib.h" | ||
8 | |||
9 | #include "shell.h" | ||
10 | #include "config.h" | ||
11 | #include "logparse.h" | ||
12 | #include "database.h" | ||
13 | #include "util.h" | ||
14 | |||
15 | #if APR_HAS_THREADS | ||
16 | #include "apr_queue.h" | ||
17 | #include "apr_thread_pool.h" | ||
18 | |||
19 | static apr_queue_t *queue; | ||
20 | |||
21 | void run_multithreaded(config_t *cfg); | ||
22 | #endif | ||
23 | |||
24 | void run_singlethreaded(config_t *cfg); | ||
25 | |||
26 | const apr_getopt_option_t _opt_config[] = { | ||
27 | {"machineid", 'm', 1, "Machine ID for the log file"}, | ||
28 | {"transaction", 't', 1, "Use a Transaction (yes,no)"}, | ||
29 | {"logformat", 'r', 1, "Use this logformat to parse files"}, | ||
30 | {"file", 'f', 1, "Parse this single log file (input dir is NOT scanned)"}, | ||
31 | {"inputdir", 'i', 1, "Input Directory to look for log files"}, | ||
32 | {"config", 'c', 1, "Configuration file to use (default mod_log_sql.conf)"}, | ||
33 | {"dryrun", 'n', 0, "Perform a dry run (do not actually alter the databse)"}, | ||
34 | {"dump", 'd', 0, "Dump the configuration after parsing and quit"}, | ||
35 | {"loglevel", 'l', 1, "Log Level (deubg, notice, error)"}, | ||
36 | {"summary", 's', 1, "Summary (yes,no)"}, | ||
37 | {"threadcount", 'p', 1, "Set thread count (a number greater than 0)"}, | ||
38 | {"help", 'h', 0, "Show Help"}, | ||
39 | {NULL} | ||
40 | }; | ||
41 | |||
42 | void show_help(const char *prog, const apr_getopt_option_t *opts, FILE *output) | ||
43 | { | ||
44 | int ptr = 0; | ||
45 | fprintf(output, "Usage: %s [OPTIONS] [files...]\n\n", prog); | ||
46 | while (opts[ptr].optch != 0) { | ||
47 | if (opts[ptr].optch > 255) { | ||
48 | if (opts[ptr].name) { | ||
49 | fprintf(output, " --%-10s", opts[ptr].name); | ||
50 | } else { | ||
51 | fprintf(output, " "); | ||
52 | } | ||
53 | } else { | ||
54 | if (opts[ptr].name) { | ||
55 | fprintf(output, " -%c --%-10s", opts[ptr].optch, opts[ptr].name); | ||
56 | } else { | ||
57 | fprintf(output, " -%c ", opts[ptr].optch); | ||
58 | } | ||
59 | } | ||
60 | if (opts[ptr].has_arg) { | ||
61 | fprintf(output, " (arg)"); | ||
62 | } else { | ||
63 | fprintf(output, " "); | ||
64 | } | ||
65 | fprintf(output, " %s\n", opts[ptr].description); | ||
66 | ptr++; | ||
67 | } | ||
68 | } | ||
69 | |||
70 | void print_summary(config_t *cfg) { | ||
71 | config_filestat_t *fstat; | ||
72 | int i,m; | ||
73 | apr_time_t totaltime = 0; | ||
74 | apr_size_t totalparsed = 0, totalskipped = 0, totalbad = 0; | ||
75 | |||
76 | fstat = (config_filestat_t *)cfg->input_files->elts; | ||
77 | |||
78 | printf("Execution Summary\nParsed %d files\n", cfg->input_files->nelts); | ||
79 | for (i=0, m=cfg->input_files->nelts; i<m; i++) { | ||
80 | totaltime += fstat[i].stop - fstat[i].start; | ||
81 | totalparsed += fstat[i].linesparsed; | ||
82 | totalskipped += fstat[i].lineskipped; | ||
83 | totalbad += fstat[i].linesbad; | ||
84 | printf(" File: %s\n" | ||
85 | " Lines Added %'d out of %'d (Skipped %'d, Bad %'d)\n" | ||
86 | " Status: %s\n" | ||
87 | " Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n" | ||
88 | "\n", | ||
89 | fstat[i].fname, | ||
90 | fstat[i].linesparsed - fstat[i].lineskipped - fstat[i].linesbad, | ||
91 | fstat[i].linesparsed, | ||
92 | fstat[i].lineskipped, | ||
93 | fstat[i].linesbad, | ||
94 | fstat[i].result, | ||
95 | apr_time_sec(fstat[i].stop - fstat[i].start)/60, | ||
96 | apr_time_sec(fstat[i].stop - fstat[i].start) % 60, | ||
97 | apr_time_msec(fstat[i].stop - fstat[i].start) | ||
98 | ); | ||
99 | } | ||
100 | printf("Totals\n" | ||
101 | " Lines Added %'d out of %'d (Skipped %'d, Bad %'d)\n" | ||
102 | " Duration: %02"APR_TIME_T_FMT":%02"APR_TIME_T_FMT".%"APR_TIME_T_FMT" (minutes, seconds, and miliseconds)\n" | ||
103 | "\n", | ||
104 | totalparsed - totalskipped - totalbad, | ||
105 | totalparsed, | ||
106 | totalskipped, | ||
107 | totalbad, | ||
108 | apr_time_sec(totaltime)/60, | ||
109 | apr_time_sec(totaltime) % 60, | ||
110 | apr_time_msec(totaltime) | ||
111 | ); | ||
112 | } | ||
113 | |||
114 | int main(int argc, const char *const argv[]) | ||
115 | { | ||
116 | apr_pool_t *pool, *ptemp; | ||
117 | apr_getopt_t *opts; | ||
118 | int opt; | ||
119 | const char *opt_arg; | ||
120 | apr_status_t rv; | ||
121 | apr_table_t *args; | ||
122 | config_t *cfg; | ||
123 | |||
124 | apr_app_initialize(&argc, &argv, NULL); | ||
125 | atexit(apr_terminate); | ||
126 | |||
127 | if (apr_pool_create(&pool, NULL) != APR_SUCCESS) { | ||
128 | fprintf(stderr, "Failed to create memory pool!\n"); | ||
129 | exit(1); | ||
130 | } | ||
131 | apr_pool_create(&ptemp, NULL); | ||
132 | |||
133 | /** Iterate over command line arguments | ||
134 | * shoving args in a apr_table for processing later*/ | ||
135 | args = apr_table_make(ptemp, 5); | ||
136 | apr_table_setn(args, "config", "mod_log_sql.conf"); | ||
137 | apr_getopt_init(&opts, ptemp, argc, argv); | ||
138 | while ((rv = apr_getopt_long(opts, _opt_config, &opt, &opt_arg)) == APR_SUCCESS) { | ||
139 | switch (opt) { | ||
140 | case 'c': | ||
141 | apr_table_setn(args,"config",opt_arg); | ||
142 | break; | ||
143 | case 'd': | ||
144 | apr_table_setn(args,"dump","yes"); | ||
145 | break; | ||
146 | case 'f': | ||
147 | apr_table_setn(args,"inputfile",opt_arg); | ||
148 | break; | ||
149 | case 'h': | ||
150 | show_help(argv[0], _opt_config, stdout); | ||
151 | exit(1); | ||
152 | break; | ||
153 | case 'i': | ||
154 | apr_table_setn(args,"inputdirectory",opt_arg); | ||
155 | break; | ||
156 | case 'l': | ||
157 | apr_table_setn(args,"loglevel",opt_arg); | ||
158 | break; | ||
159 | case 'm': | ||
160 | apr_table_setn(args,"machineid",opt_arg); | ||
161 | break; | ||
162 | case 'n': | ||
163 | apr_table_setn(args,"dryrun","yes"); | ||
164 | break; | ||
165 | case 'p': | ||
166 | apr_table_setn(args,"threadcount",opt_arg); | ||
167 | break; | ||
168 | case 'r': | ||
169 | apr_table_setn(args,"logformat",opt_arg); | ||
170 | break; | ||
171 | case 's': | ||
172 | apr_table_setn(args,"summary",opt_arg); | ||
173 | break; | ||
174 | case 't': | ||
175 | apr_table_setn(args,"usetransactions",opt_arg); | ||
176 | break; | ||
177 | } | ||
178 | } | ||
179 | if (rv != APR_EOF) { | ||
180 | show_help(argv[0], _opt_config, stderr); | ||
181 | exit(1); | ||
182 | } | ||
183 | // Check if no extra args were passed | ||
184 | if (opts->ind != opts->argc) { | ||
185 | show_help(argv[0], _opt_config, stderr); | ||
186 | fprintf(stderr, "\n%s: Extra unknown arguments passed\n\n",argv[0]); | ||
187 | exit(1); | ||
188 | } | ||
189 | |||
190 | // Initialize sub systems | ||
191 | parser_init(pool); | ||
192 | config_init(pool); | ||
193 | database_init(pool); | ||
194 | // Process configuration file | ||
195 | cfg = config_create(pool); | ||
196 | // initialize STD out error log | ||
197 | logging_preinit(cfg); | ||
198 | rv = config_read(cfg, apr_table_get(args,"Config"), args); | ||
199 | apr_pool_destroy(ptemp); | ||
200 | |||
201 | // Initialize Log system AFTER we parse the configuration | ||
202 | logging_init(cfg); | ||
203 | |||
204 | if (APR_STATUS_IS_ENOENT(rv)) { | ||
205 | logging_log(cfg,LOGLEVEL_NOISE,"Could not load configuration file: %s",apr_table_get(args,"config")); | ||
206 | } else if (rv) { | ||
207 | exit(1); | ||
208 | } | ||
209 | if (cfg->dump) { | ||
210 | config_dump(cfg); | ||
211 | exit(0); | ||
212 | } | ||
213 | |||
214 | if (config_check(cfg)) { | ||
215 | logging_log(cfg,LOGLEVEL_NOISE, "Please correct the configuration"); | ||
216 | exit(1); | ||
217 | } | ||
218 | |||
219 | // Only Find files IF no filename was passed via the command line | ||
220 | if (apr_is_empty_array(cfg->input_files)) { | ||
221 | parser_find_logs(cfg); | ||
222 | } | ||
223 | if (!apr_is_empty_array(cfg->input_files)) { | ||
224 | parser_split_logs(cfg); | ||
225 | #if APR_HAS_THREADS | ||
226 | if (cfg->thread_count > 0) { | ||
227 | run_multithreaded(cfg); | ||
228 | } else { | ||
229 | #endif | ||
230 | run_singlethreaded(cfg); | ||
231 | #if APR_HAS_THREADS | ||
232 | } | ||
233 | #endif | ||
234 | } else { | ||
235 | logging_log(cfg,LOGLEVEL_NOISE,"No log files found to parse"); | ||
236 | } | ||
237 | |||
238 | if (cfg->summary) { | ||
239 | print_summary(cfg); | ||
240 | } | ||
241 | return 0; | ||
242 | } | ||
243 | |||
244 | void run_singlethreaded(config_t *cfg) | ||
245 | { | ||
246 | config_filestat_t *filelist; | ||
247 | config_dbd_t *dbconn = NULL; | ||
248 | int f, l; | ||
249 | apr_status_t rv; | ||
250 | |||
251 | if (!cfg->dryrun) { | ||
252 | if ((rv = database_connect(cfg, &dbconn))) { | ||
253 | logging_log(cfg,LOGLEVEL_NOISE, "Error Connecting to Database"); | ||
254 | exit(1); | ||
255 | } | ||
256 | } | ||
257 | |||
258 | filelist = (config_filestat_t *)cfg->input_files->elts; | ||
259 | for (f=0, l=cfg->input_files->nelts; f < l; f++) { | ||
260 | rv = parser_parsefile(cfg, dbconn, &filelist[f]); | ||
261 | if (rv) { | ||
262 | logging_log(cfg, LOGLEVEL_NOISE, | ||
263 | "Error occured parsing log files. Aborting"); | ||
264 | break; | ||
265 | } | ||
266 | } | ||
267 | |||
268 | if (!cfg->dryrun) { | ||
269 | database_disconnect(dbconn); | ||
270 | } | ||
271 | } | ||
272 | |||
273 | #if APR_HAS_THREADS | ||
274 | void * APR_THREAD_FUNC run_filethread(apr_thread_t *thd, void *data) | ||
275 | { | ||
276 | config_t *cfg = data; | ||
277 | config_dbd_t *dbconn = NULL; | ||
278 | config_filestat_t *fileentry; | ||
279 | apr_status_t rv; | ||
280 | |||
281 | if (!cfg->dryrun) { | ||
282 | if ((rv = database_connect(cfg, &dbconn))) { | ||
283 | logging_log(cfg,LOGLEVEL_NOISE, "Error Connecting to Database"); | ||
284 | return NULL; | ||
285 | } | ||
286 | } | ||
287 | |||
288 | while (1) { | ||
289 | rv = apr_queue_pop(queue, (void **)&fileentry); | ||
290 | if (rv == APR_EINTR) | ||
291 | continue; | ||
292 | if (rv == APR_EOF) | ||
293 | break; | ||
294 | rv = parser_parsefile(cfg, dbconn, fileentry); | ||
295 | if (rv) { | ||
296 | logging_log(cfg, LOGLEVEL_NOISE, | ||
297 | "Error occured parsing log file %s", fileentry->fname); | ||
298 | } | ||
299 | } | ||
300 | |||
301 | if (!cfg->dryrun) { | ||
302 | database_disconnect(dbconn); | ||
303 | } | ||
304 | return NULL; | ||
305 | } | ||
306 | |||
307 | void run_multithreaded(config_t *cfg) | ||
308 | { | ||
309 | logging_log(cfg, LOGLEVEL_NOISE, "Running Multithreaded"); | ||
310 | |||
311 | config_filestat_t *filelist; | ||
312 | int f, l; | ||
313 | apr_status_t rv; | ||
314 | apr_pool_t *tp; | ||
315 | apr_thread_pool_t *thrp; | ||
316 | unsigned int count; | ||
317 | |||
318 | apr_pool_create(&tp, cfg->pool); | ||
319 | rv = apr_queue_create(&queue, cfg->input_files->nelts, tp); | ||
320 | |||
321 | rv = apr_thread_pool_create(&thrp, 0, cfg->thread_count, tp); | ||
322 | |||
323 | //populate queue | ||
324 | filelist = (config_filestat_t *)cfg->input_files->elts; | ||
325 | for (f=0, l=cfg->input_files->nelts; f < l; f++) { | ||
326 | rv = apr_queue_push(queue, &filelist[f]); | ||
327 | } | ||
328 | // populate the worker threads | ||
329 | for (f=0; f<cfg->thread_count; f++) { | ||
330 | rv = apr_thread_pool_push(thrp, run_filethread, cfg, 0, NULL); | ||
331 | } | ||
332 | |||
333 | do { | ||
334 | apr_sleep(apr_time_from_sec(1)); | ||
335 | count = apr_queue_size(queue); | ||
336 | } while (count > 0); | ||
337 | |||
338 | rv = apr_queue_term(queue); | ||
339 | |||
340 | rv = apr_thread_pool_destroy(thrp); | ||
341 | } | ||
342 | #endif | ||