xymon

Check-in [2c198f8f5a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add LOGFETCHSCROLLBACK and 'CURRENT' marking git-svn-id: http://svn.code.sf.net/p/xymon/code/branches/4.3.19@7607 44351d6e-118b-4698-b696-ce33095ecaa4
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | origin/4.3.19
Files: files | file ages | folders
SHA3-256: 2c198f8f5aa29184f048d3796efcf2cd1472e4c879b547294c4b3db2c33d5ced
User & Date: jccleaver@users.sf.net 2015-03-21 14:41:59
Context
2015-03-21
15:00
Add deltacount (lines matching seen since last run) option git-svn-id: http://svn.code.sf.net/p/xymon/code/branches/4.3.19@7608 44351d6e-118b-4698-b696-ce33095ecaa4 check-in: 7dbd406d51 user: jccleaver@users.sf.net tags: trunk, origin/4.3.19
14:41
Add LOGFETCHSCROLLBACK and 'CURRENT' marking git-svn-id: http://svn.code.sf.net/p/xymon/code/branches/4.3.19@7607 44351d6e-118b-4698-b696-ce33095ecaa4 check-in: 2c198f8f5a user: jccleaver@users.sf.net tags: trunk, origin/4.3.19
11:29
Add debug lines in logfetch.c; warn properly on invalid regexes git-svn-id: http://svn.code.sf.net/p/xymon/code/branches/4.3.19@7606 44351d6e-118b-4698-b696-ce33095ecaa4 check-in: 43b52d60d4 user: jccleaver@users.sf.net tags: trunk, origin/4.3.19
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to client/logfetch.c.

35
36
37
38
39
40
41
42

43
44
45
46
47
48
49
...
128
129
130
131
132
133
134
135

136
137
138
139
140
141


142
143
144
145
146
147
148
...
150
151
152
153
154
155
156








157
158
159
160
161
162
163
...
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206



207

208
209
210




211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
...
264
265
266
267
268
269
270

















271
272
273
274
275
276
277
...
286
287
288
289
290
291
292










293
294
295
296
297
298
299
300
301


302
303
304

305
306
307
308
309
310
311
#endif

#include "libxymon.h"

/* Is it ok for these to be hardcoded ? */
#define MAXCHECK   102400   /* When starting, dont look at more than 100 KB of data */
#define MAXMINUTES 30
#define POSCOUNT ((MAXMINUTES / 5) + 1)

#define LINES_AROUND_TRIGGER 5

typedef enum { C_NONE, C_LOG, C_FILE, C_DIR, C_COUNT } checktype_t;

typedef struct logdef_t {
#ifdef _LARGEFILE_SOURCE
	off_t lastpos[POSCOUNT];
................................................................................
	return buf;
}


char *logdata(char *filename, logdef_t *logdef)
{
	static char *buf, *replacement = NULL;
	char *startpos, *fillpos, *triggerstartpos, *triggerendpos;

	FILE *fd;
	struct stat st;
	size_t bytesread, bytesleft;
	int openerr, i, status, triggerlinecount, done;
	char *linepos[2*LINES_AROUND_TRIGGER+1];
	int lpidx;


	regex_t *ignexpr = NULL, *trigexpr = NULL;
#ifdef _LARGEFILE_SOURCE
	off_t bufsz;
#else
	long bufsz;
#endif

................................................................................
	unsigned int triggerptrs_count = 0;

	if (buf) free(buf);
	buf = NULL;

	if (replacement) free(replacement);
	replacement = NULL;









	fd = fileopen(filename, &openerr);
	if (fd == NULL) {
		buf = (char *)malloc(1024 + strlen(filename));
		sprintf(buf, "Cannot open logfile %s : %s\n", filename, strerror(openerr));
		return buf;
	}
................................................................................
		 * Logfile shrank - probably it was rotated.
		 * Start from beginning of file.
		 */
		errprintf("logfetch: File %s shrank from >=%zu to %zu bytes in size. Probably rotated; clearing position state\n", filename, logdef->lastpos[POSCOUNT-1], st.st_size);
		for (i=0; (i < POSCOUNT); i++) logdef->lastpos[i] = 0;
	}

	/* Go to the position we were at POSCOUNT-1 times ago (corresponds to 30 minutes) */
#ifdef _LARGEFILE_SOURCE
	fseeko(fd, logdef->lastpos[POSCOUNT-1], SEEK_SET);
	bufsz = st.st_size - ftello(fd);
	if (bufsz > MAXCHECK) {
		/*
		 * Too much data for us. We have to skip some of the old data.
		 */
		errprintf("logfetch: %s delta %zu bytes exceeds max buffer size %zu; skipping some data\n", filename, bufsz, MAXCHECK);
		logdef->lastpos[POSCOUNT-1] = st.st_size - MAXCHECK;
		fseeko(fd, logdef->lastpos[POSCOUNT-1], SEEK_SET);
		bufsz = st.st_size - ftello(fd);
	}
#else
	fseek(fd, logdef->lastpos[POSCOUNT-1], SEEK_SET);
	bufsz = st.st_size - ftell(fd);
	if (bufsz > MAXCHECK) {
		/*
		 * Too much data for us. We have to skip some of the old data.
		 */
		errprintf("logfetch: %s delta %zu bytes exceeds max buffer size %zu; skipping some data\n", filename, bufsz, MAXCHECK);
		logdef->lastpos[POSCOUNT-1] = st.st_size - MAXCHECK;
		fseek(fd, logdef->lastpos[POSCOUNT-1], SEEK_SET);
		bufsz = st.st_size - ftell(fd);
	}
#endif




	/* Shift position markers one down for the next round */

	for (i=POSCOUNT-1; (i > 0); i--) logdef->lastpos[i] = logdef->lastpos[i-1];
	logdef->lastpos[0] = st.st_size;





	/*
	 * Get our read buffer.
	 *
	 * NB: fgets() need some extra room in the input buffer.
	 *     If it is missing, we will never detect end-of-file 
	 *     because fgets() will read 0 bytes, but having read that
	 *     it still hasnt reached end-of-file status.
	 *     At least, on some platforms (Solaris, FreeBSD).
	 */
	bufsz += 1023;
	startpos = buf = (char *)malloc(bufsz + 1);
	if (buf == NULL) {
		/* Couldnt allocate the buffer */
		return "Out of memory";
	}

	/* Compile the regex patterns */
................................................................................
	 * Discard the ignored lines as we go.
	 * Remember the last trigger line we see.
	 */
	fillpos = buf;
	bytesleft = bufsz;
	done = 0;
	while (!ferror(fd) && (bytesleft > 0) && !done && (fgets_nonull(fillpos, bytesleft, fd) != NULL)) {

















		if (*fillpos == '\0') {
			/*
			 * fgets() can return an empty buffer without flagging
			 * end-of-file. It should not happen anymore now that
			 * we have extended the buffer to have room for the
			 * terminating \0 byte, but if it does then we will
			 * catch it here.
................................................................................
			int i, match = 0;

			for (i=0; ((i < logdef->ignorecount) && !match); i++) {
				match = (regexec(&ignexpr[i], fillpos, 0, NULL, 0) == 0);
				if (match) dbgprintf(" - line matched ignore %d: %s", i, fillpos); // fgets stores the newline in
			}











			if (match) continue;
		}

		linepos[lpidx] = fillpos;

		/* See if this is a trigger line */
		if (logdef->triggercount) {
			int i, match = 0;



			for (i=0; ((i < logdef->triggercount) && !match); i++) {
				match = (regexec(&trigexpr[i], fillpos, 0, NULL, 0) == 0);
				if (match) dbgprintf(" - line matched trigger %d: %s", i, fillpos); // fgets stores the newline in

			}

			if (match) {
				int sidx;
				
				sidx = lpidx - LINES_AROUND_TRIGGER; 
				if (sidx < 0) sidx += (2*LINES_AROUND_TRIGGER + 1);







|
>







 







|
>






>
>







 







>
>
>
>
>
>
>
>







 







|

|






|
|



|






|
|




>
>
>

>



>
>
>
>









|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
|





|


>
>
|


>







35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
...
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
...
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
...
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
...
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
...
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
#endif

#include "libxymon.h"

/* Is it ok for these to be hardcoded ? */
#define MAXCHECK   102400   /* When starting, dont look at more than 100 KB of data */
#define MAXMINUTES 30
#define POSCOUNT ((MAXMINUTES / 5) + 1)		/* 0 = current run */
#define DEFAULTSCROLLBACK (POSCOUNT - 1)	/* How far back to begin processing data, in runs */
#define LINES_AROUND_TRIGGER 5

typedef enum { C_NONE, C_LOG, C_FILE, C_DIR, C_COUNT } checktype_t;

typedef struct logdef_t {
#ifdef _LARGEFILE_SOURCE
	off_t lastpos[POSCOUNT];
................................................................................
	return buf;
}


char *logdata(char *filename, logdef_t *logdef)
{
	static char *buf, *replacement = NULL;
	char *startpos, *fillpos, *triggerstartpos, *triggerendpos, *curpos = NULL;
	char *curpostxt = "<...CURRENT...>\n";
	FILE *fd;
	struct stat st;
	size_t bytesread, bytesleft;
	int openerr, i, status, triggerlinecount, done;
	char *linepos[2*LINES_AROUND_TRIGGER+1];
	int lpidx;
	int scrollback = DEFAULTSCROLLBACK;
	size_t byteslast, bytestocurrent, bytesin = 0;
	regex_t *ignexpr = NULL, *trigexpr = NULL;
#ifdef _LARGEFILE_SOURCE
	off_t bufsz;
#else
	long bufsz;
#endif

................................................................................
	unsigned int triggerptrs_count = 0;

	if (buf) free(buf);
	buf = NULL;

	if (replacement) free(replacement);
	replacement = NULL;

	if (getenv("LOGFETCH_SCROLLBACK") != NULL) {
		int scroll;
		scroll = atoi(getenv("LOGFETCH_SCROLLBACK"));
		/* Don't use DEFAULTSCROLLBACK here in case we change/lower it in the future */
		if (scroll > (POSCOUNT-1)) errprintf("Scrollback requested is greater than maximum saved\n");
		else { dbgprintf("logfetch: setting scrollback to %d\n", scroll); scrollback = scroll; }
	}

	fd = fileopen(filename, &openerr);
	if (fd == NULL) {
		buf = (char *)malloc(1024 + strlen(filename));
		sprintf(buf, "Cannot open logfile %s : %s\n", filename, strerror(openerr));
		return buf;
	}
................................................................................
		 * Logfile shrank - probably it was rotated.
		 * Start from beginning of file.
		 */
		errprintf("logfetch: File %s shrank from >=%zu to %zu bytes in size. Probably rotated; clearing position state\n", filename, logdef->lastpos[POSCOUNT-1], st.st_size);
		for (i=0; (i < POSCOUNT); i++) logdef->lastpos[i] = 0;
	}

	/* Go to the position we were at $SCROLLBACK times ago (default corresponds to 6 (30 minutes when 5m/run)) */
#ifdef _LARGEFILE_SOURCE
	fseeko(fd, logdef->lastpos[scrollback], SEEK_SET);
	bufsz = st.st_size - ftello(fd);
	if (bufsz > MAXCHECK) {
		/*
		 * Too much data for us. We have to skip some of the old data.
		 */
		errprintf("logfetch: %s delta %zu bytes exceeds max buffer size %zu; skipping some data\n", filename, bufsz, MAXCHECK);
		logdef->lastpos[scrollback] = st.st_size - MAXCHECK;
		fseeko(fd, logdef->lastpos[scrollback], SEEK_SET);
		bufsz = st.st_size - ftello(fd);
	}
#else
	fseek(fd, logdef->lastpos[scrollback], SEEK_SET);
	bufsz = st.st_size - ftell(fd);
	if (bufsz > MAXCHECK) {
		/*
		 * Too much data for us. We have to skip some of the old data.
		 */
		errprintf("logfetch: %s delta %zu bytes exceeds max buffer size %zu; skipping some data\n", filename, bufsz, MAXCHECK);
		logdef->lastpos[scrollback] = st.st_size - MAXCHECK;
		fseek(fd, logdef->lastpos[scrollback], SEEK_SET);
		bufsz = st.st_size - ftell(fd);
	}
#endif

	/* Calculate delta between scrollback (where we're starting) and end of last run */
	byteslast = logdef->lastpos[scrollback];

	/* Shift position markers one down for the next round */
	/* lastpos[1] is the previous end location, lastpos[0] will be the end of this run */
	for (i=POSCOUNT-1; (i > 0); i--) logdef->lastpos[i] = logdef->lastpos[i-1];
	logdef->lastpos[0] = st.st_size;

	/* This is where we place our "CURRENT" marker -- lastpos[1] should only be 0 if we're starting from the beginning anyway */
	bytestocurrent = logdef->lastpos[1] - byteslast;
	dbgprintf("logfetch: last position count was %zu, current is %zu, scrollback is %d, bytestocurrent: %zu\n", byteslast, logdef->lastpos[1], scrollback, bytestocurrent);

	/*
	 * Get our read buffer.
	 *
	 * NB: fgets() need some extra room in the input buffer.
	 *     If it is missing, we will never detect end-of-file 
	 *     because fgets() will read 0 bytes, but having read that
	 *     it still hasnt reached end-of-file status.
	 *     At least, on some platforms (Solaris, FreeBSD).
	 */
	bufsz += 1023 + strlen(curpostxt);
	startpos = buf = (char *)malloc(bufsz + 1);
	if (buf == NULL) {
		/* Couldnt allocate the buffer */
		return "Out of memory";
	}

	/* Compile the regex patterns */
................................................................................
	 * Discard the ignored lines as we go.
	 * Remember the last trigger line we see.
	 */
	fillpos = buf;
	bytesleft = bufsz;
	done = 0;
	while (!ferror(fd) && (bytesleft > 0) && !done && (fgets_nonull(fillpos, bytesleft, fd) != NULL)) {
		int force_trigger = 0;

		/* Mark where we left off */
		bytesin += strlen(fillpos);
		if (!curpos && (bytesin >= bytestocurrent)) {
			char *t;

			dbgprintf(" - Last position was %u, found curpos location at %u in current buffer.\n", logdef->lastpos[1], bytesin);
			t = strdup(fillpos);	/* need shuffle about to insert before this line */
			strncpy(fillpos, curpostxt, strlen(curpostxt));		/* add in the CURRENT + \n */
			strncpy(fillpos+strlen(curpostxt), t, strlen(t));	/* add in whatever this line originally was */
			*(fillpos+strlen(curpostxt)+strlen(t)) = '\0';		/* and terminate it */
			xfree(t);						/* free temp */
			curpos = fillpos;					/* leave curpos to the beginning of the CURRENT flag */
			force_trigger = 1;					/* We'll force this to be saved as a trigger later on, so it always gets printed */
		}

		if (*fillpos == '\0') {
			/*
			 * fgets() can return an empty buffer without flagging
			 * end-of-file. It should not happen anymore now that
			 * we have extended the buffer to have room for the
			 * terminating \0 byte, but if it does then we will
			 * catch it here.
................................................................................
			int i, match = 0;

			for (i=0; ((i < logdef->ignorecount) && !match); i++) {
				match = (regexec(&ignexpr[i], fillpos, 0, NULL, 0) == 0);
				if (match) dbgprintf(" - line matched ignore %d: %s", i, fillpos); // fgets stores the newline in
			}

			if (force_trigger) {
				/* Oops. We actually wanted this line poked through */
				/* At the moment, we're only entering this state when we've added 'CURRENT\n' to the existing line */
				/* Since we're guaranteeing to downstream users that we're ignoring this, just truncate the line */
				/* to the first newline. If we start using force_trigger for other purposes, we might have to change */
				/* the logic here. */
				char *eoln;

				eoln = strchr(fillpos, '\n'); if (eoln) *++eoln = '\0';	// still need the final newline
			}
			else if (match) continue;
		}

		linepos[lpidx] = fillpos;

		/* See if this is a trigger line */
		if (force_trigger || logdef->triggercount) {
			int i, match = 0;

			if (force_trigger) { match = 1; dbgprintf(" - line forced as a trigger: %s", fillpos); }
			else {
			    for (i=0; ((i < logdef->triggercount) && !match); i++) {
				match = (regexec(&trigexpr[i], fillpos, 0, NULL, 0) == 0);
				if (match) dbgprintf(" - line matched trigger %d: %s", i, fillpos); // fgets stores the newline in
			    }
			}

			if (match) {
				int sidx;
				
				sidx = lpidx - LINES_AROUND_TRIGGER; 
				if (sidx < 0) sidx += (2*LINES_AROUND_TRIGGER + 1);