1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27 """
28 Log Server,
29 Part of the log which can (optionally) be in a separate process.
30 @author: Ruben Reifenberg
31 """
32
33 from sys import stderr
34 from datetime import datetime
35 from rrlog.tool import mStrftime,ListRotator,traceToShortStr
36
37 EMPTYDICT = {}
38
39
41 """
42 A configured column name is impossible because already reserved.
43 """
44 pass
45
47 """
48 Attempt to add an observer twice into the observer list
49 """
50 pass
51
52
53
54
56 """
57 @ivar clientid: int, id of the client that caused the message. Unique in the server.
58 @ivar msgid: int, id of the message. Unique in the client.
59 @ivar msg: str, msg as created by the client.
60 @ivar ts: str, timestamp
61 @ivar special: dict with custom items (see "special" argument of the log method)
62 @ivar tblen: len of the client traceback when the log method was called
63 @ivar path: client traceback path as sequence of (filename, linenumber). [0] is the latest (where the log call happened)
64 """
65 - def __init__(self,pid,clientid,msgid,msg,ts,special,cat,path,tblen,cfunc,formatter):
66 """
67 @param special: data for custom observers only
68 @param ts: timestamp, str
69 """
70 self.pid = pid
71 self.clientid = clientid
72 self.msgid = msgid
73 self.msg = msg
74 self.ts = ts
75 if special is None: self.special = EMPTYDICT
76 else: self.special = special
77 self.cat = cat
78 self.path = path
79 self.tblen = tblen
80 self._formatter = formatter
81 self.cfunc = cfunc
82 self._iCfn = None
83 for i,(cfn,cln) in enumerate(path):
84 if cfn is not None:
85 self._iCfn = i
86 break
87
89 """
90 @rtype: str
91 Callers File Name
92 """
93 if self._iCfn is not None:
94 return self._formatter.format_fname(self.path[self._iCfn][0])
95 else:
96 return None
97
99 """
100 @rtype: int
101 Callers Line Number
102 """
103 if self._iCfn is not None:
104 return self.path[self._iCfn][1]
105 else:
106 return -1
107
131
132
134 """
135 @param imin: min.index of path to use.
136 Fo example, imin==1 will skip the first path item
137 (Note: the first item is also available separately as cfn,cln)
138 @rtype: str
139 @return: path as formatted str
140 """
141 return self._formatter.pathAsStr(self.path, imin=imin)
142
144 """
145 @return: new instance with my init kwargs but updated with the given kwargs
146 """
147 oldkwargs = {
148 "pid":self.pid,
149 "clientid":self.clientid,
150 "msgid":self.msgid,
151 "msg":self.msg,
152 "ts":self.ts,
153 "special":self.special,
154 "cat":self.cat,
155 "path":self.path,
156 "tblen":self.tblen,
157 "cfunc":self.cfunc,
158 "formatter":self._formatter,
159 }
160 oldkwargs.update(kwargs)
161 return self.__class__(**oldkwargs)
162
163
164
165
166
168 - def __init__(self, configs, writerFactory):
169 assert len(configs)>0
170 self._configs = ListRotator(configs)
171 self._writerFactory = writerFactory
173 """
174 RENAMED from next(), otherwise Python3 conversion failure.
175 @param history: list of old writers to be maintained, oldest at [0]
176 Invalid (expired) writers are removed!
177 """
178 if len(history) >= self._configs.len():
179 history.pop(0)
180 return self._writerFactory(self._configs.next())
184 """
185 Assigned to a list of LogWriters,
186 rotates by creating a new one each time
187 when a line count is exceeded.
188 Maintains a history of old writers.
189 @ivar writers: History of writers, current at [-1], oldest at [0].
190 Only writers with an existing (i.e.not already overwritten) table/file are available.
191 (Migration note: This is analogous to the getWriteHistory() of version 0.1.1 but the order is ascending.)
192 """
193 - def __init__(self, getNextWriter, rotateLineMin):
194 """
195 @param getNextWriter: Callable that takes a list (history) of previous writers, and returns the next logWriter to use
196 @param rotateLineMin: rotate when ~ lines are written
197 """
198 self._rotateLineMin = rotateLineMin
199 self._nextWriter = getNextWriter,
200 self.writers = []
201 self._rotate()
202
203
205 """
206 Not threadsafe.
207 Maintains the history (self.writers).
208 A new writer is appended..
209 removes the oldest writer [0] if self._writer count is longer than self._configs.
210 """
211 writer = self._nextWriter[0](history=self.writers)
212 self.writers.append(writer)
213
214
215
217 """
218 Write without buffering, return when written
219 """
220 if (self._rotateLineMin is not None) and (self.writers[-1].estimateLineCount() >= self._rotateLineMin):
221 self._rotate()
222 self.writers[-1].writeNow(job)
223
224
225
226
227
228
229
231 """
232 This can (but must not be) in the application process.
233 @cvar CAT_INTERNALERROR: Default "I". This is used as "cat" when I write a log message about a logging failure (if possible)
234 """
235 CLIENTID_MAX = 32767
236 CFN_SHORT = 1
237 CFN_FULL = 2
238 CAT_INTERNALERROR = "I"
239
240 - def __init__(self,
241 writer,
242 filters = None,
243 observers = None,
244 cfnMode=1,
245 tsFormat=None,
246 initMsg=False,
247 jobhistSize=100,
248 oie=True,
249 ):
250 """
251 @param cfnMode: One of the MODE...constants.
252 CFN_SHORT: Caller File Names minimal,i.e.file name without directory path(=Default)
253 CFN_FULL: Caller File Names with full file path)
254 Default is CFN_SHORT
255 @param tsFormat: strftime-format string with an extension: use %3N for milliseconds.
256 [Year..second are: %Y/%y,%m,%d,%H,%M,%S]
257 None for no timestamps.
258 For convenience:
259 The String "std1" is interpreted as "%H:%M.%S;%3N".
260 The String "std2" is interpreted as "%m/%d %H:%M.%S".
261 @attention: The logged time stamps can be significantly later than
262 the log() call of your application, because time is taken by the LogServer
263 which may be remote connected.
264 @param observers: list of callables, called with args: jobhist, writer.
265 Called each time after(!) a message was written.
266 writer is the specific writer (depends on DB/file/Stdout modus.)
267 jobhist are the N last message-jobs, with the latests at [-1]. The size N of the jobhist
268 is limited by jobhistSize. Increase jobhistSize if your observer needs to see a large job history.
269 The observers are processed in the given order.
270 You can modify job content for following observers, without affecting the log (since the observers are called after
271 the line is written.)
272 For compatibility: An observer can also have an observe() method. If available, this is used (instead of __call__)
273 @param filters: list of callable objects, analogous to observers.
274 But filters are called before logging. When they modify the message, the change gets visible in the log.
275 @param initMsg: if True, write a short and witless initial message to test the LogWriter
276 @param jobhistSize: The count of recent messages that are available as a list (these may be read by observers). Default=100
277 @raise AssertionError: if an observer is >1 times in the list
278 @raise AssertionError: if a filter is >1 times in the list
279 @param oie: Observe Internal Error; defines how to handle errors occurring in the observers and filters.
280 If True, in case of an internal error in an observer/filter, all observers/filters receive an additional message describing the error.
281 If False, the observers/filters are not notified about the internal error (but the internal error still appears in the main log.)
282 """
283 if filters is None:
284 filters=[]
285 else:
286 assert hasattr(filters,"__iter__"),"filters must be a sequence, not %s"%(type(filters))
287 filters = list(filters)
288
289 if observers is None:
290 observers=[]
291 else:
292 assert hasattr(observers,"__iter__"),"observers must be a sequence, not %s"%(type(observers))
293 observers = list(observers)
294
295
296
297 self._observers = []
298 for i,x in enumerate(observers):
299 if hasattr(x,"observe") and callable(x.observe):
300 self._observers.append(x.observe)
301 else:
302 assert callable(x),"Observer %s must be callable (or need to have the deprecated observe() method)."%(x)
303 self._observers.append(x)
304
305
306 for x in observers:
307 assert observers.count(x)==1, "%s is >1 times in observers list"%(x)
308
309 for i,x in enumerate(filters):
310 assert callable(x),"Filter %s must be callable."%(x)
311 assert filters.count(x)==1, "%s is >1 times in filters list"%(x)
312
313
314 self._time0 = datetime.now()
315 self._observers = list(self._observers)
316 self._filters = filters
317 self._tsFormat = {
318 "std1": "%H:%M.%S;%3N",
319 "std2": "%m/%d %H:%M.%S",
320 }.get(tsFormat, tsFormat)
321 self._cfnMode = cfnMode
322 self._writer = writer
323 self._jobhist = []
324 assert jobhistSize>0, "need at least a history size of 1, not %s"%(jobhistSize)
325 self._jobhistSize = jobhistSize
326 self.oie = oie
327 if initMsg:
328 self._writer.writeNow(MsgJob(
329 clientid=0,
330 msgid=0,
331 cat="",
332 special=None,
333 msg="Generated by rrlog.Log",
334 path=(("LOG",-1),),
335 tblen=0,
336 ts = self._timeStr(self._time0),
337 formatter=self,
338 ))
339 self._lastClientId = 0
340
342 """
343 appends the observer at the end of the observers list
344 """
345 if observer in self._observers:
346 raise ObserverAlreadyAdded("already added: %s"%(observer))
347
348 if hasattr(observer,"observe") and callable(observer.observe):
349 self._observers.append(observer.observe)
350 else:
351 assert callable(observer),"Observer %s must be callable (or need to have the deprecated observe() method)."%(observer)
352 self._observers.append(observer)
353
355 """
356 file name in the configured format (cfnMode).
357 @return: String,men-Readable Callers File Name (None is name was None)
358 @param name:File file name incl. path, or None
359 """
360 if name is None: return None
361 if (self._cfnMode == self.CFN_SHORT):
362 """configured to use minimum name:use file name without directory path """
363 res = (name.split("/"))[-1]
364
365 if len(res) > 3:
366 if res[-3:]==".py":
367 res=res[:-3]
368 else:
369 assert self._cfnMode == self.CFN_FULL, "unknown value for cfn mode:%s"%(self._cfnMode)
370 """full name:use full path as name"""
371 res = name.replace("/","_")
372 res = res.replace(".","-")
373 return res
374
375
377 """
378 @param path: iterable
379 @param imin: min.index of path to use.
380 @return: call path, formatted as str, Empty str if path is empty
381 @rtype: str
382 """
383 res = ""
384 lastWasOmitted = False
385 for i,(cfn,cln) in enumerate(path):
386 if i >= imin:
387 if i==imin:
388 res = "|"
389 separator = ""
390 else:
391 separator = "<-"
392 if cfn is not None:
393
394 res += "%s%s(%s)"%(separator,self.format_fname(cfn),cln)
395 lastWasOmitted = False
396 else:
397
398 if not lastWasOmitted:
399 res += "<-..."
400 lastWasOmitted = True
401 return res
402
403
404
406 """
407 Not thread safe
408 @return: unique client id, >=1. Numbers start with 1 again if CLIENTID_MAX is exceeded.
409 """
410 if self._lastClientId == self.CLIENTID_MAX:
411 self._lastClientId = 0
412 self._lastClientId +=1
413 return self._lastClientId
414
416 """
417 @rtype: str
418 @return: readable time info, based on my _tsFormat
419 """
420 if self._tsFormat is None:
421 return ""
422 else:
423 return mStrftime(dt,self._tsFormat)
424
425
426 - def log(self, pid, clientid, msgid, msg, special, cat, path, tblen, cfunc):
427 """
428 @param msg: ASCII, the log message
429 @param cat: An application-specific category string of length 1.
430 Suggesting convention is "E" for error,"W" for Warning, "I" for internal error...
431 None for no category.
432 @param pid: clients os - process id
433 @param clientid: int, the id that the calling client obtained with registration.
434 @param msgid: int, recommended to be unique for this client (I do not check if unique.)
435 @param path: list, len>=0, each item = (int cfn,str cln)
436 @param cfunc: str,callers function name (additionally to cfn,cln) or None
437 """
438 kwargs = {"pid":pid,"clientid":clientid,"msgid":msgid,"msg":msg,"special":special,"cat":cat,"path":path,"tblen":tblen,"cfunc":cfunc,
439 "formatter":self,"ts":self._timeStr(datetime.now())}
440 if len(self._jobhist) >= self._jobhistSize:
441
442
443 job = self._jobhist.pop(0)
444 job.__init__(**kwargs)
445 else:
446 job = MsgJob(**kwargs)
447 self.logJob(job, self.oie)
448
450
451 self._jobhist.append(job)
452
453 for i,filter_ in enumerate(self._filters):
454 try:
455 filter_(jobhist=self._jobhist, writer=self._writer)
456 except Exception,e:
457 errorjob = job.copy_update({
458 "clientid":job.clientid,
459 "msgid":-1,
460 "msg": "[filter#%d(%s) failure.msgid:%d,raised:%s:::%s]"%(i,filter_,job.msgid,e,traceToShortStr() ),
461 "cat":self.CAT_INTERNALERROR,
462 "ts":"*"+job.ts,
463 })
464 if oie:
465
466 self.logJob(errorjob,False)
467 else:
468 try:
469
470 self._writer.writeNow(errorjob)
471 except Exception,e:
472 stderr.write("Error while reporting another error:%s,%s"%(e,traceToShortStr()))
473
474
475 self._writer.writeNow(job)
476
477 for i,observer in enumerate(self._observers):
478 try:
479 observer(jobhist=self._jobhist, writer=self._writer)
480 except Exception,e:
481 errorjob = job.copy_update({
482 "clientid":job.clientid,
483 "msgid":-1,
484 "msg": "[observer#%d(%s) failure.msgid:%d,raised:%s:::%s]"%(i,observer,job.msgid,e,traceToShortStr(8)),
485 "cat":self.CAT_INTERNALERROR,
486 "ts":"*"+job.ts,
487 })
488 if oie:
489
490 self.logJob(errorjob,False)
491 else:
492 try:
493
494 self._writer.writeNow(errorjob)
495 except Exception,e:
496 stderr.write("Error while reporting another error:%s,%s"%(e,traceToShortStr(8)))
497