Source Code for Module rrlog.contrib.catarch_dbsa

  1  #Copyright (c) 2007 
  2  #        Ruben Reifenberg, Germany, 07381. 
  3  #    All rights reserved. 
  4  # 
  5  #Redistribution and use in source and binary forms, with or without 
  6  #modification, are permitted provided that the following conditions 
  7  #are met: 
  8  #1. Redistributions of source code must retain the above copyright 
  9  #   notice, this list of conditions and the following disclaimer as 
 10  #   the first lines of this file unmodified. 
 11  #2. Redistributions in binary form must reproduce the above copyright 
 12  #   notice, this list of conditions and the following disclaimer in the 
 13  #   documentation and/or other materials provided with the distribution. 
 14  # 
 15  # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 
 16  # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
 17  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
 18  # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 
 19  # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 
 20  # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 
 21  # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
 22  # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 
 23  # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 
 24  # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 
 25  # SUCH DAMAGE. 
 26   
 27  """ 
 28  @summary: archives selected messages into own tables. 
 29  "Selected messages" are messages with certain defined "cat" values. 
 30  @author: Ruben Reifenberg 
 31  """ 
 32   
 33  from sqlalchemy import * 
 34  from rrlog import environment 
 35  from rrlog.tool import traceToShortStr 
 36  from rrlog.server import ColumnConfigurationMismatch 
 37  from rrlog.server.dbwriter_sa import DBConfig,MYSQL_ENGINE,createRotatingServer,default_format_dict 
 38  from rrlog.contrib.catarch import CatArchiver 
 39   
 40   
 41 -def _format_dict(job): 
 42          """ 
 43          Format the row fields for writing. 
 44          @return:{colname:fieldcontent} as to be written into the database row. 
 45          """ 
 46          res = default_format_dict(job) 
 47          res["trigger_id"] = job.trigger_id 
 48          return res 
 49   
 50   
 51   
 52 -def createCatArchiver( 
 53                  engineStr, 
 54                  tableNamePattern="log_archive_%s", 
 55                  metatableName="log_archive_meta", 
 56                  minArchCount=10, 
 57                  minClientArchCount=1, 
 58                  maxCount=10000, 
 59                  problemCats=None, # DEPRECATED NAME. Use cats instead. 
 60                  cats=None, # Message Categories that we want to archive 
 61                  drop=True, 
 62                  cols=None, 
 63                  observers=None, 
 64                  filters=None, 
 65                  ): 
 66          """ 
 67          See L{rrlog.contrib.catarch.CatArchiver.__init__} for specification of the parameters. 
 68          @param problemCats: DEPRECATED, use cats instead. 
 69          @param cols: Custom column configuration. See L{rrlog.server.dbwriter_sa.DBConfig.__init__} 
 70          """ 
 71          if problemCats is not None: 
 72                  assert cats is None, "use cats parameter only. problemCats (deprecated) cannot be used same time." 
 73                  import warnings 
 74                  warnings.warn("'problemCats' is renamed into 'cats'",DeprecationWarning) 
 75                  cats = problemCats 
 76   
 77          if cols is None: 
 78                  cols = ( 
 79                          ("trigger_id", DBConfig.Integer), 
 80                          ("id", DBConfig.Integer ,{"primary_key":True}), 
 81                          ) + DBConfig.COLS_DEFAULT 
 82          else: 
 83                  for (name,type) in cols: 
 84                          if name in ("trigger_id","id"): 
 85                                  raise ColumnConfigurationMismatch("Column name '%s' is reserved by the cat archiver"%(name)) 
 86   
 87          return MetatableCatArchiver( 
 88                  server = createRotatingServer( 
 89                          engineStr=engineStr, 
 90                          rotateCount=1, 
 91                          rotateLineMin=None, 
 92                          tableNamePattern=tableNamePattern, 
 93                          #logwriterFactory=_DBSALogWriter, # provides additional columns 
 94                          drop=drop, 
 95                          cols=cols, 
 96                          format_dict=_format_dict, 
 97                          observers=observers, 
 98                          filters=filters, 
 99                          ), 
100                  minArchCount=minArchCount, 
101                  minClientArchCount=minClientArchCount, 
102                  maxCount=maxCount, 
103                  cats=cats, 
104                  metatableName=metatableName, 
105                  metaEngineStr=engineStr, 
106                  metatableOverwrite=drop, 
107                  ) 
108   
109   
110   
111   
112   
113 -class MetatableCatArchiver(CatArchiver): 
114          """ 
115          Saves interesting log lines (i.e.rescues them from log rotation.) 
116          @attention: Not thread safe 
117          @attention: There is a maximum number of problems saved (see __init__ arguments). 
118          """ 
119 -        def __init__(self, 
120                  server, 
121                  minArchCount, 
122                  minClientArchCount, 
123                  cats, 
124                  maxCount, 
125                  metaEngineStr=None, 
126                  metatableName=None, 
127                  metatableOverwrite=True, 
128                  ): 
129                  """ 
130                  The metatable is a secondary archive table with exactly one line per triggering msg (i.e.no 
131                  preceding messages), an additional id (unique per triggering message) and less data columns. 
132                  @param metatableOverwrite: Drop meta table if existing 
133                  @param metaEngineStr: SQLAlchemy engine string for meta table 
134                  @param minArchCount: >=1,default: 10. This is the count of messages that are at least archived 
135                  along with any problem message (the problem message itself included.) 
136                  Note 1: When a problem is archived shortly after another, no messages are saved twice even if the "preceding messages" intervals overlap. 
137                  Note 2: The effective number of archived lines is limited by the cached jobs (job history) of the server. See LogServer parameters. 
138                  @param minClientArchCount: >=1, default: 1. This is analogous but addresses messages 
139                  with the same client id as the "trigger message". 
140                  The intention is that we often want to save more preceding messages that concern the "problematic" client only. 
141                  Note: Unlike minArchCount, messages are archived twice when the "preceding messages" intervals overlap, i.e. when problems are logged shortly after each other. 
142                  @param cats: list of cat; I archive log entries when cat is in this list.Default is "E","I","S" 
143                  (These are speaking keys that mean: 
144                  Error,InternalError,SecurityIssue. 
145                  Adapt it to the convention that your application uses.) 
146                  @param maxCount: max. problem count (== max size of the meta table). Subsequent problems are ignored. 
147                  """ 
148                  CatArchiver.__init__( 
149                          self, 
150                          server=server, 
151                          minArchCount=minArchCount, 
152                          minClientArchCount=minClientArchCount, 
153                          cats=cats, 
154                          maxCount=maxCount, 
155                          ) 
156   
157                  if metatableName is not None: 
158                          assert metaEngineStr is not None, "when name of meta table is given, metaEngineStr is required." 
159                          db = create_engine(metaEngineStr,echo=False) 
160   
161                          if environment.sa_v0_3_x: 
162                                  metadata = BoundMetaData(db) 
163                          else: 
164                                  metadata = MetaData(db) 
165                          #1 line per incoming error msg: 
166                          tMeta = Table(metatableName, metadata, 
167                                  Column("trigger_id", Integer, nullable=False),#not unique when local logging with multiple processes. 
168                                  Column("clientid", Integer, nullable=False), # client who caused archiving 
169                                  Column("msgid", Integer, nullable=False), # msg which caused archiving 
170                                  mysql_engine=MYSQL_ENGINE, 
171                                  ) 
172                          self._insertmeta = tMeta.insert().compile() 
173                          if metatableOverwrite: 
174                                  metadata.drop_all() 
175                          metadata.create_all() 
176                  else: 
177                          tMeta = None 
178                          self._insertmeta = None 
179                  self._tMeta = tMeta # for unittests 
180   
181   
182 -        def _writeMeta(self,trigger_id,clientid,msgid): 
183                  if self._insertmeta is not None: 
184                          # write single line in problem-meta table: 
185                          try: 
186                                  self._insertmeta.execute( 
187                                          trigger_id=trigger_id, 
188                                          clientid=clientid, 
189                                          msgid=msgid, 
190                                          ) 
191                          except Exception,e: 
192                                  raise Exception( 
193                                          "%s/%s, MetatableCatArchiver Data:problem%s,process%s,client%s,msg%s"%( 
194                                                  e.__class__.__name__, 
195                                                  traceToShortStr(maxLines=2,use_cr=False), 
196                                                  trigger_id, 
197                                                  process_id, 
198                                                  clientid, 
199                                                  msgid, 
200                                                  )) 
201