1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27 """
28 @summary:
29 DB Writing via SQLAlchemy (SA).
30 @author: Ruben Reifenberg
31 """
32
33 from sqlalchemy import create_engine,MetaData,Column,Table,Integer,String
34 from rrlog import Log,environment
35 from rrlog.server import LogServer,RotateLogWriter,RotateWriterFactory
36 MYSQL_ENGINE = "MyISAM"
37
38 -def createRotatingServer(engineStr, rotateCount, rotateLineMin, tableNamePattern="log_%s", tsFormat=None, filters=None, observers=None, logwriterFactory=None, drop=True, cols=None, format_dict=None):
39 """
40 @param engineStr: SQLAlchemy engine str
41 @param tableNamePattern: table name incl.placeholder for int (rotate number), e.g. "mylog%d"
42 @param filters: default = () See L{rrlog.server.LogServer.__init__}
43 @param observers: default = () See L{rrlog.server.LogServer.__init__}
44 @param tsFormat: timestamp format. See L{rrlog.server.LogServer.__init__}
45 @param rotateCount: int >=1, how many tables to use for rotation
46 @param rotateLineMin: rotate when ~ lines are written. None to switch off rotation.
47 @param logwriterFactory: Creates LogWriter instances (one per db table). If None, the module variable LOGWRITER_CLASS is used.
48 @param drop: if True, drop an eventually existing table. When False, append to an existing one (this is possible with rotateCount==1 only).
49 @raise AssertionError: when drop==False and rotateCount > 1
50 @param cols: Custom column configuration. See L{rrlog.server.dbwriter_sa.DBConfig.__init__}
51 @param format_dict: callable taking a job and returning {colname:fieldcontent}. Ignored if a logwriterFactory is given.
52 """
53 assert rotateCount >= 1, "rotate count is %s but can't be <1."%(rotateCount)
54 assert drop==True or rotateCount==1, "rotate count cannot be >1 when drop is False"
55 if logwriterFactory is None: logwriterFactory = lambda *args,**kwargs:LOGWRITER_CLASS(format_dict=format_dict,*args,**kwargs)
56 elif format_dict is not None: import warnings; warnings.warn("format_dict is ignored since logwriterFactory is already specified")
57 return LogServer(
58 writer = RotateLogWriter(
59 getNextWriter=RotateWriterFactory(
60 configs=[DBConfig(
61 engineStr = engineStr,
62 tablename=tableNamePattern%(i),
63 drop=drop,
64 cols=cols,
65 )
66 for i in range(0,rotateCount)
67 ],
68 writerFactory=logwriterFactory,
69 ).nextWriter,
70 rotateLineMin=rotateLineMin,
71 ),
72 filters = filters,
73 observers = observers,
74 tsFormat = tsFormat,
75 )
76
77 -def createLocalLog(
78 engineStr,
79 rotateCount,
80 rotateLineMin,
81 traceOffset=0,
82 tableNamePattern="log_%s",
83 filters=None,
84 observers=None,
85 tsFormat="std1",
86 stackMax=5,
87 drop=True,
88 catsEnable=None,
89 catsDisable=None,
90 seFilesExclude=None,
91 name=None,
92 cols=None,
93 extractStack=True,
94 ):
95 """
96 @param catsEnable: see L{rrlog.Log.__init__}
97 @param catsDisable: see L{rrlog.Log.__init__}
98 @param seFilesExclude: see L{rrlog.Log.__init__}
99 @param filters: see L{rrlog.server.LogServer.__init__}
100 @param observers: see L{rrlog.server.LogServer.__init__}
101 @param tsFormat: timestamp format. See L{rrlog.server.LogServer.__init__}
102 @param tableNamePattern: table name incl.placeholder for int (rotate number), e.g. "mylog%d"
103 @param rotateCount: int >>1, how many tables to use for rotation
104 @param rotateLineMin: rotate when ~ lines are written. None to switch off rotation.
105 @param stackMax: see L{rrlog.Log.__init__}, default: 5 (==log 5 stack levels.)
106 @param drop: if True, drop an eventually existing table. When False, append to an existing one (this is possible with rotateCount==1 only).
107 @param extractStack: see L{rrlog.Log.__init__}
108 @param cols: Custom column configuration. See L{rrlog.server.dbwriter_sa.DBConfig.__init__}
109 @return: callable log object, ready to use
110 """
111 return Log(
112 server = createRotatingServer(
113 engineStr,
114 rotateCount,
115 rotateLineMin,
116 tableNamePattern,
117 tsFormat=tsFormat,
118 filters=filters,
119 observers=observers,
120 drop=drop,
121 cols=cols,
122 ),
123 traceOffset=traceOffset,
124 stackMax=stackMax,
125 catsEnable=catsEnable,
126 catsDisable=catsDisable,
127 seFilesExclude=seFilesExclude,
128 name=name,
129 extractStack=extractStack,
130 )
131
132
150
157
159
160
161 COLS_DEFAULT = (
162 ("pid", _Coltypes.Integer ),
163 ("clientid", _Coltypes.Integer ),
164 ("msgid", _Coltypes.Integer ),
165 ("ts", _Coltypes.String(32) ),
166 ("cat", _Coltypes.String(1) ),
167 ("msg", _Coltypes.String(512) ),
168 ("cfunc", _Coltypes.String(32) ),
169 ("cfn", _Coltypes.String(32) ),
170 ("cln", _Coltypes.Integer ),
171 ("path", _Coltypes.String(512) ),
172 )
173 - def __init__(self, engineStr, tablename, drop=True, cols=None):
174 """
175 @param engineStr: SQLAlchemy engine str
176 @param tablename: str
177 @param drop: If False, an eventually existing table is not deleted but extended. Only true makes sense with rotation.
178 @param cols: Specifies all log table column names and types.
179 default=None. If None, the DBConfig.COLS_DEFAULT is used.
180 You may use the COLS_DEFAULT as a base for you own column configuration.
181 Required is a 3-tuple of (col-name:str,col-type,kwargs:dict) where
182 col-name is the desired DB column; this name can be used as kwarg in the log() calls later
183 col-type is DBConfig.Integer or DBConfig.String
184 kwargs is optional (can be left out) and contains kwargs for the sqlalchemy Column.
185 Example:
186 To add an own integer column, take the default columns, and add your
187 own pair of (column-name,column-type) like that:
188 cols=DBConfig.COLS_DEFAULT + (("mycolumn",DBConfig.Integer))
189 To define you column as primary key, use
190 cols=DBConfig.COLS_DEFAULT + (("mycolumn",DBConfig.Integer,{"primary_key":True}))
191 """
192 self.drop = drop
193 self.engineStr = engineStr
194 self.tablename = tablename
195 if cols is None: cols = self.__class__.COLS_DEFAULT
196 self.cols = cols
197
198
199
200 -class DBLogWriter(object):
201 """
202 USes SQLAlchemy (sa),
203 Assigned to 1 Table
204 """
205 - def __init__(self, config, format_dict=None):
206 """
207 @param config: DBConfig
208 """
209 db = create_engine(config.engineStr,echo=False)
210
211 if environment.sa_v0_3_x:
212 from sqlalchemy import BoundMetaData
213 metadata = BoundMetaData(db)
214 else:
215 metadata = MetaData(db)
216
217 self._table = Table( config.tablename, metadata,
218 Column("id", Integer, primary_key=True),
219 mysql_engine=MYSQL_ENGINE,
220 *self.createColumns(colsConfig=config.cols)
221 )
222
223 if config.drop:
224 metadata.drop_all()
225 metadata.create_all()
226
227 self._insert = self._table.insert()
228
229
230
231
232 if environment.sa_lt_v0_6_0:
233 self._insert = self._insert.compile()
234
235 self._lineCount = 0
236 if format_dict is not None:
237 self._format_dict=(format_dict,)
238 else:
239 self._format_dict=(self._format_dict,)
240
241
243 """
244 Default formatting method.
245 @rtype: {}
246 @return: {colname:fieldcontent} for a single row
247 """
248 res = default_format_dict(job)
249 res.update(job.special)
250 return res
251
252
253
254
255 - def createColumns(colsConfig):
256 """
257 There is no primary key column; these are content columns only.
258 @return: [] of SQLAlchemy Column that make up my log table
259 @param colsConfig: (col-name:str,col-type,kwargs:dict) where
260 col-type is DBConfig.Integer or DBConfig.String
261 kwargs is optional and contains kwargs for Column() of sqlalchemy
262 example: ("ipadress",DBConfig.String,{"default":"127.0.0.1"})
263 """
264 res = []
265 try:
266 for x in colsConfig:
267 assert len(x) in (2,3)
268 if len(x) == 3:
269 kwargs = x[2]
270 else:
271 kwargs = {}
272 res.append( Column(x[0],x[1],**kwargs) )
273 except Exception,e:
274 print "colsConfig was:%s"%(str(colsConfig))
275 raise
276 return res
277
278
279
280
281
282
283
284
285
286
287 createColumns = staticmethod(createColumns)
288
289
291 """
292 For performance reasons, it is allowed to estimate instead count exactly.
293 (Remark: This implementation is working exactly.)
294 @return: count of already written lines
295 """
296 return self._lineCount
297
298 - def getTable(self):
299 """
300 @rtype: SQLAlchemy Table
301 """
302 return self._table
303
304 - def writeNow(self, job):
305 """
306 Write without buffering, return when written
307 """
308 self._insert.execute(
309 self._format_dict[0](job)
310
311 )
312 self._lineCount += 1
313
314 LOGWRITER_CLASS = DBLogWriter
315