nsz Git - libfirm/blob - scripts/statev_sql.py

   1 #! /usr/bin/env python
   2
   3 import sys
   4 import os
   5 import re
   6 import time
   7 import stat
   8 import fileinput
   9 import tempfile
  10 import optparse
  11
  12 class DummyFilter:
  13         def match(self, dummy):
  14                 return True
  15
  16 class EmitBase:
  17         ctx_field_ids = {}
  18         ev_field_ids = {}
  19         types = {}
  20
  21         def create_table(self, cols, name, defaulttype, keytype, extra=""):
  22                 c  = "create table if not exists `%s` (\n" % name
  23                 c += "\t`id` %s\n" % keytype
  24
  25                 for x in cols:
  26                         name = x
  27                         type = self.types[defaulttype]
  28                         if x[0] == '$':
  29                                 name = x[1:]
  30                                 type = self.types["text"]
  31                         elif x[0] == '?':
  32                                 name = x[1:]
  33                                 type = self.types["bool"]
  34
  35                         c += "\t,`%s` %s\n" % (name, type)
  36                 c += extra
  37                 c += ");"
  38                 self.execute(c)
  39
  40 # Abstraction for mysql sql connection and sql syntax
  41 class EmitMysql(EmitBase):
  42         tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR
  43
  44         def execute(self, query, *args):
  45                 #print query + " %s\n" % str(tuple(args))
  46                 self.cursor.execute(query, *args);
  47                 self.conn.commit()
  48
  49         def connect(self, options):
  50                 import MySQLdb
  51
  52                 args = dict()
  53                 if options.password:
  54                         args['passwd'] = options.password
  55                 if not options.host:
  56                         options.host = 'localhost'
  57                 args['user'] = options.user
  58                 args['host'] = options.host
  59                 args['db']   = options.database
  60
  61                 self.conn     = MySQLdb.connect(**args)
  62                 self.options  = options
  63                 self.cursor   = self.conn.cursor()
  64
  65         def __init__(self, options, ctxcols, evcols):
  66                 self.connect(options)
  67
  68                 self.types["text"] = "varchar(80) default null";
  69                 self.types["data"] = "double default null";
  70                 self.types["bool"] = "bool";
  71
  72                 self.ctxtab = options.prefix + "ctx"
  73                 self.evtab  = options.prefix + "ev"
  74
  75                 if not options.update:
  76                         self.execute('drop table if exists `%s`' % self.evtab)
  77                         self.execute('drop table if exists `%s`' % self.ctxtab)
  78
  79                 self.create_table(ctxcols, self.ctxtab, "text", "int auto_increment", extra = ", PRIMARY KEY (`id`)")
  80                 self.create_table(evcols, self.evtab, "data", "int not null", extra = ", INDEX(`id`)")
  81
  82                 keys  = "id, " + ", ".join(evcols)
  83                 marks = ",".join(['%s'] * (len(evcols)+1))
  84                 self.evinsert = "insert into `%s` (%s) values (%s)" % (self.evtab, keys, marks)
  85
  86                 keys  = ", ".join(ctxcols)
  87                 marks = ",".join(['%s'] * len(ctxcols))
  88                 self.ctxinsert = "insert into `%s` (%s) values (%s)" % (self.ctxtab, keys, marks)
  89
  90         def ev(self, curr_id, evitems):
  91                 self.execute(self.evinsert, (curr_id,) + tuple(evitems))
  92
  93         def ctx(self, ctxitems):
  94                 self.execute(self.ctxinsert, tuple(ctxitems))
  95                 self.conn.commit()
  96                 id = self.cursor.lastrowid
  97                 return id
  98
  99         def commit(self):
 100                 self.conn.commit()
 101
 102 # Abstraction for sqlite3 databases and sql syntax
 103 class EmitSqlite3(EmitBase):
 104         def execute(self, query, *args):
 105                 #print query + " %s\n" % str(tuple(args))
 106                 self.cursor.execute(query, *args)
 107
 108         def __init__(self, options, ctxcols, evcols):
 109                 import sqlite3
 110
 111                 if options.database == None:
 112                         print "Have to specify database (file-)name for sqlite"
 113                         sys.exit(1)
 114
 115                 if not options.update:
 116                         if os.path.isfile(options.database):
 117                                 os.unlink(options.database)
 118
 119                 self.conn = sqlite3.connect(options.database)
 120                 self.cursor = self.conn.cursor()
 121
 122                 self.types["data"] = "double"
 123                 self.types["text"] = "text"
 124                 self.types["bool"] = "int"
 125                 self.ctxtab = options.prefix + "ctx"
 126                 self.evtab  = options.prefix + "ev"
 127
 128                 self.create_table(ctxcols, self.ctxtab, "text", "integer primary key")
 129                 self.create_table(evcols, self.evtab, "data", "int")
 130                 self.execute("CREATE INDEX IF NOT EXISTS `%sindex` ON `%s`(id)"
 131                                 % (self.evtab, self.evtab))
 132
 133                 keys  = "id, " + ", ".join(evcols)
 134                 marks = ",".join(["?"] * (len(evcols)+1))
 135                 self.evinsert = "insert into `%s` values (%s)" % (self.evtab, marks)
 136
 137                 keys  = ", ".join(ctxcols)
 138                 marks = ",".join(["?"] * len(ctxcols))
 139                 self.ctxinsert = "insert into `%s` (%s) values (%s)" % (self.ctxtab, keys, marks)
 140
 141                 self.nextid = 0
 142
 143         def ev(self, curr_id, evitems):
 144                 self.execute(self.evinsert, (curr_id,) + tuple(evitems))
 145
 146         def ctx(self, ctxitems):
 147                 curr_id = self.nextid
 148                 self.nextid += 1
 149                 self.execute(self.ctxinsert, tuple(ctxitems))
 150                 self.conn.commit()
 151                 return self.cursor.lastrowid
 152
 153         def commit(self):
 154                 self.conn.commit()
 155
 156 class Conv:
 157         engines = { 'sqlite3': EmitSqlite3, 'mysql': EmitMysql }
 158
 159         # Pass that determines event and context types
 160         def find_heads(self):
 161                 n_ev    = 0
 162                 ctxind  = 0
 163                 evind   = 0
 164                 ctxcols = dict()
 165                 evcols  = dict()
 166                 ctxlist = []
 167                 evlist  = []
 168                 linenr  = 0
 169
 170                 self.valid_keys = set()
 171
 172                 inp = self.input()
 173
 174                 for line in inp:
 175                         linenr += 1
 176                         fields  = line.strip().split(";")
 177                         if fields[0] == 'P':
 178                                 if (len(fields)-1) % 2 != 0:
 179                                         print "%s: Invalid number of fields after 'P'" % linenr
 180
 181                                 for i in range(1,len(fields),2):
 182                                         key = fields[i]
 183                                         if not ctxcols.has_key(key):
 184                                                 ctxcols[key] = ctxind
 185                                                 ctxlist.append(key)
 186                                                 ctxind += 1
 187
 188                         elif fields[0] == 'E':
 189                                 if (len(fields)-1) % 2 != 0:
 190                                         print "%s: Invalid number of fields after 'E'" % linenr
 191
 192                                 self.n_events += 1
 193                                 for i in range(1,len(fields),2):
 194                                         key = fields[i]
 195                                         if not self.filter.match(key):
 196                                                 continue
 197
 198                                         if not evcols.has_key(key):
 199                                                 self.valid_keys.add(key)
 200                                                 evcols[key] = evind
 201                                                 evlist.append(key)
 202                                                 evind += 1
 203
 204                 self.ctxcols = ctxcols
 205                 self.evcols = evcols
 206                 return (ctxlist, evlist)
 207
 208         def input(self):
 209                 return fileinput.FileInput(files=self.files, openhook=fileinput.hook_compressed)
 210
 211         def flush_events(self, id):
 212                 isnull = True
 213                 for e in self.evvals:
 214                         if e != None:
 215                                 isnull = False
 216                                 break
 217                 if isnull:
 218                         return
 219
 220                 self.emit.ev(id, self.evvals)
 221                 self.evvals = [None] * len(self.evvals)
 222
 223         def flush_ctx(self):
 224                 if not self.pushpending:
 225                         return
 226                 self.pushpending = False
 227                 self.curr_id = self.emit.ctx(self.ctxvals)
 228
 229         def fill_tables(self):
 230                 lineno     = 0
 231                 ids        = 0
 232                 self.curr_id = -1
 233                 keystack   = []
 234                 idstack    = []
 235                 curr_event = 0
 236                 last_prec  = -1
 237                 self.pushpending = False
 238                 self.ctxvals = [None] * len(self.ctxcols)
 239                 self.evvals  = [None] * len(self.evcols)
 240
 241                 for line in self.input():
 242                         lineno += 1
 243                         items   = line.strip().split(';')
 244                         op      = items[0]
 245
 246                         # Push context command
 247                         if op == 'P':
 248                                 self.flush_events(self.curr_id)
 249
 250                                 # push the keys
 251                                 for p in range(1,len(items),2):
 252                                         key = items[p]
 253                                         val = items[p+1]
 254
 255                                         keystack.append(key)
 256                                         idstack.append(self.curr_id)
 257
 258                                         keyidx = self.ctxcols[key]
 259                                         assert self.ctxvals[keyidx] == None
 260                                         self.ctxvals[keyidx] = val
 261                                 self.pushpending = True
 262
 263                         # Pop context command
 264                         elif op == 'O':
 265
 266                                 # For now... we could optimize this
 267                                 self.flush_ctx()
 268
 269                                 # We process fields in reverse order to makes O's match the
 270                                 # order of previous P's
 271                                 for p in range(len(items)-1,0,-1):
 272                                         self.flush_events(self.curr_id)
 273
 274                                         popkey  = items[p]
 275                                         key     = keystack.pop()
 276                                         self.curr_id = idstack.pop()
 277
 278                                         if popkey != key:
 279                                                 print "unmatched pop in line %d, push key %s, pop key: %s" % (lineno, key, popkey)
 280
 281                                         keyidx = self.ctxcols[key]
 282                                         assert self.ctxvals[keyidx] != None
 283                                         self.ctxvals[keyidx] = None
 284
 285                         elif op == 'E':
 286                                 curr_event += 1
 287
 288                                 self.flush_ctx()
 289
 290                                 # Show that we make progress
 291                                 if self.verbose:
 292                                         prec = curr_event * 10 / self.n_events
 293                                         if prec > last_prec:
 294                                                 last_prec = prec
 295                                                 print '%10d / %10d' % (curr_event, self.n_events)
 296
 297                                 for p in range(1,len(items),2):
 298                                         key = items[p]
 299                                         if key not in self.evcols:
 300                                                 continue
 301
 302                                         keyidx = self.evcols[key]
 303                                         if self.evvals[keyidx] != None:
 304                                                 self.flush_events(self.curr_id)
 305
 306                                         value = items[p+1]
 307                                         self.evvals[keyidx] = value
 308
 309         def __init__(self):
 310                 parser = optparse.OptionParser('usage: %prog [options]  <event file...>', add_help_option=False)
 311                 parser.add_option("", "--help",                        help="show this help message and exit", action="help")
 312                 parser.add_option("", "--update",     dest="update",   help="update database instead of dropping all existing values", action="store_true", default=False)
 313                 parser.add_option("-v", "--verbose",  dest="verbose",  help="verbose messages",         action="store_true", default=False)
 314                 parser.add_option("-f", "--filter",   dest="filter",   help="regexp to filter event keys", metavar="REGEXP")
 315                 parser.add_option("-u", "--user",     dest="user",     help="user",               metavar="USER")
 316                 parser.add_option("-h", "--host",     dest="host",     help="host",               metavar="HOST")
 317                 parser.add_option("-p", "--password", dest="password", help="password",           metavar="PASSWORD")
 318                 parser.add_option("-D", "--database", dest="database", help="database",           metavar="DB")
 319                 parser.add_option("-e", "--engine",   dest="engine",   help="engine (sqlite3, mysql)",             metavar="ENG", default='sqlite3')
 320                 parser.add_option("-P", "--prefix",   dest="prefix",   help="table prefix",       metavar="PREFIX", default='')
 321                 (options, args) = parser.parse_args()
 322
 323                 self.n_events = 0
 324                 self.stmts    = dict()
 325                 self.verbose  = options.verbose
 326
 327                 if len(args) < 1:
 328                         parser.print_help()
 329                         sys.exit(1)
 330
 331                 self.files  = []
 332                 files       = args
 333
 334                 for file in files:
 335                         if not os.path.isfile(file):
 336                                 print "cannot find input file %s" % (file, )
 337                         else:
 338                                 self.files.append(file)
 339
 340                 if len(self.files) < 1:
 341                         print "no input file to process"
 342                         sys.exit(3)
 343
 344                 if options.filter:
 345                         self.filter = re.compile(options.filter)
 346                 else:
 347                         self.filter = DummyFilter()
 348
 349                 if options.engine in self.engines:
 350                         engine = self.engines[options.engine]
 351                 else:
 352                         print 'engine %s not found' % options.engine
 353                         print 'we offer: %s' % self.engines.keys()
 354                         sys.exit(0)
 355
 356                 if options.verbose:
 357                         print "determining schema..."
 358
 359                 (ctxcols, evcols) = self.find_heads()
 360                 if options.verbose:
 361                         print "context schema:"
 362                         print ctxcols
 363                         print "event schema:"
 364                         print evcols
 365
 366                 self.emit = engine(options, ctxcols, evcols)
 367
 368                 if options.verbose:
 369                         print "filling tables..."
 370                 self.fill_tables()
 371                 if options.verbose:
 372                         print "comitting..."
 373                 self.emit.commit()
 374
 375 if __name__ == "__main__":
 376         Conv()