nsz Git - libfirm/blob - scripts/statev_sql.py

   1 #! /usr/bin/env python
   2 #
   3 # This file is part of libFirm.
   4 # Copyright (C) 2012 Karlsruhe Institute of Technology.
   5 import sys
   6 import os
   7 import re
   8 import time
   9 import stat
  10 import fileinput
  11 import tempfile
  12 import optparse
  13
  14 class DummyFilter:
  15         def match(self, dummy):
  16                 return True
  17
  18 class EmitBase:
  19         ctx_field_ids = {}
  20         ev_field_ids = {}
  21         types = {}
  22
  23         def create_table(self, cols, name, defaulttype, keytype, extra=""):
  24                 c  = "create table if not exists `%s` (\n" % name
  25                 c += "\t`id` %s\n" % keytype
  26
  27                 for x in cols:
  28                         name = x
  29                         type = self.types[defaulttype]
  30                         if x[0] == '$':
  31                                 name = x[1:]
  32                                 type = self.types["text"]
  33                         elif x[0] == '?':
  34                                 name = x[1:]
  35                                 type = self.types["bool"]
  36
  37                         c += "\t,`%s` %s\n" % (name, type)
  38                 c += extra
  39                 c += ");"
  40                 self.execute(c)
  41
  42 # Abstraction for mysql sql connection and sql syntax
  43 class EmitMysql(EmitBase):
  44         tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR
  45
  46         def execute(self, query, *args):
  47                 #print query + " %s\n" % str(tuple(args))
  48                 self.cursor.execute(query, *args);
  49                 self.conn.commit()
  50
  51         def connect(self, options):
  52                 import MySQLdb
  53
  54                 args = dict()
  55                 if options.password:
  56                         args['passwd'] = options.password
  57                 if not options.host:
  58                         options.host = 'localhost'
  59                 args['user'] = options.user
  60                 args['host'] = options.host
  61                 args['db']   = options.database
  62
  63                 self.conn     = MySQLdb.connect(**args)
  64                 self.options  = options
  65                 self.cursor   = self.conn.cursor()
  66
  67         def __init__(self, options, ctxcols, evcols):
  68                 self.connect(options)
  69
  70                 self.types["text"] = "varchar(80) default null";
  71                 self.types["data"] = "double default null";
  72                 self.types["bool"] = "bool";
  73
  74                 self.ctxtab = options.prefix + "ctx"
  75                 self.evtab  = options.prefix + "ev"
  76
  77                 if not options.update:
  78                         self.execute('drop table if exists `%s`' % self.evtab)
  79                         self.execute('drop table if exists `%s`' % self.ctxtab)
  80
  81                 self.create_table(ctxcols, self.ctxtab, "text", "int auto_increment", extra = ", PRIMARY KEY (`id`)")
  82                 self.create_table(evcols, self.evtab, "data", "int not null", extra = ", INDEX(`id`)")
  83
  84                 keys  = "id, " + ", ".join(evcols)
  85                 marks = ",".join(['%s'] * (len(evcols)+1))
  86                 self.evinsert = "insert into `%s` (%s) values (%s)" % (self.evtab, keys, marks)
  87
  88                 keys  = ", ".join(ctxcols)
  89                 marks = ",".join(['%s'] * len(ctxcols))
  90                 self.ctxinsert = "insert into `%s` (%s) values (%s)" % (self.ctxtab, keys, marks)
  91
  92         def ev(self, curr_id, evitems):
  93                 self.execute(self.evinsert, (curr_id,) + tuple(evitems))
  94
  95         def ctx(self, ctxitems):
  96                 self.execute(self.ctxinsert, tuple(ctxitems))
  97                 self.conn.commit()
  98                 id = self.cursor.lastrowid
  99                 return id
 100
 101         def commit(self):
 102                 self.conn.commit()
 103
 104 # Abstraction for sqlite3 databases and sql syntax
 105 class EmitSqlite3(EmitBase):
 106         def execute(self, query, *args):
 107                 #print query + " %s\n" % str(tuple(args))
 108                 self.cursor.execute(query, *args)
 109
 110         def __init__(self, options, ctxcols, evcols):
 111                 import sqlite3
 112
 113                 if options.database == None:
 114                         print "Have to specify database (file-)name for sqlite"
 115                         sys.exit(1)
 116
 117                 if not options.update:
 118                         if os.path.isfile(options.database):
 119                                 os.unlink(options.database)
 120
 121                 self.conn = sqlite3.connect(options.database)
 122                 self.cursor = self.conn.cursor()
 123
 124                 self.types["data"] = "double"
 125                 self.types["text"] = "text"
 126                 self.types["bool"] = "int"
 127                 self.ctxtab = options.prefix + "ctx"
 128                 self.evtab  = options.prefix + "ev"
 129
 130                 self.create_table(ctxcols, self.ctxtab, "text", "integer primary key")
 131                 self.create_table(evcols, self.evtab, "data", "int")
 132                 self.execute("CREATE INDEX IF NOT EXISTS `%sindex` ON `%s`(id)"
 133                                 % (self.evtab, self.evtab))
 134
 135                 keys  = "id, " + ", ".join(evcols)
 136                 marks = ",".join(["?"] * (len(evcols)+1))
 137                 self.evinsert = "insert into `%s` values (%s)" % (self.evtab, marks)
 138
 139                 keys  = ", ".join(ctxcols)
 140                 marks = ",".join(["?"] * len(ctxcols))
 141                 self.ctxinsert = "insert into `%s` (%s) values (%s)" % (self.ctxtab, keys, marks)
 142
 143                 self.nextid = 0
 144
 145         def ev(self, curr_id, evitems):
 146                 self.execute(self.evinsert, (curr_id,) + tuple(evitems))
 147
 148         def ctx(self, ctxitems):
 149                 curr_id = self.nextid
 150                 self.nextid += 1
 151                 self.execute(self.ctxinsert, tuple(ctxitems))
 152                 self.conn.commit()
 153                 return self.cursor.lastrowid
 154
 155         def commit(self):
 156                 self.conn.commit()
 157
 158 class Conv:
 159         engines = { 'sqlite3': EmitSqlite3, 'mysql': EmitMysql }
 160
 161         # Pass that determines event and context types
 162         def find_heads(self):
 163                 n_ev    = 0
 164                 ctxind  = 0
 165                 evind   = 0
 166                 ctxcols = dict()
 167                 evcols  = dict()
 168                 ctxlist = []
 169                 evlist  = []
 170                 linenr  = 0
 171
 172                 self.valid_keys = set()
 173
 174                 inp = self.input()
 175
 176                 for line in inp:
 177                         linenr += 1
 178                         fields  = line.strip().split(";")
 179                         if fields[0] == 'P':
 180                                 if (len(fields)-1) % 2 != 0:
 181                                         print "%s: Invalid number of fields after 'P'" % linenr
 182
 183                                 for i in range(1,len(fields),2):
 184                                         key = fields[i]
 185                                         if not ctxcols.has_key(key):
 186                                                 ctxcols[key] = ctxind
 187                                                 ctxlist.append(key)
 188                                                 ctxind += 1
 189
 190                         elif fields[0] == 'E':
 191                                 if (len(fields)-1) % 2 != 0:
 192                                         print "%s: Invalid number of fields after 'E'" % linenr
 193
 194                                 self.n_events += 1
 195                                 for i in range(1,len(fields),2):
 196                                         key = fields[i]
 197                                         if not self.filter.match(key):
 198                                                 continue
 199
 200                                         if not evcols.has_key(key):
 201                                                 self.valid_keys.add(key)
 202                                                 evcols[key] = evind
 203                                                 evlist.append(key)
 204                                                 evind += 1
 205
 206                 self.ctxcols = ctxcols
 207                 self.evcols = evcols
 208                 return (ctxlist, evlist)
 209
 210         def input(self):
 211                 return fileinput.FileInput(files=self.files, openhook=fileinput.hook_compressed)
 212
 213         def flush_events(self, id):
 214                 isnull = True
 215                 for e in self.evvals:
 216                         if e != None:
 217                                 isnull = False
 218                                 break
 219                 if isnull:
 220                         return
 221
 222                 self.emit.ev(id, self.evvals)
 223                 self.evvals = [None] * len(self.evvals)
 224
 225         def flush_ctx(self):
 226                 if not self.pushpending:
 227                         return
 228                 self.pushpending = False
 229                 self.curr_id = self.emit.ctx(self.ctxvals)
 230
 231         def fill_tables(self):
 232                 lineno     = 0
 233                 ids        = 0
 234                 self.curr_id = -1
 235                 keystack   = []
 236                 idstack    = []
 237                 curr_event = 0
 238                 last_prec  = -1
 239                 self.pushpending = False
 240                 self.ctxvals = [None] * len(self.ctxcols)
 241                 self.evvals  = [None] * len(self.evcols)
 242
 243                 for line in self.input():
 244                         lineno += 1
 245                         items   = line.strip().split(';')
 246                         op      = items[0]
 247
 248                         # Push context command
 249                         if op == 'P':
 250                                 self.flush_events(self.curr_id)
 251
 252                                 # push the keys
 253                                 for p in range(1,len(items),2):
 254                                         key = items[p]
 255                                         val = items[p+1]
 256
 257                                         keystack.append(key)
 258                                         idstack.append(self.curr_id)
 259
 260                                         keyidx = self.ctxcols[key]
 261                                         assert self.ctxvals[keyidx] == None
 262                                         self.ctxvals[keyidx] = val
 263                                 self.pushpending = True
 264
 265                         # Pop context command
 266                         elif op == 'O':
 267
 268                                 # For now... we could optimize this
 269                                 self.flush_ctx()
 270
 271                                 # We process fields in reverse order to makes O's match the
 272                                 # order of previous P's
 273                                 for p in range(len(items)-1,0,-1):
 274                                         self.flush_events(self.curr_id)
 275
 276                                         popkey  = items[p]
 277                                         key     = keystack.pop()
 278                                         self.curr_id = idstack.pop()
 279
 280                                         if popkey != key:
 281                                                 print "unmatched pop in line %d, push key %s, pop key: %s" % (lineno, key, popkey)
 282
 283                                         keyidx = self.ctxcols[key]
 284                                         assert self.ctxvals[keyidx] != None
 285                                         self.ctxvals[keyidx] = None
 286
 287                         elif op == 'E':
 288                                 curr_event += 1
 289
 290                                 self.flush_ctx()
 291
 292                                 # Show that we make progress
 293                                 if self.verbose:
 294                                         prec = curr_event * 10 / self.n_events
 295                                         if prec > last_prec:
 296                                                 last_prec = prec
 297                                                 print '%10d / %10d' % (curr_event, self.n_events)
 298
 299                                 for p in range(1,len(items),2):
 300                                         key = items[p]
 301                                         if key not in self.evcols:
 302                                                 continue
 303
 304                                         keyidx = self.evcols[key]
 305                                         if self.evvals[keyidx] != None:
 306                                                 self.flush_events(self.curr_id)
 307
 308                                         value = items[p+1]
 309                                         self.evvals[keyidx] = value
 310
 311         def __init__(self):
 312                 parser = optparse.OptionParser('usage: %prog [options]  <event file...>', add_help_option=False)
 313                 parser.add_option("", "--help",                        help="show this help message and exit", action="help")
 314                 parser.add_option("", "--update",     dest="update",   help="update database instead of dropping all existing values", action="store_true", default=False)
 315                 parser.add_option("-v", "--verbose",  dest="verbose",  help="verbose messages",         action="store_true", default=False)
 316                 parser.add_option("-f", "--filter",   dest="filter",   help="regexp to filter event keys", metavar="REGEXP")
 317                 parser.add_option("-u", "--user",     dest="user",     help="user",               metavar="USER")
 318                 parser.add_option("-h", "--host",     dest="host",     help="host",               metavar="HOST")
 319                 parser.add_option("-p", "--password", dest="password", help="password",           metavar="PASSWORD")
 320                 parser.add_option("-D", "--database", dest="database", help="database",           metavar="DB")
 321                 parser.add_option("-e", "--engine",   dest="engine",   help="engine (sqlite3, mysql)",             metavar="ENG", default='sqlite3')
 322                 parser.add_option("-P", "--prefix",   dest="prefix",   help="table prefix",       metavar="PREFIX", default='')
 323                 (options, args) = parser.parse_args()
 324
 325                 self.n_events = 0
 326                 self.stmts    = dict()
 327                 self.verbose  = options.verbose
 328
 329                 if len(args) < 1:
 330                         parser.print_help()
 331                         sys.exit(1)
 332
 333                 self.files  = []
 334                 files       = args
 335
 336                 for file in files:
 337                         if not os.path.isfile(file):
 338                                 print "cannot find input file %s" % (file, )
 339                         else:
 340                                 self.files.append(file)
 341
 342                 if len(self.files) < 1:
 343                         print "no input file to process"
 344                         sys.exit(3)
 345
 346                 if options.filter:
 347                         self.filter = re.compile(options.filter)
 348                 else:
 349                         self.filter = DummyFilter()
 350
 351                 if options.engine in self.engines:
 352                         engine = self.engines[options.engine]
 353                 else:
 354                         print 'engine %s not found' % options.engine
 355                         print 'we offer: %s' % self.engines.keys()
 356                         sys.exit(0)
 357
 358                 if options.verbose:
 359                         print "determining schema..."
 360
 361                 (ctxcols, evcols) = self.find_heads()
 362                 if options.verbose:
 363                         print "context schema:"
 364                         print ctxcols
 365                         print "event schema:"
 366                         print evcols
 367
 368                 self.emit = engine(options, ctxcols, evcols)
 369
 370                 if options.verbose:
 371                         print "filling tables..."
 372                 self.fill_tables()
 373                 if options.verbose:
 374                         print "comitting..."
 375                 self.emit.commit()
 376
 377 if __name__ == "__main__":
 378         Conv()