nsz Git - libfirm/blob - scripts/statev_sql.py

   1 #! /usr/bin/env python
   2
   3 import sys
   4 import os
   5 import re
   6 import time
   7 import stat
   8 import fileinput
   9 import tempfile
  10 import optparse
  11
  12 class DummyFilter:
  13         def match(self, dummy):
  14                 return True
  15
  16 class EmitBase:
  17         def create_table(self, cols, name, type, unique):
  18                 create = 'create table if not exists %s (id int %s' % (name, unique)
  19
  20                 sorted = [None] * len(cols)
  21                 for x in cols.iterkeys():
  22                         sorted[cols[x]] = x
  23                 for x in sorted:
  24                         create += (", `%s` %s" % (x, type))
  25                 create += ');'
  26                 return create
  27
  28 class EmitMysqlInfile(EmitBase):
  29         tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR
  30
  31         def execute(self, query):
  32                 c = self.conn.cursor()
  33                 print query + "\n";
  34                 c.execute(query);
  35                 self.conn.commit()
  36
  37         def __init__(self, options, tables, ctxcols, evcols):
  38                 import MySQLdb
  39
  40                 args = dict()
  41                 if options.password:
  42                         args['passwd'] = options.password
  43                 if not options.host:
  44                         options.host = 'localhost'
  45                 args['user'] = options.user
  46                 args['host'] = options.host
  47                 args['db']   = options.database
  48
  49                 self.conn     = MySQLdb.connect(**args)
  50                 self.ctxcols  = ctxcols
  51                 self.evcols   = evcols
  52                 self.options  = options
  53                 self.ctxtab   = tables['ctx']
  54                 self.evtab    = tables['ev']
  55
  56                 params = (tempfile.gettempdir(), os.sep, os.getpid())
  57                 self.evfifo  = '%s%sstatev_ev_%d' % params
  58                 self.ctxfifo = '%s%sstatev_ctx_%d' % params
  59
  60                 os.mkfifo(self.evfifo)
  61                 os.mkfifo(self.ctxfifo)
  62
  63                 os.chmod(self.evfifo,  self.tmpfile_mode)
  64                 os.chmod(self.ctxfifo, self.tmpfile_mode)
  65
  66                 self.execute('drop table if exists ' + self.evtab)
  67                 self.execute('drop table if exists ' + self.ctxtab)
  68                 table_ctx = self.create_table(self.ctxcols, self.ctxtab, 'char(80)', '')
  69                 self.execute(table_ctx)
  70                 table_ev = self.create_table(self.evcols, self.evtab, 'double default null', '')
  71                 self.execute(table_ev)
  72
  73                 if options.verbose:
  74                         print 'go for gold'
  75
  76                 n = max(len(ctxcols), len(evcols)) + 1
  77                 q = []
  78                 self.quests = []
  79                 for i in xrange(0, n):
  80                         self.quests.append(','.join(q))
  81                         q.append('%s')
  82
  83         def ev(self, curr_id, evitems):
  84                 keys = ""
  85                 first = True
  86                 for key in evitems.keys():
  87                         if first:
  88                                 first = False
  89                         else:
  90                                 keys += ", "
  91                         keys += "`%s`" % (key)
  92
  93                 stmt = "insert into `%s` (id, %s) values (%s)" % (self.evtab, keys, self.quests[len(evitems)+1])
  94                 c = self.conn.cursor()
  95                 c.execute(stmt, (curr_id,) + tuple(evitems.values()))
  96
  97         def ctx(self, curr_id, ctxitems):
  98                 keys = ""
  99                 first = True
 100                 for key in ctxitems.keys():
 101                         if first:
 102                                 first = False
 103                         else:
 104                                 keys += ", "
 105                         keys += "`%s`" % (key)
 106
 107                 stmt = "insert into `%s` (id, %s) values (%s)" % (self.ctxtab, keys, self.quests[len(ctxitems)+1])
 108                 c = self.conn.cursor()
 109                 c.execute(stmt, (curr_id,) + tuple(ctxitems.values()))
 110
 111         def commit(self):
 112                 self.conn.commit()
 113
 114 class EmitSqlite3(EmitBase):
 115         def __init__(self, options, tables, ctxcols, evcols):
 116                 import sqlite3
 117
 118                 if options.database == None:
 119                         print "Have to specify database (file-)name for sqlite"
 120                         sys.exit(1)
 121
 122                 if os.path.isfile(options.database):
 123                         os.unlink(options.database)
 124
 125                 self.ctxtab = tables['ctx']
 126                 self.evtab  = tables['ev']
 127                 self.conn = sqlite3.connect(options.database)
 128                 table_ctx = self.create_table(ctxcols, self.ctxtab, 'text', 'unique')
 129                 self.conn.execute(table_ctx)
 130                 self.conn.execute("CREATE INDEX IF NOT EXISTS ctxindex ON ctx(id)")
 131                 table_ev = self.create_table(evcols, self.evtab, 'double', '')
 132                 self.conn.execute(table_ev)
 133                 self.conn.execute("CREATE INDEX IF NOT EXISTS evindex ON ev(id)")
 134
 135                 n = max(len(ctxcols), len(evcols)) + 1
 136                 q = ['?']
 137                 self.quests = []
 138                 for i in xrange(0, n):
 139                         self.quests.append(','.join(q))
 140                         q.append('?')
 141
 142         def ev(self, curr_id, evitems):
 143                 keys = ""
 144                 first = True
 145                 for key in evitems.keys():
 146                         if first:
 147                                 first = False
 148                         else:
 149                                 keys += ", "
 150                         keys += "`%s`" % (key)
 151
 152                 stmt = "insert into `%s` (id, %s) values (%s)" % (self.evtab, keys, self.quests[len(evitems)])
 153                 self.conn.execute(stmt, (curr_id,) + tuple(evitems.values()))
 154
 155         def ctx(self, curr_id, ctxitems):
 156                 keys = ""
 157                 first = True
 158                 for key in ctxitems.keys():
 159                         if first:
 160                                 first = False
 161                         else:
 162                                 keys += ", "
 163                         keys += "`%s`" % (key)
 164
 165                 stmt = "insert into `%s` (id, %s) values (%s)" % (self.ctxtab, keys, self.quests[len(ctxitems)])
 166                 self.conn.execute(stmt, (curr_id,) + tuple(ctxitems.values()))
 167
 168         def commit(self):
 169                 self.conn.commit()
 170
 171 class Conv:
 172         engines = { 'sqlite3': EmitSqlite3, 'mysql': EmitMysqlInfile }
 173         def find_heads(self):
 174                 n_ev    = 0
 175                 ctxind  = 0
 176                 evind   = 0
 177                 ctxcols = dict()
 178                 evcols  = dict()
 179
 180                 self.valid_keys = set()
 181
 182                 inp = self.input()
 183
 184                 for line in inp:
 185                         if line[0] == 'P':
 186                                 ind = line.index(';', 2)
 187                                 key = line[2:ind]
 188                                 if not ctxcols.has_key(key):
 189                                         ctxcols[key] = ctxind
 190                                         ctxind += 1
 191
 192                         elif line[0] == 'E':
 193                                 ind = line.index(';', 2)
 194                                 key = line[2:ind]
 195                                 if self.filter.match(key):
 196                                         self.n_events += 1
 197                                         if not evcols.has_key(key):
 198                                                 self.valid_keys.add(key)
 199                                                 evcols[key] = evind
 200                                                 evind += 1
 201
 202                 return (ctxcols, evcols)
 203
 204         def input(self):
 205                 return fileinput.FileInput(files=self.files, openhook=fileinput.hook_compressed)
 206
 207         def fill_tables(self):
 208                 lineno     = 0
 209                 ids        = 0
 210                 curr_id    = 0
 211                 last_push_curr_id = 0
 212                 keystack   = []
 213                 idstack    = []
 214                 curr_event = 0
 215                 last_prec  = -1
 216                 evcols     = dict()
 217                 ctxcols    = dict()
 218
 219                 for line in self.input():
 220                         lineno += 1
 221                         items = line.strip().split(';')
 222                         op    = items[0]
 223
 224                         if op == 'P':
 225                                 # flush the current events
 226                                 if len(evcols):
 227                                         self.emit.ev(last_push_curr_id, evcols)
 228                                         evcols.clear()
 229
 230                                 # push the key
 231                                 key   = items[1]
 232                                 val   = items[2]
 233                                 keystack.append(key)
 234                                 curr_id = ids
 235                                 last_push_curr_id = curr_id
 236                                 ids += 1
 237                                 idstack.append(curr_id)
 238                                 ctxcols[key] = val
 239
 240                                 self.emit.ctx(curr_id, ctxcols)
 241
 242                         elif op == 'O':
 243                                 popkey = items[1]
 244                                 key = keystack.pop()
 245
 246                                 if popkey != key:
 247                                         print "unmatched pop in line %d, push key %s, pop key: %s" % (lineno, key, popkey)
 248
 249                                 idstack.pop()
 250                                 if len(idstack) > 0:
 251                                         if len(evcols) > 0:
 252                                                 self.emit.ev(curr_id, evcols)
 253                                                 evcols.clear()
 254                                         del ctxcols[key]
 255                                         curr_id = idstack[-1]
 256                                 else:
 257                                         curr_id = -1
 258
 259                         elif op == 'E':
 260                                 key = items[1]
 261                                 if key in self.valid_keys:
 262                                         curr_event += 1
 263                                         evcols[key] = items[2]
 264
 265                                         if self.verbose:
 266                                                 prec = curr_event * 10 / self.n_events
 267                                                 if prec > last_prec:
 268                                                         last_prec = prec
 269                                                         print '%10d / %10d' % (curr_event, self.n_events)
 270
 271         def __init__(self):
 272                 parser = optparse.OptionParser('usage: %prog [options]  <event file...>')
 273                 parser.add_option("-c", "--clean",    dest="clean",    help="delete tables in advance", action="store_true", default=False)
 274                 parser.add_option("-v", "--verbose",  dest="verbose",  help="verbose messages",         action="store_true", default=False)
 275                 parser.add_option("-f", "--filter",   dest="filter",   help="regexp to filter event keys", metavar="REGEXP")
 276                 parser.add_option("-u", "--user",     dest="user",     help="user",               metavar="USER")
 277                 parser.add_option("-H", "--host",     dest="host",     help="host",               metavar="HOST")
 278                 parser.add_option("-p", "--password", dest="password", help="password",           metavar="PASSWORD")
 279                 parser.add_option("-d", "--db",       dest="database", help="database",           metavar="DB")
 280                 parser.add_option("-e", "--engine",   dest="engine",   help="engine",             metavar="ENG", default='sqlite3')
 281                 parser.add_option("-P", "--prefix",   dest="prefix",   help="table prefix",       metavar="PREFIX", default='')
 282                 (options, args) = parser.parse_args()
 283
 284                 self.n_events = 0
 285                 self.stmts    = dict()
 286                 self.verbose  = options.verbose
 287
 288                 tables = dict()
 289                 tables['ctx'] = options.prefix + 'ctx'
 290                 tables['ev']  = options.prefix + 'ev'
 291
 292                 if len(args) < 1:
 293                         parser.print_help()
 294                         sys.exit(1)
 295
 296                 self.files  = []
 297                 files       = args
 298
 299                 for file in files:
 300                         if not os.path.isfile(file):
 301                                 print "cannot find input file %s" % (file, )
 302                         else:
 303                                 self.files.append(file)
 304
 305                 if len(self.files) < 1:
 306                         print "no input file to process"
 307                         sys.exit(3)
 308
 309                 if options.filter:
 310                         self.filter = re.compile(options.filter)
 311                 else:
 312                         self.filter = DummyFilter()
 313
 314                 if options.engine in self.engines:
 315                         engine = self.engines[options.engine]
 316                 else:
 317                         print 'engine %s not found' % options.engine
 318                         print 'we offer: %s' % self.engines.keys()
 319                         sys.exit(0)
 320
 321                 if options.verbose:
 322                         print "determining schema..."
 323
 324                 (ctxcols, evcols) = self.find_heads()
 325                 if options.verbose:
 326                         print "context schema:"
 327                         print ctxcols
 328                         print "event schema:"
 329                         print evcols
 330                         print "tables:"
 331                         print tables
 332
 333                 self.emit = engine(options, tables, ctxcols, evcols)
 334
 335                 if options.verbose:
 336                         print "filling tables..."
 337                 self.fill_tables()
 338                 if options.verbose:
 339                         print "comitting..."
 340                 self.emit.commit()
 341
 342 if __name__ == "__main__":
 343         Conv()