nsz Git - libfirm/blob - ir/be/test/statev_sql.py

   1 #! /usr/bin/env python
   2
   3 import sys
   4 import os
   5 import re
   6 import time
   7 import stat
   8 import profile
   9 import sqlite3
  10 import MySQLdb
  11 import fileinput
  12 import tempfile
  13 import optparse
  14
  15 class DummyFilter:
  16         def match(self, dummy):
  17                 return True
  18
  19 class EmitBase:
  20         def create_table(self, cols, name, type, unique):
  21                 create = 'create table if not exists %s (id int %s' % (name, unique)
  22
  23                 sorted = [None] * len(cols)
  24                 for x in cols.iterkeys():
  25                         sorted[cols[x]] = x
  26                 for x in sorted:
  27                         create += (', %s %s' % (x, type))
  28                 create += ');'
  29                 return create
  30
  31 class EmitMysqlInfile(EmitBase):
  32         tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR
  33
  34         def ex(self, args, tab, fname):
  35                 res = os.fork()
  36                 if res == 0:
  37                         stmt = """load data infile '%s' into table %s fields terminated by ';'""" % (fname, tab)
  38                         conn = MySQLdb.connect(**args)
  39                         c = conn.cursor()
  40                         c.execute(stmt)
  41                         conn.commit()
  42                         sys.exit(0)
  43                 return res
  44
  45         def __init__(self, options, ctxcols, evcols):
  46                 args = dict()
  47                 if options.password:
  48                         args['passwd'] = options.password
  49                 if not options.host:
  50                         options.host = 'localhost'
  51                 args['user'] = options.user
  52                 args['host'] = options.host
  53                 args['db']   = options.database
  54
  55                 self.conn     = MySQLdb.connect(**args)
  56                 self.ctxcols  = ctxcols
  57                 self.evcols   = evcols
  58                 self.options  = options
  59
  60                 params = (tempfile.gettempdir(), os.sep, os.getpid())
  61                 self.evfifo  = '%s%sstatev_ev_%d' % params
  62                 self.ctxfifo = '%s%sstatev_ctx_%d' % params
  63
  64                 os.mkfifo(self.evfifo)
  65                 os.mkfifo(self.ctxfifo)
  66
  67                 os.chmod(self.evfifo,  self.tmpfile_mode)
  68                 os.chmod(self.ctxfifo, self.tmpfile_mode)
  69
  70                 c = self.conn.cursor()
  71                 c.execute('drop table if exists ev')
  72                 c.execute('drop table if exists ctx')
  73                 c.execute(self.create_table(self.ctxcols, 'ctx', 'char(80)', 'unique'))
  74                 c.execute(self.create_table(self.evcols, 'ev', 'double default null', ''))
  75                 self.conn.commit()
  76
  77                 if options.verbose:
  78                         print 'go for gold'
  79
  80                 self.pidev  = self.ex(args, 'ev', self.evfifo)
  81                 self.pidctx = self.ex(args, 'ctx', self.ctxfifo)
  82
  83                 if options.verbose:
  84                         print "forked two mysql leechers: %d, %d" % (self.pidev, self.pidctx)
  85
  86                 self.evfile   = open(self.evfifo, 'w+t')
  87                 self.ctxfile  = open(self.ctxfifo, 'w+t')
  88
  89                 if options.verbose:
  90                         print 'fifo:  %s, %o' % (self.evfile.name, os.stat(self.evfile.name).st_mode)
  91                         print 'fifo:  %s, %o' % (self.ctxfile.name, os.stat(self.ctxfile.name).st_mode)
  92
  93         def ev(self, curr_id, evitems):
  94                 field = ['\N'] * len(self.evcols)
  95                 for key, val in evitems.iteritems():
  96                         index = self.evcols[key]
  97                         field[index] = val
  98                 print >> self.evfile, ('%d;' % curr_id) + ';'.join(field)
  99
 100         def ctx(self, curr_id, ctxitems):
 101                 field = ['\N'] * len(self.ctxcols)
 102                 for key, val in ctxitems.iteritems():
 103                         index = self.ctxcols[key]
 104                         field[index] = val
 105                 print >> self.ctxfile, ('%d;' % curr_id) + ';'.join(field)
 106
 107         def commit(self):
 108                 self.evfile.close()
 109                 self.ctxfile.close()
 110
 111                 os.waitpid(self.pidev, 0)
 112                 os.waitpid(self.pidctx, 0)
 113
 114                 os.unlink(self.evfile.name)
 115                 os.unlink(self.ctxfile.name)
 116
 117
 118 class EmitSqlite3(EmitBase):
 119         def __init__(self, options, ctxcols, evcols):
 120                 if os.path.isfile(options.database):
 121                         os.unlink(options.database)
 122
 123                 self.conn = sqlite3.connect(options.database)
 124                 self.conn.execute(self.create_table(ctxcols, 'ctx', 'text', 'unique'))
 125                 self.conn.execute(self.create_table(evcols, 'ev', 'double', ''))
 126
 127                 q = []
 128                 self.quests = []
 129                 for i in xrange(0, max(len(ctxcols), len(evcols))):
 130                         self.quests.append(','.join(q))
 131                         q.append('?')
 132
 133         def ev(self, curr_id, evitems):
 134                 keys = ','.join(evitems.keys())
 135                 stmt = 'insert into ev (id, %s) values (%s)' % (keys, self.quests[len(evitems) + 1])
 136                 self.conn.execute(stmt, (curr_id,) + tuple(evitems.values()))
 137
 138         def ctx(self, curr_id, ctxitems):
 139                 keys = ','.join(ctxitems.keys())
 140                 stmt = 'insert into ctx (id, %s) values (%s)' % (keys, self.quests[len(ctxitems) + 1])
 141                 self.conn.execute(stmt, (curr_id,) + tuple(ctxitems.values()))
 142
 143         def commit(self):
 144                 self.conn.commit()
 145
 146 class Conv:
 147         engines = { 'sqlite3': EmitSqlite3, 'mysql': EmitMysqlInfile }
 148         def find_heads(self):
 149                 n_ev    = 0
 150                 ctxind   = 0
 151                 evind    = 0
 152                 ctxcols  = dict()
 153                 evcols   = dict()
 154
 155                 self.valid_keys = set()
 156
 157                 for line in self.input():
 158                         heads = None
 159                         if line[0] == 'P':
 160                                 ind = line.index(';', 2)
 161                                 key = line[2:ind]
 162                                 if not key in ctxcols:
 163                                         ctxcols[key] = ctxind
 164                                         ctxind += 1
 165
 166                         elif line[0] == 'E':
 167                                 ind = line.index(';', 2)
 168                                 key = line[2:ind]
 169                                 if self.filter.match(key):
 170                                         self.n_events += 1
 171                                         if not key in evcols:
 172                                                 self.valid_keys.add(key)
 173                                                 evcols[key] = evind
 174                                                 evind += 1
 175
 176                 return (ctxcols, evcols)
 177
 178         def input(self):
 179                 return fileinput.FileInput(files=self.files, openhook=fileinput.hook_compressed)
 180
 181         def fill_tables(self):
 182                 ids        = 0
 183                 curr_id    = 0
 184                 keystack   = []
 185                 idstack    = []
 186                 curr_event = 0
 187                 last_prec  = -1
 188                 evcols     = dict()
 189                 ctxcols    = dict()
 190
 191                 for line in self.input():
 192                         items = line.strip().split(';')
 193                         op    = items[0]
 194
 195                         if op == 'P':
 196                                 # flush the current events
 197                                 if len(evcols):
 198                                         self.emit.ev(curr_id, evcols)
 199                                         evcols.clear()
 200
 201                                 # push the key
 202                                 key   = items[1]
 203                                 val   = items[2]
 204                                 keystack.append(key)
 205                                 curr_id = ids
 206                                 ids += 1
 207                                 idstack.append(curr_id)
 208                                 ctxcols[key] = val
 209
 210                                 self.emit.ctx(curr_id, ctxcols)
 211
 212                         elif op == 'O':
 213                                 key = keystack.pop()
 214                                 idstack.pop()
 215                                 if len(idstack) > 0:
 216                                         if len(evcols) > 0:
 217                                                 self.emit.ev(curr_id, evcols)
 218                                                 evcols.clear()
 219                                         del ctxcols[key]
 220                                         curr_id = idstack[-1]
 221                                 else:
 222                                         curr_id = -1
 223
 224                         elif op == 'E':
 225                                 key = items[1]
 226                                 if key in self.valid_keys:
 227                                         curr_event += 1
 228                                         evcols[key] = items[2]
 229
 230                                         if self.verbose:
 231                                                 prec = curr_event * 10 / self.n_events
 232                                                 if prec > last_prec:
 233                                                         last_prec = prec
 234                                                         print '%10d / %10d' % (curr_event, self.n_events)
 235
 236         def __init__(self):
 237                 parser = optparse.OptionParser('usage: %prog [options]  <event file...>')
 238                 parser.add_option("-c", "--clean",    dest="clean",    help="delete tables in advance", action="store_true", default=False)
 239                 parser.add_option("-v", "--verbose",  dest="verbose",  help="verbose messages",         action="store_true", default=False)
 240                 parser.add_option("-f", "--filter",   dest="filter",   help="regexp to filter event keys", metavar="REGEXP")
 241                 parser.add_option("-u", "--user",     dest="user",     help="user",               metavar="USER")
 242                 parser.add_option("-H", "--host",     dest="host",     help="host",               metavar="HOST")
 243                 parser.add_option("-p", "--password", dest="password", help="password",           metavar="PASSWORD")
 244                 parser.add_option("-d", "--db",       dest="database", help="database",           metavar="DB")
 245                 parser.add_option("-e", "--engine",   dest="engine",   help="eingine",            metavar="ENG", default='sqlite3')
 246                 (options, args) = parser.parse_args()
 247
 248                 self.n_events = 0
 249                 self.stmts    = dict()
 250                 self.verbose  = options.verbose
 251
 252                 if len(args) < 1:
 253                         parser.print_help()
 254                         sys.exit(1)
 255
 256                 self.files  = []
 257                 files       = args
 258
 259                 for file in files:
 260                         if not os.path.isfile(file):
 261                                 print "cannot find input file %s" % (file, )
 262                         else:
 263                                 self.files.append(file)
 264
 265                 if len(self.files) < 1:
 266                         print "no input file to process"
 267                         sys.exit(3)
 268
 269                 if options.filter:
 270                         self.filter = re.compile(options.filter)
 271                 else:
 272                         self.filter = DummyFilter()
 273
 274                 if options.engine in self.engines:
 275                         engine = self.engines[options.engine]
 276                 else:
 277                         print 'engine %s not found' % options.engine
 278                         print 'we offer: %s' % self.engines.keys()
 279                         sys.exit(0)
 280
 281                 if options.verbose:
 282                         print "determining schema..."
 283
 284                 (ctxcols, evcols) = self.find_heads()
 285                 if options.verbose:
 286                         print "context schema:"
 287                         print ctxcols
 288                         print "event schema:"
 289                         print evcols
 290
 291                 self.emit = engine(options, ctxcols, evcols)
 292
 293                 if options.verbose:
 294                         print "filling tables..."
 295                 self.fill_tables()
 296                 if options.verbose:
 297                         print "comitting..."
 298                 self.emit.commit()
 299
 300 if __name__ == "__main__":
 301         Conv()