From: Matthias Braun Date: Mon, 7 Jan 2008 19:21:21 +0000 (+0000) Subject: move statev_sql.py to scripts dir X-Git-Url: http://nsz.repo.hu/git/?a=commitdiff_plain;h=06430e7ba8fa72b1d9abd19fe63f09ca16f84da8;p=libfirm move statev_sql.py to scripts dir [r17200] --- diff --git a/ir/be/test/statev_sql.py b/ir/be/test/statev_sql.py deleted file mode 100755 index 8b8db52df..000000000 --- a/ir/be/test/statev_sql.py +++ /dev/null @@ -1,320 +0,0 @@ -#! /usr/bin/env python - -import sys -import os -import re -import time -import stat -import profile -import sqlite3 -import MySQLdb -import fileinput -import tempfile -import optparse - -class DummyFilter: - def match(self, dummy): - return True - -class EmitBase: - def create_table(self, cols, name, type, unique): - create = 'create table if not exists %s (id int %s' % (name, unique) - - sorted = [None] * len(cols) - for x in cols.iterkeys(): - sorted[cols[x]] = x - for x in sorted: - create += (', %s %s' % (x, type)) - create += ');' - return create - -class EmitMysqlInfile(EmitBase): - tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR - - def ex(self, args, tab, fname): - res = os.fork() - if res == 0: - stmt = """load data infile '%s' into table %s fields terminated by ';'""" % (fname, tab) - conn = MySQLdb.connect(**args) - c = conn.cursor() - c.execute(stmt) - conn.commit() - sys.exit(0) - return res - - def __init__(self, options, tables, ctxcols, evcols): - args = dict() - if options.password: - args['passwd'] = options.password - if not options.host: - options.host = 'localhost' - args['user'] = options.user - args['host'] = options.host - args['db'] = options.database - - self.conn = MySQLdb.connect(**args) - self.ctxcols = ctxcols - self.evcols = evcols - self.options = options - self.ctxtab = tables['ctx'] - self.evtab = tables['ev'] - - params = (tempfile.gettempdir(), os.sep, os.getpid()) - self.evfifo = '%s%sstatev_ev_%d' % params - self.ctxfifo = '%s%sstatev_ctx_%d' % params - - os.mkfifo(self.evfifo) - os.mkfifo(self.ctxfifo) - - os.chmod(self.evfifo, self.tmpfile_mode) - os.chmod(self.ctxfifo, self.tmpfile_mode) - - c = self.conn.cursor() - c.execute('drop table if exists ' + self.evtab) - c.execute('drop table if exists ' + self.ctxtab) - c.execute(self.create_table(self.ctxcols, self.ctxtab, 'char(80)', 'unique')) - c.execute(self.create_table(self.evcols, self.evtab, 'double default null', '')) - self.conn.commit() - - if options.verbose: - print 'go for gold' - - self.pidev = self.ex(args, self.evtab, self.evfifo) - self.pidctx = self.ex(args, self.ctxtab, self.ctxfifo) - - if options.verbose: - print "forked two mysql leechers: %d, %d" % (self.pidev, self.pidctx) - - self.evfile = open(self.evfifo, 'w+t') - self.ctxfile = open(self.ctxfifo, 'w+t') - - if options.verbose: - print 'fifo: %s, %o' % (self.evfile.name, os.stat(self.evfile.name).st_mode) - print 'fifo: %s, %o' % (self.ctxfile.name, os.stat(self.ctxfile.name).st_mode) - - def ev(self, curr_id, evitems): - field = ['\N'] * len(self.evcols) - for key, val in evitems.iteritems(): - index = self.evcols[key] - field[index] = val - print >> self.evfile, ('%d;' % curr_id) + ';'.join(field) - - def ctx(self, curr_id, ctxitems): - field = ['\N'] * len(self.ctxcols) - for key, val in ctxitems.iteritems(): - index = self.ctxcols[key] - field[index] = val - print >> self.ctxfile, ('%d;' % curr_id) + ';'.join(field) - - def commit(self): - self.evfile.close() - self.ctxfile.close() - - os.waitpid(self.pidev, 0) - os.waitpid(self.pidctx, 0) - - os.unlink(self.evfile.name) - os.unlink(self.ctxfile.name) - - -class EmitSqlite3(EmitBase): - def __init__(self, options, tables, ctxcols, evcols): - if os.path.isfile(options.database): - os.unlink(options.database) - - self.ctxtab = tables['ctx'] - self.evtab = tables['ev'] - self.conn = sqlite3.connect(options.database) - self.conn.execute(self.create_table(ctxcols, self.ctxtab, 'text', 'unique')) - self.conn.execute(self.create_table(evcols, self.evtab, 'double', '')) - - n = max(len(ctxcols), len(evcols)) + 1 - q = ['?'] - self.quests = [] - for i in xrange(0, n): - self.quests.append(','.join(q)) - q.append('?') - - def ev(self, curr_id, evitems): - keys = ','.join(evitems.keys()) - stmt = 'insert into %s (id, %s) values (%s)' % (self.evtab, keys, self.quests[len(evitems)]) - self.conn.execute(stmt, (curr_id,) + tuple(evitems.values())) - - def ctx(self, curr_id, ctxitems): - keys = ','.join(ctxitems.keys()) - stmt = 'insert into %s (id, %s) values (%s)' % (self.ctxtab, keys, self.quests[len(ctxitems)]) - self.conn.execute(stmt, (curr_id,) + tuple(ctxitems.values())) - - def commit(self): - self.conn.commit() - -class Conv: - engines = { 'sqlite3': EmitSqlite3, 'mysql': EmitMysqlInfile } - def find_heads(self): - n_ev = 0 - ctxind = 0 - evind = 0 - ctxcols = dict() - evcols = dict() - - self.valid_keys = set() - - for line in self.input(): - heads = None - if line[0] == 'P': - ind = line.index(';', 2) - key = line[2:ind] - if not key in ctxcols: - ctxcols[key] = ctxind - ctxind += 1 - - elif line[0] == 'E': - ind = line.index(';', 2) - key = line[2:ind] - if self.filter.match(key): - self.n_events += 1 - if not key in evcols: - self.valid_keys.add(key) - evcols[key] = evind - evind += 1 - - return (ctxcols, evcols) - - def input(self): - return fileinput.FileInput(files=self.files, openhook=fileinput.hook_compressed) - - def fill_tables(self): - lineno = 0 - ids = 0 - curr_id = 0 - keystack = [] - idstack = [] - curr_event = 0 - last_prec = -1 - evcols = dict() - ctxcols = dict() - - for line in self.input(): - lineno += 1 - items = line.strip().split(';') - op = items[0] - - if op == 'P': - # flush the current events - if len(evcols): - self.emit.ev(curr_id, evcols) - evcols.clear() - - # push the key - key = items[1] - val = items[2] - keystack.append(key) - curr_id = ids - ids += 1 - idstack.append(curr_id) - ctxcols[key] = val - - self.emit.ctx(curr_id, ctxcols) - - elif op == 'O': - popkey = items[1] - key = keystack.pop() - - if popkey != key: - print "unmatched pop in line %d, push key %s, pop key: %s" % (lineno, key, popkey) - - idstack.pop() - if len(idstack) > 0: - if len(evcols) > 0: - self.emit.ev(curr_id, evcols) - evcols.clear() - del ctxcols[key] - curr_id = idstack[-1] - else: - curr_id = -1 - - elif op == 'E': - key = items[1] - if key in self.valid_keys: - curr_event += 1 - evcols[key] = items[2] - - if self.verbose: - prec = curr_event * 10 / self.n_events - if prec > last_prec: - last_prec = prec - print '%10d / %10d' % (curr_event, self.n_events) - - def __init__(self): - parser = optparse.OptionParser('usage: %prog [options] ') - parser.add_option("-c", "--clean", dest="clean", help="delete tables in advance", action="store_true", default=False) - parser.add_option("-v", "--verbose", dest="verbose", help="verbose messages", action="store_true", default=False) - parser.add_option("-f", "--filter", dest="filter", help="regexp to filter event keys", metavar="REGEXP") - parser.add_option("-u", "--user", dest="user", help="user", metavar="USER") - parser.add_option("-H", "--host", dest="host", help="host", metavar="HOST") - parser.add_option("-p", "--password", dest="password", help="password", metavar="PASSWORD") - parser.add_option("-d", "--db", dest="database", help="database", metavar="DB") - parser.add_option("-e", "--engine", dest="engine", help="engine", metavar="ENG", default='sqlite3') - parser.add_option("-P", "--prefix", dest="prefix", help="table prefix", metavar="PREFIX", default='') - (options, args) = parser.parse_args() - - self.n_events = 0 - self.stmts = dict() - self.verbose = options.verbose - - tables = dict() - tables['ctx'] = options.prefix + 'ctx' - tables['ev'] = options.prefix + 'ev' - - if len(args) < 1: - parser.print_help() - sys.exit(1) - - self.files = [] - files = args - - for file in files: - if not os.path.isfile(file): - print "cannot find input file %s" % (file, ) - else: - self.files.append(file) - - if len(self.files) < 1: - print "no input file to process" - sys.exit(3) - - if options.filter: - self.filter = re.compile(options.filter) - else: - self.filter = DummyFilter() - - if options.engine in self.engines: - engine = self.engines[options.engine] - else: - print 'engine %s not found' % options.engine - print 'we offer: %s' % self.engines.keys() - sys.exit(0) - - if options.verbose: - print "determining schema..." - - (ctxcols, evcols) = self.find_heads() - if options.verbose: - print "context schema:" - print ctxcols - print "event schema:" - print evcols - print "tables:" - print tables - - self.emit = engine(options, tables, ctxcols, evcols) - - if options.verbose: - print "filling tables..." - self.fill_tables() - if options.verbose: - print "comitting..." - self.emit.commit() - -if __name__ == "__main__": - Conv() diff --git a/scripts/statev_sql.py b/scripts/statev_sql.py new file mode 100755 index 000000000..8b8db52df --- /dev/null +++ b/scripts/statev_sql.py @@ -0,0 +1,320 @@ +#! /usr/bin/env python + +import sys +import os +import re +import time +import stat +import profile +import sqlite3 +import MySQLdb +import fileinput +import tempfile +import optparse + +class DummyFilter: + def match(self, dummy): + return True + +class EmitBase: + def create_table(self, cols, name, type, unique): + create = 'create table if not exists %s (id int %s' % (name, unique) + + sorted = [None] * len(cols) + for x in cols.iterkeys(): + sorted[cols[x]] = x + for x in sorted: + create += (', %s %s' % (x, type)) + create += ');' + return create + +class EmitMysqlInfile(EmitBase): + tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR + + def ex(self, args, tab, fname): + res = os.fork() + if res == 0: + stmt = """load data infile '%s' into table %s fields terminated by ';'""" % (fname, tab) + conn = MySQLdb.connect(**args) + c = conn.cursor() + c.execute(stmt) + conn.commit() + sys.exit(0) + return res + + def __init__(self, options, tables, ctxcols, evcols): + args = dict() + if options.password: + args['passwd'] = options.password + if not options.host: + options.host = 'localhost' + args['user'] = options.user + args['host'] = options.host + args['db'] = options.database + + self.conn = MySQLdb.connect(**args) + self.ctxcols = ctxcols + self.evcols = evcols + self.options = options + self.ctxtab = tables['ctx'] + self.evtab = tables['ev'] + + params = (tempfile.gettempdir(), os.sep, os.getpid()) + self.evfifo = '%s%sstatev_ev_%d' % params + self.ctxfifo = '%s%sstatev_ctx_%d' % params + + os.mkfifo(self.evfifo) + os.mkfifo(self.ctxfifo) + + os.chmod(self.evfifo, self.tmpfile_mode) + os.chmod(self.ctxfifo, self.tmpfile_mode) + + c = self.conn.cursor() + c.execute('drop table if exists ' + self.evtab) + c.execute('drop table if exists ' + self.ctxtab) + c.execute(self.create_table(self.ctxcols, self.ctxtab, 'char(80)', 'unique')) + c.execute(self.create_table(self.evcols, self.evtab, 'double default null', '')) + self.conn.commit() + + if options.verbose: + print 'go for gold' + + self.pidev = self.ex(args, self.evtab, self.evfifo) + self.pidctx = self.ex(args, self.ctxtab, self.ctxfifo) + + if options.verbose: + print "forked two mysql leechers: %d, %d" % (self.pidev, self.pidctx) + + self.evfile = open(self.evfifo, 'w+t') + self.ctxfile = open(self.ctxfifo, 'w+t') + + if options.verbose: + print 'fifo: %s, %o' % (self.evfile.name, os.stat(self.evfile.name).st_mode) + print 'fifo: %s, %o' % (self.ctxfile.name, os.stat(self.ctxfile.name).st_mode) + + def ev(self, curr_id, evitems): + field = ['\N'] * len(self.evcols) + for key, val in evitems.iteritems(): + index = self.evcols[key] + field[index] = val + print >> self.evfile, ('%d;' % curr_id) + ';'.join(field) + + def ctx(self, curr_id, ctxitems): + field = ['\N'] * len(self.ctxcols) + for key, val in ctxitems.iteritems(): + index = self.ctxcols[key] + field[index] = val + print >> self.ctxfile, ('%d;' % curr_id) + ';'.join(field) + + def commit(self): + self.evfile.close() + self.ctxfile.close() + + os.waitpid(self.pidev, 0) + os.waitpid(self.pidctx, 0) + + os.unlink(self.evfile.name) + os.unlink(self.ctxfile.name) + + +class EmitSqlite3(EmitBase): + def __init__(self, options, tables, ctxcols, evcols): + if os.path.isfile(options.database): + os.unlink(options.database) + + self.ctxtab = tables['ctx'] + self.evtab = tables['ev'] + self.conn = sqlite3.connect(options.database) + self.conn.execute(self.create_table(ctxcols, self.ctxtab, 'text', 'unique')) + self.conn.execute(self.create_table(evcols, self.evtab, 'double', '')) + + n = max(len(ctxcols), len(evcols)) + 1 + q = ['?'] + self.quests = [] + for i in xrange(0, n): + self.quests.append(','.join(q)) + q.append('?') + + def ev(self, curr_id, evitems): + keys = ','.join(evitems.keys()) + stmt = 'insert into %s (id, %s) values (%s)' % (self.evtab, keys, self.quests[len(evitems)]) + self.conn.execute(stmt, (curr_id,) + tuple(evitems.values())) + + def ctx(self, curr_id, ctxitems): + keys = ','.join(ctxitems.keys()) + stmt = 'insert into %s (id, %s) values (%s)' % (self.ctxtab, keys, self.quests[len(ctxitems)]) + self.conn.execute(stmt, (curr_id,) + tuple(ctxitems.values())) + + def commit(self): + self.conn.commit() + +class Conv: + engines = { 'sqlite3': EmitSqlite3, 'mysql': EmitMysqlInfile } + def find_heads(self): + n_ev = 0 + ctxind = 0 + evind = 0 + ctxcols = dict() + evcols = dict() + + self.valid_keys = set() + + for line in self.input(): + heads = None + if line[0] == 'P': + ind = line.index(';', 2) + key = line[2:ind] + if not key in ctxcols: + ctxcols[key] = ctxind + ctxind += 1 + + elif line[0] == 'E': + ind = line.index(';', 2) + key = line[2:ind] + if self.filter.match(key): + self.n_events += 1 + if not key in evcols: + self.valid_keys.add(key) + evcols[key] = evind + evind += 1 + + return (ctxcols, evcols) + + def input(self): + return fileinput.FileInput(files=self.files, openhook=fileinput.hook_compressed) + + def fill_tables(self): + lineno = 0 + ids = 0 + curr_id = 0 + keystack = [] + idstack = [] + curr_event = 0 + last_prec = -1 + evcols = dict() + ctxcols = dict() + + for line in self.input(): + lineno += 1 + items = line.strip().split(';') + op = items[0] + + if op == 'P': + # flush the current events + if len(evcols): + self.emit.ev(curr_id, evcols) + evcols.clear() + + # push the key + key = items[1] + val = items[2] + keystack.append(key) + curr_id = ids + ids += 1 + idstack.append(curr_id) + ctxcols[key] = val + + self.emit.ctx(curr_id, ctxcols) + + elif op == 'O': + popkey = items[1] + key = keystack.pop() + + if popkey != key: + print "unmatched pop in line %d, push key %s, pop key: %s" % (lineno, key, popkey) + + idstack.pop() + if len(idstack) > 0: + if len(evcols) > 0: + self.emit.ev(curr_id, evcols) + evcols.clear() + del ctxcols[key] + curr_id = idstack[-1] + else: + curr_id = -1 + + elif op == 'E': + key = items[1] + if key in self.valid_keys: + curr_event += 1 + evcols[key] = items[2] + + if self.verbose: + prec = curr_event * 10 / self.n_events + if prec > last_prec: + last_prec = prec + print '%10d / %10d' % (curr_event, self.n_events) + + def __init__(self): + parser = optparse.OptionParser('usage: %prog [options] ') + parser.add_option("-c", "--clean", dest="clean", help="delete tables in advance", action="store_true", default=False) + parser.add_option("-v", "--verbose", dest="verbose", help="verbose messages", action="store_true", default=False) + parser.add_option("-f", "--filter", dest="filter", help="regexp to filter event keys", metavar="REGEXP") + parser.add_option("-u", "--user", dest="user", help="user", metavar="USER") + parser.add_option("-H", "--host", dest="host", help="host", metavar="HOST") + parser.add_option("-p", "--password", dest="password", help="password", metavar="PASSWORD") + parser.add_option("-d", "--db", dest="database", help="database", metavar="DB") + parser.add_option("-e", "--engine", dest="engine", help="engine", metavar="ENG", default='sqlite3') + parser.add_option("-P", "--prefix", dest="prefix", help="table prefix", metavar="PREFIX", default='') + (options, args) = parser.parse_args() + + self.n_events = 0 + self.stmts = dict() + self.verbose = options.verbose + + tables = dict() + tables['ctx'] = options.prefix + 'ctx' + tables['ev'] = options.prefix + 'ev' + + if len(args) < 1: + parser.print_help() + sys.exit(1) + + self.files = [] + files = args + + for file in files: + if not os.path.isfile(file): + print "cannot find input file %s" % (file, ) + else: + self.files.append(file) + + if len(self.files) < 1: + print "no input file to process" + sys.exit(3) + + if options.filter: + self.filter = re.compile(options.filter) + else: + self.filter = DummyFilter() + + if options.engine in self.engines: + engine = self.engines[options.engine] + else: + print 'engine %s not found' % options.engine + print 'we offer: %s' % self.engines.keys() + sys.exit(0) + + if options.verbose: + print "determining schema..." + + (ctxcols, evcols) = self.find_heads() + if options.verbose: + print "context schema:" + print ctxcols + print "event schema:" + print evcols + print "tables:" + print tables + + self.emit = engine(options, tables, ctxcols, evcols) + + if options.verbose: + print "filling tables..." + self.fill_tables() + if options.verbose: + print "comitting..." + self.emit.commit() + +if __name__ == "__main__": + Conv()