3 # This file is part of libFirm.
4 # Copyright (C) 2012 Karlsruhe Institute of Technology.
15 def match(self, dummy):
23 def create_table(self, cols, name, defaulttype, keytype, extra=""):
24 c = "create table if not exists `%s` (\n" % name
25 c += "\t`id` %s\n" % keytype
29 type = self.types[defaulttype]
32 type = self.types["text"]
35 type = self.types["bool"]
37 c += "\t,`%s` %s\n" % (name, type)
42 # Abstraction for mysql sql connection and sql syntax
43 class EmitMysql(EmitBase):
44 tmpfile_mode = stat.S_IREAD | stat.S_IROTH | stat.S_IWUSR
46 def execute(self, query, *args):
47 #print query + " %s\n" % str(tuple(args))
48 self.cursor.execute(query, *args);
51 def connect(self, options):
56 args['passwd'] = options.password
58 options.host = 'localhost'
59 args['user'] = options.user
60 args['host'] = options.host
61 args['db'] = options.database
63 self.conn = MySQLdb.connect(**args)
64 self.options = options
65 self.cursor = self.conn.cursor()
67 def __init__(self, options, ctxcols, evcols):
70 self.types["text"] = "varchar(80) default null";
71 self.types["data"] = "double default null";
72 self.types["bool"] = "bool";
74 self.ctxtab = options.prefix + "ctx"
75 self.evtab = options.prefix + "ev"
77 if not options.update:
78 self.execute('drop table if exists `%s`' % self.evtab)
79 self.execute('drop table if exists `%s`' % self.ctxtab)
81 self.create_table(ctxcols, self.ctxtab, "text", "int auto_increment", extra = ", PRIMARY KEY (`id`)")
82 self.create_table(evcols, self.evtab, "data", "int not null", extra = ", INDEX(`id`)")
84 keys = "id, " + ", ".join(evcols)
85 marks = ",".join(['%s'] * (len(evcols)+1))
86 self.evinsert = "insert into `%s` (%s) values (%s)" % (self.evtab, keys, marks)
88 keys = ", ".join(ctxcols)
89 marks = ",".join(['%s'] * len(ctxcols))
90 self.ctxinsert = "insert into `%s` (%s) values (%s)" % (self.ctxtab, keys, marks)
92 def ev(self, curr_id, evitems):
93 self.execute(self.evinsert, (curr_id,) + tuple(evitems))
95 def ctx(self, ctxitems):
96 self.execute(self.ctxinsert, tuple(ctxitems))
98 id = self.cursor.lastrowid
104 # Abstraction for sqlite3 databases and sql syntax
105 class EmitSqlite3(EmitBase):
106 def execute(self, query, *args):
107 #print query + " %s\n" % str(tuple(args))
108 self.cursor.execute(query, *args)
110 def __init__(self, options, ctxcols, evcols):
113 if options.database == None:
114 print "Have to specify database (file-)name for sqlite"
117 if not options.update:
118 if os.path.isfile(options.database):
119 os.unlink(options.database)
121 self.conn = sqlite3.connect(options.database)
122 self.cursor = self.conn.cursor()
124 self.types["data"] = "double"
125 self.types["text"] = "text"
126 self.types["bool"] = "int"
127 self.ctxtab = options.prefix + "ctx"
128 self.evtab = options.prefix + "ev"
130 self.create_table(ctxcols, self.ctxtab, "text", "integer primary key")
131 self.create_table(evcols, self.evtab, "data", "int")
132 self.execute("CREATE INDEX IF NOT EXISTS `%sindex` ON `%s`(id)"
133 % (self.evtab, self.evtab))
135 keys = "id, " + ", ".join(evcols)
136 marks = ",".join(["?"] * (len(evcols)+1))
137 self.evinsert = "insert into `%s` values (%s)" % (self.evtab, marks)
139 keys = ", ".join(ctxcols)
140 marks = ",".join(["?"] * len(ctxcols))
141 self.ctxinsert = "insert into `%s` (%s) values (%s)" % (self.ctxtab, keys, marks)
145 def ev(self, curr_id, evitems):
146 self.execute(self.evinsert, (curr_id,) + tuple(evitems))
148 def ctx(self, ctxitems):
149 curr_id = self.nextid
151 self.execute(self.ctxinsert, tuple(ctxitems))
153 return self.cursor.lastrowid
159 engines = { 'sqlite3': EmitSqlite3, 'mysql': EmitMysql }
161 # Pass that determines event and context types
162 def find_heads(self):
172 self.valid_keys = set()
178 fields = line.strip().split(";")
180 if (len(fields)-1) % 2 != 0:
181 print "%s: Invalid number of fields after 'P'" % linenr
183 for i in range(1,len(fields),2):
185 if not ctxcols.has_key(key):
186 ctxcols[key] = ctxind
190 elif fields[0] == 'E':
191 if (len(fields)-1) % 2 != 0:
192 print "%s: Invalid number of fields after 'E'" % linenr
195 for i in range(1,len(fields),2):
197 if not self.filter.match(key):
200 if not evcols.has_key(key):
201 self.valid_keys.add(key)
206 self.ctxcols = ctxcols
208 return (ctxlist, evlist)
211 return fileinput.FileInput(files=self.files, openhook=fileinput.hook_compressed)
213 def flush_events(self, id):
215 for e in self.evvals:
222 self.emit.ev(id, self.evvals)
223 self.evvals = [None] * len(self.evvals)
226 if not self.pushpending:
228 self.pushpending = False
229 self.curr_id = self.emit.ctx(self.ctxvals)
231 def fill_tables(self):
239 self.pushpending = False
240 self.ctxvals = [None] * len(self.ctxcols)
241 self.evvals = [None] * len(self.evcols)
243 for line in self.input():
245 items = line.strip().split(';')
248 # Push context command
250 self.flush_events(self.curr_id)
253 for p in range(1,len(items),2):
258 idstack.append(self.curr_id)
260 keyidx = self.ctxcols[key]
261 assert self.ctxvals[keyidx] == None
262 self.ctxvals[keyidx] = val
263 self.pushpending = True
265 # Pop context command
268 # For now... we could optimize this
271 # We process fields in reverse order to makes O's match the
272 # order of previous P's
273 for p in range(len(items)-1,0,-1):
274 self.flush_events(self.curr_id)
278 self.curr_id = idstack.pop()
281 print "unmatched pop in line %d, push key %s, pop key: %s" % (lineno, key, popkey)
283 keyidx = self.ctxcols[key]
284 assert self.ctxvals[keyidx] != None
285 self.ctxvals[keyidx] = None
292 # Show that we make progress
294 prec = curr_event * 10 / self.n_events
297 print '%10d / %10d' % (curr_event, self.n_events)
299 for p in range(1,len(items),2):
301 if key not in self.evcols:
304 keyidx = self.evcols[key]
305 if self.evvals[keyidx] != None:
306 self.flush_events(self.curr_id)
309 self.evvals[keyidx] = value
312 parser = optparse.OptionParser('usage: %prog [options] <event file...>', add_help_option=False)
313 parser.add_option("", "--help", help="show this help message and exit", action="help")
314 parser.add_option("", "--update", dest="update", help="update database instead of dropping all existing values", action="store_true", default=False)
315 parser.add_option("-v", "--verbose", dest="verbose", help="verbose messages", action="store_true", default=False)
316 parser.add_option("-f", "--filter", dest="filter", help="regexp to filter event keys", metavar="REGEXP")
317 parser.add_option("-u", "--user", dest="user", help="user", metavar="USER")
318 parser.add_option("-h", "--host", dest="host", help="host", metavar="HOST")
319 parser.add_option("-p", "--password", dest="password", help="password", metavar="PASSWORD")
320 parser.add_option("-D", "--database", dest="database", help="database", metavar="DB")
321 parser.add_option("-e", "--engine", dest="engine", help="engine (sqlite3, mysql)", metavar="ENG", default='sqlite3')
322 parser.add_option("-P", "--prefix", dest="prefix", help="table prefix", metavar="PREFIX", default='')
323 (options, args) = parser.parse_args()
327 self.verbose = options.verbose
337 if not os.path.isfile(file):
338 print "cannot find input file %s" % (file, )
340 self.files.append(file)
342 if len(self.files) < 1:
343 print "no input file to process"
347 self.filter = re.compile(options.filter)
349 self.filter = DummyFilter()
351 if options.engine in self.engines:
352 engine = self.engines[options.engine]
354 print 'engine %s not found' % options.engine
355 print 'we offer: %s' % self.engines.keys()
359 print "determining schema..."
361 (ctxcols, evcols) = self.find_heads()
363 print "context schema:"
365 print "event schema:"
368 self.emit = engine(options, ctxcols, evcols)
371 print "filling tables..."
377 if __name__ == "__main__":