2 * Minimizing copies with an exact algorithm using mixed integer programming (MIP).
3 * Problem statement as a 'quadratic 0-1 program with linear constraints' with
4 * n binary variables. Constraints are knapsack (enforce color for each node) and
5 * cliques of ifg (interference constraints).
6 * Transformation into a 'mixed integer program' with n binary variables and
7 * additional 2n real variables. Constraints are the above the transformed
8 * objective function and 'complementary conditions' for two var classes.
11 * NOTE: Unfortunately no good solver is available locally (or even for linking)
12 * We use CPLEX 9.0 which runs on a machine residing at the Rechenzentrum.
16 #include "becopyopt.h"
18 #define DUMP_MPS /**< dumps the problem in "CPLEX"-MPS format. NOT fixed-column-MPS. */
19 #undef USE_SOS /**< uses Special Ordered Sets when using MPS */
20 #define DO_SOLVE /**< solve the MPS output with CPLEX */
21 #undef DUMP_MATRICES /**< dumps all matrices completely. only recommended for small problems */
22 #undef DUMP_LP /**< dumps the problem in LP format. 'human-readable' equations etc... */
23 #define DELETE_FILES /**< deletes all dumped files after use */
25 /* CPLEX-account related stuff */
26 #define SSH_USER_HOST_PATH "kb61@sp-smp.rz.uni-karlsruhe.de"
27 #define SSH_PASSWD "!cplex90"
28 #define EXPECT_FILENAME "runme" /** name of the expect-script */
30 #define DEBUG_LVL 0 //SET_LEVEL_1
31 static firm_dbg_module_t *dbg = NULL;
33 #define SLOTS_NUM2POS 256
34 #define SLOTS_LIVING 32
37 * A type storing names of the x variables in the form x[NUMBER]_[COLOR]
39 typedef struct _x_name_t {
44 * For each node taking part in the opt-problem its position in the
45 * x-variable-vector is stored in a set. This set maps the node-nr (given by
46 * benumb) to the position in the vector.
48 typedef struct _num2pos_t {
53 * A type storing the unmodified '0-1 quadratic program' of the form
59 * This problem is called the original problem
61 typedef struct _problem_instance_t {
64 int x_dim, A_dim, B_dim; /**< number of: x variables, rows in A, rows in B */
65 x_name_t *x; /**< stores the names of the x variables. all possible colors for a node are ordered and occupy consecutive entries. lives in obstack ob. */
66 set *num2pos; /**< maps node numbers to positions in x. */
67 sp_matrix_t *Q, *A, *B; /**< the (sparse) matrices of this problem */
69 /* needed only for linearizations */
70 int bigM, maxQij, minQij;
72 /* overhead needed to build this */
78 /* Nodes have consecutive numbers so this hash shoud be fine */
79 #define HASH_NUM(num) num
81 static int set_cmp_num2pos(const void *x, const void *y, size_t size) {
82 return ((num2pos_t *)x)->num != ((num2pos_t *)y)->num;
86 * Sets the first position of node with number num to pos.
87 * See x_name_t *x in _problem_instance_t.
89 static INLINE void pi_set_first_pos(problem_instance_t *pi, int num, int pos) {
93 set_insert(pi->num2pos, &find, sizeof(find), HASH_NUM(num));
97 * Get position by number. (First possible color)
98 * returns -1 if not found.
100 static INLINE int pi_get_first_pos(problem_instance_t *pi, int num) {
101 num2pos_t find, *found;
103 found = set_find(pi->num2pos, &find, sizeof(find), HASH_NUM(num));
105 assert(pi->x[found->pos].n == num && (found->pos == 0 || pi->x[found->pos-1].n != num) && "pi->num2pos is broken!");
112 * Get position by number and color.
113 * returns -1 if not found.
115 static INLINE int pi_get_pos(problem_instance_t *pi, int num, int col) {
116 num2pos_t find, *found;
119 found = set_find(pi->num2pos, &find, sizeof(find), HASH_NUM(num));
123 while (pos < pi->x_dim && pi->x[pos].n == num && pi->x[pos].c < col)
126 if (pi->x[pos].n == num && pi->x[pos].c == col)
134 * Dump the raw matrices of the problem to a file for debugging.
136 static void pi_dump_matrices(problem_instance_t *pi) {
138 FILE *out = ffopen(pi->name, "matrix", "wt");
140 DBG((dbg, LEVEL_1, "Dumping raw...\n"));
141 fprintf(out, "\n\nx-names =\n");
142 for (i=0; i<pi->x_dim; ++i)
143 fprintf(out, "%5d %2d\n", pi->x[i].n, pi->x[i].c);
145 fprintf(out, "\n\n-Q =\n");
146 matrix_dump(pi->Q, out, -1);
148 fprintf(out, "\n\nA =\n");
149 matrix_dump(pi->A, out, 1);
151 fprintf(out, "\n\nB =\n");
152 matrix_dump(pi->B, out, 1);
160 * Dumps the problem instance as a MILP. The original problem is transformed into:
162 * udN: Qx -y -s +Me = 0
168 * with M >= max sum Q'ij * x_j
171 static void pi_dump_lp(problem_instance_t *pi) {
174 FILE *out = ffopen(pi->name, "lpo", "wt");
176 DBG((dbg, LEVEL_1, "Dumping lp...\n"));
177 /* calc the big M for Q */
178 max_abs_Qij = pi->maxQij;
179 if (-pi->minQij > max_abs_Qij)
180 max_abs_Qij = -pi->minQij;
181 pi->bigM = pi->A_dim * max_abs_Qij;
182 DBG((dbg, LEVEL_2, "BigM = %d\n", pi->bigM));
184 /* generate objective function */
185 fprintf(out, "min: ");
186 for (i=0; i<pi->x_dim; ++i)
187 fprintf(out, "+s%d_%d -%dx%d_%d ", pi->x[i].n, pi->x[i].c, pi->bigM, pi->x[i].n, pi->x[i].c);
188 fprintf(out, ";\n\n");
190 /* constraints for former objective function */
191 for (i=0; i<pi->x_dim; ++i) {
192 matrix_foreach_in_row(pi->Q, i, e) {
195 fprintf(out, "+x%d_%d ", pi->x[e->col].n, pi->x[e->col].c);
197 fprintf(out, "-x%d_%d ", pi->x[e->col].n, pi->x[e->col].c);
199 fprintf(out, "%+dx%d_%d ", Qio, pi->x[e->col].n, pi->x[e->col].c);
201 fprintf(out, "-y%d_%d -s%d_%d +%d= 0;\n", pi->x[i].n, pi->x[i].c, pi->x[i].n, pi->x[i].c, pi->bigM);
203 fprintf(out, "\n\n");
205 /* constraints for (special) complementary condition */
206 for (i=0; i<pi->x_dim; ++i)
207 fprintf(out, "y%d_%d <= %d - %dx%d_%d;\n", pi->x[i].n, pi->x[i].c, 2*pi->bigM, 2*pi->bigM, pi->x[i].n, pi->x[i].c);
208 fprintf(out, "\n\n");
210 /* knapsack constraints */
211 for (i=0; i<pi->A_dim; ++i) {
212 matrix_foreach_in_row(pi->Q, i, e)
213 fprintf(out, "+x%d_%d ", pi->x[e->col].n, pi->x[e->col].c);
214 fprintf(out, " = 1;\n");
216 fprintf(out, "\n\n");
218 /* interference graph constraints */
219 for (i=0; i<pi->B_dim; ++i) {
220 matrix_foreach_in_row(pi->Q, i, e)
221 fprintf(out, "+x%d_%d ", pi->x[e->col].n, pi->x[e->col].c);
222 fprintf(out, " <= 1;\n");
224 fprintf(out, "\n\n");
226 /* integer constraints */
227 fprintf(out, "int x%d_%d", pi->x[0].n, pi->x[0].c);
228 for (i=1; i<pi->x_dim; ++i)
229 fprintf(out, ", x%d_%d", pi->x[i].n, pi->x[i].c);
238 * Dumps an mps file representing the problem. This is NOT the old-style,
239 * fixed-column format. Some white spaces are important, in general spaces
240 * are separators, MARKER-lines are used in COLUMN section to define binaries.
242 //BETTER use last 2 fields in COLUMNS section
243 static void pi_dump_mps(problem_instance_t *pi) {
246 FILE *out = ffopen(pi->name, "mps", "wt");
248 DBG((dbg, LEVEL_1, "Dumping mps...\n"));
249 max_abs_Qij = pi->maxQij;
250 if (-pi->minQij > max_abs_Qij)
251 max_abs_Qij = -pi->minQij;
252 pi->bigM = pi->A_dim * max_abs_Qij;
253 DBG((dbg, LEVEL_2, "BigM = %d\n", pi->bigM));
255 fprintf(out, "NAME %s\n", pi->name);
257 fprintf(out, "ROWS\n");
258 fprintf(out, " N obj\n");
259 for (i=0; i<pi->x_dim; ++i)
260 fprintf(out, " E cQ%d\n", i);
261 for (i=0; i<pi->A_dim; ++i)
262 fprintf(out, " E cA%d\n", i);
263 for (i=0; i<pi->B_dim; ++i)
264 fprintf(out, " L cB%d\n", i);
265 for (i=0; i<pi->x_dim; ++i)
266 fprintf(out, " L cy%d\n", i);
268 fprintf(out, "COLUMNS\n");
269 /* the x vars come first */
270 /* mark them as binaries */
271 fprintf(out, " MARKI0\t'MARKER'\t'INTORG'\n");
274 fprintf(out, " S1 SOS_%d\t'MARKER'\t'SOSORG'\n", sos_cnt++);
276 for (i=0; i<pi->x_dim; ++i) {
278 if (i>0 && pi->x[i].n != pi->x[i-1].n) {
279 fprintf(out, " SOS_%d\t'MARKER'\t'SOSEND'\n", sos_cnt++);
280 fprintf(out, " S1 SOS_%d\t'MARKER'\t'SOSORG'\n", sos_cnt++);
283 /* participation in objective */
284 fprintf(out, " x%d_%d\tobj\t%d\n", pi->x[i].n, pi->x[i].c, -pi->bigM);
286 matrix_foreach_in_col(pi->Q, i, e)
287 fprintf(out, " x%d_%d\tcQ%d\t%d\n", pi->x[i].n, pi->x[i].c, e->row, e->val);
289 matrix_foreach_in_col(pi->A, i, e)
290 fprintf(out, " x%d_%d\tcA%d\t%d\n", pi->x[i].n, pi->x[i].c, e->row, e->val);
292 matrix_foreach_in_col(pi->B, i, e)
293 fprintf(out, " x%d_%d\tcB%d\t%d\n", pi->x[i].n, pi->x[i].c, e->row, e->val);
295 fprintf(out, " x%d_%d\tcy%d\t%d\n", pi->x[i].n, pi->x[i].c, i, 2*pi->bigM);
299 fprintf(out, " SOS_%d\t'MARKER'\t'SOSEND'\n", sos_cnt++);
301 fprintf(out, " MARKI1\t'MARKER'\t'INTEND'\n"); /* end of marking */
303 /* next the s vars */
304 for (i=0; i<pi->x_dim; ++i) {
305 /* participation in objective */
306 fprintf(out, " s%d_%d\tobj\t%d\n", pi->x[i].n, pi->x[i].c, 1);
308 fprintf(out, " s%d_%d\tcQ%d\t%d\n", pi->x[i].n, pi->x[i].c, i, -1);
311 /* next the y vars */
312 for (i=0; i<pi->x_dim; ++i) {
314 fprintf(out, " y%d_%d\tcQ%d\t%d\n", pi->x[i].n, pi->x[i].c, i, -1);
316 fprintf(out, " y%d_%d\tcy%d\t%d\n", pi->x[i].n, pi->x[i].c, i, 1);
319 fprintf(out, "RHS\n");
320 for (i=0; i<pi->x_dim; ++i)
321 fprintf(out, " rhs\tcQ%d\t%d\n", i, -pi->bigM);
322 for (i=0; i<pi->A_dim; ++i)
323 fprintf(out, " rhs\tcA%d\t%d\n", i, 1);
324 for (i=0; i<pi->B_dim; ++i)
325 fprintf(out, " rhs\tcB%d\t%d\n", i, 1);
326 for (i=0; i<pi->x_dim; ++i)
327 fprintf(out, " rhs\tcy%d\t%d\n", i, 2*pi->bigM);
329 fprintf(out, "ENDATA\n");
332 out = ffopen(pi->name, "mst", "wt");
333 fprintf(out, "NAME\n");
334 for (i=0; i<pi->x_dim; ++i) {
338 if (get_irn_color(get_irn_for_graph_nr(pi->irg, n)) == c)
342 fprintf(out, " x%d_%d\t%d\n", n, c, val);
344 fprintf(out, "ENDATA\n");
351 * Invoke an external solver
353 static void pi_solve_ilp(problem_instance_t *pi) {
356 DBG((dbg, LEVEL_1, "Solving with CPLEX@RZ...\n"));
357 /* write command file for CPLEX */
358 out = ffopen(pi->name, "cmd", "wt");
359 fprintf(out, "read %s.mps\n", pi->name);
360 fprintf(out, "read %s.mst\n", pi->name);
361 fprintf(out, "set mip strategy mipstart 1\n");
362 fprintf(out, "set logfile %s.sol\n", pi->name);
363 fprintf(out, "optimize\n");
364 fprintf(out, "display solution variables 1-%d\n", pi->x_dim);
365 fprintf(out, "set logfile cplex.log\n");
366 fprintf(out, "quit\n");
369 /* write expect-file for copying problem to RZ */
370 out = ffopen(EXPECT_FILENAME, "exp", "wt");
371 fprintf(out, "#! /usr/bin/expect\n");
372 fprintf(out, "spawn scp %s.mps %s.mst %s.cmd %s:\n", pi->name, pi->name, pi->name, SSH_USER_HOST_PATH);
373 fprintf(out, "expect \":\"\n");
374 fprintf(out, "send \"%s\\n\"\n", SSH_PASSWD);
375 fprintf(out, "interact\n");
377 fprintf(out, "spawn ssh %s \"./cplex90 < %s.cmd\"\n", SSH_USER_HOST_PATH, pi->name);
378 fprintf(out, "expect \":\"\n");
379 fprintf(out, "send \"%s\\n\"\n", SSH_PASSWD);
380 fprintf(out, "interact\n");
382 fprintf(out, "spawn scp %s:%s.sol .\n", SSH_USER_HOST_PATH, pi->name);
383 fprintf(out, "expect \":\"\n");
384 fprintf(out, "send \"%s\\n\"\n", SSH_PASSWD);
385 fprintf(out, "interact\n");
388 /* call the expect script */
389 chmod(EXPECT_FILENAME ".exp", 0700);
390 system(EXPECT_FILENAME ".exp");
394 * Sets the colors of irns according to the values of variables found in the
395 * output file of the solver.
397 static void pi_apply_solution(problem_instance_t *pi) {
398 FILE *in = ffopen(pi->name, "sol", "rt");
402 DBG((dbg, LEVEL_1, "Applying solution...\n"));
405 int num = -1, col = -1, val = -1;
407 //TODO No integer feasible solution exists.
409 if (fscanf(in, "x%d_%d %d.%s\n", &num, &col, &val, buf) != 3) {
410 while(fscanf(in, "%1020s\n", buf) != 1);
414 DBG((dbg, LEVEL_1, "x%d_%d = %d\n", num, col, val));
415 set_irn_color(get_irn_for_graph_nr(pi->irg, num), col);
420 #endif /* DO_SOLVE */
423 static void pi_delete_files(problem_instance_t *pi) {
425 int end = snprintf(buf, sizeof(buf), "%s", pi->name);
426 DBG((dbg, LEVEL_1, "Deleting files...\n"));
428 snprintf(buf+end, sizeof(buf)-end, ".matrix");
432 snprintf(buf+end, sizeof(buf)-end, ".mps");
434 snprintf(buf+end, sizeof(buf)-end, ".mst");
436 snprintf(buf+end, sizeof(buf)-end, ".cmd");
438 remove(EXPECT_FILENAME ".exp");
441 snprintf(buf+end, sizeof(buf)-end, ".lp");
448 * Collects all irns in currently processed register class
450 static void pi_collect_x_names(ir_node *block, void *env) {
451 problem_instance_t *pi = env;
452 struct list_head *head = &get_ra_block_info(block)->border_head;
455 list_for_each_entry_reverse(border_t, curr, head, list)
456 if (curr->is_def && curr->is_real) {
458 pi->A_dim++; /* one knapsack constraint for each node */
460 xx.n = get_irn_graph_nr(curr->irn);
461 pi_set_first_pos(pi, xx.n, pi->x_dim);
462 //TODO iterate over all possible colors !!MUST BE IN ORDER!!
463 for (xx.c=0; xx.c<MAX_COLORS; ++xx.c) {
464 if (!is_possible_color(irn, xx.c))
466 DBG((dbg, LEVEL_2, "Adding %n %d\n", curr->irn, xx.c));
467 obstack_grow(&pi->ob, &xx, sizeof(xx));
468 pi->x_dim++; /* one x variable for each node and color */
474 * Checks if all nodes in living are live_out in block block.
476 static INLINE int all_live_out(ir_node *block, pset *living) {
478 for (n = pset_first(living); n; n = pset_next(living))
479 if (!is_live_out(block, n)) {
487 * Finds cliques in the interference graph, considering only nodes
488 * for which the color pi->curr_color is possible. Finds only 'maximal-cliques',
489 * viz cliques which are not conatained in another one.
490 * This is used for the matrix B.
492 static void pi_clique_finder(ir_node *block, void *env) {
493 problem_instance_t *pi = env;
494 enum phase_t {growing, shrinking} phase = growing;
495 struct list_head *head = &get_ra_block_info(block)->border_head;
497 pset *living = pset_new_ptr(SLOTS_LIVING);
499 list_for_each_entry_reverse(border_t, b, head, list) {
500 const ir_node *irn = b->irn;
501 if (!is_possible_color(n, pi->curr_col))
505 DBG((dbg, LEVEL_2, "Def %n\n", irn));
506 pset_insert_ptr(living, irn);
508 } else { /* is_use */
509 DBG((dbg, LEVEL_2, "Use %n\n", irn));
511 /* before shrinking the set, store the current 'maximum' clique;
512 * do NOT if clique is a single node
513 * do NOT if all values are live_out */
514 if (phase == growing && pset_count(living) >= 2 && !all_live_out(block, living)) {
516 for (n = pset_first(living); n; n = pset_next(living)) {
517 int pos = pi_get_pos(pi, get_irn_graph_nr(n), pi->curr_color);
518 matrix_set(pi->B, pi->curr_row, pos, 1);
519 DBG((dbg, LEVEL_2, "B[%d, %d] := %d\n", pi->curr_row, pos, 1));
523 pset_remove_ptr(living, irn);
532 * Generate the initial problem matrices and vectors.
534 static problem_instance_t *new_pi(const copy_opt_t *co) {
535 DBG((dbg, LEVEL_1, "Generating new instance...\n"));
536 problem_instance_t *pi = calloc(1, sizeof(*pi));
538 pi->name = get_entity_name(get_irg_entity(co->irg));
539 pi->num2pos = new_set(set_cmp_num2pos, SLOTS_NUM2POS);
543 * one entry per node and possible color */
544 obstack_init(&pi->ob);
545 dom_tree_walk_irg(co->irg, pi_collect_x_names, NULL, pi);
546 pi->x = obstack_finish(&pi->ob);
549 * weights for the 'same-color-optimization' target */
552 pi->Q = new_matrix(pi->x_dim, pi->x_dim);
554 list_for_each_entry(unit_t, curr, &co->units, units) {
555 const ir_node *root, *arg;
557 unsigned rootpos, argpos;
560 root = curr->nodes[0];
561 rootnr = get_irn_graph_nr(root);
562 rootpos = pi_get_first_pos(pi, rootnr);
563 for (i = 1; i < curr->node_count; ++i) {
564 int weight = -get_weight(root, arg);
565 arg = curr->nodes[i];
566 argnr = get_irn_graph_nr(arg);
567 argpos = pi_get_first_pos(pi, argnr);
569 DBG((dbg, LEVEL_2, "Q[%n, %n] := %d\n", root, arg, weight));
570 /* for all colors root and arg have in common, set the weight for
571 * this pair in the objective function matrix Q */
572 while (rootpos < pi->x_dim && argpos < pi->x_dim &&
573 pi->x[rootpos].n == rootnr && pi->x[argpos].n == argnr) {
574 if (pi->x[rootpos].c < pi->x[argpos].c)
576 else if (pi->x[rootpos].c > pi->x[argpos].c)
579 matrix_set(pi->Q, rootpos++, argpos++, weight);
581 if (weight < pi->minQij) {
582 DBG((dbg, LEVEL_2, "minQij = %d\n", weight));
585 if (weight > pi->maxQij) {
586 DBG((dbg, LEVEL_2, "maxQij = %d\n", weight));
596 * knapsack constraint for each node */
598 int row = 0, col = 0;
599 pi->A = new_matrix(pi->A_dim, pi->x_dim);
600 while (col < pi->x_dim) {
601 int curr_n = pi->x[col].n;
602 while (col < pi->x_dim && pi->x[col].n == curr_n) {
603 DBG((dbg, LEVEL_2, "A[%d, %d] := %d\n", row, col, 1));
604 matrix_set(pi->A, row, col++, 1);
608 assert(row == pi->A_dim);
612 * interference constraints using exactly those cliques not contained in others. */
614 int color, expected_clipques = pi->A_dim/3 * MAX_COLORS;
615 pi->B = new_matrix(expected_clipques, pi->x_dim);
616 for (color = 0; color < MAX_COLORS; ++color) {
617 pi->curr_color = color;
618 dom_tree_walk_irg(pi->irg, pi_clique_finder, NULL, pi);
620 pi->B_dim = matrix_get_rowcount(pi->B);
627 * clean the problem instance
629 static void free_pi(problem_instance_t *pi) {
633 del_set(pi->num2pos);
634 obstack_free(&pi->ob, NULL);
638 void co_ilp_opt(copy_opt_t *co) {
639 dbg = firm_dbg_register("ir.be.copyoptilp");
640 firm_dbg_set_mask(dbg, DEBUG_LVL);
642 problem_instance_t *pi = new_pi(co);
645 pi_dump_matrices(pi);
658 pi_apply_solution(pi);