fixed precedence constraint
[libfirm] / ir / be / bespillslots.c
1 /*
2  * Author:      Matthias Braun
3  * Date:                26.7.06
4  * Copyright:   (c) Universitaet Karlsruhe
5  * Licence:     This file protected by GPL -  GNU GENERAL PUBLIC LICENSE.
6  */
7 #ifdef HAVE_CONFIG_H
8 #include <config.h>
9 #endif
10
11 #include <stdlib.h>
12
13 #include "set.h"
14
15 #include "irgwalk.h"
16 #include "ircons.h"
17 #include "irprintf.h"
18 #include "execfreq.h"
19 #include "unionfind.h"
20 #include "type.h"
21 #include "irdump_t.h"
22
23 #include "benode_t.h"
24 #include "besched.h"
25 #include "bespillslots.h"
26 #include "bechordal_t.h"
27 #include "bejavacoal.h"
28 #include "benodesets.h"
29 #include "bestatevent.h"
30 #include "bespilloptions.h"
31
32 #define DBG_COALESCING          1
33 #define DBG_INTERFERENCES       2
34
35 DEBUG_ONLY(
36 static firm_dbg_module_t *dbg = NULL;
37 )
38
39 typedef struct _spill_t {
40         ir_node *spill;
41         /** mode of the spilled value */
42         const ir_mode *mode;
43         /** alignment for the spilled value */
44         int alignment;
45         /** index into spillslot_unionfind unionfind structure */
46         int spillslot;
47 } spill_t;
48
49 typedef struct _affinity_edge_t {
50         double affinity;
51         int slot1, slot2;
52 } affinity_edge_t;
53
54 struct _be_fec_env_t {
55         struct obstack obst;
56         const arch_env_t *arch_env;
57         be_irg_t *birg;
58         set *spills;
59         ir_node **reloads;
60         affinity_edge_t **affinity_edges;
61         set *memperms;
62 };
63
64 /** Compare 2 affinity edges (used in quicksort) */
65 static int cmp_affinity(const void *d1, const void *d2)
66 {
67         const affinity_edge_t * const *e1 = d1;
68         const affinity_edge_t * const *e2 = d2;
69
70         // sort in descending order
71         return (*e1)->affinity < (*e2)->affinity ? 1 : -1;
72 }
73
74 static int cmp_spill(const void* d1, const void* d2, size_t size)
75 {
76         const spill_t* s1 = d1;
77         const spill_t* s2 = d2;
78         return s1->spill != s2->spill;
79 }
80
81 static spill_t *get_spill(be_fec_env_t *env, ir_node *node)
82 {
83         spill_t spill, *res;
84         int hash = nodeset_hash(node);
85
86         spill.spill = node;
87         res = set_find(env->spills, &spill, sizeof(spill), hash);
88
89         return res;
90 }
91
92 /*
93  *   ____      _ _           _     ____        _ _ _
94  *  / ___|___ | | | ___  ___| |_  / ___| _ __ (_) | |___
95  * | |   / _ \| | |/ _ \/ __| __| \___ \| '_ \| | | / __|
96  * | |__| (_) | | |  __/ (__| |_   ___) | |_) | | | \__ \
97  *  \____\___/|_|_|\___|\___|\__| |____/| .__/|_|_|_|___/
98  *                                      |_|
99  */
100
101 static INLINE ir_node *get_memory_edge(const ir_node *node)
102 {
103         int i, arity;
104
105         arity = get_irn_arity(node);
106         for(i = arity - 1; i >= 0; --i) {
107                 ir_node *arg = get_irn_n(node, i);
108                 if(get_irn_mode(arg) == mode_M)
109                         return arg;
110         }
111
112         return NULL;
113 }
114
115 static spill_t *collect_spill(be_fec_env_t *env, ir_node *node,
116                                       const ir_mode *mode, int align)
117 {
118         spill_t spill, *res;
119         int hash = nodeset_hash(node);
120
121         /* insert into set of spills if not already there */
122         spill.spill = node;
123         res = set_find(env->spills, &spill, sizeof(spill), hash);
124
125         if(res == NULL) {
126                 spill.spillslot = set_count(env->spills);
127                 spill.mode = mode;
128                 spill.alignment = align;
129                 res = set_insert(env->spills, &spill, sizeof(spill), hash);
130         } else {
131                 assert(res->mode == mode);
132                 assert(res->alignment == align);
133         }
134
135         return res;
136 }
137
138 static spill_t *collect_memphi(be_fec_env_t *env, ir_node *node,
139                                const ir_mode *mode, int align)
140 {
141         int i, arity;
142         spill_t spill, *res;
143         int hash = nodeset_hash(node);
144         const ir_exec_freq *exec_freq = be_get_birg_exec_freq(env->birg);
145
146         assert(is_Phi(node));
147
148         spill.spill = node;
149         res = set_find(env->spills, &spill, sizeof(spill), hash);
150         if(res != NULL) {
151                 assert(res->mode == mode);
152                 assert(res->alignment == align);
153                 return res;
154         }
155
156         spill.spillslot = set_count(env->spills);
157         spill.mode = mode;
158         spill.alignment = align;
159         res = set_insert(env->spills, &spill, sizeof(spill), hash);
160
161         // collect attached spills and mem-phis
162         arity = get_irn_arity(node);
163         for(i = 0; i < arity; ++i) {
164                 affinity_edge_t *affinty_edge;
165                 ir_node *arg = get_irn_n(node, i);
166                 spill_t *arg_spill;
167
168                 if(is_Phi(arg)) {
169                         arg_spill = collect_memphi(env, arg, mode, align);
170                 } else {
171                         arg_spill = collect_spill(env, arg, mode, align);
172                 }
173
174                 // add an affinity edge
175                 affinty_edge = obstack_alloc(&env->obst, sizeof(affinty_edge[0]));
176                 affinty_edge->affinity = get_block_execfreq(exec_freq, get_nodes_block(arg));
177                 affinty_edge->slot1 = res->spillslot;
178                 affinty_edge->slot2 = arg_spill->spillslot;
179                 ARR_APP1(affinity_edge_t*, env->affinity_edges, affinty_edge);
180         }
181
182         return res;
183 }
184
185 void be_node_needs_frame_entity(be_fec_env_t *env, ir_node *node,
186                                 const ir_mode *mode, int align)
187 {
188         ir_node *spillnode = get_memory_edge(node);
189         spill_t *spill;
190
191         assert(spillnode != NULL);
192
193         if (is_Phi(spillnode)) {
194                 spill = collect_memphi(env, spillnode, mode, align);
195         } else {
196                 spill = collect_spill(env, spillnode, mode, align);
197         }
198
199         ARR_APP1(ir_node *, env->reloads, node);
200 }
201
202 /*
203  *   ____            _                      ____  _       _
204  *  / ___|___   __ _| | ___  ___  ___ ___  / ___|| | ___ | |_ ___
205  * | |   / _ \ / _` | |/ _ \/ __|/ __/ _ \ \___ \| |/ _ \| __/ __|
206  * | |__| (_) | (_| | |  __/\__ \ (_|  __/  ___) | | (_) | |_\__ \
207  *  \____\___/ \__,_|_|\___||___/\___\___| |____/|_|\___/ \__|___/
208  */
209
210 static int merge_interferences(be_fec_env_t *env, bitset_t** interferences,
211                                int* spillslot_unionfind, int s1, int s2)
212 {
213         int res;
214         int i;
215         int spillcount;
216
217         // merge spillslots and interferences
218         res = uf_union(spillslot_unionfind, s1, s2);
219         // we assume that we always merge s2 to s1 so swap s1, s2 if necessary
220         if(res != 0) {
221                 int t = s1;
222                 s1 = s2;
223                 s2 = t;
224         }
225
226         bitset_or(interferences[s1], interferences[s2]);
227
228         // update other interferences
229         spillcount = set_count(env->spills);
230         for(i = 0; i < spillcount; ++i) {
231                 bitset_t *intfs = interferences[i];
232                 if(bitset_is_set(intfs, s2))
233                         bitset_set(intfs, s1);
234         }
235
236         return res;
237 }
238
239 /**
240  * A greedy coalescing algorithm for spillslots:
241  *  1. Sort the list of affinity edges
242  *  2. Try to merge slots with affinity edges (most expensive slots first)
243  *  3. Try to merge everything else that is possible
244  */
245 static void do_greedy_coalescing(be_fec_env_t *env)
246 {
247         int spillcount;
248         spill_t **spilllist;
249         spill_t *spill;
250         int i, i2;
251         int affinity_edge_count;
252         bitset_t **interferences;
253         int* spillslot_unionfind;
254         const be_lv_t *lv = be_get_birg_liveness(env->birg);
255
256         spillcount = set_count(env->spills);
257         if(spillcount == 0)
258                 return;
259
260         DBG((dbg, DBG_COALESCING, "Coalescing %d spillslots\n", spillcount));
261
262         interferences = alloca(spillcount * sizeof(interferences[0]));
263         spillslot_unionfind = alloca(spillcount * sizeof(spillslot_unionfind[0]));
264         spilllist = alloca(spillcount * sizeof(spilllist[0]));
265
266         uf_init(spillslot_unionfind, 0, spillcount);
267
268         DEBUG_ONLY(
269                 memset(spilllist, 0, spillcount * sizeof(spilllist[0]));
270         );
271
272         for(spill = set_first(env->spills), i = 0; spill != NULL; spill = set_next(env->spills), ++i) {
273                 assert(spill->spillslot < spillcount);
274                 spilllist[spill->spillslot] = spill;
275         }
276
277         for(i = 0; i < spillcount; ++i) {
278                 interferences[i] = bitset_alloca(spillcount);
279         }
280
281         // construct interferences
282         for(i = 0; i < spillcount; ++i) {
283                 for(i2 = i+1; i2 < spillcount; ++i2) {
284                         if(values_interfere(lv, spilllist[i]->spill, spilllist[i2]->spill)) {
285                                 DBG((dbg, DBG_INTERFERENCES, "Slot %d and %d interfere\n", i, i2));
286                                 bitset_set(interferences[i], i2);
287                                 bitset_set(interferences[i2], i);
288                         }
289                 }
290         }
291
292         // sort affinity edges
293         affinity_edge_count = ARR_LEN(env->affinity_edges);
294         qsort(env->affinity_edges, affinity_edge_count, sizeof(env->affinity_edges[0]), cmp_affinity);
295
296         //dump_interference_graph(env, interferences, "before");
297
298         // try to merge affine nodes
299         for(i = 0; i < affinity_edge_count; ++i) {
300                 const affinity_edge_t *edge = env->affinity_edges[i];
301                 int s1 = uf_find(spillslot_unionfind, edge->slot1);
302                 int s2 = uf_find(spillslot_unionfind, edge->slot2);
303
304                 /* test if values interfere */
305                 if(bitset_is_set(interferences[s1], s2)) {
306                         assert(bitset_is_set(interferences[s2], s1));
307                         continue;
308                 }
309
310                 DBG((dbg, DBG_COALESCING, "Merging %d and %d because of affinity edge\n", s1, s2));
311
312                 merge_interferences(env, interferences, spillslot_unionfind, s1, s2);
313         }
314
315         // try to merge as much remaining spillslots as possible
316         for(i = 0; i < spillcount; ++i) {
317                 int s1 = uf_find(spillslot_unionfind, i);
318                 if(s1 != i)
319                         continue;
320
321                 for(i2 = i+1; i2 < spillcount; ++i2) {
322                         int s2 = uf_find(spillslot_unionfind, i2);
323                         if(s2 != i2)
324                                 continue;
325
326                         /* test if values interfere
327                          * we have to test n1-n2 and n2-n1, because only 1 side gets updated
328                          * when node merging occurs
329                          */
330                         if(bitset_is_set(interferences[s1], s2)) {
331                                 assert(bitset_is_set(interferences[s2], s1));
332                                 continue;
333                         }
334
335                         DBG((dbg, DBG_COALESCING, "Merging %d and %d because it is possible\n", s1, s2));
336
337                         if(merge_interferences(env, interferences, spillslot_unionfind, s1, s2) != 0) {
338                                 // we can break the loop here, because s2 is the new supernode now
339                                 // and we'll test s2 again later anyway
340                                 break;
341                         }
342                 }
343         }
344
345         // assign spillslots to spills
346         for(i = 0; i < spillcount; ++i) {
347                 spill_t *spill = spilllist[i];
348
349                 spill->spillslot = uf_find(spillslot_unionfind, i);
350         }
351
352         //dump_interference_graph(env, interferences, "after");
353 }
354
355 /*
356  *     _            _               _____       _   _ _   _
357  *    / \   ___ ___(_) __ _ _ __   | ____|_ __ | |_(_) |_(_) ___  ___
358  *   / _ \ / __/ __| |/ _` | '_ \  |  _| | '_ \| __| | __| |/ _ \/ __|
359  *  / ___ \\__ \__ \ | (_| | | | | | |___| | | | |_| | |_| |  __/\__ \
360  * /_/   \_\___/___/_|\__, |_| |_| |_____|_| |_|\__|_|\__|_|\___||___/
361  *                    |___/
362  */
363
364 typedef struct _spill_slot_t {
365         int size;
366         int align;
367         ir_entity *entity;
368 } spill_slot_t;
369
370 typedef struct _memperm_entry_t {
371         ir_node* node;
372         int pos;
373         ir_entity *in;
374         ir_entity *out;
375         struct _memperm_entry_t *next;
376 } memperm_entry_t;
377
378 typedef struct _memperm_t {
379         ir_node *block;
380         int entrycount;
381         memperm_entry_t *entries;
382 } memperm_t;
383
384 static int cmp_memperm(const void* d1, const void* d2, size_t size)
385 {
386         const memperm_t* e1 = d1;
387         const memperm_t* e2 = d2;
388         return e1->block != e2->block;
389 }
390
391 static memperm_t *get_memperm(be_fec_env_t *env, ir_node *block)
392 {
393         memperm_t entry, *res;
394         int hash;
395
396         entry.block = block;
397         hash = nodeset_hash(block);
398
399         res = set_find(env->memperms, &entry, sizeof(entry), hash);
400
401         if(res == NULL) {
402                 entry.entrycount = 0;
403                 entry.entries = NULL;
404                 res = set_insert(env->memperms, &entry, sizeof(entry), hash);
405         }
406
407         return res;
408 }
409
410 static ir_entity* create_stack_entity(be_fec_env_t *env, spill_slot_t *slot)
411 {
412         ir_graph *irg = be_get_birg_irg(env->birg);
413         ir_type *frame = get_irg_frame_type(irg);
414         ir_entity *res = frame_alloc_area(frame, slot->size, slot->align, 0);
415
416         // adjust size of the entity type...
417         ir_type *enttype = get_entity_type(res);
418         set_type_size_bytes(enttype, slot->size);
419
420         slot->entity = res;
421
422         return res;
423 }
424
425 /**
426  * Enlarges a spillslot (if necessary) so that it can carry a value of size
427  * @p othersize and alignment @p otheralign.
428  */
429 static void enlarge_spillslot(spill_slot_t *slot, int otheralign, int othersize)
430 {
431         if(othersize > slot->size) {
432                 slot->size = othersize;
433         }
434         if(otheralign > slot->align) {
435                 if(otheralign % slot->align != 0)
436                         slot->align *= otheralign;
437                 else
438                         slot->align = otheralign;
439         } else if(slot->align % otheralign != 0) {
440                 slot->align *= otheralign;
441         }
442 }
443
444 /**
445  * Create stack entities for the spillslots and assign them to the spill and
446  * reload nodes.
447  */
448 static void assign_spillslots(be_fec_env_t *env)
449 {
450         const arch_env_t *arch_env = env->arch_env;
451         int i;
452         int spillcount;
453         spill_t *spill;
454         spill_slot_t* spillslots;
455
456         spillcount = set_count(env->spills);
457         spillslots = alloca(spillcount * sizeof(spillslots[0]));
458
459         memset(spillslots, 0, spillcount * sizeof(spillslots[0]));
460
461         // construct spillslots
462         for(spill = set_first(env->spills); spill != NULL; spill = set_next(env->spills)) {
463                 int slotid = spill->spillslot;
464                 const ir_mode *mode = spill->mode;
465                 spill_slot_t *slot = & (spillslots[slotid]);
466                 int size = get_mode_size_bytes(mode);
467                 int align = spill->alignment;
468
469                 if(slot->align == 0 && slot->size == 0) {
470                         slot->align = align;
471                         slot->size = size;
472                 } else {
473                         enlarge_spillslot(slot, align, size);
474                 }
475         }
476
477         for(spill = set_first(env->spills); spill != NULL; spill = set_next(env->spills)) {
478                 spill_slot_t *slot;
479                 ir_node *node = spill->spill;
480                 int slotid = spill->spillslot;
481
482                 slot = &spillslots[slotid];
483                 if(slot->entity == NULL) {
484                         create_stack_entity(env, slot);
485                 }
486
487                 if(is_Phi(node)) {
488                         int i, arity;
489                         ir_node *block = get_nodes_block(node);
490
491                         // should be a PhiM
492                         assert(is_Phi(node));
493
494                         for(i = 0, arity = get_irn_arity(node); i < arity; ++i) {
495                                 ir_node *arg = get_irn_n(node, i);
496                                 ir_node *predblock = get_Block_cfgpred_block(block, i);
497                                 spill_t *argspill;
498                                 int argslotid;
499
500                                 argspill = get_spill(env, arg);
501                                 assert(argspill != NULL);
502
503                                 argslotid = argspill->spillslot;
504                                 if(slotid != argslotid) {
505                                         memperm_t *memperm;
506                                         memperm_entry_t *entry;
507                                         spill_slot_t *argslot = &spillslots[argslotid];
508                                         if(argslot->entity == NULL) {
509                                                 create_stack_entity(env, argslot);
510                                         }
511
512                                         memperm = get_memperm(env, predblock);
513
514                                         entry = obstack_alloc(&env->obst, sizeof(entry[0]));
515                                         entry->node = node;
516                                         entry->pos = i;
517                                         entry->in = argslot->entity;
518                                         entry->out = slot->entity;
519                                         entry->next = memperm->entries;
520                                         memperm->entrycount++;
521                                         memperm->entries = entry;
522                                 }
523                         }
524                 } else {
525                         arch_set_frame_entity(arch_env, node, slot->entity);
526                 }
527         }
528
529         for(i = 0; i < ARR_LEN(env->reloads); ++i) {
530                 ir_node* reload = env->reloads[i];
531                 ir_node* spillnode = get_memory_edge(reload);
532                 spill_t *spill = get_spill(env, spillnode);
533                 const spill_slot_t *slot = & spillslots[spill->spillslot];
534
535                 assert(slot->entity != NULL);
536
537                 arch_set_frame_entity(arch_env, reload, slot->entity);
538         }
539 }
540
541 /**
542  * Returns the last node in a block which is no control flow changing node
543  */
544 static ir_node *get_end_of_block_insertion_point(ir_node* block)
545 {
546         ir_node* ins = sched_last(block);
547         while(is_Proj(ins) && get_irn_mode(ins) == mode_X) {
548                 ins = sched_prev(ins);
549                 assert(ins != NULL);
550         }
551
552         if(is_cfop(ins)) {
553                 while(1) {
554                         ir_node *prev = sched_prev(ins);
555                         if(!is_cfop(prev))
556                                 break;
557                         ins = prev;
558                 }
559         }
560
561         return ins;
562 }
563
564 static void create_memperms(be_fec_env_t *env)
565 {
566         const arch_env_t *arch_env = env->arch_env;
567         ir_graph *irg = be_get_birg_irg(env->birg);
568         memperm_t *memperm;
569
570         for(memperm = set_first(env->memperms); memperm != NULL; memperm = set_next(env->memperms)) {
571                 int i;
572                 memperm_entry_t *entry;
573                 ir_node *blockend;
574                 ir_node** nodes = alloca(memperm->entrycount * sizeof(nodes[0]));
575                 ir_node* mempermnode;
576
577                 assert(memperm->entrycount > 0);
578
579                 for(entry = memperm->entries, i = 0; entry != NULL; entry = entry->next, ++i) {
580                         ir_node* arg = get_irn_n(entry->node, entry->pos);
581                         nodes[i] = arg;
582                 }
583
584                 mempermnode = be_new_MemPerm(arch_env, irg, memperm->block,
585                                              memperm->entrycount, nodes);
586
587                 // insert node into schedule
588                 blockend = get_end_of_block_insertion_point(memperm->block);
589                 sched_add_before(blockend, mempermnode);
590                 be_stat_ev("mem_perm", memperm->entrycount);
591
592                 i = 0;
593                 for(entry = memperm->entries; entry != NULL; entry = entry->next, ++i) {
594                         ir_node *proj;
595                         ir_node* arg = get_irn_n(entry->node, entry->pos);
596
597                         be_set_MemPerm_in_entity(mempermnode, i, entry->in);
598                         be_set_MemPerm_out_entity(mempermnode, i, entry->out);
599                         set_irg_current_block(irg, memperm->block);
600                         proj = new_Proj(mempermnode, get_irn_mode(arg), i);
601                         sched_add_before(blockend, proj);
602
603                         set_irn_n(entry->node, entry->pos, proj);
604                 }
605         }
606 }
607
608 static int count_spillslots(const be_fec_env_t *env)
609 {
610         const spill_t *spill;
611         int spillcount = set_count(env->spills);
612         bitset_t *counted = bitset_alloca(spillcount);
613         int slotcount;
614
615         slotcount = 0;
616         for(spill = set_first(env->spills); spill != NULL;
617             spill = set_next(env->spills)) {
618                 int spillslot = spill->spillslot;
619                 if(!bitset_is_set(counted, spillslot)) {
620                         slotcount++;
621                         bitset_set(counted, spillslot);
622                 }
623         }
624
625         return slotcount;
626 }
627
628 be_fec_env_t *be_new_frame_entity_coalescer(be_irg_t *birg)
629 {
630         const arch_env_t *arch_env = birg->main_env->arch_env;
631
632         be_fec_env_t *env = xmalloc(sizeof(env[0]));
633
634         be_assure_liveness(birg);
635
636         obstack_init(&env->obst);
637         env->arch_env = arch_env;
638         env->birg = birg;
639         env->spills = new_set(cmp_spill, 10);
640         env->reloads = NEW_ARR_F(ir_node*, 0);
641         env->affinity_edges = NEW_ARR_F(affinity_edge_t*, 0);
642         env->memperms = new_set(cmp_memperm, 10);
643         FIRM_DBG_REGISTER(dbg, "firm.be.spillslots");
644
645         return env;
646 }
647
648 void be_free_frame_entity_coalescer(be_fec_env_t *env)
649 {
650         del_set(env->memperms);
651         DEL_ARR_F(env->reloads);
652         DEL_ARR_F(env->affinity_edges);
653         del_set(env->spills);
654         obstack_free(&env->obst, NULL);
655
656         free(env);
657 }
658
659 void be_assign_entities(be_fec_env_t *env)
660 {
661         if(be_stat_ev_is_active()) {
662                 int count = set_count(env->spills);
663                 be_stat_ev("spillslots", count);
664         }
665
666         if(be_coalesce_spill_slots) {
667                 do_greedy_coalescing(env);
668         }
669
670         if(be_stat_ev_is_active()) {
671                 int count = count_spillslots(env);
672                 be_stat_ev("spillslots_after_coalescing", count);
673         }
674
675         assign_spillslots(env);
676
677         create_memperms(env);
678 }
679
680 /**
681  * This walker function searches for reloads and collects all the spills
682  * and memphis attached to them.
683  */
684 static void collect_spills_walker(ir_node *node, void *data)
685 {
686         be_fec_env_t *env = data;
687         const arch_env_t *arch_env = env->arch_env;
688         const ir_mode *mode;
689         const arch_register_class_t *cls;
690         int align;
691
692         /* classify returns classification of the irn the proj is attached to */
693         if (is_Proj(node))
694                 return;
695
696         if (!arch_irn_class_is(arch_env, node, reload))
697                 return;
698
699         mode = get_irn_mode(node);
700         cls = arch_get_irn_reg_class(arch_env, node, -1);
701         align = arch_isa_get_reg_class_alignment(arch_env_get_isa(arch_env), cls);
702
703         be_node_needs_frame_entity(env, node, mode, align);
704 }
705
706 void be_coalesce_spillslots(be_irg_t *birg)
707 {
708         be_fec_env_t *env = be_new_frame_entity_coalescer(birg);
709
710         /* collect reloads */
711         irg_walk_graph(birg->irg, NULL, collect_spills_walker, env);
712
713         be_assign_entities(env);
714
715         be_free_frame_entity_coalescer(env);
716 }