Fixed the last fix again:
[libfirm] / ir / opt / ldst2.c
1 /*
2  * Copyright (C) 1995-2007 University of Karlsruhe.  All right reserved.
3  *
4  * This file is part of libFirm.
5  *
6  * This file may be distributed and/or modified under the terms of the
7  * GNU General Public License version 2 as published by the Free Software
8  * Foundation and appearing in the file LICENSE.GPL included in the
9  * packaging of this file.
10  *
11  * Licensees holding valid libFirm Professional Edition licenses may use
12  * this file in accordance with the libFirm Commercial License.
13  * Agreement provided with the Software.
14  *
15  * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
16  * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE.
18  */
19
20 /**
21  * @file
22  * @brief   parallelizing Load/Store optimisation
23  * @author  Christoph Mallon
24  * @version $Id: $
25  */
26 #ifdef HAVE_CONFIG_H
27 #include "config.h"
28 #endif
29
30 #include "array.h"
31 #include "debug.h"
32 #include "ircons.h"
33 #include "irgraph.h"
34 #include "irgmod.h"
35 #include "irgopt.h"
36 #include "irgwalk.h"
37 #include "irmemory.h"
38 #include "irnode.h"
39 #include "irnodeset.h"
40 #include "ldst2.h"
41 #include "obst.h"
42 #include "return.h"
43 #include "irdump.h"
44 #include "irflag_t.h"
45
46 #define OPTIMISE_LOAD_AFTER_LOAD
47
48
49 #define UNIMPLEMENTED abort();
50
51
52 DEBUG_ONLY(static firm_dbg_module_t *dbg);
53
54
55 static struct obstack obst;
56 static size_t count_addrs;
57 static ir_node** addrs;
58
59
60 static void AddressCollector(ir_node* node, void* env)
61 {
62         ir_nodeset_t* addrs_set = env;
63         ir_node* addr;
64         if (is_Load(node)) {
65                 addr = get_Load_ptr(node);
66         } else if (is_Store(node)) {
67                 addr = get_Store_ptr(node);
68         } else {
69                 return;
70         }
71         ir_nodeset_insert(addrs_set, addr);
72 }
73
74
75 /* Collects all unique addresses used by load and store nodes of a graph and
76  * puts them into an array for later use */
77 static void CollectAddresses(ir_graph* irg)
78 {
79         ir_nodeset_t addrs_set;
80
81         ir_nodeset_init(&addrs_set);
82         irg_walk_graph(irg, AddressCollector, NULL, &addrs_set);
83
84         count_addrs = ir_nodeset_size(&addrs_set);
85         DB((dbg, LEVEL_1, "===> %+F uses %u unique addresses\n", irg, (unsigned int)count_addrs));
86         if (count_addrs != 0) {
87                 ir_nodeset_iterator_t addr_iter;
88                 size_t i;
89
90                 addrs = NEW_ARR_D(ir_node*, &obst, count_addrs);
91                 ir_nodeset_iterator_init(&addr_iter, &addrs_set);
92                 for (i = 0; i < count_addrs; i++) {
93                         ir_node* addr = ir_nodeset_iterator_next(&addr_iter);
94                         assert(addr != NULL);
95                         set_irn_link(addr, (void *)i);
96                         addrs[i] = addr;
97                         DB((dbg, LEVEL_2, "===> Collected unique symbolic address %+F\n", addr));
98                 }
99         }
100 }
101
102
103 static void AliasSetAdder(ir_node* block, void* env)
104 {
105         ir_nodeset_t* alias_set;
106         size_t i;
107
108         alias_set = NEW_ARR_D(ir_nodeset_t, &obst, count_addrs);
109         for (i = 0; i < count_addrs; i++) {
110                 ir_nodeset_init(&alias_set[i]);
111         }
112         set_irn_link(block, alias_set);
113 }
114
115
116 static void SetStartAddressesTop(ir_graph* irg)
117 {
118         ir_node* initial_mem;
119         ir_node* start_block;
120         ir_nodeset_t* start_addrs;
121         size_t i;
122
123         initial_mem = get_irg_initial_mem(irg);
124         start_block = get_irg_start_block(irg);
125         start_addrs = get_irn_link(start_block);
126         for (i = 0; i < count_addrs; i++) {
127                 ir_nodeset_insert(&start_addrs[i], initial_mem);
128         }
129         mark_Block_block_visited(start_block);
130 }
131
132
133 static void AliasSetDestroyer(ir_node* block, void* env)
134 {
135         ir_nodeset_t* alias_set = get_irn_link(block);
136         size_t i;
137
138         for (i = 0; i < count_addrs; i++) {
139                 ir_nodeset_destroy(&alias_set[i]);
140         }
141 }
142
143
144 static ir_alias_relation AliasTest(ir_graph* irg, ir_node* addr, ir_mode* mode, ir_node* other)
145 {
146         ir_node* other_addr;
147         ir_mode* other_mode;
148
149         if (is_Proj(other)) other = get_Proj_pred(other);
150
151         if (is_Load(other)) {
152                 other_addr = get_Load_ptr(other);
153         } else if (is_Store(other)) {
154                 other_addr = get_Store_ptr(other);
155         } else {
156                 return may_alias;
157         }
158
159         other_mode = get_irn_mode(other);
160         return get_alias_relation(irg, addr, mode, other_addr, other_mode);
161 }
162
163
164 static ir_node* GenerateSync(ir_graph* irg, ir_node* block, ir_nodeset_t* after_set)
165 {
166         size_t set_size = ir_nodeset_size(after_set);
167         ir_nodeset_iterator_t iter;
168
169         assert(set_size != 0);
170
171         ir_nodeset_iterator_init(&iter, after_set);
172         if (set_size == 1) {
173                 return ir_nodeset_iterator_next(&iter);
174         } else {
175                 ir_node** in;
176                 size_t i;
177
178                 NEW_ARR_A(ir_node*, in, set_size);
179                 for (i = 0; i < set_size; i++) {
180                         in[i] = ir_nodeset_iterator_next(&iter);
181                 }
182                 return new_r_Sync(irg, block, set_size, in);
183         }
184 }
185
186
187 static ir_node** unfinished_phis;
188
189
190 static void PlaceMemPhis(ir_graph* irg, ir_node* block, ir_node* phi)
191 {
192         int unfinished = 0;
193         size_t block_n_preds = get_Block_n_cfgpreds(block);
194         ir_nodeset_t* thissets;
195         ir_node** in;
196         size_t i;
197         size_t j;
198
199         thissets = get_irn_link(block);
200         NEW_ARR_A(ir_node*, in, block_n_preds);
201         for (j = 0; j < count_addrs; j++) {
202                 ir_node* new_phi;
203
204                 for (i = 0; i < block_n_preds; i++) {
205                         ir_node* pred_block = get_nodes_block(get_Phi_pred(phi, i)); // TODO get_Block_cfgpred_block(block, i);
206                         ir_nodeset_t* predsets = get_irn_link(pred_block);
207                         size_t predset_size = ir_nodeset_size(&predsets[j]);
208
209                         if (predset_size == 0) {
210                                 in[i] = new_r_Unknown(irg, mode_M);
211                                 unfinished = 1;
212                         } else {
213                                 in[i] = GenerateSync(irg, pred_block, &predsets[j]);
214                         }
215                 }
216                 new_phi = new_r_Phi(irg, block, block_n_preds, in, mode_M);
217                 if (unfinished) {
218                         set_irn_link(new_phi, unfinished_phis[j]);
219                         unfinished_phis[j] = new_phi;
220                 }
221                 ir_nodeset_insert(&thissets[j], new_phi);
222         }
223 }
224
225
226 static int WalkMem(ir_graph* irg, ir_node* node, ir_node* last_block);
227
228
229 static void WalkMemPhi(ir_graph* irg, ir_node* block, ir_node* phi)
230 {
231         size_t n = get_Phi_n_preds(phi);
232         size_t i;
233
234         for (i = 0; i < n; i++) {
235                 WalkMem(irg, get_Phi_pred(phi, i), block);
236         }
237
238         PlaceMemPhis(irg, block, phi);
239         exchange(phi, new_Bad());
240 }
241
242
243 static void PlaceLoad(ir_graph* irg, ir_node* block, ir_node* load, ir_node* memory)
244 {
245         ir_node* addr = get_Load_ptr(load);
246         size_t addr_idx = (size_t)get_irn_link(addr);
247         ir_nodeset_t* interfere_sets = get_irn_link(block);
248         ir_nodeset_t* interfere_set = &interfere_sets[addr_idx];
249         size_t size = ir_nodeset_size(interfere_set);
250         ir_nodeset_iterator_t interfere_iter;
251         size_t i;
252
253         assert(size > 0);
254         ir_nodeset_iterator_init(&interfere_iter, interfere_set);
255         if (size == 1) {
256                 ir_node* after = ir_nodeset_iterator_next(&interfere_iter);
257                 if (is_Proj(after)) {
258                         ir_node* pred = get_Proj_pred(after);
259                         if (is_Load(pred)) {
260 #ifdef OPTIMISE_LOAD_AFTER_LOAD
261                                 if (get_Load_ptr(pred) == addr && get_Load_mode(pred) == get_Load_mode(load)) {
262                                         exchange(load, pred);
263                                         return;
264                                 }
265 #endif
266                                 after = get_Load_mem(pred);
267                         }
268                 }
269                 DB((dbg, LEVEL_3, "===> %+F must be executed after %+F\n", load, after));
270                 set_Load_mem(load, after);
271         } else {
272                 ir_node** after_set;
273                 ir_node* sync;
274
275                 NEW_ARR_A(ir_node*, after_set, size);
276                 for (i = 0; i < size; i++) {
277                         ir_node* mem = ir_nodeset_iterator_next(&interfere_iter);
278                         if (is_Proj(mem)) {
279                                 ir_node* pred = get_Proj_pred(mem);
280                                 if (is_Load(pred)) {
281 #ifdef OPTIMISE_LOAD_AFTER_LOAD
282                                         if (get_Load_ptr(pred) == addr && get_Load_mode(pred) == get_Load_mode(load)) {
283                                                 exchange(load, pred);
284                                                 return;
285                                         }
286 #endif
287                                         mem = get_Load_mem(pred);
288                                 }
289                         }
290                         DB((dbg, LEVEL_3, "===> %+F must be executed after %+F\n", load, mem));
291                         after_set[i] = mem;
292                         sync = new_r_Sync(irg, block, size, after_set);
293                 }
294                 set_Load_mem(load, sync);
295         }
296
297         for (i = 0; i < count_addrs; i++) {
298                 ir_mode* mode = get_Load_mode(load);
299                 ir_node* other_addr = addrs[i];
300                 ir_mode* other_mode = mode; // XXX second mode is nonsense
301                 ir_alias_relation rel = get_alias_relation(irg, addr, mode, other_addr, other_mode);
302                 ir_node* other_node;
303
304                 DB((dbg, LEVEL_3, "===> Testing for alias between %+F and %+F. Relation is %d\n", addr, other_addr, rel));
305                 if (rel == no_alias) {
306                         continue;
307                 }
308                 DB((dbg, LEVEL_3, "===> %+F potentially aliases address %+F\n", load, other_addr));
309
310                 ir_nodeset_iterator_init(&interfere_iter, &interfere_sets[i]);
311                 while ((other_node = ir_nodeset_iterator_next(&interfere_iter)) != NULL) {
312                         if (!is_Proj(other_node) || !is_Store(get_Proj_pred(other_node))) continue;
313                         if (AliasTest(irg, addr, mode, other_node) != no_alias) {
314                                 DB((dbg, LEVEL_3, "===> Removing %+F from execute-after set of %+F due to %+F\n", other_node, addrs[i], load));
315                                 ir_nodeset_remove_iterator(&interfere_sets[i], &interfere_iter);
316                         }
317                 }
318
319                 ir_nodeset_insert(&interfere_sets[i], memory);
320         }
321 }
322
323
324 static void PlaceStore(ir_graph* irg, ir_node* block, ir_node* store, ir_node* memory)
325 {
326         ir_node* addr = get_Store_ptr(store);
327         size_t addr_idx = (size_t)get_irn_link(addr);
328         ir_nodeset_t* interfere_sets = get_irn_link(block);
329         ir_nodeset_t* interfere_set = &interfere_sets[addr_idx];
330         ir_node* after;
331         size_t i;
332
333         after = GenerateSync(irg, block, interfere_set);
334         set_Store_mem(store, after);
335
336         for (i = 0; i < count_addrs; i++) {
337                 ir_nodeset_iterator_t interfere_iter;
338                 ir_mode* mode = get_irn_mode(get_Store_value(store));
339                 ir_node* other_addr = addrs[i];
340                 ir_mode* other_mode = mode; // XXX second mode is nonsense
341                 ir_alias_relation rel = get_alias_relation(irg, addr, mode, other_addr, other_mode);
342                 ir_node* other_node;
343
344                 DB((dbg, LEVEL_3, "===> Testing for alias between %+F and %+F. Relation is %d\n", addr, other_addr, rel));
345                 if (rel == no_alias) {
346                         continue;
347                 }
348                 DB((dbg, LEVEL_3, "===> %+F potentially aliases address %+F\n", store, other_addr));
349
350                 ir_nodeset_iterator_init(&interfere_iter, &interfere_sets[i]);
351                 while ((other_node = ir_nodeset_iterator_next(&interfere_iter)) != NULL) {
352                         if (AliasTest(irg, addr, mode, other_node) != no_alias) {
353                                 DB((dbg, LEVEL_3, "===> Removing %+F from execute-after set of %+F due to %+F\n", other_node, addrs[i], store));
354                                 ir_nodeset_remove_iterator(&interfere_sets[i], &interfere_iter);
355                         }
356                 }
357
358                 ir_nodeset_insert(&interfere_sets[i], memory);
359         }
360 }
361
362
363 static int WalkMem(ir_graph* irg, ir_node* node, ir_node* last_block)
364 {
365         int block_change = 0;
366         ir_node* block = get_nodes_block(node);
367         ir_node* pred;
368         ir_node* memory = node;
369         ir_nodeset_t* addr_sets;
370
371         if (block != last_block) {
372                 DB((dbg, LEVEL_3, "===> Changing block from %+F to %+F\n", last_block, block));
373                 block_change = 1;
374                 if (Block_not_block_visited(block)) {
375                         mark_Block_block_visited(block);
376                 } else {
377                         DB((dbg, LEVEL_2, "===> Hit already visited block at %+F\n", node));
378                         return block_change;
379                 }
380         }
381
382         // Skip projs
383         if (is_Proj(node)) node = get_Proj_pred(node);
384
385         if (is_Phi(node)) {
386                 WalkMemPhi(irg, block, node);
387                 return block_change;
388         } else if (is_Sync(node)) {
389                 UNIMPLEMENTED
390         } else if (is_Return(node)) {
391                 pred = get_Return_mem(node);
392         } else {
393                 pred = get_fragile_op_mem(node);
394         }
395
396         if (WalkMem(irg, pred, block)) {
397                 // There was a block change
398                 size_t block_arity = get_Block_n_cfgpreds(block);
399
400                 DB((dbg, LEVEL_3, "===> There is a block change before %+F\n", node));
401                 if (block_arity == 1) {
402                         // Just one predecessor, inherit its alias sets
403                         ir_node* pred_block = get_nodes_block(pred);
404                         ir_nodeset_t* predsets = get_irn_link(pred_block);
405                         ir_nodeset_t* thissets = get_irn_link(block);
406                         size_t i;
407
408                         DB((dbg, LEVEL_3, "===> Copying the only predecessor's address sets\n"));
409
410                         if (ir_nodeset_size(&predsets[0]) == 0) {
411                                 ir_node* unknown;
412
413                                 DB((dbg, LEVEL_3, "===> The predecessor was not finished yet\n"));
414                                 assert(!Block_not_block_visited(pred_block));
415
416                                 unknown = new_r_Unknown(irg, mode_M);
417                                 for (i = 0; i < count_addrs; i++) {
418                                         ir_node* phi_unk = new_r_Phi(irg, block, 1, &unknown, mode_M);
419                                         DB((dbg, LEVEL_3, "===> Placing unfinished %+F for %+F in %+F\n", phi_unk, addrs[i], block));
420                                         set_irn_link(phi_unk, unfinished_phis[i]);
421                                         unfinished_phis[i] = phi_unk;
422                                         ir_nodeset_insert(&thissets[i], phi_unk);
423                                 }
424                         } else {
425                                 for (i = 0; i < count_addrs; i++) {
426                                         ir_nodeset_iterator_t prediter;
427                                         ir_node* addr;
428
429                                         ir_nodeset_iterator_init(&prediter, &predsets[i]);
430                                         while ((addr = ir_nodeset_iterator_next(&prediter)) != NULL) {
431                                                 ir_nodeset_insert(&thissets[i], addr);
432                                         }
433                                 }
434                         }
435                 }
436         }
437
438         DB((dbg, LEVEL_3, "===> Detotalising %+F\n", node));
439
440         addr_sets = get_irn_link(block);
441
442         if (is_Load(node)) {
443                 PlaceLoad(irg, block, node, memory);
444         } else if (is_Store(node)) {
445                 PlaceStore(irg, block, node, memory);
446         } else {
447                 ir_nodeset_t sync_set;
448                 size_t i;
449                 ir_node* after;
450
451                 DB((dbg, LEVEL_3, "===> Fallback: %+F aliases everything\n", node));
452
453                 ir_nodeset_init(&sync_set);
454                 for (i = 0; i < count_addrs; i++) {
455                         ir_nodeset_iterator_t iter;
456                         ir_node* mem;
457
458                         ir_nodeset_iterator_init(&iter, &addr_sets[i]);
459                         while ((mem = ir_nodeset_iterator_next(&iter)) != NULL) {
460                                 ir_nodeset_insert(&sync_set, mem);
461                         }
462                 }
463
464                 after = GenerateSync(irg, block, &sync_set);
465                 set_irn_n(node, 0, after); // XXX unnice way to set the memory input
466
467                 for (i = 0; i < count_addrs; i++) {
468                         ir_nodeset_iterator_t iter;
469                         ir_nodeset_iterator_init(&iter, &addr_sets[i]);
470                         while (ir_nodeset_iterator_next(&iter) != NULL) {
471                                 ir_nodeset_remove_iterator(&addr_sets[i], &iter);
472                         }
473                         ir_nodeset_insert(&addr_sets[i], memory);
474                 }
475         }
476
477         return block_change;
478 }
479
480
481 static void FinalisePhis(ir_graph* irg)
482 {
483         size_t i;
484
485         for (i = 0; i < count_addrs; i++) {
486                 ir_node* next_phi;
487                 ir_node* phi;
488
489                 for (phi = unfinished_phis[i]; phi != NULL; phi = next_phi) {
490                         ir_node* block = get_nodes_block(phi);
491                         size_t block_n_preds = get_Block_n_cfgpreds(block);
492
493                         next_phi = get_irn_link(phi);
494
495                         DB((dbg, LEVEL_4, "===> Finialising phi %+F in %+F\n", phi, block));
496
497                         if (block_n_preds == 1) {
498                                 ir_node* pred_block = get_Block_cfgpred_block(block, 0);
499                                 ir_nodeset_t* pred_sets = get_irn_link(pred_block);
500                                 ir_node* after = GenerateSync(irg, pred_block, &pred_sets[i]);
501
502                                 assert(is_Unknown(get_Phi_pred(phi, 0)));
503                                 exchange(phi, after);
504                         } else {
505                                 ir_node** in;
506                                 size_t j;
507
508                                 NEW_ARR_A(ir_node*, in, block_n_preds);
509                                 for (j = 0; j < block_n_preds; j++) {
510                                         ir_node* pred_block = get_Block_cfgpred_block(block, j);
511                                         ir_nodeset_t* pred_sets = get_irn_link(pred_block);
512
513                                         if (is_Unknown(get_Phi_pred(phi, j))) {
514                                                 set_Phi_pred(phi, j, GenerateSync(irg, pred_block, &pred_sets[i]));
515                                         }
516                                 }
517                         }
518                 }
519         }
520 }
521
522
523 static void Detotalise(ir_graph* irg)
524 {
525         ir_node* end_block = get_irg_end_block(irg);
526         size_t npreds = get_Block_n_cfgpreds(end_block);
527         size_t i;
528
529         unfinished_phis = xmalloc(sizeof(*unfinished_phis) * count_addrs);
530         for (i = 0; i < count_addrs; i++) {
531                 unfinished_phis[i] = NULL;
532         }
533
534         for (i = 0; i < npreds; i++) {
535                 ir_node* pred = get_Block_cfgpred(end_block, i);
536                 assert(is_Return(pred));
537                 DB((dbg, LEVEL_2, "===> Starting memory walk at %+F\n", pred));
538                 WalkMem(irg, pred, NULL);
539         }
540
541         FinalisePhis(irg);
542         xfree(unfinished_phis);
543 }
544
545
546 static void AddSyncPreds(ir_nodeset_t* preds, ir_node* sync)
547 {
548         size_t n = get_Sync_n_preds(sync);
549         size_t i;
550
551         for (i = 0; i < n; i++) {
552                 ir_node* pred = get_Sync_pred(sync, i);
553                 if (is_Sync(pred)) {
554                         AddSyncPreds(preds, pred);
555                 } else {
556                         ir_nodeset_insert(preds, pred);
557                 }
558         }
559 }
560
561
562 static void NormaliseSync(ir_node* node, void* env)
563 {
564         ir_nodeset_t preds;
565         ir_nodeset_iterator_t iter;
566         ir_node** in;
567         size_t count_preds;
568         size_t i;
569
570         if (!is_Sync(node)) return;
571
572         ir_nodeset_init(&preds);
573         AddSyncPreds(&preds, node);
574
575         count_preds = ir_nodeset_size(&preds);
576         if (count_preds != get_Sync_n_preds(node)) {
577                 NEW_ARR_A(ir_node*, in, count_preds);
578                 ir_nodeset_iterator_init(&iter, &preds);
579                 for (i = 0; i < count_preds; i++) {
580                         ir_node* pred = ir_nodeset_iterator_next(&iter);
581                         assert(pred != NULL);
582                         in[i] = pred;
583                 }
584                 set_irn_in(node, count_preds, in);
585         }
586
587         ir_nodeset_destroy(&preds);
588 }
589
590
591 void opt_ldst2(ir_graph* irg)
592 {
593         FIRM_DBG_REGISTER(dbg, "firm.opt.ldst2");
594         DB((dbg, LEVEL_1, "===> Performing load/store optimisation on %+F\n", irg));
595
596         normalize_one_return(irg);
597
598         obstack_init(&obst);
599
600         if (1 /* XXX */ || get_opt_alias_analysis()) {
601                 assure_irg_address_taken_computed(irg);
602                 assure_irp_globals_address_taken_computed();
603         }
604
605
606         CollectAddresses(irg);
607         if (count_addrs == 0) return;
608
609         irg_block_walk_graph(irg, AliasSetAdder, NULL, NULL);
610         inc_irg_block_visited(irg);
611         SetStartAddressesTop(irg);
612         Detotalise(irg);
613
614         dump_ir_block_graph(irg, "-fluffig");
615
616         irg_block_walk_graph(irg, AliasSetDestroyer, NULL, NULL);
617         obstack_free(&obst, NULL);
618
619         normalize_proj_nodes(irg);
620         irg_walk_graph(irg, NormaliseSync, NULL, NULL);
621   optimize_graph_df(irg);
622         irg_walk_graph(irg, NormaliseSync, NULL, NULL);
623 }