2 * Copyright (C) 1995-2008 University of Karlsruhe. All right reserved.
4 * This file is part of libFirm.
6 * This file may be distributed and/or modified under the terms of the
7 * GNU General Public License version 2 as published by the Free Software
8 * Foundation and appearing in the file LICENSE.GPL included in the
9 * packaging of this file.
11 * Licensees holding valid libFirm Professional Edition licenses may use
12 * this file in accordance with the libFirm Commercial License.
13 * Agreement provided with the Software.
15 * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
16 * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * @brief Dataflow driven Load/Store optimizations, uses some ideas from
24 * @author Michael Beck
39 #include "irnodemap.h"
40 #include "raw_bitset.h"
45 * Mapping an address to an dense ID.
47 typedef struct address_entry_t {
48 unsigned id; /**< The ID */
55 FLAG_KILL_LOADS = 1, /**< KILL all Loads */
56 FLAG_KILL_STORES = 2, /**< KILL all Stores */
57 FLAG_KILLED_NODE = 4, /**< this node was killed */
58 FLAG_EXCEPTION = 8, /**< this node has exception flow */
59 FLAG_IGNORE = 16, /**< ignore this node (volatile or other) */
60 /** this memop KILLS all addresses */
61 FLAG_KILL_ALL = FLAG_KILL_LOADS|FLAG_KILL_STORES
65 * A value: This represents a value stored at a given address in
66 * memory. Do not confuse with values from value numbering.
68 typedef struct value_t value_t;
70 ir_node *address; /**< the address of this value */
71 ir_node *value; /**< the value itself */
72 ir_mode *mode; /**< the mode of the value */
73 unsigned id; /**< address id */
77 * A memop describes an memory-related operation.
78 * These are Loads/Store and all other ops that might modify
79 * memory (Calls, CopyB) or causing exceptions.
81 typedef struct memop_t memop_t;
83 value_t value; /**< the value of this memop: only defined for Load/Store */
84 ir_node *node; /**< the memory op itself */
85 ir_node *mem; /**< the memory FROM this node */
86 ir_node *replace; /**< the replacement node if this memop is replaced */
87 memop_t *next; /**< links to the next memory op in the block in forward order. */
88 memop_t *prev; /**< links to the previous memory op in the block in forward order. */
89 unsigned flags; /**< memop flags */
93 * Additional data for every basic block.
95 typedef struct block_t block_t;
97 memop_t *memop_forward; /**< topologically sorted list of memory ops in this block */
98 memop_t *memop_backward; /**< last memop in the list */
99 unsigned *avail_out; /**< out-set of available addresses */
100 memop_t **id_2_memop_avail; /**< maps avail address ids to memops */
101 unsigned *anticL_in; /**< in-set of anticipated Load addresses */
102 memop_t **id_2_memop; /**< maps address ids to memops */
103 ir_node *block; /**< the associated block */
104 block_t *forward_next; /**< next block entry for forward iteration */
105 block_t *backward_next; /**< next block entry for backward iteration */
106 memop_t *avail; /**< used locally for the avail map */
110 * Metadata for this pass.
112 typedef struct ldst_env_t {
113 struct obstack obst; /**< obstack for temporary data */
114 ir_nodemap_t adr_map; /**< Map addresses to */
115 block_t *forward; /**< Inverse post-order list of all blocks Start->End */
116 block_t *backward; /**< Inverse post-order list of all blocks End->Start */
117 ir_node *start_bl; /**< start block of the current graph */
118 ir_node *end_bl; /**< end block of the current graph */
119 unsigned *curr_set; /**< current set of addresses */
120 unsigned curr_adr_id; /**< number for address mapping */
121 unsigned n_mem_ops; /**< number of memory operations (Loads/Stores) */
122 unsigned rbs_size; /**< size of all bitsets in bytes */
123 int changed; /**< Flags for changed graph state */
128 static firm_dbg_module_t *dbg;
130 /* the one and only environment */
134 * Dumps the block list.
136 * @param ldst environment
138 static void dump_block_list(ldst_env *env) {
143 for (entry = env->forward; entry != NULL; entry = entry->forward_next) {
144 DB((dbg, LEVEL_2, "%+F {", entry->block));
147 for (op = entry->memop_forward; op != NULL; op = op->next) {
149 DB((dbg, LEVEL_2, "\n\t"));
150 } DB((dbg, LEVEL_2, "%+F", op->node));
151 if ((op->flags & FLAG_KILL_ALL) == FLAG_KILL_ALL)
152 DB((dbg, LEVEL_2, "X"));
153 else if (op->flags & FLAG_KILL_LOADS)
154 DB((dbg, LEVEL_2, "L"));
155 else if (op->flags & FLAG_KILL_STORES)
156 DB((dbg, LEVEL_2, "S"));
157 DB((dbg, LEVEL_2, ", "));
161 DB((dbg, LEVEL_2, "\n}\n\n"));
166 * Dumps the current set.
168 * @param bl current block
169 * @param s name of the set
171 static void dump_curr(block_t *bl, const char *s) {
173 unsigned end = env.n_mem_ops * 2 - 1;
176 DB((dbg, LEVEL_2, "%s[%+F] = {", s, bl->block));
178 for (pos = rbitset_next(env.curr_set, pos, 1); pos != end; pos = rbitset_next(env.curr_set, pos + 1, 1)) {
179 memop_t *op = bl->id_2_memop[pos];
182 DB((dbg, LEVEL_2, "\n\t"));
184 DB((dbg, LEVEL_2, "<%+F, %+F>, ", op->value.address, op->value.value));
187 DB((dbg, LEVEL_2, "\n}\n"));
191 #define dump_block_list()
192 #define dump_curr(bl, s)
193 #endif /* DEBUG_libfirm */
195 /** Get the block entry for a block node */
196 static block_t *get_block_entry(const ir_node *block) {
197 assert(is_Block(block));
199 return get_irn_link(block);
202 /** Get the memop entry for a memory operation node */
203 static memop_t *get_irn_memop(const ir_node *irn) {
204 assert(! is_Block(irn));
205 return get_irn_link(irn);
209 * Walk over the memory edges from definition to users.
211 * @param irn start node
212 * @param pre pre walker function
213 * @param post post walker function
214 * @param ctx context parameter for the walker functions
216 static void walk_memory(ir_node *irn, irg_walk_func *pre, irg_walk_func *post, void *ctx) {
220 mark_irn_visited(irn);
225 mode = get_irn_mode(irn);
226 if (mode == mode_M) {
227 /* every successor uses memory */
228 for (i = get_irn_n_outs(irn) - 1; i >= 0; --i) {
229 ir_node *succ = get_irn_out(irn, i);
231 if (! irn_visited(succ))
232 walk_memory(succ, pre, post, ctx);
234 } else if (mode == mode_T) {
235 /* only some Proj's uses memory */
236 for (i = get_irn_n_outs(irn) - 1; i >= 0; --i) {
237 ir_node *proj = get_irn_out(irn, i);
239 if (get_irn_mode(proj) == mode_M && ! irn_visited(proj))
240 walk_memory(proj, pre, post, ctx);
248 * Walks over all memory nodes of a graph.
251 * @param pre pre walker function
252 * @param post post walker function
253 * @param ctx context parameter for the walker functions
255 static void walk_memory_irg(ir_graph *irg, irg_walk_func pre, irg_walk_func post, void *ctx) {
256 inc_irg_visited(irg);
258 ir_reserve_resources(irg, IR_RESOURCE_IRN_VISITED);
261 * there are two possible sources for memory: initial_mem and nomem
262 * we ignore nomem as this should NOT change the memory
264 walk_memory(get_irg_initial_mem(irg), pre, post, ctx);
266 ir_free_resources(irg, IR_RESOURCE_IRN_VISITED);
270 * Block walker: allocate an block entry for every block.
272 static void prepare_blocks(ir_node *block, void *ctx) {
273 block_t *entry = obstack_alloc(&env.obst, sizeof(*entry));
277 entry->memop_forward = NULL;
278 entry->memop_backward = NULL;
279 entry->avail_out = NULL;
280 entry->id_2_memop_avail = NULL;
281 entry->anticL_in = NULL;
282 entry->id_2_memop = NULL;
283 entry->block = block;
284 entry->forward_next = env.forward;
285 entry->backward_next = NULL;
287 set_irn_link(block, entry);
289 /* create the list in inverse order */
291 /* remember temporary the last one */
292 env.backward = entry;
296 * Block walker: create backward links for the memops of a block.
298 static void collect_backward(ir_node *block, void *ctx) {
299 block_t *entry = get_block_entry(block);
303 entry->backward_next = env.backward;
305 /* create the list in inverse order */
306 env.backward = entry;
308 /* create backward links for all memory ops */
310 for (op = entry->memop_forward; op != NULL; op = op->next) {
314 entry->memop_backward = last;
320 * @param irn the IR-node representing the memop
322 static memop_t *alloc_memop(ir_node *irn) {
323 memop_t *m = obstack_alloc(&env.obst, sizeof(*m));
325 m->value.address = NULL;
326 m->value.value = NULL;
327 m->value.mode = NULL;
334 set_irn_link(irn, m);
339 * Register an address and allocate an ID for it.
341 * @param adr the IR-node representing the address
343 static unsigned register_address(ir_node *adr) {
344 address_entry *entry;
346 /* skip Confirms and Casts */
348 if (is_Confirm(adr)) {
349 adr = get_Confirm_value(adr);
353 adr = get_Cast_op(adr);
357 entry = ir_nodemap_get(&env.adr_map, adr);
361 entry = obstack_alloc(&env.obst, sizeof(*entry));
363 entry->id = env.curr_adr_id++;
364 ir_nodemap_insert(&env.adr_map, adr, entry);
370 * Return the memory properties of a call node.
372 * @param call the call node
374 * return a bitset of mtp_property_const and mtp_property_pure
376 static unsigned get_Call_memory_properties(ir_node *call) {
377 ir_type *call_tp = get_Call_type(call);
378 unsigned prop = get_method_additional_properties(call_tp);
380 /* check first the call type */
381 if ((prop & (mtp_property_const|mtp_property_pure)) == 0) {
382 /* try the called entity */
383 ir_node *ptr = get_Call_ptr(call);
385 if (is_Global(ptr)) {
386 ir_entity *ent = get_Global_entity(ptr);
388 prop = get_entity_additional_properties(ent);
391 return prop & (mtp_property_const|mtp_property_pure);
395 * Update a memop for a Load.
399 static void update_Load_memop(memop_t *m) {
401 ir_node *load = m->node;
402 ir_node *adr = get_Load_ptr(load);
404 if (get_Load_volatility(load) == volatility_is_volatile)
405 m->flags |= FLAG_IGNORE;
407 m->value.address = adr;
409 for (i = get_irn_n_outs(load) - 1; i >= 0; --i) {
410 ir_node *proj = get_irn_out(load, i);
412 /* beware of keep edges */
416 switch (get_Proj_proj(proj)) {
418 m->value.value = proj;
419 m->value.mode = get_irn_mode(proj);
421 case pn_Load_X_except:
422 m->flags |= FLAG_EXCEPTION;
430 if (m->value.value != NULL && !(m->flags & FLAG_IGNORE)) {
431 /* only create an address if this node is NOT killed immediately or ignored */
432 m->value.id = register_address(adr);
435 /* no user, KILL it */
436 m->flags |= FLAG_KILLED_NODE;
441 * Update a memop for a Store.
445 static void update_Store_memop(memop_t *m) {
447 ir_node *store = m->node;
448 ir_node *adr = get_Store_ptr(store);
450 if (get_Store_volatility(store) == volatility_is_volatile) {
451 m->flags |= FLAG_IGNORE;
453 /* only create an address if this node is NOT ignored */
454 m->value.id = register_address(adr);
458 m->value.address = adr;
460 for (i = get_irn_n_outs(store) - 1; i >= 0; --i) {
461 ir_node *proj = get_irn_out(store, i);
463 /* beware of keep edges */
467 switch (get_Proj_proj(proj)) {
468 case pn_Store_X_except:
469 m->flags |= FLAG_EXCEPTION;
476 m->value.value = get_Store_value(store);
477 m->value.mode = get_irn_mode(m->value.value);
481 * Update a memop for a Call.
485 static void update_Call_memop(memop_t *m) {
486 ir_node *call = m->node;
487 unsigned prop = get_Call_memory_properties(call);
490 if (prop & mtp_property_const) {
491 /* A constant call did NOT use memory at all, we
492 can kick it from the list. */
493 } else if (prop & mtp_property_pure) {
494 /* pure calls READ memory */
495 m->flags = FLAG_KILL_STORES;
497 m->flags = FLAG_KILL_ALL;
499 for (i = get_irn_n_outs(call) - 1; i >= 0; --i) {
500 ir_node *proj = get_irn_out(call, i);
502 /* beware of keep edges */
506 switch (get_Proj_proj(proj)) {
507 case pn_Call_X_except:
508 m->flags |= FLAG_EXCEPTION;
510 case pn_Call_M_regular:
518 * Update a memop for a Div/Mod/Quot/DivMod.
522 static void update_DivOp_memop(memop_t *m) {
523 ir_node *div = m->node;
526 for (i = get_irn_n_outs(div) - 1; i >= 0; --i) {
527 ir_node *proj = get_irn_out(div, i);
529 /* beware of keep edges */
533 switch (get_Proj_proj(proj)) {
534 case pn_Generic_X_except:
535 m->flags |= FLAG_EXCEPTION;
537 case pn_Generic_M_regular:
545 * Update a memop for a Phi.
549 static void update_Phi_memop(memop_t *m) {
550 /* the Phi is it's own mem */
555 * Memory walker: collect all memory ops and build topological lists.
557 static void collect_memops(ir_node *irn, void *ctx) {
564 /* we can safely ignore ProjM's except the initial memory */
565 if (irn != get_irg_initial_mem(current_ir_graph))
569 op = alloc_memop(irn);
570 block = get_nodes_block(irn);
571 entry = get_block_entry(block);
574 update_Phi_memop(op);
575 /* Phis must be always placed first */
576 op->next = entry->memop_forward;
577 entry->memop_forward = op;
578 if (entry->memop_backward == NULL)
579 entry->memop_backward = op;
581 switch (get_irn_opcode(irn)) {
583 update_Load_memop(op);
586 update_Store_memop(op);
589 update_Call_memop(op);
601 /* we can those to find the memory edge */
607 update_DivOp_memop(op);
611 /* TODO: handle some builtins */
613 /* unsupported operation */
614 op->flags = FLAG_KILL_ALL;
618 /* all other should be placed last */
619 if (entry->memop_backward == NULL) {
620 entry->memop_forward = entry->memop_backward = op;
622 entry->memop_backward->next = op;
623 entry->memop_backward = op;
629 * Find an address in the current set.
631 * @param bl the block
632 * @param value the value to be searched for
634 static memop_t *find_address(const block_t *bl, const value_t *value) {
635 if (rbitset_is_set(env.curr_set, value->id)) {
636 memop_t *res = bl->id_2_memop[value->id];
638 if (res->value.mode == value->mode)
640 /* allow hidden casts */
641 if (get_mode_arithmetic(res->value.mode) == irma_twos_complement &&
642 get_mode_arithmetic(value->mode) == irma_twos_complement &&
643 get_mode_size_bits(res->value.mode) == get_mode_size_bits(value->mode))
650 * Find an address in the avail_out set.
652 * @param bl the block
653 * @param value the value to be searched for
655 static memop_t *find_address_avail(const block_t *bl, const value_t *value) {
656 if (rbitset_is_set(bl->avail_out, value->id)) {
657 memop_t *res = bl->id_2_memop_avail[value->id];
659 if (res->value.mode == value->mode)
661 /* allow hidden casts */
662 if (get_mode_arithmetic(res->value.mode) == irma_twos_complement &&
663 get_mode_arithmetic(value->mode) == irma_twos_complement &&
664 get_mode_size_bits(res->value.mode) == get_mode_size_bits(value->mode))
671 * Kill all Loads from the current set.
673 * @param bl the current block
675 static void kill_all_loads(block_t *bl) {
677 unsigned end = env.n_mem_ops * 2 - 1;
679 for (pos = rbitset_next(env.curr_set, pos, 1); pos != end; pos = rbitset_next(env.curr_set, pos + 1, 1)) {
680 memop_t *op = bl->id_2_memop[pos];
682 if (! is_Store(op->node))
683 rbitset_clear(env.curr_set, pos);
688 * Kill all Stores from the current set.
690 * @param bl the current block
692 static void kill_all_stores(block_t *bl) {
694 unsigned end = env.n_mem_ops * 2 - 1;
696 for (pos = rbitset_next(env.curr_set, pos, 1); pos != end; pos = rbitset_next(env.curr_set, pos + 1, 1)) {
697 memop_t *op = bl->id_2_memop[pos];
699 if (is_Store(op->node))
700 rbitset_clear(env.curr_set, pos);
705 * Kill all addresses from the current set.
707 static void kill_all(void) {
708 rbitset_clear_all(env.curr_set, env.rbs_size);
711 rbitset_set(env.curr_set, env.rbs_size - 1);
716 * Kill Stores that are not alias free due to a Load value from the current set.
718 * @param bl the block
719 * @param value the Load value
721 static void kill_stores(block_t *bl, const value_t *value) {
723 unsigned end = env.n_mem_ops * 2 - 1;
725 for (pos = rbitset_next(env.curr_set, pos, 1); pos != end; pos = rbitset_next(env.curr_set, pos + 1, 1)) {
726 memop_t *op = bl->id_2_memop[pos];
728 if (is_Store(op->node)) {
729 if (ir_no_alias != get_alias_relation(current_ir_graph, value->address, value->mode,
730 op->value.address, op->value.mode)) {
731 rbitset_clear(env.curr_set, pos);
732 bl->id_2_memop[pos] = NULL;
739 * Kill memops that are not alias free due to a Store value from the current set.
741 * @param bl the block
742 * @param value the Store value
744 static void kill_memops(block_t *bl, const value_t *value) {
746 unsigned end = env.n_mem_ops * 2 - 1;
748 for (pos = rbitset_next(env.curr_set, pos, 1); pos != end; pos = rbitset_next(env.curr_set, pos + 1, 1)) {
749 memop_t *op = bl->id_2_memop[pos];
751 if (ir_no_alias != get_alias_relation(current_ir_graph, value->address, value->mode,
752 op->value.address, op->value.mode)) {
753 rbitset_clear(env.curr_set, pos);
754 bl->id_2_memop[pos] = NULL;
760 * Add the value of a memop to the current set.
762 * @param bl the block
763 * @param op the memory op
765 static void add_memop(block_t *bl, memop_t *op) {
766 rbitset_set(env.curr_set, op->value.id);
767 bl->id_2_memop[op->value.id] = op;
771 * Add the value of a memop to the avail_out set.
773 * @param bl the block
774 * @param op the memory op
776 static void add_memop_avail(block_t *bl, memop_t *op) {
777 rbitset_set(bl->avail_out, op->value.id);
778 bl->id_2_memop_avail[op->value.id] = op;
782 * build a phi definition for a value in the given block
784 * @param block the block
785 * @param value the value
787 * @return the definition
789 static ir_node *build_phi_definition(ir_node *block, const value_t *value) {
794 /* no: we need a new Phi */
795 n = get_Block_n_cfgpreds(block);
798 block = get_Block_cfgpred_block(block, 0);
799 n = get_Block_n_cfgpreds(block);
800 mop = find_address(get_block_entry(block), value);
806 NEW_ARR_A(ir_node *, in, n);
807 for (i = n - 1; i >= 0; --i) {
808 ir_node *pred_bl = get_Block_cfgpred_block(block, i);
811 mop = find_address(get_block_entry(pred_bl), value);
812 def_bl = get_nodes_block(mop->value.value);
814 if (block_dominates(def_bl, pred_bl))
815 in[i] = mop->value.value;
817 in[i] = build_phi_definition(pred_bl, &mop->value);
820 phi = new_r_Phi(current_ir_graph, block, n, in, value->mode);
821 DB((dbg, LEVEL_2, "Created new Phi %+F for value %+F in block %+F\n", phi, value->value, block));
826 * find a definition for a value in the given block or above.
828 * @param block the block
829 * @param op the memop: the definition must dominate it
830 * @param value the value
832 * @return the definition
834 static ir_node *find_definition(ir_node *block, const memop_t *op, const value_t *value) {
835 ir_node *def_block = get_nodes_block(value->value);
837 if (def_block == block) {
838 /* the value is in our block: check, if it is above in the memory list */
841 for (p = op->prev; p != NULL; p = p->prev) {
842 if (!(p->flags & FLAG_KILLED_NODE) &&
843 p->value.address == value->address) {
845 assert(p->value.value == value->value);
846 return p->value.value;
849 } else if (block_dominates(def_block, block)) {
850 /* strictly dominates */
854 return build_phi_definition(block, value);
858 * Mark a Load memop to be replace by a definition
860 * @param op the Load memop
862 static void mark_replace_load(memop_t *op, ir_node *def) {
864 op->flags |= FLAG_KILLED_NODE;
869 * Mark a Store memop to be removed.
871 * @param op the Store memop
873 static void mark_remove_store(memop_t *op) {
874 op->flags |= FLAG_KILLED_NODE;
878 #define BYTE_SIZE(x) (((x) + 7) >> 3)
881 * Do forward dataflow analysis on a given block to calculate the avail_out set.
883 * @param bl the block
885 * @return non-zero if the set has changed since last iteration
887 static int forward_avail(block_t *bl) {
891 ir_node *pred = get_Block_cfgpred_block(bl->block, 0);
892 block_t *pred_bl = get_block_entry(pred);
894 rbitset_cpy(env.curr_set, pred_bl->avail_out, env.rbs_size);
896 n = get_Block_n_cfgpreds(bl->block);
900 /* more than one predecessors, calculate the join */
901 for (i = n - 1; i > 0; --i) {
902 ir_node *pred = skip_Proj(get_Block_cfgpred(bl->block, i));
903 block_t *pred_bl = get_block_entry(get_nodes_block(pred));
905 rbitset_and(env.curr_set, pred_bl->avail_out, env.rbs_size);
907 if (is_Load(pred) || is_Store(pred)) {
908 /* We reached this block by an exception from a Load or Store:
909 * the memop was NOT completed than, kill it
911 memop_t *exc_op = get_irn_memop(pred);
912 rbitset_clear(env.curr_set, exc_op->value.id);
916 /* sure that all values are in the map */
917 for (pos = env.rbs_size - 1; pos >= 0; --pos) {
918 if (! rbitset_is_set(env.curr_set, pos))
919 bl->id_2_memop[pos] = NULL;
921 for (i = n - 1; i >= 0; --i) {
922 ir_node *pred = get_Block_cfgpred_block(bl->block, i);
923 block_t *pred_bl = get_block_entry(pred);
925 if (pred_bl->id_2_memop[pos] != NULL) {
926 bl->id_2_memop[pos] = pred_bl->id_2_memop[pos];
933 /* only one predecessor, simply copy the map */
934 memcpy(bl->id_2_memop, pred_bl->id_2_memop, env.rbs_size * sizeof(bl->id_2_memop[0]));
937 dump_curr(bl, "Avail_in");
939 for (op = bl->memop_forward; op != NULL; op = op->next) {
940 switch (get_irn_opcode(op->node)) {
948 if (! (op->flags & (FLAG_KILLED_NODE|FLAG_IGNORE))) {
949 /* do we have this already? */
950 memop_t *other = find_address(bl, &op->value);
952 def = find_definition(bl->block, op, &other->value);
953 if (is_Store(other->node)) {
955 DB((dbg, LEVEL_1, "RAW %+F <- %+F(%+F)\n", op->node, def, other->node));
958 DB((dbg, LEVEL_1, "RAR %+F <- %+F(%+F)\n", op->node, def, other->node));
960 mark_replace_load(op, def);
963 kill_stores(bl, &op->value);
969 if (! (op->flags & (FLAG_KILLED_NODE|FLAG_IGNORE))) {
970 /* do we have this store already */
971 memop_t *other = find_address(bl, &op->value);
973 if (is_Store(other->node) &&
974 get_nodes_block(other->node) == get_nodes_block(op->node)) {
976 * A WAW in the same block we can kick the first store.
977 * This is a shortcut: we know that the second Store will be anticipated
980 DB((dbg, LEVEL_1, "WAW %+F <- %+F\n", op->node, other->node));
981 mark_remove_store(other);
982 /* FIXME: a Load might be get freed due to this killed store */
983 } else if (other->value.value == op->value.value) {
985 DB((dbg, LEVEL_1, "WAR %+F <- %+F\n", op->node, other->node));
986 mark_remove_store(op);
988 /* we overwrite the value that was loaded */
993 kill_memops(bl, &op->value);
999 switch (op->flags & (FLAG_KILL_LOADS|FLAG_KILL_STORES)) {
1000 case FLAG_KILL_LOADS|FLAG_KILL_STORES:
1003 case FLAG_KILL_LOADS:
1006 case FLAG_KILL_STORES:
1007 kill_all_stores(bl);
1014 if (!rbitset_equal(bl->avail_out, env.curr_set, env.rbs_size)) {
1016 rbitset_cpy(bl->avail_out, env.curr_set, env.rbs_size);
1017 dump_curr(bl, "Avail_out*");
1020 dump_curr(bl, "Avail_out");
1025 * Do backward dataflow analysis on a given block to calculate the antic set
1026 * of Loaded addresses.
1028 * @param bl the block
1030 * @return non-zero if the set has changed since last iteration
1032 static int backward_antic(block_t *bl) {
1034 ir_node *succ = get_Block_cfg_out(bl->block, 0);
1035 block_t *succ_bl = get_block_entry(succ);
1038 rbitset_cpy(env.curr_set, succ_bl->anticL_in, env.rbs_size);
1039 memcpy(bl->id_2_memop, succ_bl->id_2_memop, env.rbs_size * sizeof(bl->id_2_memop[0]));
1041 n = get_Block_n_cfg_outs(bl->block);
1045 for (i = n - 1; i > 0; --i) {
1046 ir_node *succ = get_Block_cfg_out(bl->block, i);
1047 block_t *succ_bl = get_block_entry(succ);
1049 rbitset_and(env.curr_set, succ_bl->anticL_in, env.rbs_size);
1054 /* cleanup: kill those Loads which address is not available */
1055 for (pos = rbitset_next(env.curr_set, pos, 1); pos != end; pos = rbitset_next(env.curr_set, pos + 1, 1)) {
1056 memop_t *op = succ_bl->id_2_memop[pos];
1057 ir_node *ptr = get_Load_ptr(op->node);
1058 ir_node *ptr_bl = get_nodes_block(ptr);
1060 if (!block_dominates(ptr_bl, bl->block))
1061 rbitset_clear(env.curr_set, pos);
1065 dump_curr(bl, "AnticL_out");
1067 for (op = bl->memop_backward; op != NULL; op = op->prev) {
1068 switch (get_irn_opcode(op->node)) {
1076 if (! (op->flags & (FLAG_KILLED_NODE|FLAG_IGNORE))) {
1082 if (! (op->flags & (FLAG_KILLED_NODE|FLAG_IGNORE))) {
1083 /* a Store: check which memops must be killed */
1084 kill_memops(bl, &op->value);
1088 switch (op->flags & (FLAG_KILL_LOADS|FLAG_KILL_STORES)) {
1089 case FLAG_KILL_LOADS|FLAG_KILL_STORES:
1092 case FLAG_KILL_LOADS:
1095 case FLAG_KILL_STORES:
1096 kill_all_stores(bl);
1103 dump_curr(bl, "AnticL_in");
1104 if (! rbitset_equal(bl->anticL_in, env.curr_set, env.rbs_size)) {
1106 rbitset_cpy(bl->anticL_in, env.curr_set, env.rbs_size);
1113 * Replace a Load memop by a already known value.
1115 * @param op the Load memop
1117 static void replace_load(memop_t *op) {
1118 ir_node *load = op->node;
1119 ir_node *def = skip_Id(op->replace);
1124 DB((dbg, LEVEL_1, "Replacing %+F by definition %+F\n", load, is_Proj(def) ? get_Proj_pred(def) : def));
1126 if (op->flags & FLAG_EXCEPTION) {
1127 /* bad: this node is unused and executed for exception only */
1128 DB((dbg, LEVEL_1, "Unused %+F executed for exception only ...\n", load));
1131 DB((dbg, LEVEL_1, "Killing unused %+F\n", load));
1133 for (i = get_irn_n_outs(load) - 1; i >= 0; --i) {
1134 ir_node *proj = get_irn_out(load, i);
1136 switch (get_Proj_proj(proj)) {
1138 exchange(proj, get_Load_mem(load));
1141 mode = get_irn_mode(proj);
1142 if (get_irn_mode(def) != mode) {
1144 dbg_info *db = get_irn_dbg_info(load);
1145 ir_node *block = get_nodes_block(proj);
1146 def = new_rd_Conv(db, current_ir_graph, block, def, mode);
1148 exchange(proj, def);
1150 case pn_Load_X_except:
1151 exchange(proj, new_Bad());
1153 case pn_Load_X_regular:
1154 exchange(proj, new_r_Jmp(current_ir_graph, get_nodes_block(load)));
1157 panic("Unknown Proj from Load");
1163 * Remove a Store memop.
1165 * @param op the Store memop
1167 static void remove_store(memop_t *op) {
1168 ir_node *store = op->node;
1171 DB((dbg, LEVEL_1, "Removing %+F\n", store));
1172 for (i = get_irn_n_outs(store) - 1; i >= 0; --i) {
1173 ir_node *proj = get_irn_out(store, i);
1175 switch (get_Proj_proj(proj)) {
1177 exchange(proj, get_Store_mem(store));
1179 case pn_Store_X_except:
1180 exchange(proj, new_Bad());
1182 case pn_Store_X_regular:
1183 exchange(proj, new_r_Jmp(current_ir_graph, get_nodes_block(store)));
1186 panic("Unknown Proj from Store");
1193 * Do all necessary replacements for a given block.
1195 * @param bl the block
1197 static void do_replacements(block_t *bl) {
1200 for (op = bl->memop_forward; op != NULL; op = op->next) {
1201 if (op->flags & FLAG_KILLED_NODE) {
1202 switch (get_irn_opcode(op->node)) {
1215 * Calculate the Avail_out sets for all basic blocks.
1217 static void calcAvail(void) {
1221 /* calculate avail_out */
1222 DB((dbg, LEVEL_2, "Calculate Avail_out\n"));
1225 DB((dbg, LEVEL_2, "Iteration %d:\n=========\n", i));
1229 /* over all blocks in reverse post order, skip the start block */
1230 for (bl = env.forward->forward_next; bl != NULL; bl = bl->forward_next) {
1231 need_iter |= forward_avail(bl);
1234 } while (need_iter);
1236 /* copy the content of the id_2_memop map into the id_2_memop_avail map
1237 as it must be preserved for later use */
1238 for (bl = env.forward->forward_next; bl != NULL; bl = bl->forward_next) {
1239 memop_t **t = bl->id_2_memop_avail;
1241 bl->id_2_memop_avail = bl->id_2_memop;
1244 DB((dbg, LEVEL_2, "Get avail set after %d iterations\n\n", i));
1248 * Calculate the Antic_in sets for all basic blocks.
1250 static void calcAntic(void) {
1253 /* calculate antic_out */
1254 DB((dbg, LEVEL_2, "Calculate Antic_in\n"));
1259 DB((dbg, LEVEL_2, "Iteration %d:\n=========\n", i));
1263 /* over all blocks in reverse post order */
1264 for (bl = env.backward->backward_next; bl != NULL; bl = bl->backward_next) {
1265 need_iter |= backward_antic(bl);
1268 } while (need_iter);
1269 DB((dbg, LEVEL_2, "Get anticipated Load set after %d iterations\n", i));
1273 * Return the node representing the last memory in a block.
1275 * @param bl the block
1277 static ir_node *find_first_memory(block_t *bl) {
1279 if (bl->memop_forward != NULL) {
1280 return bl->memop_forward->node;
1282 /* if there is NO memory in this block, go to the post dominator */
1283 bl = get_block_entry(get_Block_ipostdom(bl->block));
1288 * Return the node representing the last memory in a block.
1290 * @param bl the block
1292 static ir_node *find_last_memory(block_t *bl) {
1294 if (bl->memop_backward != NULL) {
1295 return bl->memop_backward->mem;
1297 /* if there is NO memory in this block, go to the dominator */
1298 bl = get_block_entry(get_Block_idom(bl->block));
1303 * Reroute all memory users of old memory
1304 * to a new memory IR-node.
1306 * @param omem the old memory IR-node
1307 * @param nmem the new memory IR-node
1309 static void reroute_all_mem_users(ir_node *omem, ir_node *nmem) {
1312 for (i = get_irn_n_outs(omem) - 1; i >= 0; --i) {
1314 ir_node *user = get_irn_out_ex(omem, i, &n_pos);
1316 set_irn_n(user, n_pos, nmem);
1319 /* all edges previously point to omem now point to nmem */
1320 nmem->out = omem->out;
1324 * Reroute memory users of old memory that are dominated by a given block
1325 * to a new memory IR-node.
1327 * @param omem the old memory IR-node
1328 * @param nmem the new memory IR-node
1329 * @param pass_bl the block the memory must pass
1331 static void reroute_mem_through(ir_node *omem, ir_node *nmem, ir_node *pass_bl) {
1332 int i, j, n = get_irn_n_outs(omem);
1333 ir_def_use_edge *edges = NEW_ARR_D(ir_def_use_edge, &env.obst, n + 1);
1335 for (i = j = 0; i < n; ++i) {
1337 ir_node *user = get_irn_out_ex(omem, i, &n_pos);
1338 ir_node *use_bl = get_nodes_block(user);
1342 use_bl = get_Block_cfgpred_block(use_bl, n_pos);
1344 if (block_dominates(pass_bl, use_bl)) {
1345 /* found an user that is dominated */
1347 edges[j].pos = n_pos;
1348 edges[j].use = user;
1350 set_irn_n(user, n_pos, nmem);
1354 /* Modify the out structure: we create a new out edge array on our
1355 temporary obstack here. This should be no problem, as we invalidate the edges
1356 at the end either. */
1357 /* first entry is used for the length */
1366 static void insert_Load(ir_node *block, void *ctx) {
1367 int *new_stuff = ctx;
1369 int i, n = get_Block_n_cfgpreds(block);
1371 unsigned end = env.n_mem_ops * 2 - 1;
1376 bl = get_block_entry(block);
1378 for (pos = rbitset_next(bl->anticL_in, pos, 1); pos != end; pos = rbitset_next(bl->anticL_in, pos + 1, 1)) {
1379 memop_t *op = bl->id_2_memop[pos];
1380 int have_some, all_same;
1383 assert(is_Load(op->node));
1385 if (op->flags & FLAG_KILLED_NODE)
1391 for (i = n - 1; i >= 0; --i) {
1392 ir_node *pred = get_Block_cfgpred_block(block, i);
1393 block_t *pred_bl = get_block_entry(pred);
1394 memop_t *e = find_address_avail(pred_bl, &op->value);
1397 ir_node *block = get_nodes_block(op->value.address);
1398 if (! block_dominates(block, pred)) {
1399 /* cannot place a copy here */
1403 pred_bl->avail = NULL;
1410 else if (first != e->node)
1414 if (have_some && !all_same) {
1415 ir_mode *mode = op->value.mode;
1418 NEW_ARR_A(ir_node *, in, n);
1420 for (i = n - 1; i >= 0; --i) {
1421 ir_node *pred = get_Block_cfgpred_block(block, i);
1422 block_t *pred_bl = get_block_entry(pred);
1424 if (pred_bl->avail == NULL) {
1425 /* create a new Load here and make to make it fully redundant */
1426 dbg_info *db = get_irn_dbg_info(op->node);
1427 ir_node *last_mem = find_last_memory(pred_bl);
1428 ir_node *load, *def;
1431 assert(last_mem != NULL);
1432 load = new_rd_Load(db, current_ir_graph, pred, last_mem, op->value.address, mode, cons_none);
1433 def = new_r_Proj(current_ir_graph, pred, load, mode, pn_Load_res);
1434 DB((dbg, LEVEL_1, "Created new %+F for party redundant %+F\n", load, op->node));
1436 new_op = alloc_memop(load);
1437 new_op->mem = new_r_Proj(current_ir_graph, pred, load, mode_M, pn_Load_M);
1438 new_op->value.address = op->value.address;
1439 new_op->value.id = op->value.id;
1440 new_op->value.mode = mode;
1441 new_op->value.value = def;
1443 new_op->prev = pred_bl->memop_backward;
1444 pred_bl->memop_backward = new_op;
1446 if (get_nodes_block(last_mem) == pred) {
1447 /* We have add a new last memory op in pred block.
1448 If pred had already a last mem, reroute all memory
1450 reroute_all_mem_users(last_mem, new_op->mem);
1452 /* reroute only those memory going through the pre block */
1453 reroute_mem_through(last_mem, new_op->mem, pred);
1456 /* we added this load at the end, so it will be avail anyway */
1457 add_memop_avail(pred_bl, new_op);
1458 pred_bl->avail = new_op;
1460 in[i] = pred_bl->avail->value.value;
1462 phi = new_r_Phi(current_ir_graph, block, n, in, mode);
1463 mark_replace_load(op, phi);
1466 if (get_nodes_block(op->node) == block) {
1467 /* The redundant node was in the current block:
1468 In that case, DO NOT update avail_out. If it was NOT
1469 avail although it is executed in this bLock, it is killed by a later
1473 /* The redundant node is NOT in the current block and anticipated.
1474 This can only happen IFF nothings kills the Load in the current block,
1475 so it will be avail in the next iteration.
1477 add_memop_avail(bl, op);
1479 /* TODO propagate it downwards */
1486 * Insert Loads upwards.
1488 static void insert_Loads_upwards(void) {
1491 /* calculate antic_out */
1492 DB((dbg, LEVEL_2, "Inserting Loads\n"));
1495 DB((dbg, LEVEL_2, "Iteration %d:\n=========\n", i));
1498 dom_tree_walk_irg(current_ir_graph, insert_Load, NULL, &need_iter);
1500 } while (need_iter);
1501 DB((dbg, LEVEL_2, "Finished Load inserting after %d iterations\n", i));
1504 int opt_ldst(ir_graph *irg) {
1506 ir_graph *rem = current_ir_graph;
1508 current_ir_graph = irg;
1510 FIRM_DBG_REGISTER(dbg, "firm.opt.ldst");
1512 DB((dbg, LEVEL_1, "\nDoing Load/Store optimization on %+F\n", irg));
1514 /* we need landing pads */
1515 remove_critical_cf_edges(irg);
1517 dump_ir_block_graph(irg, "-XXX");
1519 if (get_opt_alias_analysis()) {
1520 assure_irg_entity_usage_computed(irg);
1521 assure_irp_globals_entity_usage_computed();
1524 obstack_init(&env.obst);
1525 ir_nodemap_init(&env.adr_map);
1528 env.backward = NULL;
1529 env.curr_adr_id = 0;
1532 env.start_bl = get_irg_start_block(irg);
1533 env.end_bl = get_irg_end_block(irg);
1536 assure_irg_outs(irg);
1538 ir_reserve_resources(irg, IR_RESOURCE_IRN_LINK);
1540 /* first step: allocate block entries: produces an
1541 inverse post-order list for the CFG */
1542 set_irn_link(env.end_bl, NULL);
1543 irg_out_block_walk(get_irg_start_block(irg), NULL, prepare_blocks, NULL);
1545 if (get_block_entry(env.end_bl) == NULL) {
1547 * The end block is NOT reachable due to endless loops
1548 * or no_return calls. Ensure that it is initialized.
1549 * Note that this places the entry for the end block first, so we must fix this.
1550 * env.backwards points to th last block for this purpose.
1552 prepare_blocks(env.end_bl, NULL);
1555 env.forward = bl->forward_next;
1556 bl->forward_next = NULL;
1558 env.backward->forward_next = bl;
1561 /* second step: find and sort all memory ops */
1562 walk_memory_irg(irg, collect_memops, NULL, NULL);
1564 if (env.n_mem_ops == 0) {
1569 /* create the backward links */
1570 env.backward = NULL;
1571 irg_block_walk_graph(irg, NULL, collect_backward, NULL);
1573 /* check that we really start with the start / end block */
1574 assert(env.forward->block == env.start_bl);
1575 assert(env.backward->block == env.end_bl);
1577 /* create address sets: we know that 2 * n_mem_ops - 1 is an upper bound for all possible addresses */
1578 env.rbs_size = 2 * env.n_mem_ops;
1580 /* create the current set */
1581 env.curr_set = rbitset_obstack_alloc(&env.obst, env.rbs_size);
1582 rbitset_set(env.curr_set, env.rbs_size - 1);
1584 for (bl = env.forward; bl != NULL; bl = bl->forward_next) {
1585 /* set sentinel bits */
1586 bl->avail_out = rbitset_obstack_alloc(&env.obst, env.rbs_size);
1587 rbitset_set(bl->avail_out, env.rbs_size - 1);
1589 bl->id_2_memop_avail = NEW_ARR_D(memop_t *, &env.obst, env.rbs_size);
1590 memset(bl->id_2_memop_avail, 0, env.rbs_size * sizeof(bl->id_2_memop[0]));
1592 bl->anticL_in = rbitset_obstack_alloc(&env.obst, env.rbs_size);
1593 rbitset_set(bl->anticL_in, env.rbs_size - 1);
1595 bl->id_2_memop = NEW_ARR_D(memop_t *, &env.obst, env.rbs_size);
1596 memset(bl->id_2_memop, 0, env.rbs_size * sizeof(bl->id_2_memop[0]));
1599 // dump_block_list(&env);
1604 insert_Loads_upwards();
1607 /* over all blocks in reverse post order */
1608 for (bl = env.forward; bl != NULL; bl = bl->forward_next) {
1609 do_replacements(bl);
1612 /* not only invalidate but free them. We might allocate new out arrays
1613 on our obstack which will be deleted yet. */
1615 set_irg_entity_usage_state(irg, ir_entity_usage_not_computed);
1619 ir_free_resources(irg, IR_RESOURCE_IRN_LINK);
1620 ir_nodemap_destroy(&env.adr_map);
1621 obstack_free(&env.obst, NULL);
1623 current_ir_graph = rem;
1625 return env.changed != 0;