From: Moritz Kroll Date: Wed, 4 Feb 2009 09:42:57 +0000 (+0000) Subject: Added first version of IR importer/exporter X-Git-Url: http://nsz.repo.hu/git/?a=commitdiff_plain;h=279822e9e2a4f26deaf67b5fe8423b27837a75d7;p=libfirm Added first version of IR importer/exporter [r25422] --- diff --git a/include/libfirm/firm.h b/include/libfirm/firm.h index e7c78da58..823e2cf23 100644 --- a/include/libfirm/firm.h +++ b/include/libfirm/firm.h @@ -116,6 +116,7 @@ extern "C" { #include "firm_ycomp.h" /* ycomp debugging support */ #include "irdump.h" +#include "irio.h" #include "irprintf.h" #include "irvrfy.h" diff --git a/include/libfirm/irio.h b/include/libfirm/irio.h new file mode 100644 index 000000000..89ed8b8ae --- /dev/null +++ b/include/libfirm/irio.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 1995-2008 University of Karlsruhe. All right reserved. + * + * This file is part of libFirm. + * + * This file may be distributed and/or modified under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation and appearing in the file LICENSE.GPL included in the + * packaging of this file. + * + * Licensees holding valid libFirm Professional Edition licenses may use + * this file in accordance with the libFirm Commercial License. + * Agreement provided with the Software. + * + * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE + * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE. + */ + +/** + * @file + * @brief Import/export textual representation of firm. + * @author Moritz Kroll + * @version $Id$ + */ +#ifndef FIRM_IR_IRIO_H +#define FIRM_IR_IRIO_H + +#include + +#include "firm_types.h" + +/** + * Exports the given ir graph to the given file in a textual form. + * + * @param irg the ir graph + * @param filename the name of the resulting file + * + * Exports the type graph used by the given graph and the graph itself. + */ +void ir_export_irg(ir_graph *irg, const char *filename); + +/** + * Imports the data stored in the given file. + * + * @param filename the name of the file + * + * Imports any type graphs and ir graphs contained in the file. + */ +void ir_import(const char *filename); + +#endif diff --git a/ir/ir/irio.c b/ir/ir/irio.c new file mode 100644 index 000000000..99ff27f43 --- /dev/null +++ b/ir/ir/irio.c @@ -0,0 +1,1025 @@ +/* + * Copyright (C) 1995-2009 University of Karlsruhe. All right reserved. + * + * This file is part of libFirm. + * + * This file may be distributed and/or modified under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation and appearing in the file LICENSE.GPL included in the + * packaging of this file. + * + * Licensees holding valid libFirm Professional Edition licenses may use + * this file in accordance with the libFirm Commercial License. + * Agreement provided with the Software. + * + * This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE + * WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE. + */ + +/** + * @file + * @brief Write textual representation of firm to file. + * @author Moritz Kroll + * @version $Id$ + */ +#include "config.h" + +#include + +#include "irio.h" + +#include "irprog.h" +#include "irgraph_t.h" +#include "ircons.h" +#include "irgmod.h" +#include "irflag_t.h" +#include "irgwalk.h" +#include "tv.h" +#include "array.h" +#include "error.h" +#include "adt/set.h" + +#define LEXERROR ((unsigned) ~0) + +typedef struct io_env +{ + FILE *file; + set *idset; /**< id_entry set, which maps from file ids to new Firm elements */ + int ignoreblocks; + int line, col; + ir_type **fixedtypes; +} io_env_t; + +typedef enum typetag_t +{ + tt_iro, + tt_tpo, + tt_align, + tt_allocation, + tt_peculiarity, + tt_pin_state, + tt_type_state, + tt_variability, + tt_visibility, + tt_volatility +} typetag_t; + +typedef struct lex_entry +{ + const char *str; + typetag_t typetag; + unsigned code; +} lex_entry; + +typedef struct id_entry +{ + long id; + void *elem; +} id_entry; + +/** A set of lex_entry elements. */ +static set *lexset; + + +static unsigned hash(const char *str, int len) +{ + return str[0] * 27893 ^ str[len-1] * 81 ^ str[len >> 1]; +} + +static int lex_cmp(const void *elt, const void *key, size_t size) +{ + const lex_entry *entry = (const lex_entry *) elt; + const lex_entry *keyentry = (const lex_entry *) key; + (void) size; + return strcmp(entry->str, keyentry->str); +} + +static int id_cmp(const void *elt, const void *key, size_t size) +{ + const id_entry *entry = (const id_entry *) elt; + const id_entry *keyentry = (const id_entry *) key; + (void) size; + return entry->id - keyentry->id; +} + +/** Initializes the lexer. May be called more than once without problems. */ +static void init_lexer(void) +{ + lex_entry key; + + /* Only initialize once */ + if(lexset != NULL) return; + + lexset = new_set(lex_cmp, 32); + +#define INSERT(s, tt, cod) \ + key.str = (s); \ + key.typetag = (tt); \ + key.code = (cod); \ + set_insert(lexset, &key, sizeof(key), hash(s, sizeof(s)-1)) + +#define INSERTENUM(tt, e) INSERT(#e, tt, e) + + INSERT("primitive", tt_tpo, tpo_primitive); + INSERT("method", tt_tpo, tpo_method); + INSERT("array", tt_tpo, tpo_array); + INSERT("struct", tt_tpo, tpo_struct); + INSERT("Unknown", tt_tpo, tpo_unknown); + +#include "gen_irio_lex.inl" + + INSERTENUM(tt_align, align_non_aligned); + INSERTENUM(tt_align, align_is_aligned); + + INSERTENUM(tt_allocation, allocation_automatic); + INSERTENUM(tt_allocation, allocation_parameter); + INSERTENUM(tt_allocation, allocation_dynamic); + INSERTENUM(tt_allocation, allocation_static); + + INSERTENUM(tt_pin_state, op_pin_state_floats); + INSERTENUM(tt_pin_state, op_pin_state_pinned); + INSERTENUM(tt_pin_state, op_pin_state_exc_pinned); + INSERTENUM(tt_pin_state, op_pin_state_mem_pinned); + + INSERTENUM(tt_type_state, layout_undefined); + INSERTENUM(tt_type_state, layout_fixed); + + INSERTENUM(tt_variability, variability_uninitialized); + INSERTENUM(tt_variability, variability_initialized); + INSERTENUM(tt_variability, variability_part_constant); + INSERTENUM(tt_variability, variability_constant); + + INSERTENUM(tt_visibility, visibility_local); + INSERTENUM(tt_visibility, visibility_external_visible); + INSERTENUM(tt_visibility, visibility_external_allocated); + + INSERTENUM(tt_volatility, volatility_non_volatile); + INSERTENUM(tt_volatility, volatility_is_volatile); + + INSERTENUM(tt_peculiarity, peculiarity_description); + INSERTENUM(tt_peculiarity, peculiarity_inherited); + INSERTENUM(tt_peculiarity, peculiarity_existent); + +#undef INSERTENUM +#undef INSERT +} + +/** Returns the according enum entry for the given string and tag, or LEXERROR if none was found. */ +static unsigned lex(const char *str, typetag_t typetag) +{ + lex_entry key, *entry; + + key.str = str; + + entry = set_find(lexset, &key, sizeof(key), hash(str, strlen(str))); + if (entry && entry->typetag == typetag) { + return entry->code; + } + return LEXERROR; +} + +static void *get_id(io_env_t *env, long id) +{ + id_entry key, *entry; + key.id = id; + + entry = set_find(env->idset, &key, sizeof(key), (unsigned) id); + return entry ? entry->elem : NULL; +} + +static void set_id(io_env_t *env, long id, void *elem) +{ + id_entry key; + key.id = id; + key.elem = elem; + set_insert(env->idset, &key, sizeof(key), (unsigned) id); +} + +static void write_mode(io_env_t *env, ir_mode *mode) +{ + fputs(get_mode_name(mode), env->file); + fputc(' ', env->file); +} + +static void write_pinned(io_env_t *env, ir_node *irn) +{ + fputs(get_op_pin_state_name(get_irn_pinned(irn)), env->file); + fputc(' ', env->file); +} + +static void write_volatility(io_env_t *env, ir_node *irn) +{ + ir_volatility vol; + + if(is_Load(irn)) vol = get_Load_volatility(irn); + else if(is_Store(irn)) vol = get_Store_volatility(irn); + else assert(0 && "Invalid optype for write_volatility"); + + fputs(get_volatility_name(vol), env->file); + fputc(' ', env->file); +} + +static void write_align(io_env_t *env, ir_node *irn) +{ + ir_align align; + + if(is_Load(irn)) align = get_Load_align(irn); + else if(is_Store(irn)) align = get_Store_align(irn); + else assert(0 && "Invalid optype for write_align"); + + fputs(get_align_name(align), env->file); + fputc(' ', env->file); +} + +static void export_type(io_env_t *env, ir_type *tp) +{ + FILE *f = env->file; + int i; + fprintf(f, "\ttype %ld %s \"%s\" %u %u %s %s ", + get_type_nr(tp), + get_type_tpop_name(tp), + get_type_name(tp), + get_type_size_bytes(tp), + get_type_alignment_bytes(tp), + get_type_state_name(get_type_state(tp)), + get_visibility_name(get_type_visibility(tp))); + + switch(get_type_tpop_code(tp)) + { + case tpo_array: + { + int n = get_array_n_dimensions(tp); + fprintf(f, "%i %ld ", n, get_type_nr(get_array_element_type(tp))); + for(i = 0; i < n; i++) + { + ir_node *lower = get_array_lower_bound(tp, i); + ir_node *upper = get_array_upper_bound(tp, i); + + if(is_Const(lower)) fprintf(f, "%ld ", get_tarval_long(get_Const_tarval(lower))); + else panic("Lower array bound is not constant"); + + if(is_Const(upper)) fprintf(f, "%ld ", get_tarval_long(get_Const_tarval(upper))); + else panic("Upper array bound is not constant"); + } + break; + } + + case tpo_method: + { + int nparams = get_method_n_params(tp); + int nresults = get_method_n_ress(tp); + fprintf(f, "%i %i ", nparams, nresults); + for(i = 0; i < nparams; i++) + fprintf(f, "%ld ", get_type_nr(get_method_param_type(tp, i))); + for(i = 0; i < nresults; i++) + fprintf(f, "%ld ", get_type_nr(get_method_res_type(tp, i))); + break; + } + + case tpo_primitive: + { + write_mode(env, get_type_mode(tp)); + break; + } + + case tpo_struct: + break; + + case tpo_class: + // TODO: inheritance stuff not supported yet + printf("Inheritance of classes not supported yet!\n"); + break; + + case tpo_unknown: + break; + + default: + printf("export_type: Unknown type code \"%s\".\n", get_type_tpop_name(tp)); + break; + } + fputc('\n', f); +} + +static void export_entity(io_env_t *env, ir_entity *ent) +{ + ir_type *owner = get_entity_owner(ent); + fprintf(env->file, "\tentity %ld \"%s\" %ld %ld %d %d %s %s %s %s %s\n", + get_entity_nr(ent), + get_entity_name(ent), + get_type_nr(get_entity_type(ent)), + get_type_nr(owner), + get_entity_offset(ent), + (int) get_entity_offset_bits_remainder(ent), + get_allocation_name(get_entity_allocation(ent)), + get_visibility_name(get_entity_visibility(ent)), + get_variability_name(get_entity_variability(ent)), + get_peculiarity_name(get_entity_peculiarity(ent)), + get_volatility_name(get_entity_volatility(ent))); + + // TODO: inheritance stuff for class entities not supported yet + if(is_Class_type(owner) && owner != get_glob_type()) + printf("Inheritance of class entities not supported yet!\n"); +} + +static void export_type_or_ent(type_or_ent tore, void *ctx) +{ + io_env_t *env = (io_env_t *) ctx; + + switch(get_kind(tore.ent)) + { + case k_entity: + export_entity(env, tore.ent); + break; + + case k_type: + export_type(env, tore.typ); + break; + + default: + printf("export_type_or_ent: Unknown type or entity.\n"); + break; + } +} + +static void export_node(ir_node *irn, void *ctx) +{ + io_env_t *env = (io_env_t *) ctx; + int i, n; + unsigned opcode = get_irn_opcode(irn); + char buf[1024]; + + if(env->ignoreblocks && opcode == iro_Block) return; + + n = get_irn_arity(irn); + + fprintf(env->file, "\n\t%s %ld [ ", get_irn_opname(irn), get_irn_node_nr(irn)); + + for(i = -1; i < n; i++) + { + ir_node *pred = get_irn_n(irn, i); + if(!pred) + fputs("-1 ", env->file); + else + fprintf(env->file, "%ld ", get_irn_node_nr(pred)); + } + + fprintf(env->file, "] { "); + + switch(opcode) + { + #include "gen_irio_export.inl" + } + fputc('}', env->file); +} + +/** Exports the given irg to the given file. */ +void ir_export_irg(ir_graph *irg, const char *filename) +{ + io_env_t env; + + env.file = fopen(filename, "wt"); + if(!env.file) + { + perror(filename); + return; + } + + fputs("typegraph {\n", env.file); + + type_walk_irg(irg, NULL, export_type_or_ent, &env); + + fprintf(env.file, "}\n\nirg %ld {", get_entity_nr(get_irg_entity(irg))); + + env.ignoreblocks = 0; + irg_block_walk_graph(irg, NULL, export_node, &env); + + env.ignoreblocks = 1; + irg_walk_anchors(irg, NULL, export_node, &env); + + fputs("\n}\n", env.file); + + fclose(env.file); +} + +static int read_c(io_env_t *env) +{ + int ch = fgetc(env->file); + switch(ch) + { + case '\t': + env->col += 4; + break; + + case '\n': + env->col = 0; + env->line++; + break; + + default: + env->col++; + break; + } + return ch; +} + +/** Returns the first non-whitespace character or EOF. **/ +static int skip_ws(io_env_t *env) +{ + while(1) + { + int ch = read_c(env); + switch(ch) + { + case ' ': + case '\t': + case '\n': + case '\r': + break; + + default: + return ch; + } + } +} + +static void skip_to(io_env_t *env, char to_ch) +{ + int ch; + do + { + ch = read_c(env); + } + while(ch != to_ch && ch != EOF); +} + +static int expect_char(io_env_t *env, char ch) +{ + int curch = skip_ws(env); + if(curch != ch) + { + printf("Unexpected char '%c', expected '%c' in line %i:%i\n", curch, ch, env->line, env->col); + return 0; + } + return 1; +} + +#define EXPECT(c) if(expect_char(env, (c))) {} else return 0 +#define EXPECT_OR_EXIT(c) if(expect_char(env, (c))) {} else exit(1) + +inline static const char *read_str_to(io_env_t *env, char *buf, size_t bufsize) +{ + size_t i; + for(i = 0; i < bufsize - 1; i++) + { + int ch = read_c(env); + if(ch == EOF) break; + switch(ch) + { + case ' ': + case '\t': + case '\n': + case '\r': + if(i != 0) + goto endofword; + i--; // skip whitespace + break; + + default: + buf[i] = ch; + break; + } + } +endofword: + buf[i] = 0; + return buf; +} + +static const char *read_str(io_env_t *env) +{ + static char buf[1024]; + return read_str_to(env, buf, sizeof(buf)); +} + +static const char *read_qstr_to(io_env_t *env, char *buf, size_t bufsize) +{ + size_t i; + EXPECT_OR_EXIT('\"'); + for(i = 0; i < bufsize - 1; i++) + { + int ch = read_c(env); + if(ch == EOF) + { + printf("Unexpected end of quoted string!\n"); + exit(1); + } + if(ch == '\"') break; + + buf[i] = ch; + } + if(i == bufsize - 1) + { + printf("Quoted string too long!\n"); + exit(1); + } + buf[i] = 0; + return buf; +} + +static long read_long2(io_env_t *env, char **endptr) +{ + static char buf[1024]; + return strtol(read_str_to(env, buf, sizeof(buf)), endptr, 0); +} + +static long read_long(io_env_t *env) +{ + return read_long2(env, NULL); +} + +static ir_node *get_node_or_null(io_env_t *env, long nodenr) +{ + ir_node *node = (ir_node *) get_id(env, nodenr); + if(node && node->kind != k_ir_node) + { + panic("Irn ID %ld collides with something else in line %i:%i\n", nodenr, env->line, env->col); + } + return node; +} + +static ir_node *get_node(io_env_t *env, long nodenr) +{ + ir_node *node = get_node_or_null(env, nodenr); + if(!node) + panic("Unknown node: %ld in line %i:%i\n", nodenr, env->line, env->col); + + return node; +} + +static ir_node *get_node_or_dummy(io_env_t *env, long nodenr) +{ + ir_node *node = get_node_or_null(env, nodenr); + if(!node) + { + node = new_Dummy(mode_X); + set_id(env, nodenr, node); + } + return node; +} + +static ir_type *get_type(io_env_t *env, long typenr) +{ + ir_type *type = (ir_type *) get_id(env, typenr); + if(!type) + { + panic("Unknown type: %ld in line %i:%i\n", typenr, env->line, env->col); + } + else if(type->kind != k_type) + { + panic("Type ID %ld collides with something else in line %i:%i\n", typenr, env->line, env->col); + } + return type; +} + +static ir_type *read_type(io_env_t *env) +{ + return get_type(env, read_long(env)); +} + +static ir_entity *get_entity(io_env_t *env, long entnr) +{ + ir_entity *entity = (ir_entity *) get_id(env, entnr); + if(!entity) + { + printf("Unknown entity: %ld in line %i:%i\n", entnr, env->line, env->col); + exit(1); + } + else if(entity->kind != k_entity) + { + panic("Entity ID %ld collides with something else in line %i:%i\n", entnr, env->line, env->col); + } + return entity; +} + +static ir_entity *read_entity(io_env_t *env) +{ + return get_entity(env, read_long(env)); +} + +static ir_mode *read_mode(io_env_t *env) +{ + static char buf[128]; + int i, n; + + read_str_to(env, buf, sizeof(buf)); + + n = get_irp_n_modes(); + for(i = 0; i < n; i++) + { + ir_mode *mode = get_irp_mode(i); + if(!strcmp(buf, get_mode_name(mode))) + return mode; + } + + printf("Unknown mode \"%s\" in line %i:%i\n", buf, env->line, env->col); + return mode_ANY; +} + +static const char *get_typetag_name(typetag_t typetag) +{ + switch(typetag) + { + case tt_iro: return "opcode"; + case tt_tpo: return "type"; + case tt_align: return "align"; + case tt_allocation: return "allocation"; + case tt_peculiarity: return "peculiarity"; + case tt_pin_state: return "pin state"; + case tt_type_state: return "type state"; + case tt_variability: return "variability"; + case tt_visibility: return "visibility"; + case tt_volatility: return "volatility"; + default: return ""; + } +} + +static unsigned read_enum(io_env_t *env, typetag_t typetag) +{ + static char buf[128]; + unsigned code = lex(read_str_to(env, buf, sizeof(buf)), typetag); + if(code != LEXERROR) return code; + + printf("Invalid %s: \"%s\" in %i:%i\n", get_typetag_name(typetag), buf, env->line, env->col); + return 0; +} + +#define read_align(env) ((ir_align) read_enum(env, tt_align)) +#define read_allocation(env) ((ir_allocation) read_enum(env, tt_allocation)) +#define read_peculiarity(env) ((ir_peculiarity) read_enum(env, tt_peculiarity)) +#define read_pinned(env) ((op_pin_state) read_enum(env, tt_pin_state)) +#define read_type_state(env) ((ir_type_state) read_enum(env, tt_type_state)) +#define read_variability(env) ((ir_variability) read_enum(env, tt_variability)) +#define read_visibility(env) ((ir_visibility) read_enum(env, tt_visibility)) +#define read_volatility(env) ((ir_volatility) read_enum(env, tt_volatility)) + +static tarval *read_tv(io_env_t *env) +{ + static char buf[128]; + ir_mode *tvmode = read_mode(env); + read_str_to(env, buf, sizeof(buf)); + return new_tarval_from_str(buf, strlen(buf), tvmode); +} + +/** Reads a type description and remembers it by its id. */ +static void import_type(io_env_t *env) +{ + char buf[1024]; + int i; + ir_type *type; + long typenr = read_long(env); + const char *tpop = read_str(env); + const char *name = read_qstr_to(env, buf, sizeof(buf)); + unsigned size = (unsigned) read_long(env); + unsigned align = (unsigned) read_long(env); + ir_type_state state = read_type_state(env); + ir_visibility vis = read_visibility(env); + + ident *id = new_id_from_str(name); + + switch(lex(tpop, tt_tpo)) + { + case tpo_primitive: + { + ir_mode *mode = read_mode(env); + type = new_type_primitive(id, mode); + break; + } + + case tpo_method: + { + int nparams = (int) read_long(env); + int nresults = (int) read_long(env); + + type = new_type_method(id, nparams, nresults); + + for(i = 0; i < nparams; i++) + { + long typenr = read_long(env); + ir_type *paramtype = get_type(env, typenr); + + set_method_param_type(type, i, paramtype); + } + for(i = 0; i < nresults; i++) + { + long typenr = read_long(env); + ir_type *restype = get_type(env, typenr); + + set_method_res_type(type, i, restype); + } + break; + } + + case tpo_array: + { + int ndims = (int) read_long(env); + long elemtypenr = read_long(env); + ir_type *elemtype = get_type(env, elemtypenr); + + type = new_type_array(id, ndims, elemtype); + for(i = 0; i < ndims; i++) + { + long lowerbound = read_long(env); + long upperbound = read_long(env); + set_array_bounds_int(type, i, lowerbound, upperbound); + } + set_type_size_bytes(type, size); + break; + } + + case tpo_class: + type = new_type_class(id); + set_type_size_bytes(type, size); + break; + + case tpo_struct: + type = new_type_struct(id); + set_type_size_bytes(type, size); + break; + + case tpo_union: + type = new_type_union(id); + set_type_size_bytes(type, size); + break; + + case tpo_unknown: + return; // ignore unknown type + + default: + if(typenr != 0) // ignore global type + printf("Unknown type kind: \"%s\" in line %i:%i\n", tpop, env->line, env->col); + skip_to(env, '\n'); + return; + } + + set_type_alignment_bytes(type, align); + set_type_visibility(type, vis); + + if(state == layout_fixed) + ARR_APP1(ir_type *, env->fixedtypes, type); + + set_id(env, typenr, type); + printf("Insert type %s %ld\n", name, typenr); +} + +/** Reads an entity description and remembers it by its id. */ +static void import_entity(io_env_t *env) +{ + char buf[1024]; + long entnr = read_long(env); + const char *name = read_qstr_to(env, buf, sizeof(buf)); + long typenr = read_long(env); + long ownertypenr = read_long(env); + + ir_type *type = get_type(env, typenr); + ir_type *ownertype = !ownertypenr ? get_glob_type() : get_type(env, ownertypenr); + ir_entity *entity = new_entity(ownertype, new_id_from_str(name), type); + + set_entity_offset (entity, (int) read_long(env)); + set_entity_offset_bits_remainder(entity, (unsigned char) read_long(env)); + set_entity_allocation (entity, read_allocation(env)); + set_entity_visibility (entity, read_visibility(env)); + set_entity_variability(entity, read_variability(env)); + set_entity_peculiarity(entity, read_peculiarity(env)); + set_entity_volatility (entity, read_volatility(env)); + + set_id(env, entnr, entity); + printf("Insert entity %s %ld\n", name, entnr); +} + +/** Parses the whole type graph. */ +static int parse_typegraph(io_env_t *env) +{ + const char *kind; + long curfpos; + + EXPECT('{'); + + curfpos = ftell(env->file); + + // parse all types first + while(1) + { + kind = read_str(env); + if(kind[0] == '}' && !kind[1]) break; + + if(!strcmp(kind, "type")) + import_type(env); + else + skip_to(env, '\n'); + } + + // now parse rest + fseek(env->file, curfpos, SEEK_SET); + while(1) + { + kind = read_str(env); + if(kind[0] == '}' && !kind[1]) break; + + if(!strcmp(kind, "type")) + skip_to(env, '\n'); + else if(!strcmp(kind, "entity")) + import_entity(env); + else + { + printf("Type graph element not supported yet: \"%s\"\n", kind); + skip_to(env, '\n'); + } + } + return 1; +} + +static int read_node_header(io_env_t *env, long *nodenr, long **preds, const char **nodename) +{ + int numpreds; + *nodename = read_str(env); + if((*nodename)[0] == '}' && !(*nodename)[1]) return -1; // end-of-graph + + *nodenr = read_long(env); + + ARR_RESIZE(ir_node *, *preds, 0); + + EXPECT('['); + for(numpreds = 0; !feof(env->file); numpreds++) + { + char *endptr; + ARR_APP1(long, *preds, read_long2(env, &endptr)); + if(*endptr == ']') break; + } + return numpreds; +} + +/** Parses an IRG. */ +static int parse_graph(io_env_t *env) +{ + long *preds = NEW_ARR_F(long, 16); + ir_node **prednodes = NEW_ARR_F(ir_node *, 16); + int i, numpreds, ret = 1; + long nodenr; + const char *nodename; + ir_node *node, *newnode; + + current_ir_graph = new_ir_graph(get_entity(env, read_long(env)), 0); + + EXPECT('{'); + + while(1) + { + numpreds = read_node_header(env, &nodenr, &preds, &nodename); + if(numpreds == -1) break; // end-of-graph + if(!numpreds) + { + printf("Node %s %ld is missing predecessors!", nodename, nodenr); + ret = 0; + break; + } + + ARR_RESIZE(ir_node *, prednodes, numpreds); + for(i = 0; i < numpreds - 1; i++) + prednodes[i] = get_node_or_dummy(env, preds[i + 1]); + + node = get_node_or_null(env, nodenr); + newnode = NULL; + + EXPECT('{'); + + switch(lex(nodename, tt_iro)) + { + case iro_End: + { + ir_node *newendblock = get_node(env, preds[0]); + newnode = get_irg_end(current_ir_graph); + exchange(get_nodes_block(newnode), newendblock); + break; + } + + case iro_Start: + { + ir_node *newstartblock = get_node(env, preds[0]); + newnode = get_irg_start(current_ir_graph); + exchange(get_nodes_block(newnode), newstartblock); + break; + } + + case iro_Block: + { + if(preds[0] != nodenr) + { + printf("Invalid block: preds[0] != nodenr (%ld != %ld)\n", + preds[0], nodenr); + ret = 0; + goto endloop; + } + + newnode = new_Block(numpreds - 1, prednodes); + break; + } + + case iro_Anchor: + newnode = current_ir_graph->anchor; + for(i = 0; i < numpreds - 1; i++) + set_irn_n(newnode, i, prednodes[i]); + set_irn_n(newnode, -1, get_node(env, preds[0])); + break; + + case iro_SymConst: + { + long entnr = read_long(env); + union symconst_symbol sym; + sym.entity_p = get_entity(env, entnr); + newnode = new_SymConst(mode_P, sym, symconst_addr_ent); + break; + } + + #include "gen_irio_import.inl" + + default: + goto notsupported; + } + + EXPECT('}'); + + if(!newnode) + { +notsupported: + printf("Node type not supported yet: %s in line %i:%i\n", nodename, env->line, env->col); + assert(0 && "Invalid node type"); + } + + if(node) + exchange(node, newnode); + /* Always update hash entry to avoid more uses of id nodes */ + set_id(env, nodenr, newnode); + printf("Insert %s %ld\n", nodename, nodenr); + } + +endloop: + DEL_ARR_F(preds); + DEL_ARR_F(prednodes); + + return ret; +} + +/** Imports an previously exported textual representation of an (maybe partial) irp */ +void ir_import(const char *filename) +{ + int oldoptimize = get_optimize(); + firm_verification_t oldver = get_node_verification_mode(); + io_env_t ioenv; + io_env_t *env = &ioenv; + int i, n; + + init_lexer(); + + memset(env, 0, sizeof(*env)); + env->idset = new_set(id_cmp, 128); + env->fixedtypes = NEW_ARR_F(ir_type *, 0); + + env->file = fopen(filename, "rt"); + if(!env->file) + { + perror(filename); + exit(1); + } + + set_optimize(0); + do_node_verification(FIRM_VERIFICATION_OFF); + + while(1) + { + const char *str = read_str(env); + if(!*str) break; + if(!strcmp(str, "typegraph")) + { + if(!parse_typegraph(env)) break; + } + else if(!strcmp(str, "irg")) + { + if(!parse_graph(env)) break; + } + } + + n = ARR_LEN(env->fixedtypes); + for(i = 0; i < n; i++) + set_type_state(env->fixedtypes[i], layout_fixed); + + DEL_ARR_F(env->fixedtypes); + + del_set(env->idset); + + irp_finalize_cons(); + + do_node_verification(oldver); + set_optimize(oldoptimize); + + fclose(env->file); +} diff --git a/scripts/gen_ir_io.py b/scripts/gen_ir_io.py new file mode 100755 index 000000000..6ace69fff --- /dev/null +++ b/scripts/gen_ir_io.py @@ -0,0 +1,219 @@ +#!/usr/bin/python +import sys +from jinja2 import Environment, Template +import ir_spec + +def format_args(arglist): + #argstrings = map(lambda arg : arg["name"], arglist) + #return ", ".join(argstrings) + s = ", ".join(arglist) + if len(s) == 0: + return ""; + return ", " + s; + +def format_ifnset(string, node, key): + if key in node: + return "" + return string + +def format_block(node): + if node.get("knownBlock"): + return "" + else: + return ", get_node(env, preds[0])" + +env = Environment() +env.filters['args'] = format_args +env.filters['ifnset'] = format_ifnset +env.filters['block'] = format_block + +def get_io_type(type, attrname, nodename): + if type == "tarval*": + importcmd = "tarval *%s = read_tv(env);" % attrname + exportcmd = """ + write_mode(env, get_tarval_mode(%(val)s)); + tarval_snprintf(buf, sizeof(buf), %(val)s); + fprintf(env->file, "%%s ", buf);""" + elif type == "ir_mode*": + importcmd = "ir_mode *%s = read_mode(env);" % attrname + exportcmd = "write_mode(env, %(val)s);" + elif type == "ir_entity*": + importcmd = "ir_entity *%s = read_entity(env);" % attrname + exportcmd = """fprintf(env->file, "%%ld ", get_entity_nr(%(val)s));""" + elif type == "ir_type*": + importcmd = "ir_type *%s = read_type(env);" % attrname + exportcmd = """fprintf(env->file, "%%ld ", get_type_nr(%(val)s));""" + elif type == "long" and nodename == "Proj": + importcmd = "long %s = read_long(env);" % attrname + exportcmd = """fprintf(env->file, "%%ld ", %(val)s);""" + elif type == "pn_Cmp" or type == "ir_where_alloc": + importcmd = "%s %s = (%s) read_long(env);" % (type, attrname, type) + exportcmd = """fprintf(env->file, "%%ld ", (long) %(val)s);""" + elif type == "cons_flags" and nodename == "Store": + importcmd = """ir_cons_flags %s = read_pinned(env) + | read_volatility(env) + | read_align(env);""" % attrname + exportcmd = """write_pinned(env, irn); + write_volatility(env, irn); + write_align(env, irn);""" + elif type == "cons_flags" and nodename == "Load": + importcmd = """ir_cons_flags %s = read_pinned(env) + | read_volatility(env) + | read_align(env);""" % attrname + exportcmd = """write_pinned(env, irn); + write_volatility(env, irn); + write_align(env, irn);""" + else: + print "UNKNOWN TYPE: %s" % type + importcmd = """// BAD: %s %s + %s %s = (%s) 0;""" % (type, attrname, type, attrname, type) + exportcmd = "// BAD: %s" % type + return (importcmd, exportcmd) + +""" if type == "ir_type*": + java_type = "firm.Type" + wrap_type = "Pointer" + to_wrapper = "%s.ptr" + from_wrapper = "firm.Type.createWrapper(%s)" + elif type == "ir_mode*": + java_type = "firm.Mode" + wrap_type = "Pointer" + to_wrapper = "%s.ptr" + from_wrapper = "new firm.Mode(%s)" + elif type == "tarval*": + java_type = "firm.TargetValue" + wrap_type = "Pointer" + to_wrapper = "%s.ptr" + from_wrapper = "new firm.TargetValue(%s)" + elif type == "pn_Cmp": + java_type = "int" + wrap_type = "int" + to_wrapper = "%s" + from_wrapper = "%s" + elif type == "long": + java_type = "int" + wrap_type = "com.sun.jna.NativeLong" + to_wrapper = "new com.sun.jna.NativeLong(%s)" + from_wrapper = "%s.intValue()" + elif type == "cons_flags": + java_type = "firm.bindings.binding_ircons.ir_cons_flags" + wrap_type = "int" + to_wrapper = "%s.val" + from_wrapper = "firm.bindings.binding_ircons.ir_cons_flags.getEnum(%s)" + elif type == "ir_where_alloc": + java_type = "firm.bindings.binding_ircons.ir_where_alloc" + wrap_type = "int" + to_wrapper = "%s.val" + from_wrapper = "firm.bindings.binding_ircons.ir_where_alloc.getEnum(%s)" + elif type == "ir_entity*": + java_type = "firm.Entity" + wrap_type = "Pointer" + to_wrapper = "%s.ptr" + from_wrapper = "new firm.Entity(%s)" + else: + print "UNKNOWN TYPE" + java_type = "BAD" + wrap_type = "BAD" + to_wrapper = "BAD" + from_wrapper = "BAD" + return (java_type,wrap_type,to_wrapper,from_wrapper)""" + +def prepare_attr(nodename, attr): + (importcmd,exportcmd) = get_io_type(attr["type"], attr["name"], nodename) + attr["importcmd"] = importcmd + attr["exportcmd"] = exportcmd % {"val": "get_%s_%s(irn)" % (nodename, attr["name"])} + +def preprocess_node(nodename, node): + if "is_a" in node: + parent = ir_spec.nodes[node["is_a"]] + node["ins"] = parent["ins"] + if "outs" in parent: + node["outs"] = parent["outs"] + if "ins" not in node: + node["ins"] = [] + if "outs" in node: + node["mode"] = "mode_T" + if "arity" not in node: + node["arity"] = len(node["ins"]) + if "attrs" not in node: + node["attrs"] = [] + if "constructor_args" not in node: + node["constructor_args"] = [] + + # construct node arguments + arguments = [ ] + i = 0 + for input in node["ins"]: + arguments.append("prednodes[%i]" % i) + i += 1 + + if node["arity"] == "variable" or node["arity"] == "dynamic": + arguments.append("numpreds - %i" % (i + 1)) + arguments.append("prednodes + %i" % i) + + if "mode" not in node: + arguments.append("mode") + + for attr in node["attrs"]: + prepare_attr(nodename, attr) + arguments.append(attr["name"]) + + for arg in node["constructor_args"]: + prepare_attr(nodename, arg) + arguments.append(arg["name"]) + + node["arguments"] = arguments + +export_attrs_template = env.from_string(''' + case iro_{{nodename}}: + {{"write_mode(env, get_irn_mode(irn));"|ifnset(node,"mode")}} + {% for attr in node.attrs %}{{attr.exportcmd}} + {% endfor %} + {% for attr in node.constructor_args %}{{attr.exportcmd}} + {% endfor %}break;''') + +import_attrs_template = env.from_string(''' + case iro_{{nodename}}: + { + {{"ir_mode *mode = read_mode(env);"|ifnset(node,"mode")}} + {% for attr in node.attrs %}{{attr.importcmd}} + {% endfor %} + {% for attr in node.constructor_args %}{{attr.importcmd}} + {% endfor %}newnode = new_r_{{nodename}}(current_ir_graph{{node|block}}{{node["arguments"]|args}}); + break; + } +''') + +def main(argv): + """the main function""" + + if len(argv) < 2: + print "usage: %s destdirectory" % argv[0] + sys.exit(1) + + gendir = argv[1] + + file = open(gendir + "/gen_irio_export.inl", "w"); + for nodename, node in ir_spec.nodes.iteritems(): + preprocess_node(nodename, node) + if not "abstract" in node: + file.write(export_attrs_template.render(vars())) + file.write("\n") + file.close() + + file = open(gendir + "/gen_irio_import.inl", "w"); + for nodename, node in ir_spec.nodes.iteritems(): + if not "abstract" in node and nodename != "Start" and nodename != "End" and nodename != "Anchor" and nodename != "SymConst" and nodename != "Block": + file.write(import_attrs_template.render(vars())) + # TODO: SymConst + file.write("\n") + file.close() + + file = open(gendir + "/gen_irio_lex.inl", "w"); + for nodename, node in ir_spec.nodes.iteritems(): + if not "abstract" in node: + file.write("\tINSERT(\"" + nodename + "\", tt_iro, iro_" + nodename + ");\n"); + file.close() + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/ir_spec.py b/scripts/ir_spec.py new file mode 100755 index 000000000..d0707820c --- /dev/null +++ b/scripts/ir_spec.py @@ -0,0 +1,333 @@ +nodes = dict( +Start = dict( + mode = "mode_T", + op_flags = "cfopcode", + state = "pinned", + knownBlock = True, + noconstr = True, +), + +End = dict( + mode = "mode_X", + op_flags = "cfopcode", + state = "pinned", + arity = "dynamic", + knownBlock = True, + noconstr = True, +), + +Phi = dict( + noconstr = True, + state = "pinned", + arity = "variable", +), + +Jmp = dict( + mode = "mode_X", + op_flags = "cfopcode", + state = "pinned", + ins = [], +), + +IJmp = dict( + mode = "mode_X", + op_flags = "cfopcode", + state = "pinned", + ins = [ "target" ], +), + +Const = dict( + mode = "", + knownBlock = True, + attrs = [ + dict( + type = "tarval*", + name = "tarval", + ) + ], +), + +Block = dict( + mode = "mode_BB", + knownBlock = True, + noconstr = True, + arity = "variable", + java_add = ''' + public void addPred(Node node) { + binding_cons.add_immBlock_pred(ptr, node.ptr); + } + + public void mature() { + binding_cons.mature_immBlock(ptr); + } + + @Override + public Block getBlock() { + return null; + } + + public boolean blockVisited() { + return 0 != binding.Block_block_visited(ptr); + } + + public void markBlockVisited() { + binding.mark_Block_block_visited(ptr); + }''', +), + +SymConst = dict( + mode = "mode_P", + knownBlock = True, + noconstr = True, + attrs = [ + dict( + type = "ir_entity*", + name = "entity" + ) + ], +), + +# SymConst + +Call = dict( + ins = [ "mem", "ptr" ], + arity = "variable", + outs = [ "M_regular", "X_regular", "X_except", "T_result", "M_except", "P_value_res_base" ], + attrs = [ + dict( + type = "ir_type*", + name = "type" + ) + ] +), + +binop = dict( + abstract = True, + ins = [ "left", "right" ] +), + +Add = dict( + is_a = "binop" +), + +Carry = dict( + is_a = "binop" +), + +Sub = dict( + is_a = "binop" +), + +Borrow = dict( + is_a = "binop" +), + +Mul = dict( + is_a = "binop" +), + +Mulh = dict( + is_a = "binop" +), + +Abs = dict( + is_a = "unop" +), + +And = dict( + is_a = "binop" +), + +Or = dict( + is_a = "binop" +), + +Eor = dict( + is_a = "binop" +), + +Not = dict( + is_a = "unop" +), + +Shl = dict( + is_a = "binop" +), + +Shr = dict( + is_a = "binop" +), + +Shrs = dict( + is_a = "binop" +), + +Rotl = dict( + is_a = "binop" +), + +Load = dict( + ins = [ "mem", "ptr" ], + outs = [ "M", "X_regular", "X_except", "res" ], + attrs = [ + dict( + type = "ir_mode*", + name = "mode", + java_name = "load_mode" + ), + ], + constructor_args = [ + dict( + type = "cons_flags", + name = "flags", + ), + ], +), + +Store = dict( + ins = [ "mem", "ptr", "value" ], + outs = [ "M", "X_regular", "X_except" ], + constructor_args = [ + dict( + type = "cons_flags", + name = "flags", + ), + ], +), + +Anchor = dict( + mode = "mode_ANY", + ins = [ "end_block", "start_block", "end", "start", + "end_reg", "end_except", "initial_exec", + "frame", "tls", "initial_mem", "args", + "bad", "no_mem" ], + knownBlock = True, + noconstr = True +), + +NoMem = dict( + mode = "mode_M", + knownBlock = True, +), + +Bad = dict( + mode = "mode_Bad", + knownBlock = True, +), + +Pin = dict( + ins = [ "op" ], + mode = "get_irn_mode(op);" +), + +Proj = dict( + ins = [ "pred" ], + attrs = [ + dict( + type = "long", + name = "proj" + ) + ] +), + +Sel = dict( + ins = [ "mem", "ptr" ], + arity = "variable", + mode = "mode_P", + attrs = [ + dict( + type = "ir_entity*", + name = "entity" + ) + ] +), + +Sync = dict( + mode = "mode_M", + arity = "dynamic" +), + +Tuple = dict( + arity = "variable", + mode = "mode_T", +), + +Unknown = dict( + knownBlock = True +), + +Confirm = dict( + ins = [ "value", "bound" ], + block = "get_nodes_block(value)", + mode = "get_irn_mode(value)", + attrs = [ + dict( + name = "cmp", + type = "pn_Cmp" + ), + ], +), + +Return = dict( + ins = [ "mem" ], + arity = "variable", + mode = "mode_X" +), + +unop = dict( + abstract = True, + ins = [ "op" ] +), + +Minus = dict( + is_a = "unop" +), + +Mux = dict( + ins = [ "sel", "false", "true" ] +), + +Cond = dict( + ins = [ "selector" ], + outs = [ "false", "true" ], +), + +Cmp = dict( + is_a = "binop", + outs = [ "False", "Eq", "Lt", "Le", "Gt", "Ge", "Lg", "Leg", "Uo", "Ue", "Ul", "Ule", "Ug", "Uge", "Ne", "True" ], +), + +Conv = dict( + is_a = "unop" +), + +Alloc = dict( + ins = [ "mem", "size" ], + outs = [ "M", "X_regular", "X_except", "res" ], + attrs = [ + dict( + name = "type", + type = "ir_type*" + ), + dict( + name = "where", + type = "ir_where_alloc" + ) + ] +), + +Free = dict( + ins = [ "mem", "ptr", "size" ], + mode = "mode_M", + attrs = [ + dict( + name = "type", + type = "ir_type*" + ), + dict( + name = "where", + type = "ir_where_alloc" + ) + ] +), +) diff --git a/scripts/jinja2/__init__.py b/scripts/jinja2/__init__.py new file mode 100644 index 000000000..95b2d5bc9 --- /dev/null +++ b/scripts/jinja2/__init__.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +""" + jinja2 + ~~~~~~ + + Jinja2 is a template engine written in pure Python. It provides a + Django inspired non-XML syntax but supports inline expressions and + an optional sandboxed environment. + + Nutshell + -------- + + Here a small example of a Jinja2 template:: + + {% extends 'base.html' %} + {% block title %}Memberlist{% endblock %} + {% block content %} + + {% endblock %} + + + :copyright: 2008 by Armin Ronacher, Christoph Hack. + :license: BSD, see LICENSE for more details. +""" +__docformat__ = 'restructuredtext en' +try: + __version__ = __import__('pkg_resources') \ + .get_distribution('Jinja2').version +except: + __version__ = 'unknown' + +# high level interface +from jinja2.environment import Environment, Template + +# loaders +from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \ + DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader + +# bytecode caches +from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \ + MemcachedBytecodeCache + +# undefined types +from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined + +# exceptions +from jinja2.exceptions import TemplateError, UndefinedError, \ + TemplateNotFound, TemplateSyntaxError, TemplateAssertionError + +# decorators and public utilities +from jinja2.filters import environmentfilter, contextfilter +from jinja2.utils import Markup, escape, clear_caches, \ + environmentfunction, contextfunction, is_undefined + +__all__ = [ + 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader', + 'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader', + 'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache', + 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined', + 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound', + 'TemplateSyntaxError', 'TemplateAssertionError', 'environmentfilter', + 'contextfilter', 'Markup', 'escape', 'environmentfunction', + 'contextfunction', 'clear_caches', 'is_undefined' +] diff --git a/scripts/jinja2/_ipysupport.py b/scripts/jinja2/_ipysupport.py new file mode 100644 index 000000000..22ae823ae --- /dev/null +++ b/scripts/jinja2/_ipysupport.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +""" + jinja2._ipysupport + ~~~~~~~~~~~~~~~~~~ + + IronPython support library. This library exports functionality from + the CLR to Python that is normally available in the standard library. + + :copyright: Copyright 2008 by Armin Ronacher. + :license: BSD. +""" +from System import DateTime +from System.IO import Path, File, FileInfo + + +epoch = DateTime(1970, 1, 1) + + +class _PathModule(object): + """A minimal path module.""" + + sep = str(Path.DirectorySeparatorChar) + altsep = str(Path.AltDirectorySeparatorChar) + pardir = '..' + + def join(self, path, *args): + args = list(args[::-1]) + while args: + path = Path.Combine(path, args.pop()) + return path + + def isfile(self, filename): + return File.Exists(filename) + + def getmtime(self, filename): + info = FileInfo(filename) + return int((info.LastAccessTimeUtc - epoch).TotalSeconds) + + +path = _PathModule() diff --git a/scripts/jinja2/_speedups.c b/scripts/jinja2/_speedups.c new file mode 100644 index 000000000..40bb8c0a6 --- /dev/null +++ b/scripts/jinja2/_speedups.c @@ -0,0 +1,221 @@ +/** + * jinja2._speedups + * ~~~~~~~~~~~~~~~~ + * + * This module implements functions for automatic escaping in C for better + * performance. Additionally it defines a `tb_set_next` function to patch the + * debug traceback. If the speedups module is not compiled a ctypes + * implementation of `tb_set_next` and Python implementations of the other + * functions are used. + * + * :copyright: 2008 by Armin Ronacher, Mickaël Guérin. + * :license: BSD. + */ + +#include + +#define ESCAPED_CHARS_TABLE_SIZE 63 +#define UNICHR(x) (((PyUnicodeObject*)PyUnicode_DecodeASCII(x, strlen(x), NULL))->str); + +#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) +typedef int Py_ssize_t; +#define PY_SSIZE_T_MAX INT_MAX +#define PY_SSIZE_T_MIN INT_MIN +#endif + + +static PyObject* markup; +static Py_ssize_t escaped_chars_delta_len[ESCAPED_CHARS_TABLE_SIZE]; +static Py_UNICODE *escaped_chars_repl[ESCAPED_CHARS_TABLE_SIZE]; + +static int +init_constants(void) +{ + PyObject *module; + /* happing of characters to replace */ + escaped_chars_repl['"'] = UNICHR("""); + escaped_chars_repl['\''] = UNICHR("'"); + escaped_chars_repl['&'] = UNICHR("&"); + escaped_chars_repl['<'] = UNICHR("<"); + escaped_chars_repl['>'] = UNICHR(">"); + + /* lengths of those characters when replaced - 1 */ + memset(escaped_chars_delta_len, 0, sizeof (escaped_chars_delta_len)); + escaped_chars_delta_len['"'] = escaped_chars_delta_len['\''] = \ + escaped_chars_delta_len['&'] = 4; + escaped_chars_delta_len['<'] = escaped_chars_delta_len['>'] = 3; + + /* import markup type so that we can mark the return value */ + module = PyImport_ImportModule("jinja2.utils"); + if (!module) + return 0; + markup = PyObject_GetAttrString(module, "Markup"); + Py_DECREF(module); + + return 1; +} + +static PyObject* +escape_unicode(PyUnicodeObject *in) +{ + PyUnicodeObject *out; + Py_UNICODE *inp = in->str; + const Py_UNICODE *inp_end = in->str + in->length; + Py_UNICODE *next_escp; + Py_UNICODE *outp; + Py_ssize_t delta=0, erepl=0, delta_len=0; + + /* First we need to figure out how long the escaped string will be */ + while (*(inp) || inp < inp_end) { + if (*inp < ESCAPED_CHARS_TABLE_SIZE && escaped_chars_delta_len[*inp]) { + delta += escaped_chars_delta_len[*inp]; + ++erepl; + } + ++inp; + } + + /* Do we need to escape anything at all? */ + if (!erepl) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = (PyUnicodeObject*)PyUnicode_FromUnicode(NULL, in->length + delta); + if (!out) + return NULL; + + outp = out->str; + inp = in->str; + while (erepl-- > 0) { + /* look for the next substitution */ + next_escp = inp; + while (next_escp < inp_end) { + if (*next_escp < ESCAPED_CHARS_TABLE_SIZE && + (delta_len = escaped_chars_delta_len[*next_escp])) { + ++delta_len; + break; + } + ++next_escp; + } + + if (next_escp > inp) { + /* copy unescaped chars between inp and next_escp */ + Py_UNICODE_COPY(outp, inp, next_escp-inp); + outp += next_escp - inp; + } + + /* escape 'next_escp' */ + Py_UNICODE_COPY(outp, escaped_chars_repl[*next_escp], delta_len); + outp += delta_len; + + inp = next_escp + 1; + } + if (inp < inp_end) + Py_UNICODE_COPY(outp, inp, in->length - (inp - in->str)); + + return (PyObject*)out; +} + + +static PyObject* +escape(PyObject *self, PyObject *text) +{ + PyObject *s = NULL, *rv = NULL, *html; + + /* we don't have to escape integers, bools or floats */ + if (PyInt_CheckExact(text) || PyLong_CheckExact(text) || + PyFloat_CheckExact(text) || PyBool_Check(text) || + text == Py_None) + return PyObject_CallFunctionObjArgs(markup, text, NULL); + + /* if the object has an __html__ method that performs the escaping */ + html = PyObject_GetAttrString(text, "__html__"); + if (html) { + rv = PyObject_CallObject(html, NULL); + Py_DECREF(html); + return rv; + } + + /* otherwise make the object unicode if it isn't, then escape */ + PyErr_Clear(); + if (!PyUnicode_Check(text)) { + PyObject *unicode = PyObject_Unicode(text); + if (!unicode) + return NULL; + s = escape_unicode((PyUnicodeObject*)unicode); + Py_DECREF(unicode); + } + else + s = escape_unicode((PyUnicodeObject*)text); + + /* convert the unicode string into a markup object. */ + rv = PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL); + Py_DECREF(s); + return rv; +} + + +static PyObject* +soft_unicode(PyObject *self, PyObject *s) +{ + if (!PyUnicode_Check(s)) + return PyObject_Unicode(s); + Py_INCREF(s); + return s; +} + + +static PyObject* +tb_set_next(PyObject *self, PyObject *args) +{ + PyTracebackObject *tb, *old; + PyObject *next; + + if (!PyArg_ParseTuple(args, "O!O:tb_set_next", &PyTraceBack_Type, &tb, &next)) + return NULL; + if (next == Py_None) + next = NULL; + else if (!PyTraceBack_Check(next)) { + PyErr_SetString(PyExc_TypeError, + "tb_set_next arg 2 must be traceback or None"); + return NULL; + } + else + Py_INCREF(next); + + old = tb->tb_next; + tb->tb_next = (PyTracebackObject*)next; + Py_XDECREF(old); + + Py_INCREF(Py_None); + return Py_None; +} + + +static PyMethodDef module_methods[] = { + {"escape", (PyCFunction)escape, METH_O, + "escape(s) -> markup\n\n" + "Convert the characters &, <, >, ', and \" in string s to HTML-safe\n" + "sequences. Use this if you need to display text that might contain\n" + "such characters in HTML. Marks return value as markup string."}, + {"soft_unicode", (PyCFunction)soft_unicode, METH_O, + "soft_unicode(object) -> string\n\n" + "Make a string unicode if it isn't already. That way a markup\n" + "string is not converted back to unicode."}, + {"tb_set_next", (PyCFunction)tb_set_next, METH_VARARGS, + "Set the tb_next member of a traceback object."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif +PyMODINIT_FUNC +init_speedups(void) +{ + if (!init_constants()) + return; + + Py_InitModule3("jinja2._speedups", module_methods, ""); +} diff --git a/scripts/jinja2/bccache.py b/scripts/jinja2/bccache.py new file mode 100644 index 000000000..2c57616ec --- /dev/null +++ b/scripts/jinja2/bccache.py @@ -0,0 +1,280 @@ +# -*- coding: utf-8 -*- +""" + jinja2.bccache + ~~~~~~~~~~~~~~ + + This module implements the bytecode cache system Jinja is optionally + using. This is useful if you have very complex template situations and + the compiliation of all those templates slow down your application too + much. + + Situations where this is useful are often forking web applications that + are initialized on the first request. + + :copyright: Copyright 2008 by Armin Ronacher. + :license: BSD. +""" +from os import path, listdir +import marshal +import tempfile +import cPickle as pickle +import fnmatch +from cStringIO import StringIO +try: + from hashlib import sha1 +except ImportError: + from sha import new as sha1 +from jinja2.utils import open_if_exists + + +bc_version = 1 +bc_magic = 'j2' + pickle.dumps(bc_version, 2) + + +class Bucket(object): + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment, key, checksum): + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self): + """Resets the bucket (unloads the bytecode).""" + self.code = None + + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + # now load the code. Because marshal is not able to load + # from arbitrary streams we have to work around that + if isinstance(f, file): + self.code = marshal.load(f) + else: + self.code = marshal.loads(f.read()) + + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError('can\'t write empty bucket') + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + if isinstance(f, file): + marshal.dump(self.code, f) + else: + f.write(marshal.dumps(self.code)) + + def bytecode_from_string(self, string): + """Load bytecode from a string.""" + self.load_bytecode(StringIO(string)) + + def bytecode_to_string(self): + """Return the bytecode as string.""" + out = StringIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache(object): + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with file(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with file(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja2. + """ + + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self): + """Clears the cache. This method is not used by Jinja2 but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode('utf-8')) + if filename is not None: + if isinstance(filename, unicode): + filename = filename.encode('utf-8') + hash.update('|' + filename) + return hash.hexdigest() + + def get_source_checksum(self, source): + """Returns a checksum for the source.""" + return sha1(source.encode('utf-8')).hexdigest() + + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified the system temporary items folder is used. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__(self, directory=None, pattern='__jinja2_%s.cache'): + if directory is None: + directory = tempfile.gettempdir() + self.directory = directory + self.pattern = pattern + + def _get_cache_filename(self, bucket): + return path.join(self.directory, self.pattern % bucket.key) + + def load_bytecode(self, bucket): + f = open_if_exists(self._get_cache_filename(bucket), 'rb') + if f is not None: + try: + bucket.load_bytecode(f) + finally: + f.close() + + def dump_bytecode(self, bucket): + f = file(self._get_cache_filename(bucket), 'wb') + try: + bucket.write_bytecode(f) + finally: + f.close() + + def clear(self): + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + files = fnmatch.filter(listdir(self.directory), self.pattern % '*') + for filename in files: + try: + remove(path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `werkzeug `_.contrib.cache + - `python-memcached `_ + - `cmemcache `_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only unicode. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + """ + + def __init__(self, client, prefix='jinja2/bytecode/', timeout=None): + self.client = client + self.prefix = prefix + self.timeout = timeout + + def load_bytecode(self, bucket): + code = self.client.get(self.prefix + bucket.key) + if code is not None: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket): + args = (self.prefix + bucket.key, bucket.bytecode_to_string()) + if self.timeout is not None: + args += (self.timeout,) + self.client.set(*args) diff --git a/scripts/jinja2/compiler.py b/scripts/jinja2/compiler.py new file mode 100644 index 000000000..5074a342e --- /dev/null +++ b/scripts/jinja2/compiler.py @@ -0,0 +1,1443 @@ +# -*- coding: utf-8 -*- +""" + jinja2.compiler + ~~~~~~~~~~~~~~~ + + Compiles nodes into python code. + + :copyright: Copyright 2008 by Armin Ronacher. + :license: BSD. +""" +from cStringIO import StringIO +from itertools import chain +from jinja2 import nodes +from jinja2.visitor import NodeVisitor, NodeTransformer +from jinja2.exceptions import TemplateAssertionError +from jinja2.utils import Markup, concat, escape, is_python_keyword + + +operators = { + 'eq': '==', + 'ne': '!=', + 'gt': '>', + 'gteq': '>=', + 'lt': '<', + 'lteq': '<=', + 'in': 'in', + 'notin': 'not in' +} + +try: + exec '(0 if 0 else 0)' +except SyntaxError: + have_condexpr = False +else: + have_condexpr = True + + +def generate(node, environment, name, filename, stream=None): + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError('Can\'t compile non template nodes') + generator = CodeGenerator(environment, name, filename, stream) + generator.visit(node) + if stream is None: + return generator.stream.getvalue() + + +def has_safe_repr(value): + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + if isinstance(value, (bool, int, long, float, complex, basestring, + xrange, Markup)): + return True + if isinstance(value, (tuple, list, set, frozenset)): + for item in value: + if not has_safe_repr(item): + return False + return True + elif isinstance(value, dict): + for key, value in value.iteritems(): + if not has_safe_repr(key): + return False + if not has_safe_repr(value): + return False + return True + return False + + +def find_undeclared(nodes, names): + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class Identifiers(object): + """Tracks the status of identifiers in frames.""" + + def __init__(self): + # variables that are known to be declared (probably from outer + # frames or because they are special for the frame) + self.declared = set() + + # undeclared variables from outer scopes + self.outer_undeclared = set() + + # names that are accessed without being explicitly declared by + # this one or any of the outer scopes. Names can appear both in + # declared and undeclared. + self.undeclared = set() + + # names that are declared locally + self.declared_locally = set() + + # names that are declared by parameters + self.declared_parameter = set() + + def add_special(self, name): + """Register a special name like `loop`.""" + self.undeclared.discard(name) + self.declared.add(name) + + def is_declared(self, name, local_only=False): + """Check if a name is declared in this or an outer scope.""" + if name in self.declared_locally or name in self.declared_parameter: + return True + if local_only: + return False + return name in self.declared + + def find_shadowed(self, extra=()): + """Find all the shadowed names. extra is an iterable of variables + that may be defined with `add_special` which may occour scoped. + """ + return (self.declared | self.outer_undeclared) & \ + (self.declared_locally | self.declared_parameter) | \ + set(x for x in extra if self.is_declared(x)) + + +class Frame(object): + """Holds compile time information for us.""" + + def __init__(self, parent=None): + self.identifiers = Identifiers() + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = parent and parent.require_output_check + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer = None + + # the name of the block we're in, otherwise None. + self.block = parent and parent.block or None + + # the parent of this frame + self.parent = parent + + if parent is not None: + self.identifiers.declared.update( + parent.identifiers.declared | + parent.identifiers.declared_locally | + parent.identifiers.declared_parameter | + parent.identifiers.undeclared + ) + self.identifiers.outer_undeclared.update( + parent.identifiers.undeclared - + self.identifiers.declared + ) + self.buffer = parent.buffer + + def copy(self): + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.identifiers = object.__new__(self.identifiers.__class__) + rv.identifiers.__dict__.update(self.identifiers.__dict__) + return rv + + def inspect(self, nodes, hard_scope=False): + """Walk the node and check for identifiers. If the scope is hard (eg: + enforce on a python level) overrides from outer scopes are tracked + differently. + """ + visitor = FrameIdentifierVisitor(self.identifiers, hard_scope) + for node in nodes: + visitor.visit(node) + + def inner(self): + """Return an inner frame.""" + return Frame(self) + + def soft(self): + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + """ + rv = self.copy() + rv.rootlevel = False + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self): + self.filters = set() + self.tests = set() + + def visit_Filter(self, node): + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node): + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names): + self.names = set(names) + self.undeclared = set() + + def visit_Name(self, node): + if node.ctx == 'load' and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node): + """Stop visiting a blocks.""" + + +class FrameIdentifierVisitor(NodeVisitor): + """A visitor for `Frame.inspect`.""" + + def __init__(self, identifiers, hard_scope): + self.identifiers = identifiers + self.hard_scope = hard_scope + + def visit_Name(self, node): + """All assignments to names go through this function.""" + if node.ctx == 'store': + self.identifiers.declared_locally.add(node.name) + elif node.ctx == 'param': + self.identifiers.declared_parameter.add(node.name) + elif node.ctx == 'load' and not \ + self.identifiers.is_declared(node.name, self.hard_scope): + self.identifiers.undeclared.add(node.name) + + def visit_Macro(self, node): + self.identifiers.declared_locally.add(node.name) + + def visit_Import(self, node): + self.generic_visit(node) + self.identifiers.declared_locally.add(node.target) + + def visit_FromImport(self, node): + self.generic_visit(node) + for name in node.names: + if isinstance(name, tuple): + self.identifiers.declared_locally.add(name[1]) + else: + self.identifiers.declared_locally.add(name) + + def visit_Assign(self, node): + """Visit assignments in the correct order.""" + self.visit(node.node) + self.visit(node.target) + + def visit_For(self, node): + """Visiting stops at for blocks. However the block sequence + is visited as part of the outer scope. + """ + self.visit(node.iter) + + def visit_CallBlock(self, node): + for child in node.iter_child_nodes(exclude=('body',)): + self.visit(child) + + def visit_FilterBlock(self, node): + self.visit(node.filter) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + + def __init__(self, environment, name, filename, stream=None): + if stream is None: + stream = StringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + + # aliases for imports + self.import_aliases = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests = {} + self.filters = {} + + # the debug information + self.debug_info = [] + self._write_debug_info = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # -- Various compilation helpers + + def fail(self, msg, lineno): + """Fail with a `TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self): + """Get a new unique identifier.""" + self._last_identifier += 1 + return 't_%d' % self._last_identifier + + def buffer(self, frame): + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline('%s = []' % frame.buffer) + + def return_buffer_contents(self, frame): + """Return the buffer contents of the frame.""" + if self.environment.autoescape: + self.writeline('return Markup(concat(%s))' % frame.buffer) + else: + self.writeline('return concat(%s)' % frame.buffer) + + def indent(self): + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step=1): + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame, node=None): + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline('yield ', node) + else: + self.writeline('%s.append(' % frame.buffer, node) + + def end_write(self, frame): + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(')') + + def simple_write(self, s, frame, node=None): + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes, frame): + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically + unless the force_generator parameter is set to False. + """ + if frame.buffer is None: + self.writeline('if 0: yield None') + else: + self.writeline('pass') + try: + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x): + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write('\n' * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, + self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(' ' * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline(self, x, node=None, extra=0): + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node=None, extra=0): + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature(self, node, frame, extra_kwargs=None): + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occour. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = False + for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): + if is_python_keyword(kwarg): + kwarg_workaround = True + break + + for arg in node.args: + self.write(', ') + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(', ') + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in extra_kwargs.iteritems(): + self.write(', %s=%s' % (key, value)) + if node.dyn_args: + self.write(', *') + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(', **dict({') + else: + self.write(', **{') + for kwarg in node.kwargs: + self.write('%r: ' % kwarg.key) + self.visit(kwarg.value, frame) + self.write(', ') + if extra_kwargs is not None: + for key, value in extra_kwargs.iteritems(): + self.write('%r: %s, ' % (key, value)) + if node.dyn_kwargs is not None: + self.write('}, **') + self.visit(node.dyn_kwargs, frame) + self.write(')') + else: + self.write('}') + + elif node.dyn_kwargs is not None: + self.write(', **') + self.visit(node.dyn_kwargs, frame) + + def pull_locals(self, frame): + """Pull all the references identifiers into the local scope.""" + for name in frame.identifiers.undeclared: + self.writeline('l_%s = context.resolve(%r)' % (name, name)) + + def pull_dependencies(self, nodes): + """Pull all the dependencies.""" + visitor = DependencyFinderVisitor() + for node in nodes: + visitor.visit(node) + for dependency in 'filters', 'tests': + mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): + if name not in mapping: + mapping[name] = self.temporary_identifier() + self.writeline('%s = environment.%s[%r]' % + (mapping[name], dependency, name)) + + def push_scope(self, frame, extra_vars=()): + """This function returns all the shadowed variables in a dict + in the form name: alias and will write the required assignments + into the current scope. No indentation takes place. + + This also predefines locally declared variables from the loop + body because under some circumstances it may be the case that + + `extra_vars` is passed to `Identifiers.find_shadowed`. + """ + aliases = {} + for name in frame.identifiers.find_shadowed(extra_vars): + aliases[name] = ident = self.temporary_identifier() + self.writeline('%s = l_%s' % (ident, name)) + to_declare = set() + for name in frame.identifiers.declared_locally: + if name not in aliases: + to_declare.add('l_' + name) + if to_declare: + self.writeline(' = '.join(to_declare) + ' = missing') + return aliases + + def pop_scope(self, aliases, frame): + """Restore all aliases and delete unused variables.""" + for name, alias in aliases.iteritems(): + self.writeline('l_%s = %s' % (name, alias)) + to_delete = set() + for name in frame.identifiers.declared_locally: + if name not in aliases: + to_delete.add('l_' + name) + if to_delete: + self.writeline('del ' + ', '.join(to_delete)) + + def function_scoping(self, node, frame, children=None, + find_special=True): + """In Jinja a few statements require the help of anonymous + functions. Those are currently macros and call blocks and in + the future also recursive loops. As there is currently + technical limitation that doesn't allow reading and writing a + variable in a scope where the initial value is coming from an + outer scope, this function tries to fall back with a common + error message. Additionally the frame passed is modified so + that the argumetns are collected and callers are looked up. + + This will return the modified frame. + """ + # we have to iterate twice over it, make sure that works + if children is None: + children = node.iter_child_nodes() + children = list(children) + func_frame = frame.inner() + func_frame.inspect(children, hard_scope=True) + + # variables that are undeclared (accessed before declaration) and + # declared locally *and* part of an outside scope raise a template + # assertion error. Reason: we can't generate reasonable code from + # it without aliasing all the variables. XXX: alias them ^^ + overriden_closure_vars = ( + func_frame.identifiers.undeclared & + func_frame.identifiers.declared & + (func_frame.identifiers.declared_locally | + func_frame.identifiers.declared_parameter) + ) + if overriden_closure_vars: + self.fail('It\'s not possible to set and access variables ' + 'derived from an outer scope! (affects: %s' % + ', '.join(sorted(overriden_closure_vars)), node.lineno) + + # remove variables from a closure from the frame's undeclared + # identifiers. + func_frame.identifiers.undeclared -= ( + func_frame.identifiers.undeclared & + func_frame.identifiers.declared + ) + + # no special variables for this scope, abort early + if not find_special: + return func_frame + + func_frame.accesses_kwargs = False + func_frame.accesses_varargs = False + func_frame.accesses_caller = False + func_frame.arguments = args = ['l_' + x.name for x in node.args] + + undeclared = find_undeclared(children, ('caller', 'kwargs', 'varargs')) + + if 'caller' in undeclared: + func_frame.accesses_caller = True + func_frame.identifiers.add_special('caller') + args.append('l_caller') + if 'kwargs' in undeclared: + func_frame.accesses_kwargs = True + func_frame.identifiers.add_special('kwargs') + args.append('l_kwargs') + if 'varargs' in undeclared: + func_frame.accesses_varargs = True + func_frame.identifiers.add_special('varargs') + args.append('l_varargs') + return func_frame + + def macro_body(self, node, frame, children=None): + """Dump the function def of a macro or call block.""" + frame = self.function_scoping(node, frame, children) + # macros are delayed, they never require output checks + frame.require_output_check = False + args = frame.arguments + self.writeline('def macro(%s):' % ', '.join(args), node) + self.indent() + self.buffer(frame) + self.pull_locals(frame) + self.blockvisit(node.body, frame) + self.return_buffer_contents(frame) + self.outdent() + return frame + + def macro_def(self, node, frame): + """Dump the macro definition for the def created by macro_body.""" + arg_tuple = ', '.join(repr(x.name) for x in node.args) + name = getattr(node, 'name', None) + if len(node.args) == 1: + arg_tuple += ',' + self.write('Macro(environment, macro, %r, (%s), (' % + (name, arg_tuple)) + for arg in node.defaults: + self.visit(arg, frame) + self.write(', ') + self.write('), %r, %r, %r)' % ( + bool(frame.accesses_kwargs), + bool(frame.accesses_varargs), + bool(frame.accesses_caller) + )) + + def position(self, node): + """Return a human readable position for the node.""" + rv = 'line %d' % node.lineno + if self.name is not None: + rv += ' in' + repr(self.name) + return rv + + # -- Statement Visitors + + def visit_Template(self, node, frame=None): + assert frame is None, 'no root frame allowed' + from jinja2.runtime import __all__ as exported + self.writeline('from __future__ import division') + self.writeline('from jinja2.runtime import ' + ', '.join(exported)) + + # do we have an extends tag at all? If not, we can save some + # overhead by just not processing any inheritance code. + have_extends = node.find(nodes.Extends) is not None + + # find all blocks + for block in node.find_all(nodes.Block): + if block.name in self.blocks: + self.fail('block %r defined twice' % block.name, block.lineno) + self.blocks[block.name] = block + + # find all imports and import them + for import_ in node.find_all(nodes.ImportedName): + if import_.importname not in self.import_aliases: + imp = import_.importname + self.import_aliases[imp] = alias = self.temporary_identifier() + if '.' in imp: + module, obj = imp.rsplit('.', 1) + self.writeline('from %s import %s as %s' % + (module, obj, alias)) + else: + self.writeline('import %s as %s' % (imp, alias)) + + # add the load name + self.writeline('name = %r' % self.name) + + # generate the root render function. + self.writeline('def root(context, environment=environment):', extra=1) + + # process the root + frame = Frame() + frame.inspect(node.body) + frame.toplevel = frame.rootlevel = True + frame.require_output_check = have_extends and not self.has_known_extends + self.indent() + if have_extends: + self.writeline('parent_template = None') + if 'self' in find_undeclared(node.body, ('self',)): + frame.identifiers.add_special('self') + self.writeline('l_self = TemplateReference(context)') + self.pull_locals(frame) + self.pull_dependencies(node.body) + self.blockvisit(node.body, frame) + self.outdent() + + # make sure that the parent root is called. + if have_extends: + if not self.has_known_extends: + self.indent() + self.writeline('if parent_template is not None:') + self.indent() + self.writeline('for event in parent_template.' + 'root_render_func(context):') + self.indent() + self.writeline('yield event') + self.outdent(2 + (not self.has_known_extends)) + + # at this point we now have the blocks collected and can visit them too. + for name, block in self.blocks.iteritems(): + block_frame = Frame() + block_frame.inspect(block.body) + block_frame.block = name + self.writeline('def block_%s(context, environment=environment):' + % name, block, 1) + self.indent() + undeclared = find_undeclared(block.body, ('self', 'super')) + if 'self' in undeclared: + block_frame.identifiers.add_special('self') + self.writeline('l_self = TemplateReference(context)') + if 'super' in undeclared: + block_frame.identifiers.add_special('super') + self.writeline('l_super = context.super(%r, ' + 'block_%s)' % (name, name)) + self.pull_locals(block_frame) + self.pull_dependencies(block.body) + self.blockvisit(block.body, block_frame) + self.outdent() + + self.writeline('blocks = {%s}' % ', '.join('%r: block_%s' % (x, x) + for x in self.blocks), + extra=1) + + # add a function that returns the debug info + self.writeline('debug_info = %r' % '&'.join('%s=%s' % x for x + in self.debug_info)) + + def visit_Block(self, node, frame): + """Call a block and register it for the template.""" + level = 1 + if frame.toplevel: + # if we know that we are a child template, there is no need to + # check if we are one + if self.has_known_extends: + return + if self.extends_so_far > 0: + self.writeline('if parent_template is None:') + self.indent() + level += 1 + self.writeline('for event in context.blocks[%r][0](context):' % + node.name, node) + self.indent() + self.simple_write('event', frame) + self.outdent(level) + + def visit_Extends(self, node, frame): + """Calls the extender.""" + if not frame.toplevel: + self.fail('cannot use extend from a non top-level scope', + node.lineno) + + # if the number of extends statements in general is zero so + # far, we don't have to add a check if something extended + # the template before this one. + if self.extends_so_far > 0: + + # if we have a known extends we just add a template runtime + # error into the generated code. We could catch that at compile + # time too, but i welcome it not to confuse users by throwing the + # same error at different times just "because we can". + if not self.has_known_extends: + self.writeline('if parent_template is not None:') + self.indent() + self.writeline('raise TemplateRuntimeError(%r)' % + 'extended multiple times') + self.outdent() + + # if we have a known extends already we don't need that code here + # as we know that the template execution will end here. + if self.has_known_extends: + raise CompilerExit() + + self.writeline('parent_template = environment.get_template(', node) + self.visit(node.template, frame) + self.write(', %r)' % self.name) + self.writeline('for name, parent_block in parent_template.' + 'blocks.iteritems():') + self.indent() + self.writeline('context.blocks.setdefault(name, []).' + 'append(parent_block)') + self.outdent() + + # if this extends statement was in the root level we can take + # advantage of that information and simplify the generated code + # in the top level from this point onwards + if frame.rootlevel: + self.has_known_extends = True + + # and now we have one more + self.extends_so_far += 1 + + def visit_Include(self, node, frame): + """Handles includes.""" + if node.with_context: + self.writeline('template = environment.get_template(', node) + self.visit(node.template, frame) + self.write(', %r)' % self.name) + self.writeline('for event in template.root_render_func(' + 'template.new_context(context.parent, True, ' + 'locals())):') + else: + self.writeline('for event in environment.get_template(', node) + self.visit(node.template, frame) + self.write(', %r).module._body_stream:' % + self.name) + self.indent() + self.simple_write('event', frame) + self.outdent() + + def visit_Import(self, node, frame): + """Visit regular imports.""" + self.writeline('l_%s = ' % node.target, node) + if frame.toplevel: + self.write('context.vars[%r] = ' % node.target) + self.write('environment.get_template(') + self.visit(node.template, frame) + self.write(', %r).' % self.name) + if node.with_context: + self.write('make_module(context.parent, True, locals())') + else: + self.write('module') + if frame.toplevel and not node.target.startswith('_'): + self.writeline('context.exported_vars.discard(%r)' % node.target) + + def visit_FromImport(self, node, frame): + """Visit named imports.""" + self.newline(node) + self.write('included_template = environment.get_template(') + self.visit(node.template, frame) + self.write(', %r).' % self.name) + if node.with_context: + self.write('make_module(context.parent, True)') + else: + self.write('module') + + var_names = [] + discarded_names = [] + for name in node.names: + if isinstance(name, tuple): + name, alias = name + else: + alias = name + self.writeline('l_%s = getattr(included_template, ' + '%r, missing)' % (alias, name)) + self.writeline('if l_%s is missing:' % alias) + self.indent() + self.writeline('l_%s = environment.undefined(%r %% ' + 'included_template.__name__, ' + 'name=%r)' % + (alias, 'the template %%r (imported on %s) does ' + 'not export the requested name %s' % ( + self.position(node), + repr(name) + ), name)) + self.outdent() + if frame.toplevel: + var_names.append(alias) + if not alias.startswith('_'): + discarded_names.append(alias) + + if var_names: + if len(var_names) == 1: + name = var_names[0] + self.writeline('context.vars[%r] = l_%s' % (name, name)) + else: + self.writeline('context.vars.update({%s})' % ', '.join( + '%r: l_%s' % (name, name) for name in var_names + )) + if discarded_names: + if len(discarded_names) == 1: + self.writeline('context.exported_vars.discard(%r)' % + discarded_names[0]) + else: + self.writeline('context.exported_vars.difference_' + 'update((%s))' % ', '.join(map(repr, discarded_names))) + + def visit_For(self, node, frame): + # when calculating the nodes for the inner frame we have to exclude + # the iterator contents from it + children = node.iter_child_nodes(exclude=('iter',)) + if node.recursive: + loop_frame = self.function_scoping(node, frame, children, + find_special=False) + else: + loop_frame = frame.inner() + loop_frame.inspect(children) + + # try to figure out if we have an extended loop. An extended loop + # is necessary if the loop is in recursive mode if the special loop + # variable is accessed in the body. + extended_loop = node.recursive or 'loop' in \ + find_undeclared(node.iter_child_nodes( + only=('body',)), ('loop',)) + + # if we don't have an recursive loop we have to find the shadowed + # variables at that point. Because loops can be nested but the loop + # variable is a special one we have to enforce aliasing for it. + if not node.recursive: + aliases = self.push_scope(loop_frame, ('loop',)) + + # otherwise we set up a buffer and add a function def + else: + self.writeline('def loop(reciter, loop_render_func):', node) + self.indent() + self.buffer(loop_frame) + aliases = {} + + # make sure the loop variable is a special one and raise a template + # assertion error if a loop tries to write to loop + if extended_loop: + loop_frame.identifiers.add_special('loop') + for name in node.find_all(nodes.Name): + if name.ctx == 'store' and name.name == 'loop': + self.fail('Can\'t assign to special loop variable ' + 'in for-loop target', name.lineno) + + self.pull_locals(loop_frame) + if node.else_: + iteration_indicator = self.temporary_identifier() + self.writeline('%s = 1' % iteration_indicator) + + # Create a fake parent loop if the else or test section of a + # loop is accessing the special loop variable and no parent loop + # exists. + if 'loop' not in aliases and 'loop' in find_undeclared( + node.iter_child_nodes(only=('else_', 'test')), ('loop',)): + self.writeline("l_loop = environment.undefined(%r, name='loop')" % + ("'loop' is undefined. the filter section of a loop as well " + "as the else block doesn't have access to the special 'loop'" + " variable of the current loop. Because there is no parent " + "loop it's undefined. Happened in loop on %s" % + self.position(node))) + + self.writeline('for ', node) + self.visit(node.target, loop_frame) + self.write(extended_loop and ', l_loop in LoopContext(' or ' in ') + + # if we have an extened loop and a node test, we filter in the + # "outer frame". + if extended_loop and node.test is not None: + self.write('(') + self.visit(node.target, loop_frame) + self.write(' for ') + self.visit(node.target, loop_frame) + self.write(' in ') + if node.recursive: + self.write('reciter') + else: + self.visit(node.iter, loop_frame) + self.write(' if (') + test_frame = loop_frame.copy() + self.visit(node.test, test_frame) + self.write('))') + + elif node.recursive: + self.write('reciter') + else: + self.visit(node.iter, loop_frame) + + if node.recursive: + self.write(', recurse=loop_render_func):') + else: + self.write(extended_loop and '):' or ':') + + # tests in not extended loops become a continue + if not extended_loop and node.test is not None: + self.indent() + self.writeline('if not ') + self.visit(node.test, loop_frame) + self.write(':') + self.indent() + self.writeline('continue') + self.outdent(2) + + self.indent() + self.blockvisit(node.body, loop_frame) + if node.else_: + self.writeline('%s = 0' % iteration_indicator) + self.outdent() + + if node.else_: + self.writeline('if %s:' % iteration_indicator) + self.indent() + self.blockvisit(node.else_, loop_frame) + self.outdent() + + # reset the aliases if there are any. + self.pop_scope(aliases, loop_frame) + + # if the node was recursive we have to return the buffer contents + # and start the iteration code + if node.recursive: + self.return_buffer_contents(loop_frame) + self.outdent() + self.start_write(frame, node) + self.write('loop(') + self.visit(node.iter, frame) + self.write(', loop)') + self.end_write(frame) + + def visit_If(self, node, frame): + if_frame = frame.soft() + self.writeline('if ', node) + self.visit(node.test, if_frame) + self.write(':') + self.indent() + self.blockvisit(node.body, if_frame) + self.outdent() + if node.else_: + self.writeline('else:') + self.indent() + self.blockvisit(node.else_, if_frame) + self.outdent() + + def visit_Macro(self, node, frame): + macro_frame = self.macro_body(node, frame) + self.newline() + if frame.toplevel: + if not node.name.startswith('_'): + self.write('context.exported_vars.add(%r)' % node.name) + self.writeline('context.vars[%r] = ' % node.name) + self.write('l_%s = ' % node.name) + self.macro_def(node, macro_frame) + + def visit_CallBlock(self, node, frame): + children = node.iter_child_nodes(exclude=('call',)) + call_frame = self.macro_body(node, frame, children) + self.writeline('caller = ') + self.macro_def(node, call_frame) + self.start_write(frame, node) + self.visit_Call(node.call, call_frame, forward_caller=True) + self.end_write(frame) + + def visit_FilterBlock(self, node, frame): + filter_frame = frame.inner() + filter_frame.inspect(node.iter_child_nodes()) + aliases = self.push_scope(filter_frame) + self.pull_locals(filter_frame) + self.buffer(filter_frame) + self.blockvisit(node.body, filter_frame) + self.start_write(frame, node) + self.visit_Filter(node.filter, filter_frame) + self.end_write(frame) + self.pop_scope(aliases, filter_frame) + + def visit_ExprStmt(self, node, frame): + self.newline(node) + self.visit(node.node, frame) + + def visit_Output(self, node, frame): + # if we have a known extends statement, we don't output anything + # if we are in a require_output_check section + if self.has_known_extends and frame.require_output_check: + return + + if self.environment.finalize: + finalize = lambda x: unicode(self.environment.finalize(x)) + else: + finalize = unicode + + self.newline(node) + + # if we are inside a frame that requires output checking, we do so + outdent_later = False + if frame.require_output_check: + self.writeline('if parent_template is None:') + self.indent() + outdent_later = True + + # try to evaluate as many chunks as possible into a static + # string at compile time. + body = [] + for child in node.nodes: + try: + const = child.as_const() + except nodes.Impossible: + body.append(child) + continue + try: + if self.environment.autoescape: + if hasattr(const, '__html__'): + const = const.__html__() + else: + const = escape(const) + const = finalize(const) + except: + # if something goes wrong here we evaluate the node + # at runtime for easier debugging + body.append(child) + continue + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + # if we have less than 3 nodes or a buffer we yield or extend/append + if len(body) < 3 or frame.buffer is not None: + if frame.buffer is not None: + # for one item we append, for more we extend + if len(body) == 1: + self.writeline('%s.append(' % frame.buffer) + else: + self.writeline('%s.extend((' % frame.buffer) + self.indent() + for item in body: + if isinstance(item, list): + val = repr(concat(item)) + if frame.buffer is None: + self.writeline('yield ' + val) + else: + self.writeline(val + ', ') + else: + if frame.buffer is None: + self.writeline('yield ', item) + else: + self.newline(item) + close = 1 + if self.environment.autoescape: + self.write('escape(') + else: + self.write('unicode(') + if self.environment.finalize is not None: + self.write('environment.finalize(') + close += 1 + self.visit(item, frame) + self.write(')' * close) + if frame.buffer is not None: + self.write(', ') + if frame.buffer is not None: + # close the open parentheses + self.outdent() + self.writeline(len(body) == 1 and ')' or '))') + + # otherwise we create a format string as this is faster in that case + else: + format = [] + arguments = [] + for item in body: + if isinstance(item, list): + format.append(concat(item).replace('%', '%%')) + else: + format.append('%s') + arguments.append(item) + self.writeline('yield ') + self.write(repr(concat(format)) + ' % (') + idx = -1 + self.indent() + for argument in arguments: + self.newline(argument) + close = 0 + if self.environment.autoescape: + self.write('escape(') + close += 1 + if self.environment.finalize is not None: + self.write('environment.finalize(') + close += 1 + self.visit(argument, frame) + self.write(')' * close + ', ') + self.outdent() + self.writeline(')') + + if outdent_later: + self.outdent() + + def visit_Assign(self, node, frame): + self.newline(node) + # toplevel assignments however go into the local namespace and + # the current template's context. We create a copy of the frame + # here and add a set so that the Name visitor can add the assigned + # names here. + if frame.toplevel: + assignment_frame = frame.copy() + assignment_frame.assigned_names = set() + else: + assignment_frame = frame + self.visit(node.target, assignment_frame) + self.write(' = ') + self.visit(node.node, frame) + + # make sure toplevel assignments are added to the context. + if frame.toplevel: + public_names = [x for x in assignment_frame.assigned_names + if not x.startswith('_')] + if len(assignment_frame.assigned_names) == 1: + name = iter(assignment_frame.assigned_names).next() + self.writeline('context.vars[%r] = l_%s' % (name, name)) + else: + self.writeline('context.vars.update({') + for idx, name in enumerate(assignment_frame.assigned_names): + if idx: + self.write(', ') + self.write('%r: l_%s' % (name, name)) + self.write('})') + if public_names: + if len(public_names) == 1: + self.writeline('context.exported_vars.add(%r)' % + public_names[0]) + else: + self.writeline('context.exported_vars.update((%s))' % + ', '.join(map(repr, public_names))) + + # -- Expression Visitors + + def visit_Name(self, node, frame): + if node.ctx == 'store' and frame.toplevel: + frame.assigned_names.add(node.name) + self.write('l_' + node.name) + + def visit_Const(self, node, frame): + val = node.value + if isinstance(val, float): + self.write(str(val)) + else: + self.write(repr(val)) + + def visit_TemplateData(self, node, frame): + self.write(repr(node.as_const())) + + def visit_Tuple(self, node, frame): + self.write('(') + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(idx == 0 and ',)' or ')') + + def visit_List(self, node, frame): + self.write('[') + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(']') + + def visit_Dict(self, node, frame): + self.write('{') + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item.key, frame) + self.write(': ') + self.visit(item.value, frame) + self.write('}') + + def binop(operator): + def visitor(self, node, frame): + self.write('(') + self.visit(node.left, frame) + self.write(' %s ' % operator) + self.visit(node.right, frame) + self.write(')') + return visitor + + def uaop(operator): + def visitor(self, node, frame): + self.write('(' + operator) + self.visit(node.node, frame) + self.write(')') + return visitor + + visit_Add = binop('+') + visit_Sub = binop('-') + visit_Mul = binop('*') + visit_Div = binop('/') + visit_FloorDiv = binop('//') + visit_Pow = binop('**') + visit_Mod = binop('%') + visit_And = binop('and') + visit_Or = binop('or') + visit_Pos = uaop('+') + visit_Neg = uaop('-') + visit_Not = uaop('not ') + del binop, uaop + + def visit_Concat(self, node, frame): + self.write('%s((' % (self.environment.autoescape and + 'markup_join' or 'unicode_join')) + for arg in node.nodes: + self.visit(arg, frame) + self.write(', ') + self.write('))') + + def visit_Compare(self, node, frame): + self.visit(node.expr, frame) + for op in node.ops: + self.visit(op, frame) + + def visit_Operand(self, node, frame): + self.write(' %s ' % operators[node.op]) + self.visit(node.expr, frame) + + def visit_Getattr(self, node, frame): + self.write('environment.getattr(') + self.visit(node.node, frame) + self.write(', %r)' % node.attr) + + def visit_Getitem(self, node, frame): + # slices bypass the environment getitem method. + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write('[') + self.visit(node.arg, frame) + self.write(']') + else: + self.write('environment.getitem(') + self.visit(node.node, frame) + self.write(', ') + self.visit(node.arg, frame) + self.write(')') + + def visit_Slice(self, node, frame): + if node.start is not None: + self.visit(node.start, frame) + self.write(':') + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(':') + self.visit(node.step, frame) + + def visit_Filter(self, node, frame): + self.write(self.filters[node.name] + '(') + func = self.environment.filters.get(node.name) + if func is None: + self.fail('no filter named %r' % node.name, node.lineno) + if getattr(func, 'contextfilter', False): + self.write('context, ') + elif getattr(func, 'environmentfilter', False): + self.write('environment, ') + + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif self.environment.autoescape: + self.write('Markup(concat(%s))' % frame.buffer) + else: + self.write('concat(%s)' % frame.buffer) + self.signature(node, frame) + self.write(')') + + def visit_Test(self, node, frame): + self.write(self.tests[node.name] + '(') + if node.name not in self.environment.tests: + self.fail('no test named %r' % node.name, node.lineno) + self.visit(node.node, frame) + self.signature(node, frame) + self.write(')') + + def visit_CondExpr(self, node, frame): + def write_expr2(): + if node.expr2 is not None: + return self.visit(node.expr2, frame) + self.write('environment.undefined(%r)' % ('the inline if-' + 'expression on %s evaluated to false and ' + 'no else section was defined.' % self.position(node))) + + if not have_condexpr: + self.write('((') + self.visit(node.test, frame) + self.write(') and (') + self.visit(node.expr1, frame) + self.write(',) or (') + write_expr2() + self.write(',))[0]') + else: + self.write('(') + self.visit(node.expr1, frame) + self.write(' if ') + self.visit(node.test, frame) + self.write(' else ') + write_expr2() + self.write(')') + + def visit_Call(self, node, frame, forward_caller=False): + if self.environment.sandboxed: + self.write('environment.call(context, ') + else: + self.write('context.call(') + self.visit(node.node, frame) + extra_kwargs = forward_caller and {'caller': 'caller'} or None + self.signature(node, frame, extra_kwargs) + self.write(')') + + def visit_Keyword(self, node, frame): + self.write(node.key + '=') + self.visit(node.value, frame) + + # -- Unused nodes for extensions + + def visit_MarkSafe(self, node, frame): + self.write('Markup(') + self.visit(node.expr, frame) + self.write(')') + + def visit_EnvironmentAttribute(self, node, frame): + self.write('environment.' + node.name) + + def visit_ExtensionAttribute(self, node, frame): + self.write('environment.extensions[%r].%s' % (node.identifier, node.name)) + + def visit_ImportedName(self, node, frame): + self.write(self.import_aliases[node.importname]) + + def visit_InternalName(self, node, frame): + self.write(node.name) + + def visit_ContextReference(self, node, frame): + self.write('context') + + def visit_Continue(self, node, frame): + self.writeline('continue', node) + + def visit_Break(self, node, frame): + self.writeline('break', node) diff --git a/scripts/jinja2/constants.py b/scripts/jinja2/constants.py new file mode 100644 index 000000000..c471e79ee --- /dev/null +++ b/scripts/jinja2/constants.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- +""" + jinja.constants + ~~~~~~~~~~~~~~~ + + Various constants. + + :copyright: 2007 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" + + +#: list of lorem ipsum words used by the lipsum() helper function +LOREM_IPSUM_WORDS = u'''\ +a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at +auctor augue bibendum blandit class commodo condimentum congue consectetuer +consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus +diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend +elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames +faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac +hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum +justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem +luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie +mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non +nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque +penatibus per pharetra phasellus placerat platea porta porttitor posuere +potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus +ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit +sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor +tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices +ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus +viverra volutpat vulputate''' + + +#: a dict of all html entities + apos +HTML_ENTITIES = { + 'AElig': 198, + 'Aacute': 193, + 'Acirc': 194, + 'Agrave': 192, + 'Alpha': 913, + 'Aring': 197, + 'Atilde': 195, + 'Auml': 196, + 'Beta': 914, + 'Ccedil': 199, + 'Chi': 935, + 'Dagger': 8225, + 'Delta': 916, + 'ETH': 208, + 'Eacute': 201, + 'Ecirc': 202, + 'Egrave': 200, + 'Epsilon': 917, + 'Eta': 919, + 'Euml': 203, + 'Gamma': 915, + 'Iacute': 205, + 'Icirc': 206, + 'Igrave': 204, + 'Iota': 921, + 'Iuml': 207, + 'Kappa': 922, + 'Lambda': 923, + 'Mu': 924, + 'Ntilde': 209, + 'Nu': 925, + 'OElig': 338, + 'Oacute': 211, + 'Ocirc': 212, + 'Ograve': 210, + 'Omega': 937, + 'Omicron': 927, + 'Oslash': 216, + 'Otilde': 213, + 'Ouml': 214, + 'Phi': 934, + 'Pi': 928, + 'Prime': 8243, + 'Psi': 936, + 'Rho': 929, + 'Scaron': 352, + 'Sigma': 931, + 'THORN': 222, + 'Tau': 932, + 'Theta': 920, + 'Uacute': 218, + 'Ucirc': 219, + 'Ugrave': 217, + 'Upsilon': 933, + 'Uuml': 220, + 'Xi': 926, + 'Yacute': 221, + 'Yuml': 376, + 'Zeta': 918, + 'aacute': 225, + 'acirc': 226, + 'acute': 180, + 'aelig': 230, + 'agrave': 224, + 'alefsym': 8501, + 'alpha': 945, + 'amp': 38, + 'and': 8743, + 'ang': 8736, + 'apos': 39, + 'aring': 229, + 'asymp': 8776, + 'atilde': 227, + 'auml': 228, + 'bdquo': 8222, + 'beta': 946, + 'brvbar': 166, + 'bull': 8226, + 'cap': 8745, + 'ccedil': 231, + 'cedil': 184, + 'cent': 162, + 'chi': 967, + 'circ': 710, + 'clubs': 9827, + 'cong': 8773, + 'copy': 169, + 'crarr': 8629, + 'cup': 8746, + 'curren': 164, + 'dArr': 8659, + 'dagger': 8224, + 'darr': 8595, + 'deg': 176, + 'delta': 948, + 'diams': 9830, + 'divide': 247, + 'eacute': 233, + 'ecirc': 234, + 'egrave': 232, + 'empty': 8709, + 'emsp': 8195, + 'ensp': 8194, + 'epsilon': 949, + 'equiv': 8801, + 'eta': 951, + 'eth': 240, + 'euml': 235, + 'euro': 8364, + 'exist': 8707, + 'fnof': 402, + 'forall': 8704, + 'frac12': 189, + 'frac14': 188, + 'frac34': 190, + 'frasl': 8260, + 'gamma': 947, + 'ge': 8805, + 'gt': 62, + 'hArr': 8660, + 'harr': 8596, + 'hearts': 9829, + 'hellip': 8230, + 'iacute': 237, + 'icirc': 238, + 'iexcl': 161, + 'igrave': 236, + 'image': 8465, + 'infin': 8734, + 'int': 8747, + 'iota': 953, + 'iquest': 191, + 'isin': 8712, + 'iuml': 239, + 'kappa': 954, + 'lArr': 8656, + 'lambda': 955, + 'lang': 9001, + 'laquo': 171, + 'larr': 8592, + 'lceil': 8968, + 'ldquo': 8220, + 'le': 8804, + 'lfloor': 8970, + 'lowast': 8727, + 'loz': 9674, + 'lrm': 8206, + 'lsaquo': 8249, + 'lsquo': 8216, + 'lt': 60, + 'macr': 175, + 'mdash': 8212, + 'micro': 181, + 'middot': 183, + 'minus': 8722, + 'mu': 956, + 'nabla': 8711, + 'nbsp': 160, + 'ndash': 8211, + 'ne': 8800, + 'ni': 8715, + 'not': 172, + 'notin': 8713, + 'nsub': 8836, + 'ntilde': 241, + 'nu': 957, + 'oacute': 243, + 'ocirc': 244, + 'oelig': 339, + 'ograve': 242, + 'oline': 8254, + 'omega': 969, + 'omicron': 959, + 'oplus': 8853, + 'or': 8744, + 'ordf': 170, + 'ordm': 186, + 'oslash': 248, + 'otilde': 245, + 'otimes': 8855, + 'ouml': 246, + 'para': 182, + 'part': 8706, + 'permil': 8240, + 'perp': 8869, + 'phi': 966, + 'pi': 960, + 'piv': 982, + 'plusmn': 177, + 'pound': 163, + 'prime': 8242, + 'prod': 8719, + 'prop': 8733, + 'psi': 968, + 'quot': 34, + 'rArr': 8658, + 'radic': 8730, + 'rang': 9002, + 'raquo': 187, + 'rarr': 8594, + 'rceil': 8969, + 'rdquo': 8221, + 'real': 8476, + 'reg': 174, + 'rfloor': 8971, + 'rho': 961, + 'rlm': 8207, + 'rsaquo': 8250, + 'rsquo': 8217, + 'sbquo': 8218, + 'scaron': 353, + 'sdot': 8901, + 'sect': 167, + 'shy': 173, + 'sigma': 963, + 'sigmaf': 962, + 'sim': 8764, + 'spades': 9824, + 'sub': 8834, + 'sube': 8838, + 'sum': 8721, + 'sup': 8835, + 'sup1': 185, + 'sup2': 178, + 'sup3': 179, + 'supe': 8839, + 'szlig': 223, + 'tau': 964, + 'there4': 8756, + 'theta': 952, + 'thetasym': 977, + 'thinsp': 8201, + 'thorn': 254, + 'tilde': 732, + 'times': 215, + 'trade': 8482, + 'uArr': 8657, + 'uacute': 250, + 'uarr': 8593, + 'ucirc': 251, + 'ugrave': 249, + 'uml': 168, + 'upsih': 978, + 'upsilon': 965, + 'uuml': 252, + 'weierp': 8472, + 'xi': 958, + 'yacute': 253, + 'yen': 165, + 'yuml': 255, + 'zeta': 950, + 'zwj': 8205, + 'zwnj': 8204 +} diff --git a/scripts/jinja2/debug.py b/scripts/jinja2/debug.py new file mode 100644 index 000000000..53dac4d11 --- /dev/null +++ b/scripts/jinja2/debug.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +""" + jinja2.debug + ~~~~~~~~~~~~ + + Implements the debug interface for Jinja. This module does some pretty + ugly stuff with the Python traceback system in order to achieve tracebacks + with correct line numbers, locals and contents. + + :copyright: Copyright 2008 by Armin Ronacher. + :license: BSD. +""" +import sys +from jinja2.utils import CodeType + + +def translate_exception(exc_info): + """If passed an exc_info it will automatically rewrite the exceptions + all the way down to the correct line numbers and frames. + """ + result_tb = prev_tb = None + initial_tb = tb = exc_info[2].tb_next + + while tb is not None: + template = tb.tb_frame.f_globals.get('__jinja_template__') + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + tb = fake_exc_info(exc_info[:2] + (tb,), template.filename, + lineno, prev_tb)[2] + if result_tb is None: + result_tb = tb + prev_tb = tb + tb = tb.tb_next + + return exc_info[:2] + (result_tb or initial_tb,) + + +def fake_exc_info(exc_info, filename, lineno, tb_back=None): + """Helper for `translate_exception`.""" + exc_type, exc_value, tb = exc_info + + # figure the real context out + real_locals = tb.tb_frame.f_locals.copy() + ctx = real_locals.get('context') + if ctx: + locals = ctx.get_all() + else: + locals = {} + for name, value in real_locals.iteritems(): + if name.startswith('l_'): + locals[name[2:]] = value + + # if there is a local called __jinja_exception__, we get + # rid of it to not break the debug functionality. + locals.pop('__jinja_exception__', None) + + # assamble fake globals we need + globals = { + '__name__': filename, + '__file__': filename, + '__jinja_exception__': exc_info[:2] + } + + # and fake the exception + code = compile('\n' * (lineno - 1) + 'raise __jinja_exception__[0], ' + + '__jinja_exception__[1]', filename, 'exec') + + # if it's possible, change the name of the code. This won't work + # on some python environments such as google appengine + try: + function = tb.tb_frame.f_code.co_name + if function == 'root': + location = 'top-level template code' + elif function.startswith('block_'): + location = 'block "%s"' % function[6:] + else: + location = 'template' + code = CodeType(0, code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) + except: + pass + + # execute the code and catch the new traceback + try: + exec code in globals, locals + except: + exc_info = sys.exc_info() + new_tb = exc_info[2].tb_next + + # now we can patch the exc info accordingly + if tb_set_next is not None: + if tb_back is not None: + tb_set_next(tb_back, new_tb) + if tb is not None: + tb_set_next(new_tb, tb.tb_next) + + # return without this frame + return exc_info[:2] + (new_tb,) + + +def _init_ugly_crap(): + """This function implements a few ugly things so that we can patch the + traceback objects. The function returned allows resetting `tb_next` on + any python traceback object. + """ + import ctypes + from types import TracebackType + + # figure out side of _Py_ssize_t + if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'): + _Py_ssize_t = ctypes.c_int64 + else: + _Py_ssize_t = ctypes.c_int + + # regular python + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + # python with trace + if object.__basicsize__ != ctypes.sizeof(_PyObject): + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('_ob_next', ctypes.POINTER(_PyObject)), + ('_ob_prev', ctypes.POINTER(_PyObject)), + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + class _Traceback(_PyObject): + pass + _Traceback._fields_ = [ + ('tb_next', ctypes.POINTER(_Traceback)), + ('tb_frame', ctypes.POINTER(_PyObject)), + ('tb_lasti', ctypes.c_int), + ('tb_lineno', ctypes.c_int) + ] + + def tb_set_next(tb, next): + """Set the tb_next attribute of a traceback object.""" + if not (isinstance(tb, TracebackType) and + (next is None or isinstance(next, TracebackType))): + raise TypeError('tb_set_next arguments must be traceback objects') + obj = _Traceback.from_address(id(tb)) + if tb.tb_next is not None: + old = _Traceback.from_address(id(tb.tb_next)) + old.ob_refcnt -= 1 + if next is None: + obj.tb_next = ctypes.POINTER(_Traceback)() + else: + next = _Traceback.from_address(id(next)) + next.ob_refcnt += 1 + obj.tb_next = ctypes.pointer(next) + + return tb_set_next + + +# try to get a tb_set_next implementation +try: + from jinja2._speedups import tb_set_next +except ImportError: + try: + tb_set_next = _init_ugly_crap() + except: + tb_set_next = None +del _init_ugly_crap diff --git a/scripts/jinja2/defaults.py b/scripts/jinja2/defaults.py new file mode 100644 index 000000000..3e24e7dec --- /dev/null +++ b/scripts/jinja2/defaults.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" + jinja2.defaults + ~~~~~~~~~~~~~~~ + + Jinja default filters and tags. + + :copyright: 2007-2008 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner + + +# defaults for the parser / lexer +BLOCK_START_STRING = '{%' +BLOCK_END_STRING = '%}' +VARIABLE_START_STRING = '{{' +VARIABLE_END_STRING = '}}' +COMMENT_START_STRING = '{#' +COMMENT_END_STRING = '#}' +LINE_STATEMENT_PREFIX = None +TRIM_BLOCKS = False +NEWLINE_SEQUENCE = '\n' + + +# default filters, tests and namespace +from jinja2.filters import FILTERS as DEFAULT_FILTERS +from jinja2.tests import TESTS as DEFAULT_TESTS +DEFAULT_NAMESPACE = { + 'range': xrange, + 'dict': lambda **kw: kw, + 'lipsum': generate_lorem_ipsum, + 'cycler': Cycler, + 'joiner': Joiner +} + + +# export all constants +__all__ = tuple(x for x in locals() if x.isupper()) diff --git a/scripts/jinja2/environment.py b/scripts/jinja2/environment.py new file mode 100644 index 000000000..4a9c9d102 --- /dev/null +++ b/scripts/jinja2/environment.py @@ -0,0 +1,848 @@ +# -*- coding: utf-8 -*- +""" + jinja2.environment + ~~~~~~~~~~~~~~~~~~ + + Provides a class that holds runtime and parsing time options. + + :copyright: 2008 by Armin Ronacher. + :license: BSD, see LICENSE for more details. +""" +import sys +from jinja2 import nodes +from jinja2.defaults import * +from jinja2.lexer import get_lexer, TokenStream +from jinja2.parser import Parser +from jinja2.optimizer import optimize +from jinja2.compiler import generate +from jinja2.runtime import Undefined, Context +from jinja2.exceptions import TemplateSyntaxError +from jinja2.utils import import_string, LRUCache, Markup, missing, \ + concat, consume + + +# for direct template usage we have up to ten living environments +_spontaneous_environments = LRUCache(10) + + +def get_spontaneous_environment(*args): + """Return a new spontaneous environment. A spontaneous environment is an + unnamed and unaccessible (in theory) environment that is used for + templates generated from a string and not from the file system. + """ + try: + env = _spontaneous_environments.get(args) + except TypeError: + return Environment(*args) + if env is not None: + return env + _spontaneous_environments[args] = env = Environment(*args) + env.shared = True + return env + + +def create_cache(size): + """Return the cache class for the given size.""" + if size == 0: + return None + if size < 0: + return {} + return LRUCache(size) + + +def copy_cache(cache): + """Create an empty copy of the given cache.""" + if cache is None: + return Noe + elif type(cache) is dict: + return {} + return LRUCache(cache.capacity) + + +def load_extensions(environment, extensions): + """Load the extensions from the list and bind it to the environment. + Returns a dict of instanciated environments. + """ + result = {} + for extension in extensions: + if isinstance(extension, basestring): + extension = import_string(extension) + result[extension.identifier] = extension(environment) + return result + + +def _environment_sanity_check(environment): + """Perform a sanity check on the environment.""" + assert issubclass(environment.undefined, Undefined), 'undefined must ' \ + 'be a subclass of undefined because filters depend on it.' + assert environment.block_start_string != \ + environment.variable_start_string != \ + environment.comment_start_string, 'block, variable and comment ' \ + 'start strings must be different' + assert environment.newline_sequence in ('\r', '\r\n', '\n'), \ + 'newline_sequence set to unknown line ending string.' + return environment + + +class Environment(object): + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here the possible initialization parameters: + + `block_start_string` + The string marking the begin of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the begin of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the begin of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation `. + + `optimized` + should the optimizer be enabled? Default is `True`. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that finalizes the variable. Per default no finalizing + is applied. + + `autoescape` + If set to true the XML/HTML autoescaping feature is enabled. + For more details about auto escaping see + :class:`~jinja2.utils.Markup`. + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``50`` which means + that if more than 50 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + `auto_reload` is set to `True` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox + sandboxed = False + + #: True if the environment is just an overlay + overlay = False + + #: the environment this environment is linked to if it is an overlay + linked_to = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + def __init__(self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=50, + auto_reload=True, + bytecode_cache=None): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneus environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.trim_blocks = trim_blocks + self.newline_sequence = newline_sequence + + # runtime information + self.undefined = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.tests = DEFAULT_TESTS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.bytecode_cache = None + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # load extensions + self.extensions = load_extensions(self, extensions) + + _environment_sanity_check(self) + + def extend(self, **attributes): + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions ` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in attributes.iteritems(): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay(self, block_start_string=missing, block_end_string=missing, + variable_start_string=missing, variable_end_string=missing, + comment_start_string=missing, comment_end_string=missing, + line_statement_prefix=missing, trim_blocks=missing, + extensions=missing, optimized=missing, undefined=missing, + finalize=missing, autoescape=missing, loader=missing, + cache_size=missing, auto_reload=missing, + bytecode_cache=missing): + """Create a new overlay environment that shares all the data with the + current environment except of cache and the overriden attributes. + Extensions cannot be removed for a overlayed environment. A overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + """ + args = dict(locals()) + del args['self'], args['cache_size'], args['extensions'] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlay = True + rv.linked_to = self + + for key, value in args.iteritems(): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in self.extensions.iteritems(): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(extensions)) + + return _environment_sanity_check(rv) + + lexer = property(get_lexer, doc="The lexer for this environment.") + + def getitem(self, obj, argument): + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (TypeError, LookupError): + if isinstance(argument, basestring): + try: + attr = str(argument) + except: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a bytestring. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def parse(self, source, name=None, filename=None): + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja2 extensions ` + this gives you a good overview of the node tree generated. + """ + if isinstance(filename, unicode): + filename = filename.encode('utf-8') + try: + return Parser(self, source, name, filename).parse() + except TemplateSyntaxError, e: + e.source = source + raise e + + def lex(self, source, name=None, filename=None): + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development ` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = unicode(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError, e: + e.source = source + raise e + + def preprocess(self, source, name=None, filename=None): + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce(lambda s, e: e.preprocess(s, name, filename), + self.extensions.itervalues(), unicode(source)) + + def _tokenize(self, source, name, filename=None, state=None): + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.extensions.itervalues(): + stream = ext.filter_stream(stream) + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + return stream + + def compile(self, source, name=None, filename=None, raw=False): + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + """ + if isinstance(source, basestring): + source = self.parse(source, name, filename) + if self.optimized: + source = optimize(source, self) + source = generate(source, self, name, filename) + if raw: + return source + if filename is None: + filename = '