Patch against CVS source for 2001-05-20. Index: Grammar/Grammar =================================================================== RCS file: /cvsroot/python/python/dist/src/Grammar/Grammar,v retrieving revision 1.42 diff -u -u -r1.42 Grammar --- Grammar/Grammar 2001/02/27 18:36:14 1.42 +++ Grammar/Grammar 2001/05/20 17:47:11 @@ -43,10 +43,11 @@ print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] ) del_stmt: 'del' exprlist pass_stmt: 'pass' -flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt +flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt break_stmt: 'break' continue_stmt: 'continue' return_stmt: 'return' [testlist] +yield_stmt: 'yield' testlist raise_stmt: 'raise' [test [',' test [',' test]]] import_stmt: 'import' dotted_as_name (',' dotted_as_name)* | 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*) import_as_name: NAME [NAME NAME] Index: Include/compile.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/compile.h,v retrieving revision 2.29 diff -u -u -r2.29 compile.h --- Include/compile.h 2001/03/22 02:32:48 2.29 +++ Include/compile.h 2001/05/20 17:47:12 @@ -33,6 +33,7 @@ #define CO_VARARGS 0x0004 #define CO_VARKEYWORDS 0x0008 #define CO_NESTED 0x0010 +#define CO_GENERATOR 0x0020 extern DL_IMPORT(PyTypeObject) PyCode_Type; Index: Include/frameobject.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/frameobject.h,v retrieving revision 2.31 diff -u -u -r2.31 frameobject.h --- Include/frameobject.h 2001/03/13 01:58:21 2.31 +++ Include/frameobject.h 2001/05/20 17:47:12 @@ -21,6 +21,8 @@ PyObject *f_globals; /* global symbol table (PyDictObject) */ PyObject *f_locals; /* local symbol table (PyDictObject) */ PyObject **f_valuestack; /* points after the last local */ + PyObject **f_stackbottom; /* points to the last item on the stack if + frame has yielded. */ PyObject *f_trace; /* Trace function */ PyObject *f_exc_type, *f_exc_value, *f_exc_traceback; PyThreadState *f_tstate; Index: Include/opcode.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/opcode.h,v retrieving revision 2.35 diff -u -u -r2.35 opcode.h --- Include/opcode.h 2001/04/20 19:13:01 2.35 +++ Include/opcode.h 2001/05/20 17:47:13 @@ -71,6 +71,7 @@ #define RETURN_VALUE 83 #define IMPORT_STAR 84 #define EXEC_STMT 85 +#define YIELD_VALUE 86 #define POP_BLOCK 87 #define END_FINALLY 88 Index: Include/symtable.h =================================================================== RCS file: /cvsroot/python/python/dist/src/Include/symtable.h,v retrieving revision 2.7 diff -u -u -r2.7 symtable.h --- Include/symtable.h 2001/03/22 03:57:58 2.7 +++ Include/symtable.h 2001/05/20 17:47:14 @@ -46,6 +46,7 @@ int ste_nested; /* true if scope is nested */ int ste_child_free; /* true if a child scope has free variables, including free refs to globals */ + int ste_generator; /* true if namespace is a generator */ int ste_opt_lineno; /* lineno of last exec or import * */ struct symtable *ste_table; } PySymtableEntryObject; Index: Lib/dis.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/dis.py,v retrieving revision 1.34 diff -u -u -r1.34 dis.py --- Lib/dis.py 2001/04/20 19:13:01 1.34 +++ Lib/dis.py 2001/05/20 17:47:15 @@ -223,6 +223,7 @@ def_op('RETURN_VALUE', 83) def_op('IMPORT_STAR', 84) def_op('EXEC_STMT', 85) +def_op('YIELD_STMT', 86) def_op('POP_BLOCK', 87) def_op('END_FINALLY', 88) Index: Lib/inspect.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/inspect.py,v retrieving revision 1.16 diff -u -u -r1.16 inspect.py --- Lib/inspect.py 2001/04/13 14:04:02 1.16 +++ Lib/inspect.py 2001/05/20 17:47:16 @@ -349,32 +349,28 @@ return self.lines[i] else: return '' -class EndOfBlock(Exception): pass +def getblock(lines): + """Extract the block of code at the top of the given list of lines.""" -class BlockFinder: - """Provide a tokeneater() method to detect the end of a code block.""" - def __init__(self): - self.indent = 0 - self.started = 0 - self.last = 0 + indent = 0 + started = 0 + last = 0 + tokens = tokenize.generate_tokens(ListReader(lines).readline) - def tokeneater(self, type, token, (srow, scol), (erow, ecol), line): - if not self.started: - if type == tokenize.NAME: self.started = 1 + for (type, token, (srow, scol), (erow, ecol), line) in tokens: + if not started: + if type == tokenize.NAME: + started = 1 elif type == tokenize.NEWLINE: - self.last = srow + last = srow elif type == tokenize.INDENT: - self.indent = self.indent + 1 + indent = indent + 1 elif type == tokenize.DEDENT: - self.indent = self.indent - 1 - if self.indent == 0: raise EndOfBlock, self.last - -def getblock(lines): - """Extract the block of code at the top of the given list of lines.""" - try: - tokenize.tokenize(ListReader(lines).readline, BlockFinder().tokeneater) - except EndOfBlock, eob: - return lines[:eob.args[0]] + indent = indent - 1 + if self.indent == 0: + return lines[:last] + else: + raise ValueError, "unable to find block" def getsourcelines(object): """Return a list of source lines and starting line number for an object. Index: Lib/tabnanny.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/tabnanny.py,v retrieving revision 1.13 diff -u -u -r1.13 tabnanny.py --- Lib/tabnanny.py 2001/04/08 00:38:42 1.13 +++ Lib/tabnanny.py 2001/05/20 17:47:18 @@ -77,9 +77,8 @@ if verbose > 1: print "checking", `file`, "..." - reset_globals() try: - tokenize.tokenize(f.readline, tokeneater) + process_tokens(tokenize.generate_tokens(f.readline)) except tokenize.TokenError, msg: errprint("%s: Token Error: %s" % (`file`, str(msg))) @@ -244,28 +243,19 @@ prefix = prefix + "s" return prefix + " " + string.join(firsts, ', ') -# The collection of globals, the reset_globals() function, and the -# tokeneater() function, depend on which version of tokenize is -# in use. +# Need Guido's enhancement +assert hasattr(tokenize, 'NL'), "tokenize module too old" -if hasattr(tokenize, 'NL'): - # take advantage of Guido's patch! - - indents = [] - check_equal = 0 - - def reset_globals(): - global indents, check_equal - check_equal = 0 - indents = [Whitespace("")] - - def tokeneater(type, token, start, end, line, +def process_tokens(tokens, INDENT=tokenize.INDENT, DEDENT=tokenize.DEDENT, NEWLINE=tokenize.NEWLINE, - JUNK=(tokenize.COMMENT, tokenize.NL) ): - global indents, check_equal + JUNK=(tokenize.COMMENT, tokenize.NL)): + indents = [Whitespace("")] + check_equal = 0 + + for (type, token, start, end, line) in tokens: if type == NEWLINE: # a program statement, or ENDMARKER, will eventually follow, # after some (possibly empty) run of tokens of the form @@ -311,62 +301,6 @@ msg = "indent not equal e.g. " + format_witnesses(witness) raise NannyNag(start[0], msg, line) -else: - # unpatched version of tokenize - - nesting_level = 0 - indents = [] - check_equal = 0 - - def reset_globals(): - global nesting_level, indents, check_equal - nesting_level = check_equal = 0 - indents = [Whitespace("")] - - def tokeneater(type, token, start, end, line, - INDENT=tokenize.INDENT, - DEDENT=tokenize.DEDENT, - NEWLINE=tokenize.NEWLINE, - COMMENT=tokenize.COMMENT, - OP=tokenize.OP): - global nesting_level, indents, check_equal - - if type == INDENT: - check_equal = 0 - thisguy = Whitespace(token) - if not indents[-1].less(thisguy): - witness = indents[-1].not_less_witness(thisguy) - msg = "indent not greater e.g. " + format_witnesses(witness) - raise NannyNag(start[0], msg, line) - indents.append(thisguy) - - elif type == DEDENT: - del indents[-1] - - elif type == NEWLINE: - if nesting_level == 0: - check_equal = 1 - - elif type == COMMENT: - pass - - elif check_equal: - check_equal = 0 - thisguy = Whitespace(line) - if not indents[-1].equal(thisguy): - witness = indents[-1].not_equal_witness(thisguy) - msg = "indent not equal e.g. " + format_witnesses(witness) - raise NannyNag(start[0], msg, line) - - if type == OP and token in ('{', '[', '('): - nesting_level = nesting_level + 1 - - elif type == OP and token in ('}', ']', ')'): - if nesting_level == 0: - raise NannyNag(start[0], - "unbalanced bracket '" + token + "'", - line) - nesting_level = nesting_level - 1 if __name__ == '__main__': main() Index: Lib/tokenize.py =================================================================== RCS file: /cvsroot/python/python/dist/src/Lib/tokenize.py,v retrieving revision 1.22 diff -u -u -r1.22 tokenize.py --- Lib/tokenize.py 2001/03/23 05:22:49 1.22 +++ Lib/tokenize.py 2001/05/20 17:47:19 @@ -111,7 +111,12 @@ except StopTokenizing: pass +# backwards compatible interface, probably not used def tokenize_loop(readline, tokeneater): + for token_info in generate_tokens(readline): + apply(tokeneater, token_info) + +def generate_tokens(readline): lnum = parenlev = continued = 0 namechars, numchars = string.letters + '_', string.digits contstr, needcont = '', 0 @@ -129,12 +134,12 @@ endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) - tokeneater(STRING, contstr + line[:end], + yield (STRING, contstr + line[:end], strstart, (lnum, end), contline + line) contstr, needcont = '', 0 contline = None elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - tokeneater(ERRORTOKEN, contstr + line, + yield (ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), contline) contstr = '' contline = None @@ -156,16 +161,16 @@ if pos == max: break if line[pos] in '#\r\n': # skip comments or blank lines - tokeneater((NL, COMMENT)[line[pos] == '#'], line[pos:], + yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], (lnum, pos), (lnum, len(line)), line) continue if column > indents[-1]: # count indents or dedents indents.append(column) - tokeneater(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) while column < indents[-1]: indents = indents[:-1] - tokeneater(DEDENT, '', (lnum, pos), (lnum, pos), line) + yield (DEDENT, '', (lnum, pos), (lnum, pos), line) else: # continued statement if not line: @@ -181,12 +186,12 @@ if initial in numchars or \ (initial == '.' and token != '.'): # ordinary number - tokeneater(NUMBER, token, spos, epos, line) + yield (NUMBER, token, spos, epos, line) elif initial in '\r\n': - tokeneater(parenlev > 0 and NL or NEWLINE, + yield (parenlev > 0 and NL or NEWLINE, token, spos, epos, line) elif initial == '#': - tokeneater(COMMENT, token, spos, epos, line) + yield (COMMENT, token, spos, epos, line) elif token in ("'''", '"""', # triple-quoted "r'''", 'r"""', "R'''", 'R"""', "u'''", 'u"""', "U'''", 'U"""', @@ -197,7 +202,7 @@ if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] - tokeneater(STRING, token, spos, (lnum, pos), line) + yield (STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines contstr = line[start:] @@ -216,23 +221,23 @@ contline = line break else: # ordinary string - tokeneater(STRING, token, spos, epos, line) + yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name - tokeneater(NAME, token, spos, epos, line) + yield (NAME, token, spos, epos, line) elif initial == '\\': # continued stmt continued = 1 else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 - tokeneater(OP, token, spos, epos, line) + yield (OP, token, spos, epos, line) else: - tokeneater(ERRORTOKEN, line[pos], + yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos = pos + 1 for indent in indents[1:]: # pop remaining indent levels - tokeneater(DEDENT, '', (lnum, 0), (lnum, 0), '') - tokeneater(ENDMARKER, '', (lnum, 0), (lnum, 0), '') + yield (DEDENT, '', (lnum, 0), (lnum, 0), '') + yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') if __name__ == '__main__': # testing import sys Index: Objects/frameobject.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Objects/frameobject.c,v retrieving revision 2.50 diff -u -u -r2.50 frameobject.c --- Objects/frameobject.c 2001/05/08 04:08:20 2.50 +++ Objects/frameobject.c 2001/05/20 17:47:25 @@ -67,6 +67,7 @@ { int i, slots; PyObject **fastlocals; + PyObject **p; Py_TRASHCAN_SAFE_BEGIN(f) /* Kill all local variables */ @@ -76,6 +77,10 @@ Py_XDECREF(*fastlocals); } + /* Free stack */ + for (p = f->f_valuestack; p < f->f_stackbottom; p++) { + Py_XDECREF(*p); + } Py_XDECREF(f->f_back); Py_XDECREF(f->f_code); Py_XDECREF(f->f_builtins); @@ -221,6 +226,7 @@ f->f_localsplus[extras] = NULL; f->f_valuestack = f->f_localsplus + (f->f_nlocals + ncells + nfrees); + f->f_stackbottom = f->f_valuestack; return f; } Index: Python/ceval.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/ceval.c,v retrieving revision 2.244 diff -u -u -r2.244 ceval.c --- Python/ceval.c 2001/05/18 20:53:14 2.244 +++ Python/ceval.c 2001/05/20 17:47:35 @@ -40,6 +40,7 @@ PyObject **, int, PyObject *); +static PyObject *eval_frame(PyFrameObject *); static char *get_func_name(PyObject *); static char *get_func_desc(PyObject *); static PyObject *call_object(PyObject *, PyObject *, PyObject *); @@ -97,6 +98,124 @@ #endif #endif +staticforward PyTypeObject gentype; + +typedef struct { + PyObject_HEAD + PyFrameObject *frame; + int running; /* true if generator is being executed */ +} genobject; + +static PyObject * +gen_new(PyFrameObject *f) +{ + genobject *gen = PyObject_New(genobject, &gentype); + if (gen == NULL) { + Py_DECREF(f); + return NULL; + } + gen->frame = f; + gen->running = 0; + return (PyObject *)gen; +} + +static void +gen_dealloc(genobject *gen) +{ + Py_DECREF(gen->frame); + PyObject_DEL(gen); +} + +static PyObject * +gen_iternext(genobject *gen) +{ + PyFrameObject *f = gen->frame; + PyObject *result; + + if (gen->running) { + PyErr_SetString(PyExc_ValueError, + "generator already executing"); + return NULL; + } + if (f->f_stackbottom == NULL) { + return NULL; + } + gen->running = 1; + result = eval_frame(f); + gen->running = 0; + return result; +} + +static PyObject * +gen_next(genobject *gen, PyObject *args) +{ + PyObject *result; + + if (!PyArg_ParseTuple(args, ":next")) + return NULL; + + result = gen_iternext(gen); + + if (result == NULL && !PyErr_Occurred()) { + PyErr_SetObject(PyExc_StopIteration, Py_None); + return NULL; + } + + return result; +} + +static PyObject * +gen_getiter(PyObject *gen) +{ + Py_INCREF(gen); + return gen; +} + +static struct PyMethodDef gen_methods[] = { + {"next", (PyCFunction)gen_next, METH_VARARGS, + "next() -- get the next value, or raise StopIteration"}, + {NULL, NULL} /* Sentinel */ +}; + +static PyObject * +gen_getattr(genobject *gen, char *name) +{ + return Py_FindMethod(gen_methods, (PyObject *)gen, name); +} + +statichere PyTypeObject gentype = { + PyObject_HEAD_INIT(&PyType_Type) + 0, /* ob_size */ + "generator", /* tp_name */ + sizeof(genobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)gen_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + (getattrfunc)gen_getattr, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + (getiterfunc)gen_getiter, /* tp_iter */ + (iternextfunc)gen_iternext, /* tp_iternext */ +}; + + #ifdef WITH_THREAD #ifndef DONT_HAVE_ERRNO_H @@ -328,7 +447,8 @@ WHY_RERAISE, /* Exception re-raised by 'finally' */ WHY_RETURN, /* 'return' statement */ WHY_BREAK, /* 'break' statement */ - WHY_CONTINUE /* 'continue' statement */ + WHY_CONTINUE, /* 'continue' statement */ + WHY_YIELD, /* 'yield' operator */ }; static enum why_code do_raise(PyObject *, PyObject *, PyObject *); @@ -349,10 +469,8 @@ /* Interpreter main loop */ -static PyObject * -eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals, - PyObject **args, int argcount, PyObject **kws, int kwcount, - PyObject **defs, int defcount, PyObject *closure) +PyObject * +eval_frame(PyFrameObject *f) { #ifdef DXPAIRS int lastopcode = 0; @@ -369,17 +487,17 @@ register PyObject *u; register PyObject *t; register PyObject *stream = NULL; /* for PRINT opcodes */ - register PyFrameObject *f; /* Current frame */ register PyObject **fastlocals, **freevars; PyObject *retval = NULL; /* Return value */ PyThreadState *tstate = PyThreadState_GET(); + PyCodeObject *co; unsigned char *first_instr; #ifdef LLTRACE int lltrace; #endif #if defined(Py_DEBUG) || defined(LLTRACE) /* Make it easier to find out where we are with a debugger */ - char *filename = PyString_AsString(co->co_filename); + char *filename; #endif /* Code access macros */ @@ -417,263 +535,41 @@ /* Start of code */ + if (f == NULL) + return NULL; + #ifdef USE_STACKCHECK if (tstate->recursion_depth%10 == 0 && PyOS_CheckStack()) { PyErr_SetString(PyExc_MemoryError, "Stack overflow"); return NULL; } #endif - - if (globals == NULL) { - PyErr_SetString(PyExc_SystemError, "eval_code2: NULL globals"); - return NULL; - } -#ifdef LLTRACE - lltrace = PyDict_GetItemString(globals, "__lltrace__") != NULL; -#endif - - f = PyFrame_New(tstate, /*back*/ - co, /*code*/ - globals, locals); - if (f == NULL) - return NULL; - - tstate->frame = f; - fastlocals = f->f_localsplus; - freevars = f->f_localsplus + f->f_nlocals; - - if (co->co_argcount > 0 || - co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) { - int i; - int n = argcount; - PyObject *kwdict = NULL; - if (co->co_flags & CO_VARKEYWORDS) { - kwdict = PyDict_New(); - if (kwdict == NULL) - goto fail; - i = co->co_argcount; - if (co->co_flags & CO_VARARGS) - i++; - SETLOCAL(i, kwdict); - } - if (argcount > co->co_argcount) { - if (!(co->co_flags & CO_VARARGS)) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes %s %d " - "%sargument%s (%d given)", - PyString_AsString(co->co_name), - defcount ? "at most" : "exactly", - co->co_argcount, - kwcount ? "non-keyword " : "", - co->co_argcount == 1 ? "" : "s", - argcount); - goto fail; - } - n = co->co_argcount; - } - for (i = 0; i < n; i++) { - x = args[i]; - Py_INCREF(x); - SETLOCAL(i, x); - } - if (co->co_flags & CO_VARARGS) { - u = PyTuple_New(argcount - n); - if (u == NULL) - goto fail; - SETLOCAL(co->co_argcount, u); - for (i = n; i < argcount; i++) { - x = args[i]; - Py_INCREF(x); - PyTuple_SET_ITEM(u, i-n, x); - } - } - for (i = 0; i < kwcount; i++) { - PyObject *keyword = kws[2*i]; - PyObject *value = kws[2*i + 1]; - int j; - if (keyword == NULL || !PyString_Check(keyword)) { - PyErr_Format(PyExc_TypeError, - "%.200s() keywords must be strings", - PyString_AsString(co->co_name)); - goto fail; - } - /* XXX slow -- speed up using dictionary? */ - for (j = 0; j < co->co_argcount; j++) { - PyObject *nm = PyTuple_GET_ITEM( - co->co_varnames, j); - int cmp = PyObject_RichCompareBool( - keyword, nm, Py_EQ); - if (cmp > 0) - break; - else if (cmp < 0) - goto fail; - } - /* Check errors from Compare */ - if (PyErr_Occurred()) - goto fail; - if (j >= co->co_argcount) { - if (kwdict == NULL) { - PyErr_Format(PyExc_TypeError, - "%.200s() got an unexpected " - "keyword argument '%.400s'", - PyString_AsString(co->co_name), - PyString_AsString(keyword)); - goto fail; - } - PyDict_SetItem(kwdict, keyword, value); - } - else { - if (GETLOCAL(j) != NULL) { - PyErr_Format(PyExc_TypeError, - "%.200s() got multiple " - "values for keyword " - "argument '%.400s'", - PyString_AsString(co->co_name), - PyString_AsString(keyword)); - goto fail; - } - Py_INCREF(value); - SETLOCAL(j, value); - } - } - if (argcount < co->co_argcount) { - int m = co->co_argcount - defcount; - for (i = argcount; i < m; i++) { - if (GETLOCAL(i) == NULL) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes %s %d " - "%sargument%s (%d given)", - PyString_AsString(co->co_name), - ((co->co_flags & CO_VARARGS) || - defcount) ? "at least" - : "exactly", - m, kwcount ? "non-keyword " : "", - m == 1 ? "" : "s", i); - goto fail; - } - } - if (n > m) - i = n - m; - else - i = 0; - for (; i < defcount; i++) { - if (GETLOCAL(m+i) == NULL) { - PyObject *def = defs[i]; - Py_INCREF(def); - SETLOCAL(m+i, def); - } - } - } - } - else { - if (argcount > 0 || kwcount > 0) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%d given)", - PyString_AsString(co->co_name), - argcount + kwcount); - goto fail; - } - } - /* Allocate and initialize storage for cell vars, and copy free - vars into frame. This isn't too efficient right now. */ - if (f->f_ncells) { - int i = 0, j = 0, nargs, found; - char *cellname, *argname; - PyObject *c; - - nargs = co->co_argcount; - if (co->co_flags & CO_VARARGS) - nargs++; - if (co->co_flags & CO_VARKEYWORDS) - nargs++; - - /* Check for cells that shadow args */ - for (i = 0; i < f->f_ncells && j < nargs; ++i) { - cellname = PyString_AS_STRING( - PyTuple_GET_ITEM(co->co_cellvars, i)); - found = 0; - while (j < nargs) { - argname = PyString_AS_STRING( - PyTuple_GET_ITEM(co->co_varnames, j)); - if (strcmp(cellname, argname) == 0) { - c = PyCell_New(GETLOCAL(j)); - if (c == NULL) - goto fail; - GETLOCAL(f->f_nlocals + i) = c; - found = 1; - break; - } - j++; - } - if (found == 0) { - c = PyCell_New(NULL); - if (c == NULL) - goto fail; - SETLOCAL(f->f_nlocals + i, c); - } - } - /* Initialize any that are left */ - while (i < f->f_ncells) { - c = PyCell_New(NULL); - if (c == NULL) - goto fail; - SETLOCAL(f->f_nlocals + i, c); - i++; - } - } - if (f->f_nfreevars) { - int i; - for (i = 0; i < f->f_nfreevars; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - Py_INCREF(o); - freevars[f->f_ncells + i] = o; - } - } - - if (tstate->sys_tracefunc != NULL) { - /* tstate->sys_tracefunc, if defined, is a function that - will be called on *every* entry to a code block. - Its return value, if not None, is a function that - will be called at the start of each executed line - of code. (Actually, the function must return - itself in order to continue tracing.) - The trace functions are called with three arguments: - a pointer to the current frame, a string indicating - why the function is called, and an argument which - depends on the situation. The global trace function - (sys.trace) is also called whenever an exception - is detected. */ - if (call_trace(&tstate->sys_tracefunc, - &f->f_trace, f, "call", - Py_None/*XXX how to compute arguments now?*/)) { - /* Trace function raised an error */ - goto fail; - } - } - - if (tstate->sys_profilefunc != NULL) { - /* Similar for sys_profilefunc, except it needn't return - itself and isn't called for "line" events */ - if (call_trace(&tstate->sys_profilefunc, - (PyObject**)0, f, "call", - Py_None/*XXX*/)) { - goto fail; - } - } - + /* push frame */ if (++tstate->recursion_depth > recursion_limit) { --tstate->recursion_depth; PyErr_SetString(PyExc_RuntimeError, "maximum recursion depth exceeded"); tstate->frame = f->f_back; - Py_DECREF(f); return NULL; } + f->f_back = tstate->frame; + tstate->frame = f; + co = f->f_code; + fastlocals = f->f_localsplus; + freevars = f->f_localsplus + f->f_nlocals; _PyCode_GETCODEPTR(co, &first_instr); - next_instr = first_instr; - stack_pointer = f->f_valuestack; + next_instr = first_instr + f->f_lasti; + stack_pointer = f->f_stackbottom; + f->f_stackbottom = NULL; + +#ifdef LLTRACE + lltrace = PyDict_GetItemString(f->f_globals,"__lltrace__") != NULL; +#endif +#if defined(Py_DEBUG) || defined(LLTRACE) + filename = PyString_AsString(co->co_filename); +#endif why = WHY_NOT; err = 0; @@ -1450,6 +1346,14 @@ why = WHY_RETURN; break; + case YIELD_VALUE: + retval = POP(); + f->f_stackbottom = stack_pointer; + f->f_lasti = INSTR_OFFSET(); + why = WHY_YIELD; + break; + + case EXEC_STMT: w = POP(); v = POP(); @@ -1475,6 +1379,7 @@ if (PyInt_Check(v)) { why = (enum why_code) PyInt_AsLong(v); if (why == WHY_RETURN || + why == WHY_YIELD || why == CONTINUE_LOOP) retval = POP(); } @@ -2217,7 +2122,7 @@ /* Unwind stacks if a (pseudo) exception occurred */ - while (why != WHY_NOT && f->f_iblock > 0) { + while (why != WHY_NOT && why != WHY_YIELD && f->f_iblock > 0) { PyTryBlock *b = PyFrame_BlockPop(f); if (b->b_type == SETUP_LOOP && why == WHY_CONTINUE) { @@ -2287,16 +2192,18 @@ /* Pop remaining stack entries */ + /* while (!EMPTY()) { v = POP(); Py_XDECREF(v); } + */ - if (why != WHY_RETURN) + if (why != WHY_RETURN && why != WHY_YIELD) retval = NULL; if (f->f_trace) { - if (why == WHY_RETURN) { + if (why == WHY_RETURN || why == WHY_YIELD) { if (call_trace(&f->f_trace, &f->f_trace, f, "return", retval)) { Py_XDECREF(retval); @@ -2306,7 +2213,8 @@ } } - if (tstate->sys_profilefunc && why == WHY_RETURN) { + if (tstate->sys_profilefunc && + (why == WHY_RETURN || why == WHY_YIELD)) { if (call_trace(&tstate->sys_profilefunc, (PyObject**)0, f, "return", retval)) { Py_XDECREF(retval); @@ -2317,17 +2225,271 @@ reset_exc_info(tstate); + /* pop frame */ --tstate->recursion_depth; + tstate->frame = f->f_back; - fail: /* Jump here from prelude on failure */ + return retval; +} + +static PyObject * +eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals, + PyObject **args, int argcount, PyObject **kws, int kwcount, + PyObject **defs, int defcount, PyObject *closure) +{ + register PyFrameObject *f; + register PyObject *retval = NULL; + register PyObject **fastlocals, **freevars; + PyThreadState *tstate = PyThreadState_GET(); + PyObject *x, *u; - /* Restore previous frame and release the current one */ + if (globals == NULL) { + PyErr_SetString(PyExc_SystemError, "eval_code2: NULL globals"); + return NULL; + } - tstate->frame = f->f_back; - Py_DECREF(f); + f = PyFrame_New(tstate, /*back*/ + co, /*code*/ + globals, locals); + if (f == NULL) + return NULL; + + fastlocals = f->f_localsplus; + freevars = f->f_localsplus + f->f_nlocals; + + if (co->co_argcount > 0 || + co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) { + int i; + int n = argcount; + PyObject *kwdict = NULL; + if (co->co_flags & CO_VARKEYWORDS) { + kwdict = PyDict_New(); + if (kwdict == NULL) + goto fail; + i = co->co_argcount; + if (co->co_flags & CO_VARARGS) + i++; + SETLOCAL(i, kwdict); + } + if (argcount > co->co_argcount) { + if (!(co->co_flags & CO_VARARGS)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes %s %d " + "%sargument%s (%d given)", + PyString_AsString(co->co_name), + defcount ? "at most" : "exactly", + co->co_argcount, + kwcount ? "non-keyword " : "", + co->co_argcount == 1 ? "" : "s", + argcount); + goto fail; + } + n = co->co_argcount; + } + for (i = 0; i < n; i++) { + x = args[i]; + Py_INCREF(x); + SETLOCAL(i, x); + } + if (co->co_flags & CO_VARARGS) { + u = PyTuple_New(argcount - n); + if (u == NULL) + goto fail; + SETLOCAL(co->co_argcount, u); + for (i = n; i < argcount; i++) { + x = args[i]; + Py_INCREF(x); + PyTuple_SET_ITEM(u, i-n, x); + } + } + for (i = 0; i < kwcount; i++) { + PyObject *keyword = kws[2*i]; + PyObject *value = kws[2*i + 1]; + int j; + if (keyword == NULL || !PyString_Check(keyword)) { + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", + PyString_AsString(co->co_name)); + goto fail; + } + /* XXX slow -- speed up using dictionary? */ + for (j = 0; j < co->co_argcount; j++) { + PyObject *nm = PyTuple_GET_ITEM( + co->co_varnames, j); + int cmp = PyObject_RichCompareBool( + keyword, nm, Py_EQ); + if (cmp > 0) + break; + else if (cmp < 0) + goto fail; + } + /* Check errors from Compare */ + if (PyErr_Occurred()) + goto fail; + if (j >= co->co_argcount) { + if (kwdict == NULL) { + PyErr_Format(PyExc_TypeError, + "%.200s() got an unexpected " + "keyword argument '%.400s'", + PyString_AsString(co->co_name), + PyString_AsString(keyword)); + goto fail; + } + PyDict_SetItem(kwdict, keyword, value); + } + else { + if (GETLOCAL(j) != NULL) { + PyErr_Format(PyExc_TypeError, + "%.200s() got multiple " + "values for keyword " + "argument '%.400s'", + PyString_AsString(co->co_name), + PyString_AsString(keyword)); + goto fail; + } + Py_INCREF(value); + SETLOCAL(j, value); + } + } + if (argcount < co->co_argcount) { + int m = co->co_argcount - defcount; + for (i = argcount; i < m; i++) { + if (GETLOCAL(i) == NULL) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes %s %d " + "%sargument%s (%d given)", + PyString_AsString(co->co_name), + ((co->co_flags & CO_VARARGS) || + defcount) ? "at least" + : "exactly", + m, kwcount ? "non-keyword " : "", + m == 1 ? "" : "s", i); + goto fail; + } + } + if (n > m) + i = n - m; + else + i = 0; + for (; i < defcount; i++) { + if (GETLOCAL(m+i) == NULL) { + PyObject *def = defs[i]; + Py_INCREF(def); + SETLOCAL(m+i, def); + } + } + } + } + else { + if (argcount > 0 || kwcount > 0) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%d given)", + PyString_AsString(co->co_name), + argcount + kwcount); + goto fail; + } + } + /* Allocate and initialize storage for cell vars, and copy free + vars into frame. This isn't too efficient right now. */ + if (f->f_ncells) { + int i = 0, j = 0, nargs, found; + char *cellname, *argname; + PyObject *c; + + nargs = co->co_argcount; + if (co->co_flags & CO_VARARGS) + nargs++; + if (co->co_flags & CO_VARKEYWORDS) + nargs++; + + /* Check for cells that shadow args */ + for (i = 0; i < f->f_ncells && j < nargs; ++i) { + cellname = PyString_AS_STRING( + PyTuple_GET_ITEM(co->co_cellvars, i)); + found = 0; + while (j < nargs) { + argname = PyString_AS_STRING( + PyTuple_GET_ITEM(co->co_varnames, j)); + if (strcmp(cellname, argname) == 0) { + c = PyCell_New(GETLOCAL(j)); + if (c == NULL) + goto fail; + GETLOCAL(f->f_nlocals + i) = c; + found = 1; + break; + } + j++; + } + if (found == 0) { + c = PyCell_New(NULL); + if (c == NULL) + goto fail; + SETLOCAL(f->f_nlocals + i, c); + } + } + /* Initialize any that are left */ + while (i < f->f_ncells) { + c = PyCell_New(NULL); + if (c == NULL) + goto fail; + SETLOCAL(f->f_nlocals + i, c); + i++; + } + } + if (f->f_nfreevars) { + int i; + for (i = 0; i < f->f_nfreevars; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + Py_INCREF(o); + freevars[f->f_ncells + i] = o; + } + } + + if (tstate->sys_tracefunc != NULL) { + /* tstate->sys_tracefunc, if defined, is a function that + will be called on *every* entry to a code block. + Its return value, if not None, is a function that + will be called at the start of each executed line + of code. (Actually, the function must return + itself in order to continue tracing.) + The trace functions are called with three arguments: + a pointer to the current frame, a string indicating + why the function is called, and an argument which + depends on the situation. The global trace function + (sys.trace) is also called whenever an exception + is detected. */ + if (call_trace(&tstate->sys_tracefunc, + &f->f_trace, f, "call", + Py_None/*XXX how to compute arguments now?*/)) { + /* Trace function raised an error */ + goto fail; + } + } + if (tstate->sys_profilefunc != NULL) { + /* Similar for sys_profilefunc, except it needn't return + itself and isn't called for "line" events */ + if (call_trace(&tstate->sys_profilefunc, + (PyObject**)0, f, "call", + Py_None/*XXX*/)) { + goto fail; + } + } + + if (co->co_flags & CO_GENERATOR) { + /* create a new generator that owns the ready to run frame + * and return that as the value */ + return gen_new(f); + } + + retval = eval_frame(f); + + fail: /* Jump here from prelude on failure */ + + Py_DECREF(f); return retval; } + static void set_exc_info(PyThreadState *tstate, Index: Python/compile.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/compile.c,v retrieving revision 2.200 diff -u -u -r2.200 compile.c --- Python/compile.c 2001/05/09 18:53:51 2.200 +++ Python/compile.c 2001/05/20 17:47:47 @@ -2590,17 +2590,41 @@ if (!c->c_infunction) { com_error(c, PyExc_SyntaxError, "'return' outside function"); } - if (NCH(n) < 2) { - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); + if (c->c_flags & CO_GENERATOR) { + if (NCH(n) > 1) { + com_error(c, PyExc_SyntaxError, + "'return' with argument inside generator"); + } + com_addoparg(c, LOAD_CONST, + com_addconst(c, PyExc_StopIteration)); com_push(c, 1); + com_addoparg(c, RAISE_VARARGS, 1); } - else - com_node(c, CHILD(n, 1)); - com_addbyte(c, RETURN_VALUE); + else { + if (NCH(n) < 2) { + com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); + com_push(c, 1); + } + else + com_node(c, CHILD(n, 1)); + com_addbyte(c, RETURN_VALUE); + } com_pop(c, 1); } static void +com_yield_stmt(struct compiling *c, node *n) +{ + REQ(n, yield_stmt); /* 'yield' testlist */ + if (!c->c_infunction) { + com_error(c, PyExc_SyntaxError, "'yield' outside function"); + } + com_node(c, CHILD(n, 1)); + com_addbyte(c, YIELD_VALUE); + com_pop(c, 1); +} + +static void com_raise_stmt(struct compiling *c, node *n) { int i; @@ -3411,6 +3435,9 @@ case return_stmt: com_return_stmt(c, n); break; + case yield_stmt: + com_yield_stmt(c, n); + break; case raise_stmt: com_raise_stmt(c, n); break; @@ -3630,10 +3657,19 @@ c->c_infunction = 1; com_node(c, CHILD(n, 4)); c->c_infunction = 0; - com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); - com_push(c, 1); - com_addbyte(c, RETURN_VALUE); - com_pop(c, 1); + if (c->c_flags & CO_GENERATOR) { + com_addoparg(c, LOAD_CONST, + com_addconst(c, PyExc_StopIteration)); + com_push(c, 1); + com_addoparg(c, RAISE_VARARGS, 1); + com_pop(c, 1); + } + else { + com_addoparg(c, LOAD_CONST, com_addconst(c, Py_None)); + com_push(c, 1); + com_addbyte(c, RETURN_VALUE); + com_pop(c, 1); + } } static void @@ -4298,6 +4334,8 @@ { if (c->c_future && c->c_future->ff_nested_scopes) c->c_flags |= CO_NESTED; + if (ste->ste_generator) + c->c_flags |= CO_GENERATOR; if (ste->ste_type != TYPE_MODULE) c->c_flags |= CO_NEWLOCALS; if (ste->ste_type == TYPE_FUNCTION) { @@ -4856,6 +4894,10 @@ case del_stmt: symtable_assign(st, CHILD(n, 1), 0); break; + case yield_stmt: + st->st_cur->ste_generator = 1; + n = CHILD(n, 1); + goto loop; case expr_stmt: if (NCH(n) == 1) n = CHILD(n, 0); Index: Python/marshal.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/marshal.c,v retrieving revision 1.63 diff -u -u -r1.63 marshal.c --- Python/marshal.c 2001/05/08 15:19:57 1.63 +++ Python/marshal.c 2001/05/20 17:47:50 @@ -17,6 +17,7 @@ #define TYPE_NULL '0' #define TYPE_NONE 'N' +#define TYPE_STOPITER 'S' #define TYPE_ELLIPSIS '.' #define TYPE_INT 'i' #define TYPE_INT64 'I' @@ -120,6 +121,9 @@ else if (v == Py_None) { w_byte(TYPE_NONE, p); } + else if (v == PyExc_StopIteration) { + w_byte(TYPE_STOPITER, p); + } else if (v == Py_Ellipsis) { w_byte(TYPE_ELLIPSIS, p); } @@ -375,6 +379,10 @@ case TYPE_NONE: Py_INCREF(Py_None); return Py_None; + + case TYPE_STOPITER: + Py_INCREF(PyExc_StopIteration); + return PyExc_StopIteration; case TYPE_ELLIPSIS: Py_INCREF(Py_Ellipsis); Index: Python/symtable.c =================================================================== RCS file: /cvsroot/python/python/dist/src/Python/symtable.c,v retrieving revision 2.4 diff -u -u -r2.4 symtable.c --- Python/symtable.c 2001/02/27 19:07:02 2.4 +++ Python/symtable.c 2001/05/20 17:47:50 @@ -69,6 +69,7 @@ else ste->ste_nested = 0; ste->ste_child_free = 0; + ste->ste_generator = 0; if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0) goto fail;