@@ -43,10 +43,11 @@ augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>='
 print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] )
 del_stmt: 'del' exprlist
 pass_stmt: 'pass'
-flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
 break_stmt: 'break'
 continue_stmt: 'continue'
 return_stmt: 'return' [testlist]
+yield_stmt: 'yield' testlist
 raise_stmt: 'raise' [test [',' test [',' test]]]
 import_stmt: 'import' dotted_as_name (',' dotted_as_name)* | 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*)
 import_as_name: NAME [NAME NAME]
 
@@ -33,6 +33,7 @@ typedef struct {
 #define CO_VARARGS	0x0004
 #define CO_VARKEYWORDS	0x0008
 #define CO_NESTED       0x0010
+#define CO_GENERATOR    0x0020
 
 extern DL_IMPORT(PyTypeObject) PyCode_Type;
 
 
@@ -21,6 +21,8 @@ typedef struct _frame {
     PyObject *f_globals;	/* global symbol table (PyDictObject) */
     PyObject *f_locals;		/* local symbol table (PyDictObject) */
     PyObject **f_valuestack;	/* points after the last local */
+    PyObject **f_stackbottom;   /* points to the last item on the stack if
+                                  frame has yielded. */
     PyObject *f_trace;		/* Trace function */
     PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
     PyThreadState *f_tstate;
 
@@ -18,47 +18,48 @@
 #define break_stmt 273
 #define continue_stmt 274
 #define return_stmt 275
-#define raise_stmt 276
-#define import_stmt 277
-#define import_as_name 278
-#define dotted_as_name 279
-#define dotted_name 280
-#define global_stmt 281
-#define exec_stmt 282
-#define assert_stmt 283
-#define compound_stmt 284
-#define if_stmt 285
-#define while_stmt 286
-#define for_stmt 287
-#define try_stmt 288
-#define except_clause 289
-#define suite 290
-#define test 291
-#define and_test 292
-#define not_test 293
-#define comparison 294
-#define comp_op 295
-#define expr 296
-#define xor_expr 297
-#define and_expr 298
-#define shift_expr 299
-#define arith_expr 300
-#define term 301
-#define factor 302
-#define power 303
-#define atom 304
-#define listmaker 305
-#define lambdef 306
-#define trailer 307
-#define subscriptlist 308
-#define subscript 309
-#define sliceop 310
-#define exprlist 311
-#define testlist 312
-#define dictmaker 313
-#define classdef 314
-#define arglist 315
-#define argument 316
-#define list_iter 317
-#define list_for 318
-#define list_if 319
+#define yield_stmt 276
+#define raise_stmt 277
+#define import_stmt 278
+#define import_as_name 279
+#define dotted_as_name 280
+#define dotted_name 281
+#define global_stmt 282
+#define exec_stmt 283
+#define assert_stmt 284
+#define compound_stmt 285
+#define if_stmt 286
+#define while_stmt 287
+#define for_stmt 288
+#define try_stmt 289
+#define except_clause 290
+#define suite 291
+#define test 292
+#define and_test 293
+#define not_test 294
+#define comparison 295
+#define comp_op 296
+#define expr 297
+#define xor_expr 298
+#define and_expr 299
+#define shift_expr 300
+#define arith_expr 301
+#define term 302
+#define factor 303
+#define power 304
+#define atom 305
+#define listmaker 306
+#define lambdef 307
+#define trailer 308
+#define subscriptlist 309
+#define subscript 310
+#define sliceop 311
+#define exprlist 312
+#define testlist 313
+#define dictmaker 314
+#define classdef 315
+#define arglist 316
+#define argument 317
+#define list_iter 318
+#define list_for 319
+#define list_if 320
@@ -71,6 +71,7 @@ extern "C" {
 #define RETURN_VALUE	83
 #define IMPORT_STAR	84
 #define EXEC_STMT	85
+#define YIELD_VALUE	86
 
 #define POP_BLOCK	87
 #define END_FINALLY	88
 
@@ -46,6 +46,7 @@ typedef struct _symtable_entry {
 	int ste_nested;          /* true if scope is nested */
 	int ste_child_free;      /* true if a child scope has free variables,
 				    including free refs to globals */
+	int ste_generator;       /* true if namespace is a generator */
 	int ste_opt_lineno;      /* lineno of last exec or import * */
 	struct symtable *ste_table;
 } PySymtableEntryObject;
 
@@ -223,6 +223,7 @@ def jabs_op(name, op):
 def_op('RETURN_VALUE', 83)
 def_op('IMPORT_STAR', 84)
 def_op('EXEC_STMT', 85)
+def_op('YIELD_STMT', 86)
 
 def_op('POP_BLOCK', 87)
 def_op('END_FINALLY', 88)
 
@@ -349,32 +349,28 @@ def readline(self):
             return self.lines[i]
         else: return ''
 
-class EndOfBlock(Exception): pass
-
-class BlockFinder:
-    """Provide a tokeneater() method to detect the end of a code block."""
-    def __init__(self):
-        self.indent = 0
-        self.started = 0
-        self.last = 0
-
-    def tokeneater(self, type, token, (srow, scol), (erow, ecol), line):
-        if not self.started:
-            if type == tokenize.NAME: self.started = 1
+def getblock(lines):
+    """Extract the block of code at the top of the given list of lines."""
+
+    indent = 0
+    started = 0
+    last = 0
+    tokens = tokenize.generate_tokens(ListReader(lines).readline)
+
+    for (type, token, (srow, scol), (erow, ecol), line) in tokens:
+        if not started:
+            if type == tokenize.NAME:
+                started = 1
         elif type == tokenize.NEWLINE:
-            self.last = srow
+            last = srow
         elif type == tokenize.INDENT:
-            self.indent = self.indent + 1
+            indent = indent + 1
         elif type == tokenize.DEDENT:
-            self.indent = self.indent - 1
-            if self.indent == 0: raise EndOfBlock, self.last
-
-def getblock(lines):
-    """Extract the block of code at the top of the given list of lines."""
-    try:
-        tokenize.tokenize(ListReader(lines).readline, BlockFinder().tokeneater)
-    except EndOfBlock, eob:
-        return lines[:eob.args[0]]
+            indent = indent - 1
+            if indent == 0:
+                return lines[:last]
+    else:
+        raise ValueError, "unable to find block"
 
 def getsourcelines(object):
     """Return a list of source lines and starting line number for an object.
 
@@ -77,9 +77,8 @@ def check(file):
     if verbose > 1:
         print "checking", `file`, "..."
 
-    reset_globals()
     try:
-        tokenize.tokenize(f.readline, tokeneater)
+        process_tokens(tokenize.generate_tokens(f.readline))
 
     except tokenize.TokenError, msg:
         errprint("%s: Token Error: %s" % (`file`, str(msg)))
@@ -244,28 +243,19 @@ def format_witnesses(w):
         prefix = prefix + "s"
     return prefix + " " + string.join(firsts, ', ')
 
-# The collection of globals, the reset_globals() function, and the
-# tokeneater() function, depend on which version of tokenize is
-# in use.
+# Need Guido's enhancement
+assert hasattr(tokenize, 'NL'), "tokenize module too old"
 
-if hasattr(tokenize, 'NL'):
-    # take advantage of Guido's patch!
-
-    indents = []
-    check_equal = 0
-
-    def reset_globals():
-        global indents, check_equal
-        check_equal = 0
-        indents = [Whitespace("")]
-
-    def tokeneater(type, token, start, end, line,
+def process_tokens(tokens,
                    INDENT=tokenize.INDENT,
                    DEDENT=tokenize.DEDENT,
                    NEWLINE=tokenize.NEWLINE,
-                   JUNK=(tokenize.COMMENT, tokenize.NL) ):
-        global indents, check_equal
+                   JUNK=(tokenize.COMMENT, tokenize.NL)):
 
+    indents = [Whitespace("")]
+    check_equal = 0
+
+    for (type, token, start, end, line) in tokens:
         if type == NEWLINE:
             # a program statement, or ENDMARKER, will eventually follow,
             # after some (possibly empty) run of tokens of the form
@@ -311,62 +301,6 @@ def tokeneater(type, token, start, end, line,
                 msg = "indent not equal e.g. " + format_witnesses(witness)
                 raise NannyNag(start[0], msg, line)
 
-else:
-    # unpatched version of tokenize
-
-    nesting_level = 0
-    indents = []
-    check_equal = 0
-
-    def reset_globals():
-        global nesting_level, indents, check_equal
-        nesting_level = check_equal = 0
-        indents = [Whitespace("")]
-
-    def tokeneater(type, token, start, end, line,
-                   INDENT=tokenize.INDENT,
-                   DEDENT=tokenize.DEDENT,
-                   NEWLINE=tokenize.NEWLINE,
-                   COMMENT=tokenize.COMMENT,
-                   OP=tokenize.OP):
-        global nesting_level, indents, check_equal
-
-        if type == INDENT:
-            check_equal = 0
-            thisguy = Whitespace(token)
-            if not indents[-1].less(thisguy):
-                witness = indents[-1].not_less_witness(thisguy)
-                msg = "indent not greater e.g. " + format_witnesses(witness)
-                raise NannyNag(start[0], msg, line)
-            indents.append(thisguy)
-
-        elif type == DEDENT:
-            del indents[-1]
-
-        elif type == NEWLINE:
-            if nesting_level == 0:
-                check_equal = 1
-
-        elif type == COMMENT:
-            pass
-
-        elif check_equal:
-            check_equal = 0
-            thisguy = Whitespace(line)
-            if not indents[-1].equal(thisguy):
-                witness = indents[-1].not_equal_witness(thisguy)
-                msg = "indent not equal e.g. " + format_witnesses(witness)
-                raise NannyNag(start[0], msg, line)
-
-        if type == OP and token in ('{', '[', '('):
-            nesting_level = nesting_level + 1
-
-        elif type == OP and token in ('}', ']', ')'):
-            if nesting_level == 0:
-                raise NannyNag(start[0],
-                               "unbalanced bracket '" + token + "'",
-                               line)
-            nesting_level = nesting_level - 1
 
 if __name__ == '__main__':
     main()