""" ================================================================================ Demonstrate the many mutations of generators though the years... (Copyright M. Lutz 2015) Contents: 1) In 2.3+: original model - yield 2) In 2.5+: sends (plus throws, closes) 3) In 3.3+: returns and subgenerators 4) In 3.5+: asynch/await (plus asyncio in 3.4) This is a supplement to the book "Learning Python". Its first few sections' code runs under both Python 2.X and 3.X, but later code requires recent 3.X. Generators and coroutines in Python: -Have seen a volatile and even tortured evolution, that likely qualifies as thrashing -Have advanced and obscure usage modes that are not widely used by most Python programmers -In advanced roles, are more in the realm of application programming than language, as they entail concurrent programming techniques. However, they have recently spawned new syntax that can regrettably make them a required topic for all users and learners of the core language. ================================================================================ """ # # See also: expression equivalent, in 2.4+ # >>> for x in (i ** 2 for i in range(4)): # Automatically yields values ... print(x) # But defs can use full statements ... 0 1 4 9 >>> #=============================================================================== # 1) In 2.3+: original model - yield # # A def with a "yield" statement automatically supports the iteration # protocol: yielded values are returned for next(). This suffices for # simple task switching: see http://learning-python.com/books/async1.py. #=============================================================================== >>> def a(N): # YIELD VALS ONLY ... for i in range(N): # Suspend and resume just after yield ... yield i ** 2 # Result of caller's next() or send() ... >>> G = a(4) # Manual iteration: iteration protocol >>> next(G) # Yielded values recieved at top by caller 0 >>> next(G) 1 >>> next(G) 4 >>> next(G) 9 >>> next(G) Traceback (most recent call last): File "", line 1, in StopIteration >>> for top in a(4): print(top) # Automatic iteration: all contexts ... # Calls next() * N, catches exception 0 1 4 9 >>> # # Example: classic producer/consumer model # See also task switcher link above for a more general model # Threads + thread queues achieve similar effect without manual yields # >>> queue = [] # Shared object and global name >>> def produce(N): ... for i in range(N): ... queue.append(i ** 2) ... yield # Suspend, yield control to consume() ... >>> def consume(N): ... for i in produce(N): # Start or resume produce() on each loop ... res = queue.pop() ... print(res) ... >>> consume(4) 0 1 4 9 >>> #=============================================================================== # 2) In 2.5+: sends (plus throws and closes, not shown) # # The caller can resume a generator and pass it a value via G.send(X): # X becomes the result of the "yield" expression inside the generator. #=============================================================================== >>> def a(N): # YIELD VALS + GET SENT VALS ... for i in range(N): ... X = yield i ** 2 # X is result of caller's send() ... print('gen:', X) # Display X after resumed ... # # Iterate manually # >>> G = a(4) # Create generator >>> G.send(77) # Must start with next(), not send() TypeError: can't send non-None value to a just-started generator >>> top = next(G) # Start generator: to first yield >>> top # Yielded iteration value in caller 0 >>> top = G.send(77) # 77 sent to suspended yield expression gen: 77 # Sent value printed in generator >>> top # Yielded iteration value in caller 1 >>> top = G.send(88) # 88 sent to suspended yield expression gen: 88 # Sent value printed in generator >>> top # Yielded iteration value in caller 4 >>> top = next(G) gen: None # No value (default None) sent by next() >>> top # Final yielded iteration value 9 >>> G.send(99) # Resume a(), a() exits, no new value gen: 99 Traceback (most recent call last): File "", line 1, in StopIteration >>> # # Iterate with loops # >>> for x in a(4): print('top:', x) # next() sends default None to a() ... top: 0 gen: None top: 1 gen: None top: 4 gen: None top: 9 gen: None >>> G = a(4) # Send explicit values to a() >>> v = next(G) >>> print(v) 0 >>> while True: ... v = G.send('Spam' * (v+1)) ... print('top:', v) ... gen: Spam top: 1 gen: SpamSpam top: 4 gen: SpamSpamSpamSpamSpam top: 9 gen: SpamSpamSpamSpamSpamSpamSpamSpamSpamSpam Traceback (most recent call last): File "", line 2, in StopIteration >>> # # Example: using sent value inside a generator # Yielded values can influence the caller's next action # Sent values can influence the generator's next iteration # >>> def a(N): ... X = True ... for i in range(N): ... if X: ... result = i ** 2 ... else: ... result = 1 / float(i) ... X = yield result ... >>> G = a(4) >>> next(G) # Generator runs up to first yield and suspends 0 >>> G.send(False) # Suspended yield resumed, returns sent value 1.0 >>> G.send(True) # Sent value always impacts _next_ iteration 4 >>> G.send(False) 0.3333333333333333 >>> G.send(True) Traceback (most recent call last): File "", line 1, in StopIteration >>> #=============================================================================== # 3) In 3.3+: returns and subgenerators # # A "return X" is allowed in generators: X is attached to the StopIteration # exception and becomes the result of a "yield from" subgenerator expression, # which also propagates sent and yielded values to and from subgenerators. #=============================================================================== C:\Code> py -3.5 # 3.3 and later >>> def a(N): # YIELD VALS + RETURN RESULT ... for i in range(N): ... yield i ** 2 # yield: result of iterations ... return 'SPAM' # return: result of "yield from" expression ... >>> def b(N): ... print('before...') ... res = yield from a(N) # Exports subgenerator iteration results ... print('after...', res) # res is "return" value (or None by default) ... >>> for x in a(4): print('top:', x) # Iterate through subgenerator directly ... top: 0 top: 1 top: 4 top: 9 >>> >>> for x in b(4): print('top:', x) # Iterate through subgenerator indirectly ... before... top: 0 top: 1 top: 4 top: 9 after... SPAM >>> >>> G = a(4) >>> try: ... while True: next(G) ... except StopIteration as E: # Catch return value in exception handler ... print(E.value) ... 0 1 4 9 SPAM >>> >>> b(4) # Generators are nothing till started >>> def a(N): ... for i in range(N): ... yield i ** 2 # Returns None by default (as usual) ... >>> for x in b(4): print('top:', x) # b() delegates to the new a() ... before... top: 0 top: 1 top: 4 top: 9 after... None >>> # # DRUMROLL PLEASE... (all behavior combined) # >>> def a(N): # YIELD VALS + GET SENT VALS + RETURN RESULT ... for i in range(N): ... X = yield i ** 2 # X is result of highest send() ... print('gen:', X) # Display X after resumed ... return 'SPAM' # return: result of "yield from" expression ... >>> def b(N): ... print('before...') ... res = yield from a(N) # Exports subgenerator iteration results ... print('after...', res) # res is "return" value (or None by default) ... # # Iterate manually # >>> G = b(4) # Create generator >>> next(G) # Start: to first yield in b() then a(), 0 passed up to top before... 0 >>> G.send(77) # 77 sent to b() then a(), a() yields 1 to b() then top gen: 77 1 >>> G.send(88) # ditto, but sends 88 and receives 4 gen: 88 4 >>> next(G) # next() sends default None, a() yields 9 to b() then top gen: None 9 >>> G.send(99) # Send 99 to a(), a() exits + returns 'SPAM' to b(), b() exits gen: 99 after... SPAM Traceback (most recent call last): File "", line 1, in StopIteration >>> # # Iterate with loops # >>> for x in b(4): print('top:', x) # next() sends default None to a() via b() ... before... top: 0 gen: None top: 1 gen: None top: 4 gen: None top: 9 gen: None after... SPAM >>> G = b(4) # Send explicit values to a() via b() >>> v = next(G) before... >>> v 0 >>> while True: ... v = G.send('Eggs' * (v+1)) ... print('top:', v) ... gen: Eggs top: 1 gen: EggsEggs top: 4 gen: EggsEggsEggsEggsEggs top: 9 gen: EggsEggsEggsEggsEggsEggsEggsEggsEggsEggs after... SPAM Traceback (most recent call last): File "", line 2, in StopIteration >>> # # Example: using sent values inside a subgenerator # Both "yield" and "yield from" require "()" unless alone on right of "=" # >>> def a(N): ... sum = 0 ... for i in range(N): ... sum += i + (yield i) # Add on i plus sent value ... return sum # (0+1+2+3) + (10+20+30+40) ... >>> def b(N): ... res = yield from a(N) # Get subgenerator return value ... return float(res) # Return float conversion here ... >>> G = b(4) >>> next(G) # Run to first yield in a() 0 >>> G.send(10) # Send 10 to suspended yield 1 >>> G.send(20) 2 >>> G.send(30) 3 >>> G.send(40) # Send 40, a() exits, b() exits Traceback (most recent call last): File "", line 1, in StopIteration: 106.0 >>> >>> ^Z C:\Code> py -3.2 # Return allowed in generators in 3.3+ only >>> def a(): ... for i in range(4): ... yield i ** 2 ... return 'SPAM' ... SyntaxError: 'return' with argument inside generator #=============================================================================== # 4) In 3.5+: asynch/await (plus asyncio in 3.4) # # New "async" and "await" core language syntax attempts to distinguish some # coroutine roles, by adding an entirely new and incompatible model. # # This new model in 3.5: # # -Builds on ideas in the new "asyncio" standard library module in 3.4 # -Which builds on the subgenerators added in 3.3 # -Which builds on the extended generators of 2.5 # -Which assumes the original model of 2.3 (and 2.4) # # For details, see: # # -The PEP for change 0492 # -Python 3.5's documentation # -http://learning-python.com/books/python-changes-2014-plus.html#coroutines # -Or your local priest... #=============================================================================== # This space intentionally left blank