Nodes.py 383 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458
  1. #
  2. # Parse tree nodes
  3. #
  4. from __future__ import absolute_import
  5. import cython
  6. cython.declare(sys=object, os=object, copy=object,
  7. Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object,
  8. py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object,
  9. StructOrUnionScope=object, PyClassScope=object,
  10. CppClassScope=object, UtilityCode=object, EncodedString=object,
  11. error_type=object, _py_int_types=object)
  12. import sys, os, copy
  13. from itertools import chain
  14. from . import Builtin
  15. from .Errors import error, warning, InternalError, CompileError
  16. from . import Naming
  17. from . import PyrexTypes
  18. from . import TypeSlots
  19. from .PyrexTypes import py_object_type, error_type
  20. from .Symtab import (ModuleScope, LocalScope, ClosureScope,
  21. StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope)
  22. from .Code import UtilityCode
  23. from .StringEncoding import EncodedString
  24. from . import Future
  25. from . import Options
  26. from . import DebugFlags
  27. from .Pythran import has_np_pythran, pythran_type, is_pythran_buffer
  28. from ..Utils import add_metaclass
  29. if sys.version_info[0] >= 3:
  30. _py_int_types = int
  31. else:
  32. _py_int_types = (int, long)
  33. def relative_position(pos):
  34. return (pos[0].get_filenametable_entry(), pos[1])
  35. def embed_position(pos, docstring):
  36. if not Options.embed_pos_in_docstring:
  37. return docstring
  38. pos_line = u'File: %s (starting at line %s)' % relative_position(pos)
  39. if docstring is None:
  40. # unicode string
  41. return EncodedString(pos_line)
  42. # make sure we can encode the filename in the docstring encoding
  43. # otherwise make the docstring a unicode string
  44. encoding = docstring.encoding
  45. if encoding is not None:
  46. try:
  47. pos_line.encode(encoding)
  48. except UnicodeEncodeError:
  49. encoding = None
  50. if not docstring:
  51. # reuse the string encoding of the original docstring
  52. doc = EncodedString(pos_line)
  53. else:
  54. doc = EncodedString(pos_line + u'\n' + docstring)
  55. doc.encoding = encoding
  56. return doc
  57. def analyse_type_annotation(annotation, env, assigned_value=None):
  58. base_type = None
  59. is_ambiguous = False
  60. explicit_pytype = explicit_ctype = False
  61. if annotation.is_dict_literal:
  62. warning(annotation.pos,
  63. "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.")
  64. for name, value in annotation.key_value_pairs:
  65. if not name.is_string_literal:
  66. continue
  67. if name.value in ('type', b'type'):
  68. explicit_pytype = True
  69. if not explicit_ctype:
  70. annotation = value
  71. elif name.value in ('ctype', b'ctype'):
  72. explicit_ctype = True
  73. annotation = value
  74. if explicit_pytype and explicit_ctype:
  75. warning(annotation.pos, "Duplicate type declarations found in signature annotation")
  76. arg_type = annotation.analyse_as_type(env)
  77. if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'):
  78. # Map builtin numeric Python types to C types in safe cases.
  79. if assigned_value is not None and arg_type is not None and not arg_type.is_pyobject:
  80. assigned_type = assigned_value.infer_type(env)
  81. if assigned_type and assigned_type.is_pyobject:
  82. # C type seems unsafe, e.g. due to 'None' default value => ignore annotation type
  83. is_ambiguous = True
  84. arg_type = None
  85. # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations
  86. if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type):
  87. arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type
  88. elif arg_type is not None and annotation.is_string_literal:
  89. warning(annotation.pos,
  90. "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.")
  91. if arg_type is not None:
  92. if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject:
  93. warning(annotation.pos,
  94. "Python type declaration in signature annotation does not refer to a Python type")
  95. base_type = CAnalysedBaseTypeNode(
  96. annotation.pos, type=arg_type, is_arg=True)
  97. elif is_ambiguous:
  98. warning(annotation.pos, "Ambiguous types in annotation, ignoring")
  99. else:
  100. warning(annotation.pos, "Unknown type declaration in annotation, ignoring")
  101. return base_type, arg_type
  102. def write_func_call(func, codewriter_class):
  103. def f(*args, **kwds):
  104. if len(args) > 1 and isinstance(args[1], codewriter_class):
  105. # here we annotate the code with this function call
  106. # but only if new code is generated
  107. node, code = args[:2]
  108. marker = ' /* %s -> %s.%s %s */' % (
  109. ' ' * code.call_level,
  110. node.__class__.__name__,
  111. func.__name__,
  112. node.pos[1:])
  113. pristine = code.buffer.stream.tell()
  114. code.putln(marker)
  115. start = code.buffer.stream.tell()
  116. code.call_level += 4
  117. res = func(*args, **kwds)
  118. code.call_level -= 4
  119. if start == code.buffer.stream.tell():
  120. # no code written => undo writing marker
  121. code.buffer.stream.truncate(pristine)
  122. else:
  123. marker = marker.replace('->', '<-', 1)
  124. code.putln(marker)
  125. return res
  126. else:
  127. return func(*args, **kwds)
  128. return f
  129. class VerboseCodeWriter(type):
  130. # Set this as a metaclass to trace function calls in code.
  131. # This slows down code generation and makes much larger files.
  132. def __new__(cls, name, bases, attrs):
  133. from types import FunctionType
  134. from .Code import CCodeWriter
  135. attrs = dict(attrs)
  136. for mname, m in attrs.items():
  137. if isinstance(m, FunctionType):
  138. attrs[mname] = write_func_call(m, CCodeWriter)
  139. return super(VerboseCodeWriter, cls).__new__(cls, name, bases, attrs)
  140. class CheckAnalysers(type):
  141. """Metaclass to check that type analysis functions return a node.
  142. """
  143. methods = set(['analyse_types',
  144. 'analyse_expressions',
  145. 'analyse_target_types'])
  146. def __new__(cls, name, bases, attrs):
  147. from types import FunctionType
  148. def check(name, func):
  149. def call(*args, **kwargs):
  150. retval = func(*args, **kwargs)
  151. if retval is None:
  152. print('%s %s %s' % (name, args, kwargs))
  153. return retval
  154. return call
  155. attrs = dict(attrs)
  156. for mname, m in attrs.items():
  157. if isinstance(m, FunctionType) and mname in cls.methods:
  158. attrs[mname] = check(mname, m)
  159. return super(CheckAnalysers, cls).__new__(cls, name, bases, attrs)
  160. def _with_metaclass(cls):
  161. if DebugFlags.debug_trace_code_generation:
  162. return add_metaclass(VerboseCodeWriter)(cls)
  163. #return add_metaclass(CheckAnalysers)(cls)
  164. return cls
  165. @_with_metaclass
  166. class Node(object):
  167. # pos (string, int, int) Source file position
  168. # is_name boolean Is a NameNode
  169. # is_literal boolean Is a ConstNode
  170. is_name = 0
  171. is_none = 0
  172. is_nonecheck = 0
  173. is_literal = 0
  174. is_terminator = 0
  175. is_wrapper = False # is a DefNode wrapper for a C function
  176. temps = None
  177. # All descendants should set child_attrs to a list of the attributes
  178. # containing nodes considered "children" in the tree. Each such attribute
  179. # can either contain a single node or a list of nodes. See Visitor.py.
  180. child_attrs = None
  181. # Subset of attributes that are evaluated in the outer scope (e.g. function default arguments).
  182. outer_attrs = None
  183. cf_state = None
  184. # This may be an additional (or 'actual') type that will be checked when
  185. # this node is coerced to another type. This could be useful to set when
  186. # the actual type to which it can coerce is known, but you want to leave
  187. # the type a py_object_type
  188. coercion_type = None
  189. def __init__(self, pos, **kw):
  190. self.pos = pos
  191. self.__dict__.update(kw)
  192. gil_message = "Operation"
  193. nogil_check = None
  194. in_nogil_context = False # For use only during code generation.
  195. def gil_error(self, env=None):
  196. error(self.pos, "%s not allowed without gil" % self.gil_message)
  197. cpp_message = "Operation"
  198. def cpp_check(self, env):
  199. if not env.is_cpp():
  200. self.cpp_error()
  201. def cpp_error(self):
  202. error(self.pos, "%s only allowed in c++" % self.cpp_message)
  203. def clone_node(self):
  204. """Clone the node. This is defined as a shallow copy, except for member lists
  205. amongst the child attributes (from get_child_accessors) which are also
  206. copied. Lists containing child nodes are thus seen as a way for the node
  207. to hold multiple children directly; the list is not treated as a separate
  208. level in the tree."""
  209. result = copy.copy(self)
  210. for attrname in result.child_attrs:
  211. value = getattr(result, attrname)
  212. if isinstance(value, list):
  213. setattr(result, attrname, [x for x in value])
  214. return result
  215. #
  216. # There are 3 phases of parse tree processing, applied in order to
  217. # all the statements in a given scope-block:
  218. #
  219. # (0) analyse_declarations
  220. # Make symbol table entries for all declarations at the current
  221. # level, both explicit (def, cdef, etc.) and implicit (assignment
  222. # to an otherwise undeclared name).
  223. #
  224. # (1) analyse_expressions
  225. # Determine the result types of expressions and fill in the
  226. # 'type' attribute of each ExprNode. Insert coercion nodes into the
  227. # tree where needed to convert to and from Python objects.
  228. # Allocate temporary locals for intermediate results. Fill
  229. # in the 'result_code' attribute of each ExprNode with a C code
  230. # fragment.
  231. #
  232. # (2) generate_code
  233. # Emit C code for all declarations, statements and expressions.
  234. # Recursively applies the 3 processing phases to the bodies of
  235. # functions.
  236. #
  237. def analyse_declarations(self, env):
  238. pass
  239. def analyse_expressions(self, env):
  240. raise InternalError("analyse_expressions not implemented for %s" % \
  241. self.__class__.__name__)
  242. def generate_code(self, code):
  243. raise InternalError("generate_code not implemented for %s" % \
  244. self.__class__.__name__)
  245. def annotate(self, code):
  246. # mro does the wrong thing
  247. if isinstance(self, BlockNode):
  248. self.body.annotate(code)
  249. def end_pos(self):
  250. try:
  251. return self._end_pos
  252. except AttributeError:
  253. pos = self.pos
  254. if not self.child_attrs:
  255. self._end_pos = pos
  256. return pos
  257. for attr in self.child_attrs:
  258. child = getattr(self, attr)
  259. # Sometimes lists, sometimes nodes
  260. if child is None:
  261. pass
  262. elif isinstance(child, list):
  263. for c in child:
  264. pos = max(pos, c.end_pos())
  265. else:
  266. pos = max(pos, child.end_pos())
  267. self._end_pos = pos
  268. return pos
  269. def dump(self, level=0, filter_out=("pos",), cutoff=100, encountered=None):
  270. """Debug helper method that returns a recursive string representation of this node.
  271. """
  272. if cutoff == 0:
  273. return "<...nesting level cutoff...>"
  274. if encountered is None:
  275. encountered = set()
  276. if id(self) in encountered:
  277. return "<%s (0x%x) -- already output>" % (self.__class__.__name__, id(self))
  278. encountered.add(id(self))
  279. def dump_child(x, level):
  280. if isinstance(x, Node):
  281. return x.dump(level, filter_out, cutoff-1, encountered)
  282. elif isinstance(x, list):
  283. return "[%s]" % ", ".join([dump_child(item, level) for item in x])
  284. else:
  285. return repr(x)
  286. attrs = [(key, value) for key, value in self.__dict__.items() if key not in filter_out]
  287. if len(attrs) == 0:
  288. return "<%s (0x%x)>" % (self.__class__.__name__, id(self))
  289. else:
  290. indent = " " * level
  291. res = "<%s (0x%x)\n" % (self.__class__.__name__, id(self))
  292. for key, value in attrs:
  293. res += "%s %s: %s\n" % (indent, key, dump_child(value, level + 1))
  294. res += "%s>" % indent
  295. return res
  296. def dump_pos(self, mark_column=False, marker='(#)'):
  297. """Debug helper method that returns the source code context of this node as a string.
  298. """
  299. if not self.pos:
  300. return u''
  301. source_desc, line, col = self.pos
  302. contents = source_desc.get_lines(encoding='ASCII', error_handling='ignore')
  303. # line numbers start at 1
  304. lines = contents[max(0, line-3):line]
  305. current = lines[-1]
  306. if mark_column:
  307. current = current[:col] + marker + current[col:]
  308. lines[-1] = current.rstrip() + u' # <<<<<<<<<<<<<<\n'
  309. lines += contents[line:line+2]
  310. return u'"%s":%d:%d\n%s\n' % (
  311. source_desc.get_escaped_description(), line, col, u''.join(lines))
  312. class CompilerDirectivesNode(Node):
  313. """
  314. Sets compiler directives for the children nodes
  315. """
  316. # directives {string:value} A dictionary holding the right value for
  317. # *all* possible directives.
  318. # body Node
  319. child_attrs = ["body"]
  320. def analyse_declarations(self, env):
  321. old = env.directives
  322. env.directives = self.directives
  323. self.body.analyse_declarations(env)
  324. env.directives = old
  325. def analyse_expressions(self, env):
  326. old = env.directives
  327. env.directives = self.directives
  328. self.body = self.body.analyse_expressions(env)
  329. env.directives = old
  330. return self
  331. def generate_function_definitions(self, env, code):
  332. env_old = env.directives
  333. code_old = code.globalstate.directives
  334. code.globalstate.directives = self.directives
  335. self.body.generate_function_definitions(env, code)
  336. env.directives = env_old
  337. code.globalstate.directives = code_old
  338. def generate_execution_code(self, code):
  339. old = code.globalstate.directives
  340. code.globalstate.directives = self.directives
  341. self.body.generate_execution_code(code)
  342. code.globalstate.directives = old
  343. def annotate(self, code):
  344. old = code.globalstate.directives
  345. code.globalstate.directives = self.directives
  346. self.body.annotate(code)
  347. code.globalstate.directives = old
  348. class BlockNode(object):
  349. # Mixin class for nodes representing a declaration block.
  350. def generate_cached_builtins_decls(self, env, code):
  351. entries = env.global_scope().undeclared_cached_builtins
  352. for entry in entries:
  353. code.globalstate.add_cached_builtin_decl(entry)
  354. del entries[:]
  355. def generate_lambda_definitions(self, env, code):
  356. for node in env.lambda_defs:
  357. node.generate_function_definitions(env, code)
  358. class StatListNode(Node):
  359. # stats a list of StatNode
  360. child_attrs = ["stats"]
  361. @staticmethod
  362. def create_analysed(pos, env, *args, **kw):
  363. node = StatListNode(pos, *args, **kw)
  364. return node # No node-specific analysis needed
  365. def analyse_declarations(self, env):
  366. #print "StatListNode.analyse_declarations" ###
  367. for stat in self.stats:
  368. stat.analyse_declarations(env)
  369. def analyse_expressions(self, env):
  370. #print "StatListNode.analyse_expressions" ###
  371. self.stats = [stat.analyse_expressions(env)
  372. for stat in self.stats]
  373. return self
  374. def generate_function_definitions(self, env, code):
  375. #print "StatListNode.generate_function_definitions" ###
  376. for stat in self.stats:
  377. stat.generate_function_definitions(env, code)
  378. def generate_execution_code(self, code):
  379. #print "StatListNode.generate_execution_code" ###
  380. for stat in self.stats:
  381. code.mark_pos(stat.pos)
  382. stat.generate_execution_code(code)
  383. def annotate(self, code):
  384. for stat in self.stats:
  385. stat.annotate(code)
  386. class StatNode(Node):
  387. #
  388. # Code generation for statements is split into the following subphases:
  389. #
  390. # (1) generate_function_definitions
  391. # Emit C code for the definitions of any structs,
  392. # unions, enums and functions defined in the current
  393. # scope-block.
  394. #
  395. # (2) generate_execution_code
  396. # Emit C code for executable statements.
  397. #
  398. def generate_function_definitions(self, env, code):
  399. pass
  400. def generate_execution_code(self, code):
  401. raise InternalError("generate_execution_code not implemented for %s" % \
  402. self.__class__.__name__)
  403. class CDefExternNode(StatNode):
  404. # include_file string or None
  405. # verbatim_include string or None
  406. # body StatListNode
  407. child_attrs = ["body"]
  408. def analyse_declarations(self, env):
  409. old_cinclude_flag = env.in_cinclude
  410. env.in_cinclude = 1
  411. self.body.analyse_declarations(env)
  412. env.in_cinclude = old_cinclude_flag
  413. if self.include_file or self.verbatim_include:
  414. # Determine whether include should be late
  415. stats = self.body.stats
  416. if not env.directives['preliminary_late_includes_cy28']:
  417. late = False
  418. elif not stats:
  419. # Special case: empty 'cdef extern' blocks are early
  420. late = False
  421. else:
  422. late = all(isinstance(node, CVarDefNode) for node in stats)
  423. env.add_include_file(self.include_file, self.verbatim_include, late)
  424. def analyse_expressions(self, env):
  425. return self
  426. def generate_execution_code(self, code):
  427. pass
  428. def annotate(self, code):
  429. self.body.annotate(code)
  430. class CDeclaratorNode(Node):
  431. # Part of a C declaration.
  432. #
  433. # Processing during analyse_declarations phase:
  434. #
  435. # analyse
  436. # Returns (name, type) pair where name is the
  437. # CNameDeclaratorNode of the name being declared
  438. # and type is the type it is being declared as.
  439. #
  440. # calling_convention string Calling convention of CFuncDeclaratorNode
  441. # for which this is a base
  442. child_attrs = []
  443. calling_convention = ""
  444. def analyse_templates(self):
  445. # Only C++ functions have templates.
  446. return None
  447. class CNameDeclaratorNode(CDeclaratorNode):
  448. # name string The Cython name being declared
  449. # cname string or None C name, if specified
  450. # default ExprNode or None the value assigned on declaration
  451. child_attrs = ['default']
  452. default = None
  453. def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
  454. if nonempty and self.name == '':
  455. # May have mistaken the name for the type.
  456. if base_type.is_ptr or base_type.is_array or base_type.is_buffer:
  457. error(self.pos, "Missing argument name")
  458. elif base_type.is_void:
  459. error(self.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
  460. else:
  461. self.name = base_type.declaration_code("", for_display=1, pyrex=1)
  462. base_type = py_object_type
  463. if base_type.is_fused and env.fused_to_specific:
  464. base_type = base_type.specialize(env.fused_to_specific)
  465. self.type = base_type
  466. return self, base_type
  467. class CPtrDeclaratorNode(CDeclaratorNode):
  468. # base CDeclaratorNode
  469. child_attrs = ["base"]
  470. def analyse_templates(self):
  471. return self.base.analyse_templates()
  472. def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
  473. if base_type.is_pyobject:
  474. error(self.pos, "Pointer base type cannot be a Python object")
  475. ptr_type = PyrexTypes.c_ptr_type(base_type)
  476. return self.base.analyse(ptr_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
  477. class CReferenceDeclaratorNode(CDeclaratorNode):
  478. # base CDeclaratorNode
  479. child_attrs = ["base"]
  480. def analyse_templates(self):
  481. return self.base.analyse_templates()
  482. def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
  483. if base_type.is_pyobject:
  484. error(self.pos, "Reference base type cannot be a Python object")
  485. ref_type = PyrexTypes.c_ref_type(base_type)
  486. return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
  487. class CArrayDeclaratorNode(CDeclaratorNode):
  488. # base CDeclaratorNode
  489. # dimension ExprNode
  490. child_attrs = ["base", "dimension"]
  491. def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
  492. if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction:
  493. from .ExprNodes import TupleNode
  494. if isinstance(self.dimension, TupleNode):
  495. args = self.dimension.args
  496. else:
  497. args = self.dimension,
  498. values = [v.analyse_as_type(env) for v in args]
  499. if None in values:
  500. ix = values.index(None)
  501. error(args[ix].pos, "Template parameter not a type")
  502. base_type = error_type
  503. else:
  504. base_type = base_type.specialize_here(self.pos, values)
  505. return self.base.analyse(base_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
  506. if self.dimension:
  507. self.dimension = self.dimension.analyse_const_expression(env)
  508. if not self.dimension.type.is_int:
  509. error(self.dimension.pos, "Array dimension not integer")
  510. size = self.dimension.get_constant_c_result_code()
  511. if size is not None:
  512. try:
  513. size = int(size)
  514. except ValueError:
  515. # runtime constant?
  516. pass
  517. else:
  518. size = None
  519. if not base_type.is_complete():
  520. error(self.pos, "Array element type '%s' is incomplete" % base_type)
  521. if base_type.is_pyobject:
  522. error(self.pos, "Array element cannot be a Python object")
  523. if base_type.is_cfunction:
  524. error(self.pos, "Array element cannot be a function")
  525. array_type = PyrexTypes.c_array_type(base_type, size)
  526. return self.base.analyse(array_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
  527. class CFuncDeclaratorNode(CDeclaratorNode):
  528. # base CDeclaratorNode
  529. # args [CArgDeclNode]
  530. # templates [TemplatePlaceholderType]
  531. # has_varargs boolean
  532. # exception_value ConstNode
  533. # exception_check boolean True if PyErr_Occurred check needed
  534. # nogil boolean Can be called without gil
  535. # with_gil boolean Acquire gil around function body
  536. # is_const_method boolean Whether this is a const method
  537. child_attrs = ["base", "args", "exception_value"]
  538. overridable = 0
  539. optional_arg_count = 0
  540. is_const_method = 0
  541. templates = None
  542. def analyse_templates(self):
  543. if isinstance(self.base, CArrayDeclaratorNode):
  544. from .ExprNodes import TupleNode, NameNode
  545. template_node = self.base.dimension
  546. if isinstance(template_node, TupleNode):
  547. template_nodes = template_node.args
  548. elif isinstance(template_node, NameNode):
  549. template_nodes = [template_node]
  550. else:
  551. error(template_node.pos, "Template arguments must be a list of names")
  552. return None
  553. self.templates = []
  554. for template in template_nodes:
  555. if isinstance(template, NameNode):
  556. self.templates.append(PyrexTypes.TemplatePlaceholderType(template.name))
  557. else:
  558. error(template.pos, "Template arguments must be a list of names")
  559. self.base = self.base.base
  560. return self.templates
  561. else:
  562. return None
  563. def analyse(self, return_type, env, nonempty=0, directive_locals=None, visibility=None, in_pxd=False):
  564. if directive_locals is None:
  565. directive_locals = {}
  566. if nonempty:
  567. nonempty -= 1
  568. func_type_args = []
  569. for i, arg_node in enumerate(self.args):
  570. name_declarator, type = arg_node.analyse(
  571. env, nonempty=nonempty,
  572. is_self_arg=(i == 0 and env.is_c_class_scope and 'staticmethod' not in env.directives))
  573. name = name_declarator.name
  574. if name in directive_locals:
  575. type_node = directive_locals[name]
  576. other_type = type_node.analyse_as_type(env)
  577. if other_type is None:
  578. error(type_node.pos, "Not a type")
  579. elif (type is not PyrexTypes.py_object_type
  580. and not type.same_as(other_type)):
  581. error(self.base.pos, "Signature does not agree with previous declaration")
  582. error(type_node.pos, "Previous declaration here")
  583. else:
  584. type = other_type
  585. if name_declarator.cname:
  586. error(self.pos, "Function argument cannot have C name specification")
  587. if i == 0 and env.is_c_class_scope and type.is_unspecified:
  588. # fix the type of self
  589. type = env.parent_type
  590. # Turn *[] argument into **
  591. if type.is_array:
  592. type = PyrexTypes.c_ptr_type(type.base_type)
  593. # Catch attempted C-style func(void) decl
  594. if type.is_void:
  595. error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
  596. func_type_args.append(
  597. PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
  598. if arg_node.default:
  599. self.optional_arg_count += 1
  600. elif self.optional_arg_count:
  601. error(self.pos, "Non-default argument follows default argument")
  602. exc_val = None
  603. exc_check = 0
  604. if self.exception_check == '+':
  605. env.add_include_file('ios') # for std::ios_base::failure
  606. env.add_include_file('new') # for std::bad_alloc
  607. env.add_include_file('stdexcept')
  608. env.add_include_file('typeinfo') # for std::bad_cast
  609. if (return_type.is_pyobject
  610. and (self.exception_value or self.exception_check)
  611. and self.exception_check != '+'):
  612. error(self.pos, "Exception clause not allowed for function returning Python object")
  613. else:
  614. if self.exception_value is None and self.exception_check and self.exception_check != '+':
  615. # Use an explicit exception return value to speed up exception checks.
  616. # Even if it is not declared, we can use the default exception value of the return type,
  617. # unless the function is some kind of external function that we do not control.
  618. if return_type.exception_value is not None and (visibility != 'extern' and not in_pxd):
  619. # Extension types are more difficult because the signature must match the base type signature.
  620. if not env.is_c_class_scope:
  621. from .ExprNodes import ConstNode
  622. self.exception_value = ConstNode(
  623. self.pos, value=return_type.exception_value, type=return_type)
  624. if self.exception_value:
  625. self.exception_value = self.exception_value.analyse_const_expression(env)
  626. if self.exception_check == '+':
  627. exc_val_type = self.exception_value.type
  628. if (not exc_val_type.is_error
  629. and not exc_val_type.is_pyobject
  630. and not (exc_val_type.is_cfunction
  631. and not exc_val_type.return_type.is_pyobject
  632. and not exc_val_type.args)
  633. and not (exc_val_type == PyrexTypes.c_char_type
  634. and self.exception_value.value == '*')):
  635. error(self.exception_value.pos,
  636. "Exception value must be a Python exception or cdef function with no arguments or *.")
  637. exc_val = self.exception_value
  638. else:
  639. self.exception_value = self.exception_value.coerce_to(
  640. return_type, env).analyse_const_expression(env)
  641. exc_val = self.exception_value.get_constant_c_result_code()
  642. if exc_val is None:
  643. raise InternalError(
  644. "get_constant_c_result_code not implemented for %s" %
  645. self.exception_value.__class__.__name__)
  646. if not return_type.assignable_from(self.exception_value.type):
  647. error(self.exception_value.pos,
  648. "Exception value incompatible with function return type")
  649. exc_check = self.exception_check
  650. if return_type.is_cfunction:
  651. error(self.pos, "Function cannot return a function")
  652. func_type = PyrexTypes.CFuncType(
  653. return_type, func_type_args, self.has_varargs,
  654. optional_arg_count=self.optional_arg_count,
  655. exception_value=exc_val, exception_check=exc_check,
  656. calling_convention=self.base.calling_convention,
  657. nogil=self.nogil, with_gil=self.with_gil, is_overridable=self.overridable,
  658. is_const_method=self.is_const_method,
  659. templates=self.templates)
  660. if self.optional_arg_count:
  661. if func_type.is_fused:
  662. # This is a bit of a hack... When we need to create specialized CFuncTypes
  663. # on the fly because the cdef is defined in a pxd, we need to declare the specialized optional arg
  664. # struct
  665. def declare_opt_arg_struct(func_type, fused_cname):
  666. self.declare_optional_arg_struct(func_type, env, fused_cname)
  667. func_type.declare_opt_arg_struct = declare_opt_arg_struct
  668. else:
  669. self.declare_optional_arg_struct(func_type, env)
  670. callspec = env.directives['callspec']
  671. if callspec:
  672. current = func_type.calling_convention
  673. if current and current != callspec:
  674. error(self.pos, "cannot have both '%s' and '%s' "
  675. "calling conventions" % (current, callspec))
  676. func_type.calling_convention = callspec
  677. return self.base.analyse(func_type, env, visibility=visibility, in_pxd=in_pxd)
  678. def declare_optional_arg_struct(self, func_type, env, fused_cname=None):
  679. """
  680. Declares the optional argument struct (the struct used to hold the
  681. values for optional arguments). For fused cdef functions, this is
  682. deferred as analyse_declarations is called only once (on the fused
  683. cdef function).
  684. """
  685. scope = StructOrUnionScope()
  686. arg_count_member = '%sn' % Naming.pyrex_prefix
  687. scope.declare_var(arg_count_member, PyrexTypes.c_int_type, self.pos)
  688. for arg in func_type.args[len(func_type.args) - self.optional_arg_count:]:
  689. scope.declare_var(arg.name, arg.type, arg.pos, allow_pyobject=True, allow_memoryview=True)
  690. struct_cname = env.mangle(Naming.opt_arg_prefix, self.base.name)
  691. if fused_cname is not None:
  692. struct_cname = PyrexTypes.get_fused_cname(fused_cname, struct_cname)
  693. op_args_struct = env.global_scope().declare_struct_or_union(
  694. name=struct_cname,
  695. kind='struct',
  696. scope=scope,
  697. typedef_flag=0,
  698. pos=self.pos,
  699. cname=struct_cname)
  700. op_args_struct.defined_in_pxd = 1
  701. op_args_struct.used = 1
  702. func_type.op_arg_struct = PyrexTypes.c_ptr_type(op_args_struct.type)
  703. class CConstDeclaratorNode(CDeclaratorNode):
  704. # base CDeclaratorNode
  705. child_attrs = ["base"]
  706. def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
  707. if base_type.is_pyobject:
  708. error(self.pos,
  709. "Const base type cannot be a Python object")
  710. const = PyrexTypes.c_const_type(base_type)
  711. return self.base.analyse(const, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
  712. class CArgDeclNode(Node):
  713. # Item in a function declaration argument list.
  714. #
  715. # base_type CBaseTypeNode
  716. # declarator CDeclaratorNode
  717. # not_none boolean Tagged with 'not None'
  718. # or_none boolean Tagged with 'or None'
  719. # accept_none boolean Resolved boolean for not_none/or_none
  720. # default ExprNode or None
  721. # default_value PyObjectConst constant for default value
  722. # annotation ExprNode or None Py3 function arg annotation
  723. # is_self_arg boolean Is the "self" arg of an extension type method
  724. # is_type_arg boolean Is the "class" arg of an extension type classmethod
  725. # is_kw_only boolean Is a keyword-only argument
  726. # is_dynamic boolean Non-literal arg stored inside CyFunction
  727. child_attrs = ["base_type", "declarator", "default", "annotation"]
  728. outer_attrs = ["default", "annotation"]
  729. is_self_arg = 0
  730. is_type_arg = 0
  731. is_generic = 1
  732. kw_only = 0
  733. not_none = 0
  734. or_none = 0
  735. type = None
  736. name_declarator = None
  737. default_value = None
  738. annotation = None
  739. is_dynamic = 0
  740. def analyse(self, env, nonempty=0, is_self_arg=False):
  741. if is_self_arg:
  742. self.base_type.is_self_arg = self.is_self_arg = True
  743. if self.type is None:
  744. # The parser may misinterpret names as types. We fix that here.
  745. if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '':
  746. if nonempty:
  747. if self.base_type.is_basic_c_type:
  748. # char, short, long called "int"
  749. type = self.base_type.analyse(env, could_be_name=True)
  750. arg_name = type.empty_declaration_code()
  751. else:
  752. arg_name = self.base_type.name
  753. self.declarator.name = EncodedString(arg_name)
  754. self.base_type.name = None
  755. self.base_type.is_basic_c_type = False
  756. could_be_name = True
  757. else:
  758. could_be_name = False
  759. self.base_type.is_arg = True
  760. base_type = self.base_type.analyse(env, could_be_name=could_be_name)
  761. if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name:
  762. self.declarator.name = self.base_type.arg_name
  763. # The parser is unable to resolve the ambiguity of [] as part of the
  764. # type (e.g. in buffers) or empty declarator (as with arrays).
  765. # This is only arises for empty multi-dimensional arrays.
  766. if (base_type.is_array
  767. and isinstance(self.base_type, TemplatedTypeNode)
  768. and isinstance(self.declarator, CArrayDeclaratorNode)):
  769. declarator = self.declarator
  770. while isinstance(declarator.base, CArrayDeclaratorNode):
  771. declarator = declarator.base
  772. declarator.base = self.base_type.array_declarator
  773. base_type = base_type.base_type
  774. # inject type declaration from annotations
  775. # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis
  776. if self.annotation and env and env.directives['annotation_typing'] and self.base_type.name is None:
  777. arg_type = self.inject_type_from_annotations(env)
  778. if arg_type is not None:
  779. base_type = arg_type
  780. return self.declarator.analyse(base_type, env, nonempty=nonempty)
  781. else:
  782. return self.name_declarator, self.type
  783. def inject_type_from_annotations(self, env):
  784. annotation = self.annotation
  785. if not annotation:
  786. return None
  787. base_type, arg_type = analyse_type_annotation(annotation, env, assigned_value=self.default)
  788. if base_type is not None:
  789. self.base_type = base_type
  790. return arg_type
  791. def calculate_default_value_code(self, code):
  792. if self.default_value is None:
  793. if self.default:
  794. if self.default.is_literal:
  795. # will not output any code, just assign the result_code
  796. self.default.generate_evaluation_code(code)
  797. return self.type.cast_code(self.default.result())
  798. self.default_value = code.get_argument_default_const(self.type)
  799. return self.default_value
  800. def annotate(self, code):
  801. if self.default:
  802. self.default.annotate(code)
  803. def generate_assignment_code(self, code, target=None, overloaded_assignment=False):
  804. default = self.default
  805. if default is None or default.is_literal:
  806. return
  807. if target is None:
  808. target = self.calculate_default_value_code(code)
  809. default.generate_evaluation_code(code)
  810. default.make_owned_reference(code)
  811. result = default.result() if overloaded_assignment else default.result_as(self.type)
  812. code.putln("%s = %s;" % (target, result))
  813. if self.type.is_pyobject:
  814. code.put_giveref(default.result())
  815. default.generate_post_assignment_code(code)
  816. default.free_temps(code)
  817. class CBaseTypeNode(Node):
  818. # Abstract base class for C base type nodes.
  819. #
  820. # Processing during analyse_declarations phase:
  821. #
  822. # analyse
  823. # Returns the type.
  824. def analyse_as_type(self, env):
  825. return self.analyse(env)
  826. class CAnalysedBaseTypeNode(Node):
  827. # type type
  828. child_attrs = []
  829. def analyse(self, env, could_be_name=False):
  830. return self.type
  831. class CSimpleBaseTypeNode(CBaseTypeNode):
  832. # name string
  833. # module_path [string] Qualifying name components
  834. # is_basic_c_type boolean
  835. # signed boolean
  836. # longness integer
  837. # complex boolean
  838. # is_self_arg boolean Is self argument of C method
  839. # ##is_type_arg boolean Is type argument of class method
  840. child_attrs = []
  841. arg_name = None # in case the argument name was interpreted as a type
  842. module_path = []
  843. is_basic_c_type = False
  844. complex = False
  845. def analyse(self, env, could_be_name=False):
  846. # Return type descriptor.
  847. #print "CSimpleBaseTypeNode.analyse: is_self_arg =", self.is_self_arg ###
  848. type = None
  849. if self.is_basic_c_type:
  850. type = PyrexTypes.simple_c_type(self.signed, self.longness, self.name)
  851. if not type:
  852. error(self.pos, "Unrecognised type modifier combination")
  853. elif self.name == "object" and not self.module_path:
  854. type = py_object_type
  855. elif self.name is None:
  856. if self.is_self_arg and env.is_c_class_scope:
  857. #print "CSimpleBaseTypeNode.analyse: defaulting to parent type" ###
  858. type = env.parent_type
  859. ## elif self.is_type_arg and env.is_c_class_scope:
  860. ## type = Builtin.type_type
  861. else:
  862. type = py_object_type
  863. else:
  864. if self.module_path:
  865. # Maybe it's a nested C++ class.
  866. scope = env
  867. for item in self.module_path:
  868. entry = scope.lookup(item)
  869. if entry is not None and entry.is_cpp_class:
  870. scope = entry.type.scope
  871. else:
  872. scope = None
  873. break
  874. if scope is None:
  875. # Maybe it's a cimport.
  876. scope = env.find_imported_module(self.module_path, self.pos)
  877. else:
  878. scope = env
  879. if scope:
  880. if scope.is_c_class_scope:
  881. scope = scope.global_scope()
  882. type = scope.lookup_type(self.name)
  883. if type is not None:
  884. pass
  885. elif could_be_name:
  886. if self.is_self_arg and env.is_c_class_scope:
  887. type = env.parent_type
  888. ## elif self.is_type_arg and env.is_c_class_scope:
  889. ## type = Builtin.type_type
  890. else:
  891. type = py_object_type
  892. self.arg_name = EncodedString(self.name)
  893. else:
  894. if self.templates:
  895. if not self.name in self.templates:
  896. error(self.pos, "'%s' is not a type identifier" % self.name)
  897. type = PyrexTypes.TemplatePlaceholderType(self.name)
  898. else:
  899. error(self.pos, "'%s' is not a type identifier" % self.name)
  900. if type and type.is_fused and env.fused_to_specific:
  901. type = type.specialize(env.fused_to_specific)
  902. if self.complex:
  903. if not type.is_numeric or type.is_complex:
  904. error(self.pos, "can only complexify c numeric types")
  905. type = PyrexTypes.CComplexType(type)
  906. type.create_declaration_utility_code(env)
  907. elif type is Builtin.complex_type:
  908. # Special case: optimise builtin complex type into C's
  909. # double complex. The parser cannot do this (as for the
  910. # normal scalar types) as the user may have redeclared the
  911. # 'complex' type. Testing for the exact type here works.
  912. type = PyrexTypes.c_double_complex_type
  913. type.create_declaration_utility_code(env)
  914. self.complex = True
  915. if type:
  916. return type
  917. else:
  918. return PyrexTypes.error_type
  919. class MemoryViewSliceTypeNode(CBaseTypeNode):
  920. name = 'memoryview'
  921. child_attrs = ['base_type_node', 'axes']
  922. def analyse(self, env, could_be_name=False):
  923. base_type = self.base_type_node.analyse(env)
  924. if base_type.is_error: return base_type
  925. from . import MemoryView
  926. try:
  927. axes_specs = MemoryView.get_axes_specs(env, self.axes)
  928. except CompileError as e:
  929. error(e.position, e.message_only)
  930. self.type = PyrexTypes.ErrorType()
  931. return self.type
  932. if not MemoryView.validate_axes(self.pos, axes_specs):
  933. self.type = error_type
  934. else:
  935. self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
  936. self.type.validate_memslice_dtype(self.pos)
  937. self.use_memview_utilities(env)
  938. return self.type
  939. def use_memview_utilities(self, env):
  940. from . import MemoryView
  941. env.use_utility_code(MemoryView.view_utility_code)
  942. class CNestedBaseTypeNode(CBaseTypeNode):
  943. # For C++ classes that live inside other C++ classes.
  944. # name string
  945. # base_type CBaseTypeNode
  946. child_attrs = ['base_type']
  947. def analyse(self, env, could_be_name=None):
  948. base_type = self.base_type.analyse(env)
  949. if base_type is PyrexTypes.error_type:
  950. return PyrexTypes.error_type
  951. if not base_type.is_cpp_class:
  952. error(self.pos, "'%s' is not a valid type scope" % base_type)
  953. return PyrexTypes.error_type
  954. type_entry = base_type.scope.lookup_here(self.name)
  955. if not type_entry or not type_entry.is_type:
  956. error(self.pos, "'%s.%s' is not a type identifier" % (base_type, self.name))
  957. return PyrexTypes.error_type
  958. return type_entry.type
  959. class TemplatedTypeNode(CBaseTypeNode):
  960. # After parsing:
  961. # positional_args [ExprNode] List of positional arguments
  962. # keyword_args DictNode Keyword arguments
  963. # base_type_node CBaseTypeNode
  964. # After analysis:
  965. # type PyrexTypes.BufferType or PyrexTypes.CppClassType ...containing the right options
  966. child_attrs = ["base_type_node", "positional_args",
  967. "keyword_args", "dtype_node"]
  968. dtype_node = None
  969. name = None
  970. def analyse(self, env, could_be_name=False, base_type=None):
  971. if base_type is None:
  972. base_type = self.base_type_node.analyse(env)
  973. if base_type.is_error: return base_type
  974. if base_type.is_cpp_class and base_type.is_template_type():
  975. # Templated class
  976. if self.keyword_args and self.keyword_args.key_value_pairs:
  977. error(self.pos, "c++ templates cannot take keyword arguments")
  978. self.type = PyrexTypes.error_type
  979. else:
  980. template_types = []
  981. for template_node in self.positional_args:
  982. type = template_node.analyse_as_type(env)
  983. if type is None:
  984. error(template_node.pos, "unknown type in template argument")
  985. type = error_type
  986. template_types.append(type)
  987. self.type = base_type.specialize_here(self.pos, template_types)
  988. elif base_type.is_pyobject:
  989. # Buffer
  990. from . import Buffer
  991. options = Buffer.analyse_buffer_options(
  992. self.pos,
  993. env,
  994. self.positional_args,
  995. self.keyword_args,
  996. base_type.buffer_defaults)
  997. if sys.version_info[0] < 3:
  998. # Py 2.x enforces byte strings as keyword arguments ...
  999. options = dict([(name.encode('ASCII'), value)
  1000. for name, value in options.items()])
  1001. self.type = PyrexTypes.BufferType(base_type, **options)
  1002. if has_np_pythran(env) and is_pythran_buffer(self.type):
  1003. self.type = PyrexTypes.PythranExpr(pythran_type(self.type), self.type)
  1004. else:
  1005. # Array
  1006. empty_declarator = CNameDeclaratorNode(self.pos, name="", cname=None)
  1007. if len(self.positional_args) > 1 or self.keyword_args.key_value_pairs:
  1008. error(self.pos, "invalid array declaration")
  1009. self.type = PyrexTypes.error_type
  1010. else:
  1011. # It would be nice to merge this class with CArrayDeclaratorNode,
  1012. # but arrays are part of the declaration, not the type...
  1013. if not self.positional_args:
  1014. dimension = None
  1015. else:
  1016. dimension = self.positional_args[0]
  1017. self.array_declarator = CArrayDeclaratorNode(
  1018. self.pos,
  1019. base=empty_declarator,
  1020. dimension=dimension)
  1021. self.type = self.array_declarator.analyse(base_type, env)[1]
  1022. if self.type.is_fused and env.fused_to_specific:
  1023. self.type = self.type.specialize(env.fused_to_specific)
  1024. return self.type
  1025. class CComplexBaseTypeNode(CBaseTypeNode):
  1026. # base_type CBaseTypeNode
  1027. # declarator CDeclaratorNode
  1028. child_attrs = ["base_type", "declarator"]
  1029. def analyse(self, env, could_be_name=False):
  1030. base = self.base_type.analyse(env, could_be_name)
  1031. _, type = self.declarator.analyse(base, env)
  1032. return type
  1033. class CTupleBaseTypeNode(CBaseTypeNode):
  1034. # components [CBaseTypeNode]
  1035. child_attrs = ["components"]
  1036. def analyse(self, env, could_be_name=False):
  1037. component_types = []
  1038. for c in self.components:
  1039. type = c.analyse(env)
  1040. if type.is_pyobject:
  1041. error(c.pos, "Tuple types can't (yet) contain Python objects.")
  1042. return error_type
  1043. component_types.append(type)
  1044. entry = env.declare_tuple_type(self.pos, component_types)
  1045. entry.used = True
  1046. return entry.type
  1047. class FusedTypeNode(CBaseTypeNode):
  1048. """
  1049. Represents a fused type in a ctypedef statement:
  1050. ctypedef cython.fused_type(int, long, long long) integral
  1051. name str name of this fused type
  1052. types [CSimpleBaseTypeNode] is the list of types to be fused
  1053. """
  1054. child_attrs = []
  1055. def analyse_declarations(self, env):
  1056. type = self.analyse(env)
  1057. entry = env.declare_typedef(self.name, type, self.pos)
  1058. # Omit the typedef declaration that self.declarator would produce
  1059. entry.in_cinclude = True
  1060. def analyse(self, env, could_be_name=False):
  1061. types = []
  1062. for type_node in self.types:
  1063. type = type_node.analyse_as_type(env)
  1064. if not type:
  1065. error(type_node.pos, "Not a type")
  1066. continue
  1067. if type in types:
  1068. error(type_node.pos, "Type specified multiple times")
  1069. else:
  1070. types.append(type)
  1071. # if len(self.types) == 1:
  1072. # return types[0]
  1073. return PyrexTypes.FusedType(types, name=self.name)
  1074. class CConstTypeNode(CBaseTypeNode):
  1075. # base_type CBaseTypeNode
  1076. child_attrs = ["base_type"]
  1077. def analyse(self, env, could_be_name=False):
  1078. base = self.base_type.analyse(env, could_be_name)
  1079. if base.is_pyobject:
  1080. error(self.pos,
  1081. "Const base type cannot be a Python object")
  1082. return PyrexTypes.c_const_type(base)
  1083. class CVarDefNode(StatNode):
  1084. # C variable definition or forward/extern function declaration.
  1085. #
  1086. # visibility 'private' or 'public' or 'extern'
  1087. # base_type CBaseTypeNode
  1088. # declarators [CDeclaratorNode]
  1089. # in_pxd boolean
  1090. # api boolean
  1091. # overridable boolean whether it is a cpdef
  1092. # modifiers ['inline']
  1093. # decorators [cython.locals(...)] or None
  1094. # directive_locals { string : NameNode } locals defined by cython.locals(...)
  1095. child_attrs = ["base_type", "declarators"]
  1096. decorators = None
  1097. directive_locals = None
  1098. def analyse_declarations(self, env, dest_scope=None):
  1099. if self.directive_locals is None:
  1100. self.directive_locals = {}
  1101. if not dest_scope:
  1102. dest_scope = env
  1103. self.dest_scope = dest_scope
  1104. if self.declarators:
  1105. templates = self.declarators[0].analyse_templates()
  1106. else:
  1107. templates = None
  1108. if templates is not None:
  1109. if self.visibility != 'extern':
  1110. error(self.pos, "Only extern functions allowed")
  1111. if len(self.declarators) > 1:
  1112. error(self.declarators[1].pos, "Can't multiply declare template types")
  1113. env = TemplateScope('func_template', env)
  1114. env.directives = env.outer_scope.directives
  1115. for template_param in templates:
  1116. env.declare_type(template_param.name, template_param, self.pos)
  1117. base_type = self.base_type.analyse(env)
  1118. if base_type.is_fused and not self.in_pxd and (env.is_c_class_scope or
  1119. env.is_module_scope):
  1120. error(self.pos, "Fused types not allowed here")
  1121. return error_type
  1122. self.entry = None
  1123. visibility = self.visibility
  1124. for declarator in self.declarators:
  1125. if (len(self.declarators) > 1
  1126. and not isinstance(declarator, CNameDeclaratorNode)
  1127. and env.directives['warn.multiple_declarators']):
  1128. warning(
  1129. declarator.pos,
  1130. "Non-trivial type declarators in shared declaration (e.g. mix of pointers and values). "
  1131. "Each pointer declaration should be on its own line.", 1)
  1132. create_extern_wrapper = (self.overridable
  1133. and self.visibility == 'extern'
  1134. and env.is_module_scope)
  1135. if create_extern_wrapper:
  1136. declarator.overridable = False
  1137. if isinstance(declarator, CFuncDeclaratorNode):
  1138. name_declarator, type = declarator.analyse(
  1139. base_type, env, directive_locals=self.directive_locals, visibility=visibility, in_pxd=self.in_pxd)
  1140. else:
  1141. name_declarator, type = declarator.analyse(
  1142. base_type, env, visibility=visibility, in_pxd=self.in_pxd)
  1143. if not type.is_complete():
  1144. if not (self.visibility == 'extern' and type.is_array or type.is_memoryviewslice):
  1145. error(declarator.pos, "Variable type '%s' is incomplete" % type)
  1146. if self.visibility == 'extern' and type.is_pyobject:
  1147. error(declarator.pos, "Python object cannot be declared extern")
  1148. name = name_declarator.name
  1149. cname = name_declarator.cname
  1150. if name == '':
  1151. error(declarator.pos, "Missing name in declaration.")
  1152. return
  1153. if type.is_reference and self.visibility != 'extern':
  1154. error(declarator.pos, "C++ references cannot be declared; use a pointer instead")
  1155. if type.is_cfunction:
  1156. if 'staticmethod' in env.directives:
  1157. type.is_static_method = True
  1158. self.entry = dest_scope.declare_cfunction(
  1159. name, type, declarator.pos,
  1160. cname=cname, visibility=self.visibility, in_pxd=self.in_pxd,
  1161. api=self.api, modifiers=self.modifiers, overridable=self.overridable)
  1162. if self.entry is not None:
  1163. self.entry.directive_locals = copy.copy(self.directive_locals)
  1164. if create_extern_wrapper:
  1165. self.entry.type.create_to_py_utility_code(env)
  1166. self.entry.create_wrapper = True
  1167. else:
  1168. if self.overridable:
  1169. warning(self.pos, "cpdef variables will not be supported in Cython 3; "
  1170. "currently they are no different from cdef variables", 2)
  1171. if self.directive_locals:
  1172. error(self.pos, "Decorators can only be followed by functions")
  1173. self.entry = dest_scope.declare_var(
  1174. name, type, declarator.pos,
  1175. cname=cname, visibility=visibility, in_pxd=self.in_pxd,
  1176. api=self.api, is_cdef=1)
  1177. if Options.docstrings:
  1178. self.entry.doc = embed_position(self.pos, self.doc)
  1179. class CStructOrUnionDefNode(StatNode):
  1180. # name string
  1181. # cname string or None
  1182. # kind "struct" or "union"
  1183. # typedef_flag boolean
  1184. # visibility "public" or "private"
  1185. # api boolean
  1186. # in_pxd boolean
  1187. # attributes [CVarDefNode] or None
  1188. # entry Entry
  1189. # packed boolean
  1190. child_attrs = ["attributes"]
  1191. def declare(self, env, scope=None):
  1192. self.entry = env.declare_struct_or_union(
  1193. self.name, self.kind, scope, self.typedef_flag, self.pos,
  1194. self.cname, visibility=self.visibility, api=self.api,
  1195. packed=self.packed)
  1196. def analyse_declarations(self, env):
  1197. scope = None
  1198. if self.attributes is not None:
  1199. scope = StructOrUnionScope(self.name)
  1200. self.declare(env, scope)
  1201. if self.attributes is not None:
  1202. if self.in_pxd and not env.in_cinclude:
  1203. self.entry.defined_in_pxd = 1
  1204. for attr in self.attributes:
  1205. attr.analyse_declarations(env, scope)
  1206. if self.visibility != 'extern':
  1207. for attr in scope.var_entries:
  1208. type = attr.type
  1209. while type.is_array:
  1210. type = type.base_type
  1211. if type == self.entry.type:
  1212. error(attr.pos, "Struct cannot contain itself as a member.")
  1213. def analyse_expressions(self, env):
  1214. return self
  1215. def generate_execution_code(self, code):
  1216. pass
  1217. class CppClassNode(CStructOrUnionDefNode, BlockNode):
  1218. # name string
  1219. # cname string or None
  1220. # visibility "extern"
  1221. # in_pxd boolean
  1222. # attributes [CVarDefNode] or None
  1223. # entry Entry
  1224. # base_classes [CBaseTypeNode]
  1225. # templates [(string, bool)] or None
  1226. # decorators [DecoratorNode] or None
  1227. decorators = None
  1228. def declare(self, env):
  1229. if self.templates is None:
  1230. template_types = None
  1231. else:
  1232. template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
  1233. for template_name, required in self.templates]
  1234. num_optional_templates = sum(not required for _, required in self.templates)
  1235. if num_optional_templates and not all(required for _, required in self.templates[:-num_optional_templates]):
  1236. error(self.pos, "Required template parameters must precede optional template parameters.")
  1237. self.entry = env.declare_cpp_class(
  1238. self.name, None, self.pos, self.cname,
  1239. base_classes=[], visibility=self.visibility, templates=template_types)
  1240. def analyse_declarations(self, env):
  1241. if self.templates is None:
  1242. template_types = template_names = None
  1243. else:
  1244. template_names = [template_name for template_name, _ in self.templates]
  1245. template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
  1246. for template_name, required in self.templates]
  1247. scope = None
  1248. if self.attributes is not None:
  1249. scope = CppClassScope(self.name, env, templates=template_names)
  1250. def base_ok(base_class):
  1251. if base_class.is_cpp_class or base_class.is_struct:
  1252. return True
  1253. else:
  1254. error(self.pos, "Base class '%s' not a struct or class." % base_class)
  1255. base_class_types = filter(base_ok, [b.analyse(scope or env) for b in self.base_classes])
  1256. self.entry = env.declare_cpp_class(
  1257. self.name, scope, self.pos,
  1258. self.cname, base_class_types, visibility=self.visibility, templates=template_types)
  1259. if self.entry is None:
  1260. return
  1261. self.entry.is_cpp_class = 1
  1262. if scope is not None:
  1263. scope.type = self.entry.type
  1264. defined_funcs = []
  1265. def func_attributes(attributes):
  1266. for attr in attributes:
  1267. if isinstance(attr, CFuncDefNode):
  1268. yield attr
  1269. elif isinstance(attr, CompilerDirectivesNode):
  1270. for sub_attr in func_attributes(attr.body.stats):
  1271. yield sub_attr
  1272. if self.attributes is not None:
  1273. if self.in_pxd and not env.in_cinclude:
  1274. self.entry.defined_in_pxd = 1
  1275. for attr in self.attributes:
  1276. declare = getattr(attr, 'declare', None)
  1277. if declare:
  1278. attr.declare(scope)
  1279. attr.analyse_declarations(scope)
  1280. for func in func_attributes(self.attributes):
  1281. defined_funcs.append(func)
  1282. if self.templates is not None:
  1283. func.template_declaration = "template <typename %s>" % ", typename ".join(template_names)
  1284. self.body = StatListNode(self.pos, stats=defined_funcs)
  1285. self.scope = scope
  1286. def analyse_expressions(self, env):
  1287. self.body = self.body.analyse_expressions(self.entry.type.scope)
  1288. return self
  1289. def generate_function_definitions(self, env, code):
  1290. self.body.generate_function_definitions(self.entry.type.scope, code)
  1291. def generate_execution_code(self, code):
  1292. self.body.generate_execution_code(code)
  1293. def annotate(self, code):
  1294. self.body.annotate(code)
  1295. class CEnumDefNode(StatNode):
  1296. # name string or None
  1297. # cname string or None
  1298. # items [CEnumDefItemNode]
  1299. # typedef_flag boolean
  1300. # visibility "public" or "private" or "extern"
  1301. # api boolean
  1302. # in_pxd boolean
  1303. # create_wrapper boolean
  1304. # entry Entry
  1305. child_attrs = ["items"]
  1306. def declare(self, env):
  1307. self.entry = env.declare_enum(
  1308. self.name, self.pos,
  1309. cname=self.cname, typedef_flag=self.typedef_flag,
  1310. visibility=self.visibility, api=self.api,
  1311. create_wrapper=self.create_wrapper)
  1312. def analyse_declarations(self, env):
  1313. if self.items is not None:
  1314. if self.in_pxd and not env.in_cinclude:
  1315. self.entry.defined_in_pxd = 1
  1316. for item in self.items:
  1317. item.analyse_declarations(env, self.entry)
  1318. def analyse_expressions(self, env):
  1319. return self
  1320. def generate_execution_code(self, code):
  1321. if self.visibility == 'public' or self.api:
  1322. code.mark_pos(self.pos)
  1323. temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
  1324. for item in self.entry.enum_values:
  1325. code.putln("%s = PyInt_FromLong(%s); %s" % (
  1326. temp,
  1327. item.cname,
  1328. code.error_goto_if_null(temp, item.pos)))
  1329. code.put_gotref(temp)
  1330. code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % (
  1331. Naming.moddict_cname,
  1332. item.name,
  1333. temp,
  1334. code.error_goto(item.pos)))
  1335. code.put_decref_clear(temp, PyrexTypes.py_object_type)
  1336. code.funcstate.release_temp(temp)
  1337. class CEnumDefItemNode(StatNode):
  1338. # name string
  1339. # cname string or None
  1340. # value ExprNode or None
  1341. child_attrs = ["value"]
  1342. def analyse_declarations(self, env, enum_entry):
  1343. if self.value:
  1344. self.value = self.value.analyse_const_expression(env)
  1345. if not self.value.type.is_int:
  1346. self.value = self.value.coerce_to(PyrexTypes.c_int_type, env)
  1347. self.value = self.value.analyse_const_expression(env)
  1348. entry = env.declare_const(
  1349. self.name, enum_entry.type,
  1350. self.value, self.pos, cname=self.cname,
  1351. visibility=enum_entry.visibility, api=enum_entry.api,
  1352. create_wrapper=enum_entry.create_wrapper and enum_entry.name is None)
  1353. enum_entry.enum_values.append(entry)
  1354. if enum_entry.name:
  1355. enum_entry.type.values.append(entry.name)
  1356. class CTypeDefNode(StatNode):
  1357. # base_type CBaseTypeNode
  1358. # declarator CDeclaratorNode
  1359. # visibility "public" or "private"
  1360. # api boolean
  1361. # in_pxd boolean
  1362. child_attrs = ["base_type", "declarator"]
  1363. def analyse_declarations(self, env):
  1364. base = self.base_type.analyse(env)
  1365. name_declarator, type = self.declarator.analyse(
  1366. base, env, visibility=self.visibility, in_pxd=self.in_pxd)
  1367. name = name_declarator.name
  1368. cname = name_declarator.cname
  1369. entry = env.declare_typedef(
  1370. name, type, self.pos,
  1371. cname=cname, visibility=self.visibility, api=self.api)
  1372. if type.is_fused:
  1373. entry.in_cinclude = True
  1374. if self.in_pxd and not env.in_cinclude:
  1375. entry.defined_in_pxd = 1
  1376. def analyse_expressions(self, env):
  1377. return self
  1378. def generate_execution_code(self, code):
  1379. pass
  1380. class FuncDefNode(StatNode, BlockNode):
  1381. # Base class for function definition nodes.
  1382. #
  1383. # return_type PyrexType
  1384. # #filename string C name of filename string const
  1385. # entry Symtab.Entry
  1386. # needs_closure boolean Whether or not this function has inner functions/classes/yield
  1387. # needs_outer_scope boolean Whether or not this function requires outer scope
  1388. # pymethdef_required boolean Force Python method struct generation
  1389. # directive_locals { string : ExprNode } locals defined by cython.locals(...)
  1390. # directive_returns [ExprNode] type defined by cython.returns(...)
  1391. # star_arg PyArgDeclNode or None * argument
  1392. # starstar_arg PyArgDeclNode or None ** argument
  1393. #
  1394. # is_async_def boolean is a Coroutine function
  1395. #
  1396. # has_fused_arguments boolean
  1397. # Whether this cdef function has fused parameters. This is needed
  1398. # by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
  1399. # with fused argument types with a FusedCFuncDefNode
  1400. py_func = None
  1401. needs_closure = False
  1402. needs_outer_scope = False
  1403. pymethdef_required = False
  1404. is_generator = False
  1405. is_generator_body = False
  1406. is_async_def = False
  1407. modifiers = []
  1408. has_fused_arguments = False
  1409. star_arg = None
  1410. starstar_arg = None
  1411. is_cyfunction = False
  1412. code_object = None
  1413. def analyse_default_values(self, env):
  1414. default_seen = 0
  1415. for arg in self.args:
  1416. if arg.default:
  1417. default_seen = 1
  1418. if arg.is_generic:
  1419. arg.default = arg.default.analyse_types(env)
  1420. arg.default = arg.default.coerce_to(arg.type, env)
  1421. else:
  1422. error(arg.pos, "This argument cannot have a default value")
  1423. arg.default = None
  1424. elif arg.kw_only:
  1425. default_seen = 1
  1426. elif default_seen:
  1427. error(arg.pos, "Non-default argument following default argument")
  1428. def analyse_annotation(self, env, annotation):
  1429. # Annotations can not only contain valid Python expressions but arbitrary type references.
  1430. if annotation is None:
  1431. return None
  1432. if not env.directives['annotation_typing'] or annotation.analyse_as_type(env) is None:
  1433. annotation = annotation.analyse_types(env)
  1434. return annotation
  1435. def analyse_annotations(self, env):
  1436. for arg in self.args:
  1437. if arg.annotation:
  1438. arg.annotation = self.analyse_annotation(env, arg.annotation)
  1439. def align_argument_type(self, env, arg):
  1440. # @cython.locals()
  1441. directive_locals = self.directive_locals
  1442. orig_type = arg.type
  1443. if arg.name in directive_locals:
  1444. type_node = directive_locals[arg.name]
  1445. other_type = type_node.analyse_as_type(env)
  1446. elif isinstance(arg, CArgDeclNode) and arg.annotation and env.directives['annotation_typing']:
  1447. type_node = arg.annotation
  1448. other_type = arg.inject_type_from_annotations(env)
  1449. if other_type is None:
  1450. return arg
  1451. else:
  1452. return arg
  1453. if other_type is None:
  1454. error(type_node.pos, "Not a type")
  1455. elif orig_type is not py_object_type and not orig_type.same_as(other_type):
  1456. error(arg.base_type.pos, "Signature does not agree with previous declaration")
  1457. error(type_node.pos, "Previous declaration here")
  1458. else:
  1459. arg.type = other_type
  1460. return arg
  1461. def need_gil_acquisition(self, lenv):
  1462. return 0
  1463. def create_local_scope(self, env):
  1464. genv = env
  1465. while genv.is_py_class_scope or genv.is_c_class_scope:
  1466. genv = genv.outer_scope
  1467. if self.needs_closure:
  1468. lenv = ClosureScope(name=self.entry.name,
  1469. outer_scope=genv,
  1470. parent_scope=env,
  1471. scope_name=self.entry.cname)
  1472. else:
  1473. lenv = LocalScope(name=self.entry.name,
  1474. outer_scope=genv,
  1475. parent_scope=env)
  1476. lenv.return_type = self.return_type
  1477. type = self.entry.type
  1478. if type.is_cfunction:
  1479. lenv.nogil = type.nogil and not type.with_gil
  1480. self.local_scope = lenv
  1481. lenv.directives = env.directives
  1482. return lenv
  1483. def generate_function_body(self, env, code):
  1484. self.body.generate_execution_code(code)
  1485. def generate_function_definitions(self, env, code):
  1486. from . import Buffer
  1487. if self.return_type.is_memoryviewslice:
  1488. from . import MemoryView
  1489. lenv = self.local_scope
  1490. if lenv.is_closure_scope and not lenv.is_passthrough:
  1491. outer_scope_cname = "%s->%s" % (Naming.cur_scope_cname,
  1492. Naming.outer_scope_cname)
  1493. else:
  1494. outer_scope_cname = Naming.outer_scope_cname
  1495. lenv.mangle_closure_cnames(outer_scope_cname)
  1496. # Generate closure function definitions
  1497. self.body.generate_function_definitions(lenv, code)
  1498. # generate lambda function definitions
  1499. self.generate_lambda_definitions(lenv, code)
  1500. is_getbuffer_slot = (self.entry.name == "__getbuffer__" and
  1501. self.entry.scope.is_c_class_scope)
  1502. is_releasebuffer_slot = (self.entry.name == "__releasebuffer__" and
  1503. self.entry.scope.is_c_class_scope)
  1504. is_buffer_slot = is_getbuffer_slot or is_releasebuffer_slot
  1505. if is_buffer_slot:
  1506. if 'cython_unused' not in self.modifiers:
  1507. self.modifiers = self.modifiers + ['cython_unused']
  1508. preprocessor_guard = self.get_preprocessor_guard()
  1509. profile = code.globalstate.directives['profile']
  1510. linetrace = code.globalstate.directives['linetrace']
  1511. if profile or linetrace:
  1512. code.globalstate.use_utility_code(
  1513. UtilityCode.load_cached("Profile", "Profile.c"))
  1514. # Generate C code for header and body of function
  1515. code.enter_cfunc_scope(lenv)
  1516. code.return_from_error_cleanup_label = code.new_label()
  1517. code.funcstate.gil_owned = not lenv.nogil
  1518. # ----- Top-level constants used by this function
  1519. code.mark_pos(self.pos)
  1520. self.generate_cached_builtins_decls(lenv, code)
  1521. # ----- Function header
  1522. code.putln("")
  1523. if preprocessor_guard:
  1524. code.putln(preprocessor_guard)
  1525. with_pymethdef = (self.needs_assignment_synthesis(env, code) or
  1526. self.pymethdef_required)
  1527. if self.py_func:
  1528. self.py_func.generate_function_header(
  1529. code, with_pymethdef=with_pymethdef, proto_only=True)
  1530. self.generate_function_header(code, with_pymethdef=with_pymethdef)
  1531. # ----- Local variable declarations
  1532. # Find function scope
  1533. cenv = env
  1534. while cenv.is_py_class_scope or cenv.is_c_class_scope:
  1535. cenv = cenv.outer_scope
  1536. if self.needs_closure:
  1537. code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
  1538. code.putln(";")
  1539. elif self.needs_outer_scope:
  1540. if lenv.is_passthrough:
  1541. code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
  1542. code.putln(";")
  1543. code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
  1544. code.putln(";")
  1545. self.generate_argument_declarations(lenv, code)
  1546. for entry in lenv.var_entries:
  1547. if not (entry.in_closure or entry.is_arg):
  1548. code.put_var_declaration(entry)
  1549. # Initialize the return variable __pyx_r
  1550. init = ""
  1551. if not self.return_type.is_void:
  1552. if self.return_type.is_pyobject:
  1553. init = " = NULL"
  1554. elif self.return_type.is_memoryviewslice:
  1555. init = ' = ' + MemoryView.memslice_entry_init
  1556. code.putln("%s%s;" % (
  1557. self.return_type.declaration_code(Naming.retval_cname),
  1558. init))
  1559. tempvardecl_code = code.insertion_point()
  1560. self.generate_keyword_list(code)
  1561. # ----- GIL acquisition
  1562. acquire_gil = self.acquire_gil
  1563. # See if we need to acquire the GIL for variable declarations, or for
  1564. # refnanny only
  1565. # Closures are not currently possible for cdef nogil functions,
  1566. # but check them anyway
  1567. have_object_args = self.needs_closure or self.needs_outer_scope
  1568. for arg in lenv.arg_entries:
  1569. if arg.type.is_pyobject:
  1570. have_object_args = True
  1571. break
  1572. used_buffer_entries = [entry for entry in lenv.buffer_entries if entry.used]
  1573. acquire_gil_for_var_decls_only = (
  1574. lenv.nogil and lenv.has_with_gil_block and
  1575. (have_object_args or used_buffer_entries))
  1576. acquire_gil_for_refnanny_only = (
  1577. lenv.nogil and lenv.has_with_gil_block and not
  1578. acquire_gil_for_var_decls_only)
  1579. use_refnanny = not lenv.nogil or lenv.has_with_gil_block
  1580. if acquire_gil or acquire_gil_for_var_decls_only:
  1581. code.put_ensure_gil()
  1582. code.funcstate.gil_owned = True
  1583. elif lenv.nogil and lenv.has_with_gil_block:
  1584. code.declare_gilstate()
  1585. if profile or linetrace:
  1586. if not self.is_generator:
  1587. # generators are traced when iterated, not at creation
  1588. tempvardecl_code.put_trace_declarations()
  1589. code_object = self.code_object.calculate_result_code(code) if self.code_object else None
  1590. code.put_trace_frame_init(code_object)
  1591. # ----- Special check for getbuffer
  1592. if is_getbuffer_slot:
  1593. self.getbuffer_check(code)
  1594. # ----- set up refnanny
  1595. if use_refnanny:
  1596. tempvardecl_code.put_declare_refcount_context()
  1597. code.put_setup_refcount_context(
  1598. self.entry.name, acquire_gil=acquire_gil_for_refnanny_only)
  1599. # ----- Automatic lead-ins for certain special functions
  1600. if is_getbuffer_slot:
  1601. self.getbuffer_init(code)
  1602. # ----- Create closure scope object
  1603. if self.needs_closure:
  1604. tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__')
  1605. slot_func_cname = TypeSlots.get_slot_function(lenv.scope_class.type.scope, tp_slot)
  1606. if not slot_func_cname:
  1607. slot_func_cname = '%s->tp_new' % lenv.scope_class.type.typeptr_cname
  1608. code.putln("%s = (%s)%s(%s, %s, NULL);" % (
  1609. Naming.cur_scope_cname,
  1610. lenv.scope_class.type.empty_declaration_code(),
  1611. slot_func_cname,
  1612. lenv.scope_class.type.typeptr_cname,
  1613. Naming.empty_tuple))
  1614. code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
  1615. # Scope unconditionally DECREFed on return.
  1616. code.putln("%s = %s;" % (
  1617. Naming.cur_scope_cname,
  1618. lenv.scope_class.type.cast_code("Py_None")))
  1619. code.put_incref("Py_None", py_object_type)
  1620. code.putln(code.error_goto(self.pos))
  1621. code.putln("} else {")
  1622. code.put_gotref(Naming.cur_scope_cname)
  1623. code.putln("}")
  1624. # Note that it is unsafe to decref the scope at this point.
  1625. if self.needs_outer_scope:
  1626. if self.is_cyfunction:
  1627. code.putln("%s = (%s) __Pyx_CyFunction_GetClosure(%s);" % (
  1628. outer_scope_cname,
  1629. cenv.scope_class.type.empty_declaration_code(),
  1630. Naming.self_cname))
  1631. else:
  1632. code.putln("%s = (%s) %s;" % (
  1633. outer_scope_cname,
  1634. cenv.scope_class.type.empty_declaration_code(),
  1635. Naming.self_cname))
  1636. if lenv.is_passthrough:
  1637. code.putln("%s = %s;" % (Naming.cur_scope_cname, outer_scope_cname))
  1638. elif self.needs_closure:
  1639. # inner closures own a reference to their outer parent
  1640. code.put_incref(outer_scope_cname, cenv.scope_class.type)
  1641. code.put_giveref(outer_scope_cname)
  1642. # ----- Trace function call
  1643. if profile or linetrace:
  1644. # this looks a bit late, but if we don't get here due to a
  1645. # fatal error before hand, it's not really worth tracing
  1646. if not self.is_generator:
  1647. # generators are traced when iterated, not at creation
  1648. if self.is_wrapper:
  1649. trace_name = self.entry.name + " (wrapper)"
  1650. else:
  1651. trace_name = self.entry.name
  1652. code.put_trace_call(
  1653. trace_name, self.pos, nogil=not code.funcstate.gil_owned)
  1654. code.funcstate.can_trace = True
  1655. # ----- Fetch arguments
  1656. self.generate_argument_parsing_code(env, code)
  1657. # If an argument is assigned to in the body, we must
  1658. # incref it to properly keep track of refcounts.
  1659. is_cdef = isinstance(self, CFuncDefNode)
  1660. for entry in lenv.arg_entries:
  1661. if entry.type.is_pyobject:
  1662. if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
  1663. code.put_var_incref(entry)
  1664. # Note: defaults are always incref-ed. For def functions, we
  1665. # we acquire arguments from object conversion, so we have
  1666. # new references. If we are a cdef function, we need to
  1667. # incref our arguments
  1668. elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1:
  1669. code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
  1670. for entry in lenv.var_entries:
  1671. if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure:
  1672. if entry.xdecref_cleanup:
  1673. code.put_var_xincref(entry)
  1674. else:
  1675. code.put_var_incref(entry)
  1676. # ----- Initialise local buffer auxiliary variables
  1677. for entry in lenv.var_entries + lenv.arg_entries:
  1678. if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
  1679. Buffer.put_init_vars(entry, code)
  1680. # ----- Check and convert arguments
  1681. self.generate_argument_type_tests(code)
  1682. # ----- Acquire buffer arguments
  1683. for entry in lenv.arg_entries:
  1684. if entry.type.is_buffer:
  1685. Buffer.put_acquire_arg_buffer(entry, code, self.pos)
  1686. if acquire_gil_for_var_decls_only:
  1687. code.put_release_ensured_gil()
  1688. code.funcstate.gil_owned = False
  1689. # -------------------------
  1690. # ----- Function body -----
  1691. # -------------------------
  1692. self.generate_function_body(env, code)
  1693. code.mark_pos(self.pos, trace=False)
  1694. code.putln("")
  1695. code.putln("/* function exit code */")
  1696. # ----- Default return value
  1697. if not self.body.is_terminator:
  1698. if self.return_type.is_pyobject:
  1699. #if self.return_type.is_extension_type:
  1700. # lhs = "(PyObject *)%s" % Naming.retval_cname
  1701. #else:
  1702. lhs = Naming.retval_cname
  1703. code.put_init_to_py_none(lhs, self.return_type)
  1704. else:
  1705. val = self.return_type.default_value
  1706. if val:
  1707. code.putln("%s = %s;" % (Naming.retval_cname, val))
  1708. elif not self.return_type.is_void:
  1709. code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
  1710. # ----- Error cleanup
  1711. if code.error_label in code.labels_used:
  1712. if not self.body.is_terminator:
  1713. code.put_goto(code.return_label)
  1714. code.put_label(code.error_label)
  1715. for cname, type in code.funcstate.all_managed_temps():
  1716. code.put_xdecref(cname, type, have_gil=not lenv.nogil)
  1717. # Clean up buffers -- this calls a Python function
  1718. # so need to save and restore error state
  1719. buffers_present = len(used_buffer_entries) > 0
  1720. #memslice_entries = [e for e in lenv.entries.values() if e.type.is_memoryviewslice]
  1721. if buffers_present:
  1722. code.globalstate.use_utility_code(restore_exception_utility_code)
  1723. code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
  1724. code.putln("__Pyx_PyThreadState_declare")
  1725. code.putln("__Pyx_PyThreadState_assign")
  1726. code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);")
  1727. for entry in used_buffer_entries:
  1728. Buffer.put_release_buffer_code(code, entry)
  1729. #code.putln("%s = 0;" % entry.cname)
  1730. code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
  1731. if self.return_type.is_memoryviewslice:
  1732. MemoryView.put_init_entry(Naming.retval_cname, code)
  1733. err_val = Naming.retval_cname
  1734. else:
  1735. err_val = self.error_value()
  1736. exc_check = self.caller_will_check_exceptions()
  1737. if err_val is not None or exc_check:
  1738. # TODO: Fix exception tracing (though currently unused by cProfile).
  1739. # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
  1740. # code.put_trace_exception()
  1741. if lenv.nogil and not lenv.has_with_gil_block:
  1742. code.putln("{")
  1743. code.put_ensure_gil()
  1744. code.put_add_traceback(self.entry.qualified_name)
  1745. if lenv.nogil and not lenv.has_with_gil_block:
  1746. code.put_release_ensured_gil()
  1747. code.putln("}")
  1748. else:
  1749. warning(self.entry.pos,
  1750. "Unraisable exception in function '%s'." %
  1751. self.entry.qualified_name, 0)
  1752. code.put_unraisable(self.entry.qualified_name, lenv.nogil)
  1753. default_retval = self.return_type.default_value
  1754. if err_val is None and default_retval:
  1755. err_val = default_retval
  1756. if err_val is not None:
  1757. if err_val != Naming.retval_cname:
  1758. code.putln("%s = %s;" % (Naming.retval_cname, err_val))
  1759. elif not self.return_type.is_void:
  1760. code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
  1761. if is_getbuffer_slot:
  1762. self.getbuffer_error_cleanup(code)
  1763. # If we are using the non-error cleanup section we should
  1764. # jump past it if we have an error. The if-test below determine
  1765. # whether this section is used.
  1766. if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice:
  1767. code.put_goto(code.return_from_error_cleanup_label)
  1768. # ----- Non-error return cleanup
  1769. code.put_label(code.return_label)
  1770. for entry in used_buffer_entries:
  1771. Buffer.put_release_buffer_code(code, entry)
  1772. if is_getbuffer_slot:
  1773. self.getbuffer_normal_cleanup(code)
  1774. if self.return_type.is_memoryviewslice:
  1775. # See if our return value is uninitialized on non-error return
  1776. # from . import MemoryView
  1777. # MemoryView.err_if_nogil_initialized_check(self.pos, env)
  1778. cond = code.unlikely(self.return_type.error_condition(Naming.retval_cname))
  1779. code.putln(
  1780. 'if (%s) {' % cond)
  1781. if env.nogil:
  1782. code.put_ensure_gil()
  1783. code.putln(
  1784. 'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");')
  1785. if env.nogil:
  1786. code.put_release_ensured_gil()
  1787. code.putln(
  1788. '}')
  1789. # ----- Return cleanup for both error and no-error return
  1790. code.put_label(code.return_from_error_cleanup_label)
  1791. for entry in lenv.var_entries:
  1792. if not entry.used or entry.in_closure:
  1793. continue
  1794. if entry.type.is_memoryviewslice:
  1795. code.put_xdecref_memoryviewslice(entry.cname, have_gil=not lenv.nogil)
  1796. elif entry.type.is_pyobject:
  1797. if not entry.is_arg or len(entry.cf_assignments) > 1:
  1798. if entry.xdecref_cleanup:
  1799. code.put_var_xdecref(entry)
  1800. else:
  1801. code.put_var_decref(entry)
  1802. # Decref any increfed args
  1803. for entry in lenv.arg_entries:
  1804. if entry.type.is_pyobject:
  1805. if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
  1806. code.put_var_decref(entry)
  1807. elif (entry.type.is_memoryviewslice and
  1808. (not is_cdef or len(entry.cf_assignments) > 1)):
  1809. # decref slices of def functions and acquired slices from cdef
  1810. # functions, but not borrowed slices from cdef functions.
  1811. code.put_xdecref_memoryviewslice(entry.cname,
  1812. have_gil=not lenv.nogil)
  1813. if self.needs_closure:
  1814. code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
  1815. # ----- Return
  1816. # This code is duplicated in ModuleNode.generate_module_init_func
  1817. if not lenv.nogil:
  1818. default_retval = self.return_type.default_value
  1819. err_val = self.error_value()
  1820. if err_val is None and default_retval:
  1821. err_val = default_retval # FIXME: why is err_val not used?
  1822. if self.return_type.is_pyobject:
  1823. code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
  1824. if self.entry.is_special and self.entry.name == "__hash__":
  1825. # Returning -1 for __hash__ is supposed to signal an error
  1826. # We do as Python instances and coerce -1 into -2.
  1827. code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % (
  1828. Naming.retval_cname, Naming.retval_cname))
  1829. if profile or linetrace:
  1830. code.funcstate.can_trace = False
  1831. if not self.is_generator:
  1832. # generators are traced when iterated, not at creation
  1833. if self.return_type.is_pyobject:
  1834. code.put_trace_return(
  1835. Naming.retval_cname, nogil=not code.funcstate.gil_owned)
  1836. else:
  1837. code.put_trace_return(
  1838. "Py_None", nogil=not code.funcstate.gil_owned)
  1839. if not lenv.nogil:
  1840. # GIL holding function
  1841. code.put_finish_refcount_context()
  1842. if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
  1843. # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
  1844. code.put_release_ensured_gil()
  1845. code.funcstate.gil_owned = False
  1846. if not self.return_type.is_void:
  1847. code.putln("return %s;" % Naming.retval_cname)
  1848. code.putln("}")
  1849. if preprocessor_guard:
  1850. code.putln("#endif /*!(%s)*/" % preprocessor_guard)
  1851. # ----- Go back and insert temp variable declarations
  1852. tempvardecl_code.put_temp_declarations(code.funcstate)
  1853. # ----- Python version
  1854. code.exit_cfunc_scope()
  1855. if self.py_func:
  1856. self.py_func.generate_function_definitions(env, code)
  1857. self.generate_wrapper_functions(code)
  1858. def declare_argument(self, env, arg):
  1859. if arg.type.is_void:
  1860. error(arg.pos, "Invalid use of 'void'")
  1861. elif not arg.type.is_complete() and not (arg.type.is_array or arg.type.is_memoryviewslice):
  1862. error(arg.pos, "Argument type '%s' is incomplete" % arg.type)
  1863. entry = env.declare_arg(arg.name, arg.type, arg.pos)
  1864. if arg.annotation:
  1865. entry.annotation = arg.annotation
  1866. return entry
  1867. def generate_arg_type_test(self, arg, code):
  1868. # Generate type test for one argument.
  1869. if arg.type.typeobj_is_available():
  1870. code.globalstate.use_utility_code(
  1871. UtilityCode.load_cached("ArgTypeTest", "FunctionArguments.c"))
  1872. typeptr_cname = arg.type.typeptr_cname
  1873. arg_code = "((PyObject *)%s)" % arg.entry.cname
  1874. code.putln(
  1875. 'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % (
  1876. arg_code,
  1877. typeptr_cname,
  1878. arg.accept_none,
  1879. arg.name,
  1880. arg.type.is_builtin_type and arg.type.require_exact,
  1881. code.error_goto(arg.pos)))
  1882. else:
  1883. error(arg.pos, "Cannot test type of extern C class without type object name specification")
  1884. def generate_arg_none_check(self, arg, code):
  1885. # Generate None check for one argument.
  1886. if arg.type.is_memoryviewslice:
  1887. cname = "%s.memview" % arg.entry.cname
  1888. else:
  1889. cname = arg.entry.cname
  1890. code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname)
  1891. code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % (
  1892. max(200, len(arg.name)), arg.name,
  1893. code.error_goto(arg.pos)))
  1894. code.putln('}')
  1895. def generate_wrapper_functions(self, code):
  1896. pass
  1897. def generate_execution_code(self, code):
  1898. code.mark_pos(self.pos)
  1899. # Evaluate and store argument default values
  1900. for arg in self.args:
  1901. if not arg.is_dynamic:
  1902. arg.generate_assignment_code(code)
  1903. #
  1904. # Special code for the __getbuffer__ function
  1905. #
  1906. def _get_py_buffer_info(self):
  1907. py_buffer = self.local_scope.arg_entries[1]
  1908. try:
  1909. # Check builtin definition of struct Py_buffer
  1910. obj_type = py_buffer.type.base_type.scope.entries['obj'].type
  1911. except (AttributeError, KeyError):
  1912. # User code redeclared struct Py_buffer
  1913. obj_type = None
  1914. return py_buffer, obj_type
  1915. # Old Python 3 used to support write-locks on buffer-like objects by
  1916. # calling PyObject_GetBuffer() with a view==NULL parameter. This obscure
  1917. # feature is obsolete, it was almost never used (only one instance in
  1918. # `Modules/posixmodule.c` in Python 3.1) and it is now officially removed
  1919. # (see bpo-14203). We add an extra check here to prevent legacy code from
  1920. # from trying to use the feature and prevent segmentation faults.
  1921. def getbuffer_check(self, code):
  1922. py_buffer, _ = self._get_py_buffer_info()
  1923. view = py_buffer.cname
  1924. code.putln("if (%s == NULL) {" % view)
  1925. code.putln("PyErr_SetString(PyExc_BufferError, "
  1926. "\"PyObject_GetBuffer: view==NULL argument is obsolete\");")
  1927. code.putln("return -1;")
  1928. code.putln("}")
  1929. def getbuffer_init(self, code):
  1930. py_buffer, obj_type = self._get_py_buffer_info()
  1931. view = py_buffer.cname
  1932. if obj_type and obj_type.is_pyobject:
  1933. code.put_init_to_py_none("%s->obj" % view, obj_type)
  1934. code.put_giveref("%s->obj" % view) # Do not refnanny object within structs
  1935. else:
  1936. code.putln("%s->obj = NULL;" % view)
  1937. def getbuffer_error_cleanup(self, code):
  1938. py_buffer, obj_type = self._get_py_buffer_info()
  1939. view = py_buffer.cname
  1940. if obj_type and obj_type.is_pyobject:
  1941. code.putln("if (%s->obj != NULL) {" % view)
  1942. code.put_gotref("%s->obj" % view)
  1943. code.put_decref_clear("%s->obj" % view, obj_type)
  1944. code.putln("}")
  1945. else:
  1946. code.putln("Py_CLEAR(%s->obj);" % view)
  1947. def getbuffer_normal_cleanup(self, code):
  1948. py_buffer, obj_type = self._get_py_buffer_info()
  1949. view = py_buffer.cname
  1950. if obj_type and obj_type.is_pyobject:
  1951. code.putln("if (%s->obj == Py_None) {" % view)
  1952. code.put_gotref("%s->obj" % view)
  1953. code.put_decref_clear("%s->obj" % view, obj_type)
  1954. code.putln("}")
  1955. def get_preprocessor_guard(self):
  1956. if not self.entry.is_special:
  1957. return None
  1958. name = self.entry.name
  1959. slot = TypeSlots.method_name_to_slot.get(name)
  1960. if not slot:
  1961. return None
  1962. if name == '__long__' and not self.entry.scope.lookup_here('__int__'):
  1963. return None
  1964. if name in ("__getbuffer__", "__releasebuffer__") and self.entry.scope.is_c_class_scope:
  1965. return None
  1966. return slot.preprocessor_guard_code()
  1967. class CFuncDefNode(FuncDefNode):
  1968. # C function definition.
  1969. #
  1970. # modifiers ['inline']
  1971. # visibility 'private' or 'public' or 'extern'
  1972. # base_type CBaseTypeNode
  1973. # declarator CDeclaratorNode
  1974. # cfunc_declarator the CFuncDeclarator of this function
  1975. # (this is also available through declarator or a
  1976. # base thereof)
  1977. # body StatListNode
  1978. # api boolean
  1979. # decorators [DecoratorNode] list of decorators
  1980. #
  1981. # with_gil boolean Acquire GIL around body
  1982. # type CFuncType
  1983. # py_func wrapper for calling from Python
  1984. # overridable whether or not this is a cpdef function
  1985. # inline_in_pxd whether this is an inline function in a pxd file
  1986. # template_declaration String or None Used for c++ class methods
  1987. # is_const_method whether this is a const method
  1988. # is_static_method whether this is a static method
  1989. # is_c_class_method whether this is a cclass method
  1990. child_attrs = ["base_type", "declarator", "body", "py_func_stat"]
  1991. inline_in_pxd = False
  1992. decorators = None
  1993. directive_locals = None
  1994. directive_returns = None
  1995. override = None
  1996. template_declaration = None
  1997. is_const_method = False
  1998. py_func_stat = None
  1999. def unqualified_name(self):
  2000. return self.entry.name
  2001. @property
  2002. def code_object(self):
  2003. # share the CodeObject with the cpdef wrapper (if available)
  2004. return self.py_func.code_object if self.py_func else None
  2005. def analyse_declarations(self, env):
  2006. self.is_c_class_method = env.is_c_class_scope
  2007. if self.directive_locals is None:
  2008. self.directive_locals = {}
  2009. self.directive_locals.update(env.directives.get('locals', {}))
  2010. if self.directive_returns is not None:
  2011. base_type = self.directive_returns.analyse_as_type(env)
  2012. if base_type is None:
  2013. error(self.directive_returns.pos, "Not a type")
  2014. base_type = PyrexTypes.error_type
  2015. else:
  2016. base_type = self.base_type.analyse(env)
  2017. self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod')
  2018. # The 2 here is because we need both function and argument names.
  2019. if isinstance(self.declarator, CFuncDeclaratorNode):
  2020. name_declarator, type = self.declarator.analyse(
  2021. base_type, env, nonempty=2 * (self.body is not None),
  2022. directive_locals=self.directive_locals, visibility=self.visibility)
  2023. else:
  2024. name_declarator, type = self.declarator.analyse(
  2025. base_type, env, nonempty=2 * (self.body is not None), visibility=self.visibility)
  2026. if not type.is_cfunction:
  2027. error(self.pos, "Suite attached to non-function declaration")
  2028. # Remember the actual type according to the function header
  2029. # written here, because the type in the symbol table entry
  2030. # may be different if we're overriding a C method inherited
  2031. # from the base type of an extension type.
  2032. self.type = type
  2033. type.is_overridable = self.overridable
  2034. declarator = self.declarator
  2035. while not hasattr(declarator, 'args'):
  2036. declarator = declarator.base
  2037. self.cfunc_declarator = declarator
  2038. self.args = declarator.args
  2039. opt_arg_count = self.cfunc_declarator.optional_arg_count
  2040. if (self.visibility == 'public' or self.api) and opt_arg_count:
  2041. error(self.cfunc_declarator.pos,
  2042. "Function with optional arguments may not be declared public or api")
  2043. if type.exception_check == '+' and self.visibility != 'extern':
  2044. warning(self.cfunc_declarator.pos,
  2045. "Only extern functions can throw C++ exceptions.")
  2046. for formal_arg, type_arg in zip(self.args, type.args):
  2047. self.align_argument_type(env, type_arg)
  2048. formal_arg.type = type_arg.type
  2049. formal_arg.name = type_arg.name
  2050. formal_arg.cname = type_arg.cname
  2051. self._validate_type_visibility(type_arg.type, type_arg.pos, env)
  2052. if type_arg.type.is_fused:
  2053. self.has_fused_arguments = True
  2054. if type_arg.type.is_buffer and 'inline' in self.modifiers:
  2055. warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
  2056. if type_arg.type.is_buffer or type_arg.type.is_pythran_expr:
  2057. if self.type.nogil:
  2058. error(formal_arg.pos,
  2059. "Buffer may not be acquired without the GIL. Consider using memoryview slices instead.")
  2060. elif 'inline' in self.modifiers:
  2061. warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
  2062. self._validate_type_visibility(type.return_type, self.pos, env)
  2063. name = name_declarator.name
  2064. cname = name_declarator.cname
  2065. type.is_const_method = self.is_const_method
  2066. type.is_static_method = self.is_static_method
  2067. self.entry = env.declare_cfunction(
  2068. name, type, self.pos,
  2069. cname=cname, visibility=self.visibility, api=self.api,
  2070. defining=self.body is not None, modifiers=self.modifiers,
  2071. overridable=self.overridable)
  2072. self.entry.inline_func_in_pxd = self.inline_in_pxd
  2073. self.return_type = type.return_type
  2074. if self.return_type.is_array and self.visibility != 'extern':
  2075. error(self.pos, "Function cannot return an array")
  2076. if self.return_type.is_cpp_class:
  2077. self.return_type.check_nullary_constructor(self.pos, "used as a return value")
  2078. if self.overridable and not env.is_module_scope and not self.is_static_method:
  2079. if len(self.args) < 1 or not self.args[0].type.is_pyobject:
  2080. # An error will be produced in the cdef function
  2081. self.overridable = False
  2082. self.declare_cpdef_wrapper(env)
  2083. self.create_local_scope(env)
  2084. def declare_cpdef_wrapper(self, env):
  2085. if self.overridable:
  2086. if self.is_static_method:
  2087. # TODO(robertwb): Finish this up, perhaps via more function refactoring.
  2088. error(self.pos, "static cpdef methods not yet supported")
  2089. name = self.entry.name
  2090. py_func_body = self.call_self_node(is_module_scope=env.is_module_scope)
  2091. if self.is_static_method:
  2092. from .ExprNodes import NameNode
  2093. decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))]
  2094. decorators[0].decorator.analyse_types(env)
  2095. else:
  2096. decorators = []
  2097. self.py_func = DefNode(pos=self.pos,
  2098. name=self.entry.name,
  2099. args=self.args,
  2100. star_arg=None,
  2101. starstar_arg=None,
  2102. doc=self.doc,
  2103. body=py_func_body,
  2104. decorators=decorators,
  2105. is_wrapper=1)
  2106. self.py_func.is_module_scope = env.is_module_scope
  2107. self.py_func.analyse_declarations(env)
  2108. self.py_func.entry.is_overridable = True
  2109. self.py_func_stat = StatListNode(self.pos, stats=[self.py_func])
  2110. self.py_func.type = PyrexTypes.py_object_type
  2111. self.entry.as_variable = self.py_func.entry
  2112. self.entry.used = self.entry.as_variable.used = True
  2113. # Reset scope entry the above cfunction
  2114. env.entries[name] = self.entry
  2115. if (not self.entry.is_final_cmethod and
  2116. (not env.is_module_scope or Options.lookup_module_cpdef)):
  2117. self.override = OverrideCheckNode(self.pos, py_func=self.py_func)
  2118. self.body = StatListNode(self.pos, stats=[self.override, self.body])
  2119. def _validate_type_visibility(self, type, pos, env):
  2120. """
  2121. Ensure that types used in cdef functions are public or api, or
  2122. defined in a C header.
  2123. """
  2124. public_or_api = (self.visibility == 'public' or self.api)
  2125. entry = getattr(type, 'entry', None)
  2126. if public_or_api and entry and env.is_module_scope:
  2127. if not (entry.visibility in ('public', 'extern') or
  2128. entry.api or entry.in_cinclude):
  2129. error(pos, "Function declared public or api may not have private types")
  2130. def call_self_node(self, omit_optional_args=0, is_module_scope=0):
  2131. from . import ExprNodes
  2132. args = self.type.args
  2133. if omit_optional_args:
  2134. args = args[:len(args) - self.type.optional_arg_count]
  2135. arg_names = [arg.name for arg in args]
  2136. if is_module_scope:
  2137. cfunc = ExprNodes.NameNode(self.pos, name=self.entry.name)
  2138. call_arg_names = arg_names
  2139. skip_dispatch = Options.lookup_module_cpdef
  2140. elif self.type.is_static_method:
  2141. class_entry = self.entry.scope.parent_type.entry
  2142. class_node = ExprNodes.NameNode(self.pos, name=class_entry.name)
  2143. class_node.entry = class_entry
  2144. cfunc = ExprNodes.AttributeNode(self.pos, obj=class_node, attribute=self.entry.name)
  2145. # Calling static c(p)def methods on an instance disallowed.
  2146. # TODO(robertwb): Support by passing self to check for override?
  2147. skip_dispatch = True
  2148. else:
  2149. type_entry = self.type.args[0].type.entry
  2150. type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
  2151. type_arg.entry = type_entry
  2152. cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
  2153. skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
  2154. c_call = ExprNodes.SimpleCallNode(
  2155. self.pos,
  2156. function=cfunc,
  2157. args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names],
  2158. wrapper_call=skip_dispatch)
  2159. return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)
  2160. def declare_arguments(self, env):
  2161. for arg in self.type.args:
  2162. if not arg.name:
  2163. error(arg.pos, "Missing argument name")
  2164. self.declare_argument(env, arg)
  2165. def need_gil_acquisition(self, lenv):
  2166. return self.type.with_gil
  2167. def nogil_check(self, env):
  2168. type = self.type
  2169. with_gil = type.with_gil
  2170. if type.nogil and not with_gil:
  2171. if type.return_type.is_pyobject:
  2172. error(self.pos,
  2173. "Function with Python return type cannot be declared nogil")
  2174. for entry in self.local_scope.var_entries:
  2175. if entry.type.is_pyobject and not entry.in_with_gil_block:
  2176. error(self.pos, "Function declared nogil has Python locals or temporaries")
  2177. def analyse_expressions(self, env):
  2178. self.local_scope.directives = env.directives
  2179. if self.py_func_stat is not None:
  2180. # this will also analyse the default values and the function name assignment
  2181. self.py_func_stat = self.py_func_stat.analyse_expressions(env)
  2182. elif self.py_func is not None:
  2183. # this will also analyse the default values
  2184. self.py_func = self.py_func.analyse_expressions(env)
  2185. else:
  2186. self.analyse_default_values(env)
  2187. self.analyse_annotations(env)
  2188. self.acquire_gil = self.need_gil_acquisition(self.local_scope)
  2189. return self
  2190. def needs_assignment_synthesis(self, env, code=None):
  2191. return False
  2192. def generate_function_header(self, code, with_pymethdef, with_opt_args=1, with_dispatch=1, cname=None):
  2193. scope = self.local_scope
  2194. arg_decls = []
  2195. type = self.type
  2196. for arg in type.args[:len(type.args)-type.optional_arg_count]:
  2197. arg_decl = arg.declaration_code()
  2198. entry = scope.lookup(arg.name)
  2199. if not entry.cf_used:
  2200. arg_decl = 'CYTHON_UNUSED %s' % arg_decl
  2201. arg_decls.append(arg_decl)
  2202. if with_dispatch and self.overridable:
  2203. dispatch_arg = PyrexTypes.c_int_type.declaration_code(
  2204. Naming.skip_dispatch_cname)
  2205. if self.override:
  2206. arg_decls.append(dispatch_arg)
  2207. else:
  2208. arg_decls.append('CYTHON_UNUSED %s' % dispatch_arg)
  2209. if type.optional_arg_count and with_opt_args:
  2210. arg_decls.append(type.op_arg_struct.declaration_code(Naming.optional_args_cname))
  2211. if type.has_varargs:
  2212. arg_decls.append("...")
  2213. if not arg_decls:
  2214. arg_decls = ["void"]
  2215. if cname is None:
  2216. cname = self.entry.func_cname
  2217. entity = type.function_header_code(cname, ', '.join(arg_decls))
  2218. if self.entry.visibility == 'private' and '::' not in cname:
  2219. storage_class = "static "
  2220. else:
  2221. storage_class = ""
  2222. dll_linkage = None
  2223. modifiers = code.build_function_modifiers(self.entry.func_modifiers)
  2224. header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage)
  2225. #print (storage_class, modifiers, header)
  2226. needs_proto = self.is_c_class_method
  2227. if self.template_declaration:
  2228. if needs_proto:
  2229. code.globalstate.parts['module_declarations'].putln(self.template_declaration)
  2230. code.putln(self.template_declaration)
  2231. if needs_proto:
  2232. code.globalstate.parts['module_declarations'].putln(
  2233. "%s%s%s; /* proto*/" % (storage_class, modifiers, header))
  2234. code.putln("%s%s%s {" % (storage_class, modifiers, header))
  2235. def generate_argument_declarations(self, env, code):
  2236. scope = self.local_scope
  2237. for arg in self.args:
  2238. if arg.default:
  2239. entry = scope.lookup(arg.name)
  2240. if self.override or entry.cf_used:
  2241. result = arg.calculate_default_value_code(code)
  2242. code.putln('%s = %s;' % (
  2243. arg.type.declaration_code(arg.cname), result))
  2244. def generate_keyword_list(self, code):
  2245. pass
  2246. def generate_argument_parsing_code(self, env, code):
  2247. i = 0
  2248. used = 0
  2249. scope = self.local_scope
  2250. if self.type.optional_arg_count:
  2251. code.putln('if (%s) {' % Naming.optional_args_cname)
  2252. for arg in self.args:
  2253. if arg.default:
  2254. entry = scope.lookup(arg.name)
  2255. if self.override or entry.cf_used:
  2256. code.putln('if (%s->%sn > %s) {' %
  2257. (Naming.optional_args_cname,
  2258. Naming.pyrex_prefix, i))
  2259. declarator = arg.declarator
  2260. while not hasattr(declarator, 'name'):
  2261. declarator = declarator.base
  2262. code.putln('%s = %s->%s;' %
  2263. (arg.cname, Naming.optional_args_cname,
  2264. self.type.opt_arg_cname(declarator.name)))
  2265. used += 1
  2266. i += 1
  2267. for _ in range(used):
  2268. code.putln('}')
  2269. code.putln('}')
  2270. # Move arguments into closure if required
  2271. def put_into_closure(entry):
  2272. if entry.in_closure and not arg.default:
  2273. code.putln('%s = %s;' % (entry.cname, entry.original_cname))
  2274. if entry.type.is_memoryviewslice:
  2275. code.put_incref_memoryviewslice(entry.cname, have_gil=True)
  2276. else:
  2277. code.put_var_incref(entry)
  2278. code.put_var_giveref(entry)
  2279. for arg in self.args:
  2280. put_into_closure(scope.lookup_here(arg.name))
  2281. def generate_argument_conversion_code(self, code):
  2282. pass
  2283. def generate_argument_type_tests(self, code):
  2284. # Generate type tests for args whose type in a parent
  2285. # class is a supertype of the declared type.
  2286. for arg in self.type.args:
  2287. if arg.needs_type_test:
  2288. self.generate_arg_type_test(arg, code)
  2289. elif arg.type.is_pyobject and not arg.accept_none:
  2290. self.generate_arg_none_check(arg, code)
  2291. def generate_execution_code(self, code):
  2292. if code.globalstate.directives['linetrace']:
  2293. code.mark_pos(self.pos)
  2294. code.putln("") # generate line tracing code
  2295. super(CFuncDefNode, self).generate_execution_code(code)
  2296. if self.py_func_stat:
  2297. self.py_func_stat.generate_execution_code(code)
  2298. def error_value(self):
  2299. if self.return_type.is_pyobject:
  2300. return "0"
  2301. else:
  2302. #return None
  2303. return self.entry.type.exception_value
  2304. def caller_will_check_exceptions(self):
  2305. return self.entry.type.exception_check
  2306. def generate_wrapper_functions(self, code):
  2307. # If the C signature of a function has changed, we need to generate
  2308. # wrappers to put in the slots here.
  2309. k = 0
  2310. entry = self.entry
  2311. func_type = entry.type
  2312. while entry.prev_entry is not None:
  2313. k += 1
  2314. entry = entry.prev_entry
  2315. entry.func_cname = "%s%swrap_%s" % (self.entry.func_cname, Naming.pyrex_prefix, k)
  2316. code.putln()
  2317. self.generate_function_header(
  2318. code, 0,
  2319. with_dispatch=entry.type.is_overridable,
  2320. with_opt_args=entry.type.optional_arg_count,
  2321. cname=entry.func_cname)
  2322. if not self.return_type.is_void:
  2323. code.put('return ')
  2324. args = self.type.args
  2325. arglist = [arg.cname for arg in args[:len(args)-self.type.optional_arg_count]]
  2326. if entry.type.is_overridable:
  2327. arglist.append(Naming.skip_dispatch_cname)
  2328. elif func_type.is_overridable:
  2329. arglist.append('0')
  2330. if entry.type.optional_arg_count:
  2331. arglist.append(Naming.optional_args_cname)
  2332. elif func_type.optional_arg_count:
  2333. arglist.append('NULL')
  2334. code.putln('%s(%s);' % (self.entry.func_cname, ', '.join(arglist)))
  2335. code.putln('}')
  2336. class PyArgDeclNode(Node):
  2337. # Argument which must be a Python object (used
  2338. # for * and ** arguments).
  2339. #
  2340. # name string
  2341. # entry Symtab.Entry
  2342. # annotation ExprNode or None Py3 argument annotation
  2343. child_attrs = []
  2344. is_self_arg = False
  2345. is_type_arg = False
  2346. def generate_function_definitions(self, env, code):
  2347. self.entry.generate_function_definitions(env, code)
  2348. class DecoratorNode(Node):
  2349. # A decorator
  2350. #
  2351. # decorator NameNode or CallNode or AttributeNode
  2352. child_attrs = ['decorator']
  2353. class DefNode(FuncDefNode):
  2354. # A Python function definition.
  2355. #
  2356. # name string the Python name of the function
  2357. # lambda_name string the internal name of a lambda 'function'
  2358. # decorators [DecoratorNode] list of decorators
  2359. # args [CArgDeclNode] formal arguments
  2360. # doc EncodedString or None
  2361. # body StatListNode
  2362. # return_type_annotation
  2363. # ExprNode or None the Py3 return type annotation
  2364. #
  2365. # The following subnode is constructed internally
  2366. # when the def statement is inside a Python class definition.
  2367. #
  2368. # fused_py_func DefNode The original fused cpdef DefNode
  2369. # (in case this is a specialization)
  2370. # specialized_cpdefs [DefNode] list of specialized cpdef DefNodes
  2371. # py_cfunc_node PyCFunctionNode/InnerFunctionNode The PyCFunction to create and assign
  2372. #
  2373. # decorator_indirection IndirectionNode Used to remove __Pyx_Method_ClassMethod for fused functions
  2374. child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "return_type_annotation"]
  2375. outer_attrs = ["decorators", "return_type_annotation"]
  2376. is_staticmethod = False
  2377. is_classmethod = False
  2378. lambda_name = None
  2379. reqd_kw_flags_cname = "0"
  2380. is_wrapper = 0
  2381. no_assignment_synthesis = 0
  2382. decorators = None
  2383. return_type_annotation = None
  2384. entry = None
  2385. acquire_gil = 0
  2386. self_in_stararg = 0
  2387. py_cfunc_node = None
  2388. requires_classobj = False
  2389. defaults_struct = None # Dynamic kwrds structure name
  2390. doc = None
  2391. fused_py_func = False
  2392. specialized_cpdefs = None
  2393. py_wrapper = None
  2394. py_wrapper_required = True
  2395. func_cname = None
  2396. defaults_getter = None
  2397. def __init__(self, pos, **kwds):
  2398. FuncDefNode.__init__(self, pos, **kwds)
  2399. k = rk = r = 0
  2400. for arg in self.args:
  2401. if arg.kw_only:
  2402. k += 1
  2403. if not arg.default:
  2404. rk += 1
  2405. if not arg.default:
  2406. r += 1
  2407. self.num_kwonly_args = k
  2408. self.num_required_kw_args = rk
  2409. self.num_required_args = r
  2410. def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, except_val=None, modifiers=None,
  2411. nogil=False, with_gil=False):
  2412. if self.star_arg:
  2413. error(self.star_arg.pos, "cdef function cannot have star argument")
  2414. if self.starstar_arg:
  2415. error(self.starstar_arg.pos, "cdef function cannot have starstar argument")
  2416. exception_value, exception_check = except_val or (None, False)
  2417. if cfunc is None:
  2418. cfunc_args = []
  2419. for formal_arg in self.args:
  2420. name_declarator, type = formal_arg.analyse(scope, nonempty=1)
  2421. cfunc_args.append(PyrexTypes.CFuncTypeArg(name=name_declarator.name,
  2422. cname=None,
  2423. annotation=formal_arg.annotation,
  2424. type=py_object_type,
  2425. pos=formal_arg.pos))
  2426. cfunc_type = PyrexTypes.CFuncType(return_type=py_object_type,
  2427. args=cfunc_args,
  2428. has_varargs=False,
  2429. exception_value=None,
  2430. exception_check=exception_check,
  2431. nogil=nogil,
  2432. with_gil=with_gil,
  2433. is_overridable=overridable)
  2434. cfunc = CVarDefNode(self.pos, type=cfunc_type)
  2435. else:
  2436. if scope is None:
  2437. scope = cfunc.scope
  2438. cfunc_type = cfunc.type
  2439. if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs:
  2440. error(self.pos, "wrong number of arguments")
  2441. error(cfunc.pos, "previous declaration here")
  2442. for i, (formal_arg, type_arg) in enumerate(zip(self.args, cfunc_type.args)):
  2443. name_declarator, type = formal_arg.analyse(scope, nonempty=1,
  2444. is_self_arg=(i == 0 and scope.is_c_class_scope))
  2445. if type is None or type is PyrexTypes.py_object_type:
  2446. formal_arg.type = type_arg.type
  2447. formal_arg.name_declarator = name_declarator
  2448. if exception_value is None and cfunc_type.exception_value is not None:
  2449. from .ExprNodes import ConstNode
  2450. exception_value = ConstNode(
  2451. self.pos, value=cfunc_type.exception_value, type=cfunc_type.return_type)
  2452. declarator = CFuncDeclaratorNode(self.pos,
  2453. base=CNameDeclaratorNode(self.pos, name=self.name, cname=None),
  2454. args=self.args,
  2455. has_varargs=False,
  2456. exception_check=cfunc_type.exception_check,
  2457. exception_value=exception_value,
  2458. with_gil=cfunc_type.with_gil,
  2459. nogil=cfunc_type.nogil)
  2460. return CFuncDefNode(self.pos,
  2461. modifiers=modifiers or [],
  2462. base_type=CAnalysedBaseTypeNode(self.pos, type=cfunc_type.return_type),
  2463. declarator=declarator,
  2464. body=self.body,
  2465. doc=self.doc,
  2466. overridable=cfunc_type.is_overridable,
  2467. type=cfunc_type,
  2468. with_gil=cfunc_type.with_gil,
  2469. nogil=cfunc_type.nogil,
  2470. visibility='private',
  2471. api=False,
  2472. directive_locals=getattr(cfunc, 'directive_locals', {}),
  2473. directive_returns=returns)
  2474. def is_cdef_func_compatible(self):
  2475. """Determines if the function's signature is compatible with a
  2476. cdef function. This can be used before calling
  2477. .as_cfunction() to see if that will be successful.
  2478. """
  2479. if self.needs_closure:
  2480. return False
  2481. if self.star_arg or self.starstar_arg:
  2482. return False
  2483. return True
  2484. def analyse_declarations(self, env):
  2485. if self.decorators:
  2486. for decorator in self.decorators:
  2487. func = decorator.decorator
  2488. if func.is_name:
  2489. self.is_classmethod |= func.name == 'classmethod'
  2490. self.is_staticmethod |= func.name == 'staticmethod'
  2491. if self.is_classmethod and env.lookup_here('classmethod'):
  2492. # classmethod() was overridden - not much we can do here ...
  2493. self.is_classmethod = False
  2494. if self.is_staticmethod and env.lookup_here('staticmethod'):
  2495. # staticmethod() was overridden - not much we can do here ...
  2496. self.is_staticmethod = False
  2497. if self.name == '__new__' and env.is_py_class_scope:
  2498. self.is_staticmethod = 1
  2499. self.analyse_argument_types(env)
  2500. if self.name == '<lambda>':
  2501. self.declare_lambda_function(env)
  2502. else:
  2503. self.declare_pyfunction(env)
  2504. self.analyse_signature(env)
  2505. self.return_type = self.entry.signature.return_type()
  2506. # if a signature annotation provides a more specific return object type, use it
  2507. if self.return_type is py_object_type and self.return_type_annotation:
  2508. if env.directives['annotation_typing'] and not self.entry.is_special:
  2509. _, return_type = analyse_type_annotation(self.return_type_annotation, env)
  2510. if return_type and return_type.is_pyobject:
  2511. self.return_type = return_type
  2512. self.create_local_scope(env)
  2513. self.py_wrapper = DefNodeWrapper(
  2514. self.pos,
  2515. target=self,
  2516. name=self.entry.name,
  2517. args=self.args,
  2518. star_arg=self.star_arg,
  2519. starstar_arg=self.starstar_arg,
  2520. return_type=self.return_type)
  2521. self.py_wrapper.analyse_declarations(env)
  2522. def analyse_argument_types(self, env):
  2523. self.directive_locals = env.directives.get('locals', {})
  2524. allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
  2525. f2s = env.fused_to_specific
  2526. env.fused_to_specific = None
  2527. for arg in self.args:
  2528. if hasattr(arg, 'name'):
  2529. name_declarator = None
  2530. else:
  2531. base_type = arg.base_type.analyse(env)
  2532. # If we hare in pythran mode and we got a buffer supported by
  2533. # Pythran, we change this node to a fused type
  2534. if has_np_pythran(env) and base_type.is_pythran_expr:
  2535. base_type = PyrexTypes.FusedType([
  2536. base_type,
  2537. #PyrexTypes.PythranExpr(pythran_type(self.type, "numpy_texpr")),
  2538. base_type.org_buffer])
  2539. name_declarator, type = \
  2540. arg.declarator.analyse(base_type, env)
  2541. arg.name = name_declarator.name
  2542. arg.type = type
  2543. if type.is_fused:
  2544. self.has_fused_arguments = True
  2545. self.align_argument_type(env, arg)
  2546. if name_declarator and name_declarator.cname:
  2547. error(self.pos, "Python function argument cannot have C name specification")
  2548. arg.type = arg.type.as_argument_type()
  2549. arg.hdr_type = None
  2550. arg.needs_conversion = 0
  2551. arg.needs_type_test = 0
  2552. arg.is_generic = 1
  2553. if arg.type.is_pyobject or arg.type.is_buffer or arg.type.is_memoryviewslice:
  2554. if arg.or_none:
  2555. arg.accept_none = True
  2556. elif arg.not_none:
  2557. arg.accept_none = False
  2558. elif (arg.type.is_extension_type or arg.type.is_builtin_type
  2559. or arg.type.is_buffer or arg.type.is_memoryviewslice):
  2560. if arg.default and arg.default.constant_result is None:
  2561. # special case: def func(MyType obj = None)
  2562. arg.accept_none = True
  2563. else:
  2564. # default depends on compiler directive
  2565. arg.accept_none = allow_none_for_extension_args
  2566. else:
  2567. # probably just a plain 'object'
  2568. arg.accept_none = True
  2569. else:
  2570. arg.accept_none = True # won't be used, but must be there
  2571. if arg.not_none:
  2572. error(arg.pos, "Only Python type arguments can have 'not None'")
  2573. if arg.or_none:
  2574. error(arg.pos, "Only Python type arguments can have 'or None'")
  2575. env.fused_to_specific = f2s
  2576. if has_np_pythran(env):
  2577. self.np_args_idx = [i for i,a in enumerate(self.args) if a.type.is_numpy_buffer]
  2578. else:
  2579. self.np_args_idx = []
  2580. def analyse_signature(self, env):
  2581. if self.entry.is_special:
  2582. if self.decorators:
  2583. error(self.pos, "special functions of cdef classes cannot have decorators")
  2584. self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg)
  2585. elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg):
  2586. # Use the simpler calling signature for zero- and one-argument functions.
  2587. if self.entry.signature is TypeSlots.pyfunction_signature:
  2588. if len(self.args) == 0:
  2589. self.entry.signature = TypeSlots.pyfunction_noargs
  2590. elif len(self.args) == 1:
  2591. if self.args[0].default is None and not self.args[0].kw_only:
  2592. self.entry.signature = TypeSlots.pyfunction_onearg
  2593. elif self.entry.signature is TypeSlots.pymethod_signature:
  2594. if len(self.args) == 1:
  2595. self.entry.signature = TypeSlots.unaryfunc
  2596. elif len(self.args) == 2:
  2597. if self.args[1].default is None and not self.args[1].kw_only:
  2598. self.entry.signature = TypeSlots.ibinaryfunc
  2599. sig = self.entry.signature
  2600. nfixed = sig.num_fixed_args()
  2601. if (sig is TypeSlots.pymethod_signature and nfixed == 1
  2602. and len(self.args) == 0 and self.star_arg):
  2603. # this is the only case where a diverging number of
  2604. # arguments is not an error - when we have no explicit
  2605. # 'self' parameter as in method(*args)
  2606. sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used
  2607. self.self_in_stararg = 1
  2608. nfixed = 0
  2609. if self.is_staticmethod and env.is_c_class_scope:
  2610. nfixed = 0
  2611. self.self_in_stararg = True # FIXME: why for staticmethods?
  2612. self.entry.signature = sig = copy.copy(sig)
  2613. sig.fixed_arg_format = "*"
  2614. sig.is_staticmethod = True
  2615. sig.has_generic_args = True
  2616. if ((self.is_classmethod or self.is_staticmethod) and
  2617. self.has_fused_arguments and env.is_c_class_scope):
  2618. del self.decorator_indirection.stats[:]
  2619. for i in range(min(nfixed, len(self.args))):
  2620. arg = self.args[i]
  2621. arg.is_generic = 0
  2622. if sig.is_self_arg(i) and not self.is_staticmethod:
  2623. if self.is_classmethod:
  2624. arg.is_type_arg = 1
  2625. arg.hdr_type = arg.type = Builtin.type_type
  2626. else:
  2627. arg.is_self_arg = 1
  2628. arg.hdr_type = arg.type = env.parent_type
  2629. arg.needs_conversion = 0
  2630. else:
  2631. arg.hdr_type = sig.fixed_arg_type(i)
  2632. if not arg.type.same_as(arg.hdr_type):
  2633. if arg.hdr_type.is_pyobject and arg.type.is_pyobject:
  2634. arg.needs_type_test = 1
  2635. else:
  2636. arg.needs_conversion = 1
  2637. if arg.needs_conversion:
  2638. arg.hdr_cname = Naming.arg_prefix + arg.name
  2639. else:
  2640. arg.hdr_cname = Naming.var_prefix + arg.name
  2641. if nfixed > len(self.args):
  2642. self.bad_signature()
  2643. return
  2644. elif nfixed < len(self.args):
  2645. if not sig.has_generic_args:
  2646. self.bad_signature()
  2647. for arg in self.args:
  2648. if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type):
  2649. arg.needs_type_test = 1
  2650. def bad_signature(self):
  2651. sig = self.entry.signature
  2652. expected_str = "%d" % sig.num_fixed_args()
  2653. if sig.has_generic_args:
  2654. expected_str += " or more"
  2655. name = self.name
  2656. if name.startswith("__") and name.endswith("__"):
  2657. desc = "Special method"
  2658. else:
  2659. desc = "Method"
  2660. error(self.pos, "%s %s has wrong number of arguments (%d declared, %s expected)" % (
  2661. desc, self.name, len(self.args), expected_str))
  2662. def declare_pyfunction(self, env):
  2663. #print "DefNode.declare_pyfunction:", self.name, "in", env ###
  2664. name = self.name
  2665. entry = env.lookup_here(name)
  2666. if entry:
  2667. if entry.is_final_cmethod and not env.parent_type.is_final_type:
  2668. error(self.pos, "Only final types can have final Python (def/cpdef) methods")
  2669. if entry.type.is_cfunction and not entry.is_builtin_cmethod and not self.is_wrapper:
  2670. warning(self.pos, "Overriding cdef method with def method.", 5)
  2671. entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper)
  2672. self.entry = entry
  2673. prefix = env.next_id(env.scope_prefix)
  2674. self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name
  2675. if Options.docstrings:
  2676. entry.doc = embed_position(self.pos, self.doc)
  2677. entry.doc_cname = Naming.funcdoc_prefix + prefix + name
  2678. if entry.is_special:
  2679. if entry.name in TypeSlots.invisible or not entry.doc or (
  2680. entry.name in '__getattr__' and env.directives['fast_getattr']):
  2681. entry.wrapperbase_cname = None
  2682. else:
  2683. entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name
  2684. else:
  2685. entry.doc = None
  2686. def declare_lambda_function(self, env):
  2687. entry = env.declare_lambda_function(self.lambda_name, self.pos)
  2688. entry.doc = None
  2689. self.entry = entry
  2690. self.entry.pyfunc_cname = entry.cname
  2691. def declare_arguments(self, env):
  2692. for arg in self.args:
  2693. if not arg.name:
  2694. error(arg.pos, "Missing argument name")
  2695. if arg.needs_conversion:
  2696. arg.entry = env.declare_var(arg.name, arg.type, arg.pos)
  2697. if arg.type.is_pyobject:
  2698. arg.entry.init = "0"
  2699. else:
  2700. arg.entry = self.declare_argument(env, arg)
  2701. arg.entry.is_arg = 1
  2702. arg.entry.used = 1
  2703. arg.entry.is_self_arg = arg.is_self_arg
  2704. self.declare_python_arg(env, self.star_arg)
  2705. self.declare_python_arg(env, self.starstar_arg)
  2706. def declare_python_arg(self, env, arg):
  2707. if arg:
  2708. if env.directives['infer_types'] != False:
  2709. type = PyrexTypes.unspecified_type
  2710. else:
  2711. type = py_object_type
  2712. entry = env.declare_var(arg.name, type, arg.pos)
  2713. entry.is_arg = 1
  2714. entry.used = 1
  2715. entry.init = "0"
  2716. entry.xdecref_cleanup = 1
  2717. arg.entry = entry
  2718. def analyse_expressions(self, env):
  2719. self.local_scope.directives = env.directives
  2720. self.analyse_default_values(env)
  2721. self.analyse_annotations(env)
  2722. if self.return_type_annotation:
  2723. self.return_type_annotation = self.analyse_annotation(env, self.return_type_annotation)
  2724. if not self.needs_assignment_synthesis(env) and self.decorators:
  2725. for decorator in self.decorators[::-1]:
  2726. decorator.decorator = decorator.decorator.analyse_expressions(env)
  2727. self.py_wrapper.prepare_argument_coercion(env)
  2728. return self
  2729. def needs_assignment_synthesis(self, env, code=None):
  2730. if self.is_staticmethod:
  2731. return True
  2732. if self.specialized_cpdefs or self.entry.is_fused_specialized:
  2733. return False
  2734. if self.no_assignment_synthesis:
  2735. return False
  2736. if self.entry.is_special:
  2737. return False
  2738. if self.entry.is_anonymous:
  2739. return True
  2740. if env.is_module_scope or env.is_c_class_scope:
  2741. if code is None:
  2742. return self.local_scope.directives['binding']
  2743. else:
  2744. return code.globalstate.directives['binding']
  2745. return env.is_py_class_scope or env.is_closure_scope
  2746. def error_value(self):
  2747. return self.entry.signature.error_value
  2748. def caller_will_check_exceptions(self):
  2749. return self.entry.signature.exception_check
  2750. def generate_function_definitions(self, env, code):
  2751. if self.defaults_getter:
  2752. # defaults getter must never live in class scopes, it's always a module function
  2753. self.defaults_getter.generate_function_definitions(env.global_scope(), code)
  2754. # Before closure cnames are mangled
  2755. if self.py_wrapper_required:
  2756. # func_cname might be modified by @cname
  2757. self.py_wrapper.func_cname = self.entry.func_cname
  2758. self.py_wrapper.generate_function_definitions(env, code)
  2759. FuncDefNode.generate_function_definitions(self, env, code)
  2760. def generate_function_header(self, code, with_pymethdef, proto_only=0):
  2761. if proto_only:
  2762. if self.py_wrapper_required:
  2763. self.py_wrapper.generate_function_header(
  2764. code, with_pymethdef, True)
  2765. return
  2766. arg_code_list = []
  2767. if self.entry.signature.has_dummy_arg:
  2768. self_arg = 'PyObject *%s' % Naming.self_cname
  2769. if not self.needs_outer_scope:
  2770. self_arg = 'CYTHON_UNUSED ' + self_arg
  2771. arg_code_list.append(self_arg)
  2772. def arg_decl_code(arg):
  2773. entry = arg.entry
  2774. if entry.in_closure:
  2775. cname = entry.original_cname
  2776. else:
  2777. cname = entry.cname
  2778. decl = entry.type.declaration_code(cname)
  2779. if not entry.cf_used:
  2780. decl = 'CYTHON_UNUSED ' + decl
  2781. return decl
  2782. for arg in self.args:
  2783. arg_code_list.append(arg_decl_code(arg))
  2784. if self.star_arg:
  2785. arg_code_list.append(arg_decl_code(self.star_arg))
  2786. if self.starstar_arg:
  2787. arg_code_list.append(arg_decl_code(self.starstar_arg))
  2788. if arg_code_list:
  2789. arg_code = ', '.join(arg_code_list)
  2790. else:
  2791. arg_code = 'void' # No arguments
  2792. dc = self.return_type.declaration_code(self.entry.pyfunc_cname)
  2793. decls_code = code.globalstate['decls']
  2794. preprocessor_guard = self.get_preprocessor_guard()
  2795. if preprocessor_guard:
  2796. decls_code.putln(preprocessor_guard)
  2797. decls_code.putln(
  2798. "static %s(%s); /* proto */" % (dc, arg_code))
  2799. if preprocessor_guard:
  2800. decls_code.putln("#endif")
  2801. code.putln("static %s(%s) {" % (dc, arg_code))
  2802. def generate_argument_declarations(self, env, code):
  2803. pass
  2804. def generate_keyword_list(self, code):
  2805. pass
  2806. def generate_argument_parsing_code(self, env, code):
  2807. # Move arguments into closure if required
  2808. def put_into_closure(entry):
  2809. if entry.in_closure:
  2810. code.putln('%s = %s;' % (entry.cname, entry.original_cname))
  2811. if entry.xdecref_cleanup:
  2812. # mostly applies to the starstar arg - this can sometimes be NULL
  2813. # so must be xincrefed instead
  2814. code.put_var_xincref(entry)
  2815. code.put_var_xgiveref(entry)
  2816. else:
  2817. code.put_var_incref(entry)
  2818. code.put_var_giveref(entry)
  2819. for arg in self.args:
  2820. put_into_closure(arg.entry)
  2821. for arg in self.star_arg, self.starstar_arg:
  2822. if arg:
  2823. put_into_closure(arg.entry)
  2824. def generate_argument_type_tests(self, code):
  2825. pass
  2826. class DefNodeWrapper(FuncDefNode):
  2827. # DefNode python wrapper code generator
  2828. defnode = None
  2829. target = None # Target DefNode
  2830. def __init__(self, *args, **kwargs):
  2831. FuncDefNode.__init__(self, *args, **kwargs)
  2832. self.num_kwonly_args = self.target.num_kwonly_args
  2833. self.num_required_kw_args = self.target.num_required_kw_args
  2834. self.num_required_args = self.target.num_required_args
  2835. self.self_in_stararg = self.target.self_in_stararg
  2836. self.signature = None
  2837. def analyse_declarations(self, env):
  2838. target_entry = self.target.entry
  2839. name = self.name
  2840. prefix = env.next_id(env.scope_prefix)
  2841. target_entry.func_cname = Naming.pywrap_prefix + prefix + name
  2842. target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name
  2843. self.signature = target_entry.signature
  2844. self.np_args_idx = self.target.np_args_idx
  2845. def prepare_argument_coercion(self, env):
  2846. # This is only really required for Cython utility code at this time,
  2847. # everything else can be done during code generation. But we expand
  2848. # all utility code here, simply because we cannot easily distinguish
  2849. # different code types.
  2850. for arg in self.args:
  2851. if not arg.type.is_pyobject:
  2852. if not arg.type.create_from_py_utility_code(env):
  2853. pass # will fail later
  2854. elif arg.hdr_type and not arg.hdr_type.is_pyobject:
  2855. if not arg.hdr_type.create_to_py_utility_code(env):
  2856. pass # will fail later
  2857. if self.starstar_arg and not self.starstar_arg.entry.cf_used:
  2858. # we will set the kwargs argument to NULL instead of a new dict
  2859. # and must therefore correct the control flow state
  2860. entry = self.starstar_arg.entry
  2861. entry.xdecref_cleanup = 1
  2862. for ass in entry.cf_assignments:
  2863. if not ass.is_arg and ass.lhs.is_name:
  2864. ass.lhs.cf_maybe_null = True
  2865. def signature_has_nongeneric_args(self):
  2866. argcount = len(self.args)
  2867. if argcount == 0 or (
  2868. argcount == 1 and (self.args[0].is_self_arg or
  2869. self.args[0].is_type_arg)):
  2870. return 0
  2871. return 1
  2872. def signature_has_generic_args(self):
  2873. return self.signature.has_generic_args
  2874. def generate_function_body(self, code):
  2875. args = []
  2876. if self.signature.has_dummy_arg:
  2877. args.append(Naming.self_cname)
  2878. for arg in self.args:
  2879. if arg.hdr_type and not (arg.type.is_memoryviewslice or
  2880. arg.type.is_struct or
  2881. arg.type.is_complex):
  2882. args.append(arg.type.cast_code(arg.entry.cname))
  2883. else:
  2884. args.append(arg.entry.cname)
  2885. if self.star_arg:
  2886. args.append(self.star_arg.entry.cname)
  2887. if self.starstar_arg:
  2888. args.append(self.starstar_arg.entry.cname)
  2889. args = ', '.join(args)
  2890. if not self.return_type.is_void:
  2891. code.put('%s = ' % Naming.retval_cname)
  2892. code.putln('%s(%s);' % (
  2893. self.target.entry.pyfunc_cname, args))
  2894. def generate_function_definitions(self, env, code):
  2895. lenv = self.target.local_scope
  2896. # Generate C code for header and body of function
  2897. code.mark_pos(self.pos)
  2898. code.putln("")
  2899. code.putln("/* Python wrapper */")
  2900. preprocessor_guard = self.target.get_preprocessor_guard()
  2901. if preprocessor_guard:
  2902. code.putln(preprocessor_guard)
  2903. code.enter_cfunc_scope(lenv)
  2904. code.return_from_error_cleanup_label = code.new_label()
  2905. with_pymethdef = (self.target.needs_assignment_synthesis(env, code) or
  2906. self.target.pymethdef_required)
  2907. self.generate_function_header(code, with_pymethdef)
  2908. self.generate_argument_declarations(lenv, code)
  2909. tempvardecl_code = code.insertion_point()
  2910. if self.return_type.is_pyobject:
  2911. retval_init = ' = 0'
  2912. else:
  2913. retval_init = ''
  2914. if not self.return_type.is_void:
  2915. code.putln('%s%s;' % (
  2916. self.return_type.declaration_code(Naming.retval_cname),
  2917. retval_init))
  2918. code.put_declare_refcount_context()
  2919. code.put_setup_refcount_context('%s (wrapper)' % self.name)
  2920. self.generate_argument_parsing_code(lenv, code)
  2921. self.generate_argument_type_tests(code)
  2922. self.generate_function_body(code)
  2923. # ----- Go back and insert temp variable declarations
  2924. tempvardecl_code.put_temp_declarations(code.funcstate)
  2925. code.mark_pos(self.pos)
  2926. code.putln("")
  2927. code.putln("/* function exit code */")
  2928. # ----- Error cleanup
  2929. if code.error_label in code.labels_used:
  2930. code.put_goto(code.return_label)
  2931. code.put_label(code.error_label)
  2932. for cname, type in code.funcstate.all_managed_temps():
  2933. code.put_xdecref(cname, type)
  2934. err_val = self.error_value()
  2935. if err_val is not None:
  2936. code.putln("%s = %s;" % (Naming.retval_cname, err_val))
  2937. # ----- Non-error return cleanup
  2938. code.put_label(code.return_label)
  2939. for entry in lenv.var_entries:
  2940. if entry.is_arg and entry.type.is_pyobject:
  2941. code.put_var_decref(entry)
  2942. code.put_finish_refcount_context()
  2943. if not self.return_type.is_void:
  2944. code.putln("return %s;" % Naming.retval_cname)
  2945. code.putln('}')
  2946. code.exit_cfunc_scope()
  2947. if preprocessor_guard:
  2948. code.putln("#endif /*!(%s)*/" % preprocessor_guard)
  2949. def generate_function_header(self, code, with_pymethdef, proto_only=0):
  2950. arg_code_list = []
  2951. sig = self.signature
  2952. if sig.has_dummy_arg or self.self_in_stararg:
  2953. arg_code = "PyObject *%s" % Naming.self_cname
  2954. if not sig.has_dummy_arg:
  2955. arg_code = 'CYTHON_UNUSED ' + arg_code
  2956. arg_code_list.append(arg_code)
  2957. for arg in self.args:
  2958. if not arg.is_generic:
  2959. if arg.is_self_arg or arg.is_type_arg:
  2960. arg_code_list.append("PyObject *%s" % arg.hdr_cname)
  2961. else:
  2962. arg_code_list.append(
  2963. arg.hdr_type.declaration_code(arg.hdr_cname))
  2964. entry = self.target.entry
  2965. if not entry.is_special and sig.method_flags() == [TypeSlots.method_noargs]:
  2966. arg_code_list.append("CYTHON_UNUSED PyObject *unused")
  2967. if entry.scope.is_c_class_scope and entry.name == "__ipow__":
  2968. arg_code_list.append("CYTHON_UNUSED PyObject *unused")
  2969. if sig.has_generic_args:
  2970. arg_code_list.append(
  2971. "PyObject *%s, PyObject *%s" % (
  2972. Naming.args_cname, Naming.kwds_cname))
  2973. arg_code = ", ".join(arg_code_list)
  2974. # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__'
  2975. mf = ""
  2976. if (entry.name in ("__getbuffer__", "__releasebuffer__")
  2977. and entry.scope.is_c_class_scope):
  2978. mf = "CYTHON_UNUSED "
  2979. with_pymethdef = False
  2980. dc = self.return_type.declaration_code(entry.func_cname)
  2981. header = "static %s%s(%s)" % (mf, dc, arg_code)
  2982. code.putln("%s; /*proto*/" % header)
  2983. if proto_only:
  2984. if self.target.fused_py_func:
  2985. # If we are the specialized version of the cpdef, we still
  2986. # want the prototype for the "fused cpdef", in case we're
  2987. # checking to see if our method was overridden in Python
  2988. self.target.fused_py_func.generate_function_header(
  2989. code, with_pymethdef, proto_only=True)
  2990. return
  2991. if (Options.docstrings and entry.doc and
  2992. not self.target.fused_py_func and
  2993. not entry.scope.is_property_scope and
  2994. (not entry.is_special or entry.wrapperbase_cname)):
  2995. # h_code = code.globalstate['h_code']
  2996. docstr = entry.doc
  2997. if docstr.is_unicode:
  2998. docstr = docstr.as_utf8_string()
  2999. if not (entry.is_special and entry.name in ('__getbuffer__', '__releasebuffer__')):
  3000. code.putln('static char %s[] = %s;' % (
  3001. entry.doc_cname,
  3002. docstr.as_c_string_literal()))
  3003. if entry.is_special:
  3004. code.putln('#if CYTHON_UPDATE_DESCRIPTOR_DOC')
  3005. code.putln(
  3006. "struct wrapperbase %s;" % entry.wrapperbase_cname)
  3007. code.putln('#endif')
  3008. if with_pymethdef or self.target.fused_py_func:
  3009. code.put(
  3010. "static PyMethodDef %s = " % entry.pymethdef_cname)
  3011. code.put_pymethoddef(self.target.entry, ";", allow_skip=False)
  3012. code.putln("%s {" % header)
  3013. def generate_argument_declarations(self, env, code):
  3014. for arg in self.args:
  3015. if arg.is_generic:
  3016. if arg.needs_conversion:
  3017. code.putln("PyObject *%s = 0;" % arg.hdr_cname)
  3018. else:
  3019. code.put_var_declaration(arg.entry)
  3020. for entry in env.var_entries:
  3021. if entry.is_arg:
  3022. code.put_var_declaration(entry)
  3023. def generate_argument_parsing_code(self, env, code):
  3024. # Generate fast equivalent of PyArg_ParseTuple call for
  3025. # generic arguments, if any, including args/kwargs
  3026. old_error_label = code.new_error_label()
  3027. our_error_label = code.error_label
  3028. end_label = code.new_label("argument_unpacking_done")
  3029. has_kwonly_args = self.num_kwonly_args > 0
  3030. has_star_or_kw_args = self.star_arg is not None \
  3031. or self.starstar_arg is not None or has_kwonly_args
  3032. for arg in self.args:
  3033. if not arg.type.is_pyobject:
  3034. if not arg.type.create_from_py_utility_code(env):
  3035. pass # will fail later
  3036. if not self.signature_has_generic_args():
  3037. if has_star_or_kw_args:
  3038. error(self.pos, "This method cannot have * or keyword arguments")
  3039. self.generate_argument_conversion_code(code)
  3040. elif not self.signature_has_nongeneric_args():
  3041. # func(*args) or func(**kw) or func(*args, **kw)
  3042. self.generate_stararg_copy_code(code)
  3043. else:
  3044. self.generate_tuple_and_keyword_parsing_code(self.args, end_label, code)
  3045. code.error_label = old_error_label
  3046. if code.label_used(our_error_label):
  3047. if not code.label_used(end_label):
  3048. code.put_goto(end_label)
  3049. code.put_label(our_error_label)
  3050. if has_star_or_kw_args:
  3051. self.generate_arg_decref(self.star_arg, code)
  3052. if self.starstar_arg:
  3053. if self.starstar_arg.entry.xdecref_cleanup:
  3054. code.put_var_xdecref_clear(self.starstar_arg.entry)
  3055. else:
  3056. code.put_var_decref_clear(self.starstar_arg.entry)
  3057. code.put_add_traceback(self.target.entry.qualified_name)
  3058. code.put_finish_refcount_context()
  3059. code.putln("return %s;" % self.error_value())
  3060. if code.label_used(end_label):
  3061. code.put_label(end_label)
  3062. def generate_arg_xdecref(self, arg, code):
  3063. if arg:
  3064. code.put_var_xdecref_clear(arg.entry)
  3065. def generate_arg_decref(self, arg, code):
  3066. if arg:
  3067. code.put_var_decref_clear(arg.entry)
  3068. def generate_stararg_copy_code(self, code):
  3069. if not self.star_arg:
  3070. code.globalstate.use_utility_code(
  3071. UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
  3072. code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" %
  3073. Naming.args_cname)
  3074. code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % (
  3075. self.name, Naming.args_cname, self.error_value()))
  3076. code.putln("}")
  3077. if self.starstar_arg:
  3078. if self.star_arg or not self.starstar_arg.entry.cf_used:
  3079. kwarg_check = "unlikely(%s)" % Naming.kwds_cname
  3080. else:
  3081. kwarg_check = "%s" % Naming.kwds_cname
  3082. else:
  3083. kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
  3084. Naming.kwds_cname, Naming.kwds_cname)
  3085. code.globalstate.use_utility_code(
  3086. UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c"))
  3087. code.putln(
  3088. "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
  3089. kwarg_check, Naming.kwds_cname, self.name,
  3090. bool(self.starstar_arg), self.error_value()))
  3091. if self.starstar_arg and self.starstar_arg.entry.cf_used:
  3092. if all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references):
  3093. code.putln("if (%s) {" % kwarg_check)
  3094. code.putln("%s = PyDict_Copy(%s); if (unlikely(!%s)) return %s;" % (
  3095. self.starstar_arg.entry.cname,
  3096. Naming.kwds_cname,
  3097. self.starstar_arg.entry.cname,
  3098. self.error_value()))
  3099. code.put_gotref(self.starstar_arg.entry.cname)
  3100. code.putln("} else {")
  3101. code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,))
  3102. code.putln("}")
  3103. self.starstar_arg.entry.xdecref_cleanup = 1
  3104. else:
  3105. code.put("%s = (%s) ? PyDict_Copy(%s) : PyDict_New(); " % (
  3106. self.starstar_arg.entry.cname,
  3107. Naming.kwds_cname,
  3108. Naming.kwds_cname))
  3109. code.putln("if (unlikely(!%s)) return %s;" % (
  3110. self.starstar_arg.entry.cname, self.error_value()))
  3111. self.starstar_arg.entry.xdecref_cleanup = 0
  3112. code.put_gotref(self.starstar_arg.entry.cname)
  3113. if self.self_in_stararg and not self.target.is_staticmethod:
  3114. # need to create a new tuple with 'self' inserted as first item
  3115. code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % (
  3116. self.star_arg.entry.cname,
  3117. Naming.args_cname,
  3118. self.star_arg.entry.cname))
  3119. if self.starstar_arg and self.starstar_arg.entry.cf_used:
  3120. code.putln("{")
  3121. code.put_xdecref_clear(self.starstar_arg.entry.cname, py_object_type)
  3122. code.putln("return %s;" % self.error_value())
  3123. code.putln("}")
  3124. else:
  3125. code.putln("return %s;" % self.error_value())
  3126. code.put_gotref(self.star_arg.entry.cname)
  3127. code.put_incref(Naming.self_cname, py_object_type)
  3128. code.put_giveref(Naming.self_cname)
  3129. code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % (
  3130. self.star_arg.entry.cname, Naming.self_cname))
  3131. temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
  3132. code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % (
  3133. temp, temp, Naming.args_cname, temp))
  3134. code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % (
  3135. Naming.args_cname, temp))
  3136. code.put_incref("item", py_object_type)
  3137. code.put_giveref("item")
  3138. code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % (
  3139. self.star_arg.entry.cname, temp))
  3140. code.putln("}")
  3141. code.funcstate.release_temp(temp)
  3142. self.star_arg.entry.xdecref_cleanup = 0
  3143. elif self.star_arg:
  3144. code.put_incref(Naming.args_cname, py_object_type)
  3145. code.putln("%s = %s;" % (
  3146. self.star_arg.entry.cname,
  3147. Naming.args_cname))
  3148. self.star_arg.entry.xdecref_cleanup = 0
  3149. def generate_tuple_and_keyword_parsing_code(self, args, success_label, code):
  3150. argtuple_error_label = code.new_label("argtuple_error")
  3151. positional_args = []
  3152. required_kw_only_args = []
  3153. optional_kw_only_args = []
  3154. for arg in args:
  3155. if arg.is_generic:
  3156. if arg.default:
  3157. if not arg.is_self_arg and not arg.is_type_arg:
  3158. if arg.kw_only:
  3159. optional_kw_only_args.append(arg)
  3160. else:
  3161. positional_args.append(arg)
  3162. elif arg.kw_only:
  3163. required_kw_only_args.append(arg)
  3164. elif not arg.is_self_arg and not arg.is_type_arg:
  3165. positional_args.append(arg)
  3166. # sort required kw-only args before optional ones to avoid special
  3167. # cases in the unpacking code
  3168. kw_only_args = required_kw_only_args + optional_kw_only_args
  3169. min_positional_args = self.num_required_args - self.num_required_kw_args
  3170. if len(args) > 0 and (args[0].is_self_arg or args[0].is_type_arg):
  3171. min_positional_args -= 1
  3172. max_positional_args = len(positional_args)
  3173. has_fixed_positional_count = not self.star_arg and \
  3174. min_positional_args == max_positional_args
  3175. has_kw_only_args = bool(kw_only_args)
  3176. if self.starstar_arg or self.star_arg:
  3177. self.generate_stararg_init_code(max_positional_args, code)
  3178. code.putln('{')
  3179. all_args = tuple(positional_args) + tuple(kw_only_args)
  3180. code.putln("static PyObject **%s[] = {%s,0};" % (
  3181. Naming.pykwdlist_cname,
  3182. ','.join(['&%s' % code.intern_identifier(arg.name)
  3183. for arg in all_args])))
  3184. # Before being converted and assigned to the target variables,
  3185. # borrowed references to all unpacked argument values are
  3186. # collected into a local PyObject* array called "values",
  3187. # regardless if they were taken from default arguments,
  3188. # positional arguments or keyword arguments. Note that
  3189. # C-typed default arguments are handled at conversion time,
  3190. # so their array value is NULL in the end if no argument
  3191. # was passed for them.
  3192. self.generate_argument_values_setup_code(all_args, code)
  3193. # --- optimised code when we receive keyword arguments
  3194. code.putln("if (%s(%s)) {" % (
  3195. (self.num_required_kw_args > 0) and "likely" or "unlikely",
  3196. Naming.kwds_cname))
  3197. self.generate_keyword_unpacking_code(
  3198. min_positional_args, max_positional_args,
  3199. has_fixed_positional_count, has_kw_only_args,
  3200. all_args, argtuple_error_label, code)
  3201. # --- optimised code when we do not receive any keyword arguments
  3202. if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args:
  3203. # Python raises arg tuple related errors first, so we must
  3204. # check the length here
  3205. if min_positional_args == max_positional_args and not self.star_arg:
  3206. compare = '!='
  3207. else:
  3208. compare = '<'
  3209. code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % (
  3210. Naming.args_cname, compare, min_positional_args))
  3211. code.put_goto(argtuple_error_label)
  3212. if self.num_required_kw_args:
  3213. # pure error case: keywords required but not passed
  3214. if max_positional_args > min_positional_args and not self.star_arg:
  3215. code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % (
  3216. Naming.args_cname, max_positional_args))
  3217. code.put_goto(argtuple_error_label)
  3218. code.putln('} else {')
  3219. for i, arg in enumerate(kw_only_args):
  3220. if not arg.default:
  3221. pystring_cname = code.intern_identifier(arg.name)
  3222. # required keyword-only argument missing
  3223. code.globalstate.use_utility_code(
  3224. UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
  3225. code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
  3226. self.name,
  3227. pystring_cname))
  3228. code.putln(code.error_goto(self.pos))
  3229. break
  3230. else:
  3231. # optimised tuple unpacking code
  3232. code.putln('} else {')
  3233. if min_positional_args == max_positional_args:
  3234. # parse the exact number of positional arguments from
  3235. # the args tuple
  3236. for i, arg in enumerate(positional_args):
  3237. code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
  3238. else:
  3239. # parse the positional arguments from the variable length
  3240. # args tuple and reject illegal argument tuple sizes
  3241. code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname)
  3242. if self.star_arg:
  3243. code.putln('default:')
  3244. reversed_args = list(enumerate(positional_args))[::-1]
  3245. for i, arg in reversed_args:
  3246. if i >= min_positional_args-1:
  3247. if i != reversed_args[0][0]:
  3248. code.putln('CYTHON_FALLTHROUGH;')
  3249. code.put('case %2d: ' % (i+1))
  3250. code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
  3251. if min_positional_args == 0:
  3252. code.putln('CYTHON_FALLTHROUGH;')
  3253. code.put('case 0: ')
  3254. code.putln('break;')
  3255. if self.star_arg:
  3256. if min_positional_args:
  3257. for i in range(min_positional_args-1, -1, -1):
  3258. code.putln('case %2d:' % i)
  3259. code.put_goto(argtuple_error_label)
  3260. else:
  3261. code.put('default: ')
  3262. code.put_goto(argtuple_error_label)
  3263. code.putln('}')
  3264. code.putln('}') # end of the conditional unpacking blocks
  3265. # Convert arg values to their final type and assign them.
  3266. # Also inject non-Python default arguments, which do cannot
  3267. # live in the values[] array.
  3268. for i, arg in enumerate(all_args):
  3269. self.generate_arg_assignment(arg, "values[%d]" % i, code)
  3270. code.putln('}') # end of the whole argument unpacking block
  3271. if code.label_used(argtuple_error_label):
  3272. code.put_goto(success_label)
  3273. code.put_label(argtuple_error_label)
  3274. code.globalstate.use_utility_code(
  3275. UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
  3276. code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
  3277. self.name, has_fixed_positional_count,
  3278. min_positional_args, max_positional_args,
  3279. Naming.args_cname))
  3280. code.putln(code.error_goto(self.pos))
  3281. def generate_arg_assignment(self, arg, item, code):
  3282. if arg.type.is_pyobject:
  3283. # Python default arguments were already stored in 'item' at the very beginning
  3284. if arg.is_generic:
  3285. item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
  3286. entry = arg.entry
  3287. code.putln("%s = %s;" % (entry.cname, item))
  3288. else:
  3289. if arg.type.from_py_function:
  3290. if arg.default:
  3291. # C-typed default arguments must be handled here
  3292. code.putln('if (%s) {' % item)
  3293. code.putln(arg.type.from_py_call_code(
  3294. item, arg.entry.cname, arg.pos, code))
  3295. if arg.default:
  3296. code.putln('} else {')
  3297. code.putln("%s = %s;" % (
  3298. arg.entry.cname,
  3299. arg.calculate_default_value_code(code)))
  3300. if arg.type.is_memoryviewslice:
  3301. code.put_incref_memoryviewslice(arg.entry.cname,
  3302. have_gil=True)
  3303. code.putln('}')
  3304. else:
  3305. error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type)
  3306. def generate_stararg_init_code(self, max_positional_args, code):
  3307. if self.starstar_arg:
  3308. self.starstar_arg.entry.xdecref_cleanup = 0
  3309. code.putln('%s = PyDict_New(); if (unlikely(!%s)) return %s;' % (
  3310. self.starstar_arg.entry.cname,
  3311. self.starstar_arg.entry.cname,
  3312. self.error_value()))
  3313. code.put_gotref(self.starstar_arg.entry.cname)
  3314. if self.star_arg:
  3315. self.star_arg.entry.xdecref_cleanup = 0
  3316. code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % (
  3317. Naming.args_cname,
  3318. max_positional_args))
  3319. code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % (
  3320. self.star_arg.entry.cname, Naming.args_cname,
  3321. max_positional_args, Naming.args_cname))
  3322. code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
  3323. if self.starstar_arg:
  3324. code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
  3325. code.put_finish_refcount_context()
  3326. code.putln('return %s;' % self.error_value())
  3327. code.putln('}')
  3328. code.put_gotref(self.star_arg.entry.cname)
  3329. code.putln('} else {')
  3330. code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple))
  3331. code.put_incref(Naming.empty_tuple, py_object_type)
  3332. code.putln('}')
  3333. def generate_argument_values_setup_code(self, args, code):
  3334. max_args = len(args)
  3335. # the 'values' array collects borrowed references to arguments
  3336. # before doing any type coercion etc.
  3337. code.putln("PyObject* values[%d] = {%s};" % (
  3338. max_args, ','.join('0'*max_args)))
  3339. if self.target.defaults_struct:
  3340. code.putln('%s *%s = __Pyx_CyFunction_Defaults(%s, %s);' % (
  3341. self.target.defaults_struct, Naming.dynamic_args_cname,
  3342. self.target.defaults_struct, Naming.self_cname))
  3343. # assign borrowed Python default values to the values array,
  3344. # so that they can be overwritten by received arguments below
  3345. for i, arg in enumerate(args):
  3346. if arg.default and arg.type.is_pyobject:
  3347. default_value = arg.calculate_default_value_code(code)
  3348. code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value)))
  3349. def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args,
  3350. has_fixed_positional_count, has_kw_only_args,
  3351. all_args, argtuple_error_label, code):
  3352. code.putln('Py_ssize_t kw_args;')
  3353. code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname)
  3354. # copy the values from the args tuple and check that it's not too long
  3355. code.putln('switch (pos_args) {')
  3356. if self.star_arg:
  3357. code.putln('default:')
  3358. for i in range(max_positional_args-1, -1, -1):
  3359. code.put('case %2d: ' % (i+1))
  3360. code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (
  3361. i, Naming.args_cname, i))
  3362. code.putln('CYTHON_FALLTHROUGH;')
  3363. code.putln('case 0: break;')
  3364. if not self.star_arg:
  3365. code.put('default: ') # more arguments than allowed
  3366. code.put_goto(argtuple_error_label)
  3367. code.putln('}')
  3368. # The code above is very often (but not always) the same as
  3369. # the optimised non-kwargs tuple unpacking code, so we keep
  3370. # the code block above at the very top, before the following
  3371. # 'external' PyDict_Size() call, to make it easy for the C
  3372. # compiler to merge the two separate tuple unpacking
  3373. # implementations into one when they turn out to be identical.
  3374. # If we received kwargs, fill up the positional/required
  3375. # arguments with values from the kw dict
  3376. code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname)
  3377. if self.num_required_args or max_positional_args > 0:
  3378. last_required_arg = -1
  3379. for i, arg in enumerate(all_args):
  3380. if not arg.default:
  3381. last_required_arg = i
  3382. if last_required_arg < max_positional_args:
  3383. last_required_arg = max_positional_args-1
  3384. if max_positional_args > 0:
  3385. code.putln('switch (pos_args) {')
  3386. for i, arg in enumerate(all_args[:last_required_arg+1]):
  3387. if max_positional_args > 0 and i <= max_positional_args:
  3388. if i != 0:
  3389. code.putln('CYTHON_FALLTHROUGH;')
  3390. if self.star_arg and i == max_positional_args:
  3391. code.putln('default:')
  3392. else:
  3393. code.putln('case %2d:' % i)
  3394. pystring_cname = code.intern_identifier(arg.name)
  3395. if arg.default:
  3396. if arg.kw_only:
  3397. # optional kw-only args are handled separately below
  3398. continue
  3399. code.putln('if (kw_args > 0) {')
  3400. # don't overwrite default argument
  3401. code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, %s);' % (
  3402. Naming.kwds_cname, pystring_cname))
  3403. code.putln('if (value) { values[%d] = value; kw_args--; }' % i)
  3404. code.putln('}')
  3405. else:
  3406. code.putln('if (likely((values[%d] = __Pyx_PyDict_GetItemStr(%s, %s)) != 0)) kw_args--;' % (
  3407. i, Naming.kwds_cname, pystring_cname))
  3408. if i < min_positional_args:
  3409. if i == 0:
  3410. # special case: we know arg 0 is missing
  3411. code.put('else ')
  3412. code.put_goto(argtuple_error_label)
  3413. else:
  3414. # print the correct number of values (args or
  3415. # kwargs) that were passed into positional
  3416. # arguments up to this point
  3417. code.putln('else {')
  3418. code.globalstate.use_utility_code(
  3419. UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
  3420. code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
  3421. self.name, has_fixed_positional_count,
  3422. min_positional_args, max_positional_args, i))
  3423. code.putln(code.error_goto(self.pos))
  3424. code.putln('}')
  3425. elif arg.kw_only:
  3426. code.putln('else {')
  3427. code.globalstate.use_utility_code(
  3428. UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
  3429. code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
  3430. self.name, pystring_cname))
  3431. code.putln(code.error_goto(self.pos))
  3432. code.putln('}')
  3433. if max_positional_args > 0:
  3434. code.putln('}')
  3435. if has_kw_only_args:
  3436. # unpack optional keyword-only arguments separately because
  3437. # checking for interned strings in a dict is faster than iterating
  3438. self.generate_optional_kwonly_args_unpacking_code(all_args, code)
  3439. code.putln('if (unlikely(kw_args > 0)) {')
  3440. # non-positional/-required kw args left in dict: default args,
  3441. # kw-only args, **kwargs or error
  3442. #
  3443. # This is sort of a catch-all: except for checking required
  3444. # arguments, this will always do the right thing for unpacking
  3445. # keyword arguments, so that we can concentrate on optimising
  3446. # common cases above.
  3447. if max_positional_args == 0:
  3448. pos_arg_count = "0"
  3449. elif self.star_arg:
  3450. code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % (
  3451. max_positional_args, max_positional_args))
  3452. pos_arg_count = "used_pos_args"
  3453. else:
  3454. pos_arg_count = "pos_args"
  3455. code.globalstate.use_utility_code(
  3456. UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
  3457. code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % (
  3458. Naming.kwds_cname,
  3459. Naming.pykwdlist_cname,
  3460. self.starstar_arg and self.starstar_arg.entry.cname or '0',
  3461. pos_arg_count,
  3462. self.name,
  3463. code.error_goto(self.pos)))
  3464. code.putln('}')
  3465. def generate_optional_kwonly_args_unpacking_code(self, all_args, code):
  3466. optional_args = []
  3467. first_optional_arg = -1
  3468. for i, arg in enumerate(all_args):
  3469. if not arg.kw_only or not arg.default:
  3470. continue
  3471. if not optional_args:
  3472. first_optional_arg = i
  3473. optional_args.append(arg.name)
  3474. if optional_args:
  3475. if len(optional_args) > 1:
  3476. # if we receive more than the named kwargs, we either have **kwargs
  3477. # (in which case we must iterate anyway) or it's an error (which we
  3478. # also handle during iteration) => skip this part if there are more
  3479. code.putln('if (kw_args > 0 && %s(kw_args <= %d)) {' % (
  3480. not self.starstar_arg and 'likely' or '',
  3481. len(optional_args)))
  3482. code.putln('Py_ssize_t index;')
  3483. # not unrolling the loop here reduces the C code overhead
  3484. code.putln('for (index = %d; index < %d && kw_args > 0; index++) {' % (
  3485. first_optional_arg, first_optional_arg + len(optional_args)))
  3486. else:
  3487. code.putln('if (kw_args == 1) {')
  3488. code.putln('const Py_ssize_t index = %d;' % first_optional_arg)
  3489. code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, *%s[index]);' % (
  3490. Naming.kwds_cname, Naming.pykwdlist_cname))
  3491. code.putln('if (value) { values[index] = value; kw_args--; }')
  3492. if len(optional_args) > 1:
  3493. code.putln('}')
  3494. code.putln('}')
  3495. def generate_argument_conversion_code(self, code):
  3496. # Generate code to convert arguments from signature type to
  3497. # declared type, if needed. Also copies signature arguments
  3498. # into closure fields.
  3499. for arg in self.args:
  3500. if arg.needs_conversion:
  3501. self.generate_arg_conversion(arg, code)
  3502. def generate_arg_conversion(self, arg, code):
  3503. # Generate conversion code for one argument.
  3504. old_type = arg.hdr_type
  3505. new_type = arg.type
  3506. if old_type.is_pyobject:
  3507. if arg.default:
  3508. code.putln("if (%s) {" % arg.hdr_cname)
  3509. else:
  3510. code.putln("assert(%s); {" % arg.hdr_cname)
  3511. self.generate_arg_conversion_from_pyobject(arg, code)
  3512. code.putln("}")
  3513. elif new_type.is_pyobject:
  3514. self.generate_arg_conversion_to_pyobject(arg, code)
  3515. else:
  3516. if new_type.assignable_from(old_type):
  3517. code.putln("%s = %s;" % (arg.entry.cname, arg.hdr_cname))
  3518. else:
  3519. error(arg.pos, "Cannot convert 1 argument from '%s' to '%s'" % (old_type, new_type))
  3520. def generate_arg_conversion_from_pyobject(self, arg, code):
  3521. new_type = arg.type
  3522. # copied from CoerceFromPyTypeNode
  3523. if new_type.from_py_function:
  3524. code.putln(new_type.from_py_call_code(
  3525. arg.hdr_cname,
  3526. arg.entry.cname,
  3527. arg.pos,
  3528. code,
  3529. ))
  3530. else:
  3531. error(arg.pos, "Cannot convert Python object argument to type '%s'" % new_type)
  3532. def generate_arg_conversion_to_pyobject(self, arg, code):
  3533. old_type = arg.hdr_type
  3534. func = old_type.to_py_function
  3535. if func:
  3536. code.putln("%s = %s(%s); %s" % (
  3537. arg.entry.cname,
  3538. func,
  3539. arg.hdr_cname,
  3540. code.error_goto_if_null(arg.entry.cname, arg.pos)))
  3541. code.put_var_gotref(arg.entry)
  3542. else:
  3543. error(arg.pos, "Cannot convert argument of type '%s' to Python object" % old_type)
  3544. def generate_argument_type_tests(self, code):
  3545. # Generate type tests for args whose signature
  3546. # type is PyObject * and whose declared type is
  3547. # a subtype thereof.
  3548. for arg in self.args:
  3549. if arg.needs_type_test:
  3550. self.generate_arg_type_test(arg, code)
  3551. elif not arg.accept_none and (arg.type.is_pyobject or
  3552. arg.type.is_buffer or
  3553. arg.type.is_memoryviewslice):
  3554. self.generate_arg_none_check(arg, code)
  3555. def error_value(self):
  3556. return self.signature.error_value
  3557. class GeneratorDefNode(DefNode):
  3558. # Generator function node that creates a new generator instance when called.
  3559. #
  3560. # gbody GeneratorBodyDefNode the function implementing the generator
  3561. #
  3562. is_generator = True
  3563. is_coroutine = False
  3564. is_iterable_coroutine = False
  3565. is_asyncgen = False
  3566. gen_type_name = 'Generator'
  3567. needs_closure = True
  3568. child_attrs = DefNode.child_attrs + ["gbody"]
  3569. def __init__(self, pos, **kwargs):
  3570. # XXX: don't actually needs a body
  3571. kwargs['body'] = StatListNode(pos, stats=[], is_terminator=True)
  3572. super(GeneratorDefNode, self).__init__(pos, **kwargs)
  3573. def analyse_declarations(self, env):
  3574. super(GeneratorDefNode, self).analyse_declarations(env)
  3575. self.gbody.local_scope = self.local_scope
  3576. self.gbody.analyse_declarations(env)
  3577. def generate_function_body(self, env, code):
  3578. body_cname = self.gbody.entry.func_cname
  3579. name = code.intern_identifier(self.name)
  3580. qualname = code.intern_identifier(self.qualname)
  3581. module_name = code.intern_identifier(self.module_name)
  3582. code.putln('{')
  3583. code.putln('__pyx_CoroutineObject *gen = __Pyx_%s_New('
  3584. '(__pyx_coroutine_body_t) %s, %s, (PyObject *) %s, %s, %s, %s); %s' % (
  3585. self.gen_type_name,
  3586. body_cname, self.code_object.calculate_result_code(code) if self.code_object else 'NULL',
  3587. Naming.cur_scope_cname, name, qualname, module_name,
  3588. code.error_goto_if_null('gen', self.pos)))
  3589. code.put_decref(Naming.cur_scope_cname, py_object_type)
  3590. if self.requires_classobj:
  3591. classobj_cname = 'gen->classobj'
  3592. code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
  3593. classobj_cname, Naming.self_cname))
  3594. code.put_incref(classobj_cname, py_object_type)
  3595. code.put_giveref(classobj_cname)
  3596. code.put_finish_refcount_context()
  3597. code.putln('return (PyObject *) gen;')
  3598. code.putln('}')
  3599. def generate_function_definitions(self, env, code):
  3600. env.use_utility_code(UtilityCode.load_cached(self.gen_type_name, "Coroutine.c"))
  3601. self.gbody.generate_function_header(code, proto=True)
  3602. super(GeneratorDefNode, self).generate_function_definitions(env, code)
  3603. self.gbody.generate_function_definitions(env, code)
  3604. class AsyncDefNode(GeneratorDefNode):
  3605. gen_type_name = 'Coroutine'
  3606. is_coroutine = True
  3607. class IterableAsyncDefNode(AsyncDefNode):
  3608. gen_type_name = 'IterableCoroutine'
  3609. is_iterable_coroutine = True
  3610. class AsyncGenNode(AsyncDefNode):
  3611. gen_type_name = 'AsyncGen'
  3612. is_asyncgen = True
  3613. class GeneratorBodyDefNode(DefNode):
  3614. # Main code body of a generator implemented as a DefNode.
  3615. #
  3616. is_generator_body = True
  3617. is_inlined = False
  3618. is_async_gen_body = False
  3619. inlined_comprehension_type = None # container type for inlined comprehensions
  3620. def __init__(self, pos=None, name=None, body=None, is_async_gen_body=False):
  3621. super(GeneratorBodyDefNode, self).__init__(
  3622. pos=pos, body=body, name=name, is_async_gen_body=is_async_gen_body,
  3623. doc=None, args=[], star_arg=None, starstar_arg=None)
  3624. def declare_generator_body(self, env):
  3625. prefix = env.next_id(env.scope_prefix)
  3626. name = env.next_id('generator')
  3627. cname = Naming.genbody_prefix + prefix + name
  3628. entry = env.declare_var(None, py_object_type, self.pos,
  3629. cname=cname, visibility='private')
  3630. entry.func_cname = cname
  3631. entry.qualified_name = EncodedString(self.name)
  3632. # Work-around for https://github.com/cython/cython/issues/1699
  3633. # We don't currently determine whether the generator entry is used or not,
  3634. # so mark it as used to avoid false warnings.
  3635. entry.used = True
  3636. self.entry = entry
  3637. def analyse_declarations(self, env):
  3638. self.analyse_argument_types(env)
  3639. self.declare_generator_body(env)
  3640. def generate_function_header(self, code, proto=False):
  3641. header = "static PyObject *%s(PyObject *%s_obj, CYTHON_UNUSED PyThreadState *%s, PyObject *%s)" % (
  3642. self.entry.func_cname,
  3643. Naming.generator_cname,
  3644. Naming.local_tstate_cname,
  3645. Naming.sent_value_cname)
  3646. if proto:
  3647. code.putln('%s; /* proto */' % header)
  3648. else:
  3649. code.putln('%s /* generator body */\n{' % header)
  3650. def generate_function_definitions(self, env, code):
  3651. lenv = self.local_scope
  3652. # Generate closure function definitions
  3653. self.body.generate_function_definitions(lenv, code)
  3654. # Generate C code for header and body of function
  3655. code.enter_cfunc_scope(lenv)
  3656. code.return_from_error_cleanup_label = code.new_label()
  3657. # ----- Top-level constants used by this function
  3658. code.mark_pos(self.pos)
  3659. self.generate_cached_builtins_decls(lenv, code)
  3660. # ----- Function header
  3661. code.putln("")
  3662. self.generate_function_header(code)
  3663. code.putln("__pyx_CoroutineObject *%s = (__pyx_CoroutineObject *)%s_obj;" % (Naming.generator_cname, Naming.generator_cname))
  3664. closure_init_code = code.insertion_point()
  3665. # ----- Local variables
  3666. code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
  3667. tempvardecl_code = code.insertion_point()
  3668. code.put_declare_refcount_context()
  3669. code.put_setup_refcount_context(self.entry.name or self.entry.qualified_name)
  3670. profile = code.globalstate.directives['profile']
  3671. linetrace = code.globalstate.directives['linetrace']
  3672. if profile or linetrace:
  3673. tempvardecl_code.put_trace_declarations()
  3674. code.funcstate.can_trace = True
  3675. code_object = self.code_object.calculate_result_code(code) if self.code_object else None
  3676. code.put_trace_frame_init(code_object)
  3677. # ----- Resume switch point.
  3678. code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
  3679. resume_code = code.insertion_point()
  3680. first_run_label = code.new_label('first_run')
  3681. code.use_label(first_run_label)
  3682. code.put_label(first_run_label)
  3683. code.putln('%s' %
  3684. (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))
  3685. # ----- prepare target container for inlined comprehension
  3686. if self.is_inlined and self.inlined_comprehension_type is not None:
  3687. target_type = self.inlined_comprehension_type
  3688. if target_type is Builtin.list_type:
  3689. comp_init = 'PyList_New(0)'
  3690. elif target_type is Builtin.set_type:
  3691. comp_init = 'PySet_New(NULL)'
  3692. elif target_type is Builtin.dict_type:
  3693. comp_init = 'PyDict_New()'
  3694. else:
  3695. raise InternalError(
  3696. "invalid type of inlined comprehension: %s" % target_type)
  3697. code.putln("%s = %s; %s" % (
  3698. Naming.retval_cname, comp_init,
  3699. code.error_goto_if_null(Naming.retval_cname, self.pos)))
  3700. code.put_gotref(Naming.retval_cname)
  3701. # ----- Function body
  3702. self.generate_function_body(env, code)
  3703. # ----- Closure initialization
  3704. if lenv.scope_class.type.scope.var_entries:
  3705. closure_init_code.putln('%s = %s;' % (
  3706. lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
  3707. lenv.scope_class.type.cast_code('%s->closure' %
  3708. Naming.generator_cname)))
  3709. # FIXME: this silences a potential "unused" warning => try to avoid unused closures in more cases
  3710. code.putln("CYTHON_MAYBE_UNUSED_VAR(%s);" % Naming.cur_scope_cname)
  3711. if profile or linetrace:
  3712. code.funcstate.can_trace = False
  3713. code.mark_pos(self.pos)
  3714. code.putln("")
  3715. code.putln("/* function exit code */")
  3716. # on normal generator termination, we do not take the exception propagation
  3717. # path: no traceback info is required and not creating it is much faster
  3718. if not self.is_inlined and not self.body.is_terminator:
  3719. if self.is_async_gen_body:
  3720. code.globalstate.use_utility_code(
  3721. UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
  3722. code.putln('PyErr_SetNone(%s);' % (
  3723. '__Pyx_PyExc_StopAsyncIteration' if self.is_async_gen_body else 'PyExc_StopIteration'))
  3724. # ----- Error cleanup
  3725. if code.label_used(code.error_label):
  3726. if not self.body.is_terminator:
  3727. code.put_goto(code.return_label)
  3728. code.put_label(code.error_label)
  3729. if self.is_inlined and self.inlined_comprehension_type is not None:
  3730. code.put_xdecref_clear(Naming.retval_cname, py_object_type)
  3731. if Future.generator_stop in env.global_scope().context.future_directives:
  3732. # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
  3733. code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Coroutine.c"))
  3734. code.putln("__Pyx_Generator_Replace_StopIteration(%d);" % bool(self.is_async_gen_body))
  3735. for cname, type in code.funcstate.all_managed_temps():
  3736. code.put_xdecref(cname, type)
  3737. code.put_add_traceback(self.entry.qualified_name)
  3738. # ----- Non-error return cleanup
  3739. code.put_label(code.return_label)
  3740. if self.is_inlined:
  3741. code.put_xgiveref(Naming.retval_cname)
  3742. else:
  3743. code.put_xdecref_clear(Naming.retval_cname, py_object_type)
  3744. # For Py3.7, clearing is already done below.
  3745. code.putln("#if !CYTHON_USE_EXC_INFO_STACK")
  3746. code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname)
  3747. code.putln("#endif")
  3748. code.putln('%s->resume_label = -1;' % Naming.generator_cname)
  3749. # clean up as early as possible to help breaking any reference cycles
  3750. code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname)
  3751. if profile or linetrace:
  3752. code.put_trace_return(Naming.retval_cname,
  3753. nogil=not code.funcstate.gil_owned)
  3754. code.put_finish_refcount_context()
  3755. code.putln("return %s;" % Naming.retval_cname)
  3756. code.putln("}")
  3757. # ----- Go back and insert temp variable declarations
  3758. tempvardecl_code.put_temp_declarations(code.funcstate)
  3759. # ----- Generator resume code
  3760. if profile or linetrace:
  3761. resume_code.put_trace_call(self.entry.qualified_name, self.pos,
  3762. nogil=not code.funcstate.gil_owned)
  3763. resume_code.putln("switch (%s->resume_label) {" % (
  3764. Naming.generator_cname))
  3765. resume_code.putln("case 0: goto %s;" % first_run_label)
  3766. for i, label in code.yield_labels:
  3767. resume_code.putln("case %d: goto %s;" % (i, label))
  3768. resume_code.putln("default: /* CPython raises the right error here */")
  3769. if profile or linetrace:
  3770. resume_code.put_trace_return("Py_None",
  3771. nogil=not code.funcstate.gil_owned)
  3772. resume_code.put_finish_refcount_context()
  3773. resume_code.putln("return NULL;")
  3774. resume_code.putln("}")
  3775. code.exit_cfunc_scope()
  3776. class OverrideCheckNode(StatNode):
  3777. # A Node for dispatching to the def method if it
  3778. # is overridden.
  3779. #
  3780. # py_func
  3781. #
  3782. # args
  3783. # func_temp
  3784. # body
  3785. child_attrs = ['body']
  3786. body = None
  3787. def analyse_expressions(self, env):
  3788. self.args = env.arg_entries
  3789. if self.py_func.is_module_scope:
  3790. first_arg = 0
  3791. else:
  3792. first_arg = 1
  3793. from . import ExprNodes
  3794. self.func_node = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
  3795. call_node = ExprNodes.SimpleCallNode(
  3796. self.pos, function=self.func_node,
  3797. args=[ExprNodes.NameNode(self.pos, name=arg.name)
  3798. for arg in self.args[first_arg:]])
  3799. if env.return_type.is_void or env.return_type.is_returncode:
  3800. self.body = StatListNode(self.pos, stats=[
  3801. ExprStatNode(self.pos, expr=call_node),
  3802. ReturnStatNode(self.pos, value=None)])
  3803. else:
  3804. self.body = ReturnStatNode(self.pos, value=call_node)
  3805. self.body = self.body.analyse_expressions(env)
  3806. return self
  3807. def generate_execution_code(self, code):
  3808. interned_attr_cname = code.intern_identifier(self.py_func.entry.name)
  3809. # Check to see if we are an extension type
  3810. if self.py_func.is_module_scope:
  3811. self_arg = "((PyObject *)%s)" % Naming.module_cname
  3812. else:
  3813. self_arg = "((PyObject *)%s)" % self.args[0].cname
  3814. code.putln("/* Check if called by wrapper */")
  3815. code.putln("if (unlikely(%s)) ;" % Naming.skip_dispatch_cname)
  3816. code.putln("/* Check if overridden in Python */")
  3817. if self.py_func.is_module_scope:
  3818. code.putln("else {")
  3819. else:
  3820. code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0)"
  3821. " || (Py_TYPE(%s)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % (
  3822. self_arg, self_arg))
  3823. code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
  3824. code.globalstate.use_utility_code(
  3825. UtilityCode.load_cached("PyDictVersioning", "ObjectHandling.c"))
  3826. # TODO: remove the object dict version check by 'inlining' the getattr implementation for methods.
  3827. # This would allow checking the dict versions around _PyType_Lookup() if it returns a descriptor,
  3828. # and would (tada!) make this check a pure type based thing instead of supporting only a single
  3829. # instance at a time.
  3830. code.putln("static PY_UINT64_T %s = __PYX_DICT_VERSION_INIT, %s = __PYX_DICT_VERSION_INIT;" % (
  3831. Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
  3832. code.putln("if (unlikely(!__Pyx_object_dict_version_matches(%s, %s, %s))) {" % (
  3833. self_arg, Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
  3834. code.putln("PY_UINT64_T %s = __Pyx_get_tp_dict_version(%s);" % (
  3835. Naming.type_dict_guard_temp, self_arg))
  3836. code.putln("#endif")
  3837. func_node_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
  3838. self.func_node.set_cname(func_node_temp)
  3839. # need to get attribute manually--scope would return cdef method
  3840. code.globalstate.use_utility_code(
  3841. UtilityCode.load_cached("PyObjectGetAttrStr", "ObjectHandling.c"))
  3842. err = code.error_goto_if_null(func_node_temp, self.pos)
  3843. code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % (
  3844. func_node_temp, self_arg, interned_attr_cname, err))
  3845. code.put_gotref(func_node_temp)
  3846. is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp
  3847. is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)(void*)%s)" % (
  3848. func_node_temp, self.py_func.entry.func_cname)
  3849. code.putln("if (!%s || %s) {" % (is_builtin_function_or_method, is_overridden))
  3850. self.body.generate_execution_code(code)
  3851. code.putln("}")
  3852. # NOTE: it's not 100% sure that we catch the exact versions here that were used for the lookup,
  3853. # but it is very unlikely that the versions change during lookup, and the type dict safe guard
  3854. # should increase the chance of detecting such a case.
  3855. code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
  3856. code.putln("%s = __Pyx_get_tp_dict_version(%s);" % (
  3857. Naming.tp_dict_version_temp, self_arg))
  3858. code.putln("%s = __Pyx_get_object_dict_version(%s);" % (
  3859. Naming.obj_dict_version_temp, self_arg))
  3860. # Safety check that the type dict didn't change during the lookup. Since CPython looks up the
  3861. # attribute (descriptor) first in the type dict and then in the instance dict or through the
  3862. # descriptor, the only really far-away lookup when we get here is one in the type dict. So we
  3863. # double check the type dict version before and afterwards to guard against later changes of
  3864. # the type dict during the lookup process.
  3865. code.putln("if (unlikely(%s != %s)) {" % (
  3866. Naming.type_dict_guard_temp, Naming.tp_dict_version_temp))
  3867. code.putln("%s = %s = __PYX_DICT_VERSION_INIT;" % (
  3868. Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
  3869. code.putln("}")
  3870. code.putln("#endif")
  3871. code.put_decref_clear(func_node_temp, PyrexTypes.py_object_type)
  3872. code.funcstate.release_temp(func_node_temp)
  3873. code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
  3874. code.putln("}")
  3875. code.putln("#endif")
  3876. code.putln("}")
  3877. class ClassDefNode(StatNode, BlockNode):
  3878. pass
  3879. class PyClassDefNode(ClassDefNode):
  3880. # A Python class definition.
  3881. #
  3882. # name EncodedString Name of the class
  3883. # doc string or None
  3884. # body StatNode Attribute definition code
  3885. # entry Symtab.Entry
  3886. # scope PyClassScope
  3887. # decorators [DecoratorNode] list of decorators or None
  3888. #
  3889. # The following subnodes are constructed internally:
  3890. #
  3891. # dict DictNode Class dictionary or Py3 namespace
  3892. # classobj ClassNode Class object
  3893. # target NameNode Variable to assign class object to
  3894. child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result",
  3895. "target", "class_cell", "decorators"]
  3896. decorators = None
  3897. class_result = None
  3898. is_py3_style_class = False # Python3 style class (kwargs)
  3899. metaclass = None
  3900. mkw = None
  3901. def __init__(self, pos, name, bases, doc, body, decorators=None,
  3902. keyword_args=None, force_py3_semantics=False):
  3903. StatNode.__init__(self, pos)
  3904. self.name = name
  3905. self.doc = doc
  3906. self.body = body
  3907. self.decorators = decorators
  3908. self.bases = bases
  3909. from . import ExprNodes
  3910. if self.doc and Options.docstrings:
  3911. doc = embed_position(self.pos, self.doc)
  3912. doc_node = ExprNodes.StringNode(pos, value=doc)
  3913. else:
  3914. doc_node = None
  3915. allow_py2_metaclass = not force_py3_semantics
  3916. if keyword_args:
  3917. allow_py2_metaclass = False
  3918. self.is_py3_style_class = True
  3919. if keyword_args.is_dict_literal:
  3920. if keyword_args.key_value_pairs:
  3921. for i, item in list(enumerate(keyword_args.key_value_pairs))[::-1]:
  3922. if item.key.value == 'metaclass':
  3923. if self.metaclass is not None:
  3924. error(item.pos, "keyword argument 'metaclass' passed multiple times")
  3925. # special case: we already know the metaclass,
  3926. # so we don't need to do the "build kwargs,
  3927. # find metaclass" dance at runtime
  3928. self.metaclass = item.value
  3929. del keyword_args.key_value_pairs[i]
  3930. self.mkw = keyword_args
  3931. else:
  3932. assert self.metaclass is not None
  3933. else:
  3934. # MergedDictNode
  3935. self.mkw = ExprNodes.ProxyNode(keyword_args)
  3936. if force_py3_semantics or self.bases or self.mkw or self.metaclass:
  3937. if self.metaclass is None:
  3938. if keyword_args and not keyword_args.is_dict_literal:
  3939. # **kwargs may contain 'metaclass' arg
  3940. mkdict = self.mkw
  3941. else:
  3942. mkdict = None
  3943. if (not mkdict and
  3944. self.bases.is_sequence_constructor and
  3945. not self.bases.args):
  3946. pass # no base classes => no inherited metaclass
  3947. else:
  3948. self.metaclass = ExprNodes.PyClassMetaclassNode(
  3949. pos, class_def_node=self)
  3950. needs_metaclass_calculation = False
  3951. else:
  3952. needs_metaclass_calculation = True
  3953. self.dict = ExprNodes.PyClassNamespaceNode(
  3954. pos, name=name, doc=doc_node, class_def_node=self)
  3955. self.classobj = ExprNodes.Py3ClassNode(
  3956. pos, name=name, class_def_node=self, doc=doc_node,
  3957. calculate_metaclass=needs_metaclass_calculation,
  3958. allow_py2_metaclass=allow_py2_metaclass)
  3959. else:
  3960. # no bases, no metaclass => old style class creation
  3961. self.dict = ExprNodes.DictNode(pos, key_value_pairs=[])
  3962. self.classobj = ExprNodes.ClassNode(
  3963. pos, name=name, class_def_node=self, doc=doc_node)
  3964. self.target = ExprNodes.NameNode(pos, name=name)
  3965. self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
  3966. def as_cclass(self):
  3967. """
  3968. Return this node as if it were declared as an extension class
  3969. """
  3970. if self.is_py3_style_class:
  3971. error(self.classobj.pos, "Python3 style class could not be represented as C class")
  3972. return
  3973. from . import ExprNodes
  3974. return CClassDefNode(self.pos,
  3975. visibility='private',
  3976. module_name=None,
  3977. class_name=self.name,
  3978. bases=self.bases or ExprNodes.TupleNode(self.pos, args=[]),
  3979. decorators=self.decorators,
  3980. body=self.body,
  3981. in_pxd=False,
  3982. doc=self.doc)
  3983. def create_scope(self, env):
  3984. genv = env
  3985. while genv.is_py_class_scope or genv.is_c_class_scope:
  3986. genv = genv.outer_scope
  3987. cenv = self.scope = PyClassScope(name=self.name, outer_scope=genv)
  3988. return cenv
  3989. def analyse_declarations(self, env):
  3990. class_result = self.classobj
  3991. if self.decorators:
  3992. from .ExprNodes import SimpleCallNode
  3993. for decorator in self.decorators[::-1]:
  3994. class_result = SimpleCallNode(
  3995. decorator.pos,
  3996. function=decorator.decorator,
  3997. args=[class_result])
  3998. self.decorators = None
  3999. self.class_result = class_result
  4000. if self.bases:
  4001. self.bases.analyse_declarations(env)
  4002. if self.mkw:
  4003. self.mkw.analyse_declarations(env)
  4004. self.class_result.analyse_declarations(env)
  4005. self.target.analyse_target_declaration(env)
  4006. cenv = self.create_scope(env)
  4007. cenv.directives = env.directives
  4008. cenv.class_obj_cname = self.target.entry.cname
  4009. self.body.analyse_declarations(cenv)
  4010. def analyse_expressions(self, env):
  4011. if self.bases:
  4012. self.bases = self.bases.analyse_expressions(env)
  4013. if self.mkw:
  4014. self.mkw = self.mkw.analyse_expressions(env)
  4015. if self.metaclass:
  4016. self.metaclass = self.metaclass.analyse_expressions(env)
  4017. self.dict = self.dict.analyse_expressions(env)
  4018. self.class_result = self.class_result.analyse_expressions(env)
  4019. cenv = self.scope
  4020. self.body = self.body.analyse_expressions(cenv)
  4021. self.target.analyse_target_expression(env, self.classobj)
  4022. self.class_cell = self.class_cell.analyse_expressions(cenv)
  4023. return self
  4024. def generate_function_definitions(self, env, code):
  4025. self.generate_lambda_definitions(self.scope, code)
  4026. self.body.generate_function_definitions(self.scope, code)
  4027. def generate_execution_code(self, code):
  4028. code.mark_pos(self.pos)
  4029. code.pyclass_stack.append(self)
  4030. cenv = self.scope
  4031. if self.bases:
  4032. self.bases.generate_evaluation_code(code)
  4033. if self.mkw:
  4034. self.mkw.generate_evaluation_code(code)
  4035. if self.metaclass:
  4036. self.metaclass.generate_evaluation_code(code)
  4037. self.dict.generate_evaluation_code(code)
  4038. cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
  4039. class_cell = self.class_cell
  4040. if class_cell is not None and not class_cell.is_active:
  4041. class_cell = None
  4042. if class_cell is not None:
  4043. class_cell.generate_evaluation_code(code)
  4044. self.body.generate_execution_code(code)
  4045. self.class_result.generate_evaluation_code(code)
  4046. if class_cell is not None:
  4047. class_cell.generate_injection_code(
  4048. code, self.class_result.result())
  4049. if class_cell is not None:
  4050. class_cell.generate_disposal_code(code)
  4051. class_cell.free_temps(code)
  4052. cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
  4053. self.target.generate_assignment_code(self.class_result, code)
  4054. self.dict.generate_disposal_code(code)
  4055. self.dict.free_temps(code)
  4056. if self.metaclass:
  4057. self.metaclass.generate_disposal_code(code)
  4058. self.metaclass.free_temps(code)
  4059. if self.mkw:
  4060. self.mkw.generate_disposal_code(code)
  4061. self.mkw.free_temps(code)
  4062. if self.bases:
  4063. self.bases.generate_disposal_code(code)
  4064. self.bases.free_temps(code)
  4065. code.pyclass_stack.pop()
  4066. class CClassDefNode(ClassDefNode):
  4067. # An extension type definition.
  4068. #
  4069. # visibility 'private' or 'public' or 'extern'
  4070. # typedef_flag boolean
  4071. # api boolean
  4072. # module_name string or None For import of extern type objects
  4073. # class_name string Unqualified name of class
  4074. # as_name string or None Name to declare as in this scope
  4075. # bases TupleNode Base class(es)
  4076. # objstruct_name string or None Specified C name of object struct
  4077. # typeobj_name string or None Specified C name of type object
  4078. # check_size 'warn', 'error', 'ignore' What to do if tp_basicsize does not match
  4079. # in_pxd boolean Is in a .pxd file
  4080. # decorators [DecoratorNode] list of decorators or None
  4081. # doc string or None
  4082. # body StatNode or None
  4083. # entry Symtab.Entry
  4084. # base_type PyExtensionType or None
  4085. # buffer_defaults_node DictNode or None Declares defaults for a buffer
  4086. # buffer_defaults_pos
  4087. child_attrs = ["body"]
  4088. buffer_defaults_node = None
  4089. buffer_defaults_pos = None
  4090. typedef_flag = False
  4091. api = False
  4092. objstruct_name = None
  4093. typeobj_name = None
  4094. check_size = None
  4095. decorators = None
  4096. shadow = False
  4097. def buffer_defaults(self, env):
  4098. if not hasattr(self, '_buffer_defaults'):
  4099. from . import Buffer
  4100. if self.buffer_defaults_node:
  4101. self._buffer_defaults = Buffer.analyse_buffer_options(
  4102. self.buffer_defaults_pos,
  4103. env, [], self.buffer_defaults_node,
  4104. need_complete=False)
  4105. else:
  4106. self._buffer_defaults = None
  4107. return self._buffer_defaults
  4108. def declare(self, env):
  4109. if self.module_name and self.visibility != 'extern':
  4110. module_path = self.module_name.split(".")
  4111. home_scope = env.find_imported_module(module_path, self.pos)
  4112. if not home_scope:
  4113. return None
  4114. else:
  4115. home_scope = env
  4116. self.entry = home_scope.declare_c_class(
  4117. name=self.class_name,
  4118. pos=self.pos,
  4119. defining=0,
  4120. implementing=0,
  4121. module_name=self.module_name,
  4122. base_type=None,
  4123. objstruct_cname=self.objstruct_name,
  4124. typeobj_cname=self.typeobj_name,
  4125. visibility=self.visibility,
  4126. typedef_flag=self.typedef_flag,
  4127. check_size = self.check_size,
  4128. api=self.api,
  4129. buffer_defaults=self.buffer_defaults(env),
  4130. shadow=self.shadow)
  4131. def analyse_declarations(self, env):
  4132. #print "CClassDefNode.analyse_declarations:", self.class_name
  4133. #print "...visibility =", self.visibility
  4134. #print "...module_name =", self.module_name
  4135. if env.in_cinclude and not self.objstruct_name:
  4136. error(self.pos, "Object struct name specification required for C class defined in 'extern from' block")
  4137. if self.decorators:
  4138. error(self.pos, "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
  4139. self.base_type = None
  4140. # Now that module imports are cached, we need to
  4141. # import the modules for extern classes.
  4142. if self.module_name:
  4143. self.module = None
  4144. for module in env.cimported_modules:
  4145. if module.name == self.module_name:
  4146. self.module = module
  4147. if self.module is None:
  4148. self.module = ModuleScope(self.module_name, None, env.context)
  4149. self.module.has_extern_class = 1
  4150. env.add_imported_module(self.module)
  4151. if self.bases.args:
  4152. base = self.bases.args[0]
  4153. base_type = base.analyse_as_type(env)
  4154. if base_type in (PyrexTypes.c_int_type, PyrexTypes.c_long_type, PyrexTypes.c_float_type):
  4155. # Use the Python rather than C variant of these types.
  4156. base_type = env.lookup(base_type.sign_and_name()).type
  4157. if base_type is None:
  4158. error(base.pos, "First base of '%s' is not an extension type" % self.class_name)
  4159. elif base_type == PyrexTypes.py_object_type:
  4160. base_class_scope = None
  4161. elif not base_type.is_extension_type and \
  4162. not (base_type.is_builtin_type and base_type.objstruct_cname):
  4163. error(base.pos, "'%s' is not an extension type" % base_type)
  4164. elif not base_type.is_complete():
  4165. error(base.pos, "Base class '%s' of type '%s' is incomplete" % (
  4166. base_type.name, self.class_name))
  4167. elif base_type.scope and base_type.scope.directives and \
  4168. base_type.is_final_type:
  4169. error(base.pos, "Base class '%s' of type '%s' is final" % (
  4170. base_type, self.class_name))
  4171. elif base_type.is_builtin_type and \
  4172. base_type.name in ('tuple', 'str', 'bytes'):
  4173. error(base.pos, "inheritance from PyVarObject types like '%s' is not currently supported"
  4174. % base_type.name)
  4175. else:
  4176. self.base_type = base_type
  4177. if env.directives.get('freelist', 0) > 0 and base_type != PyrexTypes.py_object_type:
  4178. warning(self.pos, "freelists cannot be used on subtypes, only the base class can manage them", 1)
  4179. has_body = self.body is not None
  4180. if has_body and self.base_type and not self.base_type.scope:
  4181. # To properly initialize inherited attributes, the base type must
  4182. # be analysed before this type.
  4183. self.base_type.defered_declarations.append(lambda : self.analyse_declarations(env))
  4184. return
  4185. if self.module_name and self.visibility != 'extern':
  4186. module_path = self.module_name.split(".")
  4187. home_scope = env.find_imported_module(module_path, self.pos)
  4188. if not home_scope:
  4189. return
  4190. else:
  4191. home_scope = env
  4192. if self.visibility == 'extern':
  4193. if (self.module_name == '__builtin__' and
  4194. self.class_name in Builtin.builtin_types and
  4195. env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython
  4196. warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1)
  4197. self.entry = home_scope.declare_c_class(
  4198. name=self.class_name,
  4199. pos=self.pos,
  4200. defining=has_body and self.in_pxd,
  4201. implementing=has_body and not self.in_pxd,
  4202. module_name=self.module_name,
  4203. base_type=self.base_type,
  4204. objstruct_cname=self.objstruct_name,
  4205. typeobj_cname=self.typeobj_name,
  4206. check_size=self.check_size,
  4207. visibility=self.visibility,
  4208. typedef_flag=self.typedef_flag,
  4209. api=self.api,
  4210. buffer_defaults=self.buffer_defaults(env),
  4211. shadow=self.shadow)
  4212. if self.shadow:
  4213. home_scope.lookup(self.class_name).as_variable = self.entry
  4214. if home_scope is not env and self.visibility == 'extern':
  4215. env.add_imported_entry(self.class_name, self.entry, self.pos)
  4216. self.scope = scope = self.entry.type.scope
  4217. if scope is not None:
  4218. scope.directives = env.directives
  4219. if self.doc and Options.docstrings:
  4220. scope.doc = embed_position(self.pos, self.doc)
  4221. if has_body:
  4222. self.body.analyse_declarations(scope)
  4223. dict_entry = self.scope.lookup_here("__dict__")
  4224. if dict_entry and dict_entry.is_variable and (not scope.defined and not scope.implemented):
  4225. dict_entry.getter_cname = self.scope.mangle_internal("__dict__getter")
  4226. self.scope.declare_property("__dict__", dict_entry.doc, dict_entry.pos)
  4227. if self.in_pxd:
  4228. scope.defined = 1
  4229. else:
  4230. scope.implemented = 1
  4231. if len(self.bases.args) > 1:
  4232. if not has_body or self.in_pxd:
  4233. error(self.bases.args[1].pos, "Only declare first base in declaration.")
  4234. # At runtime, we check that the other bases are heap types
  4235. # and that a __dict__ is added if required.
  4236. for other_base in self.bases.args[1:]:
  4237. if other_base.analyse_as_type(env):
  4238. error(other_base.pos, "Only one extension type base class allowed.")
  4239. self.entry.type.early_init = 0
  4240. from . import ExprNodes
  4241. self.type_init_args = ExprNodes.TupleNode(
  4242. self.pos,
  4243. args=[ExprNodes.IdentifierStringNode(self.pos, value=self.class_name),
  4244. self.bases,
  4245. ExprNodes.DictNode(self.pos, key_value_pairs=[])])
  4246. elif self.base_type:
  4247. self.entry.type.early_init = self.base_type.is_external or self.base_type.early_init
  4248. self.type_init_args = None
  4249. else:
  4250. self.entry.type.early_init = 1
  4251. self.type_init_args = None
  4252. env.allocate_vtable_names(self.entry)
  4253. for thunk in self.entry.type.defered_declarations:
  4254. thunk()
  4255. def analyse_expressions(self, env):
  4256. if self.body:
  4257. scope = self.entry.type.scope
  4258. self.body = self.body.analyse_expressions(scope)
  4259. if self.type_init_args:
  4260. self.type_init_args.analyse_expressions(env)
  4261. return self
  4262. def generate_function_definitions(self, env, code):
  4263. if self.body:
  4264. self.generate_lambda_definitions(self.scope, code)
  4265. self.body.generate_function_definitions(self.scope, code)
  4266. def generate_execution_code(self, code):
  4267. # This is needed to generate evaluation code for
  4268. # default values of method arguments.
  4269. code.mark_pos(self.pos)
  4270. if self.body:
  4271. self.body.generate_execution_code(code)
  4272. if not self.entry.type.early_init:
  4273. if self.type_init_args:
  4274. self.type_init_args.generate_evaluation_code(code)
  4275. bases = "PyTuple_GET_ITEM(%s, 1)" % self.type_init_args.result()
  4276. first_base = "((PyTypeObject*)PyTuple_GET_ITEM(%s, 0))" % bases
  4277. # Let Python do the base types compatibility checking.
  4278. trial_type = code.funcstate.allocate_temp(PyrexTypes.py_object_type, True)
  4279. code.putln("%s = PyType_Type.tp_new(&PyType_Type, %s, NULL);" % (
  4280. trial_type, self.type_init_args.result()))
  4281. code.putln(code.error_goto_if_null(trial_type, self.pos))
  4282. code.put_gotref(trial_type)
  4283. code.putln("if (((PyTypeObject*) %s)->tp_base != %s) {" % (
  4284. trial_type, first_base))
  4285. code.putln("PyErr_Format(PyExc_TypeError, \"best base '%s' must be equal to first base '%s'\",")
  4286. code.putln(" ((PyTypeObject*) %s)->tp_base->tp_name, %s->tp_name);" % (
  4287. trial_type, first_base))
  4288. code.putln(code.error_goto(self.pos))
  4289. code.putln("}")
  4290. code.funcstate.release_temp(trial_type)
  4291. code.put_incref(bases, PyrexTypes.py_object_type)
  4292. code.put_giveref(bases)
  4293. code.putln("%s.tp_bases = %s;" % (self.entry.type.typeobj_cname, bases))
  4294. code.put_decref_clear(trial_type, PyrexTypes.py_object_type)
  4295. self.type_init_args.generate_disposal_code(code)
  4296. self.type_init_args.free_temps(code)
  4297. self.generate_type_ready_code(self.entry, code, True)
  4298. # Also called from ModuleNode for early init types.
  4299. @staticmethod
  4300. def generate_type_ready_code(entry, code, heap_type_bases=False):
  4301. # Generate a call to PyType_Ready for an extension
  4302. # type defined in this module.
  4303. type = entry.type
  4304. typeobj_cname = type.typeobj_cname
  4305. scope = type.scope
  4306. if not scope: # could be None if there was an error
  4307. return
  4308. if entry.visibility != 'extern':
  4309. for slot in TypeSlots.slot_table:
  4310. slot.generate_dynamic_init_code(scope, code)
  4311. if heap_type_bases:
  4312. code.globalstate.use_utility_code(
  4313. UtilityCode.load_cached('PyType_Ready', 'ExtensionTypes.c'))
  4314. readyfunc = "__Pyx_PyType_Ready"
  4315. else:
  4316. readyfunc = "PyType_Ready"
  4317. code.putln(
  4318. "if (%s(&%s) < 0) %s" % (
  4319. readyfunc,
  4320. typeobj_cname,
  4321. code.error_goto(entry.pos)))
  4322. # Don't inherit tp_print from builtin types, restoring the
  4323. # behavior of using tp_repr or tp_str instead.
  4324. # ("tp_print" was renamed to "tp_vectorcall_offset" in Py3.8b1)
  4325. code.putln("#if PY_VERSION_HEX < 0x030800B1")
  4326. code.putln("%s.tp_print = 0;" % typeobj_cname)
  4327. code.putln("#endif")
  4328. # Use specialised attribute lookup for types with generic lookup but no instance dict.
  4329. getattr_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_getattro')
  4330. dictoffset_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_dictoffset')
  4331. if getattr_slot_func == '0' and dictoffset_slot_func == '0':
  4332. if type.is_final_type:
  4333. py_cfunc = "__Pyx_PyObject_GenericGetAttrNoDict" # grepable
  4334. utility_func = "PyObject_GenericGetAttrNoDict"
  4335. else:
  4336. py_cfunc = "__Pyx_PyObject_GenericGetAttr"
  4337. utility_func = "PyObject_GenericGetAttr"
  4338. code.globalstate.use_utility_code(UtilityCode.load_cached(utility_func, "ObjectHandling.c"))
  4339. code.putln("if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) &&"
  4340. " likely(!%s.tp_dictoffset && %s.tp_getattro == PyObject_GenericGetAttr)) {" % (
  4341. typeobj_cname, typeobj_cname))
  4342. code.putln("%s.tp_getattro = %s;" % (
  4343. typeobj_cname, py_cfunc))
  4344. code.putln("}")
  4345. # Fix special method docstrings. This is a bit of a hack, but
  4346. # unless we let PyType_Ready create the slot wrappers we have
  4347. # a significant performance hit. (See trac #561.)
  4348. for func in entry.type.scope.pyfunc_entries:
  4349. is_buffer = func.name in ('__getbuffer__', '__releasebuffer__')
  4350. if (func.is_special and Options.docstrings and
  4351. func.wrapperbase_cname and not is_buffer):
  4352. slot = TypeSlots.method_name_to_slot.get(func.name)
  4353. preprocessor_guard = slot.preprocessor_guard_code() if slot else None
  4354. if preprocessor_guard:
  4355. code.putln(preprocessor_guard)
  4356. code.putln('#if CYTHON_UPDATE_DESCRIPTOR_DOC')
  4357. code.putln("{")
  4358. code.putln(
  4359. 'PyObject *wrapper = PyObject_GetAttrString((PyObject *)&%s, "%s"); %s' % (
  4360. typeobj_cname,
  4361. func.name,
  4362. code.error_goto_if_null('wrapper', entry.pos)))
  4363. code.putln(
  4364. "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {")
  4365. code.putln(
  4366. "%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % (
  4367. func.wrapperbase_cname))
  4368. code.putln(
  4369. "%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname))
  4370. code.putln(
  4371. "((PyWrapperDescrObject *)wrapper)->d_base = &%s;" % (
  4372. func.wrapperbase_cname))
  4373. code.putln("}")
  4374. code.putln("}")
  4375. code.putln('#endif')
  4376. if preprocessor_guard:
  4377. code.putln('#endif')
  4378. if type.vtable_cname:
  4379. code.globalstate.use_utility_code(
  4380. UtilityCode.load_cached('SetVTable', 'ImportExport.c'))
  4381. code.putln(
  4382. "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % (
  4383. typeobj_cname,
  4384. type.vtabptr_cname,
  4385. code.error_goto(entry.pos)))
  4386. if heap_type_bases:
  4387. code.globalstate.use_utility_code(
  4388. UtilityCode.load_cached('MergeVTables', 'ImportExport.c'))
  4389. code.putln("if (__Pyx_MergeVtables(&%s) < 0) %s" % (
  4390. typeobj_cname,
  4391. code.error_goto(entry.pos)))
  4392. if not type.scope.is_internal and not type.scope.directives.get('internal'):
  4393. # scope.is_internal is set for types defined by
  4394. # Cython (such as closures), the 'internal'
  4395. # directive is set by users
  4396. code.putln(
  4397. 'if (PyObject_SetAttr(%s, %s, (PyObject *)&%s) < 0) %s' % (
  4398. Naming.module_cname,
  4399. code.intern_identifier(scope.class_name),
  4400. typeobj_cname,
  4401. code.error_goto(entry.pos)))
  4402. weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None
  4403. if weakref_entry:
  4404. if weakref_entry.type is py_object_type:
  4405. tp_weaklistoffset = "%s.tp_weaklistoffset" % typeobj_cname
  4406. if type.typedef_flag:
  4407. objstruct = type.objstruct_cname
  4408. else:
  4409. objstruct = "struct %s" % type.objstruct_cname
  4410. code.putln("if (%s == 0) %s = offsetof(%s, %s);" % (
  4411. tp_weaklistoffset,
  4412. tp_weaklistoffset,
  4413. objstruct,
  4414. weakref_entry.cname))
  4415. else:
  4416. error(weakref_entry.pos, "__weakref__ slot must be of type 'object'")
  4417. if scope.lookup_here("__reduce_cython__") if not scope.is_closure_class_scope else None:
  4418. # Unfortunately, we cannot reliably detect whether a
  4419. # superclass defined __reduce__ at compile time, so we must
  4420. # do so at runtime.
  4421. code.globalstate.use_utility_code(
  4422. UtilityCode.load_cached('SetupReduce', 'ExtensionTypes.c'))
  4423. code.putln('if (__Pyx_setup_reduce((PyObject*)&%s) < 0) %s' % (
  4424. typeobj_cname,
  4425. code.error_goto(entry.pos)))
  4426. # Generate code to initialise the typeptr of an extension
  4427. # type defined in this module to point to its type object.
  4428. if type.typeobj_cname:
  4429. code.putln(
  4430. "%s = &%s;" % (
  4431. type.typeptr_cname, type.typeobj_cname))
  4432. def annotate(self, code):
  4433. if self.type_init_args:
  4434. self.type_init_args.annotate(code)
  4435. if self.body:
  4436. self.body.annotate(code)
  4437. class PropertyNode(StatNode):
  4438. # Definition of a property in an extension type.
  4439. #
  4440. # name string
  4441. # doc EncodedString or None Doc string
  4442. # entry Symtab.Entry
  4443. # body StatListNode
  4444. child_attrs = ["body"]
  4445. def analyse_declarations(self, env):
  4446. self.entry = env.declare_property(self.name, self.doc, self.pos)
  4447. self.entry.scope.directives = env.directives
  4448. self.body.analyse_declarations(self.entry.scope)
  4449. def analyse_expressions(self, env):
  4450. self.body = self.body.analyse_expressions(env)
  4451. return self
  4452. def generate_function_definitions(self, env, code):
  4453. self.body.generate_function_definitions(env, code)
  4454. def generate_execution_code(self, code):
  4455. pass
  4456. def annotate(self, code):
  4457. self.body.annotate(code)
  4458. class GlobalNode(StatNode):
  4459. # Global variable declaration.
  4460. #
  4461. # names [string]
  4462. child_attrs = []
  4463. def analyse_declarations(self, env):
  4464. for name in self.names:
  4465. env.declare_global(name, self.pos)
  4466. def analyse_expressions(self, env):
  4467. return self
  4468. def generate_execution_code(self, code):
  4469. pass
  4470. class NonlocalNode(StatNode):
  4471. # Nonlocal variable declaration via the 'nonlocal' keyword.
  4472. #
  4473. # names [string]
  4474. child_attrs = []
  4475. def analyse_declarations(self, env):
  4476. for name in self.names:
  4477. env.declare_nonlocal(name, self.pos)
  4478. def analyse_expressions(self, env):
  4479. return self
  4480. def generate_execution_code(self, code):
  4481. pass
  4482. class ExprStatNode(StatNode):
  4483. # Expression used as a statement.
  4484. #
  4485. # expr ExprNode
  4486. child_attrs = ["expr"]
  4487. def analyse_declarations(self, env):
  4488. from . import ExprNodes
  4489. expr = self.expr
  4490. if isinstance(expr, ExprNodes.GeneralCallNode):
  4491. func = expr.function.as_cython_attribute()
  4492. if func == u'declare':
  4493. args, kwds = expr.explicit_args_kwds()
  4494. if len(args):
  4495. error(expr.pos, "Variable names must be specified.")
  4496. for var, type_node in kwds.key_value_pairs:
  4497. type = type_node.analyse_as_type(env)
  4498. if type is None:
  4499. error(type_node.pos, "Unknown type")
  4500. else:
  4501. env.declare_var(var.value, type, var.pos, is_cdef=True)
  4502. self.__class__ = PassStatNode
  4503. elif getattr(expr, 'annotation', None) is not None:
  4504. if expr.is_name:
  4505. # non-code variable annotation, e.g. "name: type"
  4506. expr.declare_from_annotation(env)
  4507. self.__class__ = PassStatNode
  4508. elif expr.is_attribute or expr.is_subscript:
  4509. # unused expression with annotation, e.g. "a[0]: type" or "a.xyz : type"
  4510. self.__class__ = PassStatNode
  4511. def analyse_expressions(self, env):
  4512. self.expr.result_is_used = False # hint that .result() may safely be left empty
  4513. self.expr = self.expr.analyse_expressions(env)
  4514. # Repeat in case of node replacement.
  4515. self.expr.result_is_used = False # hint that .result() may safely be left empty
  4516. return self
  4517. def nogil_check(self, env):
  4518. if self.expr.type.is_pyobject and self.expr.is_temp:
  4519. self.gil_error()
  4520. gil_message = "Discarding owned Python object"
  4521. def generate_execution_code(self, code):
  4522. code.mark_pos(self.pos)
  4523. self.expr.result_is_used = False # hint that .result() may safely be left empty
  4524. self.expr.generate_evaluation_code(code)
  4525. if not self.expr.is_temp and self.expr.result():
  4526. result = self.expr.result()
  4527. if not self.expr.type.is_void:
  4528. result = "(void)(%s)" % result
  4529. code.putln("%s;" % result)
  4530. self.expr.generate_disposal_code(code)
  4531. self.expr.free_temps(code)
  4532. def generate_function_definitions(self, env, code):
  4533. self.expr.generate_function_definitions(env, code)
  4534. def annotate(self, code):
  4535. self.expr.annotate(code)
  4536. class AssignmentNode(StatNode):
  4537. # Abstract base class for assignment nodes.
  4538. #
  4539. # The analyse_expressions and generate_execution_code
  4540. # phases of assignments are split into two sub-phases
  4541. # each, to enable all the right hand sides of a
  4542. # parallel assignment to be evaluated before assigning
  4543. # to any of the left hand sides.
  4544. def analyse_expressions(self, env):
  4545. node = self.analyse_types(env)
  4546. if isinstance(node, AssignmentNode) and not isinstance(node, ParallelAssignmentNode):
  4547. if node.rhs.type.is_ptr and node.rhs.is_ephemeral():
  4548. error(self.pos, "Storing unsafe C derivative of temporary Python reference")
  4549. return node
  4550. # def analyse_expressions(self, env):
  4551. # self.analyse_expressions_1(env)
  4552. # self.analyse_expressions_2(env)
  4553. def generate_execution_code(self, code):
  4554. code.mark_pos(self.pos)
  4555. self.generate_rhs_evaluation_code(code)
  4556. self.generate_assignment_code(code)
  4557. class SingleAssignmentNode(AssignmentNode):
  4558. # The simplest case:
  4559. #
  4560. # a = b
  4561. #
  4562. # lhs ExprNode Left hand side
  4563. # rhs ExprNode Right hand side
  4564. # first bool Is this guaranteed the first assignment to lhs?
  4565. # is_overloaded_assignment bool Is this assignment done via an overloaded operator=
  4566. # exception_check
  4567. # exception_value
  4568. child_attrs = ["lhs", "rhs"]
  4569. first = False
  4570. is_overloaded_assignment = False
  4571. declaration_only = False
  4572. def analyse_declarations(self, env):
  4573. from . import ExprNodes
  4574. # handle declarations of the form x = cython.foo()
  4575. if isinstance(self.rhs, ExprNodes.CallNode):
  4576. func_name = self.rhs.function.as_cython_attribute()
  4577. if func_name:
  4578. args, kwds = self.rhs.explicit_args_kwds()
  4579. if func_name in ['declare', 'typedef']:
  4580. if len(args) > 2:
  4581. error(args[2].pos, "Invalid positional argument.")
  4582. return
  4583. if kwds is not None:
  4584. kwdict = kwds.compile_time_value(None)
  4585. if func_name == 'typedef' or 'visibility' not in kwdict:
  4586. error(kwds.pos, "Invalid keyword argument.")
  4587. return
  4588. visibility = kwdict['visibility']
  4589. else:
  4590. visibility = 'private'
  4591. type = args[0].analyse_as_type(env)
  4592. if type is None:
  4593. error(args[0].pos, "Unknown type")
  4594. return
  4595. lhs = self.lhs
  4596. if func_name == 'declare':
  4597. if isinstance(lhs, ExprNodes.NameNode):
  4598. vars = [(lhs.name, lhs.pos)]
  4599. elif isinstance(lhs, ExprNodes.TupleNode):
  4600. vars = [(var.name, var.pos) for var in lhs.args]
  4601. else:
  4602. error(lhs.pos, "Invalid declaration")
  4603. return
  4604. for var, pos in vars:
  4605. env.declare_var(var, type, pos, is_cdef=True, visibility=visibility)
  4606. if len(args) == 2:
  4607. # we have a value
  4608. self.rhs = args[1]
  4609. else:
  4610. self.declaration_only = True
  4611. else:
  4612. self.declaration_only = True
  4613. if not isinstance(lhs, ExprNodes.NameNode):
  4614. error(lhs.pos, "Invalid declaration.")
  4615. env.declare_typedef(lhs.name, type, self.pos, visibility='private')
  4616. elif func_name in ['struct', 'union']:
  4617. self.declaration_only = True
  4618. if len(args) > 0 or kwds is None:
  4619. error(self.rhs.pos, "Struct or union members must be given by name.")
  4620. return
  4621. members = []
  4622. for member, type_node in kwds.key_value_pairs:
  4623. type = type_node.analyse_as_type(env)
  4624. if type is None:
  4625. error(type_node.pos, "Unknown type")
  4626. else:
  4627. members.append((member.value, type, member.pos))
  4628. if len(members) < len(kwds.key_value_pairs):
  4629. return
  4630. if not isinstance(self.lhs, ExprNodes.NameNode):
  4631. error(self.lhs.pos, "Invalid declaration.")
  4632. name = self.lhs.name
  4633. scope = StructOrUnionScope(name)
  4634. env.declare_struct_or_union(name, func_name, scope, False, self.rhs.pos)
  4635. for member, type, pos in members:
  4636. scope.declare_var(member, type, pos)
  4637. elif func_name == 'fused_type':
  4638. # dtype = cython.fused_type(...)
  4639. self.declaration_only = True
  4640. if kwds:
  4641. error(self.rhs.function.pos,
  4642. "fused_type does not take keyword arguments")
  4643. fusednode = FusedTypeNode(self.rhs.pos,
  4644. name=self.lhs.name, types=args)
  4645. fusednode.analyse_declarations(env)
  4646. if self.declaration_only:
  4647. return
  4648. else:
  4649. self.lhs.analyse_target_declaration(env)
  4650. def analyse_types(self, env, use_temp=0):
  4651. from . import ExprNodes
  4652. self.rhs = self.rhs.analyse_types(env)
  4653. unrolled_assignment = self.unroll_rhs(env)
  4654. if unrolled_assignment:
  4655. return unrolled_assignment
  4656. self.lhs = self.lhs.analyse_target_types(env)
  4657. self.lhs.gil_assignment_check(env)
  4658. unrolled_assignment = self.unroll_lhs(env)
  4659. if unrolled_assignment:
  4660. return unrolled_assignment
  4661. if isinstance(self.lhs, ExprNodes.MemoryViewIndexNode):
  4662. self.lhs.analyse_broadcast_operation(self.rhs)
  4663. self.lhs = self.lhs.analyse_as_memview_scalar_assignment(self.rhs)
  4664. elif self.lhs.type.is_array:
  4665. if not isinstance(self.lhs, ExprNodes.SliceIndexNode):
  4666. # cannot assign to C array, only to its full slice
  4667. self.lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None)
  4668. self.lhs = self.lhs.analyse_target_types(env)
  4669. if self.lhs.type.is_cpp_class:
  4670. op = env.lookup_operator_for_types(self.pos, '=', [self.lhs.type, self.rhs.type])
  4671. if op:
  4672. rhs = self.rhs
  4673. self.is_overloaded_assignment = True
  4674. self.exception_check = op.type.exception_check
  4675. self.exception_value = op.type.exception_value
  4676. if self.exception_check == '+' and self.exception_value is None:
  4677. env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp"))
  4678. else:
  4679. rhs = self.rhs.coerce_to(self.lhs.type, env)
  4680. else:
  4681. rhs = self.rhs.coerce_to(self.lhs.type, env)
  4682. if use_temp or rhs.is_attribute or (
  4683. not rhs.is_name and not rhs.is_literal and
  4684. rhs.type.is_pyobject):
  4685. # things like (cdef) attribute access are not safe (traverses pointers)
  4686. rhs = rhs.coerce_to_temp(env)
  4687. elif rhs.type.is_pyobject:
  4688. rhs = rhs.coerce_to_simple(env)
  4689. self.rhs = rhs
  4690. return self
  4691. def unroll(self, node, target_size, env):
  4692. from . import ExprNodes, UtilNodes
  4693. base = node
  4694. start_node = stop_node = step_node = check_node = None
  4695. if node.type.is_ctuple:
  4696. slice_size = node.type.size
  4697. elif node.type.is_ptr or node.type.is_array:
  4698. while isinstance(node, ExprNodes.SliceIndexNode) and not (node.start or node.stop):
  4699. base = node = node.base
  4700. if isinstance(node, ExprNodes.SliceIndexNode):
  4701. base = node.base
  4702. start_node = node.start
  4703. if start_node:
  4704. start_node = start_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
  4705. stop_node = node.stop
  4706. if stop_node:
  4707. stop_node = stop_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
  4708. else:
  4709. if node.type.is_array and node.type.size:
  4710. stop_node = ExprNodes.IntNode(
  4711. self.pos, value=str(node.type.size),
  4712. constant_result=(node.type.size if isinstance(node.type.size, _py_int_types)
  4713. else ExprNodes.constant_value_not_set))
  4714. else:
  4715. error(self.pos, "C array iteration requires known end index")
  4716. return
  4717. step_node = None #node.step
  4718. if step_node:
  4719. step_node = step_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
  4720. # TODO: Factor out SliceIndexNode.generate_slice_guard_code() for use here.
  4721. def get_const(node, none_value):
  4722. if node is None:
  4723. return none_value
  4724. elif node.has_constant_result():
  4725. return node.constant_result
  4726. else:
  4727. raise ValueError("Not a constant.")
  4728. try:
  4729. slice_size = (get_const(stop_node, None) - get_const(start_node, 0)) / get_const(step_node, 1)
  4730. except ValueError:
  4731. error(self.pos, "C array assignment currently requires known endpoints")
  4732. return
  4733. elif node.type.is_array:
  4734. slice_size = node.type.size
  4735. if not isinstance(slice_size, _py_int_types):
  4736. return # might still work when coercing to Python
  4737. else:
  4738. return
  4739. else:
  4740. return
  4741. if slice_size != target_size:
  4742. error(self.pos, "Assignment to/from slice of wrong length, expected %s, got %s" % (
  4743. slice_size, target_size))
  4744. return
  4745. items = []
  4746. base = UtilNodes.LetRefNode(base)
  4747. refs = [base]
  4748. if start_node and not start_node.is_literal:
  4749. start_node = UtilNodes.LetRefNode(start_node)
  4750. refs.append(start_node)
  4751. if stop_node and not stop_node.is_literal:
  4752. stop_node = UtilNodes.LetRefNode(stop_node)
  4753. refs.append(stop_node)
  4754. if step_node and not step_node.is_literal:
  4755. step_node = UtilNodes.LetRefNode(step_node)
  4756. refs.append(step_node)
  4757. for ix in range(target_size):
  4758. ix_node = ExprNodes.IntNode(self.pos, value=str(ix), constant_result=ix, type=PyrexTypes.c_py_ssize_t_type)
  4759. if step_node is not None:
  4760. if step_node.has_constant_result():
  4761. step_value = ix_node.constant_result * step_node.constant_result
  4762. ix_node = ExprNodes.IntNode(self.pos, value=str(step_value), constant_result=step_value)
  4763. else:
  4764. ix_node = ExprNodes.MulNode(self.pos, operator='*', operand1=step_node, operand2=ix_node)
  4765. if start_node is not None:
  4766. if start_node.has_constant_result() and ix_node.has_constant_result():
  4767. index_value = ix_node.constant_result + start_node.constant_result
  4768. ix_node = ExprNodes.IntNode(self.pos, value=str(index_value), constant_result=index_value)
  4769. else:
  4770. ix_node = ExprNodes.AddNode(
  4771. self.pos, operator='+', operand1=start_node, operand2=ix_node)
  4772. items.append(ExprNodes.IndexNode(self.pos, base=base, index=ix_node.analyse_types(env)))
  4773. return check_node, refs, items
  4774. def unroll_assignments(self, refs, check_node, lhs_list, rhs_list, env):
  4775. from . import UtilNodes
  4776. assignments = []
  4777. for lhs, rhs in zip(lhs_list, rhs_list):
  4778. assignments.append(SingleAssignmentNode(self.pos, lhs=lhs, rhs=rhs, first=self.first))
  4779. node = ParallelAssignmentNode(pos=self.pos, stats=assignments).analyse_expressions(env)
  4780. if check_node:
  4781. node = StatListNode(pos=self.pos, stats=[check_node, node])
  4782. for ref in refs[::-1]:
  4783. node = UtilNodes.LetNode(ref, node)
  4784. return node
  4785. def unroll_rhs(self, env):
  4786. from . import ExprNodes
  4787. if not isinstance(self.lhs, ExprNodes.TupleNode):
  4788. return
  4789. if any(arg.is_starred for arg in self.lhs.args):
  4790. return
  4791. unrolled = self.unroll(self.rhs, len(self.lhs.args), env)
  4792. if not unrolled:
  4793. return
  4794. check_node, refs, rhs = unrolled
  4795. return self.unroll_assignments(refs, check_node, self.lhs.args, rhs, env)
  4796. def unroll_lhs(self, env):
  4797. if self.lhs.type.is_ctuple:
  4798. # Handled directly.
  4799. return
  4800. from . import ExprNodes
  4801. if not isinstance(self.rhs, ExprNodes.TupleNode):
  4802. return
  4803. unrolled = self.unroll(self.lhs, len(self.rhs.args), env)
  4804. if not unrolled:
  4805. return
  4806. check_node, refs, lhs = unrolled
  4807. return self.unroll_assignments(refs, check_node, lhs, self.rhs.args, env)
  4808. def generate_rhs_evaluation_code(self, code):
  4809. self.rhs.generate_evaluation_code(code)
  4810. def generate_assignment_code(self, code, overloaded_assignment=False):
  4811. if self.is_overloaded_assignment:
  4812. self.lhs.generate_assignment_code(
  4813. self.rhs,
  4814. code,
  4815. overloaded_assignment=self.is_overloaded_assignment,
  4816. exception_check=self.exception_check,
  4817. exception_value=self.exception_value)
  4818. else:
  4819. self.lhs.generate_assignment_code(self.rhs, code)
  4820. def generate_function_definitions(self, env, code):
  4821. self.rhs.generate_function_definitions(env, code)
  4822. def annotate(self, code):
  4823. self.lhs.annotate(code)
  4824. self.rhs.annotate(code)
  4825. class CascadedAssignmentNode(AssignmentNode):
  4826. # An assignment with multiple left hand sides:
  4827. #
  4828. # a = b = c
  4829. #
  4830. # lhs_list [ExprNode] Left hand sides
  4831. # rhs ExprNode Right hand sides
  4832. #
  4833. # Used internally:
  4834. #
  4835. # coerced_values [ExprNode] RHS coerced to all distinct LHS types
  4836. # cloned_values [ExprNode] cloned RHS value for each LHS
  4837. # assignment_overloads [Bool] If each assignment uses a C++ operator=
  4838. child_attrs = ["lhs_list", "rhs", "coerced_values", "cloned_values"]
  4839. cloned_values = None
  4840. coerced_values = None
  4841. assignment_overloads = None
  4842. def analyse_declarations(self, env):
  4843. for lhs in self.lhs_list:
  4844. lhs.analyse_target_declaration(env)
  4845. def analyse_types(self, env, use_temp=0):
  4846. from .ExprNodes import CloneNode, ProxyNode
  4847. # collect distinct types used on the LHS
  4848. lhs_types = set()
  4849. for i, lhs in enumerate(self.lhs_list):
  4850. lhs = self.lhs_list[i] = lhs.analyse_target_types(env)
  4851. lhs.gil_assignment_check(env)
  4852. lhs_types.add(lhs.type)
  4853. rhs = self.rhs.analyse_types(env)
  4854. # common special case: only one type needed on the LHS => coerce only once
  4855. if len(lhs_types) == 1:
  4856. # Avoid coercion for overloaded assignment operators.
  4857. if next(iter(lhs_types)).is_cpp_class:
  4858. op = env.lookup_operator('=', [lhs, self.rhs])
  4859. if not op:
  4860. rhs = rhs.coerce_to(lhs_types.pop(), env)
  4861. else:
  4862. rhs = rhs.coerce_to(lhs_types.pop(), env)
  4863. if not rhs.is_name and not rhs.is_literal and (
  4864. use_temp or rhs.is_attribute or rhs.type.is_pyobject):
  4865. rhs = rhs.coerce_to_temp(env)
  4866. else:
  4867. rhs = rhs.coerce_to_simple(env)
  4868. self.rhs = ProxyNode(rhs) if rhs.is_temp else rhs
  4869. # clone RHS and coerce it to all distinct LHS types
  4870. self.coerced_values = []
  4871. coerced_values = {}
  4872. self.assignment_overloads = []
  4873. for lhs in self.lhs_list:
  4874. overloaded = lhs.type.is_cpp_class and env.lookup_operator('=', [lhs, self.rhs])
  4875. self.assignment_overloads.append(overloaded)
  4876. if lhs.type not in coerced_values and lhs.type != rhs.type:
  4877. rhs = CloneNode(self.rhs)
  4878. if not overloaded:
  4879. rhs = rhs.coerce_to(lhs.type, env)
  4880. self.coerced_values.append(rhs)
  4881. coerced_values[lhs.type] = rhs
  4882. # clone coerced values for all LHS assignments
  4883. self.cloned_values = []
  4884. for lhs in self.lhs_list:
  4885. rhs = coerced_values.get(lhs.type, self.rhs)
  4886. self.cloned_values.append(CloneNode(rhs))
  4887. return self
  4888. def generate_rhs_evaluation_code(self, code):
  4889. self.rhs.generate_evaluation_code(code)
  4890. def generate_assignment_code(self, code, overloaded_assignment=False):
  4891. # prepare all coercions
  4892. for rhs in self.coerced_values:
  4893. rhs.generate_evaluation_code(code)
  4894. # assign clones to LHS
  4895. for lhs, rhs, overload in zip(self.lhs_list, self.cloned_values, self.assignment_overloads):
  4896. rhs.generate_evaluation_code(code)
  4897. lhs.generate_assignment_code(rhs, code, overloaded_assignment=overload)
  4898. # dispose of coerced values and original RHS
  4899. for rhs_value in self.coerced_values:
  4900. rhs_value.generate_disposal_code(code)
  4901. rhs_value.free_temps(code)
  4902. self.rhs.generate_disposal_code(code)
  4903. self.rhs.free_temps(code)
  4904. def generate_function_definitions(self, env, code):
  4905. self.rhs.generate_function_definitions(env, code)
  4906. def annotate(self, code):
  4907. for rhs in self.coerced_values:
  4908. rhs.annotate(code)
  4909. for lhs, rhs in zip(self.lhs_list, self.cloned_values):
  4910. lhs.annotate(code)
  4911. rhs.annotate(code)
  4912. self.rhs.annotate(code)
  4913. class ParallelAssignmentNode(AssignmentNode):
  4914. # A combined packing/unpacking assignment:
  4915. #
  4916. # a, b, c = d, e, f
  4917. #
  4918. # This has been rearranged by the parser into
  4919. #
  4920. # a = d ; b = e ; c = f
  4921. #
  4922. # but we must evaluate all the right hand sides
  4923. # before assigning to any of the left hand sides.
  4924. #
  4925. # stats [AssignmentNode] The constituent assignments
  4926. child_attrs = ["stats"]
  4927. def analyse_declarations(self, env):
  4928. for stat in self.stats:
  4929. stat.analyse_declarations(env)
  4930. def analyse_expressions(self, env):
  4931. self.stats = [stat.analyse_types(env, use_temp=1)
  4932. for stat in self.stats]
  4933. return self
  4934. # def analyse_expressions(self, env):
  4935. # for stat in self.stats:
  4936. # stat.analyse_expressions_1(env, use_temp=1)
  4937. # for stat in self.stats:
  4938. # stat.analyse_expressions_2(env)
  4939. def generate_execution_code(self, code):
  4940. code.mark_pos(self.pos)
  4941. for stat in self.stats:
  4942. stat.generate_rhs_evaluation_code(code)
  4943. for stat in self.stats:
  4944. stat.generate_assignment_code(code)
  4945. def generate_function_definitions(self, env, code):
  4946. for stat in self.stats:
  4947. stat.generate_function_definitions(env, code)
  4948. def annotate(self, code):
  4949. for stat in self.stats:
  4950. stat.annotate(code)
  4951. class InPlaceAssignmentNode(AssignmentNode):
  4952. # An in place arithmetic operand:
  4953. #
  4954. # a += b
  4955. # a -= b
  4956. # ...
  4957. #
  4958. # lhs ExprNode Left hand side
  4959. # rhs ExprNode Right hand side
  4960. # operator char one of "+-*/%^&|"
  4961. #
  4962. # This code is a bit tricky because in order to obey Python
  4963. # semantics the sub-expressions (e.g. indices) of the lhs must
  4964. # not be evaluated twice. So we must re-use the values calculated
  4965. # in evaluation phase for the assignment phase as well.
  4966. # Fortunately, the type of the lhs node is fairly constrained
  4967. # (it must be a NameNode, AttributeNode, or IndexNode).
  4968. child_attrs = ["lhs", "rhs"]
  4969. def analyse_declarations(self, env):
  4970. self.lhs.analyse_target_declaration(env)
  4971. def analyse_types(self, env):
  4972. self.rhs = self.rhs.analyse_types(env)
  4973. self.lhs = self.lhs.analyse_target_types(env)
  4974. # When assigning to a fully indexed buffer or memoryview, coerce the rhs
  4975. if self.lhs.is_memview_index or self.lhs.is_buffer_access:
  4976. self.rhs = self.rhs.coerce_to(self.lhs.type, env)
  4977. elif self.lhs.type.is_string and self.operator in '+-':
  4978. # use pointer arithmetic for char* LHS instead of string concat
  4979. self.rhs = self.rhs.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
  4980. return self
  4981. def generate_execution_code(self, code):
  4982. code.mark_pos(self.pos)
  4983. lhs, rhs = self.lhs, self.rhs
  4984. rhs.generate_evaluation_code(code)
  4985. lhs.generate_subexpr_evaluation_code(code)
  4986. c_op = self.operator
  4987. if c_op == "//":
  4988. c_op = "/"
  4989. elif c_op == "**":
  4990. error(self.pos, "No C inplace power operator")
  4991. if lhs.is_buffer_access or lhs.is_memview_index:
  4992. if lhs.type.is_pyobject:
  4993. error(self.pos, "In-place operators not allowed on object buffers in this release.")
  4994. if c_op in ('/', '%') and lhs.type.is_int and not code.globalstate.directives['cdivision']:
  4995. error(self.pos, "In-place non-c divide operators not allowed on int buffers.")
  4996. lhs.generate_buffer_setitem_code(rhs, code, c_op)
  4997. elif lhs.is_memview_slice:
  4998. error(self.pos, "Inplace operators not supported on memoryview slices")
  4999. else:
  5000. # C++
  5001. # TODO: make sure overload is declared
  5002. code.putln("%s %s= %s;" % (lhs.result(), c_op, rhs.result()))
  5003. lhs.generate_subexpr_disposal_code(code)
  5004. lhs.free_subexpr_temps(code)
  5005. rhs.generate_disposal_code(code)
  5006. rhs.free_temps(code)
  5007. def annotate(self, code):
  5008. self.lhs.annotate(code)
  5009. self.rhs.annotate(code)
  5010. def create_binop_node(self):
  5011. from . import ExprNodes
  5012. return ExprNodes.binop_node(self.pos, self.operator, self.lhs, self.rhs)
  5013. class PrintStatNode(StatNode):
  5014. # print statement
  5015. #
  5016. # arg_tuple TupleNode
  5017. # stream ExprNode or None (stdout)
  5018. # append_newline boolean
  5019. child_attrs = ["arg_tuple", "stream"]
  5020. def analyse_expressions(self, env):
  5021. if self.stream:
  5022. stream = self.stream.analyse_expressions(env)
  5023. self.stream = stream.coerce_to_pyobject(env)
  5024. arg_tuple = self.arg_tuple.analyse_expressions(env)
  5025. self.arg_tuple = arg_tuple.coerce_to_pyobject(env)
  5026. env.use_utility_code(printing_utility_code)
  5027. if len(self.arg_tuple.args) == 1 and self.append_newline:
  5028. env.use_utility_code(printing_one_utility_code)
  5029. return self
  5030. nogil_check = Node.gil_error
  5031. gil_message = "Python print statement"
  5032. def generate_execution_code(self, code):
  5033. code.mark_pos(self.pos)
  5034. if self.stream:
  5035. self.stream.generate_evaluation_code(code)
  5036. stream_result = self.stream.py_result()
  5037. else:
  5038. stream_result = '0'
  5039. if len(self.arg_tuple.args) == 1 and self.append_newline:
  5040. arg = self.arg_tuple.args[0]
  5041. arg.generate_evaluation_code(code)
  5042. code.putln(
  5043. "if (__Pyx_PrintOne(%s, %s) < 0) %s" % (
  5044. stream_result,
  5045. arg.py_result(),
  5046. code.error_goto(self.pos)))
  5047. arg.generate_disposal_code(code)
  5048. arg.free_temps(code)
  5049. else:
  5050. self.arg_tuple.generate_evaluation_code(code)
  5051. code.putln(
  5052. "if (__Pyx_Print(%s, %s, %d) < 0) %s" % (
  5053. stream_result,
  5054. self.arg_tuple.py_result(),
  5055. self.append_newline,
  5056. code.error_goto(self.pos)))
  5057. self.arg_tuple.generate_disposal_code(code)
  5058. self.arg_tuple.free_temps(code)
  5059. if self.stream:
  5060. self.stream.generate_disposal_code(code)
  5061. self.stream.free_temps(code)
  5062. def generate_function_definitions(self, env, code):
  5063. if self.stream:
  5064. self.stream.generate_function_definitions(env, code)
  5065. self.arg_tuple.generate_function_definitions(env, code)
  5066. def annotate(self, code):
  5067. if self.stream:
  5068. self.stream.annotate(code)
  5069. self.arg_tuple.annotate(code)
  5070. class ExecStatNode(StatNode):
  5071. # exec statement
  5072. #
  5073. # args [ExprNode]
  5074. child_attrs = ["args"]
  5075. def analyse_expressions(self, env):
  5076. for i, arg in enumerate(self.args):
  5077. arg = arg.analyse_expressions(env)
  5078. arg = arg.coerce_to_pyobject(env)
  5079. self.args[i] = arg
  5080. env.use_utility_code(Builtin.pyexec_utility_code)
  5081. return self
  5082. nogil_check = Node.gil_error
  5083. gil_message = "Python exec statement"
  5084. def generate_execution_code(self, code):
  5085. code.mark_pos(self.pos)
  5086. args = []
  5087. for arg in self.args:
  5088. arg.generate_evaluation_code(code)
  5089. args.append(arg.py_result())
  5090. args = tuple(args + ['0', '0'][:3-len(args)])
  5091. temp_result = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
  5092. code.putln("%s = __Pyx_PyExec3(%s, %s, %s);" % ((temp_result,) + args))
  5093. for arg in self.args:
  5094. arg.generate_disposal_code(code)
  5095. arg.free_temps(code)
  5096. code.putln(
  5097. code.error_goto_if_null(temp_result, self.pos))
  5098. code.put_gotref(temp_result)
  5099. code.put_decref_clear(temp_result, py_object_type)
  5100. code.funcstate.release_temp(temp_result)
  5101. def annotate(self, code):
  5102. for arg in self.args:
  5103. arg.annotate(code)
  5104. class DelStatNode(StatNode):
  5105. # del statement
  5106. #
  5107. # args [ExprNode]
  5108. child_attrs = ["args"]
  5109. ignore_nonexisting = False
  5110. def analyse_declarations(self, env):
  5111. for arg in self.args:
  5112. arg.analyse_target_declaration(env)
  5113. def analyse_expressions(self, env):
  5114. for i, arg in enumerate(self.args):
  5115. arg = self.args[i] = arg.analyse_target_expression(env, None)
  5116. if arg.type.is_pyobject or (arg.is_name and arg.type.is_memoryviewslice):
  5117. if arg.is_name and arg.entry.is_cglobal:
  5118. error(arg.pos, "Deletion of global C variable")
  5119. elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
  5120. self.cpp_check(env)
  5121. elif arg.type.is_cpp_class:
  5122. error(arg.pos, "Deletion of non-heap C++ object")
  5123. elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
  5124. pass # del ba[i]
  5125. else:
  5126. error(arg.pos, "Deletion of non-Python, non-C++ object")
  5127. #arg.release_target_temp(env)
  5128. return self
  5129. def nogil_check(self, env):
  5130. for arg in self.args:
  5131. if arg.type.is_pyobject:
  5132. self.gil_error()
  5133. gil_message = "Deleting Python object"
  5134. def generate_execution_code(self, code):
  5135. code.mark_pos(self.pos)
  5136. for arg in self.args:
  5137. if (arg.type.is_pyobject or
  5138. arg.type.is_memoryviewslice or
  5139. arg.is_subscript and arg.base.type is Builtin.bytearray_type):
  5140. arg.generate_deletion_code(
  5141. code, ignore_nonexisting=self.ignore_nonexisting)
  5142. elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
  5143. arg.generate_evaluation_code(code)
  5144. code.putln("delete %s;" % arg.result())
  5145. arg.generate_disposal_code(code)
  5146. arg.free_temps(code)
  5147. # else error reported earlier
  5148. def annotate(self, code):
  5149. for arg in self.args:
  5150. arg.annotate(code)
  5151. class PassStatNode(StatNode):
  5152. # pass statement
  5153. child_attrs = []
  5154. def analyse_expressions(self, env):
  5155. return self
  5156. def generate_execution_code(self, code):
  5157. pass
  5158. class IndirectionNode(StatListNode):
  5159. """
  5160. This adds an indirection so that the node can be shared and a subtree can
  5161. be removed at any time by clearing self.stats.
  5162. """
  5163. def __init__(self, stats):
  5164. super(IndirectionNode, self).__init__(stats[0].pos, stats=stats)
  5165. class BreakStatNode(StatNode):
  5166. child_attrs = []
  5167. is_terminator = True
  5168. def analyse_expressions(self, env):
  5169. return self
  5170. def generate_execution_code(self, code):
  5171. code.mark_pos(self.pos)
  5172. if not code.break_label:
  5173. error(self.pos, "break statement not inside loop")
  5174. else:
  5175. code.put_goto(code.break_label)
  5176. class ContinueStatNode(StatNode):
  5177. child_attrs = []
  5178. is_terminator = True
  5179. def analyse_expressions(self, env):
  5180. return self
  5181. def generate_execution_code(self, code):
  5182. if not code.continue_label:
  5183. error(self.pos, "continue statement not inside loop")
  5184. return
  5185. code.mark_pos(self.pos)
  5186. code.put_goto(code.continue_label)
  5187. class ReturnStatNode(StatNode):
  5188. # return statement
  5189. #
  5190. # value ExprNode or None
  5191. # return_type PyrexType
  5192. # in_generator return inside of generator => raise StopIteration
  5193. # in_async_gen return inside of async generator
  5194. child_attrs = ["value"]
  5195. is_terminator = True
  5196. in_generator = False
  5197. in_async_gen = False
  5198. # Whether we are in a parallel section
  5199. in_parallel = False
  5200. def analyse_expressions(self, env):
  5201. return_type = env.return_type
  5202. self.return_type = return_type
  5203. if not return_type:
  5204. error(self.pos, "Return not inside a function body")
  5205. return self
  5206. if self.value:
  5207. if self.in_async_gen:
  5208. error(self.pos, "Return with value in async generator")
  5209. self.value = self.value.analyse_types(env)
  5210. if return_type.is_void or return_type.is_returncode:
  5211. error(self.value.pos, "Return with value in void function")
  5212. else:
  5213. self.value = self.value.coerce_to(env.return_type, env)
  5214. else:
  5215. if (not return_type.is_void
  5216. and not return_type.is_pyobject
  5217. and not return_type.is_returncode):
  5218. error(self.pos, "Return value required")
  5219. return self
  5220. def nogil_check(self, env):
  5221. if self.return_type.is_pyobject:
  5222. self.gil_error()
  5223. gil_message = "Returning Python object"
  5224. def generate_execution_code(self, code):
  5225. code.mark_pos(self.pos)
  5226. if not self.return_type:
  5227. # error reported earlier
  5228. return
  5229. value = self.value
  5230. if self.return_type.is_pyobject:
  5231. code.put_xdecref(Naming.retval_cname, self.return_type)
  5232. if value and value.is_none:
  5233. # Use specialised default handling for "return None".
  5234. value = None
  5235. if value:
  5236. value.generate_evaluation_code(code)
  5237. if self.return_type.is_memoryviewslice:
  5238. from . import MemoryView
  5239. MemoryView.put_acquire_memoryviewslice(
  5240. lhs_cname=Naming.retval_cname,
  5241. lhs_type=self.return_type,
  5242. lhs_pos=value.pos,
  5243. rhs=value,
  5244. code=code,
  5245. have_gil=self.in_nogil_context)
  5246. value.generate_post_assignment_code(code)
  5247. elif self.in_generator:
  5248. # return value == raise StopIteration(value), but uncatchable
  5249. code.globalstate.use_utility_code(
  5250. UtilityCode.load_cached("ReturnWithStopIteration", "Coroutine.c"))
  5251. code.putln("%s = NULL; __Pyx_ReturnWithStopIteration(%s);" % (
  5252. Naming.retval_cname,
  5253. value.py_result()))
  5254. value.generate_disposal_code(code)
  5255. else:
  5256. value.make_owned_reference(code)
  5257. code.putln("%s = %s;" % (
  5258. Naming.retval_cname,
  5259. value.result_as(self.return_type)))
  5260. value.generate_post_assignment_code(code)
  5261. value.free_temps(code)
  5262. else:
  5263. if self.return_type.is_pyobject:
  5264. if self.in_generator:
  5265. if self.in_async_gen:
  5266. code.globalstate.use_utility_code(
  5267. UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
  5268. code.put("PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); ")
  5269. code.putln("%s = NULL;" % Naming.retval_cname)
  5270. else:
  5271. code.put_init_to_py_none(Naming.retval_cname, self.return_type)
  5272. elif self.return_type.is_returncode:
  5273. self.put_return(code, self.return_type.default_value)
  5274. for cname, type in code.funcstate.temps_holding_reference():
  5275. code.put_decref_clear(cname, type)
  5276. code.put_goto(code.return_label)
  5277. def put_return(self, code, value):
  5278. if self.in_parallel:
  5279. code.putln_openmp("#pragma omp critical(__pyx_returning)")
  5280. code.putln("%s = %s;" % (Naming.retval_cname, value))
  5281. def generate_function_definitions(self, env, code):
  5282. if self.value is not None:
  5283. self.value.generate_function_definitions(env, code)
  5284. def annotate(self, code):
  5285. if self.value:
  5286. self.value.annotate(code)
  5287. class RaiseStatNode(StatNode):
  5288. # raise statement
  5289. #
  5290. # exc_type ExprNode or None
  5291. # exc_value ExprNode or None
  5292. # exc_tb ExprNode or None
  5293. # cause ExprNode or None
  5294. child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"]
  5295. is_terminator = True
  5296. def analyse_expressions(self, env):
  5297. if self.exc_type:
  5298. exc_type = self.exc_type.analyse_types(env)
  5299. self.exc_type = exc_type.coerce_to_pyobject(env)
  5300. if self.exc_value:
  5301. exc_value = self.exc_value.analyse_types(env)
  5302. self.exc_value = exc_value.coerce_to_pyobject(env)
  5303. if self.exc_tb:
  5304. exc_tb = self.exc_tb.analyse_types(env)
  5305. self.exc_tb = exc_tb.coerce_to_pyobject(env)
  5306. if self.cause:
  5307. cause = self.cause.analyse_types(env)
  5308. self.cause = cause.coerce_to_pyobject(env)
  5309. # special cases for builtin exceptions
  5310. self.builtin_exc_name = None
  5311. if self.exc_type and not self.exc_value and not self.exc_tb:
  5312. exc = self.exc_type
  5313. from . import ExprNodes
  5314. if (isinstance(exc, ExprNodes.SimpleCallNode) and
  5315. not (exc.args or (exc.arg_tuple is not None and exc.arg_tuple.args))):
  5316. exc = exc.function # extract the exception type
  5317. if exc.is_name and exc.entry.is_builtin:
  5318. self.builtin_exc_name = exc.name
  5319. if self.builtin_exc_name == 'MemoryError':
  5320. self.exc_type = None # has a separate implementation
  5321. return self
  5322. nogil_check = Node.gil_error
  5323. gil_message = "Raising exception"
  5324. def generate_execution_code(self, code):
  5325. code.mark_pos(self.pos)
  5326. if self.builtin_exc_name == 'MemoryError':
  5327. code.putln('PyErr_NoMemory(); %s' % code.error_goto(self.pos))
  5328. return
  5329. if self.exc_type:
  5330. self.exc_type.generate_evaluation_code(code)
  5331. type_code = self.exc_type.py_result()
  5332. if self.exc_type.is_name:
  5333. code.globalstate.use_entry_utility_code(self.exc_type.entry)
  5334. else:
  5335. type_code = "0"
  5336. if self.exc_value:
  5337. self.exc_value.generate_evaluation_code(code)
  5338. value_code = self.exc_value.py_result()
  5339. else:
  5340. value_code = "0"
  5341. if self.exc_tb:
  5342. self.exc_tb.generate_evaluation_code(code)
  5343. tb_code = self.exc_tb.py_result()
  5344. else:
  5345. tb_code = "0"
  5346. if self.cause:
  5347. self.cause.generate_evaluation_code(code)
  5348. cause_code = self.cause.py_result()
  5349. else:
  5350. cause_code = "0"
  5351. code.globalstate.use_utility_code(raise_utility_code)
  5352. code.putln(
  5353. "__Pyx_Raise(%s, %s, %s, %s);" % (
  5354. type_code,
  5355. value_code,
  5356. tb_code,
  5357. cause_code))
  5358. for obj in (self.exc_type, self.exc_value, self.exc_tb, self.cause):
  5359. if obj:
  5360. obj.generate_disposal_code(code)
  5361. obj.free_temps(code)
  5362. code.putln(
  5363. code.error_goto(self.pos))
  5364. def generate_function_definitions(self, env, code):
  5365. if self.exc_type is not None:
  5366. self.exc_type.generate_function_definitions(env, code)
  5367. if self.exc_value is not None:
  5368. self.exc_value.generate_function_definitions(env, code)
  5369. if self.exc_tb is not None:
  5370. self.exc_tb.generate_function_definitions(env, code)
  5371. if self.cause is not None:
  5372. self.cause.generate_function_definitions(env, code)
  5373. def annotate(self, code):
  5374. if self.exc_type:
  5375. self.exc_type.annotate(code)
  5376. if self.exc_value:
  5377. self.exc_value.annotate(code)
  5378. if self.exc_tb:
  5379. self.exc_tb.annotate(code)
  5380. if self.cause:
  5381. self.cause.annotate(code)
  5382. class ReraiseStatNode(StatNode):
  5383. child_attrs = []
  5384. is_terminator = True
  5385. def analyse_expressions(self, env):
  5386. return self
  5387. nogil_check = Node.gil_error
  5388. gil_message = "Raising exception"
  5389. def generate_execution_code(self, code):
  5390. code.mark_pos(self.pos)
  5391. vars = code.funcstate.exc_vars
  5392. if vars:
  5393. code.globalstate.use_utility_code(restore_exception_utility_code)
  5394. code.put_giveref(vars[0])
  5395. code.put_giveref(vars[1])
  5396. # fresh exceptions may not have a traceback yet (-> finally!)
  5397. code.put_xgiveref(vars[2])
  5398. code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars))
  5399. for varname in vars:
  5400. code.put("%s = 0; " % varname)
  5401. code.putln()
  5402. code.putln(code.error_goto(self.pos))
  5403. else:
  5404. code.globalstate.use_utility_code(
  5405. UtilityCode.load_cached("ReRaiseException", "Exceptions.c"))
  5406. code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos))
  5407. class AssertStatNode(StatNode):
  5408. # assert statement
  5409. #
  5410. # cond ExprNode
  5411. # value ExprNode or None
  5412. child_attrs = ["cond", "value"]
  5413. def analyse_expressions(self, env):
  5414. self.cond = self.cond.analyse_boolean_expression(env)
  5415. if self.value:
  5416. value = self.value.analyse_types(env)
  5417. if value.type is Builtin.tuple_type or not value.type.is_builtin_type:
  5418. # prevent tuple values from being interpreted as argument value tuples
  5419. from .ExprNodes import TupleNode
  5420. value = TupleNode(value.pos, args=[value], slow=True)
  5421. self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env)
  5422. else:
  5423. self.value = value.coerce_to_pyobject(env)
  5424. return self
  5425. nogil_check = Node.gil_error
  5426. gil_message = "Raising exception"
  5427. def generate_execution_code(self, code):
  5428. code.globalstate.use_utility_code(
  5429. UtilityCode.load_cached("AssertionsEnabled", "Exceptions.c"))
  5430. code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS")
  5431. code.putln("if (unlikely(__pyx_assertions_enabled())) {")
  5432. code.mark_pos(self.pos)
  5433. self.cond.generate_evaluation_code(code)
  5434. code.putln(
  5435. "if (unlikely(!%s)) {" % self.cond.result())
  5436. if self.value:
  5437. self.value.generate_evaluation_code(code)
  5438. code.putln(
  5439. "PyErr_SetObject(PyExc_AssertionError, %s);" % self.value.py_result())
  5440. self.value.generate_disposal_code(code)
  5441. self.value.free_temps(code)
  5442. else:
  5443. code.putln(
  5444. "PyErr_SetNone(PyExc_AssertionError);")
  5445. code.putln(
  5446. code.error_goto(self.pos))
  5447. code.putln(
  5448. "}")
  5449. self.cond.generate_disposal_code(code)
  5450. self.cond.free_temps(code)
  5451. code.putln(
  5452. "}")
  5453. code.putln("#endif")
  5454. def generate_function_definitions(self, env, code):
  5455. self.cond.generate_function_definitions(env, code)
  5456. if self.value is not None:
  5457. self.value.generate_function_definitions(env, code)
  5458. def annotate(self, code):
  5459. self.cond.annotate(code)
  5460. if self.value:
  5461. self.value.annotate(code)
  5462. class IfStatNode(StatNode):
  5463. # if statement
  5464. #
  5465. # if_clauses [IfClauseNode]
  5466. # else_clause StatNode or None
  5467. child_attrs = ["if_clauses", "else_clause"]
  5468. def analyse_declarations(self, env):
  5469. for if_clause in self.if_clauses:
  5470. if_clause.analyse_declarations(env)
  5471. if self.else_clause:
  5472. self.else_clause.analyse_declarations(env)
  5473. def analyse_expressions(self, env):
  5474. self.if_clauses = [if_clause.analyse_expressions(env) for if_clause in self.if_clauses]
  5475. if self.else_clause:
  5476. self.else_clause = self.else_clause.analyse_expressions(env)
  5477. return self
  5478. def generate_execution_code(self, code):
  5479. code.mark_pos(self.pos)
  5480. end_label = code.new_label()
  5481. last = len(self.if_clauses)
  5482. if self.else_clause:
  5483. # If the 'else' clause is 'unlikely', then set the preceding 'if' clause to 'likely' to reflect that.
  5484. self._set_branch_hint(self.if_clauses[-1], self.else_clause, inverse=True)
  5485. else:
  5486. last -= 1 # avoid redundant goto at end of last if-clause
  5487. for i, if_clause in enumerate(self.if_clauses):
  5488. self._set_branch_hint(if_clause, if_clause.body)
  5489. if_clause.generate_execution_code(code, end_label, is_last=i == last)
  5490. if self.else_clause:
  5491. code.mark_pos(self.else_clause.pos)
  5492. code.putln("/*else*/ {")
  5493. self.else_clause.generate_execution_code(code)
  5494. code.putln("}")
  5495. code.put_label(end_label)
  5496. def _set_branch_hint(self, clause, statements_node, inverse=False):
  5497. if not statements_node.is_terminator:
  5498. return
  5499. if not isinstance(statements_node, StatListNode) or not statements_node.stats:
  5500. return
  5501. # Anything that unconditionally raises exceptions should be considered unlikely.
  5502. if isinstance(statements_node.stats[-1], (RaiseStatNode, ReraiseStatNode)):
  5503. if len(statements_node.stats) > 1:
  5504. # Allow simple statements before the 'raise', but no conditions, loops, etc.
  5505. non_branch_nodes = (ExprStatNode, AssignmentNode, DelStatNode, GlobalNode, NonlocalNode)
  5506. for node in statements_node.stats[:-1]:
  5507. if not isinstance(node, non_branch_nodes):
  5508. return
  5509. clause.branch_hint = 'likely' if inverse else 'unlikely'
  5510. def generate_function_definitions(self, env, code):
  5511. for clause in self.if_clauses:
  5512. clause.generate_function_definitions(env, code)
  5513. if self.else_clause is not None:
  5514. self.else_clause.generate_function_definitions(env, code)
  5515. def annotate(self, code):
  5516. for if_clause in self.if_clauses:
  5517. if_clause.annotate(code)
  5518. if self.else_clause:
  5519. self.else_clause.annotate(code)
  5520. class IfClauseNode(Node):
  5521. # if or elif clause in an if statement
  5522. #
  5523. # condition ExprNode
  5524. # body StatNode
  5525. child_attrs = ["condition", "body"]
  5526. branch_hint = None
  5527. def analyse_declarations(self, env):
  5528. self.body.analyse_declarations(env)
  5529. def analyse_expressions(self, env):
  5530. self.condition = self.condition.analyse_temp_boolean_expression(env)
  5531. self.body = self.body.analyse_expressions(env)
  5532. return self
  5533. def generate_execution_code(self, code, end_label, is_last):
  5534. self.condition.generate_evaluation_code(code)
  5535. code.mark_pos(self.pos)
  5536. condition = self.condition.result()
  5537. if self.branch_hint:
  5538. condition = '%s(%s)' % (self.branch_hint, condition)
  5539. code.putln("if (%s) {" % condition)
  5540. self.condition.generate_disposal_code(code)
  5541. self.condition.free_temps(code)
  5542. self.body.generate_execution_code(code)
  5543. code.mark_pos(self.pos, trace=False)
  5544. if not (is_last or self.body.is_terminator):
  5545. code.put_goto(end_label)
  5546. code.putln("}")
  5547. def generate_function_definitions(self, env, code):
  5548. self.condition.generate_function_definitions(env, code)
  5549. self.body.generate_function_definitions(env, code)
  5550. def annotate(self, code):
  5551. self.condition.annotate(code)
  5552. self.body.annotate(code)
  5553. class SwitchCaseNode(StatNode):
  5554. # Generated in the optimization of an if-elif-else node
  5555. #
  5556. # conditions [ExprNode]
  5557. # body StatNode
  5558. child_attrs = ['conditions', 'body']
  5559. def generate_condition_evaluation_code(self, code):
  5560. for cond in self.conditions:
  5561. cond.generate_evaluation_code(code)
  5562. def generate_execution_code(self, code):
  5563. num_conditions = len(self.conditions)
  5564. line_tracing_enabled = code.globalstate.directives['linetrace']
  5565. for i, cond in enumerate(self.conditions, 1):
  5566. code.putln("case %s:" % cond.result())
  5567. code.mark_pos(cond.pos) # Tracing code must appear *after* the 'case' statement.
  5568. if line_tracing_enabled and i < num_conditions:
  5569. # Allow fall-through after the line tracing code.
  5570. code.putln('CYTHON_FALLTHROUGH;')
  5571. self.body.generate_execution_code(code)
  5572. code.mark_pos(self.pos, trace=False)
  5573. code.putln("break;")
  5574. def generate_function_definitions(self, env, code):
  5575. for cond in self.conditions:
  5576. cond.generate_function_definitions(env, code)
  5577. self.body.generate_function_definitions(env, code)
  5578. def annotate(self, code):
  5579. for cond in self.conditions:
  5580. cond.annotate(code)
  5581. self.body.annotate(code)
  5582. class SwitchStatNode(StatNode):
  5583. # Generated in the optimization of an if-elif-else node
  5584. #
  5585. # test ExprNode
  5586. # cases [SwitchCaseNode]
  5587. # else_clause StatNode or None
  5588. child_attrs = ['test', 'cases', 'else_clause']
  5589. def generate_execution_code(self, code):
  5590. self.test.generate_evaluation_code(code)
  5591. # Make sure all conditions are evaluated before going into the switch() statement.
  5592. # This is required in order to prevent any execution code from leaking into the space between the cases.
  5593. for case in self.cases:
  5594. case.generate_condition_evaluation_code(code)
  5595. code.mark_pos(self.pos)
  5596. code.putln("switch (%s) {" % self.test.result())
  5597. for case in self.cases:
  5598. case.generate_execution_code(code)
  5599. if self.else_clause is not None:
  5600. code.putln("default:")
  5601. self.else_clause.generate_execution_code(code)
  5602. code.putln("break;")
  5603. else:
  5604. # Always generate a default clause to prevent C compiler warnings
  5605. # about unmatched enum values (it was not the user who decided to
  5606. # generate the switch statement, so shouldn't be bothered).
  5607. code.putln("default: break;")
  5608. code.putln("}")
  5609. self.test.generate_disposal_code(code)
  5610. self.test.free_temps(code)
  5611. def generate_function_definitions(self, env, code):
  5612. self.test.generate_function_definitions(env, code)
  5613. for case in self.cases:
  5614. case.generate_function_definitions(env, code)
  5615. if self.else_clause is not None:
  5616. self.else_clause.generate_function_definitions(env, code)
  5617. def annotate(self, code):
  5618. self.test.annotate(code)
  5619. for case in self.cases:
  5620. case.annotate(code)
  5621. if self.else_clause is not None:
  5622. self.else_clause.annotate(code)
  5623. class LoopNode(object):
  5624. pass
  5625. class WhileStatNode(LoopNode, StatNode):
  5626. # while statement
  5627. #
  5628. # condition ExprNode
  5629. # body StatNode
  5630. # else_clause StatNode
  5631. child_attrs = ["condition", "body", "else_clause"]
  5632. def analyse_declarations(self, env):
  5633. self.body.analyse_declarations(env)
  5634. if self.else_clause:
  5635. self.else_clause.analyse_declarations(env)
  5636. def analyse_expressions(self, env):
  5637. if self.condition:
  5638. self.condition = self.condition.analyse_temp_boolean_expression(env)
  5639. self.body = self.body.analyse_expressions(env)
  5640. if self.else_clause:
  5641. self.else_clause = self.else_clause.analyse_expressions(env)
  5642. return self
  5643. def generate_execution_code(self, code):
  5644. code.mark_pos(self.pos)
  5645. old_loop_labels = code.new_loop_labels()
  5646. code.putln(
  5647. "while (1) {")
  5648. if self.condition:
  5649. self.condition.generate_evaluation_code(code)
  5650. self.condition.generate_disposal_code(code)
  5651. code.putln(
  5652. "if (!%s) break;" % self.condition.result())
  5653. self.condition.free_temps(code)
  5654. self.body.generate_execution_code(code)
  5655. code.put_label(code.continue_label)
  5656. code.putln("}")
  5657. break_label = code.break_label
  5658. code.set_loop_labels(old_loop_labels)
  5659. if self.else_clause:
  5660. code.mark_pos(self.else_clause.pos)
  5661. code.putln("/*else*/ {")
  5662. self.else_clause.generate_execution_code(code)
  5663. code.putln("}")
  5664. code.put_label(break_label)
  5665. def generate_function_definitions(self, env, code):
  5666. if self.condition:
  5667. self.condition.generate_function_definitions(env, code)
  5668. self.body.generate_function_definitions(env, code)
  5669. if self.else_clause is not None:
  5670. self.else_clause.generate_function_definitions(env, code)
  5671. def annotate(self, code):
  5672. if self.condition:
  5673. self.condition.annotate(code)
  5674. self.body.annotate(code)
  5675. if self.else_clause:
  5676. self.else_clause.annotate(code)
  5677. class DictIterationNextNode(Node):
  5678. # Helper node for calling PyDict_Next() inside of a WhileStatNode
  5679. # and checking the dictionary size for changes. Created in
  5680. # Optimize.py.
  5681. child_attrs = ['dict_obj', 'expected_size', 'pos_index_var',
  5682. 'coerced_key_var', 'coerced_value_var', 'coerced_tuple_var',
  5683. 'key_target', 'value_target', 'tuple_target', 'is_dict_flag']
  5684. coerced_key_var = key_ref = None
  5685. coerced_value_var = value_ref = None
  5686. coerced_tuple_var = tuple_ref = None
  5687. def __init__(self, dict_obj, expected_size, pos_index_var,
  5688. key_target, value_target, tuple_target, is_dict_flag):
  5689. Node.__init__(
  5690. self, dict_obj.pos,
  5691. dict_obj=dict_obj,
  5692. expected_size=expected_size,
  5693. pos_index_var=pos_index_var,
  5694. key_target=key_target,
  5695. value_target=value_target,
  5696. tuple_target=tuple_target,
  5697. is_dict_flag=is_dict_flag,
  5698. is_temp=True,
  5699. type=PyrexTypes.c_bint_type)
  5700. def analyse_expressions(self, env):
  5701. from . import ExprNodes
  5702. self.dict_obj = self.dict_obj.analyse_types(env)
  5703. self.expected_size = self.expected_size.analyse_types(env)
  5704. if self.pos_index_var:
  5705. self.pos_index_var = self.pos_index_var.analyse_types(env)
  5706. if self.key_target:
  5707. self.key_target = self.key_target.analyse_target_types(env)
  5708. self.key_ref = ExprNodes.TempNode(self.key_target.pos, PyrexTypes.py_object_type)
  5709. self.coerced_key_var = self.key_ref.coerce_to(self.key_target.type, env)
  5710. if self.value_target:
  5711. self.value_target = self.value_target.analyse_target_types(env)
  5712. self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
  5713. self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
  5714. if self.tuple_target:
  5715. self.tuple_target = self.tuple_target.analyse_target_types(env)
  5716. self.tuple_ref = ExprNodes.TempNode(self.tuple_target.pos, PyrexTypes.py_object_type)
  5717. self.coerced_tuple_var = self.tuple_ref.coerce_to(self.tuple_target.type, env)
  5718. self.is_dict_flag = self.is_dict_flag.analyse_types(env)
  5719. return self
  5720. def generate_function_definitions(self, env, code):
  5721. self.dict_obj.generate_function_definitions(env, code)
  5722. def generate_execution_code(self, code):
  5723. code.globalstate.use_utility_code(UtilityCode.load_cached("dict_iter", "Optimize.c"))
  5724. self.dict_obj.generate_evaluation_code(code)
  5725. assignments = []
  5726. temp_addresses = []
  5727. for var, result, target in [(self.key_ref, self.coerced_key_var, self.key_target),
  5728. (self.value_ref, self.coerced_value_var, self.value_target),
  5729. (self.tuple_ref, self.coerced_tuple_var, self.tuple_target)]:
  5730. if target is None:
  5731. addr = 'NULL'
  5732. else:
  5733. assignments.append((var, result, target))
  5734. var.allocate(code)
  5735. addr = '&%s' % var.result()
  5736. temp_addresses.append(addr)
  5737. result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
  5738. code.putln("%s = __Pyx_dict_iter_next(%s, %s, &%s, %s, %s, %s, %s);" % (
  5739. result_temp,
  5740. self.dict_obj.py_result(),
  5741. self.expected_size.result(),
  5742. self.pos_index_var.result(),
  5743. temp_addresses[0],
  5744. temp_addresses[1],
  5745. temp_addresses[2],
  5746. self.is_dict_flag.result()
  5747. ))
  5748. code.putln("if (unlikely(%s == 0)) break;" % result_temp)
  5749. code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
  5750. code.funcstate.release_temp(result_temp)
  5751. # evaluate all coercions before the assignments
  5752. for var, result, target in assignments:
  5753. code.put_gotref(var.result())
  5754. for var, result, target in assignments:
  5755. result.generate_evaluation_code(code)
  5756. for var, result, target in assignments:
  5757. target.generate_assignment_code(result, code)
  5758. var.release(code)
  5759. class SetIterationNextNode(Node):
  5760. # Helper node for calling _PySet_NextEntry() inside of a WhileStatNode
  5761. # and checking the set size for changes. Created in Optimize.py.
  5762. child_attrs = ['set_obj', 'expected_size', 'pos_index_var',
  5763. 'coerced_value_var', 'value_target', 'is_set_flag']
  5764. coerced_value_var = value_ref = None
  5765. def __init__(self, set_obj, expected_size, pos_index_var, value_target, is_set_flag):
  5766. Node.__init__(
  5767. self, set_obj.pos,
  5768. set_obj=set_obj,
  5769. expected_size=expected_size,
  5770. pos_index_var=pos_index_var,
  5771. value_target=value_target,
  5772. is_set_flag=is_set_flag,
  5773. is_temp=True,
  5774. type=PyrexTypes.c_bint_type)
  5775. def analyse_expressions(self, env):
  5776. from . import ExprNodes
  5777. self.set_obj = self.set_obj.analyse_types(env)
  5778. self.expected_size = self.expected_size.analyse_types(env)
  5779. self.pos_index_var = self.pos_index_var.analyse_types(env)
  5780. self.value_target = self.value_target.analyse_target_types(env)
  5781. self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
  5782. self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
  5783. self.is_set_flag = self.is_set_flag.analyse_types(env)
  5784. return self
  5785. def generate_function_definitions(self, env, code):
  5786. self.set_obj.generate_function_definitions(env, code)
  5787. def generate_execution_code(self, code):
  5788. code.globalstate.use_utility_code(UtilityCode.load_cached("set_iter", "Optimize.c"))
  5789. self.set_obj.generate_evaluation_code(code)
  5790. value_ref = self.value_ref
  5791. value_ref.allocate(code)
  5792. result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
  5793. code.putln("%s = __Pyx_set_iter_next(%s, %s, &%s, &%s, %s);" % (
  5794. result_temp,
  5795. self.set_obj.py_result(),
  5796. self.expected_size.result(),
  5797. self.pos_index_var.result(),
  5798. value_ref.result(),
  5799. self.is_set_flag.result()
  5800. ))
  5801. code.putln("if (unlikely(%s == 0)) break;" % result_temp)
  5802. code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
  5803. code.funcstate.release_temp(result_temp)
  5804. # evaluate all coercions before the assignments
  5805. code.put_gotref(value_ref.result())
  5806. self.coerced_value_var.generate_evaluation_code(code)
  5807. self.value_target.generate_assignment_code(self.coerced_value_var, code)
  5808. value_ref.release(code)
  5809. def ForStatNode(pos, **kw):
  5810. if 'iterator' in kw:
  5811. if kw['iterator'].is_async:
  5812. return AsyncForStatNode(pos, **kw)
  5813. else:
  5814. return ForInStatNode(pos, **kw)
  5815. else:
  5816. return ForFromStatNode(pos, **kw)
  5817. class _ForInStatNode(LoopNode, StatNode):
  5818. # Base class of 'for-in' statements.
  5819. #
  5820. # target ExprNode
  5821. # iterator IteratorNode | AIterAwaitExprNode(AsyncIteratorNode)
  5822. # body StatNode
  5823. # else_clause StatNode
  5824. # item NextNode | AwaitExprNode(AsyncNextNode)
  5825. # is_async boolean true for 'async for' statements
  5826. child_attrs = ["target", "item", "iterator", "body", "else_clause"]
  5827. item = None
  5828. is_async = False
  5829. def _create_item_node(self):
  5830. raise NotImplementedError("must be implemented by subclasses")
  5831. def analyse_declarations(self, env):
  5832. self.target.analyse_target_declaration(env)
  5833. self.body.analyse_declarations(env)
  5834. if self.else_clause:
  5835. self.else_clause.analyse_declarations(env)
  5836. self._create_item_node()
  5837. def analyse_expressions(self, env):
  5838. self.target = self.target.analyse_target_types(env)
  5839. self.iterator = self.iterator.analyse_expressions(env)
  5840. self._create_item_node() # must rewrap self.item after analysis
  5841. self.item = self.item.analyse_expressions(env)
  5842. if (not self.is_async and
  5843. (self.iterator.type.is_ptr or self.iterator.type.is_array) and
  5844. self.target.type.assignable_from(self.iterator.type)):
  5845. # C array slice optimization.
  5846. pass
  5847. else:
  5848. self.item = self.item.coerce_to(self.target.type, env)
  5849. self.body = self.body.analyse_expressions(env)
  5850. if self.else_clause:
  5851. self.else_clause = self.else_clause.analyse_expressions(env)
  5852. return self
  5853. def generate_execution_code(self, code):
  5854. code.mark_pos(self.pos)
  5855. old_loop_labels = code.new_loop_labels()
  5856. self.iterator.generate_evaluation_code(code)
  5857. code.putln("for (;;) {")
  5858. self.item.generate_evaluation_code(code)
  5859. self.target.generate_assignment_code(self.item, code)
  5860. self.body.generate_execution_code(code)
  5861. code.mark_pos(self.pos)
  5862. code.put_label(code.continue_label)
  5863. code.putln("}")
  5864. break_label = code.break_label
  5865. code.set_loop_labels(old_loop_labels)
  5866. if self.else_clause:
  5867. # In nested loops, the 'else' block can contain 'continue' or 'break'
  5868. # statements for the outer loop, but we may need to generate cleanup code
  5869. # before taking those paths, so we intercept them here.
  5870. orig_exit_labels = (code.continue_label, code.break_label)
  5871. code.continue_label = code.new_label('outer_continue')
  5872. code.break_label = code.new_label('outer_break')
  5873. code.putln("/*else*/ {")
  5874. self.else_clause.generate_execution_code(code)
  5875. code.putln("}")
  5876. needs_goto_end = not self.else_clause.is_terminator
  5877. for exit_label, orig_exit_label in zip([code.continue_label, code.break_label], orig_exit_labels):
  5878. if not code.label_used(exit_label):
  5879. continue
  5880. if needs_goto_end:
  5881. code.put_goto(break_label)
  5882. needs_goto_end = False
  5883. code.mark_pos(self.pos)
  5884. code.put_label(exit_label)
  5885. self.iterator.generate_disposal_code(code)
  5886. code.put_goto(orig_exit_label)
  5887. code.set_loop_labels(old_loop_labels)
  5888. code.mark_pos(self.pos)
  5889. if code.label_used(break_label):
  5890. code.put_label(break_label)
  5891. self.iterator.generate_disposal_code(code)
  5892. self.iterator.free_temps(code)
  5893. def generate_function_definitions(self, env, code):
  5894. self.target.generate_function_definitions(env, code)
  5895. self.iterator.generate_function_definitions(env, code)
  5896. self.body.generate_function_definitions(env, code)
  5897. if self.else_clause is not None:
  5898. self.else_clause.generate_function_definitions(env, code)
  5899. def annotate(self, code):
  5900. self.target.annotate(code)
  5901. self.iterator.annotate(code)
  5902. self.body.annotate(code)
  5903. if self.else_clause:
  5904. self.else_clause.annotate(code)
  5905. self.item.annotate(code)
  5906. class ForInStatNode(_ForInStatNode):
  5907. # 'for' statement
  5908. is_async = False
  5909. def _create_item_node(self):
  5910. from .ExprNodes import NextNode
  5911. self.item = NextNode(self.iterator)
  5912. class AsyncForStatNode(_ForInStatNode):
  5913. # 'async for' statement
  5914. #
  5915. # iterator AIterAwaitExprNode(AsyncIteratorNode)
  5916. # item AwaitIterNextExprNode(AsyncIteratorNode)
  5917. is_async = True
  5918. def __init__(self, pos, **kw):
  5919. assert 'item' not in kw
  5920. from . import ExprNodes
  5921. # AwaitExprNodes must appear before running MarkClosureVisitor
  5922. kw['item'] = ExprNodes.AwaitIterNextExprNode(kw['iterator'].pos, arg=None)
  5923. _ForInStatNode.__init__(self, pos, **kw)
  5924. def _create_item_node(self):
  5925. from . import ExprNodes
  5926. self.item.arg = ExprNodes.AsyncNextNode(self.iterator)
  5927. class ForFromStatNode(LoopNode, StatNode):
  5928. # for name from expr rel name rel expr
  5929. #
  5930. # target NameNode
  5931. # bound1 ExprNode
  5932. # relation1 string
  5933. # relation2 string
  5934. # bound2 ExprNode
  5935. # step ExprNode or None
  5936. # body StatNode
  5937. # else_clause StatNode or None
  5938. #
  5939. # Used internally:
  5940. #
  5941. # from_range bool
  5942. # is_py_target bool
  5943. # loopvar_node ExprNode (usually a NameNode or temp node)
  5944. # py_loopvar_node PyTempNode or None
  5945. child_attrs = ["target", "bound1", "bound2", "step", "body", "else_clause"]
  5946. is_py_target = False
  5947. loopvar_node = None
  5948. py_loopvar_node = None
  5949. from_range = False
  5950. gil_message = "For-loop using object bounds or target"
  5951. def nogil_check(self, env):
  5952. for x in (self.target, self.bound1, self.bound2):
  5953. if x.type.is_pyobject:
  5954. self.gil_error()
  5955. def analyse_declarations(self, env):
  5956. self.target.analyse_target_declaration(env)
  5957. self.body.analyse_declarations(env)
  5958. if self.else_clause:
  5959. self.else_clause.analyse_declarations(env)
  5960. def analyse_expressions(self, env):
  5961. from . import ExprNodes
  5962. self.target = self.target.analyse_target_types(env)
  5963. self.bound1 = self.bound1.analyse_types(env)
  5964. self.bound2 = self.bound2.analyse_types(env)
  5965. if self.step is not None:
  5966. if isinstance(self.step, ExprNodes.UnaryMinusNode):
  5967. warning(self.step.pos, "Probable infinite loop in for-from-by statement. "
  5968. "Consider switching the directions of the relations.", 2)
  5969. self.step = self.step.analyse_types(env)
  5970. self.set_up_loop(env)
  5971. target_type = self.target.type
  5972. if not (target_type.is_pyobject or target_type.is_numeric):
  5973. error(self.target.pos, "for-from loop variable must be c numeric type or Python object")
  5974. self.body = self.body.analyse_expressions(env)
  5975. if self.else_clause:
  5976. self.else_clause = self.else_clause.analyse_expressions(env)
  5977. return self
  5978. def set_up_loop(self, env):
  5979. from . import ExprNodes
  5980. target_type = self.target.type
  5981. if target_type.is_numeric:
  5982. loop_type = target_type
  5983. else:
  5984. if target_type.is_enum:
  5985. warning(self.target.pos,
  5986. "Integer loops over enum values are fragile. Please cast to a safe integer type instead.")
  5987. loop_type = PyrexTypes.c_long_type if target_type.is_pyobject else PyrexTypes.c_int_type
  5988. if not self.bound1.type.is_pyobject:
  5989. loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound1.type)
  5990. if not self.bound2.type.is_pyobject:
  5991. loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound2.type)
  5992. if self.step is not None and not self.step.type.is_pyobject:
  5993. loop_type = PyrexTypes.widest_numeric_type(loop_type, self.step.type)
  5994. self.bound1 = self.bound1.coerce_to(loop_type, env)
  5995. self.bound2 = self.bound2.coerce_to(loop_type, env)
  5996. if not self.bound2.is_literal:
  5997. self.bound2 = self.bound2.coerce_to_temp(env)
  5998. if self.step is not None:
  5999. self.step = self.step.coerce_to(loop_type, env)
  6000. if not self.step.is_literal:
  6001. self.step = self.step.coerce_to_temp(env)
  6002. if target_type.is_numeric or target_type.is_enum:
  6003. self.is_py_target = False
  6004. if isinstance(self.target, ExprNodes.BufferIndexNode):
  6005. raise error(self.pos, "Buffer or memoryview slicing/indexing not allowed as for-loop target.")
  6006. self.loopvar_node = self.target
  6007. self.py_loopvar_node = None
  6008. else:
  6009. self.is_py_target = True
  6010. c_loopvar_node = ExprNodes.TempNode(self.pos, loop_type, env)
  6011. self.loopvar_node = c_loopvar_node
  6012. self.py_loopvar_node = ExprNodes.CloneNode(c_loopvar_node).coerce_to_pyobject(env)
  6013. def generate_execution_code(self, code):
  6014. code.mark_pos(self.pos)
  6015. old_loop_labels = code.new_loop_labels()
  6016. from_range = self.from_range
  6017. self.bound1.generate_evaluation_code(code)
  6018. self.bound2.generate_evaluation_code(code)
  6019. offset, incop = self.relation_table[self.relation1]
  6020. if self.step is not None:
  6021. self.step.generate_evaluation_code(code)
  6022. step = self.step.result()
  6023. incop = "%s=%s" % (incop[0], step) # e.g. '++' => '+= STEP'
  6024. else:
  6025. step = '1'
  6026. from . import ExprNodes
  6027. if isinstance(self.loopvar_node, ExprNodes.TempNode):
  6028. self.loopvar_node.allocate(code)
  6029. if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
  6030. self.py_loopvar_node.allocate(code)
  6031. loopvar_type = PyrexTypes.c_long_type if self.target.type.is_enum else self.target.type
  6032. if from_range and not self.is_py_target:
  6033. loopvar_name = code.funcstate.allocate_temp(loopvar_type, False)
  6034. else:
  6035. loopvar_name = self.loopvar_node.result()
  6036. if loopvar_type.is_int and not loopvar_type.signed and self.relation2[0] == '>':
  6037. # Handle the case where the endpoint of an unsigned int iteration
  6038. # is within step of 0.
  6039. code.putln("for (%s = %s%s + %s; %s %s %s + %s; ) { %s%s;" % (
  6040. loopvar_name,
  6041. self.bound1.result(), offset, step,
  6042. loopvar_name, self.relation2, self.bound2.result(), step,
  6043. loopvar_name, incop))
  6044. else:
  6045. code.putln("for (%s = %s%s; %s %s %s; %s%s) {" % (
  6046. loopvar_name,
  6047. self.bound1.result(), offset,
  6048. loopvar_name, self.relation2, self.bound2.result(),
  6049. loopvar_name, incop))
  6050. coerced_loopvar_node = self.py_loopvar_node
  6051. if coerced_loopvar_node is None and from_range:
  6052. coerced_loopvar_node = ExprNodes.RawCNameExprNode(self.target.pos, loopvar_type, loopvar_name)
  6053. if coerced_loopvar_node is not None:
  6054. coerced_loopvar_node.generate_evaluation_code(code)
  6055. self.target.generate_assignment_code(coerced_loopvar_node, code)
  6056. self.body.generate_execution_code(code)
  6057. code.put_label(code.continue_label)
  6058. if not from_range and self.py_loopvar_node:
  6059. # This mess is to make for..from loops with python targets behave
  6060. # exactly like those with C targets with regards to re-assignment
  6061. # of the loop variable.
  6062. if self.target.entry.is_pyglobal:
  6063. # We know target is a NameNode, this is the only ugly case.
  6064. target_node = ExprNodes.PyTempNode(self.target.pos, None)
  6065. target_node.allocate(code)
  6066. interned_cname = code.intern_identifier(self.target.entry.name)
  6067. if self.target.entry.scope.is_module_scope:
  6068. code.globalstate.use_utility_code(
  6069. UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c"))
  6070. lookup_func = '__Pyx_GetModuleGlobalName(%s, %s); %s'
  6071. else:
  6072. code.globalstate.use_utility_code(
  6073. UtilityCode.load_cached("GetNameInClass", "ObjectHandling.c"))
  6074. lookup_func = '__Pyx_GetNameInClass(%s, {}, %s); %s'.format(
  6075. self.target.entry.scope.namespace_cname)
  6076. code.putln(lookup_func % (
  6077. target_node.result(),
  6078. interned_cname,
  6079. code.error_goto_if_null(target_node.result(), self.target.pos)))
  6080. code.put_gotref(target_node.result())
  6081. else:
  6082. target_node = self.target
  6083. from_py_node = ExprNodes.CoerceFromPyTypeNode(
  6084. self.loopvar_node.type, target_node, self.target.entry.scope)
  6085. from_py_node.temp_code = loopvar_name
  6086. from_py_node.generate_result_code(code)
  6087. if self.target.entry.is_pyglobal:
  6088. code.put_decref(target_node.result(), target_node.type)
  6089. target_node.release(code)
  6090. code.putln("}")
  6091. if not from_range and self.py_loopvar_node:
  6092. # This is potentially wasteful, but we don't want the semantics to
  6093. # depend on whether or not the loop is a python type.
  6094. self.py_loopvar_node.generate_evaluation_code(code)
  6095. self.target.generate_assignment_code(self.py_loopvar_node, code)
  6096. if from_range and not self.is_py_target:
  6097. code.funcstate.release_temp(loopvar_name)
  6098. break_label = code.break_label
  6099. code.set_loop_labels(old_loop_labels)
  6100. if self.else_clause:
  6101. code.putln("/*else*/ {")
  6102. self.else_clause.generate_execution_code(code)
  6103. code.putln("}")
  6104. code.put_label(break_label)
  6105. self.bound1.generate_disposal_code(code)
  6106. self.bound1.free_temps(code)
  6107. self.bound2.generate_disposal_code(code)
  6108. self.bound2.free_temps(code)
  6109. if isinstance(self.loopvar_node, ExprNodes.TempNode):
  6110. self.loopvar_node.release(code)
  6111. if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
  6112. self.py_loopvar_node.release(code)
  6113. if self.step is not None:
  6114. self.step.generate_disposal_code(code)
  6115. self.step.free_temps(code)
  6116. relation_table = {
  6117. # {relop : (initial offset, increment op)}
  6118. '<=': ("", "++"),
  6119. '<' : ("+1", "++"),
  6120. '>=': ("", "--"),
  6121. '>' : ("-1", "--"),
  6122. }
  6123. def generate_function_definitions(self, env, code):
  6124. self.target.generate_function_definitions(env, code)
  6125. self.bound1.generate_function_definitions(env, code)
  6126. self.bound2.generate_function_definitions(env, code)
  6127. if self.step is not None:
  6128. self.step.generate_function_definitions(env, code)
  6129. self.body.generate_function_definitions(env, code)
  6130. if self.else_clause is not None:
  6131. self.else_clause.generate_function_definitions(env, code)
  6132. def annotate(self, code):
  6133. self.target.annotate(code)
  6134. self.bound1.annotate(code)
  6135. self.bound2.annotate(code)
  6136. if self.step:
  6137. self.step.annotate(code)
  6138. self.body.annotate(code)
  6139. if self.else_clause:
  6140. self.else_clause.annotate(code)
  6141. class WithStatNode(StatNode):
  6142. """
  6143. Represents a Python with statement.
  6144. Implemented by the WithTransform as follows:
  6145. MGR = EXPR
  6146. EXIT = MGR.__exit__
  6147. VALUE = MGR.__enter__()
  6148. EXC = True
  6149. try:
  6150. try:
  6151. TARGET = VALUE # optional
  6152. BODY
  6153. except:
  6154. EXC = False
  6155. if not EXIT(*EXCINFO):
  6156. raise
  6157. finally:
  6158. if EXC:
  6159. EXIT(None, None, None)
  6160. MGR = EXIT = VALUE = None
  6161. """
  6162. # manager The with statement manager object
  6163. # target ExprNode the target lhs of the __enter__() call
  6164. # body StatNode
  6165. # enter_call ExprNode the call to the __enter__() method
  6166. # exit_var String the cname of the __exit__() method reference
  6167. child_attrs = ["manager", "enter_call", "target", "body"]
  6168. enter_call = None
  6169. target_temp = None
  6170. def analyse_declarations(self, env):
  6171. self.manager.analyse_declarations(env)
  6172. self.enter_call.analyse_declarations(env)
  6173. self.body.analyse_declarations(env)
  6174. def analyse_expressions(self, env):
  6175. self.manager = self.manager.analyse_types(env)
  6176. self.enter_call = self.enter_call.analyse_types(env)
  6177. if self.target:
  6178. # set up target_temp before descending into body (which uses it)
  6179. from .ExprNodes import TempNode
  6180. self.target_temp = TempNode(self.enter_call.pos, self.enter_call.type)
  6181. self.body = self.body.analyse_expressions(env)
  6182. return self
  6183. def generate_function_definitions(self, env, code):
  6184. self.manager.generate_function_definitions(env, code)
  6185. self.enter_call.generate_function_definitions(env, code)
  6186. self.body.generate_function_definitions(env, code)
  6187. def generate_execution_code(self, code):
  6188. code.mark_pos(self.pos)
  6189. code.putln("/*with:*/ {")
  6190. self.manager.generate_evaluation_code(code)
  6191. self.exit_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
  6192. code.globalstate.use_utility_code(
  6193. UtilityCode.load_cached("PyObjectLookupSpecial", "ObjectHandling.c"))
  6194. code.putln("%s = __Pyx_PyObject_LookupSpecial(%s, %s); %s" % (
  6195. self.exit_var,
  6196. self.manager.py_result(),
  6197. code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')),
  6198. code.error_goto_if_null(self.exit_var, self.pos),
  6199. ))
  6200. code.put_gotref(self.exit_var)
  6201. # need to free exit_var in the face of exceptions during setup
  6202. old_error_label = code.new_error_label()
  6203. intermediate_error_label = code.error_label
  6204. self.enter_call.generate_evaluation_code(code)
  6205. if self.target:
  6206. # The temp result will be cleaned up by the WithTargetAssignmentStatNode
  6207. # after assigning its result to the target of the 'with' statement.
  6208. self.target_temp.allocate(code)
  6209. self.enter_call.make_owned_reference(code)
  6210. code.putln("%s = %s;" % (self.target_temp.result(), self.enter_call.result()))
  6211. self.enter_call.generate_post_assignment_code(code)
  6212. else:
  6213. self.enter_call.generate_disposal_code(code)
  6214. self.enter_call.free_temps(code)
  6215. self.manager.generate_disposal_code(code)
  6216. self.manager.free_temps(code)
  6217. code.error_label = old_error_label
  6218. self.body.generate_execution_code(code)
  6219. if code.label_used(intermediate_error_label):
  6220. step_over_label = code.new_label()
  6221. code.put_goto(step_over_label)
  6222. code.put_label(intermediate_error_label)
  6223. code.put_decref_clear(self.exit_var, py_object_type)
  6224. code.put_goto(old_error_label)
  6225. code.put_label(step_over_label)
  6226. code.funcstate.release_temp(self.exit_var)
  6227. code.putln('}')
  6228. class WithTargetAssignmentStatNode(AssignmentNode):
  6229. # The target assignment of the 'with' statement value (return
  6230. # value of the __enter__() call).
  6231. #
  6232. # This is a special cased assignment that properly cleans up the RHS.
  6233. #
  6234. # lhs ExprNode the assignment target
  6235. # rhs ExprNode a (coerced) TempNode for the rhs (from WithStatNode)
  6236. # with_node WithStatNode the surrounding with-statement
  6237. child_attrs = ["rhs", "lhs"]
  6238. with_node = None
  6239. rhs = None
  6240. def analyse_declarations(self, env):
  6241. self.lhs.analyse_target_declaration(env)
  6242. def analyse_expressions(self, env):
  6243. self.lhs = self.lhs.analyse_target_types(env)
  6244. self.lhs.gil_assignment_check(env)
  6245. self.rhs = self.with_node.target_temp.coerce_to(self.lhs.type, env)
  6246. return self
  6247. def generate_execution_code(self, code):
  6248. self.rhs.generate_evaluation_code(code)
  6249. self.lhs.generate_assignment_code(self.rhs, code)
  6250. self.with_node.target_temp.release(code)
  6251. def annotate(self, code):
  6252. self.lhs.annotate(code)
  6253. self.rhs.annotate(code)
  6254. class TryExceptStatNode(StatNode):
  6255. # try .. except statement
  6256. #
  6257. # body StatNode
  6258. # except_clauses [ExceptClauseNode]
  6259. # else_clause StatNode or None
  6260. child_attrs = ["body", "except_clauses", "else_clause"]
  6261. in_generator = False
  6262. def analyse_declarations(self, env):
  6263. self.body.analyse_declarations(env)
  6264. for except_clause in self.except_clauses:
  6265. except_clause.analyse_declarations(env)
  6266. if self.else_clause:
  6267. self.else_clause.analyse_declarations(env)
  6268. def analyse_expressions(self, env):
  6269. self.body = self.body.analyse_expressions(env)
  6270. default_clause_seen = 0
  6271. for i, except_clause in enumerate(self.except_clauses):
  6272. except_clause = self.except_clauses[i] = except_clause.analyse_expressions(env)
  6273. if default_clause_seen:
  6274. error(except_clause.pos, "default 'except:' must be last")
  6275. if not except_clause.pattern:
  6276. default_clause_seen = 1
  6277. self.has_default_clause = default_clause_seen
  6278. if self.else_clause:
  6279. self.else_clause = self.else_clause.analyse_expressions(env)
  6280. return self
  6281. nogil_check = Node.gil_error
  6282. gil_message = "Try-except statement"
  6283. def generate_execution_code(self, code):
  6284. code.mark_pos(self.pos) # before changing the error label, in case of tracing errors
  6285. code.putln("{")
  6286. old_return_label = code.return_label
  6287. old_break_label = code.break_label
  6288. old_continue_label = code.continue_label
  6289. old_error_label = code.new_error_label()
  6290. our_error_label = code.error_label
  6291. except_end_label = code.new_label('exception_handled')
  6292. except_error_label = code.new_label('except_error')
  6293. except_return_label = code.new_label('except_return')
  6294. try_return_label = code.new_label('try_return')
  6295. try_break_label = code.new_label('try_break') if old_break_label else None
  6296. try_continue_label = code.new_label('try_continue') if old_continue_label else None
  6297. try_end_label = code.new_label('try_end')
  6298. exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
  6299. for _ in range(3)]
  6300. save_exc = code.insertion_point()
  6301. code.putln(
  6302. "/*try:*/ {")
  6303. code.return_label = try_return_label
  6304. code.break_label = try_break_label
  6305. code.continue_label = try_continue_label
  6306. self.body.generate_execution_code(code)
  6307. code.mark_pos(self.pos, trace=False)
  6308. code.putln(
  6309. "}")
  6310. temps_to_clean_up = code.funcstate.all_free_managed_temps()
  6311. can_raise = code.label_used(our_error_label)
  6312. if can_raise:
  6313. # inject code before the try block to save away the exception state
  6314. code.globalstate.use_utility_code(reset_exception_utility_code)
  6315. if not self.in_generator:
  6316. save_exc.putln("__Pyx_PyThreadState_declare")
  6317. save_exc.putln("__Pyx_PyThreadState_assign")
  6318. save_exc.putln("__Pyx_ExceptionSave(%s);" % (
  6319. ', '.join(['&%s' % var for var in exc_save_vars])))
  6320. for var in exc_save_vars:
  6321. save_exc.put_xgotref(var)
  6322. def restore_saved_exception():
  6323. for name in exc_save_vars:
  6324. code.put_xgiveref(name)
  6325. code.putln("__Pyx_ExceptionReset(%s);" %
  6326. ', '.join(exc_save_vars))
  6327. else:
  6328. # try block cannot raise exceptions, but we had to allocate the temps above,
  6329. # so just keep the C compiler from complaining about them being unused
  6330. mark_vars_used = ["(void)%s;" % var for var in exc_save_vars]
  6331. save_exc.putln("%s /* mark used */" % ' '.join(mark_vars_used))
  6332. def restore_saved_exception():
  6333. pass
  6334. code.error_label = except_error_label
  6335. code.return_label = except_return_label
  6336. normal_case_terminates = self.body.is_terminator
  6337. if self.else_clause:
  6338. code.mark_pos(self.else_clause.pos)
  6339. code.putln(
  6340. "/*else:*/ {")
  6341. self.else_clause.generate_execution_code(code)
  6342. code.putln(
  6343. "}")
  6344. if not normal_case_terminates:
  6345. normal_case_terminates = self.else_clause.is_terminator
  6346. if can_raise:
  6347. if not normal_case_terminates:
  6348. for var in exc_save_vars:
  6349. code.put_xdecref_clear(var, py_object_type)
  6350. code.put_goto(try_end_label)
  6351. code.put_label(our_error_label)
  6352. for temp_name, temp_type in temps_to_clean_up:
  6353. code.put_xdecref_clear(temp_name, temp_type)
  6354. outer_except = code.funcstate.current_except
  6355. # Currently points to self, but the ExceptClauseNode would also be ok. Change if needed.
  6356. code.funcstate.current_except = self
  6357. for except_clause in self.except_clauses:
  6358. except_clause.generate_handling_code(code, except_end_label)
  6359. code.funcstate.current_except = outer_except
  6360. if not self.has_default_clause:
  6361. code.put_goto(except_error_label)
  6362. for exit_label, old_label in [(except_error_label, old_error_label),
  6363. (try_break_label, old_break_label),
  6364. (try_continue_label, old_continue_label),
  6365. (try_return_label, old_return_label),
  6366. (except_return_label, old_return_label)]:
  6367. if code.label_used(exit_label):
  6368. if not normal_case_terminates and not code.label_used(try_end_label):
  6369. code.put_goto(try_end_label)
  6370. code.put_label(exit_label)
  6371. code.mark_pos(self.pos, trace=False)
  6372. if can_raise:
  6373. restore_saved_exception()
  6374. code.put_goto(old_label)
  6375. if code.label_used(except_end_label):
  6376. if not normal_case_terminates and not code.label_used(try_end_label):
  6377. code.put_goto(try_end_label)
  6378. code.put_label(except_end_label)
  6379. if can_raise:
  6380. restore_saved_exception()
  6381. if code.label_used(try_end_label):
  6382. code.put_label(try_end_label)
  6383. code.putln("}")
  6384. for cname in exc_save_vars:
  6385. code.funcstate.release_temp(cname)
  6386. code.return_label = old_return_label
  6387. code.break_label = old_break_label
  6388. code.continue_label = old_continue_label
  6389. code.error_label = old_error_label
  6390. def generate_function_definitions(self, env, code):
  6391. self.body.generate_function_definitions(env, code)
  6392. for except_clause in self.except_clauses:
  6393. except_clause.generate_function_definitions(env, code)
  6394. if self.else_clause is not None:
  6395. self.else_clause.generate_function_definitions(env, code)
  6396. def annotate(self, code):
  6397. self.body.annotate(code)
  6398. for except_node in self.except_clauses:
  6399. except_node.annotate(code)
  6400. if self.else_clause:
  6401. self.else_clause.annotate(code)
  6402. class ExceptClauseNode(Node):
  6403. # Part of try ... except statement.
  6404. #
  6405. # pattern [ExprNode]
  6406. # target ExprNode or None
  6407. # body StatNode
  6408. # excinfo_target TupleNode(3*ResultRefNode) or None optional target for exception info (not owned here!)
  6409. # match_flag string result of exception match
  6410. # exc_value ExcValueNode used internally
  6411. # function_name string qualified name of enclosing function
  6412. # exc_vars (string * 3) local exception variables
  6413. # is_except_as bool Py3-style "except ... as xyz"
  6414. # excinfo_target is never set by the parser, but can be set by a transform
  6415. # in order to extract more extensive information about the exception as a
  6416. # sys.exc_info()-style tuple into a target variable
  6417. child_attrs = ["pattern", "target", "body", "exc_value"]
  6418. exc_value = None
  6419. excinfo_target = None
  6420. is_except_as = False
  6421. def analyse_declarations(self, env):
  6422. if self.target:
  6423. self.target.analyse_target_declaration(env)
  6424. self.body.analyse_declarations(env)
  6425. def analyse_expressions(self, env):
  6426. self.function_name = env.qualified_name
  6427. if self.pattern:
  6428. # normalise/unpack self.pattern into a list
  6429. for i, pattern in enumerate(self.pattern):
  6430. pattern = pattern.analyse_expressions(env)
  6431. self.pattern[i] = pattern.coerce_to_pyobject(env)
  6432. if self.target:
  6433. from . import ExprNodes
  6434. self.exc_value = ExprNodes.ExcValueNode(self.pos)
  6435. self.target = self.target.analyse_target_expression(env, self.exc_value)
  6436. self.body = self.body.analyse_expressions(env)
  6437. return self
  6438. def generate_handling_code(self, code, end_label):
  6439. code.mark_pos(self.pos)
  6440. if self.pattern:
  6441. has_non_literals = not all(
  6442. pattern.is_literal or pattern.is_simple() and not pattern.is_temp
  6443. for pattern in self.pattern)
  6444. if has_non_literals:
  6445. # For non-trivial exception check expressions, hide the live exception from C-API calls.
  6446. exc_vars = [code.funcstate.allocate_temp(py_object_type, manage_ref=True)
  6447. for _ in range(3)]
  6448. code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
  6449. code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % tuple(exc_vars))
  6450. code.globalstate.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c"))
  6451. exc_test_func = "__Pyx_PyErr_GivenExceptionMatches(%s, %%s)" % exc_vars[0]
  6452. else:
  6453. exc_vars = ()
  6454. code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c"))
  6455. exc_test_func = "__Pyx_PyErr_ExceptionMatches(%s)"
  6456. exc_tests = []
  6457. for pattern in self.pattern:
  6458. pattern.generate_evaluation_code(code)
  6459. exc_tests.append(exc_test_func % pattern.py_result())
  6460. match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
  6461. code.putln("%s = %s;" % (match_flag, ' || '.join(exc_tests)))
  6462. for pattern in self.pattern:
  6463. pattern.generate_disposal_code(code)
  6464. pattern.free_temps(code)
  6465. if has_non_literals:
  6466. code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(exc_vars))
  6467. code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
  6468. for temp in exc_vars:
  6469. code.funcstate.release_temp(temp)
  6470. code.putln(
  6471. "if (%s) {" %
  6472. match_flag)
  6473. code.funcstate.release_temp(match_flag)
  6474. else:
  6475. code.putln("/*except:*/ {")
  6476. if (not getattr(self.body, 'stats', True)
  6477. and self.excinfo_target is None
  6478. and self.target is None):
  6479. # most simple case: no exception variable, empty body (pass)
  6480. # => reset the exception state, done
  6481. code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
  6482. code.putln("__Pyx_ErrRestore(0,0,0);")
  6483. code.put_goto(end_label)
  6484. code.putln("}")
  6485. return
  6486. exc_vars = [code.funcstate.allocate_temp(py_object_type, manage_ref=True)
  6487. for _ in range(3)]
  6488. code.put_add_traceback(self.function_name)
  6489. # We always have to fetch the exception value even if
  6490. # there is no target, because this also normalises the
  6491. # exception and stores it in the thread state.
  6492. code.globalstate.use_utility_code(get_exception_utility_code)
  6493. exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
  6494. code.putln("if (__Pyx_GetException(%s) < 0) %s" % (
  6495. exc_args, code.error_goto(self.pos)))
  6496. for var in exc_vars:
  6497. code.put_gotref(var)
  6498. if self.target:
  6499. self.exc_value.set_var(exc_vars[1])
  6500. self.exc_value.generate_evaluation_code(code)
  6501. self.target.generate_assignment_code(self.exc_value, code)
  6502. if self.excinfo_target is not None:
  6503. for tempvar, node in zip(exc_vars, self.excinfo_target.args):
  6504. node.set_var(tempvar)
  6505. old_break_label, old_continue_label = code.break_label, code.continue_label
  6506. code.break_label = code.new_label('except_break')
  6507. code.continue_label = code.new_label('except_continue')
  6508. old_exc_vars = code.funcstate.exc_vars
  6509. code.funcstate.exc_vars = exc_vars
  6510. self.body.generate_execution_code(code)
  6511. code.funcstate.exc_vars = old_exc_vars
  6512. if not self.body.is_terminator:
  6513. for var in exc_vars:
  6514. # FIXME: XDECREF() is needed to allow re-raising (which clears the exc_vars),
  6515. # but I don't think it's the right solution.
  6516. code.put_xdecref_clear(var, py_object_type)
  6517. code.put_goto(end_label)
  6518. for new_label, old_label in [(code.break_label, old_break_label),
  6519. (code.continue_label, old_continue_label)]:
  6520. if code.label_used(new_label):
  6521. code.put_label(new_label)
  6522. for var in exc_vars:
  6523. code.put_decref_clear(var, py_object_type)
  6524. code.put_goto(old_label)
  6525. code.break_label = old_break_label
  6526. code.continue_label = old_continue_label
  6527. for temp in exc_vars:
  6528. code.funcstate.release_temp(temp)
  6529. code.putln(
  6530. "}")
  6531. def generate_function_definitions(self, env, code):
  6532. if self.target is not None:
  6533. self.target.generate_function_definitions(env, code)
  6534. self.body.generate_function_definitions(env, code)
  6535. def annotate(self, code):
  6536. if self.pattern:
  6537. for pattern in self.pattern:
  6538. pattern.annotate(code)
  6539. if self.target:
  6540. self.target.annotate(code)
  6541. self.body.annotate(code)
  6542. class TryFinallyStatNode(StatNode):
  6543. # try ... finally statement
  6544. #
  6545. # body StatNode
  6546. # finally_clause StatNode
  6547. # finally_except_clause deep-copy of finally_clause for exception case
  6548. # in_generator inside of generator => must store away current exception also in return case
  6549. #
  6550. # Each of the continue, break, return and error gotos runs
  6551. # into its own deep-copy of the finally block code.
  6552. # In addition, if we're doing an error, we save the
  6553. # exception on entry to the finally block and restore
  6554. # it on exit.
  6555. child_attrs = ["body", "finally_clause", "finally_except_clause"]
  6556. preserve_exception = 1
  6557. # handle exception case, in addition to return/break/continue
  6558. handle_error_case = True
  6559. func_return_type = None
  6560. finally_except_clause = None
  6561. is_try_finally_in_nogil = False
  6562. in_generator = False
  6563. @staticmethod
  6564. def create_analysed(pos, env, body, finally_clause):
  6565. node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
  6566. return node
  6567. def analyse_declarations(self, env):
  6568. self.body.analyse_declarations(env)
  6569. self.finally_except_clause = copy.deepcopy(self.finally_clause)
  6570. self.finally_except_clause.analyse_declarations(env)
  6571. self.finally_clause.analyse_declarations(env)
  6572. def analyse_expressions(self, env):
  6573. self.body = self.body.analyse_expressions(env)
  6574. self.finally_clause = self.finally_clause.analyse_expressions(env)
  6575. self.finally_except_clause = self.finally_except_clause.analyse_expressions(env)
  6576. if env.return_type and not env.return_type.is_void:
  6577. self.func_return_type = env.return_type
  6578. return self
  6579. nogil_check = Node.gil_error
  6580. gil_message = "Try-finally statement"
  6581. def generate_execution_code(self, code):
  6582. code.mark_pos(self.pos) # before changing the error label, in case of tracing errors
  6583. code.putln("/*try:*/ {")
  6584. old_error_label = code.error_label
  6585. old_labels = code.all_new_labels()
  6586. new_labels = code.get_all_labels()
  6587. new_error_label = code.error_label
  6588. if not self.handle_error_case:
  6589. code.error_label = old_error_label
  6590. catch_label = code.new_label()
  6591. was_in_try_finally = code.funcstate.in_try_finally
  6592. code.funcstate.in_try_finally = 1
  6593. self.body.generate_execution_code(code)
  6594. code.funcstate.in_try_finally = was_in_try_finally
  6595. code.putln("}")
  6596. temps_to_clean_up = code.funcstate.all_free_managed_temps()
  6597. code.mark_pos(self.finally_clause.pos)
  6598. code.putln("/*finally:*/ {")
  6599. # Reset labels only after writing out a potential line trace call for correct nogil error handling.
  6600. code.set_all_labels(old_labels)
  6601. def fresh_finally_clause(_next=[self.finally_clause]):
  6602. # generate the original subtree once and always keep a fresh copy
  6603. node = _next[0]
  6604. node_copy = copy.deepcopy(node)
  6605. if node is self.finally_clause:
  6606. _next[0] = node_copy
  6607. else:
  6608. node = node_copy
  6609. return node
  6610. preserve_error = self.preserve_exception and code.label_used(new_error_label)
  6611. needs_success_cleanup = not self.finally_clause.is_terminator
  6612. if not self.body.is_terminator:
  6613. code.putln('/*normal exit:*/{')
  6614. fresh_finally_clause().generate_execution_code(code)
  6615. if not self.finally_clause.is_terminator:
  6616. code.put_goto(catch_label)
  6617. code.putln('}')
  6618. if preserve_error:
  6619. code.put_label(new_error_label)
  6620. code.putln('/*exception exit:*/{')
  6621. if not self.in_generator:
  6622. code.putln("__Pyx_PyThreadState_declare")
  6623. if self.is_try_finally_in_nogil:
  6624. code.declare_gilstate()
  6625. if needs_success_cleanup:
  6626. exc_lineno_cnames = tuple([
  6627. code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
  6628. for _ in range(2)])
  6629. exc_filename_cname = code.funcstate.allocate_temp(
  6630. PyrexTypes.CPtrType(PyrexTypes.c_const_type(PyrexTypes.c_char_type)),
  6631. manage_ref=False)
  6632. else:
  6633. exc_lineno_cnames = exc_filename_cname = None
  6634. exc_vars = tuple([
  6635. code.funcstate.allocate_temp(py_object_type, manage_ref=False)
  6636. for _ in range(6)])
  6637. self.put_error_catcher(
  6638. code, temps_to_clean_up, exc_vars, exc_lineno_cnames, exc_filename_cname)
  6639. finally_old_labels = code.all_new_labels()
  6640. code.putln('{')
  6641. old_exc_vars = code.funcstate.exc_vars
  6642. code.funcstate.exc_vars = exc_vars[:3]
  6643. self.finally_except_clause.generate_execution_code(code)
  6644. code.funcstate.exc_vars = old_exc_vars
  6645. code.putln('}')
  6646. if needs_success_cleanup:
  6647. self.put_error_uncatcher(code, exc_vars, exc_lineno_cnames, exc_filename_cname)
  6648. if exc_lineno_cnames:
  6649. for cname in exc_lineno_cnames:
  6650. code.funcstate.release_temp(cname)
  6651. if exc_filename_cname:
  6652. code.funcstate.release_temp(exc_filename_cname)
  6653. code.put_goto(old_error_label)
  6654. for new_label, old_label in zip(code.get_all_labels(), finally_old_labels):
  6655. if not code.label_used(new_label):
  6656. continue
  6657. code.put_label(new_label)
  6658. self.put_error_cleaner(code, exc_vars)
  6659. code.put_goto(old_label)
  6660. for cname in exc_vars:
  6661. code.funcstate.release_temp(cname)
  6662. code.putln('}')
  6663. code.set_all_labels(old_labels)
  6664. return_label = code.return_label
  6665. exc_vars = ()
  6666. for i, (new_label, old_label) in enumerate(zip(new_labels, old_labels)):
  6667. if not code.label_used(new_label):
  6668. continue
  6669. if new_label == new_error_label and preserve_error:
  6670. continue # handled above
  6671. code.putln('%s: {' % new_label)
  6672. ret_temp = None
  6673. if old_label == return_label:
  6674. # return actually raises an (uncatchable) exception in generators that we must preserve
  6675. if self.in_generator:
  6676. exc_vars = tuple([
  6677. code.funcstate.allocate_temp(py_object_type, manage_ref=False)
  6678. for _ in range(6)])
  6679. self.put_error_catcher(code, [], exc_vars)
  6680. if not self.finally_clause.is_terminator:
  6681. # store away return value for later reuse
  6682. if (self.func_return_type and
  6683. not self.is_try_finally_in_nogil and
  6684. not isinstance(self.finally_clause, GILExitNode)):
  6685. ret_temp = code.funcstate.allocate_temp(
  6686. self.func_return_type, manage_ref=False)
  6687. code.putln("%s = %s;" % (ret_temp, Naming.retval_cname))
  6688. if self.func_return_type.is_pyobject:
  6689. code.putln("%s = 0;" % Naming.retval_cname)
  6690. fresh_finally_clause().generate_execution_code(code)
  6691. if old_label == return_label:
  6692. if ret_temp:
  6693. code.putln("%s = %s;" % (Naming.retval_cname, ret_temp))
  6694. if self.func_return_type.is_pyobject:
  6695. code.putln("%s = 0;" % ret_temp)
  6696. code.funcstate.release_temp(ret_temp)
  6697. if self.in_generator:
  6698. self.put_error_uncatcher(code, exc_vars)
  6699. for cname in exc_vars:
  6700. code.funcstate.release_temp(cname)
  6701. if not self.finally_clause.is_terminator:
  6702. code.put_goto(old_label)
  6703. code.putln('}')
  6704. # End finally
  6705. code.put_label(catch_label)
  6706. code.putln(
  6707. "}")
  6708. def generate_function_definitions(self, env, code):
  6709. self.body.generate_function_definitions(env, code)
  6710. self.finally_clause.generate_function_definitions(env, code)
  6711. if self.finally_except_clause:
  6712. self.finally_except_clause.generate_function_definitions(env, code)
  6713. def put_error_catcher(self, code, temps_to_clean_up, exc_vars,
  6714. exc_lineno_cnames=None, exc_filename_cname=None):
  6715. code.globalstate.use_utility_code(restore_exception_utility_code)
  6716. code.globalstate.use_utility_code(get_exception_utility_code)
  6717. code.globalstate.use_utility_code(swap_exception_utility_code)
  6718. if self.is_try_finally_in_nogil:
  6719. code.put_ensure_gil(declare_gilstate=False)
  6720. code.putln("__Pyx_PyThreadState_assign")
  6721. code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
  6722. for temp_name, type in temps_to_clean_up:
  6723. code.put_xdecref_clear(temp_name, type)
  6724. # not using preprocessor here to avoid warnings about
  6725. # unused utility functions and/or temps
  6726. code.putln("if (PY_MAJOR_VERSION >= 3)"
  6727. " __Pyx_ExceptionSwap(&%s, &%s, &%s);" % exc_vars[3:])
  6728. code.putln("if ((PY_MAJOR_VERSION < 3) ||"
  6729. # if __Pyx_GetException() fails in Py3,
  6730. # store the newly raised exception instead
  6731. " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) "
  6732. "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2))
  6733. for var in exc_vars:
  6734. code.put_xgotref(var)
  6735. if exc_lineno_cnames:
  6736. code.putln("%s = %s; %s = %s; %s = %s;" % (
  6737. exc_lineno_cnames[0], Naming.lineno_cname,
  6738. exc_lineno_cnames[1], Naming.clineno_cname,
  6739. exc_filename_cname, Naming.filename_cname))
  6740. if self.is_try_finally_in_nogil:
  6741. code.put_release_ensured_gil()
  6742. def put_error_uncatcher(self, code, exc_vars, exc_lineno_cnames=None, exc_filename_cname=None):
  6743. code.globalstate.use_utility_code(restore_exception_utility_code)
  6744. code.globalstate.use_utility_code(reset_exception_utility_code)
  6745. if self.is_try_finally_in_nogil:
  6746. code.put_ensure_gil(declare_gilstate=False)
  6747. # not using preprocessor here to avoid warnings about
  6748. # unused utility functions and/or temps
  6749. code.putln("if (PY_MAJOR_VERSION >= 3) {")
  6750. for var in exc_vars[3:]:
  6751. code.put_xgiveref(var)
  6752. code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
  6753. code.putln("}")
  6754. for var in exc_vars[:3]:
  6755. code.put_xgiveref(var)
  6756. code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3])
  6757. if self.is_try_finally_in_nogil:
  6758. code.put_release_ensured_gil()
  6759. code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
  6760. if exc_lineno_cnames:
  6761. code.putln("%s = %s; %s = %s; %s = %s;" % (
  6762. Naming.lineno_cname, exc_lineno_cnames[0],
  6763. Naming.clineno_cname, exc_lineno_cnames[1],
  6764. Naming.filename_cname, exc_filename_cname))
  6765. def put_error_cleaner(self, code, exc_vars):
  6766. code.globalstate.use_utility_code(reset_exception_utility_code)
  6767. if self.is_try_finally_in_nogil:
  6768. code.put_ensure_gil(declare_gilstate=False)
  6769. # not using preprocessor here to avoid warnings about
  6770. # unused utility functions and/or temps
  6771. code.putln("if (PY_MAJOR_VERSION >= 3) {")
  6772. for var in exc_vars[3:]:
  6773. code.put_xgiveref(var)
  6774. code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
  6775. code.putln("}")
  6776. for var in exc_vars[:3]:
  6777. code.put_xdecref_clear(var, py_object_type)
  6778. if self.is_try_finally_in_nogil:
  6779. code.put_release_ensured_gil()
  6780. code.putln(' '.join(["%s = 0;"]*3) % exc_vars[3:])
  6781. def annotate(self, code):
  6782. self.body.annotate(code)
  6783. self.finally_clause.annotate(code)
  6784. class NogilTryFinallyStatNode(TryFinallyStatNode):
  6785. """
  6786. A try/finally statement that may be used in nogil code sections.
  6787. """
  6788. preserve_exception = False
  6789. nogil_check = None
  6790. class GILStatNode(NogilTryFinallyStatNode):
  6791. # 'with gil' or 'with nogil' statement
  6792. #
  6793. # state string 'gil' or 'nogil'
  6794. state_temp = None
  6795. def __init__(self, pos, state, body):
  6796. self.state = state
  6797. self.create_state_temp_if_needed(pos, state, body)
  6798. TryFinallyStatNode.__init__(
  6799. self, pos,
  6800. body=body,
  6801. finally_clause=GILExitNode(
  6802. pos, state=state, state_temp=self.state_temp))
  6803. def create_state_temp_if_needed(self, pos, state, body):
  6804. from .ParseTreeTransforms import YieldNodeCollector
  6805. collector = YieldNodeCollector()
  6806. collector.visitchildren(body)
  6807. if not collector.yields:
  6808. return
  6809. if state == 'gil':
  6810. temp_type = PyrexTypes.c_gilstate_type
  6811. else:
  6812. temp_type = PyrexTypes.c_threadstate_ptr_type
  6813. from . import ExprNodes
  6814. self.state_temp = ExprNodes.TempNode(pos, temp_type)
  6815. def analyse_declarations(self, env):
  6816. env._in_with_gil_block = (self.state == 'gil')
  6817. if self.state == 'gil':
  6818. env.has_with_gil_block = True
  6819. return super(GILStatNode, self).analyse_declarations(env)
  6820. def analyse_expressions(self, env):
  6821. env.use_utility_code(
  6822. UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
  6823. was_nogil = env.nogil
  6824. env.nogil = self.state == 'nogil'
  6825. node = TryFinallyStatNode.analyse_expressions(self, env)
  6826. env.nogil = was_nogil
  6827. return node
  6828. def generate_execution_code(self, code):
  6829. code.mark_pos(self.pos)
  6830. code.begin_block()
  6831. if self.state_temp:
  6832. self.state_temp.allocate(code)
  6833. variable = self.state_temp.result()
  6834. else:
  6835. variable = None
  6836. old_gil_config = code.funcstate.gil_owned
  6837. if self.state == 'gil':
  6838. code.put_ensure_gil(variable=variable)
  6839. code.funcstate.gil_owned = True
  6840. else:
  6841. code.put_release_gil(variable=variable)
  6842. code.funcstate.gil_owned = False
  6843. TryFinallyStatNode.generate_execution_code(self, code)
  6844. if self.state_temp:
  6845. self.state_temp.release(code)
  6846. code.funcstate.gil_owned = old_gil_config
  6847. code.end_block()
  6848. class GILExitNode(StatNode):
  6849. """
  6850. Used as the 'finally' block in a GILStatNode
  6851. state string 'gil' or 'nogil'
  6852. """
  6853. child_attrs = []
  6854. state_temp = None
  6855. def analyse_expressions(self, env):
  6856. return self
  6857. def generate_execution_code(self, code):
  6858. if self.state_temp:
  6859. variable = self.state_temp.result()
  6860. else:
  6861. variable = None
  6862. if self.state == 'gil':
  6863. code.put_release_ensured_gil(variable)
  6864. else:
  6865. code.put_acquire_gil(variable)
  6866. class EnsureGILNode(GILExitNode):
  6867. """
  6868. Ensure the GIL in nogil functions for cleanup before returning.
  6869. """
  6870. def generate_execution_code(self, code):
  6871. code.put_ensure_gil(declare_gilstate=False)
  6872. def cython_view_utility_code():
  6873. from . import MemoryView
  6874. return MemoryView.view_utility_code
  6875. utility_code_for_cimports = {
  6876. # utility code (or inlining c) in a pxd (or pyx) file.
  6877. # TODO: Consider a generic user-level mechanism for importing
  6878. 'cpython.array' : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
  6879. 'cpython.array.array' : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
  6880. 'cython.view' : cython_view_utility_code,
  6881. }
  6882. utility_code_for_imports = {
  6883. # utility code used when special modules are imported.
  6884. # TODO: Consider a generic user-level mechanism for importing
  6885. 'asyncio': ("__Pyx_patch_asyncio", "PatchAsyncIO", "Coroutine.c"),
  6886. 'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"),
  6887. }
  6888. class CImportStatNode(StatNode):
  6889. # cimport statement
  6890. #
  6891. # module_name string Qualified name of module being imported
  6892. # as_name string or None Name specified in "as" clause, if any
  6893. # is_absolute bool True for absolute imports, False otherwise
  6894. child_attrs = []
  6895. is_absolute = False
  6896. def analyse_declarations(self, env):
  6897. if not env.is_module_scope:
  6898. error(self.pos, "cimport only allowed at module level")
  6899. return
  6900. module_scope = env.find_module(
  6901. self.module_name, self.pos, relative_level=0 if self.is_absolute else -1)
  6902. if "." in self.module_name:
  6903. names = [EncodedString(name) for name in self.module_name.split(".")]
  6904. top_name = names[0]
  6905. top_module_scope = env.context.find_submodule(top_name)
  6906. module_scope = top_module_scope
  6907. for name in names[1:]:
  6908. submodule_scope = module_scope.find_submodule(name)
  6909. module_scope.declare_module(name, submodule_scope, self.pos)
  6910. module_scope = submodule_scope
  6911. if self.as_name:
  6912. env.declare_module(self.as_name, module_scope, self.pos)
  6913. else:
  6914. env.add_imported_module(module_scope)
  6915. env.declare_module(top_name, top_module_scope, self.pos)
  6916. else:
  6917. name = self.as_name or self.module_name
  6918. env.declare_module(name, module_scope, self.pos)
  6919. if self.module_name in utility_code_for_cimports:
  6920. env.use_utility_code(utility_code_for_cimports[self.module_name]())
  6921. def analyse_expressions(self, env):
  6922. return self
  6923. def generate_execution_code(self, code):
  6924. pass
  6925. class FromCImportStatNode(StatNode):
  6926. # from ... cimport statement
  6927. #
  6928. # module_name string Qualified name of module
  6929. # relative_level int or None Relative import: number of dots before module_name
  6930. # imported_names [(pos, name, as_name, kind)] Names to be imported
  6931. child_attrs = []
  6932. module_name = None
  6933. relative_level = None
  6934. imported_names = None
  6935. def analyse_declarations(self, env):
  6936. if not env.is_module_scope:
  6937. error(self.pos, "cimport only allowed at module level")
  6938. return
  6939. if self.relative_level and self.relative_level > env.qualified_name.count('.'):
  6940. error(self.pos, "relative cimport beyond main package is not allowed")
  6941. return
  6942. module_scope = env.find_module(self.module_name, self.pos, relative_level=self.relative_level, need_pxd=0)
  6943. module_name = module_scope.qualified_name
  6944. env.add_imported_module(module_scope)
  6945. for pos, name, as_name, kind in self.imported_names:
  6946. if name == "*":
  6947. for local_name, entry in list(module_scope.entries.items()):
  6948. env.add_imported_entry(local_name, entry, pos)
  6949. else:
  6950. entry = module_scope.lookup(name)
  6951. if entry:
  6952. if kind and not self.declaration_matches(entry, kind):
  6953. entry.redeclared(pos)
  6954. entry.used = 1
  6955. else:
  6956. if kind == 'struct' or kind == 'union':
  6957. entry = module_scope.declare_struct_or_union(
  6958. name, kind=kind, scope=None, typedef_flag=0, pos=pos)
  6959. elif kind == 'class':
  6960. entry = module_scope.declare_c_class(name, pos=pos, module_name=module_name)
  6961. else:
  6962. submodule_scope = env.context.find_module(
  6963. name, relative_to=module_scope, pos=self.pos, absolute_fallback=False)
  6964. if submodule_scope.parent_module is module_scope:
  6965. env.declare_module(as_name or name, submodule_scope, self.pos)
  6966. else:
  6967. error(pos, "Name '%s' not declared in module '%s'" % (name, module_name))
  6968. if entry:
  6969. local_name = as_name or name
  6970. env.add_imported_entry(local_name, entry, pos)
  6971. if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now
  6972. if module_name in utility_code_for_cimports:
  6973. env.use_utility_code(utility_code_for_cimports[module_name]())
  6974. for _, name, _, _ in self.imported_names:
  6975. fqname = '%s.%s' % (module_name, name)
  6976. if fqname in utility_code_for_cimports:
  6977. env.use_utility_code(utility_code_for_cimports[fqname]())
  6978. def declaration_matches(self, entry, kind):
  6979. if not entry.is_type:
  6980. return 0
  6981. type = entry.type
  6982. if kind == 'class':
  6983. if not type.is_extension_type:
  6984. return 0
  6985. else:
  6986. if not type.is_struct_or_union:
  6987. return 0
  6988. if kind != type.kind:
  6989. return 0
  6990. return 1
  6991. def analyse_expressions(self, env):
  6992. return self
  6993. def generate_execution_code(self, code):
  6994. pass
  6995. class FromImportStatNode(StatNode):
  6996. # from ... import statement
  6997. #
  6998. # module ImportNode
  6999. # items [(string, NameNode)]
  7000. # interned_items [(string, NameNode, ExprNode)]
  7001. # item PyTempNode used internally
  7002. # import_star boolean used internally
  7003. child_attrs = ["module"]
  7004. import_star = 0
  7005. def analyse_declarations(self, env):
  7006. for name, target in self.items:
  7007. if name == "*":
  7008. if not env.is_module_scope:
  7009. error(self.pos, "import * only allowed at module level")
  7010. return
  7011. env.has_import_star = 1
  7012. self.import_star = 1
  7013. else:
  7014. target.analyse_target_declaration(env)
  7015. def analyse_expressions(self, env):
  7016. from . import ExprNodes
  7017. self.module = self.module.analyse_expressions(env)
  7018. self.item = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
  7019. self.interned_items = []
  7020. for name, target in self.items:
  7021. if name == '*':
  7022. for _, entry in env.entries.items():
  7023. if not entry.is_type and entry.type.is_extension_type:
  7024. env.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
  7025. break
  7026. else:
  7027. entry = env.lookup(target.name)
  7028. # check whether or not entry is already cimported
  7029. if (entry.is_type and entry.type.name == name
  7030. and hasattr(entry.type, 'module_name')):
  7031. if entry.type.module_name == self.module.module_name.value:
  7032. # cimported with absolute name
  7033. continue
  7034. try:
  7035. # cimported with relative name
  7036. module = env.find_module(self.module.module_name.value, pos=self.pos,
  7037. relative_level=self.module.level)
  7038. if entry.type.module_name == module.qualified_name:
  7039. continue
  7040. except AttributeError:
  7041. pass
  7042. target = target.analyse_target_expression(env, None) # FIXME?
  7043. if target.type is py_object_type:
  7044. coerced_item = None
  7045. else:
  7046. coerced_item = self.item.coerce_to(target.type, env)
  7047. self.interned_items.append((name, target, coerced_item))
  7048. return self
  7049. def generate_execution_code(self, code):
  7050. code.mark_pos(self.pos)
  7051. self.module.generate_evaluation_code(code)
  7052. if self.import_star:
  7053. code.putln(
  7054. 'if (%s(%s) < 0) %s;' % (
  7055. Naming.import_star,
  7056. self.module.py_result(),
  7057. code.error_goto(self.pos)))
  7058. item_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
  7059. self.item.set_cname(item_temp)
  7060. if self.interned_items:
  7061. code.globalstate.use_utility_code(
  7062. UtilityCode.load_cached("ImportFrom", "ImportExport.c"))
  7063. for name, target, coerced_item in self.interned_items:
  7064. code.putln(
  7065. '%s = __Pyx_ImportFrom(%s, %s); %s' % (
  7066. item_temp,
  7067. self.module.py_result(),
  7068. code.intern_identifier(name),
  7069. code.error_goto_if_null(item_temp, self.pos)))
  7070. code.put_gotref(item_temp)
  7071. if coerced_item is None:
  7072. target.generate_assignment_code(self.item, code)
  7073. else:
  7074. coerced_item.allocate_temp_result(code)
  7075. coerced_item.generate_result_code(code)
  7076. target.generate_assignment_code(coerced_item, code)
  7077. code.put_decref_clear(item_temp, py_object_type)
  7078. code.funcstate.release_temp(item_temp)
  7079. self.module.generate_disposal_code(code)
  7080. self.module.free_temps(code)
  7081. class ParallelNode(Node):
  7082. """
  7083. Base class for cython.parallel constructs.
  7084. """
  7085. nogil_check = None
  7086. class ParallelStatNode(StatNode, ParallelNode):
  7087. """
  7088. Base class for 'with cython.parallel.parallel():' and 'for i in prange():'.
  7089. assignments { Entry(var) : (var.pos, inplace_operator_or_None) }
  7090. assignments to variables in this parallel section
  7091. parent parent ParallelStatNode or None
  7092. is_parallel indicates whether this node is OpenMP parallel
  7093. (true for #pragma omp parallel for and
  7094. #pragma omp parallel)
  7095. is_parallel is true for:
  7096. #pragma omp parallel
  7097. #pragma omp parallel for
  7098. sections, but NOT for
  7099. #pragma omp for
  7100. We need this to determine the sharing attributes.
  7101. privatization_insertion_point a code insertion point used to make temps
  7102. private (esp. the "nsteps" temp)
  7103. args tuple the arguments passed to the parallel construct
  7104. kwargs DictNode the keyword arguments passed to the parallel
  7105. construct (replaced by its compile time value)
  7106. """
  7107. child_attrs = ['body', 'num_threads']
  7108. body = None
  7109. is_prange = False
  7110. is_nested_prange = False
  7111. error_label_used = False
  7112. num_threads = None
  7113. chunksize = None
  7114. parallel_exc = (
  7115. Naming.parallel_exc_type,
  7116. Naming.parallel_exc_value,
  7117. Naming.parallel_exc_tb,
  7118. )
  7119. parallel_pos_info = (
  7120. Naming.parallel_filename,
  7121. Naming.parallel_lineno,
  7122. Naming.parallel_clineno,
  7123. )
  7124. pos_info = (
  7125. Naming.filename_cname,
  7126. Naming.lineno_cname,
  7127. Naming.clineno_cname,
  7128. )
  7129. critical_section_counter = 0
  7130. def __init__(self, pos, **kwargs):
  7131. super(ParallelStatNode, self).__init__(pos, **kwargs)
  7132. # All assignments in this scope
  7133. self.assignments = kwargs.get('assignments') or {}
  7134. # All seen closure cnames and their temporary cnames
  7135. self.seen_closure_vars = set()
  7136. # Dict of variables that should be declared (first|last|)private or
  7137. # reduction { Entry: (op, lastprivate) }.
  7138. # If op is not None, it's a reduction.
  7139. self.privates = {}
  7140. # [NameNode]
  7141. self.assigned_nodes = []
  7142. def analyse_declarations(self, env):
  7143. self.body.analyse_declarations(env)
  7144. self.num_threads = None
  7145. if self.kwargs:
  7146. # Try to find num_threads and chunksize keyword arguments
  7147. pairs = []
  7148. seen = set()
  7149. for dictitem in self.kwargs.key_value_pairs:
  7150. if dictitem.key.value in seen:
  7151. error(self.pos, "Duplicate keyword argument found: %s" % dictitem.key.value)
  7152. seen.add(dictitem.key.value)
  7153. if dictitem.key.value == 'num_threads':
  7154. if not dictitem.value.is_none:
  7155. self.num_threads = dictitem.value
  7156. elif self.is_prange and dictitem.key.value == 'chunksize':
  7157. if not dictitem.value.is_none:
  7158. self.chunksize = dictitem.value
  7159. else:
  7160. pairs.append(dictitem)
  7161. self.kwargs.key_value_pairs = pairs
  7162. try:
  7163. self.kwargs = self.kwargs.compile_time_value(env)
  7164. except Exception as e:
  7165. error(self.kwargs.pos, "Only compile-time values may be "
  7166. "supplied as keyword arguments")
  7167. else:
  7168. self.kwargs = {}
  7169. for kw, val in self.kwargs.items():
  7170. if kw not in self.valid_keyword_arguments:
  7171. error(self.pos, "Invalid keyword argument: %s" % kw)
  7172. else:
  7173. setattr(self, kw, val)
  7174. def analyse_expressions(self, env):
  7175. if self.num_threads:
  7176. self.num_threads = self.num_threads.analyse_expressions(env)
  7177. if self.chunksize:
  7178. self.chunksize = self.chunksize.analyse_expressions(env)
  7179. self.body = self.body.analyse_expressions(env)
  7180. self.analyse_sharing_attributes(env)
  7181. if self.num_threads is not None:
  7182. if self.parent and self.parent.num_threads is not None and not self.parent.is_prange:
  7183. error(self.pos, "num_threads already declared in outer section")
  7184. elif self.parent and not self.parent.is_prange:
  7185. error(self.pos, "num_threads must be declared in the parent parallel section")
  7186. elif (self.num_threads.type.is_int and
  7187. self.num_threads.is_literal and
  7188. self.num_threads.compile_time_value(env) <= 0):
  7189. error(self.pos, "argument to num_threads must be greater than 0")
  7190. if not self.num_threads.is_simple() or self.num_threads.type.is_pyobject:
  7191. self.num_threads = self.num_threads.coerce_to(
  7192. PyrexTypes.c_int_type, env).coerce_to_temp(env)
  7193. return self
  7194. def analyse_sharing_attributes(self, env):
  7195. """
  7196. Analyse the privates for this block and set them in self.privates.
  7197. This should be called in a post-order fashion during the
  7198. analyse_expressions phase
  7199. """
  7200. for entry, (pos, op) in self.assignments.items():
  7201. if self.is_prange and not self.is_parallel:
  7202. # closely nested prange in a with parallel block, disallow
  7203. # assigning to privates in the with parallel block (we
  7204. # consider it too implicit and magicky for users)
  7205. if entry in self.parent.assignments:
  7206. error(pos, "Cannot assign to private of outer parallel block")
  7207. continue
  7208. if not self.is_prange and op:
  7209. # Again possible, but considered to magicky
  7210. error(pos, "Reductions not allowed for parallel blocks")
  7211. continue
  7212. # By default all variables should have the same values as if
  7213. # executed sequentially
  7214. lastprivate = True
  7215. self.propagate_var_privatization(entry, pos, op, lastprivate)
  7216. def propagate_var_privatization(self, entry, pos, op, lastprivate):
  7217. """
  7218. Propagate the sharing attributes of a variable. If the privatization is
  7219. determined by a parent scope, done propagate further.
  7220. If we are a prange, we propagate our sharing attributes outwards to
  7221. other pranges. If we are a prange in parallel block and the parallel
  7222. block does not determine the variable private, we propagate to the
  7223. parent of the parent. Recursion stops at parallel blocks, as they have
  7224. no concept of lastprivate or reduction.
  7225. So the following cases propagate:
  7226. sum is a reduction for all loops:
  7227. for i in prange(n):
  7228. for j in prange(n):
  7229. for k in prange(n):
  7230. sum += i * j * k
  7231. sum is a reduction for both loops, local_var is private to the
  7232. parallel with block:
  7233. for i in prange(n):
  7234. with parallel:
  7235. local_var = ... # private to the parallel
  7236. for j in prange(n):
  7237. sum += i * j
  7238. Nested with parallel blocks are disallowed, because they wouldn't
  7239. allow you to propagate lastprivates or reductions:
  7240. #pragma omp parallel for lastprivate(i)
  7241. for i in prange(n):
  7242. sum = 0
  7243. #pragma omp parallel private(j, sum)
  7244. with parallel:
  7245. #pragma omp parallel
  7246. with parallel:
  7247. #pragma omp for lastprivate(j) reduction(+:sum)
  7248. for j in prange(n):
  7249. sum += i
  7250. # sum and j are well-defined here
  7251. # sum and j are undefined here
  7252. # sum and j are undefined here
  7253. """
  7254. self.privates[entry] = (op, lastprivate)
  7255. if entry.type.is_memoryviewslice:
  7256. error(pos, "Memoryview slices can only be shared in parallel sections")
  7257. return
  7258. if self.is_prange:
  7259. if not self.is_parallel and entry not in self.parent.assignments:
  7260. # Parent is a parallel with block
  7261. parent = self.parent.parent
  7262. else:
  7263. parent = self.parent
  7264. # We don't need to propagate privates, only reductions and
  7265. # lastprivates
  7266. if parent and (op or lastprivate):
  7267. parent.propagate_var_privatization(entry, pos, op, lastprivate)
  7268. def _allocate_closure_temp(self, code, entry):
  7269. """
  7270. Helper function that allocate a temporary for a closure variable that
  7271. is assigned to.
  7272. """
  7273. if self.parent:
  7274. return self.parent._allocate_closure_temp(code, entry)
  7275. if entry.cname in self.seen_closure_vars:
  7276. return entry.cname
  7277. cname = code.funcstate.allocate_temp(entry.type, True)
  7278. # Add both the actual cname and the temp cname, as the actual cname
  7279. # will be replaced with the temp cname on the entry
  7280. self.seen_closure_vars.add(entry.cname)
  7281. self.seen_closure_vars.add(cname)
  7282. self.modified_entries.append((entry, entry.cname))
  7283. code.putln("%s = %s;" % (cname, entry.cname))
  7284. entry.cname = cname
  7285. def initialize_privates_to_nan(self, code, exclude=None):
  7286. first = True
  7287. for entry, (op, lastprivate) in sorted(self.privates.items()):
  7288. if not op and (not exclude or entry != exclude):
  7289. invalid_value = entry.type.invalid_value()
  7290. if invalid_value:
  7291. if first:
  7292. code.putln("/* Initialize private variables to "
  7293. "invalid values */")
  7294. first = False
  7295. code.putln("%s = %s;" % (entry.cname,
  7296. entry.type.cast_code(invalid_value)))
  7297. def evaluate_before_block(self, code, expr):
  7298. c = self.begin_of_parallel_control_block_point_after_decls
  7299. # we need to set the owner to ourselves temporarily, as
  7300. # allocate_temp may generate a comment in the middle of our pragma
  7301. # otherwise when DebugFlags.debug_temp_code_comments is in effect
  7302. owner = c.funcstate.owner
  7303. c.funcstate.owner = c
  7304. expr.generate_evaluation_code(c)
  7305. c.funcstate.owner = owner
  7306. return expr.result()
  7307. def put_num_threads(self, code):
  7308. """
  7309. Write self.num_threads if set as the num_threads OpenMP directive
  7310. """
  7311. if self.num_threads is not None:
  7312. code.put(" num_threads(%s)" % self.evaluate_before_block(code, self.num_threads))
  7313. def declare_closure_privates(self, code):
  7314. """
  7315. If a variable is in a scope object, we need to allocate a temp and
  7316. assign the value from the temp to the variable in the scope object
  7317. after the parallel section. This kind of copying should be done only
  7318. in the outermost parallel section.
  7319. """
  7320. self.modified_entries = []
  7321. for entry in sorted(self.assignments):
  7322. if entry.from_closure or entry.in_closure:
  7323. self._allocate_closure_temp(code, entry)
  7324. def release_closure_privates(self, code):
  7325. """
  7326. Release any temps used for variables in scope objects. As this is the
  7327. outermost parallel block, we don't need to delete the cnames from
  7328. self.seen_closure_vars.
  7329. """
  7330. for entry, original_cname in self.modified_entries:
  7331. code.putln("%s = %s;" % (original_cname, entry.cname))
  7332. code.funcstate.release_temp(entry.cname)
  7333. entry.cname = original_cname
  7334. def privatize_temps(self, code, exclude_temps=()):
  7335. """
  7336. Make any used temporaries private. Before the relevant code block
  7337. code.start_collecting_temps() should have been called.
  7338. """
  7339. c = self.privatization_insertion_point
  7340. self.privatization_insertion_point = None
  7341. if self.is_parallel:
  7342. self.temps = temps = code.funcstate.stop_collecting_temps()
  7343. privates, firstprivates = [], []
  7344. for temp, type in sorted(temps):
  7345. if type.is_pyobject or type.is_memoryviewslice:
  7346. firstprivates.append(temp)
  7347. else:
  7348. privates.append(temp)
  7349. if privates:
  7350. c.put(" private(%s)" % ", ".join(privates))
  7351. if firstprivates:
  7352. c.put(" firstprivate(%s)" % ", ".join(firstprivates))
  7353. if self.breaking_label_used:
  7354. shared_vars = [Naming.parallel_why]
  7355. if self.error_label_used:
  7356. shared_vars.extend(self.parallel_exc)
  7357. c.put(" private(%s, %s, %s)" % self.pos_info)
  7358. c.put(" shared(%s)" % ', '.join(shared_vars))
  7359. def cleanup_temps(self, code):
  7360. # Now clean up any memoryview slice and object temporaries
  7361. if self.is_parallel and not self.is_nested_prange:
  7362. code.putln("/* Clean up any temporaries */")
  7363. for temp, type in sorted(self.temps):
  7364. if type.is_memoryviewslice:
  7365. code.put_xdecref_memoryviewslice(temp, have_gil=False)
  7366. elif type.is_pyobject:
  7367. code.put_xdecref(temp, type)
  7368. code.putln("%s = NULL;" % temp)
  7369. def setup_parallel_control_flow_block(self, code):
  7370. """
  7371. Sets up a block that surrounds the parallel block to determine
  7372. how the parallel section was exited. Any kind of return is
  7373. trapped (break, continue, return, exceptions). This is the idea:
  7374. {
  7375. int why = 0;
  7376. #pragma omp parallel
  7377. {
  7378. return # -> goto new_return_label;
  7379. goto end_parallel;
  7380. new_return_label:
  7381. why = 3;
  7382. goto end_parallel;
  7383. end_parallel:;
  7384. #pragma omp flush(why) # we need to flush for every iteration
  7385. }
  7386. if (why == 3)
  7387. goto old_return_label;
  7388. }
  7389. """
  7390. self.old_loop_labels = code.new_loop_labels()
  7391. self.old_error_label = code.new_error_label()
  7392. self.old_return_label = code.return_label
  7393. code.return_label = code.new_label(name="return")
  7394. code.begin_block() # parallel control flow block
  7395. self.begin_of_parallel_control_block_point = code.insertion_point()
  7396. self.begin_of_parallel_control_block_point_after_decls = code.insertion_point()
  7397. self.undef_builtin_expect_apple_gcc_bug(code)
  7398. def begin_parallel_block(self, code):
  7399. """
  7400. Each OpenMP thread in a parallel section that contains a with gil block
  7401. must have the thread-state initialized. The call to
  7402. PyGILState_Release() then deallocates our threadstate. If we wouldn't
  7403. do this, each with gil block would allocate and deallocate one, thereby
  7404. losing exception information before it can be saved before leaving the
  7405. parallel section.
  7406. """
  7407. self.begin_of_parallel_block = code.insertion_point()
  7408. def end_parallel_block(self, code):
  7409. """
  7410. To ensure all OpenMP threads have thread states, we ensure the GIL
  7411. in each thread (which creates a thread state if it doesn't exist),
  7412. after which we release the GIL.
  7413. On exit, reacquire the GIL and release the thread state.
  7414. If compiled without OpenMP support (at the C level), then we still have
  7415. to acquire the GIL to decref any object temporaries.
  7416. """
  7417. begin_code = self.begin_of_parallel_block
  7418. self.begin_of_parallel_block = None
  7419. if self.error_label_used:
  7420. end_code = code
  7421. begin_code.putln("#ifdef _OPENMP")
  7422. begin_code.put_ensure_gil(declare_gilstate=True)
  7423. begin_code.putln("Py_BEGIN_ALLOW_THREADS")
  7424. begin_code.putln("#endif /* _OPENMP */")
  7425. end_code.putln("#ifdef _OPENMP")
  7426. end_code.putln("Py_END_ALLOW_THREADS")
  7427. end_code.putln("#else")
  7428. end_code.put_safe("{\n")
  7429. end_code.put_ensure_gil()
  7430. end_code.putln("#endif /* _OPENMP */")
  7431. self.cleanup_temps(end_code)
  7432. end_code.put_release_ensured_gil()
  7433. end_code.putln("#ifndef _OPENMP")
  7434. end_code.put_safe("}\n")
  7435. end_code.putln("#endif /* _OPENMP */")
  7436. def trap_parallel_exit(self, code, should_flush=False):
  7437. """
  7438. Trap any kind of return inside a parallel construct. 'should_flush'
  7439. indicates whether the variable should be flushed, which is needed by
  7440. prange to skip the loop. It also indicates whether we need to register
  7441. a continue (we need this for parallel blocks, but not for prange
  7442. loops, as it is a direct jump there).
  7443. It uses the same mechanism as try/finally:
  7444. 1 continue
  7445. 2 break
  7446. 3 return
  7447. 4 error
  7448. """
  7449. save_lastprivates_label = code.new_label()
  7450. dont_return_label = code.new_label()
  7451. self.any_label_used = False
  7452. self.breaking_label_used = False
  7453. self.error_label_used = False
  7454. self.parallel_private_temps = []
  7455. all_labels = code.get_all_labels()
  7456. # Figure this out before starting to generate any code
  7457. for label in all_labels:
  7458. if code.label_used(label):
  7459. self.breaking_label_used = (self.breaking_label_used or
  7460. label != code.continue_label)
  7461. self.any_label_used = True
  7462. if self.any_label_used:
  7463. code.put_goto(dont_return_label)
  7464. for i, label in enumerate(all_labels):
  7465. if not code.label_used(label):
  7466. continue
  7467. is_continue_label = label == code.continue_label
  7468. code.put_label(label)
  7469. if not (should_flush and is_continue_label):
  7470. if label == code.error_label:
  7471. self.error_label_used = True
  7472. self.fetch_parallel_exception(code)
  7473. code.putln("%s = %d;" % (Naming.parallel_why, i + 1))
  7474. if (self.breaking_label_used and self.is_prange and not
  7475. is_continue_label):
  7476. code.put_goto(save_lastprivates_label)
  7477. else:
  7478. code.put_goto(dont_return_label)
  7479. if self.any_label_used:
  7480. if self.is_prange and self.breaking_label_used:
  7481. # Don't rely on lastprivate, save our lastprivates
  7482. code.put_label(save_lastprivates_label)
  7483. self.save_parallel_vars(code)
  7484. code.put_label(dont_return_label)
  7485. if should_flush and self.breaking_label_used:
  7486. code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_why)
  7487. def save_parallel_vars(self, code):
  7488. """
  7489. The following shenanigans are instated when we break, return or
  7490. propagate errors from a prange. In this case we cannot rely on
  7491. lastprivate() to do its job, as no iterations may have executed yet
  7492. in the last thread, leaving the values undefined. It is most likely
  7493. that the breaking thread has well-defined values of the lastprivate
  7494. variables, so we keep those values.
  7495. """
  7496. section_name = "__pyx_parallel_lastprivates%d" % self.critical_section_counter
  7497. code.putln_openmp("#pragma omp critical(%s)" % section_name)
  7498. ParallelStatNode.critical_section_counter += 1
  7499. code.begin_block() # begin critical section
  7500. c = self.begin_of_parallel_control_block_point
  7501. temp_count = 0
  7502. for entry, (op, lastprivate) in sorted(self.privates.items()):
  7503. if not lastprivate or entry.type.is_pyobject:
  7504. continue
  7505. type_decl = entry.type.empty_declaration_code()
  7506. temp_cname = "__pyx_parallel_temp%d" % temp_count
  7507. private_cname = entry.cname
  7508. temp_count += 1
  7509. invalid_value = entry.type.invalid_value()
  7510. if invalid_value:
  7511. init = ' = ' + entry.type.cast_code(invalid_value)
  7512. else:
  7513. init = ''
  7514. # Declare the parallel private in the outer block
  7515. c.putln("%s %s%s;" % (type_decl, temp_cname, init))
  7516. # Initialize before escaping
  7517. code.putln("%s = %s;" % (temp_cname, private_cname))
  7518. self.parallel_private_temps.append((temp_cname, private_cname))
  7519. code.end_block() # end critical section
  7520. def fetch_parallel_exception(self, code):
  7521. """
  7522. As each OpenMP thread may raise an exception, we need to fetch that
  7523. exception from the threadstate and save it for after the parallel
  7524. section where it can be re-raised in the master thread.
  7525. Although it would seem that __pyx_filename, __pyx_lineno and
  7526. __pyx_clineno are only assigned to under exception conditions (i.e.,
  7527. when we have the GIL), and thus should be allowed to be shared without
  7528. any race condition, they are in fact subject to the same race
  7529. conditions that they were previously when they were global variables
  7530. and functions were allowed to release the GIL:
  7531. thread A thread B
  7532. acquire
  7533. set lineno
  7534. release
  7535. acquire
  7536. set lineno
  7537. release
  7538. acquire
  7539. fetch exception
  7540. release
  7541. skip the fetch
  7542. deallocate threadstate deallocate threadstate
  7543. """
  7544. code.begin_block()
  7545. code.put_ensure_gil(declare_gilstate=True)
  7546. code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_exc_type)
  7547. code.putln(
  7548. "if (!%s) {" % Naming.parallel_exc_type)
  7549. code.putln("__Pyx_ErrFetchWithState(&%s, &%s, &%s);" % self.parallel_exc)
  7550. pos_info = chain(*zip(self.parallel_pos_info, self.pos_info))
  7551. code.funcstate.uses_error_indicator = True
  7552. code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
  7553. code.put_gotref(Naming.parallel_exc_type)
  7554. code.putln(
  7555. "}")
  7556. code.put_release_ensured_gil()
  7557. code.end_block()
  7558. def restore_parallel_exception(self, code):
  7559. "Re-raise a parallel exception"
  7560. code.begin_block()
  7561. code.put_ensure_gil(declare_gilstate=True)
  7562. code.put_giveref(Naming.parallel_exc_type)
  7563. code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc)
  7564. pos_info = chain(*zip(self.pos_info, self.parallel_pos_info))
  7565. code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
  7566. code.put_release_ensured_gil()
  7567. code.end_block()
  7568. def restore_labels(self, code):
  7569. """
  7570. Restore all old labels. Call this before the 'else' clause to for
  7571. loops and always before ending the parallel control flow block.
  7572. """
  7573. code.set_all_labels(self.old_loop_labels + (self.old_return_label,
  7574. self.old_error_label))
  7575. def end_parallel_control_flow_block(
  7576. self, code, break_=False, continue_=False, return_=False):
  7577. """
  7578. This ends the parallel control flow block and based on how the parallel
  7579. section was exited, takes the corresponding action. The break_ and
  7580. continue_ parameters indicate whether these should be propagated
  7581. outwards:
  7582. for i in prange(...):
  7583. with cython.parallel.parallel():
  7584. continue
  7585. Here break should be trapped in the parallel block, and propagated to
  7586. the for loop.
  7587. """
  7588. c = self.begin_of_parallel_control_block_point
  7589. self.begin_of_parallel_control_block_point = None
  7590. self.begin_of_parallel_control_block_point_after_decls = None
  7591. if self.num_threads is not None:
  7592. # FIXME: is it the right place? should not normally produce code.
  7593. self.num_threads.generate_disposal_code(code)
  7594. self.num_threads.free_temps(code)
  7595. # Firstly, always prefer errors over returning, continue or break
  7596. if self.error_label_used:
  7597. c.putln("const char *%s = NULL; int %s = 0, %s = 0;" % self.parallel_pos_info)
  7598. c.putln("PyObject *%s = NULL, *%s = NULL, *%s = NULL;" % self.parallel_exc)
  7599. code.putln(
  7600. "if (%s) {" % Naming.parallel_exc_type)
  7601. code.putln("/* This may have been overridden by a continue, "
  7602. "break or return in another thread. Prefer the error. */")
  7603. code.putln("%s = 4;" % Naming.parallel_why)
  7604. code.putln(
  7605. "}")
  7606. if continue_:
  7607. any_label_used = self.any_label_used
  7608. else:
  7609. any_label_used = self.breaking_label_used
  7610. if any_label_used:
  7611. # __pyx_parallel_why is used, declare and initialize
  7612. c.putln("int %s;" % Naming.parallel_why)
  7613. c.putln("%s = 0;" % Naming.parallel_why)
  7614. code.putln(
  7615. "if (%s) {" % Naming.parallel_why)
  7616. for temp_cname, private_cname in self.parallel_private_temps:
  7617. code.putln("%s = %s;" % (private_cname, temp_cname))
  7618. code.putln("switch (%s) {" % Naming.parallel_why)
  7619. if continue_:
  7620. code.put(" case 1: ")
  7621. code.put_goto(code.continue_label)
  7622. if break_:
  7623. code.put(" case 2: ")
  7624. code.put_goto(code.break_label)
  7625. if return_:
  7626. code.put(" case 3: ")
  7627. code.put_goto(code.return_label)
  7628. if self.error_label_used:
  7629. code.globalstate.use_utility_code(restore_exception_utility_code)
  7630. code.putln(" case 4:")
  7631. self.restore_parallel_exception(code)
  7632. code.put_goto(code.error_label)
  7633. code.putln("}") # end switch
  7634. code.putln(
  7635. "}") # end if
  7636. code.end_block() # end parallel control flow block
  7637. self.redef_builtin_expect_apple_gcc_bug(code)
  7638. # FIXME: improve with version number for OS X Lion
  7639. buggy_platform_macro_condition = "(defined(__APPLE__) || defined(__OSX__))"
  7640. have_expect_condition = "(defined(__GNUC__) && " \
  7641. "(__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))))"
  7642. redef_condition = "(%s && %s)" % (buggy_platform_macro_condition, have_expect_condition)
  7643. def undef_builtin_expect_apple_gcc_bug(self, code):
  7644. """
  7645. A bug on OS X Lion disallows __builtin_expect macros. This code avoids them
  7646. """
  7647. if not self.parent:
  7648. code.undef_builtin_expect(self.redef_condition)
  7649. def redef_builtin_expect_apple_gcc_bug(self, code):
  7650. if not self.parent:
  7651. code.redef_builtin_expect(self.redef_condition)
  7652. class ParallelWithBlockNode(ParallelStatNode):
  7653. """
  7654. This node represents a 'with cython.parallel.parallel():' block
  7655. """
  7656. valid_keyword_arguments = ['num_threads']
  7657. num_threads = None
  7658. def analyse_declarations(self, env):
  7659. super(ParallelWithBlockNode, self).analyse_declarations(env)
  7660. if self.args:
  7661. error(self.pos, "cython.parallel.parallel() does not take "
  7662. "positional arguments")
  7663. def generate_execution_code(self, code):
  7664. self.declare_closure_privates(code)
  7665. self.setup_parallel_control_flow_block(code)
  7666. code.putln("#ifdef _OPENMP")
  7667. code.put("#pragma omp parallel ")
  7668. if self.privates:
  7669. privates = [e.cname for e in self.privates
  7670. if not e.type.is_pyobject]
  7671. code.put('private(%s)' % ', '.join(sorted(privates)))
  7672. self.privatization_insertion_point = code.insertion_point()
  7673. self.put_num_threads(code)
  7674. code.putln("")
  7675. code.putln("#endif /* _OPENMP */")
  7676. code.begin_block() # parallel block
  7677. self.begin_parallel_block(code)
  7678. self.initialize_privates_to_nan(code)
  7679. code.funcstate.start_collecting_temps()
  7680. self.body.generate_execution_code(code)
  7681. self.trap_parallel_exit(code)
  7682. self.privatize_temps(code)
  7683. self.end_parallel_block(code)
  7684. code.end_block() # end parallel block
  7685. continue_ = code.label_used(code.continue_label)
  7686. break_ = code.label_used(code.break_label)
  7687. return_ = code.label_used(code.return_label)
  7688. self.restore_labels(code)
  7689. self.end_parallel_control_flow_block(code, break_=break_,
  7690. continue_=continue_,
  7691. return_=return_)
  7692. self.release_closure_privates(code)
  7693. class ParallelRangeNode(ParallelStatNode):
  7694. """
  7695. This node represents a 'for i in cython.parallel.prange():' construct.
  7696. target NameNode the target iteration variable
  7697. else_clause Node or None the else clause of this loop
  7698. """
  7699. child_attrs = ['body', 'target', 'else_clause', 'args', 'num_threads',
  7700. 'chunksize']
  7701. body = target = else_clause = args = None
  7702. start = stop = step = None
  7703. is_prange = True
  7704. nogil = None
  7705. schedule = None
  7706. valid_keyword_arguments = ['schedule', 'nogil', 'num_threads', 'chunksize']
  7707. def __init__(self, pos, **kwds):
  7708. super(ParallelRangeNode, self).__init__(pos, **kwds)
  7709. # Pretend to be a ForInStatNode for control flow analysis
  7710. self.iterator = PassStatNode(pos)
  7711. def analyse_declarations(self, env):
  7712. super(ParallelRangeNode, self).analyse_declarations(env)
  7713. self.target.analyse_target_declaration(env)
  7714. if self.else_clause is not None:
  7715. self.else_clause.analyse_declarations(env)
  7716. if not self.args or len(self.args) > 3:
  7717. error(self.pos, "Invalid number of positional arguments to prange")
  7718. return
  7719. if len(self.args) == 1:
  7720. self.stop, = self.args
  7721. elif len(self.args) == 2:
  7722. self.start, self.stop = self.args
  7723. else:
  7724. self.start, self.stop, self.step = self.args
  7725. if hasattr(self.schedule, 'decode'):
  7726. self.schedule = self.schedule.decode('ascii')
  7727. if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'):
  7728. error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,))
  7729. def analyse_expressions(self, env):
  7730. was_nogil = env.nogil
  7731. if self.nogil:
  7732. env.nogil = True
  7733. if self.target is None:
  7734. error(self.pos, "prange() can only be used as part of a for loop")
  7735. return self
  7736. self.target = self.target.analyse_target_types(env)
  7737. if not self.target.type.is_numeric:
  7738. # Not a valid type, assume one for now anyway
  7739. if not self.target.type.is_pyobject:
  7740. # nogil_check will catch the is_pyobject case
  7741. error(self.target.pos,
  7742. "Must be of numeric type, not %s" % self.target.type)
  7743. self.index_type = PyrexTypes.c_py_ssize_t_type
  7744. else:
  7745. self.index_type = self.target.type
  7746. # Setup start, stop and step, allocating temps if needed
  7747. self.names = 'start', 'stop', 'step'
  7748. start_stop_step = self.start, self.stop, self.step
  7749. for node, name in zip(start_stop_step, self.names):
  7750. if node is not None:
  7751. node.analyse_types(env)
  7752. if not node.type.is_numeric:
  7753. error(node.pos, "%s argument must be numeric" % name)
  7754. continue
  7755. if not node.is_literal:
  7756. node = node.coerce_to_temp(env)
  7757. setattr(self, name, node)
  7758. # As we range from 0 to nsteps, computing the index along the
  7759. # way, we need a fitting type for 'i' and 'nsteps'
  7760. self.index_type = PyrexTypes.widest_numeric_type(
  7761. self.index_type, node.type)
  7762. if self.else_clause is not None:
  7763. self.else_clause = self.else_clause.analyse_expressions(env)
  7764. # Although not actually an assignment in this scope, it should be
  7765. # treated as such to ensure it is unpacked if a closure temp, and to
  7766. # ensure lastprivate behaviour and propagation. If the target index is
  7767. # not a NameNode, it won't have an entry, and an error was issued by
  7768. # ParallelRangeTransform
  7769. if hasattr(self.target, 'entry'):
  7770. self.assignments[self.target.entry] = self.target.pos, None
  7771. node = super(ParallelRangeNode, self).analyse_expressions(env)
  7772. if node.chunksize:
  7773. if not node.schedule:
  7774. error(node.chunksize.pos,
  7775. "Must provide schedule with chunksize")
  7776. elif node.schedule == 'runtime':
  7777. error(node.chunksize.pos,
  7778. "Chunksize not valid for the schedule runtime")
  7779. elif (node.chunksize.type.is_int and
  7780. node.chunksize.is_literal and
  7781. node.chunksize.compile_time_value(env) <= 0):
  7782. error(node.chunksize.pos, "Chunksize must not be negative")
  7783. node.chunksize = node.chunksize.coerce_to(
  7784. PyrexTypes.c_int_type, env).coerce_to_temp(env)
  7785. if node.nogil:
  7786. env.nogil = was_nogil
  7787. node.is_nested_prange = node.parent and node.parent.is_prange
  7788. if node.is_nested_prange:
  7789. parent = node
  7790. while parent.parent and parent.parent.is_prange:
  7791. parent = parent.parent
  7792. parent.assignments.update(node.assignments)
  7793. parent.privates.update(node.privates)
  7794. parent.assigned_nodes.extend(node.assigned_nodes)
  7795. return node
  7796. def nogil_check(self, env):
  7797. names = 'start', 'stop', 'step', 'target'
  7798. nodes = self.start, self.stop, self.step, self.target
  7799. for name, node in zip(names, nodes):
  7800. if node is not None and node.type.is_pyobject:
  7801. error(node.pos, "%s may not be a Python object "
  7802. "as we don't have the GIL" % name)
  7803. def generate_execution_code(self, code):
  7804. """
  7805. Generate code in the following steps
  7806. 1) copy any closure variables determined thread-private
  7807. into temporaries
  7808. 2) allocate temps for start, stop and step
  7809. 3) generate a loop that calculates the total number of steps,
  7810. which then computes the target iteration variable for every step:
  7811. for i in prange(start, stop, step):
  7812. ...
  7813. becomes
  7814. nsteps = (stop - start) / step;
  7815. i = start;
  7816. #pragma omp parallel for lastprivate(i)
  7817. for (temp = 0; temp < nsteps; temp++) {
  7818. i = start + step * temp;
  7819. ...
  7820. }
  7821. Note that accumulation of 'i' would have a data dependency
  7822. between iterations.
  7823. Also, you can't do this
  7824. for (i = start; i < stop; i += step)
  7825. ...
  7826. as the '<' operator should become '>' for descending loops.
  7827. 'for i from x < i < y:' does not suffer from this problem
  7828. as the relational operator is known at compile time!
  7829. 4) release our temps and write back any private closure variables
  7830. """
  7831. self.declare_closure_privates(code)
  7832. # This can only be a NameNode
  7833. target_index_cname = self.target.entry.cname
  7834. # This will be used as the dict to format our code strings, holding
  7835. # the start, stop , step, temps and target cnames
  7836. fmt_dict = {
  7837. 'target': target_index_cname,
  7838. 'target_type': self.target.type.empty_declaration_code()
  7839. }
  7840. # Setup start, stop and step, allocating temps if needed
  7841. start_stop_step = self.start, self.stop, self.step
  7842. defaults = '0', '0', '1'
  7843. for node, name, default in zip(start_stop_step, self.names, defaults):
  7844. if node is None:
  7845. result = default
  7846. elif node.is_literal:
  7847. result = node.get_constant_c_result_code()
  7848. else:
  7849. node.generate_evaluation_code(code)
  7850. result = node.result()
  7851. fmt_dict[name] = result
  7852. fmt_dict['i'] = code.funcstate.allocate_temp(self.index_type, False)
  7853. fmt_dict['nsteps'] = code.funcstate.allocate_temp(self.index_type, False)
  7854. # TODO: check if the step is 0 and if so, raise an exception in a
  7855. # 'with gil' block. For now, just abort
  7856. code.putln("if ((%(step)s == 0)) abort();" % fmt_dict)
  7857. self.setup_parallel_control_flow_block(code) # parallel control flow block
  7858. # Note: nsteps is private in an outer scope if present
  7859. code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict)
  7860. # The target iteration variable might not be initialized, do it only if
  7861. # we are executing at least 1 iteration, otherwise we should leave the
  7862. # target unaffected. The target iteration variable is firstprivate to
  7863. # shut up compiler warnings caused by lastprivate, as the compiler
  7864. # erroneously believes that nsteps may be <= 0, leaving the private
  7865. # target index uninitialized
  7866. code.putln("if (%(nsteps)s > 0)" % fmt_dict)
  7867. code.begin_block() # if block
  7868. self.generate_loop(code, fmt_dict)
  7869. code.end_block() # end if block
  7870. self.restore_labels(code)
  7871. if self.else_clause:
  7872. if self.breaking_label_used:
  7873. code.put("if (%s < 2)" % Naming.parallel_why)
  7874. code.begin_block() # else block
  7875. code.putln("/* else */")
  7876. self.else_clause.generate_execution_code(code)
  7877. code.end_block() # end else block
  7878. # ------ cleanup ------
  7879. self.end_parallel_control_flow_block(code) # end parallel control flow block
  7880. # And finally, release our privates and write back any closure
  7881. # variables
  7882. for temp in start_stop_step + (self.chunksize,):
  7883. if temp is not None:
  7884. temp.generate_disposal_code(code)
  7885. temp.free_temps(code)
  7886. code.funcstate.release_temp(fmt_dict['i'])
  7887. code.funcstate.release_temp(fmt_dict['nsteps'])
  7888. self.release_closure_privates(code)
  7889. def generate_loop(self, code, fmt_dict):
  7890. if self.is_nested_prange:
  7891. code.putln("#if 0")
  7892. else:
  7893. code.putln("#ifdef _OPENMP")
  7894. if not self.is_parallel:
  7895. code.put("#pragma omp for")
  7896. self.privatization_insertion_point = code.insertion_point()
  7897. reduction_codepoint = self.parent.privatization_insertion_point
  7898. else:
  7899. code.put("#pragma omp parallel")
  7900. self.privatization_insertion_point = code.insertion_point()
  7901. reduction_codepoint = self.privatization_insertion_point
  7902. code.putln("")
  7903. code.putln("#endif /* _OPENMP */")
  7904. code.begin_block() # pragma omp parallel begin block
  7905. # Initialize the GIL if needed for this thread
  7906. self.begin_parallel_block(code)
  7907. if self.is_nested_prange:
  7908. code.putln("#if 0")
  7909. else:
  7910. code.putln("#ifdef _OPENMP")
  7911. code.put("#pragma omp for")
  7912. for entry, (op, lastprivate) in sorted(self.privates.items()):
  7913. # Don't declare the index variable as a reduction
  7914. if op and op in "+*-&^|" and entry != self.target.entry:
  7915. if entry.type.is_pyobject:
  7916. error(self.pos, "Python objects cannot be reductions")
  7917. else:
  7918. #code.put(" reduction(%s:%s)" % (op, entry.cname))
  7919. # This is the only way reductions + nesting works in gcc4.5
  7920. reduction_codepoint.put(
  7921. " reduction(%s:%s)" % (op, entry.cname))
  7922. else:
  7923. if entry == self.target.entry:
  7924. code.put(" firstprivate(%s)" % entry.cname)
  7925. code.put(" lastprivate(%s)" % entry.cname)
  7926. continue
  7927. if not entry.type.is_pyobject:
  7928. if lastprivate:
  7929. private = 'lastprivate'
  7930. else:
  7931. private = 'private'
  7932. code.put(" %s(%s)" % (private, entry.cname))
  7933. if self.schedule:
  7934. if self.chunksize:
  7935. chunksize = ", %s" % self.evaluate_before_block(code, self.chunksize)
  7936. else:
  7937. chunksize = ""
  7938. code.put(" schedule(%s%s)" % (self.schedule, chunksize))
  7939. self.put_num_threads(reduction_codepoint)
  7940. code.putln("")
  7941. code.putln("#endif /* _OPENMP */")
  7942. code.put("for (%(i)s = 0; %(i)s < %(nsteps)s; %(i)s++)" % fmt_dict)
  7943. code.begin_block() # for loop block
  7944. guard_around_body_codepoint = code.insertion_point()
  7945. # Start if guard block around the body. This may be unnecessary, but
  7946. # at least it doesn't spoil indentation
  7947. code.begin_block()
  7948. code.putln("%(target)s = (%(target_type)s)(%(start)s + %(step)s * %(i)s);" % fmt_dict)
  7949. self.initialize_privates_to_nan(code, exclude=self.target.entry)
  7950. if self.is_parallel and not self.is_nested_prange:
  7951. # nested pranges are not omp'ified, temps go to outer loops
  7952. code.funcstate.start_collecting_temps()
  7953. self.body.generate_execution_code(code)
  7954. self.trap_parallel_exit(code, should_flush=True)
  7955. if self.is_parallel and not self.is_nested_prange:
  7956. # nested pranges are not omp'ified, temps go to outer loops
  7957. self.privatize_temps(code)
  7958. if self.breaking_label_used:
  7959. # Put a guard around the loop body in case return, break or
  7960. # exceptions might be used
  7961. guard_around_body_codepoint.putln("if (%s < 2)" % Naming.parallel_why)
  7962. code.end_block() # end guard around loop body
  7963. code.end_block() # end for loop block
  7964. if self.is_parallel:
  7965. # Release the GIL and deallocate the thread state
  7966. self.end_parallel_block(code)
  7967. code.end_block() # pragma omp parallel end block
  7968. class CnameDecoratorNode(StatNode):
  7969. """
  7970. This node is for the cname decorator in CythonUtilityCode:
  7971. @cname('the_cname')
  7972. cdef func(...):
  7973. ...
  7974. In case of a cdef class the cname specifies the objstruct_cname.
  7975. node the node to which the cname decorator is applied
  7976. cname the cname the node should get
  7977. """
  7978. child_attrs = ['node']
  7979. def analyse_declarations(self, env):
  7980. self.node.analyse_declarations(env)
  7981. node = self.node
  7982. if isinstance(node, CompilerDirectivesNode):
  7983. node = node.body.stats[0]
  7984. self.is_function = isinstance(node, FuncDefNode)
  7985. is_struct_or_enum = isinstance(node, (CStructOrUnionDefNode, CEnumDefNode))
  7986. e = node.entry
  7987. if self.is_function:
  7988. e.cname = self.cname
  7989. e.func_cname = self.cname
  7990. e.used = True
  7991. if e.pyfunc_cname and '.' in e.pyfunc_cname:
  7992. e.pyfunc_cname = self.mangle(e.pyfunc_cname)
  7993. elif is_struct_or_enum:
  7994. e.cname = e.type.cname = self.cname
  7995. else:
  7996. scope = node.scope
  7997. e.cname = self.cname
  7998. e.type.objstruct_cname = self.cname + '_obj'
  7999. e.type.typeobj_cname = Naming.typeobj_prefix + self.cname
  8000. e.type.typeptr_cname = self.cname + '_type'
  8001. e.type.scope.namespace_cname = e.type.typeptr_cname
  8002. e.as_variable.cname = e.type.typeptr_cname
  8003. scope.scope_prefix = self.cname + "_"
  8004. for name, entry in scope.entries.items():
  8005. if entry.func_cname:
  8006. entry.func_cname = self.mangle(entry.cname)
  8007. if entry.pyfunc_cname:
  8008. entry.pyfunc_cname = self.mangle(entry.pyfunc_cname)
  8009. def mangle(self, cname):
  8010. if '.' in cname:
  8011. # remove __pyx_base from func_cname
  8012. cname = cname.split('.')[-1]
  8013. return '%s_%s' % (self.cname, cname)
  8014. def analyse_expressions(self, env):
  8015. self.node = self.node.analyse_expressions(env)
  8016. return self
  8017. def generate_function_definitions(self, env, code):
  8018. "Ensure a prototype for every @cname method in the right place"
  8019. if self.is_function and env.is_c_class_scope:
  8020. # method in cdef class, generate a prototype in the header
  8021. h_code = code.globalstate['utility_code_proto']
  8022. if isinstance(self.node, DefNode):
  8023. self.node.generate_function_header(
  8024. h_code, with_pymethdef=False, proto_only=True)
  8025. else:
  8026. from . import ModuleNode
  8027. entry = self.node.entry
  8028. cname = entry.cname
  8029. entry.cname = entry.func_cname
  8030. ModuleNode.generate_cfunction_declaration(
  8031. entry,
  8032. env.global_scope(),
  8033. h_code,
  8034. definition=True)
  8035. entry.cname = cname
  8036. self.node.generate_function_definitions(env, code)
  8037. def generate_execution_code(self, code):
  8038. self.node.generate_execution_code(code)
  8039. #------------------------------------------------------------------------------------
  8040. #
  8041. # Runtime support code
  8042. #
  8043. #------------------------------------------------------------------------------------
  8044. if Options.gcc_branch_hints:
  8045. branch_prediction_macros = """
  8046. /* Test for GCC > 2.95 */
  8047. #if defined(__GNUC__) \
  8048. && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
  8049. #define likely(x) __builtin_expect(!!(x), 1)
  8050. #define unlikely(x) __builtin_expect(!!(x), 0)
  8051. #else /* !__GNUC__ or GCC < 2.95 */
  8052. #define likely(x) (x)
  8053. #define unlikely(x) (x)
  8054. #endif /* __GNUC__ */
  8055. """
  8056. else:
  8057. branch_prediction_macros = """
  8058. #define likely(x) (x)
  8059. #define unlikely(x) (x)
  8060. """
  8061. #------------------------------------------------------------------------------------
  8062. printing_utility_code = UtilityCode.load_cached("Print", "Printing.c")
  8063. printing_one_utility_code = UtilityCode.load_cached("PrintOne", "Printing.c")
  8064. #------------------------------------------------------------------------------------
  8065. # Exception raising code
  8066. #
  8067. # Exceptions are raised by __Pyx_Raise() and stored as plain
  8068. # type/value/tb in PyThreadState->curexc_*. When being caught by an
  8069. # 'except' statement, curexc_* is moved over to exc_* by
  8070. # __Pyx_GetException()
  8071. restore_exception_utility_code = UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")
  8072. raise_utility_code = UtilityCode.load_cached("RaiseException", "Exceptions.c")
  8073. get_exception_utility_code = UtilityCode.load_cached("GetException", "Exceptions.c")
  8074. swap_exception_utility_code = UtilityCode.load_cached("SwapException", "Exceptions.c")
  8075. reset_exception_utility_code = UtilityCode.load_cached("SaveResetException", "Exceptions.c")
  8076. traceback_utility_code = UtilityCode.load_cached("AddTraceback", "Exceptions.c")
  8077. #------------------------------------------------------------------------------------
  8078. get_exception_tuple_utility_code = UtilityCode(
  8079. proto="""
  8080. static PyObject *__Pyx_GetExceptionTuple(PyThreadState *__pyx_tstate); /*proto*/
  8081. """,
  8082. # I doubt that calling __Pyx_GetException() here is correct as it moves
  8083. # the exception from tstate->curexc_* to tstate->exc_*, which prevents
  8084. # exception handlers later on from receiving it.
  8085. # NOTE: "__pyx_tstate" may be used by __Pyx_GetException() macro
  8086. impl = """
  8087. static PyObject *__Pyx_GetExceptionTuple(CYTHON_UNUSED PyThreadState *__pyx_tstate) {
  8088. PyObject *type = NULL, *value = NULL, *tb = NULL;
  8089. if (__Pyx_GetException(&type, &value, &tb) == 0) {
  8090. PyObject* exc_info = PyTuple_New(3);
  8091. if (exc_info) {
  8092. Py_INCREF(type);
  8093. Py_INCREF(value);
  8094. Py_INCREF(tb);
  8095. PyTuple_SET_ITEM(exc_info, 0, type);
  8096. PyTuple_SET_ITEM(exc_info, 1, value);
  8097. PyTuple_SET_ITEM(exc_info, 2, tb);
  8098. return exc_info;
  8099. }
  8100. }
  8101. return NULL;
  8102. }
  8103. """,
  8104. requires=[get_exception_utility_code])