1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519 |
- //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
- //
- // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
- // See https://llvm.org/LICENSE.txt for license information.
- // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
- //
- //===----------------------------------------------------------------------===//
- //
- // This contains code to emit OpenMP nodes as LLVM code.
- //
- //===----------------------------------------------------------------------===//
- #include "CGCleanup.h"
- #include "CGOpenMPRuntime.h"
- #include "CodeGenFunction.h"
- #include "CodeGenModule.h"
- #include "TargetInfo.h"
- #include "clang/AST/ASTContext.h"
- #include "clang/AST/Attr.h"
- #include "clang/AST/DeclOpenMP.h"
- #include "clang/AST/OpenMPClause.h"
- #include "clang/AST/Stmt.h"
- #include "clang/AST/StmtOpenMP.h"
- #include "clang/AST/StmtVisitor.h"
- #include "clang/Basic/OpenMPKinds.h"
- #include "clang/Basic/PrettyStackTrace.h"
- #include "llvm/BinaryFormat/Dwarf.h"
- #include "llvm/Frontend/OpenMP/OMPConstants.h"
- #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
- #include "llvm/IR/Constants.h"
- #include "llvm/IR/DebugInfoMetadata.h"
- #include "llvm/IR/Instructions.h"
- #include "llvm/IR/IntrinsicInst.h"
- #include "llvm/IR/Metadata.h"
- #include "llvm/Support/AtomicOrdering.h"
- using namespace clang;
- using namespace CodeGen;
- using namespace llvm::omp;
- static const VarDecl *getBaseDecl(const Expr *Ref);
- namespace {
- /// Lexical scope for OpenMP executable constructs, that handles correct codegen
- /// for captured expressions.
- class OMPLexicalScope : public CodeGenFunction::LexicalScope {
- void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
- for (const auto *C : S.clauses()) {
- if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
- if (const auto *PreInit =
- cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
- for (const auto *I : PreInit->decls()) {
- if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
- CGF.EmitVarDecl(cast<VarDecl>(*I));
- } else {
- CodeGenFunction::AutoVarEmission Emission =
- CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
- CGF.EmitAutoVarCleanups(Emission);
- }
- }
- }
- }
- }
- }
- CodeGenFunction::OMPPrivateScope InlinedShareds;
- static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
- return CGF.LambdaCaptureFields.lookup(VD) ||
- (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
- (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
- cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
- }
- public:
- OMPLexicalScope(
- CodeGenFunction &CGF, const OMPExecutableDirective &S,
- const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
- const bool EmitPreInitStmt = true)
- : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
- InlinedShareds(CGF) {
- if (EmitPreInitStmt)
- emitPreInitStmt(CGF, S);
- if (!CapturedRegion.hasValue())
- return;
- assert(S.hasAssociatedStmt() &&
- "Expected associated statement for inlined directive.");
- const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
- for (const auto &C : CS->captures()) {
- if (C.capturesVariable() || C.capturesVariableByCopy()) {
- auto *VD = C.getCapturedVar();
- assert(VD == VD->getCanonicalDecl() &&
- "Canonical decl must be captured.");
- DeclRefExpr DRE(
- CGF.getContext(), const_cast<VarDecl *>(VD),
- isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
- InlinedShareds.isGlobalVarCaptured(VD)),
- VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
- InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
- return CGF.EmitLValue(&DRE).getAddress(CGF);
- });
- }
- }
- (void)InlinedShareds.Privatize();
- }
- };
- /// Lexical scope for OpenMP parallel construct, that handles correct codegen
- /// for captured expressions.
- class OMPParallelScope final : public OMPLexicalScope {
- bool EmitPreInitStmt(const OMPExecutableDirective &S) {
- OpenMPDirectiveKind Kind = S.getDirectiveKind();
- return !(isOpenMPTargetExecutionDirective(Kind) ||
- isOpenMPLoopBoundSharingDirective(Kind)) &&
- isOpenMPParallelDirective(Kind);
- }
- public:
- OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
- : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
- EmitPreInitStmt(S)) {}
- };
- /// Lexical scope for OpenMP teams construct, that handles correct codegen
- /// for captured expressions.
- class OMPTeamsScope final : public OMPLexicalScope {
- bool EmitPreInitStmt(const OMPExecutableDirective &S) {
- OpenMPDirectiveKind Kind = S.getDirectiveKind();
- return !isOpenMPTargetExecutionDirective(Kind) &&
- isOpenMPTeamsDirective(Kind);
- }
- public:
- OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
- : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
- EmitPreInitStmt(S)) {}
- };
- /// Private scope for OpenMP loop-based directives, that supports capturing
- /// of used expression from loop statement.
- class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
- void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopBasedDirective &S) {
- const DeclStmt *PreInits;
- CodeGenFunction::OMPMapVars PreCondVars;
- if (auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
- llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
- for (const auto *E : LD->counters()) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- EmittedAsPrivate.insert(VD->getCanonicalDecl());
- (void)PreCondVars.setVarAddr(
- CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
- }
- // Mark private vars as undefs.
- for (const auto *C : LD->getClausesOfKind<OMPPrivateClause>()) {
- for (const Expr *IRef : C->varlists()) {
- const auto *OrigVD =
- cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
- if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
- (void)PreCondVars.setVarAddr(
- CGF, OrigVD,
- Address(llvm::UndefValue::get(CGF.ConvertTypeForMem(
- CGF.getContext().getPointerType(
- OrigVD->getType().getNonReferenceType()))),
- CGF.getContext().getDeclAlign(OrigVD)));
- }
- }
- }
- (void)PreCondVars.apply(CGF);
- // Emit init, __range and __end variables for C++ range loops.
- (void)OMPLoopBasedDirective::doForAllLoops(
- LD->getInnermostCapturedStmt()->getCapturedStmt(),
- /*TryImperfectlyNestedLoops=*/true, LD->getLoopsNumber(),
- [&CGF](unsigned Cnt, const Stmt *CurStmt) {
- if (const auto *CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {
- if (const Stmt *Init = CXXFor->getInit())
- CGF.EmitStmt(Init);
- CGF.EmitStmt(CXXFor->getRangeStmt());
- CGF.EmitStmt(CXXFor->getEndStmt());
- }
- return false;
- });
- PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
- } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
- PreInits = cast_or_null<DeclStmt>(Tile->getPreInits());
- } else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
- PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
- } else {
- llvm_unreachable("Unknown loop-based directive kind.");
- }
- if (PreInits) {
- for (const auto *I : PreInits->decls())
- CGF.EmitVarDecl(cast<VarDecl>(*I));
- }
- PreCondVars.restore(CGF);
- }
- public:
- OMPLoopScope(CodeGenFunction &CGF, const OMPLoopBasedDirective &S)
- : CodeGenFunction::RunCleanupsScope(CGF) {
- emitPreInitStmt(CGF, S);
- }
- };
- class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
- CodeGenFunction::OMPPrivateScope InlinedShareds;
- static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
- return CGF.LambdaCaptureFields.lookup(VD) ||
- (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
- (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
- cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
- }
- public:
- OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
- : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
- InlinedShareds(CGF) {
- for (const auto *C : S.clauses()) {
- if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
- if (const auto *PreInit =
- cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
- for (const auto *I : PreInit->decls()) {
- if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
- CGF.EmitVarDecl(cast<VarDecl>(*I));
- } else {
- CodeGenFunction::AutoVarEmission Emission =
- CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
- CGF.EmitAutoVarCleanups(Emission);
- }
- }
- }
- } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
- for (const Expr *E : UDP->varlists()) {
- const Decl *D = cast<DeclRefExpr>(E)->getDecl();
- if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
- CGF.EmitVarDecl(*OED);
- }
- } else if (const auto *UDP = dyn_cast<OMPUseDeviceAddrClause>(C)) {
- for (const Expr *E : UDP->varlists()) {
- const Decl *D = getBaseDecl(E);
- if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
- CGF.EmitVarDecl(*OED);
- }
- }
- }
- if (!isOpenMPSimdDirective(S.getDirectiveKind()))
- CGF.EmitOMPPrivateClause(S, InlinedShareds);
- if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
- if (const Expr *E = TG->getReductionRef())
- CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
- }
- // Temp copy arrays for inscan reductions should not be emitted as they are
- // not used in simd only mode.
- llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;
- for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
- if (C->getModifier() != OMPC_REDUCTION_inscan)
- continue;
- for (const Expr *E : C->copy_array_temps())
- CopyArrayTemps.insert(cast<DeclRefExpr>(E)->getDecl());
- }
- const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
- while (CS) {
- for (auto &C : CS->captures()) {
- if (C.capturesVariable() || C.capturesVariableByCopy()) {
- auto *VD = C.getCapturedVar();
- if (CopyArrayTemps.contains(VD))
- continue;
- assert(VD == VD->getCanonicalDecl() &&
- "Canonical decl must be captured.");
- DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
- isCapturedVar(CGF, VD) ||
- (CGF.CapturedStmtInfo &&
- InlinedShareds.isGlobalVarCaptured(VD)),
- VD->getType().getNonReferenceType(), VK_LValue,
- C.getLocation());
- InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
- return CGF.EmitLValue(&DRE).getAddress(CGF);
- });
- }
- }
- CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
- }
- (void)InlinedShareds.Privatize();
- }
- };
- } // namespace
- static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
- const OMPExecutableDirective &S,
- const RegionCodeGenTy &CodeGen);
- LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
- if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
- if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
- OrigVD = OrigVD->getCanonicalDecl();
- bool IsCaptured =
- LambdaCaptureFields.lookup(OrigVD) ||
- (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
- (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
- OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
- return EmitLValue(&DRE);
- }
- }
- return EmitLValue(E);
- }
- llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
- ASTContext &C = getContext();
- llvm::Value *Size = nullptr;
- auto SizeInChars = C.getTypeSizeInChars(Ty);
- if (SizeInChars.isZero()) {
- // getTypeSizeInChars() returns 0 for a VLA.
- while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
- VlaSizePair VlaSize = getVLASize(VAT);
- Ty = VlaSize.Type;
- Size =
- Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;
- }
- SizeInChars = C.getTypeSizeInChars(Ty);
- if (SizeInChars.isZero())
- return llvm::ConstantInt::get(SizeTy, /*V=*/0);
- return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
- }
- return CGM.getSize(SizeInChars);
- }
- void CodeGenFunction::GenerateOpenMPCapturedVars(
- const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
- const RecordDecl *RD = S.getCapturedRecordDecl();
- auto CurField = RD->field_begin();
- auto CurCap = S.captures().begin();
- for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
- E = S.capture_init_end();
- I != E; ++I, ++CurField, ++CurCap) {
- if (CurField->hasCapturedVLAType()) {
- const VariableArrayType *VAT = CurField->getCapturedVLAType();
- llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
- CapturedVars.push_back(Val);
- } else if (CurCap->capturesThis()) {
- CapturedVars.push_back(CXXThisValue);
- } else if (CurCap->capturesVariableByCopy()) {
- llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
- // If the field is not a pointer, we need to save the actual value
- // and load it as a void pointer.
- if (!CurField->getType()->isAnyPointerType()) {
- ASTContext &Ctx = getContext();
- Address DstAddr = CreateMemTemp(
- Ctx.getUIntPtrType(),
- Twine(CurCap->getCapturedVar()->getName(), ".casted"));
- LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
- llvm::Value *SrcAddrVal = EmitScalarConversion(
- DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
- Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
- LValue SrcLV =
- MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
- // Store the value using the source type pointer.
- EmitStoreThroughLValue(RValue::get(CV), SrcLV);
- // Load the value using the destination type pointer.
- CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
- }
- CapturedVars.push_back(CV);
- } else {
- assert(CurCap->capturesVariable() && "Expected capture by reference.");
- CapturedVars.push_back(EmitLValue(*I).getAddress(*this).getPointer());
- }
- }
- }
- static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
- QualType DstType, StringRef Name,
- LValue AddrLV) {
- ASTContext &Ctx = CGF.getContext();
- llvm::Value *CastedPtr = CGF.EmitScalarConversion(
- AddrLV.getAddress(CGF).getPointer(), Ctx.getUIntPtrType(),
- Ctx.getPointerType(DstType), Loc);
- Address TmpAddr =
- CGF.MakeNaturalAlignAddrLValue(CastedPtr, DstType).getAddress(CGF);
- return TmpAddr;
- }
- static QualType getCanonicalParamType(ASTContext &C, QualType T) {
- if (T->isLValueReferenceType())
- return C.getLValueReferenceType(
- getCanonicalParamType(C, T.getNonReferenceType()),
- /*SpelledAsLValue=*/false);
- if (T->isPointerType())
- return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
- if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
- if (const auto *VLA = dyn_cast<VariableArrayType>(A))
- return getCanonicalParamType(C, VLA->getElementType());
- if (!A->isVariablyModifiedType())
- return C.getCanonicalType(T);
- }
- return C.getCanonicalParamType(T);
- }
- namespace {
- /// Contains required data for proper outlined function codegen.
- struct FunctionOptions {
- /// Captured statement for which the function is generated.
- const CapturedStmt *S = nullptr;
- /// true if cast to/from UIntPtr is required for variables captured by
- /// value.
- const bool UIntPtrCastRequired = true;
- /// true if only casted arguments must be registered as local args or VLA
- /// sizes.
- const bool RegisterCastedArgsOnly = false;
- /// Name of the generated function.
- const StringRef FunctionName;
- /// Location of the non-debug version of the outlined function.
- SourceLocation Loc;
- explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
- bool RegisterCastedArgsOnly, StringRef FunctionName,
- SourceLocation Loc)
- : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
- RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
- FunctionName(FunctionName), Loc(Loc) {}
- };
- } // namespace
- static llvm::Function *emitOutlinedFunctionPrologue(
- CodeGenFunction &CGF, FunctionArgList &Args,
- llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
- &LocalAddrs,
- llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
- &VLASizes,
- llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
- const CapturedDecl *CD = FO.S->getCapturedDecl();
- const RecordDecl *RD = FO.S->getCapturedRecordDecl();
- assert(CD->hasBody() && "missing CapturedDecl body");
- CXXThisValue = nullptr;
- // Build the argument list.
- CodeGenModule &CGM = CGF.CGM;
- ASTContext &Ctx = CGM.getContext();
- FunctionArgList TargetArgs;
- Args.append(CD->param_begin(),
- std::next(CD->param_begin(), CD->getContextParamPosition()));
- TargetArgs.append(
- CD->param_begin(),
- std::next(CD->param_begin(), CD->getContextParamPosition()));
- auto I = FO.S->captures().begin();
- FunctionDecl *DebugFunctionDecl = nullptr;
- if (!FO.UIntPtrCastRequired) {
- FunctionProtoType::ExtProtoInfo EPI;
- QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
- DebugFunctionDecl = FunctionDecl::Create(
- Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
- SourceLocation(), DeclarationName(), FunctionTy,
- Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
- /*UsesFPIntrin=*/false, /*isInlineSpecified=*/false,
- /*hasWrittenPrototype=*/false);
- }
- for (const FieldDecl *FD : RD->fields()) {
- QualType ArgType = FD->getType();
- IdentifierInfo *II = nullptr;
- VarDecl *CapVar = nullptr;
- // If this is a capture by copy and the type is not a pointer, the outlined
- // function argument type should be uintptr and the value properly casted to
- // uintptr. This is necessary given that the runtime library is only able to
- // deal with pointers. We can pass in the same way the VLA type sizes to the
- // outlined function.
- if (FO.UIntPtrCastRequired &&
- ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
- I->capturesVariableArrayType()))
- ArgType = Ctx.getUIntPtrType();
- if (I->capturesVariable() || I->capturesVariableByCopy()) {
- CapVar = I->getCapturedVar();
- II = CapVar->getIdentifier();
- } else if (I->capturesThis()) {
- II = &Ctx.Idents.get("this");
- } else {
- assert(I->capturesVariableArrayType());
- II = &Ctx.Idents.get("vla");
- }
- if (ArgType->isVariablyModifiedType())
- ArgType = getCanonicalParamType(Ctx, ArgType);
- VarDecl *Arg;
- if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
- Arg = ParmVarDecl::Create(
- Ctx, DebugFunctionDecl,
- CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
- CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
- /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
- } else {
- Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
- II, ArgType, ImplicitParamDecl::Other);
- }
- Args.emplace_back(Arg);
- // Do not cast arguments if we emit function with non-original types.
- TargetArgs.emplace_back(
- FO.UIntPtrCastRequired
- ? Arg
- : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
- ++I;
- }
- Args.append(std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
- CD->param_end());
- TargetArgs.append(
- std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
- CD->param_end());
- // Create the function declaration.
- const CGFunctionInfo &FuncInfo =
- CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
- llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
- auto *F =
- llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
- FO.FunctionName, &CGM.getModule());
- CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
- if (CD->isNothrow())
- F->setDoesNotThrow();
- F->setDoesNotRecurse();
- // Always inline the outlined function if optimizations are enabled.
- if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
- F->removeFnAttr(llvm::Attribute::NoInline);
- F->addFnAttr(llvm::Attribute::AlwaysInline);
- }
- // Generate the function.
- CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
- FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),
- FO.UIntPtrCastRequired ? FO.Loc
- : CD->getBody()->getBeginLoc());
- unsigned Cnt = CD->getContextParamPosition();
- I = FO.S->captures().begin();
- for (const FieldDecl *FD : RD->fields()) {
- // Do not map arguments if we emit function with non-original types.
- Address LocalAddr(Address::invalid());
- if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
- LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
- TargetArgs[Cnt]);
- } else {
- LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
- }
- // If we are capturing a pointer by copy we don't need to do anything, just
- // use the value that we get from the arguments.
- if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
- const VarDecl *CurVD = I->getCapturedVar();
- if (!FO.RegisterCastedArgsOnly)
- LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
- ++Cnt;
- ++I;
- continue;
- }
- LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
- AlignmentSource::Decl);
- if (FD->hasCapturedVLAType()) {
- if (FO.UIntPtrCastRequired) {
- ArgLVal = CGF.MakeAddrLValue(
- castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
- Args[Cnt]->getName(), ArgLVal),
- FD->getType(), AlignmentSource::Decl);
- }
- llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
- const VariableArrayType *VAT = FD->getCapturedVLAType();
- VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
- } else if (I->capturesVariable()) {
- const VarDecl *Var = I->getCapturedVar();
- QualType VarTy = Var->getType();
- Address ArgAddr = ArgLVal.getAddress(CGF);
- if (ArgLVal.getType()->isLValueReferenceType()) {
- ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
- } else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
- assert(ArgLVal.getType()->isPointerType());
- ArgAddr = CGF.EmitLoadOfPointer(
- ArgAddr, ArgLVal.getType()->castAs<PointerType>());
- }
- if (!FO.RegisterCastedArgsOnly) {
- LocalAddrs.insert(
- {Args[Cnt],
- {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
- }
- } else if (I->capturesVariableByCopy()) {
- assert(!FD->getType()->isAnyPointerType() &&
- "Not expecting a captured pointer.");
- const VarDecl *Var = I->getCapturedVar();
- LocalAddrs.insert({Args[Cnt],
- {Var, FO.UIntPtrCastRequired
- ? castValueFromUintptr(
- CGF, I->getLocation(), FD->getType(),
- Args[Cnt]->getName(), ArgLVal)
- : ArgLVal.getAddress(CGF)}});
- } else {
- // If 'this' is captured, load it into CXXThisValue.
- assert(I->capturesThis());
- CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
- LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress(CGF)}});
- }
- ++Cnt;
- ++I;
- }
- return F;
- }
- llvm::Function *
- CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
- SourceLocation Loc) {
- assert(
- CapturedStmtInfo &&
- "CapturedStmtInfo should be set when generating the captured function");
- const CapturedDecl *CD = S.getCapturedDecl();
- // Build the argument list.
- bool NeedWrapperFunction =
- getDebugInfo() && CGM.getCodeGenOpts().hasReducedDebugInfo();
- FunctionArgList Args;
- llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
- llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
- SmallString<256> Buffer;
- llvm::raw_svector_ostream Out(Buffer);
- Out << CapturedStmtInfo->getHelperName();
- if (NeedWrapperFunction)
- Out << "_debug__";
- FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
- Out.str(), Loc);
- llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
- VLASizes, CXXThisValue, FO);
- CodeGenFunction::OMPPrivateScope LocalScope(*this);
- for (const auto &LocalAddrPair : LocalAddrs) {
- if (LocalAddrPair.second.first) {
- LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
- return LocalAddrPair.second.second;
- });
- }
- }
- (void)LocalScope.Privatize();
- for (const auto &VLASizePair : VLASizes)
- VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
- PGO.assignRegionCounters(GlobalDecl(CD), F);
- CapturedStmtInfo->EmitBody(*this, CD->getBody());
- (void)LocalScope.ForceCleanup();
- FinishFunction(CD->getBodyRBrace());
- if (!NeedWrapperFunction)
- return F;
- FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
- /*RegisterCastedArgsOnly=*/true,
- CapturedStmtInfo->getHelperName(), Loc);
- CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
- WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
- Args.clear();
- LocalAddrs.clear();
- VLASizes.clear();
- llvm::Function *WrapperF =
- emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
- WrapperCGF.CXXThisValue, WrapperFO);
- llvm::SmallVector<llvm::Value *, 4> CallArgs;
- auto *PI = F->arg_begin();
- for (const auto *Arg : Args) {
- llvm::Value *CallArg;
- auto I = LocalAddrs.find(Arg);
- if (I != LocalAddrs.end()) {
- LValue LV = WrapperCGF.MakeAddrLValue(
- I->second.second,
- I->second.first ? I->second.first->getType() : Arg->getType(),
- AlignmentSource::Decl);
- if (LV.getType()->isAnyComplexType())
- LV.setAddress(WrapperCGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- LV.getAddress(WrapperCGF),
- PI->getType()->getPointerTo(
- LV.getAddress(WrapperCGF).getAddressSpace())));
- CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
- } else {
- auto EI = VLASizes.find(Arg);
- if (EI != VLASizes.end()) {
- CallArg = EI->second.second;
- } else {
- LValue LV =
- WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
- Arg->getType(), AlignmentSource::Decl);
- CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
- }
- }
- CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
- ++PI;
- }
- CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, Loc, F, CallArgs);
- WrapperCGF.FinishFunction();
- return WrapperF;
- }
- //===----------------------------------------------------------------------===//
- // OpenMP Directive Emission
- //===----------------------------------------------------------------------===//
- void CodeGenFunction::EmitOMPAggregateAssign(
- Address DestAddr, Address SrcAddr, QualType OriginalType,
- const llvm::function_ref<void(Address, Address)> CopyGen) {
- // Perform element-by-element initialization.
- QualType ElementTy;
- // Drill down to the base element type on both arrays.
- const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
- llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
- SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
- llvm::Value *SrcBegin = SrcAddr.getPointer();
- llvm::Value *DestBegin = DestAddr.getPointer();
- // Cast from pointer to array type to pointer to single element.
- llvm::Value *DestEnd =
- Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements);
- // The basic structure here is a while-do loop.
- llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
- llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
- llvm::Value *IsEmpty =
- Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
- Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
- // Enter the loop body, making that address the current address.
- llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
- EmitBlock(BodyBB);
- CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
- llvm::PHINode *SrcElementPHI =
- Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
- SrcElementPHI->addIncoming(SrcBegin, EntryBB);
- Address SrcElementCurrent =
- Address(SrcElementPHI,
- SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
- llvm::PHINode *DestElementPHI = Builder.CreatePHI(
- DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
- DestElementPHI->addIncoming(DestBegin, EntryBB);
- Address DestElementCurrent =
- Address(DestElementPHI,
- DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
- // Emit copy.
- CopyGen(DestElementCurrent, SrcElementCurrent);
- // Shift the address forward by one element.
- llvm::Value *DestElementNext =
- Builder.CreateConstGEP1_32(DestAddr.getElementType(), DestElementPHI,
- /*Idx0=*/1, "omp.arraycpy.dest.element");
- llvm::Value *SrcElementNext =
- Builder.CreateConstGEP1_32(SrcAddr.getElementType(), SrcElementPHI,
- /*Idx0=*/1, "omp.arraycpy.src.element");
- // Check whether we've reached the end.
- llvm::Value *Done =
- Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
- Builder.CreateCondBr(Done, DoneBB, BodyBB);
- DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
- SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
- // Done.
- EmitBlock(DoneBB, /*IsFinished=*/true);
- }
- void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
- Address SrcAddr, const VarDecl *DestVD,
- const VarDecl *SrcVD, const Expr *Copy) {
- if (OriginalType->isArrayType()) {
- const auto *BO = dyn_cast<BinaryOperator>(Copy);
- if (BO && BO->getOpcode() == BO_Assign) {
- // Perform simple memcpy for simple copying.
- LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
- LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
- EmitAggregateAssign(Dest, Src, OriginalType);
- } else {
- // For arrays with complex element types perform element by element
- // copying.
- EmitOMPAggregateAssign(
- DestAddr, SrcAddr, OriginalType,
- [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
- // Working with the single array element, so have to remap
- // destination and source variables to corresponding array
- // elements.
- CodeGenFunction::OMPPrivateScope Remap(*this);
- Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
- Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
- (void)Remap.Privatize();
- EmitIgnoredExpr(Copy);
- });
- }
- } else {
- // Remap pseudo source variable to private copy.
- CodeGenFunction::OMPPrivateScope Remap(*this);
- Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
- Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
- (void)Remap.Privatize();
- // Emit copying of the whole variable.
- EmitIgnoredExpr(Copy);
- }
- }
- bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
- OMPPrivateScope &PrivateScope) {
- if (!HaveInsertPoint())
- return false;
- bool DeviceConstTarget =
- getLangOpts().OpenMPIsDevice &&
- isOpenMPTargetExecutionDirective(D.getDirectiveKind());
- bool FirstprivateIsLastprivate = false;
- llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;
- for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
- for (const auto *D : C->varlists())
- Lastprivates.try_emplace(
- cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl(),
- C->getKind());
- }
- llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
- llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
- getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
- // Force emission of the firstprivate copy if the directive does not emit
- // outlined function, like omp for, omp simd, omp distribute etc.
- bool MustEmitFirstprivateCopy =
- CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
- for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
- const auto *IRef = C->varlist_begin();
- const auto *InitsRef = C->inits().begin();
- for (const Expr *IInit : C->private_copies()) {
- const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
- bool ThisFirstprivateIsLastprivate =
- Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
- const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
- if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
- !FD->getType()->isReferenceType() &&
- (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
- EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
- ++IRef;
- ++InitsRef;
- continue;
- }
- // Do not emit copy for firstprivate constant variables in target regions,
- // captured by reference.
- if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
- FD && FD->getType()->isReferenceType() &&
- (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
- EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
- ++IRef;
- ++InitsRef;
- continue;
- }
- FirstprivateIsLastprivate =
- FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
- if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
- const auto *VDInit =
- cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
- bool IsRegistered;
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
- /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
- (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
- LValue OriginalLVal;
- if (!FD) {
- // Check if the firstprivate variable is just a constant value.
- ConstantEmission CE = tryEmitAsConstant(&DRE);
- if (CE && !CE.isReference()) {
- // Constant value, no need to create a copy.
- ++IRef;
- ++InitsRef;
- continue;
- }
- if (CE && CE.isReference()) {
- OriginalLVal = CE.getReferenceLValue(*this, &DRE);
- } else {
- assert(!CE && "Expected non-constant firstprivate.");
- OriginalLVal = EmitLValue(&DRE);
- }
- } else {
- OriginalLVal = EmitLValue(&DRE);
- }
- QualType Type = VD->getType();
- if (Type->isArrayType()) {
- // Emit VarDecl with copy init for arrays.
- // Get the address of the original variable captured in current
- // captured region.
- IsRegistered = PrivateScope.addPrivate(
- OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
- AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
- const Expr *Init = VD->getInit();
- if (!isa<CXXConstructExpr>(Init) ||
- isTrivialInitializer(Init)) {
- // Perform simple memcpy.
- LValue Dest =
- MakeAddrLValue(Emission.getAllocatedAddress(), Type);
- EmitAggregateAssign(Dest, OriginalLVal, Type);
- } else {
- EmitOMPAggregateAssign(
- Emission.getAllocatedAddress(),
- OriginalLVal.getAddress(*this), Type,
- [this, VDInit, Init](Address DestElement,
- Address SrcElement) {
- // Clean up any temporaries needed by the
- // initialization.
- RunCleanupsScope InitScope(*this);
- // Emit initialization for single element.
- setAddrOfLocalVar(VDInit, SrcElement);
- EmitAnyExprToMem(Init, DestElement,
- Init->getType().getQualifiers(),
- /*IsInitializer*/ false);
- LocalDeclMap.erase(VDInit);
- });
- }
- EmitAutoVarCleanups(Emission);
- return Emission.getAllocatedAddress();
- });
- } else {
- Address OriginalAddr = OriginalLVal.getAddress(*this);
- IsRegistered =
- PrivateScope.addPrivate(OrigVD, [this, VDInit, OriginalAddr, VD,
- ThisFirstprivateIsLastprivate,
- OrigVD, &Lastprivates, IRef]() {
- // Emit private VarDecl with copy init.
- // Remap temp VDInit variable to the address of the original
- // variable (for proper handling of captured global variables).
- setAddrOfLocalVar(VDInit, OriginalAddr);
- EmitDecl(*VD);
- LocalDeclMap.erase(VDInit);
- if (ThisFirstprivateIsLastprivate &&
- Lastprivates[OrigVD->getCanonicalDecl()] ==
- OMPC_LASTPRIVATE_conditional) {
- // Create/init special variable for lastprivate conditionals.
- Address VDAddr =
- CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
- *this, OrigVD);
- llvm::Value *V = EmitLoadOfScalar(
- MakeAddrLValue(GetAddrOfLocalVar(VD), (*IRef)->getType(),
- AlignmentSource::Decl),
- (*IRef)->getExprLoc());
- EmitStoreOfScalar(V,
- MakeAddrLValue(VDAddr, (*IRef)->getType(),
- AlignmentSource::Decl));
- LocalDeclMap.erase(VD);
- setAddrOfLocalVar(VD, VDAddr);
- return VDAddr;
- }
- return GetAddrOfLocalVar(VD);
- });
- }
- assert(IsRegistered &&
- "firstprivate var already registered as private");
- // Silence the warning about unused variable.
- (void)IsRegistered;
- }
- ++IRef;
- ++InitsRef;
- }
- }
- return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
- }
- void CodeGenFunction::EmitOMPPrivateClause(
- const OMPExecutableDirective &D,
- CodeGenFunction::OMPPrivateScope &PrivateScope) {
- if (!HaveInsertPoint())
- return;
- llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
- for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
- auto IRef = C->varlist_begin();
- for (const Expr *IInit : C->private_copies()) {
- const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
- if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
- bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
- // Emit private VarDecl with copy init.
- EmitDecl(*VD);
- return GetAddrOfLocalVar(VD);
- });
- assert(IsRegistered && "private var already registered as private");
- // Silence the warning about unused variable.
- (void)IsRegistered;
- }
- ++IRef;
- }
- }
- }
- bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
- if (!HaveInsertPoint())
- return false;
- // threadprivate_var1 = master_threadprivate_var1;
- // operator=(threadprivate_var2, master_threadprivate_var2);
- // ...
- // __kmpc_barrier(&loc, global_tid);
- llvm::DenseSet<const VarDecl *> CopiedVars;
- llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
- for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
- auto IRef = C->varlist_begin();
- auto ISrcRef = C->source_exprs().begin();
- auto IDestRef = C->destination_exprs().begin();
- for (const Expr *AssignOp : C->assignment_ops()) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
- QualType Type = VD->getType();
- if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
- // Get the address of the master variable. If we are emitting code with
- // TLS support, the address is passed from the master as field in the
- // captured declaration.
- Address MasterAddr = Address::invalid();
- if (getLangOpts().OpenMPUseTLS &&
- getContext().getTargetInfo().isTLSSupported()) {
- assert(CapturedStmtInfo->lookup(VD) &&
- "Copyin threadprivates should have been captured!");
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
- (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
- MasterAddr = EmitLValue(&DRE).getAddress(*this);
- LocalDeclMap.erase(VD);
- } else {
- MasterAddr =
- Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
- : CGM.GetAddrOfGlobal(VD),
- getContext().getDeclAlign(VD));
- }
- // Get the address of the threadprivate variable.
- Address PrivateAddr = EmitLValue(*IRef).getAddress(*this);
- if (CopiedVars.size() == 1) {
- // At first check if current thread is a master thread. If it is, no
- // need to copy data.
- CopyBegin = createBasicBlock("copyin.not.master");
- CopyEnd = createBasicBlock("copyin.not.master.end");
- // TODO: Avoid ptrtoint conversion.
- auto *MasterAddrInt =
- Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy);
- auto *PrivateAddrInt =
- Builder.CreatePtrToInt(PrivateAddr.getPointer(), CGM.IntPtrTy);
- Builder.CreateCondBr(
- Builder.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,
- CopyEnd);
- EmitBlock(CopyBegin);
- }
- const auto *SrcVD =
- cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
- const auto *DestVD =
- cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
- EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
- }
- ++IRef;
- ++ISrcRef;
- ++IDestRef;
- }
- }
- if (CopyEnd) {
- // Exit out of copying procedure for non-master thread.
- EmitBlock(CopyEnd, /*IsFinished=*/true);
- return true;
- }
- return false;
- }
- bool CodeGenFunction::EmitOMPLastprivateClauseInit(
- const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
- if (!HaveInsertPoint())
- return false;
- bool HasAtLeastOneLastprivate = false;
- llvm::DenseSet<const VarDecl *> SIMDLCVs;
- if (isOpenMPSimdDirective(D.getDirectiveKind())) {
- const auto *LoopDirective = cast<OMPLoopDirective>(&D);
- for (const Expr *C : LoopDirective->counters()) {
- SIMDLCVs.insert(
- cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
- }
- }
- llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
- for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
- HasAtLeastOneLastprivate = true;
- if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
- !getLangOpts().OpenMPSimd)
- break;
- const auto *IRef = C->varlist_begin();
- const auto *IDestRef = C->destination_exprs().begin();
- for (const Expr *IInit : C->private_copies()) {
- // Keep the address of the original variable for future update at the end
- // of the loop.
- const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
- // Taskloops do not require additional initialization, it is done in
- // runtime support library.
- if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
- const auto *DestVD =
- cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
- PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
- /*RefersToEnclosingVariableOrCapture=*/
- CapturedStmtInfo->lookup(OrigVD) != nullptr,
- (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
- return EmitLValue(&DRE).getAddress(*this);
- });
- // Check if the variable is also a firstprivate: in this case IInit is
- // not generated. Initialization of this variable will happen in codegen
- // for 'firstprivate' clause.
- if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
- bool IsRegistered =
- PrivateScope.addPrivate(OrigVD, [this, VD, C, OrigVD]() {
- if (C->getKind() == OMPC_LASTPRIVATE_conditional) {
- Address VDAddr =
- CGM.getOpenMPRuntime().emitLastprivateConditionalInit(
- *this, OrigVD);
- setAddrOfLocalVar(VD, VDAddr);
- return VDAddr;
- }
- // Emit private VarDecl with copy init.
- EmitDecl(*VD);
- return GetAddrOfLocalVar(VD);
- });
- assert(IsRegistered &&
- "lastprivate var already registered as private");
- (void)IsRegistered;
- }
- }
- ++IRef;
- ++IDestRef;
- }
- }
- return HasAtLeastOneLastprivate;
- }
- void CodeGenFunction::EmitOMPLastprivateClauseFinal(
- const OMPExecutableDirective &D, bool NoFinals,
- llvm::Value *IsLastIterCond) {
- if (!HaveInsertPoint())
- return;
- // Emit following code:
- // if (<IsLastIterCond>) {
- // orig_var1 = private_orig_var1;
- // ...
- // orig_varn = private_orig_varn;
- // }
- llvm::BasicBlock *ThenBB = nullptr;
- llvm::BasicBlock *DoneBB = nullptr;
- if (IsLastIterCond) {
- // Emit implicit barrier if at least one lastprivate conditional is found
- // and this is not a simd mode.
- if (!getLangOpts().OpenMPSimd &&
- llvm::any_of(D.getClausesOfKind<OMPLastprivateClause>(),
- [](const OMPLastprivateClause *C) {
- return C->getKind() == OMPC_LASTPRIVATE_conditional;
- })) {
- CGM.getOpenMPRuntime().emitBarrierCall(*this, D.getBeginLoc(),
- OMPD_unknown,
- /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
- }
- ThenBB = createBasicBlock(".omp.lastprivate.then");
- DoneBB = createBasicBlock(".omp.lastprivate.done");
- Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
- EmitBlock(ThenBB);
- }
- llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
- llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
- if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
- auto IC = LoopDirective->counters().begin();
- for (const Expr *F : LoopDirective->finals()) {
- const auto *D =
- cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
- if (NoFinals)
- AlreadyEmittedVars.insert(D);
- else
- LoopCountersAndUpdates[D] = F;
- ++IC;
- }
- }
- for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
- auto IRef = C->varlist_begin();
- auto ISrcRef = C->source_exprs().begin();
- auto IDestRef = C->destination_exprs().begin();
- for (const Expr *AssignOp : C->assignment_ops()) {
- const auto *PrivateVD =
- cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
- QualType Type = PrivateVD->getType();
- const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
- if (AlreadyEmittedVars.insert(CanonicalVD).second) {
- // If lastprivate variable is a loop control variable for loop-based
- // directive, update its value before copyin back to original
- // variable.
- if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
- EmitIgnoredExpr(FinalExpr);
- const auto *SrcVD =
- cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
- const auto *DestVD =
- cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
- // Get the address of the private variable.
- Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
- if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
- PrivateAddr =
- Address(Builder.CreateLoad(PrivateAddr),
- CGM.getNaturalTypeAlignment(RefTy->getPointeeType()));
- // Store the last value to the private copy in the last iteration.
- if (C->getKind() == OMPC_LASTPRIVATE_conditional)
- CGM.getOpenMPRuntime().emitLastprivateConditionalFinalUpdate(
- *this, MakeAddrLValue(PrivateAddr, (*IRef)->getType()), PrivateVD,
- (*IRef)->getExprLoc());
- // Get the address of the original variable.
- Address OriginalAddr = GetAddrOfLocalVar(DestVD);
- EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
- }
- ++IRef;
- ++ISrcRef;
- ++IDestRef;
- }
- if (const Expr *PostUpdate = C->getPostUpdateExpr())
- EmitIgnoredExpr(PostUpdate);
- }
- if (IsLastIterCond)
- EmitBlock(DoneBB, /*IsFinished=*/true);
- }
- void CodeGenFunction::EmitOMPReductionClauseInit(
- const OMPExecutableDirective &D,
- CodeGenFunction::OMPPrivateScope &PrivateScope, bool ForInscan) {
- if (!HaveInsertPoint())
- return;
- SmallVector<const Expr *, 4> Shareds;
- SmallVector<const Expr *, 4> Privates;
- SmallVector<const Expr *, 4> ReductionOps;
- SmallVector<const Expr *, 4> LHSs;
- SmallVector<const Expr *, 4> RHSs;
- OMPTaskDataTy Data;
- SmallVector<const Expr *, 4> TaskLHSs;
- SmallVector<const Expr *, 4> TaskRHSs;
- for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
- if (ForInscan != (C->getModifier() == OMPC_REDUCTION_inscan))
- continue;
- Shareds.append(C->varlist_begin(), C->varlist_end());
- Privates.append(C->privates().begin(), C->privates().end());
- ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
- LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
- RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
- if (C->getModifier() == OMPC_REDUCTION_task) {
- Data.ReductionVars.append(C->privates().begin(), C->privates().end());
- Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
- Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
- Data.ReductionOps.append(C->reduction_ops().begin(),
- C->reduction_ops().end());
- TaskLHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
- TaskRHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
- }
- }
- ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
- unsigned Count = 0;
- auto *ILHS = LHSs.begin();
- auto *IRHS = RHSs.begin();
- auto *IPriv = Privates.begin();
- for (const Expr *IRef : Shareds) {
- const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
- // Emit private VarDecl with reduction init.
- RedCG.emitSharedOrigLValue(*this, Count);
- RedCG.emitAggregateType(*this, Count);
- AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
- RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
- RedCG.getSharedLValue(Count).getAddress(*this),
- [&Emission](CodeGenFunction &CGF) {
- CGF.EmitAutoVarInit(Emission);
- return true;
- });
- EmitAutoVarCleanups(Emission);
- Address BaseAddr = RedCG.adjustPrivateAddress(
- *this, Count, Emission.getAllocatedAddress());
- bool IsRegistered = PrivateScope.addPrivate(
- RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
- assert(IsRegistered && "private var already registered as private");
- // Silence the warning about unused variable.
- (void)IsRegistered;
- const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
- const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
- QualType Type = PrivateVD->getType();
- bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
- if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
- // Store the address of the original variable associated with the LHS
- // implicit variable.
- PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() {
- return RedCG.getSharedLValue(Count).getAddress(*this);
- });
- PrivateScope.addPrivate(
- RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
- } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
- isa<ArraySubscriptExpr>(IRef)) {
- // Store the address of the original variable associated with the LHS
- // implicit variable.
- PrivateScope.addPrivate(LHSVD, [&RedCG, Count, this]() {
- return RedCG.getSharedLValue(Count).getAddress(*this);
- });
- PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
- return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
- ConvertTypeForMem(RHSVD->getType()),
- "rhs.begin");
- });
- } else {
- QualType Type = PrivateVD->getType();
- bool IsArray = getContext().getAsArrayType(Type) != nullptr;
- Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress(*this);
- // Store the address of the original variable associated with the LHS
- // implicit variable.
- if (IsArray) {
- OriginalAddr = Builder.CreateElementBitCast(
- OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
- }
- PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
- PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
- return IsArray ? Builder.CreateElementBitCast(
- GetAddrOfLocalVar(PrivateVD),
- ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
- : GetAddrOfLocalVar(PrivateVD);
- });
- }
- ++ILHS;
- ++IRHS;
- ++IPriv;
- ++Count;
- }
- if (!Data.ReductionVars.empty()) {
- Data.IsReductionWithTaskMod = true;
- Data.IsWorksharingReduction =
- isOpenMPWorksharingDirective(D.getDirectiveKind());
- llvm::Value *ReductionDesc = CGM.getOpenMPRuntime().emitTaskReductionInit(
- *this, D.getBeginLoc(), TaskLHSs, TaskRHSs, Data);
- const Expr *TaskRedRef = nullptr;
- switch (D.getDirectiveKind()) {
- case OMPD_parallel:
- TaskRedRef = cast<OMPParallelDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_for:
- TaskRedRef = cast<OMPForDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_sections:
- TaskRedRef = cast<OMPSectionsDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_parallel_for:
- TaskRedRef = cast<OMPParallelForDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_parallel_master:
- TaskRedRef =
- cast<OMPParallelMasterDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_parallel_sections:
- TaskRedRef =
- cast<OMPParallelSectionsDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_target_parallel:
- TaskRedRef =
- cast<OMPTargetParallelDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_target_parallel_for:
- TaskRedRef =
- cast<OMPTargetParallelForDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_distribute_parallel_for:
- TaskRedRef =
- cast<OMPDistributeParallelForDirective>(D).getTaskReductionRefExpr();
- break;
- case OMPD_teams_distribute_parallel_for:
- TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(D)
- .getTaskReductionRefExpr();
- break;
- case OMPD_target_teams_distribute_parallel_for:
- TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(D)
- .getTaskReductionRefExpr();
- break;
- case OMPD_simd:
- case OMPD_for_simd:
- case OMPD_section:
- case OMPD_single:
- case OMPD_master:
- case OMPD_critical:
- case OMPD_parallel_for_simd:
- case OMPD_task:
- case OMPD_taskyield:
- case OMPD_barrier:
- case OMPD_taskwait:
- case OMPD_taskgroup:
- case OMPD_flush:
- case OMPD_depobj:
- case OMPD_scan:
- case OMPD_ordered:
- case OMPD_atomic:
- case OMPD_teams:
- case OMPD_target:
- case OMPD_cancellation_point:
- case OMPD_cancel:
- case OMPD_target_data:
- case OMPD_target_enter_data:
- case OMPD_target_exit_data:
- case OMPD_taskloop:
- case OMPD_taskloop_simd:
- case OMPD_master_taskloop:
- case OMPD_master_taskloop_simd:
- case OMPD_parallel_master_taskloop:
- case OMPD_parallel_master_taskloop_simd:
- case OMPD_distribute:
- case OMPD_target_update:
- case OMPD_distribute_parallel_for_simd:
- case OMPD_distribute_simd:
- case OMPD_target_parallel_for_simd:
- case OMPD_target_simd:
- case OMPD_teams_distribute:
- case OMPD_teams_distribute_simd:
- case OMPD_teams_distribute_parallel_for_simd:
- case OMPD_target_teams:
- case OMPD_target_teams_distribute:
- case OMPD_target_teams_distribute_parallel_for_simd:
- case OMPD_target_teams_distribute_simd:
- case OMPD_declare_target:
- case OMPD_end_declare_target:
- case OMPD_threadprivate:
- case OMPD_allocate:
- case OMPD_declare_reduction:
- case OMPD_declare_mapper:
- case OMPD_declare_simd:
- case OMPD_requires:
- case OMPD_declare_variant:
- case OMPD_begin_declare_variant:
- case OMPD_end_declare_variant:
- case OMPD_unknown:
- default:
- llvm_unreachable("Enexpected directive with task reductions.");
- }
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());
- EmitVarDecl(*VD);
- EmitStoreOfScalar(ReductionDesc, GetAddrOfLocalVar(VD),
- /*Volatile=*/false, TaskRedRef->getType());
- }
- }
- void CodeGenFunction::EmitOMPReductionClauseFinal(
- const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
- if (!HaveInsertPoint())
- return;
- llvm::SmallVector<const Expr *, 8> Privates;
- llvm::SmallVector<const Expr *, 8> LHSExprs;
- llvm::SmallVector<const Expr *, 8> RHSExprs;
- llvm::SmallVector<const Expr *, 8> ReductionOps;
- bool HasAtLeastOneReduction = false;
- bool IsReductionWithTaskMod = false;
- for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
- // Do not emit for inscan reductions.
- if (C->getModifier() == OMPC_REDUCTION_inscan)
- continue;
- HasAtLeastOneReduction = true;
- Privates.append(C->privates().begin(), C->privates().end());
- LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
- RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
- ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
- IsReductionWithTaskMod =
- IsReductionWithTaskMod || C->getModifier() == OMPC_REDUCTION_task;
- }
- if (HasAtLeastOneReduction) {
- if (IsReductionWithTaskMod) {
- CGM.getOpenMPRuntime().emitTaskReductionFini(
- *this, D.getBeginLoc(),
- isOpenMPWorksharingDirective(D.getDirectiveKind()));
- }
- bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
- isOpenMPParallelDirective(D.getDirectiveKind()) ||
- ReductionKind == OMPD_simd;
- bool SimpleReduction = ReductionKind == OMPD_simd;
- // Emit nowait reduction if nowait clause is present or directive is a
- // parallel directive (it always has implicit barrier).
- CGM.getOpenMPRuntime().emitReduction(
- *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
- {WithNowait, SimpleReduction, ReductionKind});
- }
- }
- static void emitPostUpdateForReductionClause(
- CodeGenFunction &CGF, const OMPExecutableDirective &D,
- const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
- if (!CGF.HaveInsertPoint())
- return;
- llvm::BasicBlock *DoneBB = nullptr;
- for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
- if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
- if (!DoneBB) {
- if (llvm::Value *Cond = CondGen(CGF)) {
- // If the first post-update expression is found, emit conditional
- // block if it was requested.
- llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
- DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
- CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
- CGF.EmitBlock(ThenBB);
- }
- }
- CGF.EmitIgnoredExpr(PostUpdate);
- }
- }
- if (DoneBB)
- CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
- }
- namespace {
- /// Codegen lambda for appending distribute lower and upper bounds to outlined
- /// parallel function. This is necessary for combined constructs such as
- /// 'distribute parallel for'
- typedef llvm::function_ref<void(CodeGenFunction &,
- const OMPExecutableDirective &,
- llvm::SmallVectorImpl<llvm::Value *> &)>
- CodeGenBoundParametersTy;
- } // anonymous namespace
- static void
- checkForLastprivateConditionalUpdate(CodeGenFunction &CGF,
- const OMPExecutableDirective &S) {
- if (CGF.getLangOpts().OpenMP < 50)
- return;
- llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;
- for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
- for (const Expr *Ref : C->varlists()) {
- if (!Ref->getType()->isScalarType())
- continue;
- const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
- if (!DRE)
- continue;
- PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
- }
- }
- for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
- for (const Expr *Ref : C->varlists()) {
- if (!Ref->getType()->isScalarType())
- continue;
- const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
- if (!DRE)
- continue;
- PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
- }
- }
- for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
- for (const Expr *Ref : C->varlists()) {
- if (!Ref->getType()->isScalarType())
- continue;
- const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
- if (!DRE)
- continue;
- PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, Ref);
- }
- }
- // Privates should ne analyzed since they are not captured at all.
- // Task reductions may be skipped - tasks are ignored.
- // Firstprivates do not return value but may be passed by reference - no need
- // to check for updated lastprivate conditional.
- for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
- for (const Expr *Ref : C->varlists()) {
- if (!Ref->getType()->isScalarType())
- continue;
- const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
- if (!DRE)
- continue;
- PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));
- }
- }
- CGF.CGM.getOpenMPRuntime().checkAndEmitSharedLastprivateConditional(
- CGF, S, PrivateDecls);
- }
- static void emitCommonOMPParallelDirective(
- CodeGenFunction &CGF, const OMPExecutableDirective &S,
- OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
- const CodeGenBoundParametersTy &CodeGenBoundParameters) {
- const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
- llvm::Value *NumThreads = nullptr;
- llvm::Function *OutlinedFn =
- CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
- S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
- if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
- CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
- NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
- /*IgnoreResultAssign=*/true);
- CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
- CGF, NumThreads, NumThreadsClause->getBeginLoc());
- }
- if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
- CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
- CGF.CGM.getOpenMPRuntime().emitProcBindClause(
- CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
- }
- const Expr *IfCond = nullptr;
- for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
- if (C->getNameModifier() == OMPD_unknown ||
- C->getNameModifier() == OMPD_parallel) {
- IfCond = C->getCondition();
- break;
- }
- }
- OMPParallelScope Scope(CGF, S);
- llvm::SmallVector<llvm::Value *, 16> CapturedVars;
- // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
- // lower and upper bounds with the pragma 'for' chunking mechanism.
- // The following lambda takes care of appending the lower and upper bound
- // parameters when necessary
- CodeGenBoundParameters(CGF, S, CapturedVars);
- CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
- CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
- CapturedVars, IfCond, NumThreads);
- }
- static bool isAllocatableDecl(const VarDecl *VD) {
- const VarDecl *CVD = VD->getCanonicalDecl();
- if (!CVD->hasAttr<OMPAllocateDeclAttr>())
- return false;
- const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
- // Use the default allocation.
- return !((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||
- AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&
- !AA->getAllocator());
- }
- static void emitEmptyBoundParameters(CodeGenFunction &,
- const OMPExecutableDirective &,
- llvm::SmallVectorImpl<llvm::Value *> &) {}
- Address CodeGenFunction::OMPBuilderCBHelpers::getAddressOfLocalVariable(
- CodeGenFunction &CGF, const VarDecl *VD) {
- CodeGenModule &CGM = CGF.CGM;
- auto &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- if (!VD)
- return Address::invalid();
- const VarDecl *CVD = VD->getCanonicalDecl();
- if (!isAllocatableDecl(CVD))
- return Address::invalid();
- llvm::Value *Size;
- CharUnits Align = CGM.getContext().getDeclAlign(CVD);
- if (CVD->getType()->isVariablyModifiedType()) {
- Size = CGF.getTypeSize(CVD->getType());
- // Align the size: ((size + align - 1) / align) * align
- Size = CGF.Builder.CreateNUWAdd(
- Size, CGM.getSize(Align - CharUnits::fromQuantity(1)));
- Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align));
- Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align));
- } else {
- CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType());
- Size = CGM.getSize(Sz.alignTo(Align));
- }
- const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
- assert(AA->getAllocator() &&
- "Expected allocator expression for non-default allocator.");
- llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator());
- // According to the standard, the original allocator type is a enum (integer).
- // Convert to pointer type, if required.
- if (Allocator->getType()->isIntegerTy())
- Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy);
- else if (Allocator->getType()->isPointerTy())
- Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator,
- CGM.VoidPtrTy);
- llvm::Value *Addr = OMPBuilder.createOMPAlloc(
- CGF.Builder, Size, Allocator,
- getNameWithSeparators({CVD->getName(), ".void.addr"}, ".", "."));
- llvm::CallInst *FreeCI =
- OMPBuilder.createOMPFree(CGF.Builder, Addr, Allocator);
- CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FreeCI);
- Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- Addr,
- CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())),
- getNameWithSeparators({CVD->getName(), ".addr"}, ".", "."));
- return Address(Addr, Align);
- }
- Address CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
- CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr,
- SourceLocation Loc) {
- CodeGenModule &CGM = CGF.CGM;
- if (CGM.getLangOpts().OpenMPUseTLS &&
- CGM.getContext().getTargetInfo().isTLSSupported())
- return VDAddr;
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- llvm::Type *VarTy = VDAddr.getElementType();
- llvm::Value *Data =
- CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.Int8PtrTy);
- llvm::ConstantInt *Size = CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy));
- std::string Suffix = getNameWithSeparators({"cache", ""});
- llvm::Twine CacheName = Twine(CGM.getMangledName(VD)).concat(Suffix);
- llvm::CallInst *ThreadPrivateCacheCall =
- OMPBuilder.createCachedThreadPrivate(CGF.Builder, Data, Size, CacheName);
- return Address(ThreadPrivateCacheCall, VDAddr.getAlignment());
- }
- std::string CodeGenFunction::OMPBuilderCBHelpers::getNameWithSeparators(
- ArrayRef<StringRef> Parts, StringRef FirstSeparator, StringRef Separator) {
- SmallString<128> Buffer;
- llvm::raw_svector_ostream OS(Buffer);
- StringRef Sep = FirstSeparator;
- for (StringRef Part : Parts) {
- OS << Sep << Part;
- Sep = Separator;
- }
- return OS.str().str();
- }
- void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
- if (CGM.getLangOpts().OpenMPIRBuilder) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- // Check if we have any if clause associated with the directive.
- llvm::Value *IfCond = nullptr;
- if (const auto *C = S.getSingleClause<OMPIfClause>())
- IfCond = EmitScalarExpr(C->getCondition(),
- /*IgnoreResultAssign=*/true);
- llvm::Value *NumThreads = nullptr;
- if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>())
- NumThreads = EmitScalarExpr(NumThreadsClause->getNumThreads(),
- /*IgnoreResultAssign=*/true);
- ProcBindKind ProcBind = OMP_PROC_BIND_default;
- if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>())
- ProcBind = ProcBindClause->getProcBindKind();
- using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- // The cleanup callback that finalizes all variabels at the given location,
- // thus calls destructors etc.
- auto FiniCB = [this](InsertPointTy IP) {
- OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
- };
- // Privatization callback that performs appropriate action for
- // shared/private/firstprivate/lastprivate/copyin/... variables.
- //
- // TODO: This defaults to shared right now.
- auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
- llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
- // The next line is appropriate only for variables (Val) with the
- // data-sharing attribute "shared".
- ReplVal = &Val;
- return CodeGenIP;
- };
- const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
- const Stmt *ParallelRegionBodyStmt = CS->getCapturedStmt();
- auto BodyGenCB = [ParallelRegionBodyStmt,
- this](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
- llvm::BasicBlock &ContinuationBB) {
- OMPBuilderCBHelpers::OutlinedRegionBodyRAII ORB(*this, AllocaIP,
- ContinuationBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, ParallelRegionBodyStmt,
- CodeGenIP, ContinuationBB);
- };
- CGCapturedStmtInfo CGSI(*CS, CR_OpenMP);
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
- llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
- AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
- Builder.restoreIP(
- OMPBuilder.createParallel(Builder, AllocaIP, BodyGenCB, PrivCB, FiniCB,
- IfCond, NumThreads, ProcBind, S.hasCancel()));
- return;
- }
- // Emit parallel region as a standalone region.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- OMPPrivateScope PrivateScope(CGF);
- bool Copyins = CGF.EmitOMPCopyinClause(S);
- (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
- if (Copyins) {
- // Emit implicit barrier to synchronize threads and avoid data races on
- // propagation master's thread values of threadprivate variables to local
- // instances of that variables of all other implicit threads.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(
- CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
- }
- CGF.EmitOMPPrivateClause(S, PrivateScope);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
- };
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
- emitEmptyBoundParameters);
- emitPostUpdateForReductionClause(*this, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- void CodeGenFunction::EmitOMPMetaDirective(const OMPMetaDirective &S) {
- EmitStmt(S.getIfStmt());
- }
- namespace {
- /// RAII to handle scopes for loop transformation directives.
- class OMPTransformDirectiveScopeRAII {
- OMPLoopScope *Scope = nullptr;
- CodeGenFunction::CGCapturedStmtInfo *CGSI = nullptr;
- CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII = nullptr;
- public:
- OMPTransformDirectiveScopeRAII(CodeGenFunction &CGF, const Stmt *S) {
- if (const auto *Dir = dyn_cast<OMPLoopBasedDirective>(S)) {
- Scope = new OMPLoopScope(CGF, *Dir);
- CGSI = new CodeGenFunction::CGCapturedStmtInfo(CR_OpenMP);
- CapInfoRAII = new CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);
- }
- }
- ~OMPTransformDirectiveScopeRAII() {
- if (!Scope)
- return;
- delete CapInfoRAII;
- delete CGSI;
- delete Scope;
- }
- };
- } // namespace
- static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
- int MaxLevel, int Level = 0) {
- assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
- const Stmt *SimplifiedS = S->IgnoreContainers();
- if (const auto *CS = dyn_cast<CompoundStmt>(SimplifiedS)) {
- PrettyStackTraceLoc CrashInfo(
- CGF.getContext().getSourceManager(), CS->getLBracLoc(),
- "LLVM IR generation of compound statement ('{}')");
- // Keep track of the current cleanup stack depth, including debug scopes.
- CodeGenFunction::LexicalScope Scope(CGF, S->getSourceRange());
- for (const Stmt *CurStmt : CS->body())
- emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
- return;
- }
- if (SimplifiedS == NextLoop) {
- if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
- SimplifiedS = Dir->getTransformedStmt();
- if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
- SimplifiedS = CanonLoop->getLoopStmt();
- if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
- S = For->getBody();
- } else {
- assert(isa<CXXForRangeStmt>(SimplifiedS) &&
- "Expected canonical for loop or range-based for loop.");
- const auto *CXXFor = cast<CXXForRangeStmt>(SimplifiedS);
- CGF.EmitStmt(CXXFor->getLoopVarStmt());
- S = CXXFor->getBody();
- }
- if (Level + 1 < MaxLevel) {
- NextLoop = OMPLoopDirective::tryToFindNextInnerLoop(
- S, /*TryImperfectlyNestedLoops=*/true);
- emitBody(CGF, S, NextLoop, MaxLevel, Level + 1);
- return;
- }
- }
- CGF.EmitStmt(S);
- }
- void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
- JumpDest LoopExit) {
- RunCleanupsScope BodyScope(*this);
- // Update counters values on current iteration.
- for (const Expr *UE : D.updates())
- EmitIgnoredExpr(UE);
- // Update the linear variables.
- // In distribute directives only loop counters may be marked as linear, no
- // need to generate the code for them.
- if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
- for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
- for (const Expr *UE : C->updates())
- EmitIgnoredExpr(UE);
- }
- }
- // On a continue in the body, jump to the end.
- JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
- BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
- for (const Expr *E : D.finals_conditions()) {
- if (!E)
- continue;
- // Check that loop counter in non-rectangular nest fits into the iteration
- // space.
- llvm::BasicBlock *NextBB = createBasicBlock("omp.body.next");
- EmitBranchOnBoolExpr(E, NextBB, Continue.getBlock(),
- getProfileCount(D.getBody()));
- EmitBlock(NextBB);
- }
- OMPPrivateScope InscanScope(*this);
- EmitOMPReductionClauseInit(D, InscanScope, /*ForInscan=*/true);
- bool IsInscanRegion = InscanScope.Privatize();
- if (IsInscanRegion) {
- // Need to remember the block before and after scan directive
- // to dispatch them correctly depending on the clause used in
- // this directive, inclusive or exclusive. For inclusive scan the natural
- // order of the blocks is used, for exclusive clause the blocks must be
- // executed in reverse order.
- OMPBeforeScanBlock = createBasicBlock("omp.before.scan.bb");
- OMPAfterScanBlock = createBasicBlock("omp.after.scan.bb");
- // No need to allocate inscan exit block, in simd mode it is selected in the
- // codegen for the scan directive.
- if (D.getDirectiveKind() != OMPD_simd && !getLangOpts().OpenMPSimd)
- OMPScanExitBlock = createBasicBlock("omp.exit.inscan.bb");
- OMPScanDispatch = createBasicBlock("omp.inscan.dispatch");
- EmitBranch(OMPScanDispatch);
- EmitBlock(OMPBeforeScanBlock);
- }
- // Emit loop variables for C++ range loops.
- const Stmt *Body =
- D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();
- // Emit loop body.
- emitBody(*this, Body,
- OMPLoopBasedDirective::tryToFindNextInnerLoop(
- Body, /*TryImperfectlyNestedLoops=*/true),
- D.getLoopsNumber());
- // Jump to the dispatcher at the end of the loop body.
- if (IsInscanRegion)
- EmitBranch(OMPScanExitBlock);
- // The end (updates/cleanups).
- EmitBlock(Continue.getBlock());
- BreakContinueStack.pop_back();
- }
- using EmittedClosureTy = std::pair<llvm::Function *, llvm::Value *>;
- /// Emit a captured statement and return the function as well as its captured
- /// closure context.
- static EmittedClosureTy emitCapturedStmtFunc(CodeGenFunction &ParentCGF,
- const CapturedStmt *S) {
- LValue CapStruct = ParentCGF.InitCapturedStruct(*S);
- CodeGenFunction CGF(ParentCGF.CGM, /*suppressNewContext=*/true);
- std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =
- std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());
- llvm::Function *F = CGF.GenerateCapturedStmtFunction(*S);
- return {F, CapStruct.getPointer(ParentCGF)};
- }
- /// Emit a call to a previously captured closure.
- static llvm::CallInst *
- emitCapturedStmtCall(CodeGenFunction &ParentCGF, EmittedClosureTy Cap,
- llvm::ArrayRef<llvm::Value *> Args) {
- // Append the closure context to the argument.
- SmallVector<llvm::Value *> EffectiveArgs;
- EffectiveArgs.reserve(Args.size() + 1);
- llvm::append_range(EffectiveArgs, Args);
- EffectiveArgs.push_back(Cap.second);
- return ParentCGF.Builder.CreateCall(Cap.first, EffectiveArgs);
- }
- llvm::CanonicalLoopInfo *
- CodeGenFunction::EmitOMPCollapsedCanonicalLoopNest(const Stmt *S, int Depth) {
- assert(Depth == 1 && "Nested loops with OpenMPIRBuilder not yet implemented");
- // The caller is processing the loop-associated directive processing the \p
- // Depth loops nested in \p S. Put the previous pending loop-associated
- // directive to the stack. If the current loop-associated directive is a loop
- // transformation directive, it will push its generated loops onto the stack
- // such that together with the loops left here they form the combined loop
- // nest for the parent loop-associated directive.
- int ParentExpectedOMPLoopDepth = ExpectedOMPLoopDepth;
- ExpectedOMPLoopDepth = Depth;
- EmitStmt(S);
- assert(OMPLoopNestStack.size() >= (size_t)Depth && "Found too few loops");
- // The last added loop is the outermost one.
- llvm::CanonicalLoopInfo *Result = OMPLoopNestStack.back();
- // Pop the \p Depth loops requested by the call from that stack and restore
- // the previous context.
- OMPLoopNestStack.pop_back_n(Depth);
- ExpectedOMPLoopDepth = ParentExpectedOMPLoopDepth;
- return Result;
- }
- void CodeGenFunction::EmitOMPCanonicalLoop(const OMPCanonicalLoop *S) {
- const Stmt *SyntacticalLoop = S->getLoopStmt();
- if (!getLangOpts().OpenMPIRBuilder) {
- // Ignore if OpenMPIRBuilder is not enabled.
- EmitStmt(SyntacticalLoop);
- return;
- }
- LexicalScope ForScope(*this, S->getSourceRange());
- // Emit init statements. The Distance/LoopVar funcs may reference variable
- // declarations they contain.
- const Stmt *BodyStmt;
- if (const auto *For = dyn_cast<ForStmt>(SyntacticalLoop)) {
- if (const Stmt *InitStmt = For->getInit())
- EmitStmt(InitStmt);
- BodyStmt = For->getBody();
- } else if (const auto *RangeFor =
- dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {
- if (const DeclStmt *RangeStmt = RangeFor->getRangeStmt())
- EmitStmt(RangeStmt);
- if (const DeclStmt *BeginStmt = RangeFor->getBeginStmt())
- EmitStmt(BeginStmt);
- if (const DeclStmt *EndStmt = RangeFor->getEndStmt())
- EmitStmt(EndStmt);
- if (const DeclStmt *LoopVarStmt = RangeFor->getLoopVarStmt())
- EmitStmt(LoopVarStmt);
- BodyStmt = RangeFor->getBody();
- } else
- llvm_unreachable("Expected for-stmt or range-based for-stmt");
- // Emit closure for later use. By-value captures will be captured here.
- const CapturedStmt *DistanceFunc = S->getDistanceFunc();
- EmittedClosureTy DistanceClosure = emitCapturedStmtFunc(*this, DistanceFunc);
- const CapturedStmt *LoopVarFunc = S->getLoopVarFunc();
- EmittedClosureTy LoopVarClosure = emitCapturedStmtFunc(*this, LoopVarFunc);
- // Call the distance function to get the number of iterations of the loop to
- // come.
- QualType LogicalTy = DistanceFunc->getCapturedDecl()
- ->getParam(0)
- ->getType()
- .getNonReferenceType();
- Address CountAddr = CreateMemTemp(LogicalTy, ".count.addr");
- emitCapturedStmtCall(*this, DistanceClosure, {CountAddr.getPointer()});
- llvm::Value *DistVal = Builder.CreateLoad(CountAddr, ".count");
- // Emit the loop structure.
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- auto BodyGen = [&, this](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,
- llvm::Value *IndVar) {
- Builder.restoreIP(CodeGenIP);
- // Emit the loop body: Convert the logical iteration number to the loop
- // variable and emit the body.
- const DeclRefExpr *LoopVarRef = S->getLoopVarRef();
- LValue LCVal = EmitLValue(LoopVarRef);
- Address LoopVarAddress = LCVal.getAddress(*this);
- emitCapturedStmtCall(*this, LoopVarClosure,
- {LoopVarAddress.getPointer(), IndVar});
- RunCleanupsScope BodyScope(*this);
- EmitStmt(BodyStmt);
- };
- llvm::CanonicalLoopInfo *CL =
- OMPBuilder.createCanonicalLoop(Builder, BodyGen, DistVal);
- // Finish up the loop.
- Builder.restoreIP(CL->getAfterIP());
- ForScope.ForceCleanup();
- // Remember the CanonicalLoopInfo for parent AST nodes consuming it.
- OMPLoopNestStack.push_back(CL);
- }
- void CodeGenFunction::EmitOMPInnerLoop(
- const OMPExecutableDirective &S, bool RequiresCleanup, const Expr *LoopCond,
- const Expr *IncExpr,
- const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
- const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
- auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
- // Start the loop with a block that tests the condition.
- auto CondBlock = createBasicBlock("omp.inner.for.cond");
- EmitBlock(CondBlock);
- const SourceRange R = S.getSourceRange();
- // If attributes are attached, push to the basic block with them.
- const auto &OMPED = cast<OMPExecutableDirective>(S);
- const CapturedStmt *ICS = OMPED.getInnermostCapturedStmt();
- const Stmt *SS = ICS->getCapturedStmt();
- const AttributedStmt *AS = dyn_cast_or_null<AttributedStmt>(SS);
- OMPLoopNestStack.clear();
- if (AS)
- LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(),
- AS->getAttrs(), SourceLocToDebugLoc(R.getBegin()),
- SourceLocToDebugLoc(R.getEnd()));
- else
- LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
- SourceLocToDebugLoc(R.getEnd()));
- // If there are any cleanups between here and the loop-exit scope,
- // create a block to stage a loop exit along.
- llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
- if (RequiresCleanup)
- ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
- llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
- // Emit condition.
- EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
- if (ExitBlock != LoopExit.getBlock()) {
- EmitBlock(ExitBlock);
- EmitBranchThroughCleanup(LoopExit);
- }
- EmitBlock(LoopBody);
- incrementProfileCounter(&S);
- // Create a block for the increment.
- JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
- BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
- BodyGen(*this);
- // Emit "IV = IV + 1" and a back-edge to the condition block.
- EmitBlock(Continue.getBlock());
- EmitIgnoredExpr(IncExpr);
- PostIncGen(*this);
- BreakContinueStack.pop_back();
- EmitBranch(CondBlock);
- LoopStack.pop();
- // Emit the fall-through block.
- EmitBlock(LoopExit.getBlock());
- }
- bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
- if (!HaveInsertPoint())
- return false;
- // Emit inits for the linear variables.
- bool HasLinears = false;
- for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
- for (const Expr *Init : C->inits()) {
- HasLinears = true;
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
- if (const auto *Ref =
- dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
- AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
- const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
- CapturedStmtInfo->lookup(OrigVD) != nullptr,
- VD->getInit()->getType(), VK_LValue,
- VD->getInit()->getExprLoc());
- EmitExprAsInit(
- &DRE, VD,
- MakeAddrLValue(Emission.getAllocatedAddress(), VD->getType()),
- /*capturedByInit=*/false);
- EmitAutoVarCleanups(Emission);
- } else {
- EmitVarDecl(*VD);
- }
- }
- // Emit the linear steps for the linear clauses.
- // If a step is not constant, it is pre-calculated before the loop.
- if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
- if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
- EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
- // Emit calculation of the linear step.
- EmitIgnoredExpr(CS);
- }
- }
- return HasLinears;
- }
- void CodeGenFunction::EmitOMPLinearClauseFinal(
- const OMPLoopDirective &D,
- const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
- if (!HaveInsertPoint())
- return;
- llvm::BasicBlock *DoneBB = nullptr;
- // Emit the final values of the linear variables.
- for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
- auto IC = C->varlist_begin();
- for (const Expr *F : C->finals()) {
- if (!DoneBB) {
- if (llvm::Value *Cond = CondGen(*this)) {
- // If the first post-update expression is found, emit conditional
- // block if it was requested.
- llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
- DoneBB = createBasicBlock(".omp.linear.pu.done");
- Builder.CreateCondBr(Cond, ThenBB, DoneBB);
- EmitBlock(ThenBB);
- }
- }
- const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
- CapturedStmtInfo->lookup(OrigVD) != nullptr,
- (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
- Address OrigAddr = EmitLValue(&DRE).getAddress(*this);
- CodeGenFunction::OMPPrivateScope VarScope(*this);
- VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
- (void)VarScope.Privatize();
- EmitIgnoredExpr(F);
- ++IC;
- }
- if (const Expr *PostUpdate = C->getPostUpdateExpr())
- EmitIgnoredExpr(PostUpdate);
- }
- if (DoneBB)
- EmitBlock(DoneBB, /*IsFinished=*/true);
- }
- static void emitAlignedClause(CodeGenFunction &CGF,
- const OMPExecutableDirective &D) {
- if (!CGF.HaveInsertPoint())
- return;
- for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
- llvm::APInt ClauseAlignment(64, 0);
- if (const Expr *AlignmentExpr = Clause->getAlignment()) {
- auto *AlignmentCI =
- cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
- ClauseAlignment = AlignmentCI->getValue();
- }
- for (const Expr *E : Clause->varlists()) {
- llvm::APInt Alignment(ClauseAlignment);
- if (Alignment == 0) {
- // OpenMP [2.8.1, Description]
- // If no optional parameter is specified, implementation-defined default
- // alignments for SIMD instructions on the target platforms are assumed.
- Alignment =
- CGF.getContext()
- .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
- E->getType()->getPointeeType()))
- .getQuantity();
- }
- assert((Alignment == 0 || Alignment.isPowerOf2()) &&
- "alignment is not power of 2");
- if (Alignment != 0) {
- llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
- CGF.emitAlignmentAssumption(
- PtrValue, E, /*No second loc needed*/ SourceLocation(),
- llvm::ConstantInt::get(CGF.getLLVMContext(), Alignment));
- }
- }
- }
- }
- void CodeGenFunction::EmitOMPPrivateLoopCounters(
- const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
- if (!HaveInsertPoint())
- return;
- auto I = S.private_counters().begin();
- for (const Expr *E : S.counters()) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
- // Emit var without initialization.
- AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
- EmitAutoVarCleanups(VarEmission);
- LocalDeclMap.erase(PrivateVD);
- (void)LoopScope.addPrivate(
- VD, [&VarEmission]() { return VarEmission.getAllocatedAddress(); });
- if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
- VD->hasGlobalStorage()) {
- (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
- LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
- E->getType(), VK_LValue, E->getExprLoc());
- return EmitLValue(&DRE).getAddress(*this);
- });
- } else {
- (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
- return VarEmission.getAllocatedAddress();
- });
- }
- ++I;
- }
- // Privatize extra loop counters used in loops for ordered(n) clauses.
- for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
- if (!C->getNumForLoops())
- continue;
- for (unsigned I = S.getLoopsNumber(), E = C->getLoopNumIterations().size();
- I < E; ++I) {
- const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
- const auto *VD = cast<VarDecl>(DRE->getDecl());
- // Override only those variables that can be captured to avoid re-emission
- // of the variables declared within the loops.
- if (DRE->refersToEnclosingVariableOrCapture()) {
- (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
- return CreateMemTemp(DRE->getType(), VD->getName());
- });
- }
- }
- }
- }
- static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
- const Expr *Cond, llvm::BasicBlock *TrueBlock,
- llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
- if (!CGF.HaveInsertPoint())
- return;
- {
- CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
- CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
- (void)PreCondScope.Privatize();
- // Get initial values of real counters.
- for (const Expr *I : S.inits()) {
- CGF.EmitIgnoredExpr(I);
- }
- }
- // Create temp loop control variables with their init values to support
- // non-rectangular loops.
- CodeGenFunction::OMPMapVars PreCondVars;
- for (const Expr *E : S.dependent_counters()) {
- if (!E)
- continue;
- assert(!E->getType().getNonReferenceType()->isRecordType() &&
- "dependent counter must not be an iterator.");
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- Address CounterAddr =
- CGF.CreateMemTemp(VD->getType().getNonReferenceType());
- (void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);
- }
- (void)PreCondVars.apply(CGF);
- for (const Expr *E : S.dependent_inits()) {
- if (!E)
- continue;
- CGF.EmitIgnoredExpr(E);
- }
- // Check that loop is executed at least one time.
- CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
- PreCondVars.restore(CGF);
- }
- void CodeGenFunction::EmitOMPLinearClause(
- const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
- if (!HaveInsertPoint())
- return;
- llvm::DenseSet<const VarDecl *> SIMDLCVs;
- if (isOpenMPSimdDirective(D.getDirectiveKind())) {
- const auto *LoopDirective = cast<OMPLoopDirective>(&D);
- for (const Expr *C : LoopDirective->counters()) {
- SIMDLCVs.insert(
- cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
- }
- }
- for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
- auto CurPrivate = C->privates().begin();
- for (const Expr *E : C->varlists()) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- const auto *PrivateVD =
- cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
- if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
- bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
- // Emit private VarDecl with copy init.
- EmitVarDecl(*PrivateVD);
- return GetAddrOfLocalVar(PrivateVD);
- });
- assert(IsRegistered && "linear var already registered as private");
- // Silence the warning about unused variable.
- (void)IsRegistered;
- } else {
- EmitVarDecl(*PrivateVD);
- }
- ++CurPrivate;
- }
- }
- }
- static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
- const OMPExecutableDirective &D) {
- if (!CGF.HaveInsertPoint())
- return;
- if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
- RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
- /*ignoreResult=*/true);
- auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
- CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
- // In presence of finite 'safelen', it may be unsafe to mark all
- // the memory instructions parallel, because loop-carried
- // dependences of 'safelen' iterations are possible.
- CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
- } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
- RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
- /*ignoreResult=*/true);
- auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
- CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
- // In presence of finite 'safelen', it may be unsafe to mark all
- // the memory instructions parallel, because loop-carried
- // dependences of 'safelen' iterations are possible.
- CGF.LoopStack.setParallel(/*Enable=*/false);
- }
- }
- void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) {
- // Walk clauses and process safelen/lastprivate.
- LoopStack.setParallel(/*Enable=*/true);
- LoopStack.setVectorizeEnable();
- emitSimdlenSafelenClause(*this, D);
- if (const auto *C = D.getSingleClause<OMPOrderClause>())
- if (C->getKind() == OMPC_ORDER_concurrent)
- LoopStack.setParallel(/*Enable=*/true);
- if ((D.getDirectiveKind() == OMPD_simd ||
- (getLangOpts().OpenMPSimd &&
- isOpenMPSimdDirective(D.getDirectiveKind()))) &&
- llvm::any_of(D.getClausesOfKind<OMPReductionClause>(),
- [](const OMPReductionClause *C) {
- return C->getModifier() == OMPC_REDUCTION_inscan;
- }))
- // Disable parallel access in case of prefix sum.
- LoopStack.setParallel(/*Enable=*/false);
- }
- void CodeGenFunction::EmitOMPSimdFinal(
- const OMPLoopDirective &D,
- const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
- if (!HaveInsertPoint())
- return;
- llvm::BasicBlock *DoneBB = nullptr;
- auto IC = D.counters().begin();
- auto IPC = D.private_counters().begin();
- for (const Expr *F : D.finals()) {
- const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
- const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
- const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
- if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
- OrigVD->hasGlobalStorage() || CED) {
- if (!DoneBB) {
- if (llvm::Value *Cond = CondGen(*this)) {
- // If the first post-update expression is found, emit conditional
- // block if it was requested.
- llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
- DoneBB = createBasicBlock(".omp.final.done");
- Builder.CreateCondBr(Cond, ThenBB, DoneBB);
- EmitBlock(ThenBB);
- }
- }
- Address OrigAddr = Address::invalid();
- if (CED) {
- OrigAddr =
- EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress(*this);
- } else {
- DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
- /*RefersToEnclosingVariableOrCapture=*/false,
- (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
- OrigAddr = EmitLValue(&DRE).getAddress(*this);
- }
- OMPPrivateScope VarScope(*this);
- VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
- (void)VarScope.Privatize();
- EmitIgnoredExpr(F);
- }
- ++IC;
- ++IPC;
- }
- if (DoneBB)
- EmitBlock(DoneBB, /*IsFinished=*/true);
- }
- static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
- const OMPLoopDirective &S,
- CodeGenFunction::JumpDest LoopExit) {
- CGF.EmitOMPLoopBody(S, LoopExit);
- CGF.EmitStopPoint(&S);
- }
- /// Emit a helper variable and return corresponding lvalue.
- static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
- const DeclRefExpr *Helper) {
- auto VDecl = cast<VarDecl>(Helper->getDecl());
- CGF.EmitVarDecl(*VDecl);
- return CGF.EmitLValue(Helper);
- }
- static void emitCommonSimdLoop(CodeGenFunction &CGF, const OMPLoopDirective &S,
- const RegionCodeGenTy &SimdInitGen,
- const RegionCodeGenTy &BodyCodeGen) {
- auto &&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](CodeGenFunction &CGF,
- PrePostActionTy &) {
- CGOpenMPRuntime::NontemporalDeclsRAII NontemporalsRegion(CGF.CGM, S);
- CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
- SimdInitGen(CGF);
- BodyCodeGen(CGF);
- };
- auto &&ElseGen = [&BodyCodeGen](CodeGenFunction &CGF, PrePostActionTy &) {
- CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
- CGF.LoopStack.setVectorizeEnable(/*Enable=*/false);
- BodyCodeGen(CGF);
- };
- const Expr *IfCond = nullptr;
- if (isOpenMPSimdDirective(S.getDirectiveKind())) {
- for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
- if (CGF.getLangOpts().OpenMP >= 50 &&
- (C->getNameModifier() == OMPD_unknown ||
- C->getNameModifier() == OMPD_simd)) {
- IfCond = C->getCondition();
- break;
- }
- }
- }
- if (IfCond) {
- CGF.CGM.getOpenMPRuntime().emitIfClause(CGF, IfCond, ThenGen, ElseGen);
- } else {
- RegionCodeGenTy ThenRCG(ThenGen);
- ThenRCG(CGF);
- }
- }
- static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
- "Expected simd directive");
- OMPLoopScope PreInitScope(CGF, S);
- // if (PreCond) {
- // for (IV in 0..LastIteration) BODY;
- // <Final counter/linear vars updates>;
- // }
- //
- if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
- isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
- isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
- (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
- (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
- }
- // Emit: if (PreCond) - begin.
- // If the condition constant folds and can be elided, avoid emitting the
- // whole loop.
- bool CondConstant;
- llvm::BasicBlock *ContBlock = nullptr;
- if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
- if (!CondConstant)
- return;
- } else {
- llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
- ContBlock = CGF.createBasicBlock("simd.if.end");
- emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
- CGF.getProfileCount(&S));
- CGF.EmitBlock(ThenBlock);
- CGF.incrementProfileCounter(&S);
- }
- // Emit the loop iteration variable.
- const Expr *IVExpr = S.getIterationVariable();
- const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
- CGF.EmitVarDecl(*IVDecl);
- CGF.EmitIgnoredExpr(S.getInit());
- // Emit the iterations count variable.
- // If it is not a variable, Sema decided to calculate iterations count on
- // each iteration (e.g., it is foldable into a constant).
- if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
- CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
- // Emit calculation of the iterations count.
- CGF.EmitIgnoredExpr(S.getCalcLastIteration());
- }
- emitAlignedClause(CGF, S);
- (void)CGF.EmitOMPLinearClauseInit(S);
- {
- CodeGenFunction::OMPPrivateScope LoopScope(CGF);
- CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
- CGF.EmitOMPLinearClause(S, LoopScope);
- CGF.EmitOMPPrivateClause(S, LoopScope);
- CGF.EmitOMPReductionClauseInit(S, LoopScope);
- CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
- CGF, S, CGF.EmitLValue(S.getIterationVariable()));
- bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
- (void)LoopScope.Privatize();
- if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
- CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
- emitCommonSimdLoop(
- CGF, S,
- [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPSimdInit(S);
- },
- [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPInnerLoop(
- S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
- [&S](CodeGenFunction &CGF) {
- emitOMPLoopBodyWithStopPoint(CGF, S,
- CodeGenFunction::JumpDest());
- },
- [](CodeGenFunction &) {});
- });
- CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
- // Emit final copy of the lastprivate variables at the end of loops.
- if (HasLastprivateClause)
- CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
- emitPostUpdateForReductionClause(CGF, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
- // Emit: if (PreCond) - end.
- if (ContBlock) {
- CGF.EmitBranch(ContBlock);
- CGF.EmitBlock(ContBlock, true);
- }
- }
- static bool isSupportedByOpenMPIRBuilder(const OMPExecutableDirective &S) {
- // Check for unsupported clauses
- if (!S.clauses().empty()) {
- // Currently no clause is supported
- return false;
- }
- // Check if we have a statement with the ordered directive.
- // Visit the statement hierarchy to find a compound statement
- // with a ordered directive in it.
- if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {
- if (const Stmt *SyntacticalLoop = CanonLoop->getLoopStmt()) {
- for (const Stmt *SubStmt : SyntacticalLoop->children()) {
- if (!SubStmt)
- continue;
- if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(SubStmt)) {
- for (const Stmt *CSSubStmt : CS->children()) {
- if (!CSSubStmt)
- continue;
- if (isa<OMPOrderedDirective>(CSSubStmt)) {
- return false;
- }
- }
- }
- }
- }
- }
- return true;
- }
- void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
- bool UseOMPIRBuilder =
- CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
- if (UseOMPIRBuilder) {
- auto &&CodeGenIRBuilder = [this, &S, UseOMPIRBuilder](CodeGenFunction &CGF,
- PrePostActionTy &) {
- // Use the OpenMPIRBuilder if enabled.
- if (UseOMPIRBuilder) {
- // Emit the associated statement and get its loop representation.
- llvm::DebugLoc DL = SourceLocToDebugLoc(S.getBeginLoc());
- const Stmt *Inner = S.getRawStmt();
- llvm::CanonicalLoopInfo *CLI =
- EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
- llvm::OpenMPIRBuilder &OMPBuilder =
- CGM.getOpenMPRuntime().getOMPBuilder();
- // Add SIMD specific metadata
- OMPBuilder.applySimd(DL, CLI);
- return;
- }
- };
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd,
- CodeGenIRBuilder);
- }
- return;
- }
- ParentLoopDirectiveForScanRegion ScanRegion(*this, S);
- OMPFirstScanLoop = true;
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitOMPSimdRegion(CGF, S, Action);
- };
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- void CodeGenFunction::EmitOMPTileDirective(const OMPTileDirective &S) {
- // Emit the de-sugared statement.
- OMPTransformDirectiveScopeRAII TileScope(*this, &S);
- EmitStmt(S.getTransformedStmt());
- }
- void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {
- bool UseOMPIRBuilder = CGM.getLangOpts().OpenMPIRBuilder;
- if (UseOMPIRBuilder) {
- auto DL = SourceLocToDebugLoc(S.getBeginLoc());
- const Stmt *Inner = S.getRawStmt();
- // Consume nested loop. Clear the entire remaining loop stack because a
- // fully unrolled loop is non-transformable. For partial unrolling the
- // generated outer loop is pushed back to the stack.
- llvm::CanonicalLoopInfo *CLI = EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
- OMPLoopNestStack.clear();
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- bool NeedsUnrolledCLI = ExpectedOMPLoopDepth >= 1;
- llvm::CanonicalLoopInfo *UnrolledCLI = nullptr;
- if (S.hasClausesOfKind<OMPFullClause>()) {
- assert(ExpectedOMPLoopDepth == 0);
- OMPBuilder.unrollLoopFull(DL, CLI);
- } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
- uint64_t Factor = 0;
- if (Expr *FactorExpr = PartialClause->getFactor()) {
- Factor = FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
- assert(Factor >= 1 && "Only positive factors are valid");
- }
- OMPBuilder.unrollLoopPartial(DL, CLI, Factor,
- NeedsUnrolledCLI ? &UnrolledCLI : nullptr);
- } else {
- OMPBuilder.unrollLoopHeuristic(DL, CLI);
- }
- assert((!NeedsUnrolledCLI || UnrolledCLI) &&
- "NeedsUnrolledCLI implies UnrolledCLI to be set");
- if (UnrolledCLI)
- OMPLoopNestStack.push_back(UnrolledCLI);
- return;
- }
- // This function is only called if the unrolled loop is not consumed by any
- // other loop-associated construct. Such a loop-associated construct will have
- // used the transformed AST.
- // Set the unroll metadata for the next emitted loop.
- LoopStack.setUnrollState(LoopAttributes::Enable);
- if (S.hasClausesOfKind<OMPFullClause>()) {
- LoopStack.setUnrollState(LoopAttributes::Full);
- } else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
- if (Expr *FactorExpr = PartialClause->getFactor()) {
- uint64_t Factor =
- FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
- assert(Factor >= 1 && "Only positive factors are valid");
- LoopStack.setUnrollCount(Factor);
- }
- }
- EmitStmt(S.getAssociatedStmt());
- }
- void CodeGenFunction::EmitOMPOuterLoop(
- bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
- CodeGenFunction::OMPPrivateScope &LoopScope,
- const CodeGenFunction::OMPLoopArguments &LoopArgs,
- const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
- const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
- CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
- const Expr *IVExpr = S.getIterationVariable();
- const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
- const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
- JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
- // Start the loop with a block that tests the condition.
- llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
- EmitBlock(CondBlock);
- const SourceRange R = S.getSourceRange();
- OMPLoopNestStack.clear();
- LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
- SourceLocToDebugLoc(R.getEnd()));
- llvm::Value *BoolCondVal = nullptr;
- if (!DynamicOrOrdered) {
- // UB = min(UB, GlobalUB) or
- // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
- // 'distribute parallel for')
- EmitIgnoredExpr(LoopArgs.EUB);
- // IV = LB
- EmitIgnoredExpr(LoopArgs.Init);
- // IV < UB
- BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
- } else {
- BoolCondVal =
- RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
- LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
- }
- // If there are any cleanups between here and the loop-exit scope,
- // create a block to stage a loop exit along.
- llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
- if (LoopScope.requiresCleanups())
- ExitBlock = createBasicBlock("omp.dispatch.cleanup");
- llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
- Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
- if (ExitBlock != LoopExit.getBlock()) {
- EmitBlock(ExitBlock);
- EmitBranchThroughCleanup(LoopExit);
- }
- EmitBlock(LoopBody);
- // Emit "IV = LB" (in case of static schedule, we have already calculated new
- // LB for loop condition and emitted it above).
- if (DynamicOrOrdered)
- EmitIgnoredExpr(LoopArgs.Init);
- // Create a block for the increment.
- JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
- BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
- emitCommonSimdLoop(
- *this, S,
- [&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
- // Generate !llvm.loop.parallel metadata for loads and stores for loops
- // with dynamic/guided scheduling and without ordered clause.
- if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
- CGF.LoopStack.setParallel(!IsMonotonic);
- if (const auto *C = S.getSingleClause<OMPOrderClause>())
- if (C->getKind() == OMPC_ORDER_concurrent)
- CGF.LoopStack.setParallel(/*Enable=*/true);
- } else {
- CGF.EmitOMPSimdInit(S);
- }
- },
- [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
- &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
- SourceLocation Loc = S.getBeginLoc();
- // when 'distribute' is not combined with a 'for':
- // while (idx <= UB) { BODY; ++idx; }
- // when 'distribute' is combined with a 'for'
- // (e.g. 'distribute parallel for')
- // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
- CGF.EmitOMPInnerLoop(
- S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
- [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
- CodeGenLoop(CGF, S, LoopExit);
- },
- [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
- CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
- });
- });
- EmitBlock(Continue.getBlock());
- BreakContinueStack.pop_back();
- if (!DynamicOrOrdered) {
- // Emit "LB = LB + Stride", "UB = UB + Stride".
- EmitIgnoredExpr(LoopArgs.NextLB);
- EmitIgnoredExpr(LoopArgs.NextUB);
- }
- EmitBranch(CondBlock);
- OMPLoopNestStack.clear();
- LoopStack.pop();
- // Emit the fall-through block.
- EmitBlock(LoopExit.getBlock());
- // Tell the runtime we are done.
- auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
- if (!DynamicOrOrdered)
- CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
- S.getDirectiveKind());
- };
- OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
- }
- void CodeGenFunction::EmitOMPForOuterLoop(
- const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
- const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
- const OMPLoopArguments &LoopArgs,
- const CodeGenDispatchBoundsTy &CGDispatchBounds) {
- CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
- // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
- const bool DynamicOrOrdered = Ordered || RT.isDynamic(ScheduleKind.Schedule);
- assert((Ordered || !RT.isStaticNonchunked(ScheduleKind.Schedule,
- LoopArgs.Chunk != nullptr)) &&
- "static non-chunked schedule does not need outer loop");
- // Emit outer loop.
- //
- // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
- // When schedule(dynamic,chunk_size) is specified, the iterations are
- // distributed to threads in the team in chunks as the threads request them.
- // Each thread executes a chunk of iterations, then requests another chunk,
- // until no chunks remain to be distributed. Each chunk contains chunk_size
- // iterations, except for the last chunk to be distributed, which may have
- // fewer iterations. When no chunk_size is specified, it defaults to 1.
- //
- // When schedule(guided,chunk_size) is specified, the iterations are assigned
- // to threads in the team in chunks as the executing threads request them.
- // Each thread executes a chunk of iterations, then requests another chunk,
- // until no chunks remain to be assigned. For a chunk_size of 1, the size of
- // each chunk is proportional to the number of unassigned iterations divided
- // by the number of threads in the team, decreasing to 1. For a chunk_size
- // with value k (greater than 1), the size of each chunk is determined in the
- // same way, with the restriction that the chunks do not contain fewer than k
- // iterations (except for the last chunk to be assigned, which may have fewer
- // than k iterations).
- //
- // When schedule(auto) is specified, the decision regarding scheduling is
- // delegated to the compiler and/or runtime system. The programmer gives the
- // implementation the freedom to choose any possible mapping of iterations to
- // threads in the team.
- //
- // When schedule(runtime) is specified, the decision regarding scheduling is
- // deferred until run time, and the schedule and chunk size are taken from the
- // run-sched-var ICV. If the ICV is set to auto, the schedule is
- // implementation defined
- //
- // while(__kmpc_dispatch_next(&LB, &UB)) {
- // idx = LB;
- // while (idx <= UB) { BODY; ++idx;
- // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
- // } // inner loop
- // }
- //
- // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
- // When schedule(static, chunk_size) is specified, iterations are divided into
- // chunks of size chunk_size, and the chunks are assigned to the threads in
- // the team in a round-robin fashion in the order of the thread number.
- //
- // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
- // while (idx <= UB) { BODY; ++idx; } // inner loop
- // LB = LB + ST;
- // UB = UB + ST;
- // }
- //
- const Expr *IVExpr = S.getIterationVariable();
- const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
- const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
- if (DynamicOrOrdered) {
- const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
- CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
- llvm::Value *LBVal = DispatchBounds.first;
- llvm::Value *UBVal = DispatchBounds.second;
- CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
- LoopArgs.Chunk};
- RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
- IVSigned, Ordered, DipatchRTInputValues);
- } else {
- CGOpenMPRuntime::StaticRTInput StaticInit(
- IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
- LoopArgs.ST, LoopArgs.Chunk);
- RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
- ScheduleKind, StaticInit);
- }
- auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
- const unsigned IVSize,
- const bool IVSigned) {
- if (Ordered) {
- CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
- IVSigned);
- }
- };
- OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
- LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
- OuterLoopArgs.IncExpr = S.getInc();
- OuterLoopArgs.Init = S.getInit();
- OuterLoopArgs.Cond = S.getCond();
- OuterLoopArgs.NextLB = S.getNextLowerBound();
- OuterLoopArgs.NextUB = S.getNextUpperBound();
- EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
- emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
- }
- static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
- const unsigned IVSize, const bool IVSigned) {}
- void CodeGenFunction::EmitOMPDistributeOuterLoop(
- OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
- OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
- const CodeGenLoopTy &CodeGenLoopContent) {
- CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
- // Emit outer loop.
- // Same behavior as a OMPForOuterLoop, except that schedule cannot be
- // dynamic
- //
- const Expr *IVExpr = S.getIterationVariable();
- const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
- const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
- CGOpenMPRuntime::StaticRTInput StaticInit(
- IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
- LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
- RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
- // for combined 'distribute' and 'for' the increment expression of distribute
- // is stored in DistInc. For 'distribute' alone, it is in Inc.
- Expr *IncExpr;
- if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
- IncExpr = S.getDistInc();
- else
- IncExpr = S.getInc();
- // this routine is shared by 'omp distribute parallel for' and
- // 'omp distribute': select the right EUB expression depending on the
- // directive
- OMPLoopArguments OuterLoopArgs;
- OuterLoopArgs.LB = LoopArgs.LB;
- OuterLoopArgs.UB = LoopArgs.UB;
- OuterLoopArgs.ST = LoopArgs.ST;
- OuterLoopArgs.IL = LoopArgs.IL;
- OuterLoopArgs.Chunk = LoopArgs.Chunk;
- OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedEnsureUpperBound()
- : S.getEnsureUpperBound();
- OuterLoopArgs.IncExpr = IncExpr;
- OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedInit()
- : S.getInit();
- OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedCond()
- : S.getCond();
- OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedNextLowerBound()
- : S.getNextLowerBound();
- OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedNextUpperBound()
- : S.getNextUpperBound();
- EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
- LoopScope, OuterLoopArgs, CodeGenLoopContent,
- emitEmptyOrdered);
- }
- static std::pair<LValue, LValue>
- emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
- const OMPExecutableDirective &S) {
- const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
- LValue LB =
- EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
- LValue UB =
- EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
- // When composing 'distribute' with 'for' (e.g. as in 'distribute
- // parallel for') we need to use the 'distribute'
- // chunk lower and upper bounds rather than the whole loop iteration
- // space. These are parameters to the outlined function for 'parallel'
- // and we copy the bounds of the previous schedule into the
- // the current ones.
- LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
- LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
- llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
- PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
- PrevLBVal = CGF.EmitScalarConversion(
- PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
- LS.getIterationVariable()->getType(),
- LS.getPrevLowerBoundVariable()->getExprLoc());
- llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
- PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
- PrevUBVal = CGF.EmitScalarConversion(
- PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
- LS.getIterationVariable()->getType(),
- LS.getPrevUpperBoundVariable()->getExprLoc());
- CGF.EmitStoreOfScalar(PrevLBVal, LB);
- CGF.EmitStoreOfScalar(PrevUBVal, UB);
- return {LB, UB};
- }
- /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
- /// we need to use the LB and UB expressions generated by the worksharing
- /// code generation support, whereas in non combined situations we would
- /// just emit 0 and the LastIteration expression
- /// This function is necessary due to the difference of the LB and UB
- /// types for the RT emission routines for 'for_static_init' and
- /// 'for_dispatch_init'
- static std::pair<llvm::Value *, llvm::Value *>
- emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
- const OMPExecutableDirective &S,
- Address LB, Address UB) {
- const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
- const Expr *IVExpr = LS.getIterationVariable();
- // when implementing a dynamic schedule for a 'for' combined with a
- // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
- // is not normalized as each team only executes its own assigned
- // distribute chunk
- QualType IteratorTy = IVExpr->getType();
- llvm::Value *LBVal =
- CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
- llvm::Value *UBVal =
- CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
- return {LBVal, UBVal};
- }
- static void emitDistributeParallelForDistributeInnerBoundParams(
- CodeGenFunction &CGF, const OMPExecutableDirective &S,
- llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
- const auto &Dir = cast<OMPLoopDirective>(S);
- LValue LB =
- CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
- llvm::Value *LBCast =
- CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(LB.getAddress(CGF)),
- CGF.SizeTy, /*isSigned=*/false);
- CapturedVars.push_back(LBCast);
- LValue UB =
- CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
- llvm::Value *UBCast =
- CGF.Builder.CreateIntCast(CGF.Builder.CreateLoad(UB.getAddress(CGF)),
- CGF.SizeTy, /*isSigned=*/false);
- CapturedVars.push_back(UBCast);
- }
- static void
- emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
- const OMPLoopDirective &S,
- CodeGenFunction::JumpDest LoopExit) {
- auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- bool HasCancel = false;
- if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
- if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
- HasCancel = D->hasCancel();
- else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
- HasCancel = D->hasCancel();
- else if (const auto *D =
- dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
- HasCancel = D->hasCancel();
- }
- CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
- HasCancel);
- CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
- emitDistributeParallelForInnerBounds,
- emitDistributeParallelForDispatchBounds);
- };
- emitCommonOMPParallelDirective(
- CGF, S,
- isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
- CGInlinedWorksharingLoop,
- emitDistributeParallelForDistributeInnerBoundParams);
- }
- void CodeGenFunction::EmitOMPDistributeParallelForDirective(
- const OMPDistributeParallelForDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
- S.getDistInc());
- };
- OMPLexicalScope Scope(*this, S, OMPD_parallel);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
- }
- void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
- const OMPDistributeParallelForSimdDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
- S.getDistInc());
- };
- OMPLexicalScope Scope(*this, S, OMPD_parallel);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
- }
- void CodeGenFunction::EmitOMPDistributeSimdDirective(
- const OMPDistributeSimdDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
- };
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
- }
- void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
- // Emit SPMD target parallel for region as a standalone region.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitOMPSimdRegion(CGF, S, Action);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetSimdDirective(
- const OMPTargetSimdDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitOMPSimdRegion(CGF, S, Action);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- namespace {
- struct ScheduleKindModifiersTy {
- OpenMPScheduleClauseKind Kind;
- OpenMPScheduleClauseModifier M1;
- OpenMPScheduleClauseModifier M2;
- ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
- OpenMPScheduleClauseModifier M1,
- OpenMPScheduleClauseModifier M2)
- : Kind(Kind), M1(M1), M2(M2) {}
- };
- } // namespace
- bool CodeGenFunction::EmitOMPWorksharingLoop(
- const OMPLoopDirective &S, Expr *EUB,
- const CodeGenLoopBoundsTy &CodeGenLoopBounds,
- const CodeGenDispatchBoundsTy &CGDispatchBounds) {
- // Emit the loop iteration variable.
- const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
- const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
- EmitVarDecl(*IVDecl);
- // Emit the iterations count variable.
- // If it is not a variable, Sema decided to calculate iterations count on each
- // iteration (e.g., it is foldable into a constant).
- if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
- EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
- // Emit calculation of the iterations count.
- EmitIgnoredExpr(S.getCalcLastIteration());
- }
- CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
- bool HasLastprivateClause;
- // Check pre-condition.
- {
- OMPLoopScope PreInitScope(*this, S);
- // Skip the entire loop if we don't meet the precondition.
- // If the condition constant folds and can be elided, avoid emitting the
- // whole loop.
- bool CondConstant;
- llvm::BasicBlock *ContBlock = nullptr;
- if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
- if (!CondConstant)
- return false;
- } else {
- llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
- ContBlock = createBasicBlock("omp.precond.end");
- emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
- getProfileCount(&S));
- EmitBlock(ThenBlock);
- incrementProfileCounter(&S);
- }
- RunCleanupsScope DoacrossCleanupScope(*this);
- bool Ordered = false;
- if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
- if (OrderedClause->getNumForLoops())
- RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
- else
- Ordered = true;
- }
- llvm::DenseSet<const Expr *> EmittedFinals;
- emitAlignedClause(*this, S);
- bool HasLinears = EmitOMPLinearClauseInit(S);
- // Emit helper vars inits.
- std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
- LValue LB = Bounds.first;
- LValue UB = Bounds.second;
- LValue ST =
- EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
- LValue IL =
- EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
- // Emit 'then' code.
- {
- OMPPrivateScope LoopScope(*this);
- if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
- // Emit implicit barrier to synchronize threads and avoid data races on
- // initialization of firstprivate variables and post-update of
- // lastprivate variables.
- CGM.getOpenMPRuntime().emitBarrierCall(
- *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
- }
- EmitOMPPrivateClause(S, LoopScope);
- CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(
- *this, S, EmitLValue(S.getIterationVariable()));
- HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
- EmitOMPReductionClauseInit(S, LoopScope);
- EmitOMPPrivateLoopCounters(S, LoopScope);
- EmitOMPLinearClause(S, LoopScope);
- (void)LoopScope.Privatize();
- if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
- CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
- // Detect the loop schedule kind and chunk.
- const Expr *ChunkExpr = nullptr;
- OpenMPScheduleTy ScheduleKind;
- if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
- ScheduleKind.Schedule = C->getScheduleKind();
- ScheduleKind.M1 = C->getFirstScheduleModifier();
- ScheduleKind.M2 = C->getSecondScheduleModifier();
- ChunkExpr = C->getChunkSize();
- } else {
- // Default behaviour for schedule clause.
- CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
- *this, S, ScheduleKind.Schedule, ChunkExpr);
- }
- bool HasChunkSizeOne = false;
- llvm::Value *Chunk = nullptr;
- if (ChunkExpr) {
- Chunk = EmitScalarExpr(ChunkExpr);
- Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
- S.getIterationVariable()->getType(),
- S.getBeginLoc());
- Expr::EvalResult Result;
- if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
- llvm::APSInt EvaluatedChunk = Result.Val.getInt();
- HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
- }
- }
- const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
- const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
- // OpenMP 4.5, 2.7.1 Loop Construct, Description.
- // If the static schedule kind is specified or if the ordered clause is
- // specified, and if no monotonic modifier is specified, the effect will
- // be as if the monotonic modifier was specified.
- bool StaticChunkedOne =
- RT.isStaticChunked(ScheduleKind.Schedule,
- /* Chunked */ Chunk != nullptr) &&
- HasChunkSizeOne &&
- isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
- bool IsMonotonic =
- Ordered ||
- (ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
- !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
- ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
- ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
- ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
- if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
- /* Chunked */ Chunk != nullptr) ||
- StaticChunkedOne) &&
- !Ordered) {
- JumpDest LoopExit =
- getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
- emitCommonSimdLoop(
- *this, S,
- [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- if (isOpenMPSimdDirective(S.getDirectiveKind())) {
- CGF.EmitOMPSimdInit(S);
- } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
- if (C->getKind() == OMPC_ORDER_concurrent)
- CGF.LoopStack.setParallel(/*Enable=*/true);
- }
- },
- [IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,
- &S, ScheduleKind, LoopExit,
- &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
- // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
- // When no chunk_size is specified, the iteration space is divided
- // into chunks that are approximately equal in size, and at most
- // one chunk is distributed to each thread. Note that the size of
- // the chunks is unspecified in this case.
- CGOpenMPRuntime::StaticRTInput StaticInit(
- IVSize, IVSigned, Ordered, IL.getAddress(CGF),
- LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF),
- StaticChunkedOne ? Chunk : nullptr);
- CGF.CGM.getOpenMPRuntime().emitForStaticInit(
- CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind,
- StaticInit);
- // UB = min(UB, GlobalUB);
- if (!StaticChunkedOne)
- CGF.EmitIgnoredExpr(S.getEnsureUpperBound());
- // IV = LB;
- CGF.EmitIgnoredExpr(S.getInit());
- // For unchunked static schedule generate:
- //
- // while (idx <= UB) {
- // BODY;
- // ++idx;
- // }
- //
- // For static schedule with chunk one:
- //
- // while (IV <= PrevUB) {
- // BODY;
- // IV += ST;
- // }
- CGF.EmitOMPInnerLoop(
- S, LoopScope.requiresCleanups(),
- StaticChunkedOne ? S.getCombinedParForInDistCond()
- : S.getCond(),
- StaticChunkedOne ? S.getDistInc() : S.getInc(),
- [&S, LoopExit](CodeGenFunction &CGF) {
- emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);
- },
- [](CodeGenFunction &) {});
- });
- EmitBlock(LoopExit.getBlock());
- // Tell the runtime we are done.
- auto &&CodeGen = [&S](CodeGenFunction &CGF) {
- CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
- S.getDirectiveKind());
- };
- OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
- } else {
- // Emit the outer loop, which requests its work chunk [LB..UB] from
- // runtime and runs the inner loop to process it.
- const OMPLoopArguments LoopArguments(
- LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
- IL.getAddress(*this), Chunk, EUB);
- EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
- LoopArguments, CGDispatchBounds);
- }
- if (isOpenMPSimdDirective(S.getDirectiveKind())) {
- EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
- return CGF.Builder.CreateIsNotNull(
- CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
- });
- }
- EmitOMPReductionClauseFinal(
- S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
- ? /*Parallel and Simd*/ OMPD_parallel_for_simd
- : /*Parallel only*/ OMPD_parallel);
- // Emit post-update of the reduction variables if IsLastIter != 0.
- emitPostUpdateForReductionClause(
- *this, S, [IL, &S](CodeGenFunction &CGF) {
- return CGF.Builder.CreateIsNotNull(
- CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
- });
- // Emit final copy of the lastprivate variables if IsLastIter != 0.
- if (HasLastprivateClause)
- EmitOMPLastprivateClauseFinal(
- S, isOpenMPSimdDirective(S.getDirectiveKind()),
- Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
- }
- EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
- return CGF.Builder.CreateIsNotNull(
- CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
- });
- DoacrossCleanupScope.ForceCleanup();
- // We're now done with the loop, so jump to the continuation block.
- if (ContBlock) {
- EmitBranch(ContBlock);
- EmitBlock(ContBlock, /*IsFinished=*/true);
- }
- }
- return HasLastprivateClause;
- }
- /// The following two functions generate expressions for the loop lower
- /// and upper bounds in case of static and dynamic (dispatch) schedule
- /// of the associated 'for' or 'distribute' loop.
- static std::pair<LValue, LValue>
- emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
- const auto &LS = cast<OMPLoopDirective>(S);
- LValue LB =
- EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
- LValue UB =
- EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
- return {LB, UB};
- }
- /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
- /// consider the lower and upper bound expressions generated by the
- /// worksharing loop support, but we use 0 and the iteration space size as
- /// constants
- static std::pair<llvm::Value *, llvm::Value *>
- emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
- Address LB, Address UB) {
- const auto &LS = cast<OMPLoopDirective>(S);
- const Expr *IVExpr = LS.getIterationVariable();
- const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
- llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
- llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
- return {LBVal, UBVal};
- }
- /// Emits internal temp array declarations for the directive with inscan
- /// reductions.
- /// The code is the following:
- /// \code
- /// size num_iters = <num_iters>;
- /// <type> buffer[num_iters];
- /// \endcode
- static void emitScanBasedDirectiveDecls(
- CodeGenFunction &CGF, const OMPLoopDirective &S,
- llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen) {
- llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
- NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
- SmallVector<const Expr *, 4> Shareds;
- SmallVector<const Expr *, 4> Privates;
- SmallVector<const Expr *, 4> ReductionOps;
- SmallVector<const Expr *, 4> CopyArrayTemps;
- for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
- assert(C->getModifier() == OMPC_REDUCTION_inscan &&
- "Only inscan reductions are expected.");
- Shareds.append(C->varlist_begin(), C->varlist_end());
- Privates.append(C->privates().begin(), C->privates().end());
- ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
- CopyArrayTemps.append(C->copy_array_temps().begin(),
- C->copy_array_temps().end());
- }
- {
- // Emit buffers for each reduction variables.
- // ReductionCodeGen is required to emit correctly the code for array
- // reductions.
- ReductionCodeGen RedCG(Shareds, Shareds, Privates, ReductionOps);
- unsigned Count = 0;
- auto *ITA = CopyArrayTemps.begin();
- for (const Expr *IRef : Privates) {
- const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
- // Emit variably modified arrays, used for arrays/array sections
- // reductions.
- if (PrivateVD->getType()->isVariablyModifiedType()) {
- RedCG.emitSharedOrigLValue(CGF, Count);
- RedCG.emitAggregateType(CGF, Count);
- }
- CodeGenFunction::OpaqueValueMapping DimMapping(
- CGF,
- cast<OpaqueValueExpr>(
- cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())
- ->getSizeExpr()),
- RValue::get(OMPScanNumIterations));
- // Emit temp buffer.
- CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));
- ++ITA;
- ++Count;
- }
- }
- }
- /// Emits the code for the directive with inscan reductions.
- /// The code is the following:
- /// \code
- /// #pragma omp ...
- /// for (i: 0..<num_iters>) {
- /// <input phase>;
- /// buffer[i] = red;
- /// }
- /// #pragma omp master // in parallel region
- /// for (int k = 0; k != ceil(log2(num_iters)); ++k)
- /// for (size cnt = last_iter; cnt >= pow(2, k); --k)
- /// buffer[i] op= buffer[i-pow(2,k)];
- /// #pragma omp barrier // in parallel region
- /// #pragma omp ...
- /// for (0..<num_iters>) {
- /// red = InclusiveScan ? buffer[i] : buffer[i-1];
- /// <scan phase>;
- /// }
- /// \endcode
- static void emitScanBasedDirective(
- CodeGenFunction &CGF, const OMPLoopDirective &S,
- llvm::function_ref<llvm::Value *(CodeGenFunction &)> NumIteratorsGen,
- llvm::function_ref<void(CodeGenFunction &)> FirstGen,
- llvm::function_ref<void(CodeGenFunction &)> SecondGen) {
- llvm::Value *OMPScanNumIterations = CGF.Builder.CreateIntCast(
- NumIteratorsGen(CGF), CGF.SizeTy, /*isSigned=*/false);
- SmallVector<const Expr *, 4> Privates;
- SmallVector<const Expr *, 4> ReductionOps;
- SmallVector<const Expr *, 4> LHSs;
- SmallVector<const Expr *, 4> RHSs;
- SmallVector<const Expr *, 4> CopyArrayElems;
- for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
- assert(C->getModifier() == OMPC_REDUCTION_inscan &&
- "Only inscan reductions are expected.");
- Privates.append(C->privates().begin(), C->privates().end());
- ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
- LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
- RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
- CopyArrayElems.append(C->copy_array_elems().begin(),
- C->copy_array_elems().end());
- }
- CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);
- {
- // Emit loop with input phase:
- // #pragma omp ...
- // for (i: 0..<num_iters>) {
- // <input phase>;
- // buffer[i] = red;
- // }
- CGF.OMPFirstScanLoop = true;
- CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
- FirstGen(CGF);
- }
- // #pragma omp barrier // in parallel region
- auto &&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,
- &ReductionOps,
- &Privates](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- // Emit prefix reduction:
- // #pragma omp master // in parallel region
- // for (int k = 0; k <= ceil(log2(n)); ++k)
- llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();
- llvm::BasicBlock *LoopBB = CGF.createBasicBlock("omp.outer.log.scan.body");
- llvm::BasicBlock *ExitBB = CGF.createBasicBlock("omp.outer.log.scan.exit");
- llvm::Function *F =
- CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);
- llvm::Value *Arg =
- CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);
- llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);
- F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);
- LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);
- LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);
- llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(
- OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));
- auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getBeginLoc());
- CGF.EmitBlock(LoopBB);
- auto *Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);
- // size pow2k = 1;
- auto *Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
- Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);
- Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);
- // for (size i = n - 1; i >= 2 ^ k; --i)
- // tmp[i] op= tmp[i-pow2k];
- llvm::BasicBlock *InnerLoopBB =
- CGF.createBasicBlock("omp.inner.log.scan.body");
- llvm::BasicBlock *InnerExitBB =
- CGF.createBasicBlock("omp.inner.log.scan.exit");
- llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);
- CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
- CGF.EmitBlock(InnerLoopBB);
- auto *IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);
- IVal->addIncoming(NMin1, LoopBB);
- {
- CodeGenFunction::OMPPrivateScope PrivScope(CGF);
- auto *ILHS = LHSs.begin();
- auto *IRHS = RHSs.begin();
- for (const Expr *CopyArrayElem : CopyArrayElems) {
- const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
- const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
- Address LHSAddr = Address::invalid();
- {
- CodeGenFunction::OpaqueValueMapping IdxMapping(
- CGF,
- cast<OpaqueValueExpr>(
- cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
- RValue::get(IVal));
- LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
- }
- PrivScope.addPrivate(LHSVD, [LHSAddr]() { return LHSAddr; });
- Address RHSAddr = Address::invalid();
- {
- llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);
- CodeGenFunction::OpaqueValueMapping IdxMapping(
- CGF,
- cast<OpaqueValueExpr>(
- cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
- RValue::get(OffsetIVal));
- RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress(CGF);
- }
- PrivScope.addPrivate(RHSVD, [RHSAddr]() { return RHSAddr; });
- ++ILHS;
- ++IRHS;
- }
- PrivScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitReduction(
- CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
- {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_unknown});
- }
- llvm::Value *NextIVal =
- CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));
- IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());
- CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);
- CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
- CGF.EmitBlock(InnerExitBB);
- llvm::Value *Next =
- CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));
- Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());
- // pow2k <<= 1;
- llvm::Value *NextPow2K =
- CGF.Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
- Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());
- llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);
- CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
- auto DL1 = ApplyDebugLocation::CreateDefaultArtificial(CGF, S.getEndLoc());
- CGF.EmitBlock(ExitBB);
- };
- if (isOpenMPParallelDirective(S.getDirectiveKind())) {
- CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(
- CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
- } else {
- RegionCodeGenTy RCG(CodeGen);
- RCG(CGF);
- }
- CGF.OMPFirstScanLoop = false;
- SecondGen(CGF);
- }
- static bool emitWorksharingDirective(CodeGenFunction &CGF,
- const OMPLoopDirective &S,
- bool HasCancel) {
- bool HasLastprivates;
- if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
- [](const OMPReductionClause *C) {
- return C->getModifier() == OMPC_REDUCTION_inscan;
- })) {
- const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
- CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
- OMPLoopScope LoopScope(CGF, S);
- return CGF.EmitScalarExpr(S.getNumIterations());
- };
- const auto &&FirstGen = [&S, HasCancel](CodeGenFunction &CGF) {
- CodeGenFunction::OMPCancelStackRAII CancelRegion(
- CGF, S.getDirectiveKind(), HasCancel);
- (void)CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
- emitForLoopBounds,
- emitDispatchForLoopBounds);
- // Emit an implicit barrier at the end.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(CGF, S.getBeginLoc(),
- OMPD_for);
- };
- const auto &&SecondGen = [&S, HasCancel,
- &HasLastprivates](CodeGenFunction &CGF) {
- CodeGenFunction::OMPCancelStackRAII CancelRegion(
- CGF, S.getDirectiveKind(), HasCancel);
- HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
- emitForLoopBounds,
- emitDispatchForLoopBounds);
- };
- if (!isOpenMPParallelDirective(S.getDirectiveKind()))
- emitScanBasedDirectiveDecls(CGF, S, NumIteratorsGen);
- emitScanBasedDirective(CGF, S, NumIteratorsGen, FirstGen, SecondGen);
- } else {
- CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
- HasCancel);
- HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
- emitForLoopBounds,
- emitDispatchForLoopBounds);
- }
- return HasLastprivates;
- }
- static bool isSupportedByOpenMPIRBuilder(const OMPForDirective &S) {
- if (S.hasCancel())
- return false;
- for (OMPClause *C : S.clauses())
- if (!isa<OMPNowaitClause>(C))
- return false;
- return true;
- }
- void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
- bool HasLastprivates = false;
- bool UseOMPIRBuilder =
- CGM.getLangOpts().OpenMPIRBuilder && isSupportedByOpenMPIRBuilder(S);
- auto &&CodeGen = [this, &S, &HasLastprivates,
- UseOMPIRBuilder](CodeGenFunction &CGF, PrePostActionTy &) {
- // Use the OpenMPIRBuilder if enabled.
- if (UseOMPIRBuilder) {
- // Emit the associated statement and get its loop representation.
- const Stmt *Inner = S.getRawStmt();
- llvm::CanonicalLoopInfo *CLI =
- EmitOMPCollapsedCanonicalLoopNest(Inner, 1);
- bool NeedsBarrier = !S.getSingleClause<OMPNowaitClause>();
- llvm::OpenMPIRBuilder &OMPBuilder =
- CGM.getOpenMPRuntime().getOMPBuilder();
- llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
- AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
- OMPBuilder.applyWorkshareLoop(Builder.getCurrentDebugLocation(), CLI,
- AllocaIP, NeedsBarrier);
- return;
- }
- HasLastprivates = emitWorksharingDirective(CGF, S, S.hasCancel());
- };
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
- S.hasCancel());
- }
- if (!UseOMPIRBuilder) {
- // Emit an implicit barrier at the end.
- if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
- CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
- bool HasLastprivates = false;
- auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
- PrePostActionTy &) {
- HasLastprivates = emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
- };
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
- }
- // Emit an implicit barrier at the end.
- if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
- CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
- const Twine &Name,
- llvm::Value *Init = nullptr) {
- LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
- if (Init)
- CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
- return LVal;
- }
- void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
- const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
- const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
- bool HasLastprivates = false;
- auto &&CodeGen = [&S, CapturedStmt, CS,
- &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
- const ASTContext &C = CGF.getContext();
- QualType KmpInt32Ty =
- C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
- // Emit helper vars inits.
- LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
- CGF.Builder.getInt32(0));
- llvm::ConstantInt *GlobalUBVal = CS != nullptr
- ? CGF.Builder.getInt32(CS->size() - 1)
- : CGF.Builder.getInt32(0);
- LValue UB =
- createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
- LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
- CGF.Builder.getInt32(1));
- LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
- CGF.Builder.getInt32(0));
- // Loop counter.
- LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
- OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
- CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
- OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
- CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
- // Generate condition for loop.
- BinaryOperator *Cond = BinaryOperator::Create(
- C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,
- S.getBeginLoc(), FPOptionsOverride());
- // Increment for loop counter.
- UnaryOperator *Inc = UnaryOperator::Create(
- C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,
- S.getBeginLoc(), true, FPOptionsOverride());
- auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
- // Iterate through all sections and emit a switch construct:
- // switch (IV) {
- // case 0:
- // <SectionStmt[0]>;
- // break;
- // ...
- // case <NumSection> - 1:
- // <SectionStmt[<NumSection> - 1]>;
- // break;
- // }
- // .omp.sections.exit:
- llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
- llvm::SwitchInst *SwitchStmt =
- CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
- ExitBB, CS == nullptr ? 1 : CS->size());
- if (CS) {
- unsigned CaseNumber = 0;
- for (const Stmt *SubStmt : CS->children()) {
- auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
- CGF.EmitBlock(CaseBB);
- SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
- CGF.EmitStmt(SubStmt);
- CGF.EmitBranch(ExitBB);
- ++CaseNumber;
- }
- } else {
- llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
- CGF.EmitBlock(CaseBB);
- SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
- CGF.EmitStmt(CapturedStmt);
- CGF.EmitBranch(ExitBB);
- }
- CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
- };
- CodeGenFunction::OMPPrivateScope LoopScope(CGF);
- if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
- // Emit implicit barrier to synchronize threads and avoid data races on
- // initialization of firstprivate variables and post-update of lastprivate
- // variables.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(
- CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
- }
- CGF.EmitOMPPrivateClause(S, LoopScope);
- CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion(CGF, S, IV);
- HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
- CGF.EmitOMPReductionClauseInit(S, LoopScope);
- (void)LoopScope.Privatize();
- if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
- CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
- // Emit static non-chunked loop.
- OpenMPScheduleTy ScheduleKind;
- ScheduleKind.Schedule = OMPC_SCHEDULE_static;
- CGOpenMPRuntime::StaticRTInput StaticInit(
- /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(CGF),
- LB.getAddress(CGF), UB.getAddress(CGF), ST.getAddress(CGF));
- CGF.CGM.getOpenMPRuntime().emitForStaticInit(
- CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
- // UB = min(UB, GlobalUB);
- llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
- llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
- CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
- CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
- // IV = LB;
- CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
- // while (idx <= UB) { BODY; ++idx; }
- CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, Cond, Inc, BodyGen,
- [](CodeGenFunction &) {});
- // Tell the runtime we are done.
- auto &&CodeGen = [&S](CodeGenFunction &CGF) {
- CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
- S.getDirectiveKind());
- };
- CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
- // Emit post-update of the reduction variables if IsLastIter != 0.
- emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
- return CGF.Builder.CreateIsNotNull(
- CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
- });
- // Emit final copy of the lastprivate variables if IsLastIter != 0.
- if (HasLastprivates)
- CGF.EmitOMPLastprivateClauseFinal(
- S, /*NoFinals=*/false,
- CGF.Builder.CreateIsNotNull(
- CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
- };
- bool HasCancel = false;
- if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
- HasCancel = OSD->hasCancel();
- else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
- HasCancel = OPSD->hasCancel();
- OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
- HasCancel);
- // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
- // clause. Otherwise the barrier will be generated by the codegen for the
- // directive.
- if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
- // Emit implicit barrier to synchronize threads and avoid data races on
- // initialization of firstprivate variables.
- CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
- OMPD_unknown);
- }
- }
- void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
- if (CGM.getLangOpts().OpenMPIRBuilder) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
- auto FiniCB = [this](InsertPointTy IP) {
- OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
- };
- const CapturedStmt *ICS = S.getInnermostCapturedStmt();
- const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
- const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
- llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
- if (CS) {
- for (const Stmt *SubStmt : CS->children()) {
- auto SectionCB = [this, SubStmt](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP,
- FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, SubStmt, CodeGenIP,
- FiniBB);
- };
- SectionCBVector.push_back(SectionCB);
- }
- } else {
- auto SectionCB = [this, CapturedStmt](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, CapturedStmt, CodeGenIP,
- FiniBB);
- };
- SectionCBVector.push_back(SectionCB);
- }
- // Privatization callback that performs appropriate action for
- // shared/private/firstprivate/lastprivate/copyin/... variables.
- //
- // TODO: This defaults to shared right now.
- auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
- llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
- // The next line is appropriate only for variables (Val) with the
- // data-sharing attribute "shared".
- ReplVal = &Val;
- return CodeGenIP;
- };
- CGCapturedStmtInfo CGSI(*ICS, CR_OpenMP);
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*this, &CGSI);
- llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
- AllocaInsertPt->getParent(), AllocaInsertPt->getIterator());
- Builder.restoreIP(OMPBuilder.createSections(
- Builder, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),
- S.getSingleClause<OMPNowaitClause>()));
- return;
- }
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- EmitSections(S);
- }
- // Emit an implicit barrier at the end.
- if (!S.getSingleClause<OMPNowaitClause>()) {
- CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
- OMPD_sections);
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
- if (CGM.getLangOpts().OpenMPIRBuilder) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- const Stmt *SectionRegionBodyStmt = S.getAssociatedStmt();
- auto FiniCB = [this](InsertPointTy IP) {
- OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
- };
- auto BodyGenCB = [SectionRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, SectionRegionBodyStmt,
- CodeGenIP, FiniBB);
- };
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(&S);
- Builder.restoreIP(OMPBuilder.createSection(Builder, BodyGenCB, FiniCB));
- return;
- }
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(&S);
- EmitStmt(S.getAssociatedStmt());
- }
- void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
- llvm::SmallVector<const Expr *, 8> CopyprivateVars;
- llvm::SmallVector<const Expr *, 8> DestExprs;
- llvm::SmallVector<const Expr *, 8> SrcExprs;
- llvm::SmallVector<const Expr *, 8> AssignmentOps;
- // Check if there are any 'copyprivate' clauses associated with this
- // 'single' construct.
- // Build a list of copyprivate variables along with helper expressions
- // (<source>, <destination>, <destination>=<source> expressions)
- for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
- CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
- DestExprs.append(C->destination_exprs().begin(),
- C->destination_exprs().end());
- SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
- AssignmentOps.append(C->assignment_ops().begin(),
- C->assignment_ops().end());
- }
- // Emit code for 'single' region along with 'copyprivate' clauses
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- OMPPrivateScope SingleScope(CGF);
- (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
- CGF.EmitOMPPrivateClause(S, SingleScope);
- (void)SingleScope.Privatize();
- CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
- };
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
- CopyprivateVars, DestExprs,
- SrcExprs, AssignmentOps);
- }
- // Emit an implicit barrier at the end (to avoid data race on firstprivate
- // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
- if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
- CGM.getOpenMPRuntime().emitBarrierCall(
- *this, S.getBeginLoc(),
- S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- static void emitMaster(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.EmitStmt(S.getRawStmt());
- };
- CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());
- }
- void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
- if (CGM.getLangOpts().OpenMPIRBuilder) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- const Stmt *MasterRegionBodyStmt = S.getAssociatedStmt();
- auto FiniCB = [this](InsertPointTy IP) {
- OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
- };
- auto BodyGenCB = [MasterRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, MasterRegionBodyStmt,
- CodeGenIP, FiniBB);
- };
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(&S);
- Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
- return;
- }
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(&S);
- emitMaster(*this, S);
- }
- static void emitMasked(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.EmitStmt(S.getRawStmt());
- };
- Expr *Filter = nullptr;
- if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
- Filter = FilterClause->getThreadID();
- CGF.CGM.getOpenMPRuntime().emitMaskedRegion(CGF, CodeGen, S.getBeginLoc(),
- Filter);
- }
- void CodeGenFunction::EmitOMPMaskedDirective(const OMPMaskedDirective &S) {
- if (CGM.getLangOpts().OpenMPIRBuilder) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- const Stmt *MaskedRegionBodyStmt = S.getAssociatedStmt();
- const Expr *Filter = nullptr;
- if (const auto *FilterClause = S.getSingleClause<OMPFilterClause>())
- Filter = FilterClause->getThreadID();
- llvm::Value *FilterVal = Filter
- ? EmitScalarExpr(Filter, CGM.Int32Ty)
- : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0);
- auto FiniCB = [this](InsertPointTy IP) {
- OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
- };
- auto BodyGenCB = [MaskedRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, MaskedRegionBodyStmt,
- CodeGenIP, FiniBB);
- };
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(&S);
- Builder.restoreIP(
- OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, FilterVal));
- return;
- }
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(&S);
- emitMasked(*this, S);
- }
- void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
- if (CGM.getLangOpts().OpenMPIRBuilder) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- const Stmt *CriticalRegionBodyStmt = S.getAssociatedStmt();
- const Expr *Hint = nullptr;
- if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
- Hint = HintClause->getHint();
- // TODO: This is slightly different from what's currently being done in
- // clang. Fix the Int32Ty to IntPtrTy (pointer width size) when everything
- // about typing is final.
- llvm::Value *HintInst = nullptr;
- if (Hint)
- HintInst =
- Builder.CreateIntCast(EmitScalarExpr(Hint), CGM.Int32Ty, false);
- auto FiniCB = [this](InsertPointTy IP) {
- OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
- };
- auto BodyGenCB = [CriticalRegionBodyStmt, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP, FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, CriticalRegionBodyStmt,
- CodeGenIP, FiniBB);
- };
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(&S);
- Builder.restoreIP(OMPBuilder.createCritical(
- Builder, BodyGenCB, FiniCB, S.getDirectiveName().getAsString(),
- HintInst));
- return;
- }
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.EmitStmt(S.getAssociatedStmt());
- };
- const Expr *Hint = nullptr;
- if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
- Hint = HintClause->getHint();
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(&S);
- CGM.getOpenMPRuntime().emitCriticalRegion(*this,
- S.getDirectiveName().getAsString(),
- CodeGen, S.getBeginLoc(), Hint);
- }
- void CodeGenFunction::EmitOMPParallelForDirective(
- const OMPParallelForDirective &S) {
- // Emit directive as a combined directive that consists of two implicit
- // directives: 'parallel' with 'for' directive.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- (void)emitWorksharingDirective(CGF, S, S.hasCancel());
- };
- {
- if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
- [](const OMPReductionClause *C) {
- return C->getModifier() == OMPC_REDUCTION_inscan;
- })) {
- const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
- CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
- CGCapturedStmtInfo CGSI(CR_OpenMP);
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
- OMPLoopScope LoopScope(CGF, S);
- return CGF.EmitScalarExpr(S.getNumIterations());
- };
- emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
- }
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
- emitEmptyBoundParameters);
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- void CodeGenFunction::EmitOMPParallelForSimdDirective(
- const OMPParallelForSimdDirective &S) {
- // Emit directive as a combined directive that consists of two implicit
- // directives: 'parallel' with 'for' directive.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- (void)emitWorksharingDirective(CGF, S, /*HasCancel=*/false);
- };
- {
- if (llvm::any_of(S.getClausesOfKind<OMPReductionClause>(),
- [](const OMPReductionClause *C) {
- return C->getModifier() == OMPC_REDUCTION_inscan;
- })) {
- const auto &&NumIteratorsGen = [&S](CodeGenFunction &CGF) {
- CodeGenFunction::OMPLocalDeclMapRAII Scope(CGF);
- CGCapturedStmtInfo CGSI(CR_OpenMP);
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);
- OMPLoopScope LoopScope(CGF, S);
- return CGF.EmitScalarExpr(S.getNumIterations());
- };
- emitScanBasedDirectiveDecls(*this, S, NumIteratorsGen);
- }
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- emitCommonOMPParallelDirective(*this, S, OMPD_for_simd, CodeGen,
- emitEmptyBoundParameters);
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- void CodeGenFunction::EmitOMPParallelMasterDirective(
- const OMPParallelMasterDirective &S) {
- // Emit directive as a combined directive that consists of two implicit
- // directives: 'parallel' with 'master' directive.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- OMPPrivateScope PrivateScope(CGF);
- bool Copyins = CGF.EmitOMPCopyinClause(S);
- (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
- if (Copyins) {
- // Emit implicit barrier to synchronize threads and avoid data races on
- // propagation master's thread values of threadprivate variables to local
- // instances of that variables of all other implicit threads.
- CGF.CGM.getOpenMPRuntime().emitBarrierCall(
- CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
- }
- CGF.EmitOMPPrivateClause(S, PrivateScope);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- emitMaster(CGF, S);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
- };
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- emitCommonOMPParallelDirective(*this, S, OMPD_master, CodeGen,
- emitEmptyBoundParameters);
- emitPostUpdateForReductionClause(*this, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- void CodeGenFunction::EmitOMPParallelSectionsDirective(
- const OMPParallelSectionsDirective &S) {
- // Emit directive as a combined directive that consists of two implicit
- // directives: 'parallel' with 'sections' directive.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.EmitSections(S);
- };
- {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
- emitEmptyBoundParameters);
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, S);
- }
- namespace {
- /// Get the list of variables declared in the context of the untied tasks.
- class CheckVarsEscapingUntiedTaskDeclContext final
- : public ConstStmtVisitor<CheckVarsEscapingUntiedTaskDeclContext> {
- llvm::SmallVector<const VarDecl *, 4> PrivateDecls;
- public:
- explicit CheckVarsEscapingUntiedTaskDeclContext() = default;
- virtual ~CheckVarsEscapingUntiedTaskDeclContext() = default;
- void VisitDeclStmt(const DeclStmt *S) {
- if (!S)
- return;
- // Need to privatize only local vars, static locals can be processed as is.
- for (const Decl *D : S->decls()) {
- if (const auto *VD = dyn_cast_or_null<VarDecl>(D))
- if (VD->hasLocalStorage())
- PrivateDecls.push_back(VD);
- }
- }
- void VisitOMPExecutableDirective(const OMPExecutableDirective *) {}
- void VisitCapturedStmt(const CapturedStmt *) {}
- void VisitLambdaExpr(const LambdaExpr *) {}
- void VisitBlockExpr(const BlockExpr *) {}
- void VisitStmt(const Stmt *S) {
- if (!S)
- return;
- for (const Stmt *Child : S->children())
- if (Child)
- Visit(Child);
- }
- /// Swaps list of vars with the provided one.
- ArrayRef<const VarDecl *> getPrivateDecls() const { return PrivateDecls; }
- };
- } // anonymous namespace
- void CodeGenFunction::EmitOMPTaskBasedDirective(
- const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
- const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
- OMPTaskDataTy &Data) {
- // Emit outlined function for task construct.
- const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
- auto I = CS->getCapturedDecl()->param_begin();
- auto PartId = std::next(I);
- auto TaskT = std::next(I, 4);
- // Check if the task is final
- if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
- // If the condition constant folds and can be elided, try to avoid emitting
- // the condition and the dead arm of the if/else.
- const Expr *Cond = Clause->getCondition();
- bool CondConstant;
- if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
- Data.Final.setInt(CondConstant);
- else
- Data.Final.setPointer(EvaluateExprAsBool(Cond));
- } else {
- // By default the task is not final.
- Data.Final.setInt(/*IntVal=*/false);
- }
- // Check if the task has 'priority' clause.
- if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
- const Expr *Prio = Clause->getPriority();
- Data.Priority.setInt(/*IntVal=*/true);
- Data.Priority.setPointer(EmitScalarConversion(
- EmitScalarExpr(Prio), Prio->getType(),
- getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
- Prio->getExprLoc()));
- }
- // The first function argument for tasks is a thread id, the second one is a
- // part id (0 for tied tasks, >=0 for untied task).
- llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
- // Get list of private variables.
- for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
- auto IRef = C->varlist_begin();
- for (const Expr *IInit : C->private_copies()) {
- const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
- if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
- Data.PrivateVars.push_back(*IRef);
- Data.PrivateCopies.push_back(IInit);
- }
- ++IRef;
- }
- }
- EmittedAsPrivate.clear();
- // Get list of firstprivate variables.
- for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
- auto IRef = C->varlist_begin();
- auto IElemInitRef = C->inits().begin();
- for (const Expr *IInit : C->private_copies()) {
- const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
- if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
- Data.FirstprivateVars.push_back(*IRef);
- Data.FirstprivateCopies.push_back(IInit);
- Data.FirstprivateInits.push_back(*IElemInitRef);
- }
- ++IRef;
- ++IElemInitRef;
- }
- }
- // Get list of lastprivate variables (for taskloops).
- llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
- for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
- auto IRef = C->varlist_begin();
- auto ID = C->destination_exprs().begin();
- for (const Expr *IInit : C->private_copies()) {
- const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
- if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
- Data.LastprivateVars.push_back(*IRef);
- Data.LastprivateCopies.push_back(IInit);
- }
- LastprivateDstsOrigs.insert(
- std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
- cast<DeclRefExpr>(*IRef)));
- ++IRef;
- ++ID;
- }
- }
- SmallVector<const Expr *, 4> LHSs;
- SmallVector<const Expr *, 4> RHSs;
- for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
- Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
- Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
- Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
- Data.ReductionOps.append(C->reduction_ops().begin(),
- C->reduction_ops().end());
- LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
- RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
- }
- Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
- *this, S.getBeginLoc(), LHSs, RHSs, Data);
- // Build list of dependences.
- for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
- OMPTaskDataTy::DependData &DD =
- Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
- DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
- }
- // Get list of local vars for untied tasks.
- if (!Data.Tied) {
- CheckVarsEscapingUntiedTaskDeclContext Checker;
- Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());
- Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
- Checker.getPrivateDecls().end());
- }
- auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
- CapturedRegion](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
- std::pair<Address, Address>>
- UntiedLocalVars;
- // Set proper addresses for generated private copies.
- OMPPrivateScope Scope(CGF);
- // Generate debug info for variables present in shared clause.
- if (auto *DI = CGF.getDebugInfo()) {
- llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =
- CGF.CapturedStmtInfo->getCaptureFields();
- llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();
- if (CaptureFields.size() && ContextValue) {
- unsigned CharWidth = CGF.getContext().getCharWidth();
- // The shared variables are packed together as members of structure.
- // So the address of each shared variable can be computed by adding
- // offset of it (within record) to the base address of record. For each
- // shared variable, debug intrinsic llvm.dbg.declare is generated with
- // appropriate expressions (DIExpression).
- // Ex:
- // %12 = load %struct.anon*, %struct.anon** %__context.addr.i
- // call void @llvm.dbg.declare(metadata %struct.anon* %12,
- // metadata !svar1,
- // metadata !DIExpression(DW_OP_deref))
- // call void @llvm.dbg.declare(metadata %struct.anon* %12,
- // metadata !svar2,
- // metadata !DIExpression(DW_OP_plus_uconst, 8, DW_OP_deref))
- for (auto It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {
- const VarDecl *SharedVar = It->first;
- RecordDecl *CaptureRecord = It->second->getParent();
- const ASTRecordLayout &Layout =
- CGF.getContext().getASTRecordLayout(CaptureRecord);
- unsigned Offset =
- Layout.getFieldOffset(It->second->getFieldIndex()) / CharWidth;
- if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
- (void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,
- CGF.Builder, false);
- llvm::Instruction &Last = CGF.Builder.GetInsertBlock()->back();
- // Get the call dbg.declare instruction we just created and update
- // its DIExpression to add offset to base address.
- if (auto DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&Last)) {
- SmallVector<uint64_t, 8> Ops;
- // Add offset to the base address if non zero.
- if (Offset) {
- Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);
- Ops.push_back(Offset);
- }
- Ops.push_back(llvm::dwarf::DW_OP_deref);
- auto &Ctx = DDI->getContext();
- llvm::DIExpression *DIExpr = llvm::DIExpression::get(Ctx, Ops);
- Last.setOperand(2, llvm::MetadataAsValue::get(Ctx, DIExpr));
- }
- }
- }
- }
- llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> FirstprivatePtrs;
- if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
- !Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
- enum { PrivatesParam = 2, CopyFnParam = 3 };
- llvm::Value *CopyFn = CGF.Builder.CreateLoad(
- CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
- llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
- CS->getCapturedDecl()->getParam(PrivatesParam)));
- // Map privates.
- llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
- llvm::SmallVector<llvm::Value *, 16> CallArgs;
- llvm::SmallVector<llvm::Type *, 4> ParamTypes;
- CallArgs.push_back(PrivatesPtr);
- ParamTypes.push_back(PrivatesPtr->getType());
- for (const Expr *E : Data.PrivateVars) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- Address PrivatePtr = CGF.CreateMemTemp(
- CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
- PrivatePtrs.emplace_back(VD, PrivatePtr);
- CallArgs.push_back(PrivatePtr.getPointer());
- ParamTypes.push_back(PrivatePtr.getType());
- }
- for (const Expr *E : Data.FirstprivateVars) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- Address PrivatePtr =
- CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
- ".firstpriv.ptr.addr");
- PrivatePtrs.emplace_back(VD, PrivatePtr);
- FirstprivatePtrs.emplace_back(VD, PrivatePtr);
- CallArgs.push_back(PrivatePtr.getPointer());
- ParamTypes.push_back(PrivatePtr.getType());
- }
- for (const Expr *E : Data.LastprivateVars) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- Address PrivatePtr =
- CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
- ".lastpriv.ptr.addr");
- PrivatePtrs.emplace_back(VD, PrivatePtr);
- CallArgs.push_back(PrivatePtr.getPointer());
- ParamTypes.push_back(PrivatePtr.getType());
- }
- for (const VarDecl *VD : Data.PrivateLocals) {
- QualType Ty = VD->getType().getNonReferenceType();
- if (VD->getType()->isLValueReferenceType())
- Ty = CGF.getContext().getPointerType(Ty);
- if (isAllocatableDecl(VD))
- Ty = CGF.getContext().getPointerType(Ty);
- Address PrivatePtr = CGF.CreateMemTemp(
- CGF.getContext().getPointerType(Ty), ".local.ptr.addr");
- auto Result = UntiedLocalVars.insert(
- std::make_pair(VD, std::make_pair(PrivatePtr, Address::invalid())));
- // If key exists update in place.
- if (Result.second == false)
- *Result.first = std::make_pair(
- VD, std::make_pair(PrivatePtr, Address::invalid()));
- CallArgs.push_back(PrivatePtr.getPointer());
- ParamTypes.push_back(PrivatePtr.getType());
- }
- auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
- ParamTypes, /*isVarArg=*/false);
- CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CopyFn, CopyFnTy->getPointerTo());
- CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
- CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
- for (const auto &Pair : LastprivateDstsOrigs) {
- const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
- DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
- /*RefersToEnclosingVariableOrCapture=*/
- CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
- Pair.second->getType(), VK_LValue,
- Pair.second->getExprLoc());
- Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
- return CGF.EmitLValue(&DRE).getAddress(CGF);
- });
- }
- for (const auto &Pair : PrivatePtrs) {
- Address Replacement(CGF.Builder.CreateLoad(Pair.second),
- CGF.getContext().getDeclAlign(Pair.first));
- Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
- if (auto *DI = CGF.getDebugInfo())
- if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
- (void)DI->EmitDeclareOfAutoVariable(
- Pair.first, Pair.second.getPointer(), CGF.Builder,
- /*UsePointerValue*/ true);
- }
- // Adjust mapping for internal locals by mapping actual memory instead of
- // a pointer to this memory.
- for (auto &Pair : UntiedLocalVars) {
- if (isAllocatableDecl(Pair.first)) {
- llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
- Address Replacement(Ptr, CGF.getPointerAlign());
- Pair.second.first = Replacement;
- Ptr = CGF.Builder.CreateLoad(Replacement);
- Replacement = Address(Ptr, CGF.getContext().getDeclAlign(Pair.first));
- Pair.second.second = Replacement;
- } else {
- llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);
- Address Replacement(Ptr, CGF.getContext().getDeclAlign(Pair.first));
- Pair.second.first = Replacement;
- }
- }
- }
- if (Data.Reductions) {
- OMPPrivateScope FirstprivateScope(CGF);
- for (const auto &Pair : FirstprivatePtrs) {
- Address Replacement(CGF.Builder.CreateLoad(Pair.second),
- CGF.getContext().getDeclAlign(Pair.first));
- FirstprivateScope.addPrivate(Pair.first,
- [Replacement]() { return Replacement; });
- }
- (void)FirstprivateScope.Privatize();
- OMPLexicalScope LexScope(CGF, S, CapturedRegion);
- ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionVars,
- Data.ReductionCopies, Data.ReductionOps);
- llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
- CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
- for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
- RedCG.emitSharedOrigLValue(CGF, Cnt);
- RedCG.emitAggregateType(CGF, Cnt);
- // FIXME: This must removed once the runtime library is fixed.
- // Emit required threadprivate variables for
- // initializer/combiner/finalizer.
- CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
- RedCG, Cnt);
- Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
- CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
- Replacement =
- Address(CGF.EmitScalarConversion(
- Replacement.getPointer(), CGF.getContext().VoidPtrTy,
- CGF.getContext().getPointerType(
- Data.ReductionCopies[Cnt]->getType()),
- Data.ReductionCopies[Cnt]->getExprLoc()),
- Replacement.getAlignment());
- Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
- Scope.addPrivate(RedCG.getBaseDecl(Cnt),
- [Replacement]() { return Replacement; });
- }
- }
- // Privatize all private variables except for in_reduction items.
- (void)Scope.Privatize();
- SmallVector<const Expr *, 4> InRedVars;
- SmallVector<const Expr *, 4> InRedPrivs;
- SmallVector<const Expr *, 4> InRedOps;
- SmallVector<const Expr *, 4> TaskgroupDescriptors;
- for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
- auto IPriv = C->privates().begin();
- auto IRed = C->reduction_ops().begin();
- auto ITD = C->taskgroup_descriptors().begin();
- for (const Expr *Ref : C->varlists()) {
- InRedVars.emplace_back(Ref);
- InRedPrivs.emplace_back(*IPriv);
- InRedOps.emplace_back(*IRed);
- TaskgroupDescriptors.emplace_back(*ITD);
- std::advance(IPriv, 1);
- std::advance(IRed, 1);
- std::advance(ITD, 1);
- }
- }
- // Privatize in_reduction items here, because taskgroup descriptors must be
- // privatized earlier.
- OMPPrivateScope InRedScope(CGF);
- if (!InRedVars.empty()) {
- ReductionCodeGen RedCG(InRedVars, InRedVars, InRedPrivs, InRedOps);
- for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
- RedCG.emitSharedOrigLValue(CGF, Cnt);
- RedCG.emitAggregateType(CGF, Cnt);
- // The taskgroup descriptor variable is always implicit firstprivate and
- // privatized already during processing of the firstprivates.
- // FIXME: This must removed once the runtime library is fixed.
- // Emit required threadprivate variables for
- // initializer/combiner/finalizer.
- CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
- RedCG, Cnt);
- llvm::Value *ReductionsPtr;
- if (const Expr *TRExpr = TaskgroupDescriptors[Cnt]) {
- ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),
- TRExpr->getExprLoc());
- } else {
- ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
- }
- Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
- CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
- Replacement = Address(
- CGF.EmitScalarConversion(
- Replacement.getPointer(), CGF.getContext().VoidPtrTy,
- CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
- InRedPrivs[Cnt]->getExprLoc()),
- Replacement.getAlignment());
- Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
- InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
- [Replacement]() { return Replacement; });
- }
- }
- (void)InRedScope.Privatize();
- CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF,
- UntiedLocalVars);
- Action.Enter(CGF);
- BodyGen(CGF);
- };
- llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
- S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
- Data.NumberOfParts);
- OMPLexicalScope Scope(*this, S, llvm::None,
- !isOpenMPParallelDirective(S.getDirectiveKind()) &&
- !isOpenMPSimdDirective(S.getDirectiveKind()));
- TaskGen(*this, OutlinedFn, Data);
- }
- static ImplicitParamDecl *
- createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
- QualType Ty, CapturedDecl *CD,
- SourceLocation Loc) {
- auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
- ImplicitParamDecl::Other);
- auto *OrigRef = DeclRefExpr::Create(
- C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
- /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
- auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
- ImplicitParamDecl::Other);
- auto *PrivateRef = DeclRefExpr::Create(
- C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
- /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
- QualType ElemType = C.getBaseElementType(Ty);
- auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
- ImplicitParamDecl::Other);
- auto *InitRef = DeclRefExpr::Create(
- C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
- /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
- PrivateVD->setInitStyle(VarDecl::CInit);
- PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
- InitRef, /*BasePath=*/nullptr,
- VK_PRValue, FPOptionsOverride()));
- Data.FirstprivateVars.emplace_back(OrigRef);
- Data.FirstprivateCopies.emplace_back(PrivateRef);
- Data.FirstprivateInits.emplace_back(InitRef);
- return OrigVD;
- }
- void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
- const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
- OMPTargetDataInfo &InputInfo) {
- // Emit outlined function for task construct.
- const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
- Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
- QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
- auto I = CS->getCapturedDecl()->param_begin();
- auto PartId = std::next(I);
- auto TaskT = std::next(I, 4);
- OMPTaskDataTy Data;
- // The task is not final.
- Data.Final.setInt(/*IntVal=*/false);
- // Get list of firstprivate variables.
- for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
- auto IRef = C->varlist_begin();
- auto IElemInitRef = C->inits().begin();
- for (auto *IInit : C->private_copies()) {
- Data.FirstprivateVars.push_back(*IRef);
- Data.FirstprivateCopies.push_back(IInit);
- Data.FirstprivateInits.push_back(*IElemInitRef);
- ++IRef;
- ++IElemInitRef;
- }
- }
- OMPPrivateScope TargetScope(*this);
- VarDecl *BPVD = nullptr;
- VarDecl *PVD = nullptr;
- VarDecl *SVD = nullptr;
- VarDecl *MVD = nullptr;
- if (InputInfo.NumberOfTargetItems > 0) {
- auto *CD = CapturedDecl::Create(
- getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
- llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
- QualType BaseAndPointerAndMapperType = getContext().getConstantArrayType(
- getContext().VoidPtrTy, ArrSize, nullptr, ArrayType::Normal,
- /*IndexTypeQuals=*/0);
- BPVD = createImplicitFirstprivateForType(
- getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
- PVD = createImplicitFirstprivateForType(
- getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
- QualType SizesType = getContext().getConstantArrayType(
- getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1),
- ArrSize, nullptr, ArrayType::Normal,
- /*IndexTypeQuals=*/0);
- SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
- S.getBeginLoc());
- TargetScope.addPrivate(
- BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
- TargetScope.addPrivate(PVD,
- [&InputInfo]() { return InputInfo.PointersArray; });
- TargetScope.addPrivate(SVD,
- [&InputInfo]() { return InputInfo.SizesArray; });
- // If there is no user-defined mapper, the mapper array will be nullptr. In
- // this case, we don't need to privatize it.
- if (!isa_and_nonnull<llvm::ConstantPointerNull>(
- InputInfo.MappersArray.getPointer())) {
- MVD = createImplicitFirstprivateForType(
- getContext(), Data, BaseAndPointerAndMapperType, CD, S.getBeginLoc());
- TargetScope.addPrivate(MVD,
- [&InputInfo]() { return InputInfo.MappersArray; });
- }
- }
- (void)TargetScope.Privatize();
- // Build list of dependences.
- for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
- OMPTaskDataTy::DependData &DD =
- Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
- DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
- }
- auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD,
- &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
- // Set proper addresses for generated private copies.
- OMPPrivateScope Scope(CGF);
- if (!Data.FirstprivateVars.empty()) {
- enum { PrivatesParam = 2, CopyFnParam = 3 };
- llvm::Value *CopyFn = CGF.Builder.CreateLoad(
- CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
- llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
- CS->getCapturedDecl()->getParam(PrivatesParam)));
- // Map privates.
- llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
- llvm::SmallVector<llvm::Value *, 16> CallArgs;
- llvm::SmallVector<llvm::Type *, 4> ParamTypes;
- CallArgs.push_back(PrivatesPtr);
- ParamTypes.push_back(PrivatesPtr->getType());
- for (const Expr *E : Data.FirstprivateVars) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- Address PrivatePtr =
- CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
- ".firstpriv.ptr.addr");
- PrivatePtrs.emplace_back(VD, PrivatePtr);
- CallArgs.push_back(PrivatePtr.getPointer());
- ParamTypes.push_back(PrivatePtr.getType());
- }
- auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
- ParamTypes, /*isVarArg=*/false);
- CopyFn = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CopyFn, CopyFnTy->getPointerTo());
- CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
- CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);
- for (const auto &Pair : PrivatePtrs) {
- Address Replacement(CGF.Builder.CreateLoad(Pair.second),
- CGF.getContext().getDeclAlign(Pair.first));
- Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
- }
- }
- // Privatize all private variables except for in_reduction items.
- (void)Scope.Privatize();
- if (InputInfo.NumberOfTargetItems > 0) {
- InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
- CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0);
- InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
- CGF.GetAddrOfLocalVar(PVD), /*Index=*/0);
- InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
- CGF.GetAddrOfLocalVar(SVD), /*Index=*/0);
- // If MVD is nullptr, the mapper array is not privatized
- if (MVD)
- InputInfo.MappersArray = CGF.Builder.CreateConstArrayGEP(
- CGF.GetAddrOfLocalVar(MVD), /*Index=*/0);
- }
- Action.Enter(CGF);
- OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
- BodyGen(CGF);
- };
- llvm::Function *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
- S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
- Data.NumberOfParts);
- llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
- IntegerLiteral IfCond(getContext(), TrueOrFalse,
- getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
- SourceLocation());
- CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
- SharedsTy, CapturedStruct, &IfCond, Data);
- }
- void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
- // Emit outlined function for task construct.
- const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
- Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
- QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
- const Expr *IfCond = nullptr;
- for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
- if (C->getNameModifier() == OMPD_unknown ||
- C->getNameModifier() == OMPD_task) {
- IfCond = C->getCondition();
- break;
- }
- }
- OMPTaskDataTy Data;
- // Check if we should emit tied or untied task.
- Data.Tied = !S.getSingleClause<OMPUntiedClause>();
- auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitStmt(CS->getCapturedStmt());
- };
- auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
- IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
- const OMPTaskDataTy &Data) {
- CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
- SharedsTy, CapturedStruct, IfCond,
- Data);
- };
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
- }
- void CodeGenFunction::EmitOMPTaskyieldDirective(
- const OMPTaskyieldDirective &S) {
- CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
- }
- void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
- CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
- }
- void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
- OMPTaskDataTy Data;
- // Build list of dependences
- for (const auto *C : S.getClausesOfKind<OMPDependClause>()) {
- OMPTaskDataTy::DependData &DD =
- Data.Dependences.emplace_back(C->getDependencyKind(), C->getModifier());
- DD.DepExprs.append(C->varlist_begin(), C->varlist_end());
- }
- CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
- }
- void CodeGenFunction::EmitOMPTaskgroupDirective(
- const OMPTaskgroupDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- if (const Expr *E = S.getReductionRef()) {
- SmallVector<const Expr *, 4> LHSs;
- SmallVector<const Expr *, 4> RHSs;
- OMPTaskDataTy Data;
- for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
- Data.ReductionVars.append(C->varlist_begin(), C->varlist_end());
- Data.ReductionOrigs.append(C->varlist_begin(), C->varlist_end());
- Data.ReductionCopies.append(C->privates().begin(), C->privates().end());
- Data.ReductionOps.append(C->reduction_ops().begin(),
- C->reduction_ops().end());
- LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
- RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
- }
- llvm::Value *ReductionDesc =
- CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
- LHSs, RHSs, Data);
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- CGF.EmitVarDecl(*VD);
- CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
- /*Volatile=*/false, E->getType());
- }
- CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
- };
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
- }
- void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
- llvm::AtomicOrdering AO = S.getSingleClause<OMPFlushClause>()
- ? llvm::AtomicOrdering::NotAtomic
- : llvm::AtomicOrdering::AcquireRelease;
- CGM.getOpenMPRuntime().emitFlush(
- *this,
- [&S]() -> ArrayRef<const Expr *> {
- if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
- return llvm::makeArrayRef(FlushClause->varlist_begin(),
- FlushClause->varlist_end());
- return llvm::None;
- }(),
- S.getBeginLoc(), AO);
- }
- void CodeGenFunction::EmitOMPDepobjDirective(const OMPDepobjDirective &S) {
- const auto *DO = S.getSingleClause<OMPDepobjClause>();
- LValue DOLVal = EmitLValue(DO->getDepobj());
- if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
- OMPTaskDataTy::DependData Dependencies(DC->getDependencyKind(),
- DC->getModifier());
- Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end());
- Address DepAddr = CGM.getOpenMPRuntime().emitDepobjDependClause(
- *this, Dependencies, DC->getBeginLoc());
- EmitStoreOfScalar(DepAddr.getPointer(), DOLVal);
- return;
- }
- if (const auto *DC = S.getSingleClause<OMPDestroyClause>()) {
- CGM.getOpenMPRuntime().emitDestroyClause(*this, DOLVal, DC->getBeginLoc());
- return;
- }
- if (const auto *UC = S.getSingleClause<OMPUpdateClause>()) {
- CGM.getOpenMPRuntime().emitUpdateClause(
- *this, DOLVal, UC->getDependencyKind(), UC->getBeginLoc());
- return;
- }
- }
- void CodeGenFunction::EmitOMPScanDirective(const OMPScanDirective &S) {
- if (!OMPParentLoopDirectiveForScan)
- return;
- const OMPExecutableDirective &ParentDir = *OMPParentLoopDirectiveForScan;
- bool IsInclusive = S.hasClausesOfKind<OMPInclusiveClause>();
- SmallVector<const Expr *, 4> Shareds;
- SmallVector<const Expr *, 4> Privates;
- SmallVector<const Expr *, 4> LHSs;
- SmallVector<const Expr *, 4> RHSs;
- SmallVector<const Expr *, 4> ReductionOps;
- SmallVector<const Expr *, 4> CopyOps;
- SmallVector<const Expr *, 4> CopyArrayTemps;
- SmallVector<const Expr *, 4> CopyArrayElems;
- for (const auto *C : ParentDir.getClausesOfKind<OMPReductionClause>()) {
- if (C->getModifier() != OMPC_REDUCTION_inscan)
- continue;
- Shareds.append(C->varlist_begin(), C->varlist_end());
- Privates.append(C->privates().begin(), C->privates().end());
- LHSs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
- RHSs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
- ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
- CopyOps.append(C->copy_ops().begin(), C->copy_ops().end());
- CopyArrayTemps.append(C->copy_array_temps().begin(),
- C->copy_array_temps().end());
- CopyArrayElems.append(C->copy_array_elems().begin(),
- C->copy_array_elems().end());
- }
- if (ParentDir.getDirectiveKind() == OMPD_simd ||
- (getLangOpts().OpenMPSimd &&
- isOpenMPSimdDirective(ParentDir.getDirectiveKind()))) {
- // For simd directive and simd-based directives in simd only mode, use the
- // following codegen:
- // int x = 0;
- // #pragma omp simd reduction(inscan, +: x)
- // for (..) {
- // <first part>
- // #pragma omp scan inclusive(x)
- // <second part>
- // }
- // is transformed to:
- // int x = 0;
- // for (..) {
- // int x_priv = 0;
- // <first part>
- // x = x_priv + x;
- // x_priv = x;
- // <second part>
- // }
- // and
- // int x = 0;
- // #pragma omp simd reduction(inscan, +: x)
- // for (..) {
- // <first part>
- // #pragma omp scan exclusive(x)
- // <second part>
- // }
- // to
- // int x = 0;
- // for (..) {
- // int x_priv = 0;
- // <second part>
- // int temp = x;
- // x = x_priv + x;
- // x_priv = temp;
- // <first part>
- // }
- llvm::BasicBlock *OMPScanReduce = createBasicBlock("omp.inscan.reduce");
- EmitBranch(IsInclusive
- ? OMPScanReduce
- : BreakContinueStack.back().ContinueBlock.getBlock());
- EmitBlock(OMPScanDispatch);
- {
- // New scope for correct construction/destruction of temp variables for
- // exclusive scan.
- LexicalScope Scope(*this, S.getSourceRange());
- EmitBranch(IsInclusive ? OMPBeforeScanBlock : OMPAfterScanBlock);
- EmitBlock(OMPScanReduce);
- if (!IsInclusive) {
- // Create temp var and copy LHS value to this temp value.
- // TMP = LHS;
- for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
- const Expr *PrivateExpr = Privates[I];
- const Expr *TempExpr = CopyArrayTemps[I];
- EmitAutoVarDecl(
- *cast<VarDecl>(cast<DeclRefExpr>(TempExpr)->getDecl()));
- LValue DestLVal = EmitLValue(TempExpr);
- LValue SrcLVal = EmitLValue(LHSs[I]);
- EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this),
- SrcLVal.getAddress(*this),
- cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
- cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
- CopyOps[I]);
- }
- }
- CGM.getOpenMPRuntime().emitReduction(
- *this, ParentDir.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,
- {/*WithNowait=*/true, /*SimpleReduction=*/true, OMPD_simd});
- for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
- const Expr *PrivateExpr = Privates[I];
- LValue DestLVal;
- LValue SrcLVal;
- if (IsInclusive) {
- DestLVal = EmitLValue(RHSs[I]);
- SrcLVal = EmitLValue(LHSs[I]);
- } else {
- const Expr *TempExpr = CopyArrayTemps[I];
- DestLVal = EmitLValue(RHSs[I]);
- SrcLVal = EmitLValue(TempExpr);
- }
- EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this),
- SrcLVal.getAddress(*this),
- cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
- cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
- CopyOps[I]);
- }
- }
- EmitBranch(IsInclusive ? OMPAfterScanBlock : OMPBeforeScanBlock);
- OMPScanExitBlock = IsInclusive
- ? BreakContinueStack.back().ContinueBlock.getBlock()
- : OMPScanReduce;
- EmitBlock(OMPAfterScanBlock);
- return;
- }
- if (!IsInclusive) {
- EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
- EmitBlock(OMPScanExitBlock);
- }
- if (OMPFirstScanLoop) {
- // Emit buffer[i] = red; at the end of the input phase.
- const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
- .getIterationVariable()
- ->IgnoreParenImpCasts();
- LValue IdxLVal = EmitLValue(IVExpr);
- llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
- IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);
- for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
- const Expr *PrivateExpr = Privates[I];
- const Expr *OrigExpr = Shareds[I];
- const Expr *CopyArrayElem = CopyArrayElems[I];
- OpaqueValueMapping IdxMapping(
- *this,
- cast<OpaqueValueExpr>(
- cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
- RValue::get(IdxVal));
- LValue DestLVal = EmitLValue(CopyArrayElem);
- LValue SrcLVal = EmitLValue(OrigExpr);
- EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this),
- SrcLVal.getAddress(*this),
- cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
- cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
- CopyOps[I]);
- }
- }
- EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
- if (IsInclusive) {
- EmitBlock(OMPScanExitBlock);
- EmitBranch(BreakContinueStack.back().ContinueBlock.getBlock());
- }
- EmitBlock(OMPScanDispatch);
- if (!OMPFirstScanLoop) {
- // Emit red = buffer[i]; at the entrance to the scan phase.
- const auto *IVExpr = cast<OMPLoopDirective>(ParentDir)
- .getIterationVariable()
- ->IgnoreParenImpCasts();
- LValue IdxLVal = EmitLValue(IVExpr);
- llvm::Value *IdxVal = EmitLoadOfScalar(IdxLVal, IVExpr->getExprLoc());
- IdxVal = Builder.CreateIntCast(IdxVal, SizeTy, /*isSigned=*/false);
- llvm::BasicBlock *ExclusiveExitBB = nullptr;
- if (!IsInclusive) {
- llvm::BasicBlock *ContBB = createBasicBlock("omp.exclusive.dec");
- ExclusiveExitBB = createBasicBlock("omp.exclusive.copy.exit");
- llvm::Value *Cmp = Builder.CreateIsNull(IdxVal);
- Builder.CreateCondBr(Cmp, ExclusiveExitBB, ContBB);
- EmitBlock(ContBB);
- // Use idx - 1 iteration for exclusive scan.
- IdxVal = Builder.CreateNUWSub(IdxVal, llvm::ConstantInt::get(SizeTy, 1));
- }
- for (unsigned I = 0, E = CopyArrayElems.size(); I < E; ++I) {
- const Expr *PrivateExpr = Privates[I];
- const Expr *OrigExpr = Shareds[I];
- const Expr *CopyArrayElem = CopyArrayElems[I];
- OpaqueValueMapping IdxMapping(
- *this,
- cast<OpaqueValueExpr>(
- cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),
- RValue::get(IdxVal));
- LValue SrcLVal = EmitLValue(CopyArrayElem);
- LValue DestLVal = EmitLValue(OrigExpr);
- EmitOMPCopy(PrivateExpr->getType(), DestLVal.getAddress(*this),
- SrcLVal.getAddress(*this),
- cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),
- cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()),
- CopyOps[I]);
- }
- if (!IsInclusive) {
- EmitBlock(ExclusiveExitBB);
- }
- }
- EmitBranch((OMPFirstScanLoop == IsInclusive) ? OMPBeforeScanBlock
- : OMPAfterScanBlock);
- EmitBlock(OMPAfterScanBlock);
- }
- void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
- const CodeGenLoopTy &CodeGenLoop,
- Expr *IncExpr) {
- // Emit the loop iteration variable.
- const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
- const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
- EmitVarDecl(*IVDecl);
- // Emit the iterations count variable.
- // If it is not a variable, Sema decided to calculate iterations count on each
- // iteration (e.g., it is foldable into a constant).
- if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
- EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
- // Emit calculation of the iterations count.
- EmitIgnoredExpr(S.getCalcLastIteration());
- }
- CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
- bool HasLastprivateClause = false;
- // Check pre-condition.
- {
- OMPLoopScope PreInitScope(*this, S);
- // Skip the entire loop if we don't meet the precondition.
- // If the condition constant folds and can be elided, avoid emitting the
- // whole loop.
- bool CondConstant;
- llvm::BasicBlock *ContBlock = nullptr;
- if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
- if (!CondConstant)
- return;
- } else {
- llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
- ContBlock = createBasicBlock("omp.precond.end");
- emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
- getProfileCount(&S));
- EmitBlock(ThenBlock);
- incrementProfileCounter(&S);
- }
- emitAlignedClause(*this, S);
- // Emit 'then' code.
- {
- // Emit helper vars inits.
- LValue LB = EmitOMPHelperVar(
- *this, cast<DeclRefExpr>(
- (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedLowerBoundVariable()
- : S.getLowerBoundVariable())));
- LValue UB = EmitOMPHelperVar(
- *this, cast<DeclRefExpr>(
- (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedUpperBoundVariable()
- : S.getUpperBoundVariable())));
- LValue ST =
- EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
- LValue IL =
- EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
- OMPPrivateScope LoopScope(*this);
- if (EmitOMPFirstprivateClause(S, LoopScope)) {
- // Emit implicit barrier to synchronize threads and avoid data races
- // on initialization of firstprivate variables and post-update of
- // lastprivate variables.
- CGM.getOpenMPRuntime().emitBarrierCall(
- *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
- /*ForceSimpleCall=*/true);
- }
- EmitOMPPrivateClause(S, LoopScope);
- if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
- !isOpenMPParallelDirective(S.getDirectiveKind()) &&
- !isOpenMPTeamsDirective(S.getDirectiveKind()))
- EmitOMPReductionClauseInit(S, LoopScope);
- HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
- EmitOMPPrivateLoopCounters(S, LoopScope);
- (void)LoopScope.Privatize();
- if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
- CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
- // Detect the distribute schedule kind and chunk.
- llvm::Value *Chunk = nullptr;
- OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
- if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
- ScheduleKind = C->getDistScheduleKind();
- if (const Expr *Ch = C->getChunkSize()) {
- Chunk = EmitScalarExpr(Ch);
- Chunk = EmitScalarConversion(Chunk, Ch->getType(),
- S.getIterationVariable()->getType(),
- S.getBeginLoc());
- }
- } else {
- // Default behaviour for dist_schedule clause.
- CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
- *this, S, ScheduleKind, Chunk);
- }
- const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
- const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
- // OpenMP [2.10.8, distribute Construct, Description]
- // If dist_schedule is specified, kind must be static. If specified,
- // iterations are divided into chunks of size chunk_size, chunks are
- // assigned to the teams of the league in a round-robin fashion in the
- // order of the team number. When no chunk_size is specified, the
- // iteration space is divided into chunks that are approximately equal
- // in size, and at most one chunk is distributed to each team of the
- // league. The size of the chunks is unspecified in this case.
- bool StaticChunked =
- RT.isStaticChunked(ScheduleKind, /* Chunked */ Chunk != nullptr) &&
- isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
- if (RT.isStaticNonchunked(ScheduleKind,
- /* Chunked */ Chunk != nullptr) ||
- StaticChunked) {
- CGOpenMPRuntime::StaticRTInput StaticInit(
- IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(*this),
- LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
- StaticChunked ? Chunk : nullptr);
- RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
- StaticInit);
- JumpDest LoopExit =
- getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
- // UB = min(UB, GlobalUB);
- EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedEnsureUpperBound()
- : S.getEnsureUpperBound());
- // IV = LB;
- EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedInit()
- : S.getInit());
- const Expr *Cond =
- isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
- ? S.getCombinedCond()
- : S.getCond();
- if (StaticChunked)
- Cond = S.getCombinedDistCond();
- // For static unchunked schedules generate:
- //
- // 1. For distribute alone, codegen
- // while (idx <= UB) {
- // BODY;
- // ++idx;
- // }
- //
- // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
- // while (idx <= UB) {
- // <CodeGen rest of pragma>(LB, UB);
- // idx += ST;
- // }
- //
- // For static chunk one schedule generate:
- //
- // while (IV <= GlobalUB) {
- // <CodeGen rest of pragma>(LB, UB);
- // LB += ST;
- // UB += ST;
- // UB = min(UB, GlobalUB);
- // IV = LB;
- // }
- //
- emitCommonSimdLoop(
- *this, S,
- [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- if (isOpenMPSimdDirective(S.getDirectiveKind()))
- CGF.EmitOMPSimdInit(S);
- },
- [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
- StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPInnerLoop(
- S, LoopScope.requiresCleanups(), Cond, IncExpr,
- [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
- CodeGenLoop(CGF, S, LoopExit);
- },
- [&S, StaticChunked](CodeGenFunction &CGF) {
- if (StaticChunked) {
- CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
- CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
- CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
- CGF.EmitIgnoredExpr(S.getCombinedInit());
- }
- });
- });
- EmitBlock(LoopExit.getBlock());
- // Tell the runtime we are done.
- RT.emitForStaticFinish(*this, S.getEndLoc(), S.getDirectiveKind());
- } else {
- // Emit the outer loop, which requests its work chunk [LB..UB] from
- // runtime and runs the inner loop to process it.
- const OMPLoopArguments LoopArguments = {
- LB.getAddress(*this), UB.getAddress(*this), ST.getAddress(*this),
- IL.getAddress(*this), Chunk};
- EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
- CodeGenLoop);
- }
- if (isOpenMPSimdDirective(S.getDirectiveKind())) {
- EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
- return CGF.Builder.CreateIsNotNull(
- CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
- });
- }
- if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
- !isOpenMPParallelDirective(S.getDirectiveKind()) &&
- !isOpenMPTeamsDirective(S.getDirectiveKind())) {
- EmitOMPReductionClauseFinal(S, OMPD_simd);
- // Emit post-update of the reduction variables if IsLastIter != 0.
- emitPostUpdateForReductionClause(
- *this, S, [IL, &S](CodeGenFunction &CGF) {
- return CGF.Builder.CreateIsNotNull(
- CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
- });
- }
- // Emit final copy of the lastprivate variables if IsLastIter != 0.
- if (HasLastprivateClause) {
- EmitOMPLastprivateClauseFinal(
- S, /*NoFinals=*/false,
- Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
- }
- }
- // We're now done with the loop, so jump to the continuation block.
- if (ContBlock) {
- EmitBranch(ContBlock);
- EmitBlock(ContBlock, true);
- }
- }
- }
- void CodeGenFunction::EmitOMPDistributeDirective(
- const OMPDistributeDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
- };
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
- }
- static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
- const CapturedStmt *S,
- SourceLocation Loc) {
- CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
- CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
- CGF.CapturedStmtInfo = &CapStmtInfo;
- llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S, Loc);
- Fn->setDoesNotRecurse();
- return Fn;
- }
- void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
- if (CGM.getLangOpts().OpenMPIRBuilder) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
- if (S.hasClausesOfKind<OMPDependClause>()) {
- // The ordered directive with depend clause.
- assert(!S.hasAssociatedStmt() &&
- "No associated statement must be in ordered depend construct.");
- InsertPointTy AllocaIP(AllocaInsertPt->getParent(),
- AllocaInsertPt->getIterator());
- for (const auto *DC : S.getClausesOfKind<OMPDependClause>()) {
- unsigned NumLoops = DC->getNumLoops();
- QualType Int64Ty = CGM.getContext().getIntTypeForBitwidth(
- /*DestWidth=*/64, /*Signed=*/1);
- llvm::SmallVector<llvm::Value *> StoreValues;
- for (unsigned I = 0; I < NumLoops; I++) {
- const Expr *CounterVal = DC->getLoopData(I);
- assert(CounterVal);
- llvm::Value *StoreValue = EmitScalarConversion(
- EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty,
- CounterVal->getExprLoc());
- StoreValues.emplace_back(StoreValue);
- }
- bool IsDependSource = false;
- if (DC->getDependencyKind() == OMPC_DEPEND_source)
- IsDependSource = true;
- Builder.restoreIP(OMPBuilder.createOrderedDepend(
- Builder, AllocaIP, NumLoops, StoreValues, ".cnt.addr",
- IsDependSource));
- }
- } else {
- // The ordered directive with threads or simd clause, or without clause.
- // Without clause, it behaves as if the threads clause is specified.
- const auto *C = S.getSingleClause<OMPSIMDClause>();
- auto FiniCB = [this](InsertPointTy IP) {
- OMPBuilderCBHelpers::FinalizeOMPRegion(*this, IP);
- };
- auto BodyGenCB = [&S, C, this](InsertPointTy AllocaIP,
- InsertPointTy CodeGenIP,
- llvm::BasicBlock &FiniBB) {
- const CapturedStmt *CS = S.getInnermostCapturedStmt();
- if (C) {
- llvm::SmallVector<llvm::Value *, 16> CapturedVars;
- GenerateOpenMPCapturedVars(*CS, CapturedVars);
- llvm::Function *OutlinedFn =
- emitOutlinedOrderedFunction(CGM, CS, S.getBeginLoc());
- assert(S.getBeginLoc().isValid() &&
- "Outlined function call location must be valid.");
- ApplyDebugLocation::CreateDefaultArtificial(*this, S.getBeginLoc());
- OMPBuilderCBHelpers::EmitCaptureStmt(*this, CodeGenIP, FiniBB,
- OutlinedFn, CapturedVars);
- } else {
- OMPBuilderCBHelpers::InlinedRegionBodyRAII IRB(*this, AllocaIP,
- FiniBB);
- OMPBuilderCBHelpers::EmitOMPRegionBody(*this, CS->getCapturedStmt(),
- CodeGenIP, FiniBB);
- }
- };
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- Builder.restoreIP(
- OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, !C));
- }
- return;
- }
- if (S.hasClausesOfKind<OMPDependClause>()) {
- assert(!S.hasAssociatedStmt() &&
- "No associated statement must be in ordered depend construct.");
- for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
- CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
- return;
- }
- const auto *C = S.getSingleClause<OMPSIMDClause>();
- auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- const CapturedStmt *CS = S.getInnermostCapturedStmt();
- if (C) {
- llvm::SmallVector<llvm::Value *, 16> CapturedVars;
- CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
- llvm::Function *OutlinedFn =
- emitOutlinedOrderedFunction(CGM, CS, S.getBeginLoc());
- CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
- OutlinedFn, CapturedVars);
- } else {
- Action.Enter(CGF);
- CGF.EmitStmt(CS->getCapturedStmt());
- }
- };
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
- }
- static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
- QualType SrcType, QualType DestType,
- SourceLocation Loc) {
- assert(CGF.hasScalarEvaluationKind(DestType) &&
- "DestType must have scalar evaluation kind.");
- assert(!Val.isAggregate() && "Must be a scalar or complex.");
- return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
- DestType, Loc)
- : CGF.EmitComplexToScalarConversion(
- Val.getComplexVal(), SrcType, DestType, Loc);
- }
- static CodeGenFunction::ComplexPairTy
- convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
- QualType DestType, SourceLocation Loc) {
- assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
- "DestType must have complex evaluation kind.");
- CodeGenFunction::ComplexPairTy ComplexVal;
- if (Val.isScalar()) {
- // Convert the input element to the element type of the complex.
- QualType DestElementType =
- DestType->castAs<ComplexType>()->getElementType();
- llvm::Value *ScalarVal = CGF.EmitScalarConversion(
- Val.getScalarVal(), SrcType, DestElementType, Loc);
- ComplexVal = CodeGenFunction::ComplexPairTy(
- ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
- } else {
- assert(Val.isComplex() && "Must be a scalar or complex.");
- QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
- QualType DestElementType =
- DestType->castAs<ComplexType>()->getElementType();
- ComplexVal.first = CGF.EmitScalarConversion(
- Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
- ComplexVal.second = CGF.EmitScalarConversion(
- Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
- }
- return ComplexVal;
- }
- static void emitSimpleAtomicStore(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
- LValue LVal, RValue RVal) {
- if (LVal.isGlobalReg())
- CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
- else
- CGF.EmitAtomicStore(RVal, LVal, AO, LVal.isVolatile(), /*isInit=*/false);
- }
- static RValue emitSimpleAtomicLoad(CodeGenFunction &CGF,
- llvm::AtomicOrdering AO, LValue LVal,
- SourceLocation Loc) {
- if (LVal.isGlobalReg())
- return CGF.EmitLoadOfLValue(LVal, Loc);
- return CGF.EmitAtomicLoad(
- LVal, Loc, llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO),
- LVal.isVolatile());
- }
- void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
- QualType RValTy, SourceLocation Loc) {
- switch (getEvaluationKind(LVal.getType())) {
- case TEK_Scalar:
- EmitStoreThroughLValue(RValue::get(convertToScalarValue(
- *this, RVal, RValTy, LVal.getType(), Loc)),
- LVal);
- break;
- case TEK_Complex:
- EmitStoreOfComplex(
- convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
- /*isInit=*/false);
- break;
- case TEK_Aggregate:
- llvm_unreachable("Must be a scalar or complex.");
- }
- }
- static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, llvm::AtomicOrdering AO,
- const Expr *X, const Expr *V,
- SourceLocation Loc) {
- // v = x;
- assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
- assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
- LValue XLValue = CGF.EmitLValue(X);
- LValue VLValue = CGF.EmitLValue(V);
- RValue Res = emitSimpleAtomicLoad(CGF, AO, XLValue, Loc);
- // OpenMP, 2.17.7, atomic Construct
- // If the read or capture clause is specified and the acquire, acq_rel, or
- // seq_cst clause is specified then the strong flush on exit from the atomic
- // operation is also an acquire flush.
- switch (AO) {
- case llvm::AtomicOrdering::Acquire:
- case llvm::AtomicOrdering::AcquireRelease:
- case llvm::AtomicOrdering::SequentiallyConsistent:
- CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc,
- llvm::AtomicOrdering::Acquire);
- break;
- case llvm::AtomicOrdering::Monotonic:
- case llvm::AtomicOrdering::Release:
- break;
- case llvm::AtomicOrdering::NotAtomic:
- case llvm::AtomicOrdering::Unordered:
- llvm_unreachable("Unexpected ordering.");
- }
- CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V);
- }
- static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF,
- llvm::AtomicOrdering AO, const Expr *X,
- const Expr *E, SourceLocation Loc) {
- // x = expr;
- assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
- emitSimpleAtomicStore(CGF, AO, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
- // OpenMP, 2.17.7, atomic Construct
- // If the write, update, or capture clause is specified and the release,
- // acq_rel, or seq_cst clause is specified then the strong flush on entry to
- // the atomic operation is also a release flush.
- switch (AO) {
- case llvm::AtomicOrdering::Release:
- case llvm::AtomicOrdering::AcquireRelease:
- case llvm::AtomicOrdering::SequentiallyConsistent:
- CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc,
- llvm::AtomicOrdering::Release);
- break;
- case llvm::AtomicOrdering::Acquire:
- case llvm::AtomicOrdering::Monotonic:
- break;
- case llvm::AtomicOrdering::NotAtomic:
- case llvm::AtomicOrdering::Unordered:
- llvm_unreachable("Unexpected ordering.");
- }
- }
- static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
- RValue Update,
- BinaryOperatorKind BO,
- llvm::AtomicOrdering AO,
- bool IsXLHSInRHSPart) {
- ASTContext &Context = CGF.getContext();
- // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
- // expression is simple and atomic is allowed for the given type for the
- // target platform.
- if (BO == BO_Comma || !Update.isScalar() ||
- !Update.getScalarVal()->getType()->isIntegerTy() || !X.isSimple() ||
- (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
- (Update.getScalarVal()->getType() !=
- X.getAddress(CGF).getElementType())) ||
- !X.getAddress(CGF).getElementType()->isIntegerTy() ||
- !Context.getTargetInfo().hasBuiltinAtomic(
- Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
- return std::make_pair(false, RValue::get(nullptr));
- llvm::AtomicRMWInst::BinOp RMWOp;
- switch (BO) {
- case BO_Add:
- RMWOp = llvm::AtomicRMWInst::Add;
- break;
- case BO_Sub:
- if (!IsXLHSInRHSPart)
- return std::make_pair(false, RValue::get(nullptr));
- RMWOp = llvm::AtomicRMWInst::Sub;
- break;
- case BO_And:
- RMWOp = llvm::AtomicRMWInst::And;
- break;
- case BO_Or:
- RMWOp = llvm::AtomicRMWInst::Or;
- break;
- case BO_Xor:
- RMWOp = llvm::AtomicRMWInst::Xor;
- break;
- case BO_LT:
- RMWOp = X.getType()->hasSignedIntegerRepresentation()
- ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
- : llvm::AtomicRMWInst::Max)
- : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
- : llvm::AtomicRMWInst::UMax);
- break;
- case BO_GT:
- RMWOp = X.getType()->hasSignedIntegerRepresentation()
- ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
- : llvm::AtomicRMWInst::Min)
- : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
- : llvm::AtomicRMWInst::UMin);
- break;
- case BO_Assign:
- RMWOp = llvm::AtomicRMWInst::Xchg;
- break;
- case BO_Mul:
- case BO_Div:
- case BO_Rem:
- case BO_Shl:
- case BO_Shr:
- case BO_LAnd:
- case BO_LOr:
- return std::make_pair(false, RValue::get(nullptr));
- case BO_PtrMemD:
- case BO_PtrMemI:
- case BO_LE:
- case BO_GE:
- case BO_EQ:
- case BO_NE:
- case BO_Cmp:
- case BO_AddAssign:
- case BO_SubAssign:
- case BO_AndAssign:
- case BO_OrAssign:
- case BO_XorAssign:
- case BO_MulAssign:
- case BO_DivAssign:
- case BO_RemAssign:
- case BO_ShlAssign:
- case BO_ShrAssign:
- case BO_Comma:
- llvm_unreachable("Unsupported atomic update operation");
- }
- llvm::Value *UpdateVal = Update.getScalarVal();
- if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
- UpdateVal = CGF.Builder.CreateIntCast(
- IC, X.getAddress(CGF).getElementType(),
- X.getType()->hasSignedIntegerRepresentation());
- }
- llvm::Value *Res =
- CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(CGF), UpdateVal, AO);
- return std::make_pair(true, RValue::get(Res));
- }
- std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
- LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
- llvm::AtomicOrdering AO, SourceLocation Loc,
- const llvm::function_ref<RValue(RValue)> CommonGen) {
- // Update expressions are allowed to have the following forms:
- // x binop= expr; -> xrval + expr;
- // x++, ++x -> xrval + 1;
- // x--, --x -> xrval - 1;
- // x = x binop expr; -> xrval binop expr
- // x = expr Op x; - > expr binop xrval;
- auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
- if (!Res.first) {
- if (X.isGlobalReg()) {
- // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
- // 'xrval'.
- EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
- } else {
- // Perform compare-and-swap procedure.
- EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
- }
- }
- return Res;
- }
- static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF,
- llvm::AtomicOrdering AO, const Expr *X,
- const Expr *E, const Expr *UE,
- bool IsXLHSInRHSPart, SourceLocation Loc) {
- assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
- "Update expr in 'atomic update' must be a binary operator.");
- const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
- // Update expressions are allowed to have the following forms:
- // x binop= expr; -> xrval + expr;
- // x++, ++x -> xrval + 1;
- // x--, --x -> xrval - 1;
- // x = x binop expr; -> xrval binop expr
- // x = expr Op x; - > expr binop xrval;
- assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
- LValue XLValue = CGF.EmitLValue(X);
- RValue ExprRValue = CGF.EmitAnyExpr(E);
- const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
- const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
- const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
- const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
- auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
- CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
- CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
- return CGF.EmitAnyExpr(UE);
- };
- (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
- XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
- // OpenMP, 2.17.7, atomic Construct
- // If the write, update, or capture clause is specified and the release,
- // acq_rel, or seq_cst clause is specified then the strong flush on entry to
- // the atomic operation is also a release flush.
- switch (AO) {
- case llvm::AtomicOrdering::Release:
- case llvm::AtomicOrdering::AcquireRelease:
- case llvm::AtomicOrdering::SequentiallyConsistent:
- CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc,
- llvm::AtomicOrdering::Release);
- break;
- case llvm::AtomicOrdering::Acquire:
- case llvm::AtomicOrdering::Monotonic:
- break;
- case llvm::AtomicOrdering::NotAtomic:
- case llvm::AtomicOrdering::Unordered:
- llvm_unreachable("Unexpected ordering.");
- }
- }
- static RValue convertToType(CodeGenFunction &CGF, RValue Value,
- QualType SourceType, QualType ResType,
- SourceLocation Loc) {
- switch (CGF.getEvaluationKind(ResType)) {
- case TEK_Scalar:
- return RValue::get(
- convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
- case TEK_Complex: {
- auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
- return RValue::getComplex(Res.first, Res.second);
- }
- case TEK_Aggregate:
- break;
- }
- llvm_unreachable("Must be a scalar or complex.");
- }
- static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF,
- llvm::AtomicOrdering AO,
- bool IsPostfixUpdate, const Expr *V,
- const Expr *X, const Expr *E,
- const Expr *UE, bool IsXLHSInRHSPart,
- SourceLocation Loc) {
- assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
- assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
- RValue NewVVal;
- LValue VLValue = CGF.EmitLValue(V);
- LValue XLValue = CGF.EmitLValue(X);
- RValue ExprRValue = CGF.EmitAnyExpr(E);
- QualType NewVValType;
- if (UE) {
- // 'x' is updated with some additional value.
- assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
- "Update expr in 'atomic capture' must be a binary operator.");
- const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
- // Update expressions are allowed to have the following forms:
- // x binop= expr; -> xrval + expr;
- // x++, ++x -> xrval + 1;
- // x--, --x -> xrval - 1;
- // x = x binop expr; -> xrval binop expr
- // x = expr Op x; - > expr binop xrval;
- const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
- const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
- const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
- NewVValType = XRValExpr->getType();
- const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
- auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
- IsPostfixUpdate](RValue XRValue) {
- CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
- CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
- RValue Res = CGF.EmitAnyExpr(UE);
- NewVVal = IsPostfixUpdate ? XRValue : Res;
- return Res;
- };
- auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
- XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
- if (Res.first) {
- // 'atomicrmw' instruction was generated.
- if (IsPostfixUpdate) {
- // Use old value from 'atomicrmw'.
- NewVVal = Res.second;
- } else {
- // 'atomicrmw' does not provide new value, so evaluate it using old
- // value of 'x'.
- CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
- CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
- NewVVal = CGF.EmitAnyExpr(UE);
- }
- }
- } else {
- // 'x' is simply rewritten with some 'expr'.
- NewVValType = X->getType().getNonReferenceType();
- ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
- X->getType().getNonReferenceType(), Loc);
- auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
- NewVVal = XRValue;
- return ExprRValue;
- };
- // Try to perform atomicrmw xchg, otherwise simple exchange.
- auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
- XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
- Loc, Gen);
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, X);
- if (Res.first) {
- // 'atomicrmw' instruction was generated.
- NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
- }
- }
- // Emit post-update store to 'v' of old/new 'x' value.
- CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
- CGF.CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(CGF, V);
- // OpenMP 5.1 removes the required flush for capture clause.
- if (CGF.CGM.getLangOpts().OpenMP < 51) {
- // OpenMP, 2.17.7, atomic Construct
- // If the write, update, or capture clause is specified and the release,
- // acq_rel, or seq_cst clause is specified then the strong flush on entry to
- // the atomic operation is also a release flush.
- // If the read or capture clause is specified and the acquire, acq_rel, or
- // seq_cst clause is specified then the strong flush on exit from the atomic
- // operation is also an acquire flush.
- switch (AO) {
- case llvm::AtomicOrdering::Release:
- CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc,
- llvm::AtomicOrdering::Release);
- break;
- case llvm::AtomicOrdering::Acquire:
- CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc,
- llvm::AtomicOrdering::Acquire);
- break;
- case llvm::AtomicOrdering::AcquireRelease:
- case llvm::AtomicOrdering::SequentiallyConsistent:
- CGF.CGM.getOpenMPRuntime().emitFlush(
- CGF, llvm::None, Loc, llvm::AtomicOrdering::AcquireRelease);
- break;
- case llvm::AtomicOrdering::Monotonic:
- break;
- case llvm::AtomicOrdering::NotAtomic:
- case llvm::AtomicOrdering::Unordered:
- llvm_unreachable("Unexpected ordering.");
- }
- }
- }
- static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
- llvm::AtomicOrdering AO, bool IsPostfixUpdate,
- const Expr *X, const Expr *V, const Expr *E,
- const Expr *UE, bool IsXLHSInRHSPart,
- SourceLocation Loc) {
- switch (Kind) {
- case OMPC_read:
- emitOMPAtomicReadExpr(CGF, AO, X, V, Loc);
- break;
- case OMPC_write:
- emitOMPAtomicWriteExpr(CGF, AO, X, E, Loc);
- break;
- case OMPC_unknown:
- case OMPC_update:
- emitOMPAtomicUpdateExpr(CGF, AO, X, E, UE, IsXLHSInRHSPart, Loc);
- break;
- case OMPC_capture:
- emitOMPAtomicCaptureExpr(CGF, AO, IsPostfixUpdate, V, X, E, UE,
- IsXLHSInRHSPart, Loc);
- break;
- case OMPC_compare:
- // Do nothing here as we already emit an error.
- break;
- case OMPC_if:
- case OMPC_final:
- case OMPC_num_threads:
- case OMPC_private:
- case OMPC_firstprivate:
- case OMPC_lastprivate:
- case OMPC_reduction:
- case OMPC_task_reduction:
- case OMPC_in_reduction:
- case OMPC_safelen:
- case OMPC_simdlen:
- case OMPC_sizes:
- case OMPC_full:
- case OMPC_partial:
- case OMPC_allocator:
- case OMPC_allocate:
- case OMPC_collapse:
- case OMPC_default:
- case OMPC_seq_cst:
- case OMPC_acq_rel:
- case OMPC_acquire:
- case OMPC_release:
- case OMPC_relaxed:
- case OMPC_shared:
- case OMPC_linear:
- case OMPC_aligned:
- case OMPC_copyin:
- case OMPC_copyprivate:
- case OMPC_flush:
- case OMPC_depobj:
- case OMPC_proc_bind:
- case OMPC_schedule:
- case OMPC_ordered:
- case OMPC_nowait:
- case OMPC_untied:
- case OMPC_threadprivate:
- case OMPC_depend:
- case OMPC_mergeable:
- case OMPC_device:
- case OMPC_threads:
- case OMPC_simd:
- case OMPC_map:
- case OMPC_num_teams:
- case OMPC_thread_limit:
- case OMPC_priority:
- case OMPC_grainsize:
- case OMPC_nogroup:
- case OMPC_num_tasks:
- case OMPC_hint:
- case OMPC_dist_schedule:
- case OMPC_defaultmap:
- case OMPC_uniform:
- case OMPC_to:
- case OMPC_from:
- case OMPC_use_device_ptr:
- case OMPC_use_device_addr:
- case OMPC_is_device_ptr:
- case OMPC_unified_address:
- case OMPC_unified_shared_memory:
- case OMPC_reverse_offload:
- case OMPC_dynamic_allocators:
- case OMPC_atomic_default_mem_order:
- case OMPC_device_type:
- case OMPC_match:
- case OMPC_nontemporal:
- case OMPC_order:
- case OMPC_destroy:
- case OMPC_detach:
- case OMPC_inclusive:
- case OMPC_exclusive:
- case OMPC_uses_allocators:
- case OMPC_affinity:
- case OMPC_init:
- case OMPC_inbranch:
- case OMPC_notinbranch:
- case OMPC_link:
- case OMPC_indirect:
- case OMPC_use:
- case OMPC_novariants:
- case OMPC_nocontext:
- case OMPC_filter:
- case OMPC_when:
- case OMPC_adjust_args:
- case OMPC_append_args:
- case OMPC_memory_order:
- case OMPC_bind:
- case OMPC_align:
- llvm_unreachable("Clause is not allowed in 'omp atomic'.");
- }
- }
- void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
- llvm::AtomicOrdering AO = llvm::AtomicOrdering::Monotonic;
- bool MemOrderingSpecified = false;
- if (S.getSingleClause<OMPSeqCstClause>()) {
- AO = llvm::AtomicOrdering::SequentiallyConsistent;
- MemOrderingSpecified = true;
- } else if (S.getSingleClause<OMPAcqRelClause>()) {
- AO = llvm::AtomicOrdering::AcquireRelease;
- MemOrderingSpecified = true;
- } else if (S.getSingleClause<OMPAcquireClause>()) {
- AO = llvm::AtomicOrdering::Acquire;
- MemOrderingSpecified = true;
- } else if (S.getSingleClause<OMPReleaseClause>()) {
- AO = llvm::AtomicOrdering::Release;
- MemOrderingSpecified = true;
- } else if (S.getSingleClause<OMPRelaxedClause>()) {
- AO = llvm::AtomicOrdering::Monotonic;
- MemOrderingSpecified = true;
- }
- OpenMPClauseKind Kind = OMPC_unknown;
- for (const OMPClause *C : S.clauses()) {
- // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
- // if it is first).
- if (C->getClauseKind() != OMPC_seq_cst &&
- C->getClauseKind() != OMPC_acq_rel &&
- C->getClauseKind() != OMPC_acquire &&
- C->getClauseKind() != OMPC_release &&
- C->getClauseKind() != OMPC_relaxed && C->getClauseKind() != OMPC_hint) {
- Kind = C->getClauseKind();
- break;
- }
- }
- if (!MemOrderingSpecified) {
- llvm::AtomicOrdering DefaultOrder =
- CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
- if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
- DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
- (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
- Kind == OMPC_capture)) {
- AO = DefaultOrder;
- } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
- if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
- AO = llvm::AtomicOrdering::Release;
- } else if (Kind == OMPC_read) {
- assert(Kind == OMPC_read && "Unexpected atomic kind.");
- AO = llvm::AtomicOrdering::Acquire;
- }
- }
- }
- LexicalScope Scope(*this, S.getSourceRange());
- EmitStopPoint(S.getAssociatedStmt());
- emitOMPAtomicExpr(*this, Kind, AO, S.isPostfixUpdate(), S.getX(), S.getV(),
- S.getExpr(), S.getUpdateExpr(), S.isXLHSInRHSPart(),
- S.getBeginLoc());
- }
- static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
- const OMPExecutableDirective &S,
- const RegionCodeGenTy &CodeGen) {
- assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
- CodeGenModule &CGM = CGF.CGM;
- // On device emit this construct as inlined code.
- if (CGM.getLangOpts().OpenMPIsDevice) {
- OMPLexicalScope Scope(CGF, S, OMPD_target);
- CGM.getOpenMPRuntime().emitInlinedDirective(
- CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
- });
- return;
- }
- auto LPCRegion = CGOpenMPRuntime::LastprivateConditionalRAII::disable(CGF, S);
- llvm::Function *Fn = nullptr;
- llvm::Constant *FnID = nullptr;
- const Expr *IfCond = nullptr;
- // Check for the at most one if clause associated with the target region.
- for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
- if (C->getNameModifier() == OMPD_unknown ||
- C->getNameModifier() == OMPD_target) {
- IfCond = C->getCondition();
- break;
- }
- }
- // Check if we have any device clause associated with the directive.
- llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device(
- nullptr, OMPC_DEVICE_unknown);
- if (auto *C = S.getSingleClause<OMPDeviceClause>())
- Device.setPointerAndInt(C->getDevice(), C->getModifier());
- // Check if we have an if clause whose conditional always evaluates to false
- // or if we do not have any targets specified. If so the target region is not
- // an offload entry point.
- bool IsOffloadEntry = true;
- if (IfCond) {
- bool Val;
- if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
- IsOffloadEntry = false;
- }
- if (CGM.getLangOpts().OMPTargetTriples.empty())
- IsOffloadEntry = false;
- assert(CGF.CurFuncDecl && "No parent declaration for target region!");
- StringRef ParentName;
- // In case we have Ctors/Dtors we use the complete type variant to produce
- // the mangling of the device outlined kernel.
- if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
- ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
- else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
- ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
- else
- ParentName =
- CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
- IsOffloadEntry, CodeGen);
- OMPLexicalScope Scope(CGF, S, OMPD_task);
- auto &&SizeEmitter =
- [IsOffloadEntry](CodeGenFunction &CGF,
- const OMPLoopDirective &D) -> llvm::Value * {
- if (IsOffloadEntry) {
- OMPLoopScope(CGF, D);
- // Emit calculation of the iterations count.
- llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
- NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
- /*isSigned=*/false);
- return NumIterations;
- }
- return nullptr;
- };
- CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
- SizeEmitter);
- }
- static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
- CGF.EmitOMPPrivateClause(S, PrivateScope);
- (void)PrivateScope.Privatize();
- if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
- CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
- CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
- CGF.EnsureInsertPoint();
- }
- void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
- StringRef ParentName,
- const OMPTargetDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetRegion(CGF, S, Action);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetRegion(CGF, S, Action);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
- const OMPExecutableDirective &S,
- OpenMPDirectiveKind InnermostKind,
- const RegionCodeGenTy &CodeGen) {
- const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
- llvm::Function *OutlinedFn =
- CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
- S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
- const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
- const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
- if (NT || TL) {
- const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
- const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
- CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
- S.getBeginLoc());
- }
- OMPTeamsScope Scope(CGF, S);
- llvm::SmallVector<llvm::Value *, 16> CapturedVars;
- CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
- CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
- CapturedVars);
- }
- void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
- // Emit teams region as a standalone region.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- OMPPrivateScope PrivateScope(CGF);
- (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
- CGF.EmitOMPPrivateClause(S, PrivateScope);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
- emitPostUpdateForReductionClause(*this, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
- const OMPTargetTeamsDirective &S) {
- auto *CS = S.getCapturedStmt(OMPD_teams);
- Action.Enter(CGF);
- // Emit teams region as a standalone region.
- auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
- CGF.EmitOMPPrivateClause(S, PrivateScope);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
- CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
- CGF.EmitStmt(CS->getCapturedStmt());
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
- emitPostUpdateForReductionClause(CGF, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName,
- const OMPTargetTeamsDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsRegion(CGF, Action, S);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetTeamsDirective(
- const OMPTargetTeamsDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsRegion(CGF, Action, S);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- static void
- emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
- const OMPTargetTeamsDistributeDirective &S) {
- Action.Enter(CGF);
- auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
- };
- // Emit teams region as a standalone region.
- auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
- CodeGenDistribute);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
- emitPostUpdateForReductionClause(CGF, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName,
- const OMPTargetTeamsDistributeDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsDistributeRegion(CGF, Action, S);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
- const OMPTargetTeamsDistributeDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsDistributeRegion(CGF, Action, S);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- static void emitTargetTeamsDistributeSimdRegion(
- CodeGenFunction &CGF, PrePostActionTy &Action,
- const OMPTargetTeamsDistributeSimdDirective &S) {
- Action.Enter(CGF);
- auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
- };
- // Emit teams region as a standalone region.
- auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
- CodeGenDistribute);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
- emitPostUpdateForReductionClause(CGF, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName,
- const OMPTargetTeamsDistributeSimdDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
- const OMPTargetTeamsDistributeSimdDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- void CodeGenFunction::EmitOMPTeamsDistributeDirective(
- const OMPTeamsDistributeDirective &S) {
- auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
- };
- // Emit teams region as a standalone region.
- auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
- CodeGenDistribute);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
- emitPostUpdateForReductionClause(*this, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
- const OMPTeamsDistributeSimdDirective &S) {
- auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
- };
- // Emit teams region as a standalone region.
- auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
- CodeGenDistribute);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
- emitPostUpdateForReductionClause(*this, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
- const OMPTeamsDistributeParallelForDirective &S) {
- auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
- S.getDistInc());
- };
- // Emit teams region as a standalone region.
- auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
- CodeGenDistribute);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
- emitPostUpdateForReductionClause(*this, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
- const OMPTeamsDistributeParallelForSimdDirective &S) {
- auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
- S.getDistInc());
- };
- // Emit teams region as a standalone region.
- auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
- CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for_simd,
- CodeGen);
- emitPostUpdateForReductionClause(*this, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPInteropDirective(const OMPInteropDirective &S) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- llvm::Value *Device = nullptr;
- if (const auto *C = S.getSingleClause<OMPDeviceClause>())
- Device = EmitScalarExpr(C->getDevice());
- llvm::Value *NumDependences = nullptr;
- llvm::Value *DependenceAddress = nullptr;
- if (const auto *DC = S.getSingleClause<OMPDependClause>()) {
- OMPTaskDataTy::DependData Dependencies(DC->getDependencyKind(),
- DC->getModifier());
- Dependencies.DepExprs.append(DC->varlist_begin(), DC->varlist_end());
- std::pair<llvm::Value *, Address> DependencePair =
- CGM.getOpenMPRuntime().emitDependClause(*this, Dependencies,
- DC->getBeginLoc());
- NumDependences = DependencePair.first;
- DependenceAddress = Builder.CreatePointerCast(
- DependencePair.second.getPointer(), CGM.Int8PtrTy);
- }
- assert(!(S.hasClausesOfKind<OMPNowaitClause>() &&
- !(S.getSingleClause<OMPInitClause>() ||
- S.getSingleClause<OMPDestroyClause>() ||
- S.getSingleClause<OMPUseClause>())) &&
- "OMPNowaitClause clause is used separately in OMPInteropDirective.");
- if (const auto *C = S.getSingleClause<OMPInitClause>()) {
- llvm::Value *InteropvarPtr =
- EmitLValue(C->getInteropVar()).getPointer(*this);
- llvm::omp::OMPInteropType InteropType = llvm::omp::OMPInteropType::Unknown;
- if (C->getIsTarget()) {
- InteropType = llvm::omp::OMPInteropType::Target;
- } else {
- assert(C->getIsTargetSync() && "Expected interop-type target/targetsync");
- InteropType = llvm::omp::OMPInteropType::TargetSync;
- }
- OMPBuilder.createOMPInteropInit(Builder, InteropvarPtr, InteropType, Device,
- NumDependences, DependenceAddress,
- S.hasClausesOfKind<OMPNowaitClause>());
- } else if (const auto *C = S.getSingleClause<OMPDestroyClause>()) {
- llvm::Value *InteropvarPtr =
- EmitLValue(C->getInteropVar()).getPointer(*this);
- OMPBuilder.createOMPInteropDestroy(Builder, InteropvarPtr, Device,
- NumDependences, DependenceAddress,
- S.hasClausesOfKind<OMPNowaitClause>());
- } else if (const auto *C = S.getSingleClause<OMPUseClause>()) {
- llvm::Value *InteropvarPtr =
- EmitLValue(C->getInteropVar()).getPointer(*this);
- OMPBuilder.createOMPInteropUse(Builder, InteropvarPtr, Device,
- NumDependences, DependenceAddress,
- S.hasClausesOfKind<OMPNowaitClause>());
- }
- }
- static void emitTargetTeamsDistributeParallelForRegion(
- CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
- S.getDistInc());
- };
- // Emit teams region as a standalone region.
- auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
- CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
- CodeGenTeams);
- emitPostUpdateForReductionClause(CGF, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName,
- const OMPTargetTeamsDistributeParallelForDirective &S) {
- // Emit SPMD target teams distribute parallel for region as a standalone
- // region.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
- const OMPTargetTeamsDistributeParallelForDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- static void emitTargetTeamsDistributeParallelForSimdRegion(
- CodeGenFunction &CGF,
- const OMPTargetTeamsDistributeParallelForSimdDirective &S,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
- S.getDistInc());
- };
- // Emit teams region as a standalone region.
- auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
- CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
- };
- emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
- CodeGenTeams);
- emitPostUpdateForReductionClause(CGF, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName,
- const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
- // Emit SPMD target teams distribute parallel for simd region as a standalone
- // region.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
- const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- void CodeGenFunction::EmitOMPCancellationPointDirective(
- const OMPCancellationPointDirective &S) {
- CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
- S.getCancelRegion());
- }
- void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
- const Expr *IfCond = nullptr;
- for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
- if (C->getNameModifier() == OMPD_unknown ||
- C->getNameModifier() == OMPD_cancel) {
- IfCond = C->getCondition();
- break;
- }
- }
- if (CGM.getLangOpts().OpenMPIRBuilder) {
- llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
- // TODO: This check is necessary as we only generate `omp parallel` through
- // the OpenMPIRBuilder for now.
- if (S.getCancelRegion() == OMPD_parallel ||
- S.getCancelRegion() == OMPD_sections ||
- S.getCancelRegion() == OMPD_section) {
- llvm::Value *IfCondition = nullptr;
- if (IfCond)
- IfCondition = EmitScalarExpr(IfCond,
- /*IgnoreResultAssign=*/true);
- return Builder.restoreIP(
- OMPBuilder.createCancel(Builder, IfCondition, S.getCancelRegion()));
- }
- }
- CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
- S.getCancelRegion());
- }
- CodeGenFunction::JumpDest
- CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
- if (Kind == OMPD_parallel || Kind == OMPD_task ||
- Kind == OMPD_target_parallel || Kind == OMPD_taskloop ||
- Kind == OMPD_master_taskloop || Kind == OMPD_parallel_master_taskloop)
- return ReturnBlock;
- assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
- Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
- Kind == OMPD_distribute_parallel_for ||
- Kind == OMPD_target_parallel_for ||
- Kind == OMPD_teams_distribute_parallel_for ||
- Kind == OMPD_target_teams_distribute_parallel_for);
- return OMPCancelStack.getExitBlock();
- }
- void CodeGenFunction::EmitOMPUseDevicePtrClause(
- const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,
- const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
- auto OrigVarIt = C.varlist_begin();
- auto InitIt = C.inits().begin();
- for (const Expr *PvtVarIt : C.private_copies()) {
- const auto *OrigVD =
- cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
- const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
- const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
- // In order to identify the right initializer we need to match the
- // declaration used by the mapping logic. In some cases we may get
- // OMPCapturedExprDecl that refers to the original declaration.
- const ValueDecl *MatchingVD = OrigVD;
- if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
- // OMPCapturedExprDecl are used to privative fields of the current
- // structure.
- const auto *ME = cast<MemberExpr>(OED->getInit());
- assert(isa<CXXThisExpr>(ME->getBase()) &&
- "Base should be the current struct!");
- MatchingVD = ME->getMemberDecl();
- }
- // If we don't have information about the current list item, move on to
- // the next one.
- auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
- if (InitAddrIt == CaptureDeviceAddrMap.end())
- continue;
- bool IsRegistered = PrivateScope.addPrivate(
- OrigVD, [this, OrigVD, InitAddrIt, InitVD, PvtVD]() {
- // Initialize the temporary initialization variable with the address
- // we get from the runtime library. We have to cast the source address
- // because it is always a void *. References are materialized in the
- // privatization scope, so the initialization here disregards the fact
- // the original variable is a reference.
- QualType AddrQTy = getContext().getPointerType(
- OrigVD->getType().getNonReferenceType());
- llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
- Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
- setAddrOfLocalVar(InitVD, InitAddr);
- // Emit private declaration, it will be initialized by the value we
- // declaration we just added to the local declarations map.
- EmitDecl(*PvtVD);
- // The initialization variables reached its purpose in the emission
- // of the previous declaration, so we don't need it anymore.
- LocalDeclMap.erase(InitVD);
- // Return the address of the private variable.
- return GetAddrOfLocalVar(PvtVD);
- });
- assert(IsRegistered && "firstprivate var already registered as private");
- // Silence the warning about unused variable.
- (void)IsRegistered;
- ++OrigVarIt;
- ++InitIt;
- }
- }
- static const VarDecl *getBaseDecl(const Expr *Ref) {
- const Expr *Base = Ref->IgnoreParenImpCasts();
- while (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Base))
- Base = OASE->getBase()->IgnoreParenImpCasts();
- while (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Base))
- Base = ASE->getBase()->IgnoreParenImpCasts();
- return cast<VarDecl>(cast<DeclRefExpr>(Base)->getDecl());
- }
- void CodeGenFunction::EmitOMPUseDeviceAddrClause(
- const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,
- const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
- llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
- for (const Expr *Ref : C.varlists()) {
- const VarDecl *OrigVD = getBaseDecl(Ref);
- if (!Processed.insert(OrigVD).second)
- continue;
- // In order to identify the right initializer we need to match the
- // declaration used by the mapping logic. In some cases we may get
- // OMPCapturedExprDecl that refers to the original declaration.
- const ValueDecl *MatchingVD = OrigVD;
- if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
- // OMPCapturedExprDecl are used to privative fields of the current
- // structure.
- const auto *ME = cast<MemberExpr>(OED->getInit());
- assert(isa<CXXThisExpr>(ME->getBase()) &&
- "Base should be the current struct!");
- MatchingVD = ME->getMemberDecl();
- }
- // If we don't have information about the current list item, move on to
- // the next one.
- auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
- if (InitAddrIt == CaptureDeviceAddrMap.end())
- continue;
- Address PrivAddr = InitAddrIt->getSecond();
- // For declrefs and variable length array need to load the pointer for
- // correct mapping, since the pointer to the data was passed to the runtime.
- if (isa<DeclRefExpr>(Ref->IgnoreParenImpCasts()) ||
- MatchingVD->getType()->isArrayType())
- PrivAddr =
- EmitLoadOfPointer(PrivAddr, getContext()
- .getPointerType(OrigVD->getType())
- ->castAs<PointerType>());
- llvm::Type *RealTy =
- ConvertTypeForMem(OrigVD->getType().getNonReferenceType())
- ->getPointerTo();
- PrivAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(PrivAddr, RealTy);
- (void)PrivateScope.addPrivate(OrigVD, [PrivAddr]() { return PrivAddr; });
- }
- }
- // Generate the instructions for '#pragma omp target data' directive.
- void CodeGenFunction::EmitOMPTargetDataDirective(
- const OMPTargetDataDirective &S) {
- CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true,
- /*SeparateBeginEndCalls=*/true);
- // Create a pre/post action to signal the privatization of the device pointer.
- // This action can be replaced by the OpenMP runtime code generation to
- // deactivate privatization.
- bool PrivatizeDevicePointers = false;
- class DevicePointerPrivActionTy : public PrePostActionTy {
- bool &PrivatizeDevicePointers;
- public:
- explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
- : PrivatizeDevicePointers(PrivatizeDevicePointers) {}
- void Enter(CodeGenFunction &CGF) override {
- PrivatizeDevicePointers = true;
- }
- };
- DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
- auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
- CodeGenFunction &CGF, PrePostActionTy &Action) {
- auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
- };
- // Codegen that selects whether to generate the privatization code or not.
- auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
- &InnermostCodeGen](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- RegionCodeGenTy RCG(InnermostCodeGen);
- PrivatizeDevicePointers = false;
- // Call the pre-action to change the status of PrivatizeDevicePointers if
- // needed.
- Action.Enter(CGF);
- if (PrivatizeDevicePointers) {
- OMPPrivateScope PrivateScope(CGF);
- // Emit all instances of the use_device_ptr clause.
- for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
- CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
- Info.CaptureDeviceAddrMap);
- for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
- CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope,
- Info.CaptureDeviceAddrMap);
- (void)PrivateScope.Privatize();
- RCG(CGF);
- } else {
- OMPLexicalScope Scope(CGF, S, OMPD_unknown);
- RCG(CGF);
- }
- };
- // Forward the provided action to the privatization codegen.
- RegionCodeGenTy PrivRCG(PrivCodeGen);
- PrivRCG.setAction(Action);
- // Notwithstanding the body of the region is emitted as inlined directive,
- // we don't use an inline scope as changes in the references inside the
- // region are expected to be visible outside, so we do not privative them.
- OMPLexicalScope Scope(CGF, S);
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
- PrivRCG);
- };
- RegionCodeGenTy RCG(CodeGen);
- // If we don't have target devices, don't bother emitting the data mapping
- // code.
- if (CGM.getLangOpts().OMPTargetTriples.empty()) {
- RCG(*this);
- return;
- }
- // Check if we have any if clause associated with the directive.
- const Expr *IfCond = nullptr;
- if (const auto *C = S.getSingleClause<OMPIfClause>())
- IfCond = C->getCondition();
- // Check if we have any device clause associated with the directive.
- const Expr *Device = nullptr;
- if (const auto *C = S.getSingleClause<OMPDeviceClause>())
- Device = C->getDevice();
- // Set the action to signal privatization of device pointers.
- RCG.setAction(PrivAction);
- // Emit region code.
- CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
- Info);
- }
- void CodeGenFunction::EmitOMPTargetEnterDataDirective(
- const OMPTargetEnterDataDirective &S) {
- // If we don't have target devices, don't bother emitting the data mapping
- // code.
- if (CGM.getLangOpts().OMPTargetTriples.empty())
- return;
- // Check if we have any if clause associated with the directive.
- const Expr *IfCond = nullptr;
- if (const auto *C = S.getSingleClause<OMPIfClause>())
- IfCond = C->getCondition();
- // Check if we have any device clause associated with the directive.
- const Expr *Device = nullptr;
- if (const auto *C = S.getSingleClause<OMPDeviceClause>())
- Device = C->getDevice();
- OMPLexicalScope Scope(*this, S, OMPD_task);
- CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
- }
- void CodeGenFunction::EmitOMPTargetExitDataDirective(
- const OMPTargetExitDataDirective &S) {
- // If we don't have target devices, don't bother emitting the data mapping
- // code.
- if (CGM.getLangOpts().OMPTargetTriples.empty())
- return;
- // Check if we have any if clause associated with the directive.
- const Expr *IfCond = nullptr;
- if (const auto *C = S.getSingleClause<OMPIfClause>())
- IfCond = C->getCondition();
- // Check if we have any device clause associated with the directive.
- const Expr *Device = nullptr;
- if (const auto *C = S.getSingleClause<OMPDeviceClause>())
- Device = C->getDevice();
- OMPLexicalScope Scope(*this, S, OMPD_task);
- CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
- }
- static void emitTargetParallelRegion(CodeGenFunction &CGF,
- const OMPTargetParallelDirective &S,
- PrePostActionTy &Action) {
- // Get the captured statement associated with the 'parallel' region.
- const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
- Action.Enter(CGF);
- auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
- (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
- CGF.EmitOMPPrivateClause(S, PrivateScope);
- CGF.EmitOMPReductionClauseInit(S, PrivateScope);
- (void)PrivateScope.Privatize();
- if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
- CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
- // TODO: Add support for clauses.
- CGF.EmitStmt(CS->getCapturedStmt());
- CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
- };
- emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
- emitEmptyBoundParameters);
- emitPostUpdateForReductionClause(CGF, S,
- [](CodeGenFunction &) { return nullptr; });
- }
- void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName,
- const OMPTargetParallelDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetParallelRegion(CGF, S, Action);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetParallelDirective(
- const OMPTargetParallelDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetParallelRegion(CGF, S, Action);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- static void emitTargetParallelForRegion(CodeGenFunction &CGF,
- const OMPTargetParallelForDirective &S,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- // Emit directive as a combined directive that consists of two implicit
- // directives: 'parallel' with 'for' directive.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CodeGenFunction::OMPCancelStackRAII CancelRegion(
- CGF, OMPD_target_parallel_for, S.hasCancel());
- CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
- emitDispatchForLoopBounds);
- };
- emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
- emitEmptyBoundParameters);
- }
- void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName,
- const OMPTargetParallelForDirective &S) {
- // Emit SPMD target parallel for region as a standalone region.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetParallelForRegion(CGF, S, Action);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetParallelForDirective(
- const OMPTargetParallelForDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetParallelForRegion(CGF, S, Action);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- static void
- emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
- const OMPTargetParallelForSimdDirective &S,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- // Emit directive as a combined directive that consists of two implicit
- // directives: 'parallel' with 'for' directive.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
- emitDispatchForLoopBounds);
- };
- emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
- emitEmptyBoundParameters);
- }
- void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
- CodeGenModule &CGM, StringRef ParentName,
- const OMPTargetParallelForSimdDirective &S) {
- // Emit SPMD target parallel for region as a standalone region.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetParallelForSimdRegion(CGF, S, Action);
- };
- llvm::Function *Fn;
- llvm::Constant *Addr;
- // Emit target region as a standalone region.
- CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
- S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
- assert(Fn && Addr && "Target device function emission failed.");
- }
- void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
- const OMPTargetParallelForSimdDirective &S) {
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- emitTargetParallelForSimdRegion(CGF, S, Action);
- };
- emitCommonOMPTargetDirective(*this, S, CodeGen);
- }
- /// Emit a helper variable and return corresponding lvalue.
- static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
- const ImplicitParamDecl *PVD,
- CodeGenFunction::OMPPrivateScope &Privates) {
- const auto *VDecl = cast<VarDecl>(Helper->getDecl());
- Privates.addPrivate(VDecl,
- [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
- }
- void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
- assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
- // Emit outlined function for task construct.
- const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
- Address CapturedStruct = Address::invalid();
- {
- OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
- CapturedStruct = GenerateCapturedStmtArgument(*CS);
- }
- QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
- const Expr *IfCond = nullptr;
- for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
- if (C->getNameModifier() == OMPD_unknown ||
- C->getNameModifier() == OMPD_taskloop) {
- IfCond = C->getCondition();
- break;
- }
- }
- OMPTaskDataTy Data;
- // Check if taskloop must be emitted without taskgroup.
- Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
- // TODO: Check if we should emit tied or untied task.
- Data.Tied = true;
- // Set scheduling for taskloop
- if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
- // grainsize clause
- Data.Schedule.setInt(/*IntVal=*/false);
- Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
- } else if (const auto *Clause = S.getSingleClause<OMPNumTasksClause>()) {
- // num_tasks clause
- Data.Schedule.setInt(/*IntVal=*/true);
- Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
- }
- auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
- // if (PreCond) {
- // for (IV in 0..LastIteration) BODY;
- // <Final counter/linear vars updates>;
- // }
- //
- // Emit: if (PreCond) - begin.
- // If the condition constant folds and can be elided, avoid emitting the
- // whole loop.
- bool CondConstant;
- llvm::BasicBlock *ContBlock = nullptr;
- OMPLoopScope PreInitScope(CGF, S);
- if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
- if (!CondConstant)
- return;
- } else {
- llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
- ContBlock = CGF.createBasicBlock("taskloop.if.end");
- emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
- CGF.getProfileCount(&S));
- CGF.EmitBlock(ThenBlock);
- CGF.incrementProfileCounter(&S);
- }
- (void)CGF.EmitOMPLinearClauseInit(S);
- OMPPrivateScope LoopScope(CGF);
- // Emit helper vars inits.
- enum { LowerBound = 5, UpperBound, Stride, LastIter };
- auto *I = CS->getCapturedDecl()->param_begin();
- auto *LBP = std::next(I, LowerBound);
- auto *UBP = std::next(I, UpperBound);
- auto *STP = std::next(I, Stride);
- auto *LIP = std::next(I, LastIter);
- mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
- LoopScope);
- mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
- LoopScope);
- mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
- mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
- LoopScope);
- CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
- CGF.EmitOMPLinearClause(S, LoopScope);
- bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
- (void)LoopScope.Privatize();
- // Emit the loop iteration variable.
- const Expr *IVExpr = S.getIterationVariable();
- const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
- CGF.EmitVarDecl(*IVDecl);
- CGF.EmitIgnoredExpr(S.getInit());
- // Emit the iterations count variable.
- // If it is not a variable, Sema decided to calculate iterations count on
- // each iteration (e.g., it is foldable into a constant).
- if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
- CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
- // Emit calculation of the iterations count.
- CGF.EmitIgnoredExpr(S.getCalcLastIteration());
- }
- {
- OMPLexicalScope Scope(CGF, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
- emitCommonSimdLoop(
- CGF, S,
- [&S](CodeGenFunction &CGF, PrePostActionTy &) {
- if (isOpenMPSimdDirective(S.getDirectiveKind()))
- CGF.EmitOMPSimdInit(S);
- },
- [&S, &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
- CGF.EmitOMPInnerLoop(
- S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),
- [&S](CodeGenFunction &CGF) {
- emitOMPLoopBodyWithStopPoint(CGF, S,
- CodeGenFunction::JumpDest());
- },
- [](CodeGenFunction &) {});
- });
- }
- // Emit: if (PreCond) - end.
- if (ContBlock) {
- CGF.EmitBranch(ContBlock);
- CGF.EmitBlock(ContBlock, true);
- }
- // Emit final copy of the lastprivate variables if IsLastIter != 0.
- if (HasLastprivateClause) {
- CGF.EmitOMPLastprivateClauseFinal(
- S, isOpenMPSimdDirective(S.getDirectiveKind()),
- CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
- CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
- (*LIP)->getType(), S.getBeginLoc())));
- }
- CGF.EmitOMPLinearClauseFinal(S, [LIP, &S](CodeGenFunction &CGF) {
- return CGF.Builder.CreateIsNotNull(
- CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
- (*LIP)->getType(), S.getBeginLoc()));
- });
- };
- auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
- IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
- const OMPTaskDataTy &Data) {
- auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
- &Data](CodeGenFunction &CGF, PrePostActionTy &) {
- OMPLoopScope PreInitScope(CGF, S);
- CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
- OutlinedFn, SharedsTy,
- CapturedStruct, IfCond, Data);
- };
- CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
- CodeGen);
- };
- if (Data.Nogroup) {
- EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
- } else {
- CGM.getOpenMPRuntime().emitTaskgroupRegion(
- *this,
- [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
- Data);
- },
- S.getBeginLoc());
- }
- }
- void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- EmitOMPTaskLoopBasedDirective(S);
- }
- void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
- const OMPTaskLoopSimdDirective &S) {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S);
- EmitOMPTaskLoopBasedDirective(S);
- }
- void CodeGenFunction::EmitOMPMasterTaskLoopDirective(
- const OMPMasterTaskLoopDirective &S) {
- auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- EmitOMPTaskLoopBasedDirective(S);
- };
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S, llvm::None, /*EmitPreInitStmt=*/false);
- CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
- }
- void CodeGenFunction::EmitOMPMasterTaskLoopSimdDirective(
- const OMPMasterTaskLoopSimdDirective &S) {
- auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- EmitOMPTaskLoopBasedDirective(S);
- };
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- OMPLexicalScope Scope(*this, S);
- CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
- }
- void CodeGenFunction::EmitOMPParallelMasterTaskLoopDirective(
- const OMPParallelMasterTaskLoopDirective &S) {
- auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.EmitOMPTaskLoopBasedDirective(S);
- };
- OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
- CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
- S.getBeginLoc());
- };
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop, CodeGen,
- emitEmptyBoundParameters);
- }
- void CodeGenFunction::EmitOMPParallelMasterTaskLoopSimdDirective(
- const OMPParallelMasterTaskLoopSimdDirective &S) {
- auto &&CodeGen = [this, &S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- auto &&TaskLoopCodeGen = [&S](CodeGenFunction &CGF,
- PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.EmitOMPTaskLoopBasedDirective(S);
- };
- OMPLexicalScope Scope(CGF, S, OMPD_parallel, /*EmitPreInitStmt=*/false);
- CGM.getOpenMPRuntime().emitMasterRegion(CGF, TaskLoopCodeGen,
- S.getBeginLoc());
- };
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
- emitCommonOMPParallelDirective(*this, S, OMPD_master_taskloop_simd, CodeGen,
- emitEmptyBoundParameters);
- }
- // Generate the instructions for '#pragma omp target update' directive.
- void CodeGenFunction::EmitOMPTargetUpdateDirective(
- const OMPTargetUpdateDirective &S) {
- // If we don't have target devices, don't bother emitting the data mapping
- // code.
- if (CGM.getLangOpts().OMPTargetTriples.empty())
- return;
- // Check if we have any if clause associated with the directive.
- const Expr *IfCond = nullptr;
- if (const auto *C = S.getSingleClause<OMPIfClause>())
- IfCond = C->getCondition();
- // Check if we have any device clause associated with the directive.
- const Expr *Device = nullptr;
- if (const auto *C = S.getSingleClause<OMPDeviceClause>())
- Device = C->getDevice();
- OMPLexicalScope Scope(*this, S, OMPD_task);
- CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
- }
- void CodeGenFunction::EmitOMPGenericLoopDirective(
- const OMPGenericLoopDirective &S) {
- // Unimplemented, just inline the underlying statement for now.
- auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
- CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
- };
- OMPLexicalScope Scope(*this, S, OMPD_unknown);
- CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_loop, CodeGen);
- }
- void CodeGenFunction::EmitSimpleOMPExecutableDirective(
- const OMPExecutableDirective &D) {
- if (const auto *SD = dyn_cast<OMPScanDirective>(&D)) {
- EmitOMPScanDirective(*SD);
- return;
- }
- if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
- return;
- auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
- OMPPrivateScope GlobalsScope(CGF);
- if (isOpenMPTaskingDirective(D.getDirectiveKind())) {
- // Capture global firstprivates to avoid crash.
- for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
- for (const Expr *Ref : C->varlists()) {
- const auto *DRE = cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());
- if (!DRE)
- continue;
- const auto *VD = dyn_cast<VarDecl>(DRE->getDecl());
- if (!VD || VD->hasLocalStorage())
- continue;
- if (!CGF.LocalDeclMap.count(VD)) {
- LValue GlobLVal = CGF.EmitLValue(Ref);
- GlobalsScope.addPrivate(
- VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); });
- }
- }
- }
- }
- if (isOpenMPSimdDirective(D.getDirectiveKind())) {
- (void)GlobalsScope.Privatize();
- ParentLoopDirectiveForScanRegion ScanRegion(CGF, D);
- emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
- } else {
- if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
- for (const Expr *E : LD->counters()) {
- const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
- if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
- LValue GlobLVal = CGF.EmitLValue(E);
- GlobalsScope.addPrivate(
- VD, [&GlobLVal, &CGF]() { return GlobLVal.getAddress(CGF); });
- }
- if (isa<OMPCapturedExprDecl>(VD)) {
- // Emit only those that were not explicitly referenced in clauses.
- if (!CGF.LocalDeclMap.count(VD))
- CGF.EmitVarDecl(*VD);
- }
- }
- for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
- if (!C->getNumForLoops())
- continue;
- for (unsigned I = LD->getLoopsNumber(),
- E = C->getLoopNumIterations().size();
- I < E; ++I) {
- if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
- cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
- // Emit only those that were not explicitly referenced in clauses.
- if (!CGF.LocalDeclMap.count(VD))
- CGF.EmitVarDecl(*VD);
- }
- }
- }
- }
- (void)GlobalsScope.Privatize();
- CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
- }
- };
- if (D.getDirectiveKind() == OMPD_atomic ||
- D.getDirectiveKind() == OMPD_critical ||
- D.getDirectiveKind() == OMPD_section ||
- D.getDirectiveKind() == OMPD_master ||
- D.getDirectiveKind() == OMPD_masked) {
- EmitStmt(D.getAssociatedStmt());
- } else {
- auto LPCRegion =
- CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, D);
- OMPSimdLexicalScope Scope(*this, D);
- CGM.getOpenMPRuntime().emitInlinedDirective(
- *this,
- isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
- : D.getDirectiveKind(),
- CodeGen);
- }
- // Check for outer lastprivate conditional update.
- checkForLastprivateConditionalUpdate(*this, D);
- }
|