CbC/CbC_llvm: clang/lib/CodeGen/CGStmtOpenMP.cpp comparison

comparison clang/lib/CodeGen/CGStmtOpenMP.cpp @ 223:5f17cb93ff66 llvm-original

LLVM13 (2021/7/18)

author	Shinji KONO <kono@ie.u-ryukyu.ac.jp>
date	Sun, 18 Jul 2021 22:43:00 +0900
parents	79ff65ed7e25
children	5f20bc1ed4ff

comparison

equal deleted inserted replaced

-:81f6424ef0e3
+:5f17cb93ff66
 return false;
 });
 PreInits = cast_or_null<DeclStmt>(LD->getPreInits());
 } else if (const auto *Tile = dyn_cast<OMPTileDirective>(&S)) {
 PreInits = cast_or_null<DeclStmt>(Tile->getPreInits());
+} else if (const auto *Unroll = dyn_cast<OMPUnrollDirective>(&S)) {
+PreInits = cast_or_null<DeclStmt>(Unroll->getPreInits());
 } else {
 llvm_unreachable("Unknown loop-based directive kind.");
 }
 if (PreInits) {
 for (const auto *I : PreInits->decls())
 return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
 }
 return CGM.getSize(SizeInChars);
 }
+void CodeGenFunction::GenerateOpenMPCapturedVarsAggregate(
+const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
+const RecordDecl *RD = S.getCapturedRecordDecl();
+QualType RecordTy = getContext().getRecordType(RD);
+// Create the aggregate argument struct for the outlined function.
+LValue AggLV = MakeAddrLValue(
+CreateMemTemp(RecordTy, "omp.outlined.arg.agg."), RecordTy);
+// Initialize the aggregate with captured values.
+auto CurField = RD->field_begin();
+for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
+E = S.capture_init_end();
+I != E; ++I, ++CurField) {
+LValue LV = EmitLValueForFieldInitialization(AggLV, *CurField);
+// Initialize for VLA.
+if (CurField->hasCapturedVLAType()) {
+EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);
+} else
+// Initialize for capturesThis, capturesVariableByCopy,
+// capturesVariable
+EmitInitializerForField(*CurField, LV, *I);
+}
+CapturedVars.push_back(AggLV.getPointer(*this));
+}
 void CodeGenFunction::GenerateOpenMPCapturedVars(
 const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
 const RecordDecl *RD = S.getCapturedRecordDecl();
 auto CurField = RD->field_begin();
 auto CurCap = S.captures().begin();
 : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
 RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
 FunctionName(FunctionName), Loc(Loc) {}
 };
 } // namespace
+static llvm::Function *emitOutlinedFunctionPrologueAggregate(
+CodeGenFunction &CGF, FunctionArgList &Args,
+llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
+&LocalAddrs,
+llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
+&VLASizes,
+llvm::Value *&CXXThisValue, const CapturedStmt &CS, SourceLocation Loc,
+StringRef FunctionName) {
+const CapturedDecl *CD = CS.getCapturedDecl();
+const RecordDecl *RD = CS.getCapturedRecordDecl();
+assert(CD->hasBody() && "missing CapturedDecl body");
+CXXThisValue = nullptr;
+// Build the argument list.
+CodeGenModule &CGM = CGF.CGM;
+ASTContext &Ctx = CGM.getContext();
+Args.append(CD->param_begin(), CD->param_end());
+// Create the function declaration.
+const CGFunctionInfo &FuncInfo =
+CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
+llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
+auto *F =
+llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
+FunctionName, &CGM.getModule());
+CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
+if (CD->isNothrow())
+F->setDoesNotThrow();
+F->setDoesNotRecurse();
+// Generate the function.
+CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, Loc, Loc);
+Address ContextAddr = CGF.GetAddrOfLocalVar(CD->getContextParam());
+llvm::Value *ContextV = CGF.Builder.CreateLoad(ContextAddr);
+LValue ContextLV = CGF.MakeNaturalAlignAddrLValue(
+ContextV, CGM.getContext().getTagDeclType(RD));
+auto I = CS.captures().begin();
+for (const FieldDecl *FD : RD->fields()) {
+LValue FieldLV = CGF.EmitLValueForFieldInitialization(ContextLV, FD);
+// Do not map arguments if we emit function with non-original types.
+Address LocalAddr = FieldLV.getAddress(CGF);
+// If we are capturing a pointer by copy we don't need to do anything, just
+// use the value that we get from the arguments.
+if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
+const VarDecl *CurVD = I->getCapturedVar();
+LocalAddrs.insert({FD, {CurVD, LocalAddr}});
+++I;
+continue;
+}
+LValue ArgLVal =
+CGF.MakeAddrLValue(LocalAddr, FD->getType(), AlignmentSource::Decl);
+if (FD->hasCapturedVLAType()) {
+llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
+const VariableArrayType *VAT = FD->getCapturedVLAType();
+VLASizes.try_emplace(FD, VAT->getSizeExpr(), ExprArg);
+} else if (I->capturesVariable()) {
+const VarDecl *Var = I->getCapturedVar();
+QualType VarTy = Var->getType();
+Address ArgAddr = ArgLVal.getAddress(CGF);
+if (ArgLVal.getType()->isLValueReferenceType()) {
+ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
+} else if (!VarTy->isVariablyModifiedType() || !VarTy->isPointerType()) {
+assert(ArgLVal.getType()->isPointerType());
+ArgAddr = CGF.EmitLoadOfPointer(
+ArgAddr, ArgLVal.getType()->castAs<PointerType>());
+}
+LocalAddrs.insert(
+{FD, {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
+} else if (I->capturesVariableByCopy()) {
+assert(!FD->getType()->isAnyPointerType() &&
+"Not expecting a captured pointer.");
+const VarDecl *Var = I->getCapturedVar();
+Address CopyAddr = CGF.CreateMemTemp(FD->getType(), Ctx.getDeclAlign(FD),
+Var->getName());
+LValue CopyLVal =
+CGF.MakeAddrLValue(CopyAddr, FD->getType(), AlignmentSource::Decl);
+RValue ArgRVal = CGF.EmitLoadOfLValue(ArgLVal, I->getLocation());
+CGF.EmitStoreThroughLValue(ArgRVal, CopyLVal);
+LocalAddrs.insert({FD, {Var, CopyAddr}});
+} else {
+// If 'this' is captured, load it into CXXThisValue.
+assert(I->capturesThis());
+CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
+LocalAddrs.insert({FD, {nullptr, ArgLVal.getAddress(CGF)}});
+}
+++I;
+}
+return F;
+}
 static llvm::Function *emitOutlinedFunctionPrologue(
 CodeGenFunction &CGF, FunctionArgList &Args,
 llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
 &LocalAddrs,
 }
 return F;
 }
+llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunctionAggregate(
+const CapturedStmt &S, SourceLocation Loc) {
+assert(
+CapturedStmtInfo &&
+"CapturedStmtInfo should be set when generating the captured function");
+const CapturedDecl *CD = S.getCapturedDecl();
+// Build the argument list.
+FunctionArgList Args;
+llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
+llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
+StringRef FunctionName = CapturedStmtInfo->getHelperName();
+llvm::Function *F = emitOutlinedFunctionPrologueAggregate(
+*this, Args, LocalAddrs, VLASizes, CXXThisValue, S, Loc, FunctionName);
+CodeGenFunction::OMPPrivateScope LocalScope(*this);
+for (const auto &LocalAddrPair : LocalAddrs) {
+if (LocalAddrPair.second.first) {
+LocalScope.addPrivate(LocalAddrPair.second.first, [&LocalAddrPair]() {
+return LocalAddrPair.second.second;
+});
+}
+}
+(void)LocalScope.Privatize();
+for (const auto &VLASizePair : VLASizes)
+VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
+PGO.assignRegionCounters(GlobalDecl(CD), F);
+CapturedStmtInfo->EmitBody(*this, CD->getBody());
+(void)LocalScope.ForceCleanup();
+FinishFunction(CD->getBodyRBrace());
+return F;
+}
 llvm::Function *
 CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,
 SourceLocation Loc) {
 assert(
 CapturedStmtInfo &&
 SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
 llvm::Value *SrcBegin = SrcAddr.getPointer();
 llvm::Value *DestBegin = DestAddr.getPointer();
 // Cast from pointer to array type to pointer to single element.
-llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
+llvm::Value *DestEnd =
+Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements);
 // The basic structure here is a while-do loop.
 llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
 llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
 llvm::Value *IsEmpty =
 Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
 // Emit copy.
 CopyGen(DestElementCurrent, SrcElementCurrent);
 // Shift the address forward by one element.
 llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
-DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
+DestAddr.getElementType(), DestElementPHI, /*Idx0=*/1,
+"omp.arraycpy.dest.element");
 llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
-SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
+SrcAddr.getElementType(), SrcElementPHI, /*Idx0=*/1,
+"omp.arraycpy.src.element");
 // Check whether we've reached the end.
 llvm::Value *Done =
 Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
 Builder.CreateCondBr(Done, DoneBB, BodyBB);
 DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
 // Do not emit copy for firstprivate constant variables in target regions,
 // captured by reference.
 if (DeviceConstTarget && OrigVD->getType().isConstant(getContext()) &&
 FD && FD->getType()->isReferenceType() &&
 (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())) {
-(void)CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(*this,
+EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
-OrigVD);
 ++IRef;
 ++InitsRef;
 continue;
 }
 FirstprivateIsLastprivate =
 // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
 // lower and upper bounds with the pragma 'for' chunking mechanism.
 // The following lambda takes care of appending the lower and upper bound
 // parameters when necessary
 CodeGenBoundParameters(CGF, S, CapturedVars);
-CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
+CGF.GenerateOpenMPCapturedVarsAggregate(*CS, CapturedVars);
 CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
 CapturedVars, IfCond);
 }
 static bool isAllocatableDecl(const VarDecl *VD) {
 emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
 return;
 }
 if (SimplifiedS == NextLoop) {
 if (auto *Dir = dyn_cast<OMPTileDirective>(SimplifiedS))
+SimplifiedS = Dir->getTransformedStmt();
+if (auto *Dir = dyn_cast<OMPUnrollDirective>(SimplifiedS))
 SimplifiedS = Dir->getTransformedStmt();
 if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
 SimplifiedS = CanonLoop->getLoopStmt();
 if (const auto *For = dyn_cast<ForStmt>(SimplifiedS)) {
 S = For->getBody();
 }
 }
 }
 static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
-const OMPExecutableDirective &D,
+const OMPExecutableDirective &D) {
-bool IsMonotonic) {
 if (!CGF.HaveInsertPoint())
 return;
 if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
 RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
 /*ignoreResult=*/true);
 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
 // In presence of finite 'safelen', it may be unsafe to mark all
 // the memory instructions parallel, because loop-carried
 // dependences of 'safelen' iterations are possible.
-if (!IsMonotonic)
+CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
-CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
 } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
 RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
 /*ignoreResult=*/true);
 auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
 CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
 // dependences of 'safelen' iterations are possible.
 CGF.LoopStack.setParallel(/*Enable=*/false);
 }
 }
-void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
+void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D) {
-bool IsMonotonic) {
 // Walk clauses and process safelen/lastprivate.
-LoopStack.setParallel(!IsMonotonic);
+LoopStack.setParallel(/*Enable=*/true);
 LoopStack.setVectorizeEnable();
-emitSimdlenSafelenClause(*this, D, IsMonotonic);
+emitSimdlenSafelenClause(*this, D);
 if (const auto *C = D.getSingleClause<OMPOrderClause>())
 if (C->getKind() == OMPC_ORDER_concurrent)
 LoopStack.setParallel(/*Enable=*/true);
 if ((D.getDirectiveKind() == OMPD_simd ||
 (getLangOpts().OpenMPSimd &&
 // Emit the de-sugared statement.
 OMPTransformDirectiveScopeRAII TileScope(*this, &S);
 EmitStmt(S.getTransformedStmt());
 }
+void CodeGenFunction::EmitOMPUnrollDirective(const OMPUnrollDirective &S) {
+// This function is only called if the unrolled loop is not consumed by any
+// other loop-associated construct. Such a loop-associated construct will have
+// used the transformed AST.
+// Set the unroll metadata for the next emitted loop.
+LoopStack.setUnrollState(LoopAttributes::Enable);
+if (S.hasClausesOfKind<OMPFullClause>()) {
+LoopStack.setUnrollState(LoopAttributes::Full);
+} else if (auto *PartialClause = S.getSingleClause<OMPPartialClause>()) {
+if (Expr *FactorExpr = PartialClause->getFactor()) {
+uint64_t Factor =
+FactorExpr->EvaluateKnownConstInt(getContext()).getZExtValue();
+assert(Factor >= 1 && "Only positive factors are valid");
+LoopStack.setUnrollCount(Factor);
+}
+}
+EmitStmt(S.getAssociatedStmt());
+}
 void CodeGenFunction::EmitOMPOuterLoop(
 bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
 CodeGenFunction::OMPPrivateScope &LoopScope,
 const CodeGenFunction::OMPLoopArguments &LoopArgs,
 const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
 CGF.LoopStack.setParallel(!IsMonotonic);
 if (const auto *C = S.getSingleClause<OMPOrderClause>())
 if (C->getKind() == OMPC_ORDER_concurrent)
 CGF.LoopStack.setParallel(/*Enable=*/true);
 } else {
-CGF.EmitOMPSimdInit(S, IsMonotonic);
+CGF.EmitOMPSimdInit(S);
 }
 },
 [&S, &LoopArgs, LoopExit, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,
 &LoopScope](CodeGenFunction &CGF, PrePostActionTy &) {
 SourceLocation Loc = S.getBeginLoc();
 bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
 /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
 isOpenMPLoopBoundSharingDirective(S.getDirectiveKind());
 bool IsMonotonic =
 Ordered ||
-((ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
+(ScheduleKind.Schedule == OMPC_SCHEDULE_static &&
-ScheduleKind.Schedule == OMPC_SCHEDULE_unknown) &&
 !(ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||
 ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
 ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
 if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
 !Ordered) {
 JumpDest LoopExit =
 getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
 emitCommonSimdLoop(
 *this, S,
-[&S, IsMonotonic](CodeGenFunction &CGF, PrePostActionTy &) {
+[&S](CodeGenFunction &CGF, PrePostActionTy &) {
 if (isOpenMPSimdDirective(S.getDirectiveKind())) {
-CGF.EmitOMPSimdInit(S, IsMonotonic);
+CGF.EmitOMPSimdInit(S);
 } else if (const auto *C = S.getSingleClause<OMPOrderClause>()) {
 if (C->getKind() == OMPC_ORDER_concurrent)
 CGF.LoopStack.setParallel(/*Enable=*/true);
 }
 },
 CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
 OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
 CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
 // Generate condition for loop.
 BinaryOperator *Cond = BinaryOperator::Create(
-C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue, OK_Ordinary,
+C, &IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_PRValue, OK_Ordinary,
 S.getBeginLoc(), FPOptionsOverride());
 // Increment for loop counter.
 UnaryOperator *Inc = UnaryOperator::Create(
-C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
+C, &IVRefExpr, UO_PreInc, KmpInt32Ty, VK_PRValue, OK_Ordinary,
 S.getBeginLoc(), true, FPOptionsOverride());
 auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
 // Iterate through all sections and emit a switch construct:
 // switch (IV) {
 //   case 0:
 C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
 /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
 PrivateVD->setInitStyle(VarDecl::CInit);
 PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
 InitRef, /*BasePath=*/nullptr,
-VK_RValue, FPOptionsOverride()));
+VK_PRValue, FPOptionsOverride()));
 Data.FirstprivateVars.emplace_back(OrigRef);
 Data.FirstprivateCopies.emplace_back(PrivateRef);
 Data.FirstprivateInits.emplace_back(InitRef);
 return OrigVD;
 }
 //
 emitCommonSimdLoop(
 *this, S,
 [&S](CodeGenFunction &CGF, PrePostActionTy &) {
 if (isOpenMPSimdDirective(S.getDirectiveKind()))
-CGF.EmitOMPSimdInit(S, /*IsMonotonic=*/true);
+CGF.EmitOMPSimdInit(S);
 },
 [&S, &LoopScope, Cond, IncExpr, LoopExit, &CodeGenLoop,
 StaticChunked](CodeGenFunction &CGF, PrePostActionTy &) {
 CGF.EmitOMPInnerLoop(
 S, LoopScope.requiresCleanups(), Cond, IncExpr,
 case OMPC_task_reduction:
 case OMPC_in_reduction:
 case OMPC_safelen:
 case OMPC_simdlen:
 case OMPC_sizes:
+case OMPC_full:
+case OMPC_partial:
 case OMPC_allocator:
 case OMPC_allocate:
 case OMPC_collapse:
 case OMPC_default:
 case OMPC_seq_cst:
 S.getBeginLoc());
 }
 OMPTeamsScope Scope(CGF, S);
 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
-CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
+CGF.GenerateOpenMPCapturedVarsAggregate(*CS, CapturedVars);
 CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
 CapturedVars);
 }
 void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {

Mercurial > hg > CbC > CbC_llvm

comparison clang/lib/CodeGen/CGStmtOpenMP.cpp @ 223:5f17cb93ff66 llvm-original