CbC/CbC_llvm: lib/IR/AutoUpgrade.cpp comparison

comparison lib/IR/AutoUpgrade.cpp @ 148:63bd29f05246

merged

author	Shinji KONO <kono@ie.u-ryukyu.ac.jp>
date	Wed, 14 Aug 2019 19:46:37 +0900
parents	c2174574ed3a
children

comparison

equal deleted inserted replaced

-:3fc4d5c3e21e
+:63bd29f05246
 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
 //
-//                     The LLVM Compiler Infrastructure
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-//
+// See https://llvm.org/LICENSE.txt for license information.
-// This file is distributed under the University of Illinois Open Source
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-// License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the auto-upgrade helper functions.
 // This is where deprecated IR intrinsics and other IR features are updated to
 rename(F);
 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
 return true;
 }
-// Upgrade the declaration of fp compare intrinsics that change return type
-// from scalar to vXi1 mask.
-static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
-Function *&NewFn) {
-// Check if the return type is a vector.
-if (F->getReturnType()->isVectorTy())
-return false;
-rename(F);
-NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
-return true;
-}
 static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
 // All of the intrinsics matches below should be marked with which llvm
 // version started autoupgrading them. At some point in the future we would
 // like to use this information to remove upgrade code for some older
 // intrinsics. It is currently undecided how we will determine that future
 // point.
-if (Name=="ssse3.pabs.b.128" || // Added in 6.0
+if (Name == "addcarryx.u32" || // Added in 8.0
+Name == "addcarryx.u64" || // Added in 8.0
+Name == "addcarry.u32" || // Added in 8.0
+Name == "addcarry.u64" || // Added in 8.0
+Name == "subborrow.u32" || // Added in 8.0
+Name == "subborrow.u64" || // Added in 8.0
+Name.startswith("sse2.padds.") || // Added in 8.0
+Name.startswith("sse2.psubs.") || // Added in 8.0
+Name.startswith("sse2.paddus.") || // Added in 8.0
+Name.startswith("sse2.psubus.") || // Added in 8.0
+Name.startswith("avx2.padds.") || // Added in 8.0
+Name.startswith("avx2.psubs.") || // Added in 8.0
+Name.startswith("avx2.paddus.") || // Added in 8.0
+Name.startswith("avx2.psubus.") || // Added in 8.0
+Name.startswith("avx512.padds.") || // Added in 8.0
+Name.startswith("avx512.psubs.") || // Added in 8.0
+Name.startswith("avx512.mask.padds.") || // Added in 8.0
+Name.startswith("avx512.mask.psubs.") || // Added in 8.0
+Name.startswith("avx512.mask.paddus.") || // Added in 8.0
+Name.startswith("avx512.mask.psubus.") || // Added in 8.0
+Name=="ssse3.pabs.b.128" || // Added in 6.0
 Name=="ssse3.pabs.w.128" || // Added in 6.0
 Name=="ssse3.pabs.d.128" || // Added in 6.0
+Name.startswith("fma4.vfmadd.s") || // Added in 7.0
+Name.startswith("fma.vfmadd.") || // Added in 7.0
+Name.startswith("fma.vfmsub.") || // Added in 7.0
+Name.startswith("fma.vfmaddsub.") || // Added in 7.0
+Name.startswith("fma.vfmsubadd.") || // Added in 7.0
+Name.startswith("fma.vfnmadd.") || // Added in 7.0
+Name.startswith("fma.vfnmsub.") || // Added in 7.0
+Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0
+Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0
+Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0
+Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0
+Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0
+Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0
+Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0
+Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0
+Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0
+Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0
+Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0
 Name.startswith("avx512.mask.shuf.i") || // Added in 6.0
 Name.startswith("avx512.mask.shuf.f") || // Added in 6.0
 Name.startswith("avx512.kunpck") || //added in 6.0
 Name.startswith("avx2.pabs.") || // Added in 6.0
 Name.startswith("avx512.mask.pabs.") || // Added in 6.0
 Name.startswith("avx512.broadcastm") || // Added in 6.0
+Name == "sse.sqrt.ss" || // Added in 7.0
+Name == "sse2.sqrt.sd" || // Added in 7.0
+Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0
+Name.startswith("avx.sqrt.p") || // Added in 7.0
+Name.startswith("sse2.sqrt.p") || // Added in 7.0
+Name.startswith("sse.sqrt.p") || // Added in 7.0
 Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
 Name.startswith("sse2.pcmpeq.") || // Added in 3.1
 Name.startswith("sse2.pcmpgt.") || // Added in 3.1
 Name.startswith("avx2.pcmpeq.") || // Added in 3.1
 Name.startswith("avx2.pcmpgt.") || // Added in 3.1
 Name.startswith("avx512.mask.padd.") || // Added in 4.0
 Name.startswith("avx512.mask.psub.") || // Added in 4.0
 Name.startswith("avx512.mask.pmull.") || // Added in 4.0
 Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
 Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
+Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
+Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
+Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
+Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
+Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
+Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
+Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
+Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
+Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
+Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
+Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
+Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0
+Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0
+Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0
+Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0
+Name == "avx512.cvtusi2sd" || // Added in 7.0
+Name.startswith("avx512.mask.permvar.") || // Added in 7.0
+Name == "sse2.pmulu.dq" || // Added in 7.0
+Name == "sse41.pmuldq" || // Added in 7.0
+Name == "avx2.pmulu.dq" || // Added in 7.0
+Name == "avx2.pmul.dq" || // Added in 7.0
+Name == "avx512.pmulu.dq.512" || // Added in 7.0
+Name == "avx512.pmul.dq.512" || // Added in 7.0
 Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0
 Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0
+Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0
+Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0
+Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0
+Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0
+Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0
 Name.startswith("avx512.mask.packsswb.") || // Added in 5.0
 Name.startswith("avx512.mask.packssdw.") || // Added in 5.0
 Name.startswith("avx512.mask.packuswb.") || // Added in 5.0
 Name.startswith("avx512.mask.packusdw.") || // Added in 5.0
 Name.startswith("avx512.mask.cmp.b") || // Added in 5.0
 Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
 Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
 Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
+Name.startswith("avx512.mask.cmp.p") || // Added in 7.0
 Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
 Name.startswith("avx512.cvtb2mask.") || // Added in 7.0
 Name.startswith("avx512.cvtw2mask.") || // Added in 7.0
 Name.startswith("avx512.cvtd2mask.") || // Added in 7.0
 Name.startswith("avx512.cvtq2mask.") || // Added in 7.0
-Name == "avx512.mask.add.pd.128" || // Added in 4.0
-Name == "avx512.mask.add.pd.256" || // Added in 4.0
-Name == "avx512.mask.add.ps.128" || // Added in 4.0
-Name == "avx512.mask.add.ps.256" || // Added in 4.0
-Name == "avx512.mask.div.pd.128" || // Added in 4.0
-Name == "avx512.mask.div.pd.256" || // Added in 4.0
-Name == "avx512.mask.div.ps.128" || // Added in 4.0
-Name == "avx512.mask.div.ps.256" || // Added in 4.0
-Name == "avx512.mask.mul.pd.128" || // Added in 4.0
-Name == "avx512.mask.mul.pd.256" || // Added in 4.0
-Name == "avx512.mask.mul.ps.128" || // Added in 4.0
-Name == "avx512.mask.mul.ps.256" || // Added in 4.0
-Name == "avx512.mask.sub.pd.128" || // Added in 4.0
-Name == "avx512.mask.sub.pd.256" || // Added in 4.0
-Name == "avx512.mask.sub.ps.128" || // Added in 4.0
-Name == "avx512.mask.sub.ps.256" || // Added in 4.0
-Name == "avx512.mask.max.pd.128" || // Added in 5.0
-Name == "avx512.mask.max.pd.256" || // Added in 5.0
-Name == "avx512.mask.max.ps.128" || // Added in 5.0
-Name == "avx512.mask.max.ps.256" || // Added in 5.0
-Name == "avx512.mask.min.pd.128" || // Added in 5.0
-Name == "avx512.mask.min.pd.256" || // Added in 5.0
-Name == "avx512.mask.min.ps.128" || // Added in 5.0
-Name == "avx512.mask.min.ps.256" || // Added in 5.0
 Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0
 Name.startswith("avx512.mask.psll.d") || // Added in 4.0
 Name.startswith("avx512.mask.psll.q") || // Added in 4.0
 Name.startswith("avx512.mask.psll.w") || // Added in 4.0
 Name.startswith("avx512.mask.psra.d") || // Added in 4.0
 Name.startswith("avx2.pmovsx") || // Added in 3.9
 Name.startswith("avx2.pmovzx") || // Added in 3.9
 Name.startswith("avx512.mask.pmovsx") || // Added in 4.0
 Name.startswith("avx512.mask.pmovzx") || // Added in 4.0
 Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0
+Name.startswith("avx512.mask.pternlog.") || // Added in 7.0
+Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0
+Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0
+Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0
+Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0
+Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0
+Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0
+Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0
+Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0
+Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0
+Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0
+Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0
+Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0
+Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0
+Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0
+Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0
+Name.startswith("avx512.mask.vpshld.") || // Added in 7.0
+Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0
+Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0
+Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0
+Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0
+Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0
+Name.startswith("avx512.vpshld.") || // Added in 8.0
+Name.startswith("avx512.vpshrd.") || // Added in 8.0
+Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0
+Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0
+Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0
+Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0
+Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0
+Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0
+Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
+Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
+Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
+Name.startswith("avx512.mask.conflict.") || // Added in 9.0
+Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
+Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
+Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
+Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
+Name == "sse.cvtsi2ss" || // Added in 7.0
+Name == "sse.cvtsi642ss" || // Added in 7.0
+Name == "sse2.cvtsi2sd" || // Added in 7.0
+Name == "sse2.cvtsi642sd" || // Added in 7.0
+Name == "sse2.cvtss2sd" || // Added in 7.0
 Name == "sse2.cvtdq2pd" || // Added in 3.9
+Name == "sse2.cvtdq2ps" || // Added in 7.0
 Name == "sse2.cvtps2pd" || // Added in 3.9
 Name == "avx.cvtdq2.pd.256" || // Added in 3.9
+Name == "avx.cvtdq2.ps.256" || // Added in 7.0
 Name == "avx.cvt.ps2.pd.256" || // Added in 3.9
 Name.startswith("avx.vinsertf128.") || // Added in 3.7
 Name == "avx2.vinserti128" || // Added in 3.7
 Name.startswith("avx512.mask.insert") || // Added in 4.0
 Name.startswith("avx.vextractf128.") || // Added in 3.7
 Name.startswith("avx512.mask.store.p") || // Added in 3.9
 Name.startswith("avx512.mask.store.b.") || // Added in 3.9
 Name.startswith("avx512.mask.store.w.") || // Added in 3.9
 Name.startswith("avx512.mask.store.d.") || // Added in 3.9
 Name.startswith("avx512.mask.store.q.") || // Added in 3.9
+Name == "avx512.mask.store.ss" || // Added in 7.0
 Name.startswith("avx512.mask.loadu.") || // Added in 3.9
 Name.startswith("avx512.mask.load.") || // Added in 3.9
+Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
+Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
+Name.startswith("avx512.mask.expand.b") || // Added in 9.0
+Name.startswith("avx512.mask.expand.w") || // Added in 9.0
+Name.startswith("avx512.mask.expand.d") || // Added in 9.0
+Name.startswith("avx512.mask.expand.q") || // Added in 9.0
+Name.startswith("avx512.mask.expand.p") || // Added in 9.0
+Name.startswith("avx512.mask.compress.b") || // Added in 9.0
+Name.startswith("avx512.mask.compress.w") || // Added in 9.0
+Name.startswith("avx512.mask.compress.d") || // Added in 9.0
+Name.startswith("avx512.mask.compress.q") || // Added in 9.0
+Name.startswith("avx512.mask.compress.p") || // Added in 9.0
 Name == "sse42.crc32.64.8" || // Added in 3.4
 Name.startswith("avx.vbroadcast.s") || // Added in 3.5
+Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
 Name.startswith("avx512.mask.palignr.") || // Added in 3.9
 Name.startswith("avx512.mask.valign.") || // Added in 4.0
 Name.startswith("sse2.psll.dq") || // Added in 3.7
 Name.startswith("sse2.psrl.dq") || // Added in 3.7
 Name.startswith("avx2.psll.dq") || // Added in 3.7
 Name.startswith("avx512.mask.broadcasti") || // Added in 6.0
 Name == "xop.vpcmov" || // Added in 3.8
 Name == "xop.vpcmov.256" || // Added in 5.0
 Name.startswith("avx512.mask.move.s") || // Added in 4.0
 Name.startswith("avx512.cvtmask2") || // Added in 5.0
-(Name.startswith("xop.vpcom") && // Added in 3.2
+Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
-F->arg_size() == 2) ||
+Name.startswith("xop.vprot") || // Added in 8.0
+Name.startswith("avx512.prol") || // Added in 8.0
+Name.startswith("avx512.pror") || // Added in 8.0
+Name.startswith("avx512.mask.prorv.") || // Added in 8.0
+Name.startswith("avx512.mask.pror.") ||  // Added in 8.0
+Name.startswith("avx512.mask.prolv.") || // Added in 8.0
+Name.startswith("avx512.mask.prol.") ||  // Added in 8.0
 Name.startswith("avx512.ptestm") || //Added in 6.0
 Name.startswith("avx512.ptestnm") || //Added in 6.0
-Name.startswith("sse2.pavg") || // Added in 6.0
-Name.startswith("avx2.pavg") || // Added in 6.0
 Name.startswith("avx512.mask.pavg")) // Added in 6.0
 return true;
 return false;
 }
 // Remove "x86." prefix.
 Name = Name.substr(4);
 if (ShouldUpgradeX86Intrinsic(F, Name)) {
 NewFn = nullptr;
+return true;
+}
+if (Name == "rdtscp") { // Added in 8.0
+// If this intrinsic has 0 operands, it's the new version.
+if (F->getFunctionType()->getNumParams() == 0)
+return false;
+rename(F);
+NewFn = Intrinsic::getDeclaration(F->getParent(),
+Intrinsic::x86_rdtscp);
 return true;
 }
 // SSE4.1 ptest functions may have an old signature.
 if (Name.startswith("sse41.ptest")) { // Added in 3.2
 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
 NewFn);
 if (Name == "avx2.mpsadbw") // Added in 3.6
 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
 NewFn);
-if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0
-return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,
-NewFn);
-if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0
-return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,
-NewFn);
-if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0
-return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
-NewFn);
-if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0
-return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,
-NewFn);
-if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0
-return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,
-NewFn);
-if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
-return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
-NewFn);
 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
 if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
 rename(F);
 NewFn = Intrinsic::getDeclaration(F->getParent(),
 NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
 return true;
 }
 }
+if (Name == "seh.recoverfp") {
+NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
+return true;
+}
 return false;
 }
 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
 assert(F && "Illegal to upgrade a non-existent Function.");
 };
 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
 // the end of the name. Change name from llvm.arm.neon.vclz.* to
 //  llvm.ctlz.*
 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
-NewFn = Function::Create(fType, F->getLinkage(),
+NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
 "llvm.ctlz." + Name.substr(14), F->getParent());
 return true;
 }
 if (Name.startswith("arm.neon.vcnt")) {
 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
 auto fArgs = F->getFunctionType()->params();
 SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());
 // Can't use Intrinsic::getDeclaration here as the return types might
 // then only be structurally equal.
 FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);
-NewFn = Function::Create(fType, F->getLinkage(),
+NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
 "llvm." + Name + ".p0i8", F->getParent());
 return true;
 }
 Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
 if (vstRegex.match(Name)) {
 }
 if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
 return true;
 }
+if (Name.startswith("aarch64.neon.addp")) {
+if (F->arg_size() != 2)
+break; // Invalid IR.
+auto fArgs = F->getFunctionType()->params();
+VectorType *ArgTy = dyn_cast<VectorType>(fArgs[0]);
+if (ArgTy && ArgTy->getElementType()->isFloatingPointTy()) {
+NewFn = Intrinsic::getDeclaration(F->getParent(),
+Intrinsic::aarch64_neon_faddp, fArgs);
+return true;
+}
+}
 break;
 }
 case 'c': {
 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
 return true;
 }
 break;
 }
+case 'e': {
+SmallVector<StringRef, 2> Groups;
+Regex R("^experimental.vector.reduce.([a-z]+)\\.[fi][0-9]+");
+if (R.match(Name, &Groups)) {
+Intrinsic::ID ID = Intrinsic::not_intrinsic;
+if (Groups[1] == "fadd")
+ID = Intrinsic::experimental_vector_reduce_v2_fadd;
+if (Groups[1] == "fmul")
+ID = Intrinsic::experimental_vector_reduce_v2_fmul;
+if (ID != Intrinsic::not_intrinsic) {
+rename(F);
+auto Args = F->getFunctionType()->params();
+Type *Tys[] = {F->getFunctionType()->getReturnType(), Args[1]};
+NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+return true;
+}
+}
+break;
+}
 case 'i':
 case 'l': {
 bool IsLifetimeStart = Name.startswith("lifetime.start");
 if (IsLifetimeStart || Name.startswith("invariant.start")) {
 Intrinsic::ID ID = IsLifetimeStart ?
 rename(F);
 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
 return true;
 }
 }
+if (Name.startswith("invariant.group.barrier")) {
+// Rename invariant.group.barrier to launder.invariant.group
+auto Args = F->getFunctionType()->params();
+Type* ObjectPtr[1] = {Args[0]};
+rename(F);
+NewFn = Intrinsic::getDeclaration(F->getParent(),
+Intrinsic::launder_invariant_group, ObjectPtr);
+return true;
+}
 break;
 }
 case 'm': {
 if (Name.startswith("masked.load.")) {
 Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };
 bool Expand = StringSwitch<bool>(Name)
 .Cases("abs.i", "abs.ll", true)
 .Cases("clz.ll", "popc.ll", "h2f", true)
 .Cases("max.i", "max.ll", "max.ui", "max.ull", true)
 .Cases("min.i", "min.ll", "min.ui", "min.ull", true)
+.StartsWith("atomic.load.add.f32.p", true)
+.StartsWith("atomic.load.add.f64.p", true)
 .Default(false);
 if (Expand) {
 NewFn = nullptr;
 return true;
 }
 case 'o':
 // We only need to change the name to match the mangling including the
 // address space.
 if (Name.startswith("objectsize.")) {
 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
-if (F->arg_size() == 2 ||
+if (F->arg_size() == 2 || F->arg_size() == 3 ||
 F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
 rename(F);
 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
 Tys);
 return true;
 }
 }
 break;
+case 'p':
+if (Name == "prefetch") {
+// Handle address space overloading.
+Type *Tys[] = {F->arg_begin()->getType()};
+if (F->getName() != Intrinsic::getName(Intrinsic::prefetch, Tys)) {
+rename(F);
+NewFn =
+Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);
+return true;
+}
+}
+break;
 case 's':
 if (Name == "stackprotectorcheck") {
 NewFn = nullptr;
 return true;
 }
 if (Intrinsic::ID id = F->getIntrinsicID())
 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
 return Upgraded;
 }
-bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
+GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
-// Nothing to do yet.
+if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
-return false;
+GV->getName() == "llvm.global_dtors")) ||
+!GV->hasInitializer())
+return nullptr;
+ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
+if (!ATy)
+return nullptr;
+StructType *STy = dyn_cast<StructType>(ATy->getElementType());
+if (!STy || STy->getNumElements() != 2)
+return nullptr;
+LLVMContext &C = GV->getContext();
+IRBuilder<> IRB(C);
+auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
+IRB.getInt8PtrTy());
+Constant *Init = GV->getInitializer();
+unsigned N = Init->getNumOperands();
+std::vector<Constant *> NewCtors(N);
+for (unsigned i = 0; i != N; ++i) {
+auto Ctor = cast<Constant>(Init->getOperand(i));
+NewCtors[i] = ConstantStruct::get(
+EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
+Constant::getNullValue(IRB.getInt8PtrTy()));
+}
+Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
+return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
+NewInit, GV->getName());
 }
 // Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
 // to byte shuffles.
 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
 return Mask;
 }
 static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
 Value *Op0, Value *Op1) {
-// If the mask is all ones just emit the align operation.
+// If the mask is all ones just emit the first operation.
 if (const auto *C = dyn_cast<Constant>(Mask))
 if (C->isAllOnesValue())
 return Op0;
 Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements());
+return Builder.CreateSelect(Mask, Op0, Op1);
+}
+static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,
+Value *Op0, Value *Op1) {
+// If the mask is all ones just emit the first operation.
+if (const auto *C = dyn_cast<Constant>(Mask))
+if (C->isAllOnesValue())
+return Op0;
+llvm::VectorType *MaskTy =
+llvm::VectorType::get(Builder.getInt1Ty(),
+Mask->getType()->getIntegerBitWidth());
+Mask = Builder.CreateBitCast(Mask, MaskTy);
+Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
 return Builder.CreateSelect(Mask, Op0, Op1);
 }
 // Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
 // PALIGNR handles large immediates by shifting while VALIGN masks the immediate
 "palignr");
 return EmitX86Select(Builder, Mask, Align, Passthru);
 }
+static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,
+bool ZeroMask, bool IndexForm) {
+Type *Ty = CI.getType();
+unsigned VecWidth = Ty->getPrimitiveSizeInBits();
+unsigned EltWidth = Ty->getScalarSizeInBits();
+bool IsFloat = Ty->isFPOrFPVectorTy();
+Intrinsic::ID IID;
+if (VecWidth == 128 && EltWidth == 32 && IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
+else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_d_128;
+else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
+else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_q_128;
+else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
+else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_d_256;
+else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
+else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_q_256;
+else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
+else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_d_512;
+else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
+else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
+IID = Intrinsic::x86_avx512_vpermi2var_q_512;
+else if (VecWidth == 128 && EltWidth == 16)
+IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
+else if (VecWidth == 256 && EltWidth == 16)
+IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
+else if (VecWidth == 512 && EltWidth == 16)
+IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
+else if (VecWidth == 128 && EltWidth == 8)
+IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
+else if (VecWidth == 256 && EltWidth == 8)
+IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
+else if (VecWidth == 512 && EltWidth == 8)
+IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
+CI.getArgOperand(2) };
+// If this isn't index form we need to swap operand 0 and 1.
+if (!IndexForm)
+std::swap(Args[0], Args[1]);
+Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
+Args);
+Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
+: Builder.CreateBitCast(CI.getArgOperand(1),
+Ty);
+return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
+}
+static Value *UpgradeX86AddSubSatIntrinsics(IRBuilder<> &Builder, CallInst &CI,
+bool IsSigned, bool IsAddition) {
+Type *Ty = CI.getType();
+Value *Op0 = CI.getOperand(0);
+Value *Op1 = CI.getOperand(1);
+Intrinsic::ID IID =
+IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat)
+: (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat);
+Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
+if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
+Value *VecSrc = CI.getOperand(2);
+Value *Mask = CI.getOperand(3);
+Res = EmitX86Select(Builder, Mask, Res, VecSrc);
+}
+return Res;
+}
+static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
+bool IsRotateRight) {
+Type *Ty = CI.getType();
+Value *Src = CI.getArgOperand(0);
+Value *Amt = CI.getArgOperand(1);
+// Amount may be scalar immediate, in which case create a splat vector.
+// Funnel shifts amounts are treated as modulo and types are all power-of-2 so
+// we only care about the lowest log2 bits anyway.
+if (Amt->getType() != Ty) {
+unsigned NumElts = Ty->getVectorNumElements();
+Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
+Amt = Builder.CreateVectorSplat(NumElts, Amt);
+}
+Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
+Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
+if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
+Value *VecSrc = CI.getOperand(2);
+Value *Mask = CI.getOperand(3);
+Res = EmitX86Select(Builder, Mask, Res, VecSrc);
+}
+return Res;
+}
+static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,
+bool IsSigned) {
+Type *Ty = CI.getType();
+Value *LHS = CI.getArgOperand(0);
+Value *RHS = CI.getArgOperand(1);
+CmpInst::Predicate Pred;
+switch (Imm) {
+case 0x0:
+Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
+break;
+case 0x1:
+Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
+break;
+case 0x2:
+Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
+break;
+case 0x3:
+Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
+break;
+case 0x4:
+Pred = ICmpInst::ICMP_EQ;
+break;
+case 0x5:
+Pred = ICmpInst::ICMP_NE;
+break;
+case 0x6:
+return Constant::getNullValue(Ty); // FALSE
+case 0x7:
+return Constant::getAllOnesValue(Ty); // TRUE
+default:
+llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
+}
+Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
+Value *Ext = Builder.CreateSExt(Cmp, Ty);
+return Ext;
+}
+static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
+bool IsShiftRight, bool ZeroMask) {
+Type *Ty = CI.getType();
+Value *Op0 = CI.getArgOperand(0);
+Value *Op1 = CI.getArgOperand(1);
+Value *Amt = CI.getArgOperand(2);
+if (IsShiftRight)
+std::swap(Op0, Op1);
+// Amount may be scalar immediate, in which case create a splat vector.
+// Funnel shifts amounts are treated as modulo and types are all power-of-2 so
+// we only care about the lowest log2 bits anyway.
+if (Amt->getType() != Ty) {
+unsigned NumElts = Ty->getVectorNumElements();
+Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
+Amt = Builder.CreateVectorSplat(NumElts, Amt);
+}
+Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
+Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
+unsigned NumArgs = CI.getNumArgOperands();
+if (NumArgs >= 4) { // For masked intrinsics.
+Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
+ZeroMask     ? ConstantAggregateZero::get(CI.getType()) :
+CI.getArgOperand(0);
+Value *Mask = CI.getOperand(NumArgs - 1);
+Res = EmitX86Select(Builder, Mask, Res, VecSrc);
+}
+return Res;
+}
 static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
 Value *Ptr, Value *Data, Value *Mask,
 bool Aligned) {
 // Cast the pointer to the right type.
 Ptr = Builder.CreateBitCast(Ptr,
 }
 static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
 Value *Ptr, Value *Passthru, Value *Mask,
 bool Aligned) {
+Type *ValTy = Passthru->getType();
 // Cast the pointer to the right type.
-Ptr = Builder.CreateBitCast(Ptr,
+Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
-llvm::PointerType::getUnqual(Passthru->getType()));
 unsigned Align =
 Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1;
 // If the mask is all ones just emit a regular store.
 if (const auto *C = dyn_cast<Constant>(Mask))
 if (C->isAllOnesValue())
-return Builder.CreateAlignedLoad(Ptr, Align);
+return Builder.CreateAlignedLoad(ValTy, Ptr, Align);
 // Convert the mask from an integer type to a vector of i1.
 unsigned NumElts = Passthru->getType()->getVectorNumElements();
 Mask = getX86MaskVec(Builder, Mask, NumElts);
 return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru);
 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
 return Res;
 }
+static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) {
+Type *Ty = CI.getType();
+// Arguments have a vXi32 type so cast to vXi64.
+Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
+Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
+if (IsSigned) {
+// Shift left then arithmetic shift right.
+Constant *ShiftAmt = ConstantInt::get(Ty, 32);
+LHS = Builder.CreateShl(LHS, ShiftAmt);
+LHS = Builder.CreateAShr(LHS, ShiftAmt);
+RHS = Builder.CreateShl(RHS, ShiftAmt);
+RHS = Builder.CreateAShr(RHS, ShiftAmt);
+} else {
+// Clear the upper bits.
+Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
+LHS = Builder.CreateAnd(LHS, Mask);
+RHS = Builder.CreateAnd(RHS, Mask);
+}
+Value *Res = Builder.CreateMul(LHS, RHS);
+if (CI.getNumArgOperands() == 4)
+Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
+return Res;
+}
 // Applying mask on vector of i1's and make sure result is at least 8 bits wide.
-static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder,Value *Vec, Value *Mask,
+static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
-unsigned NumElts) {
+Value *Mask) {
+unsigned NumElts = Vec->getType()->getVectorNumElements();
 if (Mask) {
 const auto *C = dyn_cast<Constant>(Mask);
 if (!C || !C->isAllOnesValue())
 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
 }
 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
 }
 Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1);
-return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask, NumElts);
+return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);
 }
 // Replace a masked intrinsic with an older unmasked intrinsic.
 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,
 Intrinsic::ID IID) {
-Function *F = CI.getCalledFunction();
+Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
-Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);
 Value *Rep = Builder.CreateCall(Intrin,
 { CI.getArgOperand(0), CI.getArgOperand(1) });
 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
 }
 Value* Op = CI.getArgOperand(0);
 Type* ReturnOp = CI.getType();
 unsigned NumElts = CI.getType()->getVectorNumElements();
 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
+}
+// Replace intrinsic with unmasked version and a select.
+static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
+CallInst &CI, Value *&Rep) {
+Name = Name.substr(12); // Remove avx512.mask.
+unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
+unsigned EltWidth = CI.getType()->getScalarSizeInBits();
+Intrinsic::ID IID;
+if (Name.startswith("max.p")) {
+if (VecWidth == 128 && EltWidth == 32)
+IID = Intrinsic::x86_sse_max_ps;
+else if (VecWidth == 128 && EltWidth == 64)
+IID = Intrinsic::x86_sse2_max_pd;
+else if (VecWidth == 256 && EltWidth == 32)
+IID = Intrinsic::x86_avx_max_ps_256;
+else if (VecWidth == 256 && EltWidth == 64)
+IID = Intrinsic::x86_avx_max_pd_256;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("min.p")) {
+if (VecWidth == 128 && EltWidth == 32)
+IID = Intrinsic::x86_sse_min_ps;
+else if (VecWidth == 128 && EltWidth == 64)
+IID = Intrinsic::x86_sse2_min_pd;
+else if (VecWidth == 256 && EltWidth == 32)
+IID = Intrinsic::x86_avx_min_ps_256;
+else if (VecWidth == 256 && EltWidth == 64)
+IID = Intrinsic::x86_avx_min_pd_256;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("pshuf.b.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_ssse3_pshuf_b_128;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_pshuf_b;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_pshuf_b_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("pmul.hr.sw.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_pmul_hr_sw;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("pmulh.w.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_sse2_pmulh_w;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_pmulh_w;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_pmulh_w_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("pmulhu.w.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_sse2_pmulhu_w;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_pmulhu_w;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_pmulhu_w_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("pmaddw.d.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_sse2_pmadd_wd;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_pmadd_wd;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_pmaddw_d_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("pmaddubs.w.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_pmadd_ub_sw;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_pmaddubs_w_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("packsswb.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_sse2_packsswb_128;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_packsswb;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_packsswb_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("packssdw.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_sse2_packssdw_128;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_packssdw;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_packssdw_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("packuswb.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_sse2_packuswb_128;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_packuswb;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_packuswb_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("packusdw.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_sse41_packusdw;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx2_packusdw;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_packusdw_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("vpermilvar.")) {
+if (VecWidth == 128 && EltWidth == 32)
+IID = Intrinsic::x86_avx_vpermilvar_ps;
+else if (VecWidth == 128 && EltWidth == 64)
+IID = Intrinsic::x86_avx_vpermilvar_pd;
+else if (VecWidth == 256 && EltWidth == 32)
+IID = Intrinsic::x86_avx_vpermilvar_ps_256;
+else if (VecWidth == 256 && EltWidth == 64)
+IID = Intrinsic::x86_avx_vpermilvar_pd_256;
+else if (VecWidth == 512 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
+else if (VecWidth == 512 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name == "cvtpd2dq.256") {
+IID = Intrinsic::x86_avx_cvt_pd2dq_256;
+} else if (Name == "cvtpd2ps.256") {
+IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
+} else if (Name == "cvttpd2dq.256") {
+IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
+} else if (Name == "cvttps2dq.128") {
+IID = Intrinsic::x86_sse2_cvttps2dq;
+} else if (Name == "cvttps2dq.256") {
+IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
+} else if (Name.startswith("permvar.")) {
+bool IsFloat = CI.getType()->isFPOrFPVectorTy();
+if (VecWidth == 256 && EltWidth == 32 && IsFloat)
+IID = Intrinsic::x86_avx2_permps;
+else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
+IID = Intrinsic::x86_avx2_permd;
+else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
+IID = Intrinsic::x86_avx512_permvar_df_256;
+else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
+IID = Intrinsic::x86_avx512_permvar_di_256;
+else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
+IID = Intrinsic::x86_avx512_permvar_sf_512;
+else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
+IID = Intrinsic::x86_avx512_permvar_si_512;
+else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
+IID = Intrinsic::x86_avx512_permvar_df_512;
+else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
+IID = Intrinsic::x86_avx512_permvar_di_512;
+else if (VecWidth == 128 && EltWidth == 16)
+IID = Intrinsic::x86_avx512_permvar_hi_128;
+else if (VecWidth == 256 && EltWidth == 16)
+IID = Intrinsic::x86_avx512_permvar_hi_256;
+else if (VecWidth == 512 && EltWidth == 16)
+IID = Intrinsic::x86_avx512_permvar_hi_512;
+else if (VecWidth == 128 && EltWidth == 8)
+IID = Intrinsic::x86_avx512_permvar_qi_128;
+else if (VecWidth == 256 && EltWidth == 8)
+IID = Intrinsic::x86_avx512_permvar_qi_256;
+else if (VecWidth == 512 && EltWidth == 8)
+IID = Intrinsic::x86_avx512_permvar_qi_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("dbpsadbw.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_avx512_dbpsadbw_128;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx512_dbpsadbw_256;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_dbpsadbw_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("pmultishift.qb.")) {
+if (VecWidth == 128)
+IID = Intrinsic::x86_avx512_pmultishift_qb_128;
+else if (VecWidth == 256)
+IID = Intrinsic::x86_avx512_pmultishift_qb_256;
+else if (VecWidth == 512)
+IID = Intrinsic::x86_avx512_pmultishift_qb_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("conflict.")) {
+if (Name[9] == 'd' && VecWidth == 128)
+IID = Intrinsic::x86_avx512_conflict_d_128;
+else if (Name[9] == 'd' && VecWidth == 256)
+IID = Intrinsic::x86_avx512_conflict_d_256;
+else if (Name[9] == 'd' && VecWidth == 512)
+IID = Intrinsic::x86_avx512_conflict_d_512;
+else if (Name[9] == 'q' && VecWidth == 128)
+IID = Intrinsic::x86_avx512_conflict_q_128;
+else if (Name[9] == 'q' && VecWidth == 256)
+IID = Intrinsic::x86_avx512_conflict_q_256;
+else if (Name[9] == 'q' && VecWidth == 512)
+IID = Intrinsic::x86_avx512_conflict_q_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else if (Name.startswith("pavg.")) {
+if (Name[5] == 'b' && VecWidth == 128)
+IID = Intrinsic::x86_sse2_pavg_b;
+else if (Name[5] == 'b' && VecWidth == 256)
+IID = Intrinsic::x86_avx2_pavg_b;
+else if (Name[5] == 'b' && VecWidth == 512)
+IID = Intrinsic::x86_avx512_pavg_b_512;
+else if (Name[5] == 'w' && VecWidth == 128)
+IID = Intrinsic::x86_sse2_pavg_w;
+else if (Name[5] == 'w' && VecWidth == 256)
+IID = Intrinsic::x86_avx2_pavg_w;
+else if (Name[5] == 'w' && VecWidth == 512)
+IID = Intrinsic::x86_avx512_pavg_w_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+} else
+return false;
+SmallVector<Value *, 4> Args(CI.arg_operands().begin(),
+CI.arg_operands().end());
+Args.pop_back();
+Args.pop_back();
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
+Args);
+unsigned NumArgs = CI.getNumArgOperands();
+Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
+CI.getArgOperand(NumArgs - 2));
+return true;
+}
+/// Upgrade comment in call to inline asm that represents an objc retain release
+/// marker.
+void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
+size_t Pos;
+if (AsmStr->find("mov\tfp") == 0 &&
+AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
+(Pos = AsmStr->find("# marker")) != std::string::npos) {
+AsmStr->replace(Pos, 1, ";");
+}
+return;
 }
 /// Upgrade a call to an old intrinsic. All argument and return casting must be
 /// provided to seamlessly integrate with existing context.
 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
 Arg0 = Builder.CreateBitCast(Arg0,
 PointerType::getUnqual(Arg1->getType()),
 "cast");
 Builder.CreateAlignedStore(Arg1, Arg0, 1);
+// Remove intrinsic.
+CI->eraseFromParent();
+return;
+}
+if (IsX86 && Name == "avx512.mask.store.ss") {
+Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
+UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
+Mask, false);
 // Remove intrinsic.
 CI->eraseFromParent();
 return;
 }
 ExtTy = Type::getInt64Ty(C);
 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
 ExtTy->getPrimitiveSizeInBits();
 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
 Rep = Builder.CreateVectorSplat(NumElts, Rep);
+} else if (IsX86 && (Name == "sse.sqrt.ss" ||
+Name == "sse2.sqrt.sd")) {
+Value *Vec = CI->getArgOperand(0);
+Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
+Function *Intr = Intrinsic::getDeclaration(F->getParent(),
+Intrinsic::sqrt, Elt0->getType());
+Elt0 = Builder.CreateCall(Intr, Elt0);
+Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
+} else if (IsX86 && (Name.startswith("avx.sqrt.p") ||
+Name.startswith("sse2.sqrt.p") ||
+Name.startswith("sse.sqrt.p"))) {
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
+Intrinsic::sqrt,
+CI->getType()),
+{CI->getArgOperand(0)});
+} else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) {
+if (CI->getNumArgOperands() == 4 &&
+(!isa<ConstantInt>(CI->getArgOperand(3)) ||
+cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
+Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
+: Intrinsic::x86_avx512_sqrt_pd_512;
+Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
+IID), Args);
+} else {
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
+Intrinsic::sqrt,
+CI->getType()),
+{CI->getArgOperand(0)});
+}
+Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
+CI->getArgOperand(1));
 } else if (IsX86 && (Name.startswith("avx512.ptestm") ||
 Name.startswith("avx512.ptestnm"))) {
 Value *Op0 = CI->getArgOperand(0);
 Value *Op1 = CI->getArgOperand(1);
 Value *Mask = CI->getArgOperand(2);
 llvm::Type *Ty = Op0->getType();
 Value *Zero = llvm::Constant::getNullValue(Ty);
 ICmpInst::Predicate Pred =
 Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
 Rep = Builder.CreateICmp(Pred, Rep, Zero);
-unsigned NumElts = Op0->getType()->getVectorNumElements();
+Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
-Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask, NumElts);
 } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
 unsigned NumElts =
 CI->getArgOperand(1)->getType()->getVectorNumElements();
 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
 else
 C = ConstantInt::getNullValue(Builder.getInt16Ty());
 Rep = Builder.CreateICmpEQ(Rep, C);
 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
-} else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd")) {
+} else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
+Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
+Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
+Name == "sse.div.ss" || Name == "sse2.div.sd")) {
 Type *I32Ty = Type::getInt32Ty(C);
 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
 ConstantInt::get(I32Ty, 0));
 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
 ConstantInt::get(I32Ty, 0));
-Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
+Value *EltOp;
-Builder.CreateFAdd(Elt0, Elt1),
+if (Name.contains(".add."))
-ConstantInt::get(I32Ty, 0));
+EltOp = Builder.CreateFAdd(Elt0, Elt1);
-} else if (IsX86 && (Name == "sse.sub.ss" || Name == "sse2.sub.sd")) {
+else if (Name.contains(".sub."))
-Type *I32Ty = Type::getInt32Ty(C);
+EltOp = Builder.CreateFSub(Elt0, Elt1);
-Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
+else if (Name.contains(".mul."))
-ConstantInt::get(I32Ty, 0));
+EltOp = Builder.CreateFMul(Elt0, Elt1);
-Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
+else
-ConstantInt::get(I32Ty, 0));
+EltOp = Builder.CreateFDiv(Elt0, Elt1);
-Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
+Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
-Builder.CreateFSub(Elt0, Elt1),
-ConstantInt::get(I32Ty, 0));
-} else if (IsX86 && (Name == "sse.mul.ss" || Name == "sse2.mul.sd")) {
-Type *I32Ty = Type::getInt32Ty(C);
-Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
-ConstantInt::get(I32Ty, 0));
-Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
-ConstantInt::get(I32Ty, 0));
-Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
-Builder.CreateFMul(Elt0, Elt1),
-ConstantInt::get(I32Ty, 0));
-} else if (IsX86 && (Name == "sse.div.ss" || Name == "sse2.div.sd")) {
-Type *I32Ty = Type::getInt32Ty(C);
-Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
-ConstantInt::get(I32Ty, 0));
-Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
-ConstantInt::get(I32Ty, 0));
-Rep = Builder.CreateInsertElement(CI->getArgOperand(0),
-Builder.CreateFDiv(Elt0, Elt1),
 ConstantInt::get(I32Ty, 0));
 } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {
 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
 bool CmpEq = Name[16] == 'e';
 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
-} else if (IsX86 && Name.startswith("avx512.mask.cmp")) {
+} else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {
+Type *OpTy = CI->getArgOperand(0)->getType();
+unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
+Intrinsic::ID IID;
+switch (VecWidth) {
+default: llvm_unreachable("Unexpected intrinsic");
+case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
+case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
+case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
+}
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{ CI->getOperand(0), CI->getArgOperand(1) });
+Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
+} else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {
+Type *OpTy = CI->getArgOperand(0)->getType();
+unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
+unsigned EltWidth = OpTy->getScalarSizeInBits();
+Intrinsic::ID IID;
+if (VecWidth == 128 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_fpclass_ps_128;
+else if (VecWidth == 256 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_fpclass_ps_256;
+else if (VecWidth == 512 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_fpclass_ps_512;
+else if (VecWidth == 128 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_fpclass_pd_128;
+else if (VecWidth == 256 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_fpclass_pd_256;
+else if (VecWidth == 512 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_fpclass_pd_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{ CI->getOperand(0), CI->getArgOperand(1) });
+Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
+} else if (IsX86 && Name.startswith("avx512.mask.cmp.p")) {
+Type *OpTy = CI->getArgOperand(0)->getType();
+unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
+unsigned EltWidth = OpTy->getScalarSizeInBits();
+Intrinsic::ID IID;
+if (VecWidth == 128 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_cmp_ps_128;
+else if (VecWidth == 256 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_cmp_ps_256;
+else if (VecWidth == 512 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_cmp_ps_512;
+else if (VecWidth == 128 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_cmp_pd_128;
+else if (VecWidth == 256 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_cmp_pd_256;
+else if (VecWidth == 512 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_cmp_pd_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+SmallVector<Value *, 4> Args;
+Args.push_back(CI->getArgOperand(0));
+Args.push_back(CI->getArgOperand(1));
+Args.push_back(CI->getArgOperand(2));
+if (CI->getNumArgOperands() == 5)
+Args.push_back(CI->getArgOperand(4));
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+Args);
+Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(3));
+} else if (IsX86 && Name.startswith("avx512.mask.cmp.") &&
+Name[16] != 'p') {
+// Integer compare intrinsics.
 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
-} else if (IsX86 && Name.startswith("avx512.mask.ucmp")) {
+} else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) {
 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
 } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") ||
 Name.startswith("avx512.cvtw2mask.") ||
 Name.startswith("avx512.cvtd2mask.") ||
 Name.startswith("avx512.cvtq2mask."))) {
 Value *Op = CI->getArgOperand(0);
 Value *Zero = llvm::Constant::getNullValue(Op->getType());
 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
-Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr,
+Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);
-Op->getType()->getVectorNumElements());
 } else if(IsX86 && (Name == "ssse3.pabs.b.128" ||
 Name == "ssse3.pabs.w.128" ||
 Name == "ssse3.pabs.d.128" ||
 Name.startswith("avx2.pabs") ||
 Name.startswith("avx512.mask.pabs"))) {
 Name == "sse41.pminuw" ||
 Name == "sse41.pminud" ||
 Name.startswith("avx2.pminu") ||
 Name.startswith("avx512.mask.pminu"))) {
 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT);
+} else if (IsX86 && (Name == "sse2.pmulu.dq" ||
+Name == "avx2.pmulu.dq" ||
+Name == "avx512.pmulu.dq.512" ||
+Name.startswith("avx512.mask.pmulu.dq."))) {
+Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
+} else if (IsX86 && (Name == "sse41.pmuldq" ||
+Name == "avx2.pmul.dq" ||
+Name == "avx512.pmul.dq.512" ||
+Name.startswith("avx512.mask.pmul.dq."))) {
+Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
+} else if (IsX86 && (Name == "sse.cvtsi2ss" ||
+Name == "sse2.cvtsi2sd" ||
+Name == "sse.cvtsi642ss" ||
+Name == "sse2.cvtsi642sd")) {
+Rep = Builder.CreateSIToFP(CI->getArgOperand(1),
+CI->getType()->getVectorElementType());
+Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
+} else if (IsX86 && Name == "avx512.cvtusi2sd") {
+Rep = Builder.CreateUIToFP(CI->getArgOperand(1),
+CI->getType()->getVectorElementType());
+Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
+} else if (IsX86 && Name == "sse2.cvtss2sd") {
+Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
+Rep = Builder.CreateFPExt(Rep, CI->getType()->getVectorElementType());
+Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
 } else if (IsX86 && (Name == "sse2.cvtdq2pd" ||
+Name == "sse2.cvtdq2ps" ||
+Name == "avx.cvtdq2.pd.256" ||
+Name == "avx.cvtdq2.ps.256" ||
+Name.startswith("avx512.mask.cvtdq2pd.") ||
+Name.startswith("avx512.mask.cvtudq2pd.") ||
+Name.startswith("avx512.mask.cvtdq2ps.") ||
+Name.startswith("avx512.mask.cvtudq2ps.") ||
+Name.startswith("avx512.mask.cvtqq2pd.") ||
+Name.startswith("avx512.mask.cvtuqq2pd.") ||
+Name == "avx512.mask.cvtqq2ps.256" ||
+Name == "avx512.mask.cvtqq2ps.512" ||
+Name == "avx512.mask.cvtuqq2ps.256" ||
+Name == "avx512.mask.cvtuqq2ps.512" ||
 Name == "sse2.cvtps2pd" ||
-Name == "avx.cvtdq2.pd.256" ||
 Name == "avx.cvt.ps2.pd.256" ||
-Name.startswith("avx512.mask.cvtdq2pd.") ||
+Name == "avx512.mask.cvtps2pd.128" ||
-Name.startswith("avx512.mask.cvtudq2pd."))) {
+Name == "avx512.mask.cvtps2pd.256")) {
-// Lossless i32/float to double conversion.
+Type *DstTy = CI->getType();
-// Extract the bottom elements if necessary and convert to double vector.
-Value *Src = CI->getArgOperand(0);
-VectorType *SrcTy = cast<VectorType>(Src->getType());
-VectorType *DstTy = cast<VectorType>(CI->getType());
 Rep = CI->getArgOperand(0);
+Type *SrcTy = Rep->getType();
-unsigned NumDstElts = DstTy->getNumElements();
-if (NumDstElts < SrcTy->getNumElements()) {
+unsigned NumDstElts = DstTy->getVectorNumElements();
+if (NumDstElts < SrcTy->getVectorNumElements()) {
 assert(NumDstElts == 2 && "Unexpected vector size");
 uint32_t ShuffleMask[2] = { 0, 1 };
-Rep = Builder.CreateShuffleVector(Rep, UndefValue::get(SrcTy),
+Rep = Builder.CreateShuffleVector(Rep, Rep, ShuffleMask);
-ShuffleMask);
 }
-bool SInt2Double = (StringRef::npos != Name.find("cvtdq2"));
+bool IsPS2PD = SrcTy->getVectorElementType()->isFloatTy();
-bool UInt2Double = (StringRef::npos != Name.find("cvtudq2"));
+bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
-if (SInt2Double)
+if (IsPS2PD)
-Rep = Builder.CreateSIToFP(Rep, DstTy, "cvtdq2pd");
-else if (UInt2Double)
-Rep = Builder.CreateUIToFP(Rep, DstTy, "cvtudq2pd");
-else
 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
+else if (CI->getNumArgOperands() == 4 &&
-if (CI->getNumArgOperands() == 3)
+(!isa<ConstantInt>(CI->getArgOperand(3)) ||
+cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
+Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
+: Intrinsic::x86_avx512_sitofp_round;
+Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
+{ DstTy, SrcTy });
+Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
+} else {
+Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
+: Builder.CreateSIToFP(Rep, DstTy, "cvt");
+}
+if (CI->getNumArgOperands() >= 3)
 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 CI->getArgOperand(1));
 } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
 CI->getArgOperand(1), CI->getArgOperand(2),
 /*Aligned*/false);
 } else if (IsX86 && (Name.startswith("avx512.mask.load."))) {
 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
 CI->getArgOperand(1),CI->getArgOperand(2),
 /*Aligned*/true);
+} else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
+Type *ResultTy = CI->getType();
+Type *PtrTy = ResultTy->getVectorElementType();
+// Cast the pointer to element type.
+Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
+llvm::PointerType::getUnqual(PtrTy));
+Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
+ResultTy->getVectorNumElements());
+Function *ELd = Intrinsic::getDeclaration(F->getParent(),
+Intrinsic::masked_expandload,
+ResultTy);
+Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
+} else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) {
+Type *ResultTy = CI->getArgOperand(1)->getType();
+Type *PtrTy = ResultTy->getVectorElementType();
+// Cast the pointer to element type.
+Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
+llvm::PointerType::getUnqual(PtrTy));
+Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
+ResultTy->getVectorNumElements());
+Function *CSt = Intrinsic::getDeclaration(F->getParent(),
+Intrinsic::masked_compressstore,
+ResultTy);
+Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
+} else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
+Name.startswith("avx512.mask.expand."))) {
+Type *ResultTy = CI->getType();
+Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
+ResultTy->getVectorNumElements());
+bool IsCompress = Name[12] == 'c';
+Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
+: Intrinsic::x86_avx512_mask_expand;
+Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
+Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
+MaskVec });
 } else if (IsX86 && Name.startswith("xop.vpcom")) {
-Intrinsic::ID intID;
+bool IsSigned;
-if (Name.endswith("ub"))
+if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
-intID = Intrinsic::x86_xop_vpcomub;
+Name.endswith("uq"))
-else if (Name.endswith("uw"))
+IsSigned = false;
-intID = Intrinsic::x86_xop_vpcomuw;
+else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
-else if (Name.endswith("ud"))
+Name.endswith("q"))
-intID = Intrinsic::x86_xop_vpcomud;
+IsSigned = true;
-else if (Name.endswith("uq"))
-intID = Intrinsic::x86_xop_vpcomuq;
-else if (Name.endswith("b"))
-intID = Intrinsic::x86_xop_vpcomb;
-else if (Name.endswith("w"))
-intID = Intrinsic::x86_xop_vpcomw;
-else if (Name.endswith("d"))
-intID = Intrinsic::x86_xop_vpcomd;
-else if (Name.endswith("q"))
-intID = Intrinsic::x86_xop_vpcomq;
 else
 llvm_unreachable("Unknown suffix");
-Name = Name.substr(9); // strip off "xop.vpcom"
 unsigned Imm;
-if (Name.startswith("lt"))
+if (CI->getNumArgOperands() == 3) {
-Imm = 0;
+Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
-else if (Name.startswith("le"))
+} else {
-Imm = 1;
+Name = Name.substr(9); // strip off "xop.vpcom"
-else if (Name.startswith("gt"))
+if (Name.startswith("lt"))
-Imm = 2;
+Imm = 0;
-else if (Name.startswith("ge"))
+else if (Name.startswith("le"))
-Imm = 3;
+Imm = 1;
-else if (Name.startswith("eq"))
+else if (Name.startswith("gt"))
-Imm = 4;
+Imm = 2;
-else if (Name.startswith("ne"))
+else if (Name.startswith("ge"))
-Imm = 5;
+Imm = 3;
-else if (Name.startswith("false"))
+else if (Name.startswith("eq"))
-Imm = 6;
+Imm = 4;
-else if (Name.startswith("true"))
+else if (Name.startswith("ne"))
-Imm = 7;
+Imm = 5;
-else
+else if (Name.startswith("false"))
-llvm_unreachable("Unknown condition");
+Imm = 6;
+else if (Name.startswith("true"))
-Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
+Imm = 7;
-Rep =
+else
-Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1),
+llvm_unreachable("Unknown condition");
-Builder.getInt8(Imm)});
+}
+Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
 } else if (IsX86 && Name.startswith("xop.vpcmov")) {
 Value *Sel = CI->getArgOperand(2);
 Value *NotSel = Builder.CreateNot(Sel);
 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
 Rep = Builder.CreateOr(Sel0, Sel1);
+} else if (IsX86 && (Name.startswith("xop.vprot") ||
+Name.startswith("avx512.prol") ||
+Name.startswith("avx512.mask.prol"))) {
+Rep = upgradeX86Rotate(Builder, *CI, false);
+} else if (IsX86 && (Name.startswith("avx512.pror") ||
+Name.startswith("avx512.mask.pror"))) {
+Rep = upgradeX86Rotate(Builder, *CI, true);
+} else if (IsX86 && (Name.startswith("avx512.vpshld.") ||
+Name.startswith("avx512.mask.vpshld") ||
+Name.startswith("avx512.maskz.vpshld"))) {
+bool ZeroMask = Name[11] == 'z';
+Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
+} else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||
+Name.startswith("avx512.mask.vpshrd") ||
+Name.startswith("avx512.maskz.vpshrd"))) {
+bool ZeroMask = Name[11] == 'z';
+Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
 } else if (IsX86 && Name == "sse42.crc32.64.8") {
 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
 Intrinsic::x86_sse42_crc32_32_8);
 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
-} else if (IsX86 && Name.startswith("avx.vbroadcast.s")) {
+} else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||
+Name.startswith("avx512.vbroadcast.s"))) {
 // Replace broadcasts with a series of insertelements.
 Type *VecTy = CI->getType();
 Type *EltTy = VecTy->getVectorElementType();
 unsigned EltNum = VecTy->getVectorNumElements();
 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
 : Builder.CreateZExt(SV, DstTy);
 // If there are 3 arguments, it's a masked intrinsic so we need a select.
 if (CI->getNumArgOperands() == 3)
 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 CI->getArgOperand(1));
+} else if (Name == "avx512.mask.pmov.qd.256" ||
+Name == "avx512.mask.pmov.qd.512" ||
+Name == "avx512.mask.pmov.wb.256" ||
+Name == "avx512.mask.pmov.wb.512") {
+Type *Ty = CI->getArgOperand(1)->getType();
+Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
+Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
+CI->getArgOperand(1));
 } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
 Name == "avx2.vbroadcasti128")) {
 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
 Type *EltTy = CI->getType()->getVectorElementType();
 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
 Type *VT = VectorType::get(EltTy, NumSrcElts);
 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
 PointerType::getUnqual(VT));
-Value *Load = Builder.CreateAlignedLoad(Op, 1);
+Value *Load = Builder.CreateAlignedLoad(VT, Op, 1);
 if (NumSrcElts == 2)
 Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
 { 0, 1, 0, 1 });
 else
 Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
 Constant::getNullValue(MaskTy));
 if (CI->getNumArgOperands() == 3)
 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 CI->getArgOperand(1));
+} else if (IsX86 && (Name.startswith("sse2.padds.") ||
+Name.startswith("sse2.psubs.") ||
+Name.startswith("avx2.padds.") ||
+Name.startswith("avx2.psubs.") ||
+Name.startswith("avx512.padds.") ||
+Name.startswith("avx512.psubs.") ||
+Name.startswith("avx512.mask.padds.") ||
+Name.startswith("avx512.mask.psubs."))) {
+bool IsAdd = Name.contains(".padds");
+Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, true, IsAdd);
+} else if (IsX86 && (Name.startswith("sse2.paddus.") ||
+Name.startswith("sse2.psubus.") ||
+Name.startswith("avx2.paddus.") ||
+Name.startswith("avx2.psubus.") ||
+Name.startswith("avx512.mask.paddus.") ||
+Name.startswith("avx512.mask.psubus."))) {
+bool IsAdd = Name.contains(".paddus");
+Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, false, IsAdd);
 } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
 CI->getArgOperand(1),
 CI->getArgOperand(2),
 CI->getArgOperand(3),
 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.pand.")) {
+} else if (IsX86 && (Name.startswith("avx512.mask.and.") ||
-Rep = Builder.CreateAnd(CI->getArgOperand(0), CI->getArgOperand(1));
+Name.startswith("avx512.mask.pand."))) {
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.pandn.")) {
-Rep = Builder.CreateAnd(Builder.CreateNot(CI->getArgOperand(0)),
-CI->getArgOperand(1));
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.por.")) {
-Rep = Builder.CreateOr(CI->getArgOperand(0), CI->getArgOperand(1));
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.pxor.")) {
-Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1));
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.and.")) {
 VectorType *FTy = cast<VectorType>(CI->getType());
 VectorType *ITy = VectorType::getInteger(FTy);
 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
 Rep = Builder.CreateBitCast(Rep, FTy);
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.andn.")) {
+} else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||
+Name.startswith("avx512.mask.pandn."))) {
 VectorType *FTy = cast<VectorType>(CI->getType());
 VectorType *ITy = VectorType::getInteger(FTy);
 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
 Rep = Builder.CreateAnd(Rep,
 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
 Rep = Builder.CreateBitCast(Rep, FTy);
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.or.")) {
+} else if (IsX86 && (Name.startswith("avx512.mask.or.") ||
+Name.startswith("avx512.mask.por."))) {
 VectorType *FTy = cast<VectorType>(CI->getType());
 VectorType *ITy = VectorType::getInteger(FTy);
 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
 Rep = Builder.CreateBitCast(Rep, FTy);
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.xor.")) {
+} else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||
+Name.startswith("avx512.mask.pxor."))) {
 VectorType *FTy = cast<VectorType>(CI->getType());
 VectorType *ITy = VectorType::getInteger(FTy);
 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
 Rep = Builder.CreateBitCast(Rep, FTy);
 CI->getArgOperand(2));
 } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {
 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
-} else if (IsX86 && (Name.startswith("avx512.mask.add.p"))) {
+} else if (IsX86 && Name.startswith("avx512.mask.add.p")) {
-Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
+if (Name.endswith(".512")) {
+Intrinsic::ID IID;
+if (Name[17] == 's')
+IID = Intrinsic::x86_avx512_add_ps_512;
+else
+IID = Intrinsic::x86_avx512_add_pd_512;
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{ CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(4) });
+} else {
+Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
+}
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
 } else if (IsX86 && Name.startswith("avx512.mask.div.p")) {
-Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
+if (Name.endswith(".512")) {
+Intrinsic::ID IID;
+if (Name[17] == 's')
+IID = Intrinsic::x86_avx512_div_ps_512;
+else
+IID = Intrinsic::x86_avx512_div_pd_512;
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{ CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(4) });
+} else {
+Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
+}
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
 } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {
-Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
+if (Name.endswith(".512")) {
+Intrinsic::ID IID;
+if (Name[17] == 's')
+IID = Intrinsic::x86_avx512_mul_ps_512;
+else
+IID = Intrinsic::x86_avx512_mul_pd_512;
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{ CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(4) });
+} else {
+Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
+}
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
 } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {
-Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
+if (Name.endswith(".512")) {
+Intrinsic::ID IID;
+if (Name[17] == 's')
+IID = Intrinsic::x86_avx512_sub_ps_512;
+else
+IID = Intrinsic::x86_avx512_sub_pd_512;
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{ CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(4) });
+} else {
+Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
+}
+Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
+CI->getArgOperand(2));
+} else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
+Name.startswith("avx512.mask.min.p")) &&
+Name.drop_front(18) == ".512") {
+bool IsDouble = Name[17] == 'd';
+bool IsMin = Name[13] == 'i';
+static const Intrinsic::ID MinMaxTbl[2][2] = {
+{ Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
+{ Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
+};
+Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{ CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(4) });
 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
 CI->getArgOperand(2));
 } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {
 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
 Intrinsic::ctlz,
 CI->getType()),
 { CI->getArgOperand(0), Builder.getInt1(false) });
 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
 CI->getArgOperand(1));
-} else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
-Name.startswith("avx512.mask.min.p"))) {
-bool IsMin = Name[13] == 'i';
-VectorType *VecTy = cast<VectorType>(CI->getType());
-unsigned VecWidth = VecTy->getPrimitiveSizeInBits();
-unsigned EltWidth = VecTy->getScalarSizeInBits();
-Intrinsic::ID IID;
-if (!IsMin && VecWidth == 128 && EltWidth == 32)
-IID = Intrinsic::x86_sse_max_ps;
-else if (!IsMin && VecWidth == 128 && EltWidth == 64)
-IID = Intrinsic::x86_sse2_max_pd;
-else if (!IsMin && VecWidth == 256 && EltWidth == 32)
-IID = Intrinsic::x86_avx_max_ps_256;
-else if (!IsMin && VecWidth == 256 && EltWidth == 64)
-IID = Intrinsic::x86_avx_max_pd_256;
-else if (IsMin && VecWidth == 128 && EltWidth == 32)
-IID = Intrinsic::x86_sse_min_ps;
-else if (IsMin && VecWidth == 128 && EltWidth == 64)
-IID = Intrinsic::x86_sse2_min_pd;
-else if (IsMin && VecWidth == 256 && EltWidth == 32)
-IID = Intrinsic::x86_avx_min_ps_256;
-else if (IsMin && VecWidth == 256 && EltWidth == 64)
-IID = Intrinsic::x86_avx_min_pd_256;
-else
-llvm_unreachable("Unexpected intrinsic");
-Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-{ CI->getArgOperand(0), CI->getArgOperand(1) });
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.pshuf.b.")) {
-VectorType *VecTy = cast<VectorType>(CI->getType());
-Intrinsic::ID IID;
-if (VecTy->getPrimitiveSizeInBits() == 128)
-IID = Intrinsic::x86_ssse3_pshuf_b_128;
-else if (VecTy->getPrimitiveSizeInBits() == 256)
-IID = Intrinsic::x86_avx2_pshuf_b;
-else if (VecTy->getPrimitiveSizeInBits() == 512)
-IID = Intrinsic::x86_avx512_pshuf_b_512;
-else
-llvm_unreachable("Unexpected intrinsic");
-Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-{ CI->getArgOperand(0), CI->getArgOperand(1) });
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
-} else if (IsX86 && (Name.startswith("avx512.mask.pmul.dq.") ||
-Name.startswith("avx512.mask.pmulu.dq."))) {
-bool IsUnsigned = Name[16] == 'u';
-VectorType *VecTy = cast<VectorType>(CI->getType());
-Intrinsic::ID IID;
-if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 128)
-IID = Intrinsic::x86_sse41_pmuldq;
-else if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 256)
-IID = Intrinsic::x86_avx2_pmul_dq;
-else if (!IsUnsigned && VecTy->getPrimitiveSizeInBits() == 512)
-IID = Intrinsic::x86_avx512_pmul_dq_512;
-else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 128)
-IID = Intrinsic::x86_sse2_pmulu_dq;
-else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 256)
-IID = Intrinsic::x86_avx2_pmulu_dq;
-else if (IsUnsigned && VecTy->getPrimitiveSizeInBits() == 512)
-IID = Intrinsic::x86_avx512_pmulu_dq_512;
-else
-llvm_unreachable("Unexpected intrinsic");
-Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-{ CI->getArgOperand(0), CI->getArgOperand(1) });
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
-} else if (IsX86 && Name.startswith("avx512.mask.pack")) {
-bool IsUnsigned = Name[16] == 'u';
-bool IsDW = Name[18] == 'd';
-VectorType *VecTy = cast<VectorType>(CI->getType());
-Intrinsic::ID IID;
-if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 128)
-IID = Intrinsic::x86_sse2_packsswb_128;
-else if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 256)
-IID = Intrinsic::x86_avx2_packsswb;
-else if (!IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 512)
-IID = Intrinsic::x86_avx512_packsswb_512;
-else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 128)
-IID = Intrinsic::x86_sse2_packssdw_128;
-else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 256)
-IID = Intrinsic::x86_avx2_packssdw;
-else if (!IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 512)
-IID = Intrinsic::x86_avx512_packssdw_512;
-else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 128)
-IID = Intrinsic::x86_sse2_packuswb_128;
-else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 256)
-IID = Intrinsic::x86_avx2_packuswb;
-else if (IsUnsigned && !IsDW && VecTy->getPrimitiveSizeInBits() == 512)
-IID = Intrinsic::x86_avx512_packuswb_512;
-else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 128)
-IID = Intrinsic::x86_sse41_packusdw;
-else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 256)
-IID = Intrinsic::x86_avx2_packusdw;
-else if (IsUnsigned && IsDW && VecTy->getPrimitiveSizeInBits() == 512)
-IID = Intrinsic::x86_avx512_packusdw_512;
-else
-llvm_unreachable("Unexpected intrinsic");
-Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-{ CI->getArgOperand(0), CI->getArgOperand(1) });
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
 } else if (IsX86 && Name.startswith("avx512.mask.psll")) {
 bool IsImmediate = Name[16] == 'i' ||
 (Name.size() > 18 && Name[18] == 'i');
 bool IsVariable = Name[16] == 'v';
 char Size = Name[16] == '.' ? Name[17] :
 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
 } else if (IsX86 && Name.startswith("avx512.mask.move.s")) {
 Rep = upgradeMaskedMove(Builder, *CI);
 } else if (IsX86 && Name.startswith("avx512.cvtmask2")) {
 Rep = UpgradeMaskToInt(Builder, *CI);
-} else if (IsX86 && Name.startswith("avx512.mask.vpermilvar.")) {
-Intrinsic::ID IID;
-if (Name.endswith("ps.128"))
-IID = Intrinsic::x86_avx_vpermilvar_ps;
-else if (Name.endswith("pd.128"))
-IID = Intrinsic::x86_avx_vpermilvar_pd;
-else if (Name.endswith("ps.256"))
-IID = Intrinsic::x86_avx_vpermilvar_ps_256;
-else if (Name.endswith("pd.256"))
-IID = Intrinsic::x86_avx_vpermilvar_pd_256;
-else if (Name.endswith("ps.512"))
-IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
-else if (Name.endswith("pd.512"))
-IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
-else
-llvm_unreachable("Unexpected vpermilvar intrinsic");
-Function *Intrin = Intrinsic::getDeclaration(F->getParent(), IID);
-Rep = Builder.CreateCall(Intrin,
-{ CI->getArgOperand(0), CI->getArgOperand(1) });
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-CI->getArgOperand(2));
 } else if (IsX86 && Name.endswith(".movntdqa")) {
 Module *M = F->getParent();
 MDNode *Node = MDNode::get(
 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
 VectorType *VTy = cast<VectorType>(CI->getType());
 // Convert the type of the pointer to a pointer to the stored type.
 Value *BC =
 Builder.CreateBitCast(Ptr, PointerType::getUnqual(VTy), "cast");
-LoadInst *LI = Builder.CreateAlignedLoad(BC, VTy->getBitWidth() / 8);
+LoadInst *LI = Builder.CreateAlignedLoad(VTy, BC, VTy->getBitWidth() / 8);
 LI->setMetadata(M->getMDKindID("nontemporal"), Node);
 Rep = LI;
-} else if (IsX86 &&
+} else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
-(Name.startswith("sse2.pavg") || Name.startswith("avx2.pavg") ||
+Name.startswith("fma.vfmsub.") ||
-Name.startswith("avx512.mask.pavg"))) {
+Name.startswith("fma.vfnmadd.") ||
-// llvm.x86.sse2.pavg.b/w, llvm.x86.avx2.pavg.b/w,
+Name.startswith("fma.vfnmsub."))) {
-// llvm.x86.avx512.mask.pavg.b/w
+bool NegMul = Name[6] == 'n';
+bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
+bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
+Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2) };
+if (IsScalar) {
+Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
+Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
+Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
+}
+if (NegMul && !IsScalar)
+Ops[0] = Builder.CreateFNeg(Ops[0]);
+if (NegMul && IsScalar)
+Ops[1] = Builder.CreateFNeg(Ops[1]);
+if (NegAcc)
+Ops[2] = Builder.CreateFNeg(Ops[2]);
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
+Intrinsic::fma,
+Ops[0]->getType()),
+Ops);
+if (IsScalar)
+Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
+(uint64_t)0);
+} else if (IsX86 && Name.startswith("fma4.vfmadd.s")) {
+Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2) };
+Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
+Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
+Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
+Intrinsic::fma,
+Ops[0]->getType()),
+Ops);
+Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
+Rep, (uint64_t)0);
+} else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") ||
+Name.startswith("avx512.maskz.vfmadd.s") ||
+Name.startswith("avx512.mask3.vfmadd.s") ||
+Name.startswith("avx512.mask3.vfmsub.s") ||
+Name.startswith("avx512.mask3.vfnmsub.s"))) {
+bool IsMask3 = Name[11] == '3';
+bool IsMaskZ = Name[11] == 'z';
+// Drop the "avx512.mask." to make it easier.
+Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
+bool NegMul = Name[2] == 'n';
+bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
 Value *A = CI->getArgOperand(0);
 Value *B = CI->getArgOperand(1);
-VectorType *ZextType = VectorType::getExtendedElementVectorType(
+Value *C = CI->getArgOperand(2);
-cast<VectorType>(A->getType()));
-Value *ExtendedA = Builder.CreateZExt(A, ZextType);
+if (NegMul && (IsMask3 || IsMaskZ))
-Value *ExtendedB = Builder.CreateZExt(B, ZextType);
+A = Builder.CreateFNeg(A);
-Value *Sum = Builder.CreateAdd(ExtendedA, ExtendedB);
+if (NegMul && !(IsMask3 || IsMaskZ))
-Value *AddOne = Builder.CreateAdd(Sum, ConstantInt::get(ZextType, 1));
+B = Builder.CreateFNeg(B);
-Value *ShiftR = Builder.CreateLShr(AddOne, ConstantInt::get(ZextType, 1));
+if (NegAcc)
-Rep = Builder.CreateTrunc(ShiftR, A->getType());
+C = Builder.CreateFNeg(C);
-if (CI->getNumArgOperands() > 2) {
-Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
+A = Builder.CreateExtractElement(A, (uint64_t)0);
-CI->getArgOperand(2));
+B = Builder.CreateExtractElement(B, (uint64_t)0);
+C = Builder.CreateExtractElement(C, (uint64_t)0);
+if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
+cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
+Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
+Intrinsic::ID IID;
+if (Name.back() == 'd')
+IID = Intrinsic::x86_avx512_vfmadd_f64;
+else
+IID = Intrinsic::x86_avx512_vfmadd_f32;
+Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
+Rep = Builder.CreateCall(FMA, Ops);
+} else {
+Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
+Intrinsic::fma,
+A->getType());
+Rep = Builder.CreateCall(FMA, { A, B, C });
 }
+Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) :
+IsMask3 ? C : A;
+// For Mask3 with NegAcc, we need to create a new extractelement that
+// avoids the negation above.
+if (NegAcc && IsMask3)
+PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
+(uint64_t)0);
+Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
+Rep, PassThru);
+Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
+Rep, (uint64_t)0);
+} else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") ||
+Name.startswith("avx512.mask.vfnmadd.p") ||
+Name.startswith("avx512.mask.vfnmsub.p") ||
+Name.startswith("avx512.mask3.vfmadd.p") ||
+Name.startswith("avx512.mask3.vfmsub.p") ||
+Name.startswith("avx512.mask3.vfnmsub.p") ||
+Name.startswith("avx512.maskz.vfmadd.p"))) {
+bool IsMask3 = Name[11] == '3';
+bool IsMaskZ = Name[11] == 'z';
+// Drop the "avx512.mask." to make it easier.
+Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
+bool NegMul = Name[2] == 'n';
+bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
+Value *A = CI->getArgOperand(0);
+Value *B = CI->getArgOperand(1);
+Value *C = CI->getArgOperand(2);
+if (NegMul && (IsMask3 || IsMaskZ))
+A = Builder.CreateFNeg(A);
+if (NegMul && !(IsMask3 || IsMaskZ))
+B = Builder.CreateFNeg(B);
+if (NegAcc)
+C = Builder.CreateFNeg(C);
+if (CI->getNumArgOperands() == 5 &&
+(!isa<ConstantInt>(CI->getArgOperand(4)) ||
+cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
+Intrinsic::ID IID;
+// Check the character before ".512" in string.
+if (Name[Name.size()-5] == 's')
+IID = Intrinsic::x86_avx512_vfmadd_ps_512;
+else
+IID = Intrinsic::x86_avx512_vfmadd_pd_512;
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{ A, B, C, CI->getArgOperand(4) });
+} else {
+Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
+Intrinsic::fma,
+A->getType());
+Rep = Builder.CreateCall(FMA, { A, B, C });
+}
+Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
+IsMask3 ? CI->getArgOperand(2) :
+CI->getArgOperand(0);
+Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
+} else if (IsX86 && (Name.startswith("fma.vfmaddsub.p") ||
+Name.startswith("fma.vfmsubadd.p"))) {
+bool IsSubAdd = Name[7] == 's';
+int NumElts = CI->getType()->getVectorNumElements();
+Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2) };
+Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
+Ops[0]->getType());
+Value *Odd = Builder.CreateCall(FMA, Ops);
+Ops[2] = Builder.CreateFNeg(Ops[2]);
+Value *Even = Builder.CreateCall(FMA, Ops);
+if (IsSubAdd)
+std::swap(Even, Odd);
+SmallVector<uint32_t, 32> Idxs(NumElts);
+for (int i = 0; i != NumElts; ++i)
+Idxs[i] = i + (i % 2) * NumElts;
+Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
+} else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") ||
+Name.startswith("avx512.mask3.vfmaddsub.p") ||
+Name.startswith("avx512.maskz.vfmaddsub.p") ||
+Name.startswith("avx512.mask3.vfmsubadd.p"))) {
+bool IsMask3 = Name[11] == '3';
+bool IsMaskZ = Name[11] == 'z';
+// Drop the "avx512.mask." to make it easier.
+Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
+bool IsSubAdd = Name[3] == 's';
+if (CI->getNumArgOperands() == 5 &&
+(!isa<ConstantInt>(CI->getArgOperand(4)) ||
+cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
+Intrinsic::ID IID;
+// Check the character before ".512" in string.
+if (Name[Name.size()-5] == 's')
+IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
+else
+IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
+Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2), CI->getArgOperand(4) };
+if (IsSubAdd)
+Ops[2] = Builder.CreateFNeg(Ops[2]);
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+{CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2), CI->getArgOperand(4)});
+} else {
+int NumElts = CI->getType()->getVectorNumElements();
+Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2) };
+Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
+Ops[0]->getType());
+Value *Odd = Builder.CreateCall(FMA, Ops);
+Ops[2] = Builder.CreateFNeg(Ops[2]);
+Value *Even = Builder.CreateCall(FMA, Ops);
+if (IsSubAdd)
+std::swap(Even, Odd);
+SmallVector<uint32_t, 32> Idxs(NumElts);
+for (int i = 0; i != NumElts; ++i)
+Idxs[i] = i + (i % 2) * NumElts;
+Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
+}
+Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
+IsMask3 ? CI->getArgOperand(2) :
+CI->getArgOperand(0);
+Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
+} else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") ||
+Name.startswith("avx512.maskz.pternlog."))) {
+bool ZeroMask = Name[11] == 'z';
+unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
+unsigned EltWidth = CI->getType()->getScalarSizeInBits();
+Intrinsic::ID IID;
+if (VecWidth == 128 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_pternlog_d_128;
+else if (VecWidth == 256 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_pternlog_d_256;
+else if (VecWidth == 512 && EltWidth == 32)
+IID = Intrinsic::x86_avx512_pternlog_d_512;
+else if (VecWidth == 128 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_pternlog_q_128;
+else if (VecWidth == 256 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_pternlog_q_256;
+else if (VecWidth == 512 && EltWidth == 64)
+IID = Intrinsic::x86_avx512_pternlog_q_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
+CI->getArgOperand(2), CI->getArgOperand(3) };
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
+Args);
+Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
+: CI->getArgOperand(0);
+Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
+} else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") ||
+Name.startswith("avx512.maskz.vpmadd52"))) {
+bool ZeroMask = Name[11] == 'z';
+bool High = Name[20] == 'h' || Name[21] == 'h';
+unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
+Intrinsic::ID IID;
+if (VecWidth == 128 && !High)
+IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
+else if (VecWidth == 256 && !High)
+IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
+else if (VecWidth == 512 && !High)
+IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
+else if (VecWidth == 128 && High)
+IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
+else if (VecWidth == 256 && High)
+IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
+else if (VecWidth == 512 && High)
+IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
+CI->getArgOperand(2) };
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
+Args);
+Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
+: CI->getArgOperand(0);
+Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
+} else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") ||
+Name.startswith("avx512.mask.vpermt2var.") ||
+Name.startswith("avx512.maskz.vpermt2var."))) {
+bool ZeroMask = Name[11] == 'z';
+bool IndexForm = Name[17] == 'i';
+Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
+} else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||
+Name.startswith("avx512.maskz.vpdpbusd.") ||
+Name.startswith("avx512.mask.vpdpbusds.") ||
+Name.startswith("avx512.maskz.vpdpbusds."))) {
+bool ZeroMask = Name[11] == 'z';
+bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
+unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
+Intrinsic::ID IID;
+if (VecWidth == 128 && !IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpbusd_128;
+else if (VecWidth == 256 && !IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpbusd_256;
+else if (VecWidth == 512 && !IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpbusd_512;
+else if (VecWidth == 128 && IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpbusds_128;
+else if (VecWidth == 256 && IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpbusds_256;
+else if (VecWidth == 512 && IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpbusds_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2)  };
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
+Args);
+Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
+: CI->getArgOperand(0);
+Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
+} else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") ||
+Name.startswith("avx512.maskz.vpdpwssd.") ||
+Name.startswith("avx512.mask.vpdpwssds.") ||
+Name.startswith("avx512.maskz.vpdpwssds."))) {
+bool ZeroMask = Name[11] == 'z';
+bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
+unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
+Intrinsic::ID IID;
+if (VecWidth == 128 && !IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpwssd_128;
+else if (VecWidth == 256 && !IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpwssd_256;
+else if (VecWidth == 512 && !IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpwssd_512;
+else if (VecWidth == 128 && IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpwssds_128;
+else if (VecWidth == 256 && IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpwssds_256;
+else if (VecWidth == 512 && IsSaturating)
+IID = Intrinsic::x86_avx512_vpdpwssds_512;
+else
+llvm_unreachable("Unexpected intrinsic");
+Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2)  };
+Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
+Args);
+Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
+: CI->getArgOperand(0);
+Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
+} else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
+Name == "addcarry.u32" || Name == "addcarry.u64" ||
+Name == "subborrow.u32" || Name == "subborrow.u64")) {
+Intrinsic::ID IID;
+if (Name[0] == 'a' && Name.back() == '2')
+IID = Intrinsic::x86_addcarry_32;
+else if (Name[0] == 'a' && Name.back() == '4')
+IID = Intrinsic::x86_addcarry_64;
+else if (Name[0] == 's' && Name.back() == '2')
+IID = Intrinsic::x86_subborrow_32;
+else if (Name[0] == 's' && Name.back() == '4')
+IID = Intrinsic::x86_subborrow_64;
+else
+llvm_unreachable("Unexpected intrinsic");
+// Make a call with 3 operands.
+Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
+CI->getArgOperand(2)};
+Value *NewCall = Builder.CreateCall(
+Intrinsic::getDeclaration(CI->getModule(), IID),
+Args);
+// Extract the second result and store it.
+Value *Data = Builder.CreateExtractValue(NewCall, 1);
+// Cast the pointer to the right type.
+Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
+llvm::PointerType::getUnqual(Data->getType()));
+Builder.CreateAlignedStore(Data, Ptr, 1);
+// Replace the original call result with the first result of the new call.
+Value *CF = Builder.CreateExtractValue(NewCall, 0);
+CI->replaceAllUsesWith(CF);
+Rep = nullptr;
+} else if (IsX86 && Name.startswith("avx512.mask.") &&
+upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
+// Rep will be updated by the call in the condition.
 } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
 Value *Arg = CI->getArgOperand(0);
 Value *Neg = Builder.CreateNeg(Arg, "neg");
 Value *Cmp = Builder.CreateICmpSGE(
 Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
+} else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||
+Name.startswith("atomic.load.add.f64.p"))) {
+Value *Ptr = CI->getArgOperand(0);
+Value *Val = CI->getArgOperand(1);
+Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val,
+AtomicOrdering::SequentiallyConsistent);
 } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
 Name == "max.ui" || Name == "max.ull")) {
 Value *Arg0 = CI->getArgOperand(0);
 Value *Arg1 = CI->getArgOperand(1);
 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
 switch (NewFn->getIntrinsicID()) {
 default: {
 DefaultCase();
 return;
 }
+case Intrinsic::experimental_vector_reduce_v2_fmul: {
+SmallVector<Value *, 2> Args;
+if (CI->isFast())
+Args.push_back(ConstantFP::get(CI->getOperand(0)->getType(), 1.0));
+else
+Args.push_back(CI->getOperand(0));
+Args.push_back(CI->getOperand(1));
+NewCall = Builder.CreateCall(NewFn, Args);
+cast<Instruction>(NewCall)->copyFastMathFlags(CI);
+break;
+}
+case Intrinsic::experimental_vector_reduce_v2_fadd: {
+SmallVector<Value *, 2> Args;
+if (CI->isFast())
+Args.push_back(Constant::getNullValue(CI->getOperand(0)->getType()));
+else
+Args.push_back(CI->getOperand(0));
+Args.push_back(CI->getOperand(1));
+NewCall = Builder.CreateCall(NewFn, Args);
+cast<Instruction>(NewCall)->copyFastMathFlags(CI);
+break;
+}
 case Intrinsic::arm_neon_vld1:
 case Intrinsic::arm_neon_vld2:
 case Intrinsic::arm_neon_vld3:
 case Intrinsic::arm_neon_vld4:
 case Intrinsic::arm_neon_vld2lane:
 case Intrinsic::objectsize: {
 Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
 ? Builder.getFalse()
 : CI->getArgOperand(2);
+Value *Dynamic =
+CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
 NewCall = Builder.CreateCall(
-NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize});
+NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
 break;
 }
 case Intrinsic::ctpop:
 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
 break;
 }
+case Intrinsic::x86_rdtscp: {
+// This used to take 1 arguments. If we have no arguments, it is already
+// upgraded.
+if (CI->getNumOperands() == 0)
+return;
+NewCall = Builder.CreateCall(NewFn);
+// Extract the second result and store it.
+Value *Data = Builder.CreateExtractValue(NewCall, 1);
+// Cast the pointer to the right type.
+Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
+llvm::PointerType::getUnqual(Data->getType()));
+Builder.CreateAlignedStore(Data, Ptr, 1);
+// Replace the original call result with the first result of the new call.
+Value *TSC = Builder.CreateExtractValue(NewCall, 0);
+std::string Name = CI->getName();
+if (!Name.empty()) {
+CI->setName(Name + ".old");
+NewCall->setName(Name);
+}
+CI->replaceAllUsesWith(TSC);
+CI->eraseFromParent();
+return;
+}
 case Intrinsic::x86_sse41_insertps:
 case Intrinsic::x86_sse41_dppd:
 case Intrinsic::x86_sse41_dpps:
 case Intrinsic::x86_sse41_mpsadbw:
 case Intrinsic::x86_avx_dp_ps_256:
 // Replace the last argument with a trunc.
 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
 NewCall = Builder.CreateCall(NewFn, Args);
 break;
-}
-case Intrinsic::x86_avx512_mask_cmp_pd_128:
-case Intrinsic::x86_avx512_mask_cmp_pd_256:
-case Intrinsic::x86_avx512_mask_cmp_pd_512:
-case Intrinsic::x86_avx512_mask_cmp_ps_128:
-case Intrinsic::x86_avx512_mask_cmp_ps_256:
-case Intrinsic::x86_avx512_mask_cmp_ps_512: {
-SmallVector<Value *, 4> Args;
-Args.push_back(CI->getArgOperand(0));
-Args.push_back(CI->getArgOperand(1));
-Args.push_back(CI->getArgOperand(2));
-if (CI->getNumArgOperands() == 5)
-Args.push_back(CI->getArgOperand(4));
-NewCall = Builder.CreateCall(NewFn, Args);
-unsigned NumElts = Args[0]->getType()->getVectorNumElements();
-Value *Res = ApplyX86MaskOn1BitsVec(Builder, NewCall, CI->getArgOperand(3),
-NumElts);
-std::string Name = CI->getName();
-if (!Name.empty()) {
-CI->setName(Name + ".old");
-NewCall->setName(Name);
-}
-CI->replaceAllUsesWith(Res);
-CI->eraseFromParent();
-return;
 }
 case Intrinsic::thread_pointer: {
 NewCall = Builder.CreateCall(NewFn, {});
 break;
 // Diagnose a version mismatch.
 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
 M.getContext().diagnose(DiagVersion);
 }
 return Modified;
+}
+/// This checks for objc retain release marker which should be upgraded. It
+/// returns true if module is modified.
+static bool UpgradeRetainReleaseMarker(Module &M) {
+bool Changed = false;
+const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
+NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
+if (ModRetainReleaseMarker) {
+MDNode *Op = ModRetainReleaseMarker->getOperand(0);
+if (Op) {
+MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
+if (ID) {
+SmallVector<StringRef, 4> ValueComp;
+ID->getString().split(ValueComp, "#");
+if (ValueComp.size() == 2) {
+std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
+ID = MDString::get(M.getContext(), NewValue);
+}
+M.addModuleFlag(Module::Error, MarkerKey, ID);
+M.eraseNamedMetadata(ModRetainReleaseMarker);
+Changed = true;
+}
+}
+}
+return Changed;
+}
+void llvm::UpgradeARCRuntime(Module &M) {
+// This lambda converts normal function calls to ARC runtime functions to
+// intrinsic calls.
+auto UpgradeToIntrinsic = [&](const char *OldFunc,
+llvm::Intrinsic::ID IntrinsicFunc) {
+Function *Fn = M.getFunction(OldFunc);
+if (!Fn)
+return;
+Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
+for (auto I = Fn->user_begin(), E = Fn->user_end(); I != E;) {
+CallInst *CI = dyn_cast<CallInst>(*I++);
+if (!CI || CI->getCalledFunction() != Fn)
+continue;
+IRBuilder<> Builder(CI->getParent(), CI->getIterator());
+FunctionType *NewFuncTy = NewFn->getFunctionType();
+SmallVector<Value *, 2> Args;
+for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) {
+Value *Arg = CI->getArgOperand(I);
+// Bitcast argument to the parameter type of the new function if it's
+// not a variadic argument.
+if (I < NewFuncTy->getNumParams())
+Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
+Args.push_back(Arg);
+}
+// Create a call instruction that calls the new function.
+CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
+NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
+NewCall->setName(CI->getName());
+// Bitcast the return value back to the type of the old call.
+Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
+if (!CI->use_empty())
+CI->replaceAllUsesWith(NewRetVal);
+CI->eraseFromParent();
+}
+if (Fn->use_empty())
+Fn->eraseFromParent();
+};
+// Unconditionally convert a call to "clang.arc.use" to a call to
+// "llvm.objc.clang.arc.use".
+UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
+// Upgrade the retain release marker. If there is no need to upgrade
+// the marker, that means either the module is already new enough to contain
+// new intrinsics or it is not ARC. There is no need to upgrade runtime call.
+if (!UpgradeRetainReleaseMarker(M))
+return;
+std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
+{"objc_autorelease", llvm::Intrinsic::objc_autorelease},
+{"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
+{"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
+{"objc_autoreleaseReturnValue",
+llvm::Intrinsic::objc_autoreleaseReturnValue},
+{"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
+{"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
+{"objc_initWeak", llvm::Intrinsic::objc_initWeak},
+{"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
+{"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
+{"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
+{"objc_release", llvm::Intrinsic::objc_release},
+{"objc_retain", llvm::Intrinsic::objc_retain},
+{"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
+{"objc_retainAutoreleaseReturnValue",
+llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
+{"objc_retainAutoreleasedReturnValue",
+llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
+{"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
+{"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
+{"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
+{"objc_unsafeClaimAutoreleasedReturnValue",
+llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
+{"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
+{"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
+{"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
+{"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
+{"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
+{"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
+{"objc_arc_annotation_topdown_bbstart",
+llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
+{"objc_arc_annotation_topdown_bbend",
+llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
+{"objc_arc_annotation_bottomup_bbstart",
+llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
+{"objc_arc_annotation_bottomup_bbend",
+llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
+for (auto &I : RuntimeFuncs)
+UpgradeToIntrinsic(I.first, I.second);
 }
 bool llvm::UpgradeModuleFlags(Module &M) {
 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
 if (!ModFlags)

Mercurial > hg > CbC > CbC_llvm

comparison lib/IR/AutoUpgrade.cpp @ 148:63bd29f05246