Mercurial > hg > CbC > CbC_gcc

comparison gcc/omp-simd-clone.c @ 111:04ced10e8804

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
gcc 7
author kono
date Fri, 27 Oct 2017 22:46:09 +0900
parents
children 84e7813d76e9
comparison
equal deleted inserted replaced
68:561a7518be6b 111:04ced10e8804
1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2017 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-prop.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
51 #include "stringpool.h"
52 #include "attribs.h"
53
54 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
55 of arguments to reserve space for. */
56
57 static struct cgraph_simd_clone *
58 simd_clone_struct_alloc (int nargs)
59 {
60 struct cgraph_simd_clone *clone_info;
61 size_t len = (sizeof (struct cgraph_simd_clone)
62 + nargs * sizeof (struct cgraph_simd_clone_arg));
63 clone_info = (struct cgraph_simd_clone *)
64 ggc_internal_cleared_alloc (len);
65 return clone_info;
66 }
67
68 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
69
70 static inline void
71 simd_clone_struct_copy (struct cgraph_simd_clone *to,
72 struct cgraph_simd_clone *from)
73 {
74 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
75 + ((from->nargs - from->inbranch)
76 * sizeof (struct cgraph_simd_clone_arg))));
77 }
78
79 /* Return vector of parameter types of function FNDECL. This uses
80 TYPE_ARG_TYPES if available, otherwise falls back to types of
81 DECL_ARGUMENTS types. */
82
83 static vec<tree>
84 simd_clone_vector_of_formal_parm_types (tree fndecl)
85 {
86 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
87 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
88 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
89 unsigned int i;
90 tree arg;
91 FOR_EACH_VEC_ELT (args, i, arg)
92 args[i] = TREE_TYPE (args[i]);
93 return args;
94 }
95
96 /* Given a simd function in NODE, extract the simd specific
97 information from the OMP clauses passed in CLAUSES, and return
98 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
99 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
100 otherwise set to FALSE. */
101
102 static struct cgraph_simd_clone *
103 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
104 bool *inbranch_specified)
105 {
106 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
107 tree t;
108 int n;
109 *inbranch_specified = false;
110
111 n = args.length ();
112 if (n > 0 && args.last () == void_type_node)
113 n--;
114
115 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
116 be cloned have a distinctive artificial label in addition to "omp
117 declare simd". */
118 bool cilk_clone
119 = (flag_cilkplus
120 && lookup_attribute ("cilk simd function",
121 DECL_ATTRIBUTES (node->decl)));
122
123 /* Allocate one more than needed just in case this is an in-branch
124 clone which will require a mask argument. */
125 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
126 clone_info->nargs = n;
127 clone_info->cilk_elemental = cilk_clone;
128
129 if (!clauses)
130 goto out;
131
132 clauses = TREE_VALUE (clauses);
133 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
134 goto out;
135
136 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
137 {
138 switch (OMP_CLAUSE_CODE (t))
139 {
140 case OMP_CLAUSE_INBRANCH:
141 clone_info->inbranch = 1;
142 *inbranch_specified = true;
143 break;
144 case OMP_CLAUSE_NOTINBRANCH:
145 clone_info->inbranch = 0;
146 *inbranch_specified = true;
147 break;
148 case OMP_CLAUSE_SIMDLEN:
149 clone_info->simdlen
150 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
151 break;
152 case OMP_CLAUSE_LINEAR:
153 {
154 tree decl = OMP_CLAUSE_DECL (t);
155 tree step = OMP_CLAUSE_LINEAR_STEP (t);
156 int argno = TREE_INT_CST_LOW (decl);
157 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
158 {
159 enum cgraph_simd_clone_arg_type arg_type;
160 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
161 switch (OMP_CLAUSE_LINEAR_KIND (t))
162 {
163 case OMP_CLAUSE_LINEAR_REF:
164 arg_type
165 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
166 break;
167 case OMP_CLAUSE_LINEAR_UVAL:
168 arg_type
169 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
170 break;
171 case OMP_CLAUSE_LINEAR_VAL:
172 case OMP_CLAUSE_LINEAR_DEFAULT:
173 arg_type
174 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
175 break;
176 default:
177 gcc_unreachable ();
178 }
179 else
180 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
181 clone_info->args[argno].arg_type = arg_type;
182 clone_info->args[argno].linear_step = tree_to_shwi (step);
183 gcc_assert (clone_info->args[argno].linear_step >= 0
184 && clone_info->args[argno].linear_step < n);
185 }
186 else
187 {
188 if (POINTER_TYPE_P (args[argno]))
189 step = fold_convert (ssizetype, step);
190 if (!tree_fits_shwi_p (step))
191 {
192 warning_at (OMP_CLAUSE_LOCATION (t), 0,
193 "ignoring large linear step");
194 args.release ();
195 return NULL;
196 }
197 else if (integer_zerop (step))
198 {
199 warning_at (OMP_CLAUSE_LOCATION (t), 0,
200 "ignoring zero linear step");
201 args.release ();
202 return NULL;
203 }
204 else
205 {
206 enum cgraph_simd_clone_arg_type arg_type;
207 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
208 switch (OMP_CLAUSE_LINEAR_KIND (t))
209 {
210 case OMP_CLAUSE_LINEAR_REF:
211 arg_type
212 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
213 break;
214 case OMP_CLAUSE_LINEAR_UVAL:
215 arg_type
216 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
217 break;
218 case OMP_CLAUSE_LINEAR_VAL:
219 case OMP_CLAUSE_LINEAR_DEFAULT:
220 arg_type
221 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
222 break;
223 default:
224 gcc_unreachable ();
225 }
226 else
227 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
228 clone_info->args[argno].arg_type = arg_type;
229 clone_info->args[argno].linear_step = tree_to_shwi (step);
230 }
231 }
232 break;
233 }
234 case OMP_CLAUSE_UNIFORM:
235 {
236 tree decl = OMP_CLAUSE_DECL (t);
237 int argno = tree_to_uhwi (decl);
238 clone_info->args[argno].arg_type
239 = SIMD_CLONE_ARG_TYPE_UNIFORM;
240 break;
241 }
242 case OMP_CLAUSE_ALIGNED:
243 {
244 tree decl = OMP_CLAUSE_DECL (t);
245 int argno = tree_to_uhwi (decl);
246 clone_info->args[argno].alignment
247 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
248 break;
249 }
250 default:
251 break;
252 }
253 }
254
255 out:
256 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
257 {
258 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
259 "ignoring %<#pragma omp declare simd%> on function "
260 "with %<_Atomic%> qualified return type");
261 args.release ();
262 return NULL;
263 }
264
265 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
266 if (TYPE_ATOMIC (args[argno])
267 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
268 {
269 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
270 "ignoring %<#pragma omp declare simd%> on function "
271 "with %<_Atomic%> qualified non-%<uniform%> argument");
272 args.release ();
273 return NULL;
274 }
275
276 args.release ();
277 return clone_info;
278 }
279
280 /* Given a SIMD clone in NODE, calculate the characteristic data
281 type and return the coresponding type. The characteristic data
282 type is computed as described in the Intel Vector ABI. */
283
284 static tree
285 simd_clone_compute_base_data_type (struct cgraph_node *node,
286 struct cgraph_simd_clone *clone_info)
287 {
288 tree type = integer_type_node;
289 tree fndecl = node->decl;
290
291 /* a) For non-void function, the characteristic data type is the
292 return type. */
293 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
294 type = TREE_TYPE (TREE_TYPE (fndecl));
295
296 /* b) If the function has any non-uniform, non-linear parameters,
297 then the characteristic data type is the type of the first
298 such parameter. */
299 else
300 {
301 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
302 for (unsigned int i = 0; i < clone_info->nargs; ++i)
303 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
304 {
305 type = map[i];
306 break;
307 }
308 map.release ();
309 }
310
311 /* c) If the characteristic data type determined by a) or b) above
312 is struct, union, or class type which is pass-by-value (except
313 for the type that maps to the built-in complex data type), the
314 characteristic data type is int. */
315 if (RECORD_OR_UNION_TYPE_P (type)
316 && !aggregate_value_p (type, NULL)
317 && TREE_CODE (type) != COMPLEX_TYPE)
318 return integer_type_node;
319
320 /* d) If none of the above three classes is applicable, the
321 characteristic data type is int. */
322
323 return type;
324
325 /* e) For Intel Xeon Phi native and offload compilation, if the
326 resulting characteristic data type is 8-bit or 16-bit integer
327 data type, the characteristic data type is int. */
328 /* Well, we don't handle Xeon Phi yet. */
329 }
330
331 static tree
332 simd_clone_mangle (struct cgraph_node *node,
333 struct cgraph_simd_clone *clone_info)
334 {
335 char vecsize_mangle = clone_info->vecsize_mangle;
336 char mask = clone_info->inbranch ? 'M' : 'N';
337 unsigned int simdlen = clone_info->simdlen;
338 unsigned int n;
339 pretty_printer pp;
340
341 gcc_assert (vecsize_mangle && simdlen);
342
343 pp_string (&pp, "_ZGV");
344 pp_character (&pp, vecsize_mangle);
345 pp_character (&pp, mask);
346 pp_decimal_int (&pp, simdlen);
347
348 for (n = 0; n < clone_info->nargs; ++n)
349 {
350 struct cgraph_simd_clone_arg arg = clone_info->args[n];
351
352 switch (arg.arg_type)
353 {
354 case SIMD_CLONE_ARG_TYPE_UNIFORM:
355 pp_character (&pp, 'u');
356 break;
357 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
358 pp_character (&pp, 'l');
359 goto mangle_linear;
360 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
361 pp_character (&pp, 'R');
362 goto mangle_linear;
363 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
364 pp_character (&pp, 'L');
365 goto mangle_linear;
366 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
367 pp_character (&pp, 'U');
368 goto mangle_linear;
369 mangle_linear:
370 gcc_assert (arg.linear_step != 0);
371 if (arg.linear_step > 1)
372 pp_unsigned_wide_integer (&pp, arg.linear_step);
373 else if (arg.linear_step < 0)
374 {
375 pp_character (&pp, 'n');
376 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
377 arg.linear_step));
378 }
379 break;
380 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
381 pp_string (&pp, "ls");
382 pp_unsigned_wide_integer (&pp, arg.linear_step);
383 break;
384 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
385 pp_string (&pp, "Rs");
386 pp_unsigned_wide_integer (&pp, arg.linear_step);
387 break;
388 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
389 pp_string (&pp, "Ls");
390 pp_unsigned_wide_integer (&pp, arg.linear_step);
391 break;
392 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
393 pp_string (&pp, "Us");
394 pp_unsigned_wide_integer (&pp, arg.linear_step);
395 break;
396 default:
397 pp_character (&pp, 'v');
398 }
399 if (arg.alignment)
400 {
401 pp_character (&pp, 'a');
402 pp_decimal_int (&pp, arg.alignment);
403 }
404 }
405
406 pp_underscore (&pp);
407 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
408 if (*str == '*')
409 ++str;
410 pp_string (&pp, str);
411 str = pp_formatted_text (&pp);
412
413 /* If there already is a SIMD clone with the same mangled name, don't
414 add another one. This can happen e.g. for
415 #pragma omp declare simd
416 #pragma omp declare simd simdlen(8)
417 int foo (int, int);
418 if the simdlen is assumed to be 8 for the first one, etc. */
419 for (struct cgraph_node *clone = node->simd_clones; clone;
420 clone = clone->simdclone->next_clone)
421 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
422 return NULL_TREE;
423
424 return get_identifier (str);
425 }
426
427 /* Create a simd clone of OLD_NODE and return it. */
428
429 static struct cgraph_node *
430 simd_clone_create (struct cgraph_node *old_node)
431 {
432 struct cgraph_node *new_node;
433 if (old_node->definition)
434 {
435 if (!old_node->has_gimple_body_p ())
436 return NULL;
437 old_node->get_body ();
438 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
439 false, NULL, NULL,
440 "simdclone");
441 }
442 else
443 {
444 tree old_decl = old_node->decl;
445 tree new_decl = copy_node (old_node->decl);
446 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
447 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
448 SET_DECL_RTL (new_decl, NULL);
449 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
450 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
451 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
452 if (old_node->in_other_partition)
453 new_node->in_other_partition = 1;
454 }
455 if (new_node == NULL)
456 return new_node;
457
458 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
459 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
460 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
461 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
462 DECL_VISIBILITY_SPECIFIED (new_node->decl)
463 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
464 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
465 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
466 if (DECL_ONE_ONLY (old_node->decl))
467 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
468
469 /* The method cgraph_version_clone_with_body () will force the new
470 symbol local. Undo this, and inherit external visibility from
471 the old node. */
472 new_node->local.local = old_node->local.local;
473 new_node->externally_visible = old_node->externally_visible;
474
475 return new_node;
476 }
477
478 /* Adjust the return type of the given function to its appropriate
479 vector counterpart. Returns a simd array to be used throughout the
480 function as a return value. */
481
482 static tree
483 simd_clone_adjust_return_type (struct cgraph_node *node)
484 {
485 tree fndecl = node->decl;
486 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
487 unsigned int veclen;
488 tree t;
489
490 /* Adjust the function return type. */
491 if (orig_rettype == void_type_node)
492 return NULL_TREE;
493 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
494 t = TREE_TYPE (TREE_TYPE (fndecl));
495 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
496 veclen = node->simdclone->vecsize_int;
497 else
498 veclen = node->simdclone->vecsize_float;
499 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t));
500 if (veclen > node->simdclone->simdlen)
501 veclen = node->simdclone->simdlen;
502 if (POINTER_TYPE_P (t))
503 t = pointer_sized_int_node;
504 if (veclen == node->simdclone->simdlen)
505 t = build_vector_type (t, node->simdclone->simdlen);
506 else
507 {
508 t = build_vector_type (t, veclen);
509 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
510 }
511 TREE_TYPE (TREE_TYPE (fndecl)) = t;
512 if (!node->definition)
513 return NULL_TREE;
514
515 t = DECL_RESULT (fndecl);
516 /* Adjust the DECL_RESULT. */
517 gcc_assert (TREE_TYPE (t) != void_type_node);
518 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
519 relayout_decl (t);
520
521 tree atype = build_array_type_nelts (orig_rettype,
522 node->simdclone->simdlen);
523 if (veclen != node->simdclone->simdlen)
524 return build1 (VIEW_CONVERT_EXPR, atype, t);
525
526 /* Set up a SIMD array to use as the return value. */
527 tree retval = create_tmp_var_raw (atype, "retval");
528 gimple_add_tmp_var (retval);
529 return retval;
530 }
531
532 /* Each vector argument has a corresponding array to be used locally
533 as part of the eventual loop. Create such temporary array and
534 return it.
535
536 PREFIX is the prefix to be used for the temporary.
537
538 TYPE is the inner element type.
539
540 SIMDLEN is the number of elements. */
541
542 static tree
543 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
544 {
545 tree atype = build_array_type_nelts (type, simdlen);
546 tree avar = create_tmp_var_raw (atype, prefix);
547 gimple_add_tmp_var (avar);
548 return avar;
549 }
550
551 /* Modify the function argument types to their corresponding vector
552 counterparts if appropriate. Also, create one array for each simd
553 argument to be used locally when using the function arguments as
554 part of the loop.
555
556 NODE is the function whose arguments are to be adjusted.
557
558 Returns an adjustment vector that will be filled describing how the
559 argument types will be adjusted. */
560
561 static ipa_parm_adjustment_vec
562 simd_clone_adjust_argument_types (struct cgraph_node *node)
563 {
564 vec<tree> args;
565 ipa_parm_adjustment_vec adjustments;
566
567 if (node->definition)
568 args = ipa_get_vector_of_formal_parms (node->decl);
569 else
570 args = simd_clone_vector_of_formal_parm_types (node->decl);
571 adjustments.create (args.length ());
572 unsigned i, j, veclen;
573 struct ipa_parm_adjustment adj;
574 struct cgraph_simd_clone *sc = node->simdclone;
575
576 for (i = 0; i < sc->nargs; ++i)
577 {
578 memset (&adj, 0, sizeof (adj));
579 tree parm = args[i];
580 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
581 adj.base_index = i;
582 adj.base = parm;
583
584 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
585 sc->args[i].orig_type = parm_type;
586
587 switch (sc->args[i].arg_type)
588 {
589 default:
590 /* No adjustment necessary for scalar arguments. */
591 adj.op = IPA_PARM_OP_COPY;
592 break;
593 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
594 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
595 if (node->definition)
596 sc->args[i].simd_array
597 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
598 TREE_TYPE (parm_type),
599 sc->simdlen);
600 adj.op = IPA_PARM_OP_COPY;
601 break;
602 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
603 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
604 case SIMD_CLONE_ARG_TYPE_VECTOR:
605 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
606 veclen = sc->vecsize_int;
607 else
608 veclen = sc->vecsize_float;
609 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type));
610 if (veclen > sc->simdlen)
611 veclen = sc->simdlen;
612 adj.arg_prefix = "simd";
613 if (POINTER_TYPE_P (parm_type))
614 adj.type = build_vector_type (pointer_sized_int_node, veclen);
615 else
616 adj.type = build_vector_type (parm_type, veclen);
617 sc->args[i].vector_type = adj.type;
618 for (j = veclen; j < sc->simdlen; j += veclen)
619 {
620 adjustments.safe_push (adj);
621 if (j == veclen)
622 {
623 memset (&adj, 0, sizeof (adj));
624 adj.op = IPA_PARM_OP_NEW;
625 adj.arg_prefix = "simd";
626 adj.base_index = i;
627 adj.type = sc->args[i].vector_type;
628 }
629 }
630
631 if (node->definition)
632 sc->args[i].simd_array
633 = create_tmp_simd_array (DECL_NAME (parm)
634 ? IDENTIFIER_POINTER (DECL_NAME (parm))
635 : NULL, parm_type, sc->simdlen);
636 }
637 adjustments.safe_push (adj);
638 }
639
640 if (sc->inbranch)
641 {
642 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
643
644 memset (&adj, 0, sizeof (adj));
645 adj.op = IPA_PARM_OP_NEW;
646 adj.arg_prefix = "mask";
647
648 adj.base_index = i;
649 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
650 veclen = sc->vecsize_int;
651 else
652 veclen = sc->vecsize_float;
653 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type));
654 if (veclen > sc->simdlen)
655 veclen = sc->simdlen;
656 if (sc->mask_mode != VOIDmode)
657 adj.type
658 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
659 else if (POINTER_TYPE_P (base_type))
660 adj.type = build_vector_type (pointer_sized_int_node, veclen);
661 else
662 adj.type = build_vector_type (base_type, veclen);
663 adjustments.safe_push (adj);
664
665 for (j = veclen; j < sc->simdlen; j += veclen)
666 adjustments.safe_push (adj);
667
668 /* We have previously allocated one extra entry for the mask. Use
669 it and fill it. */
670 sc->nargs++;
671 if (sc->mask_mode != VOIDmode)
672 base_type = boolean_type_node;
673 if (node->definition)
674 {
675 sc->args[i].orig_arg
676 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
677 if (sc->mask_mode == VOIDmode)
678 sc->args[i].simd_array
679 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
680 else if (veclen < sc->simdlen)
681 sc->args[i].simd_array
682 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
683 else
684 sc->args[i].simd_array = NULL_TREE;
685 }
686 sc->args[i].orig_type = base_type;
687 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
688 }
689
690 if (node->definition)
691 ipa_modify_formal_parameters (node->decl, adjustments);
692 else
693 {
694 tree new_arg_types = NULL_TREE, new_reversed;
695 bool last_parm_void = false;
696 if (args.length () > 0 && args.last () == void_type_node)
697 last_parm_void = true;
698
699 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
700 j = adjustments.length ();
701 for (i = 0; i < j; i++)
702 {
703 struct ipa_parm_adjustment *adj = &adjustments[i];
704 tree ptype;
705 if (adj->op == IPA_PARM_OP_COPY)
706 ptype = args[adj->base_index];
707 else
708 ptype = adj->type;
709 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
710 }
711 new_reversed = nreverse (new_arg_types);
712 if (last_parm_void)
713 {
714 if (new_reversed)
715 TREE_CHAIN (new_arg_types) = void_list_node;
716 else
717 new_reversed = void_list_node;
718 }
719
720 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
721 TYPE_ARG_TYPES (new_type) = new_reversed;
722 TREE_TYPE (node->decl) = new_type;
723
724 adjustments.release ();
725 }
726 args.release ();
727 return adjustments;
728 }
729
730 /* Initialize and copy the function arguments in NODE to their
731 corresponding local simd arrays. Returns a fresh gimple_seq with
732 the instruction sequence generated. */
733
734 static gimple_seq
735 simd_clone_init_simd_arrays (struct cgraph_node *node,
736 ipa_parm_adjustment_vec adjustments)
737 {
738 gimple_seq seq = NULL;
739 unsigned i = 0, j = 0, k;
740
741 for (tree arg = DECL_ARGUMENTS (node->decl);
742 arg;
743 arg = DECL_CHAIN (arg), i++, j++)
744 {
745 if (adjustments[j].op == IPA_PARM_OP_COPY
746 || POINTER_TYPE_P (TREE_TYPE (arg)))
747 continue;
748
749 node->simdclone->args[i].vector_arg = arg;
750
751 tree array = node->simdclone->args[i].simd_array;
752 if (node->simdclone->mask_mode != VOIDmode
753 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
754 {
755 if (array == NULL_TREE)
756 continue;
757 unsigned int l
758 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
759 for (k = 0; k <= l; k++)
760 {
761 if (k)
762 {
763 arg = DECL_CHAIN (arg);
764 j++;
765 }
766 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
767 array, size_int (k), NULL, NULL);
768 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
769 gimplify_and_add (t, &seq);
770 }
771 continue;
772 }
773 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
774 {
775 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
776 tree ptr = build_fold_addr_expr (array);
777 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
778 build_int_cst (ptype, 0));
779 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
780 gimplify_and_add (t, &seq);
781 }
782 else
783 {
784 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
785 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
786 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
787 {
788 tree ptr = build_fold_addr_expr (array);
789 int elemsize;
790 if (k)
791 {
792 arg = DECL_CHAIN (arg);
793 j++;
794 }
795 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
796 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
797 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
798 build_int_cst (ptype, k * elemsize));
799 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
800 gimplify_and_add (t, &seq);
801 }
802 }
803 }
804 return seq;
805 }
806
807 /* Callback info for ipa_simd_modify_stmt_ops below. */
808
809 struct modify_stmt_info {
810 ipa_parm_adjustment_vec adjustments;
811 gimple *stmt;
812 /* True if the parent statement was modified by
813 ipa_simd_modify_stmt_ops. */
814 bool modified;
815 };
816
817 /* Callback for walk_gimple_op.
818
819 Adjust operands from a given statement as specified in the
820 adjustments vector in the callback data. */
821
822 static tree
823 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
824 {
825 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
826 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
827 tree *orig_tp = tp;
828 if (TREE_CODE (*tp) == ADDR_EXPR)
829 tp = &TREE_OPERAND (*tp, 0);
830 struct ipa_parm_adjustment *cand = NULL;
831 if (TREE_CODE (*tp) == PARM_DECL)
832 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
833 else
834 {
835 if (TYPE_P (*tp))
836 *walk_subtrees = 0;
837 }
838
839 tree repl = NULL_TREE;
840 if (cand)
841 repl = unshare_expr (cand->new_decl);
842 else
843 {
844 if (tp != orig_tp)
845 {
846 *walk_subtrees = 0;
847 bool modified = info->modified;
848 info->modified = false;
849 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
850 if (!info->modified)
851 {
852 info->modified = modified;
853 return NULL_TREE;
854 }
855 info->modified = modified;
856 repl = *tp;
857 }
858 else
859 return NULL_TREE;
860 }
861
862 if (tp != orig_tp)
863 {
864 repl = build_fold_addr_expr (repl);
865 gimple *stmt;
866 if (is_gimple_debug (info->stmt))
867 {
868 tree vexpr = make_node (DEBUG_EXPR_DECL);
869 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
870 DECL_ARTIFICIAL (vexpr) = 1;
871 TREE_TYPE (vexpr) = TREE_TYPE (repl);
872 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
873 repl = vexpr;
874 }
875 else
876 {
877 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
878 repl = gimple_assign_lhs (stmt);
879 }
880 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
881 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
882 *orig_tp = repl;
883 }
884 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
885 {
886 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
887 *tp = vce;
888 }
889 else
890 *tp = repl;
891
892 info->modified = true;
893 return NULL_TREE;
894 }
895
896 /* Traverse the function body and perform all modifications as
897 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
898 modified such that the replacement/reduction value will now be an
899 offset into the corresponding simd_array.
900
901 This function will replace all function argument uses with their
902 corresponding simd array elements, and ajust the return values
903 accordingly. */
904
905 static void
906 ipa_simd_modify_function_body (struct cgraph_node *node,
907 ipa_parm_adjustment_vec adjustments,
908 tree retval_array, tree iter)
909 {
910 basic_block bb;
911 unsigned int i, j, l;
912
913 /* Re-use the adjustments array, but this time use it to replace
914 every function argument use to an offset into the corresponding
915 simd_array. */
916 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
917 {
918 if (!node->simdclone->args[i].vector_arg)
919 continue;
920
921 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
922 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
923 adjustments[j].new_decl
924 = build4 (ARRAY_REF,
925 basetype,
926 node->simdclone->args[i].simd_array,
927 iter,
928 NULL_TREE, NULL_TREE);
929 if (adjustments[j].op == IPA_PARM_OP_NONE
930 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
931 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
932 }
933
934 l = adjustments.length ();
935 tree name;
936
937 FOR_EACH_SSA_NAME (i, name, cfun)
938 {
939 if (SSA_NAME_VAR (name)
940 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
941 {
942 for (j = 0; j < l; j++)
943 if (SSA_NAME_VAR (name) == adjustments[j].base
944 && adjustments[j].new_decl)
945 {
946 tree base_var;
947 if (adjustments[j].new_ssa_base == NULL_TREE)
948 {
949 base_var
950 = copy_var_decl (adjustments[j].base,
951 DECL_NAME (adjustments[j].base),
952 TREE_TYPE (adjustments[j].base));
953 adjustments[j].new_ssa_base = base_var;
954 }
955 else
956 base_var = adjustments[j].new_ssa_base;
957 if (SSA_NAME_IS_DEFAULT_DEF (name))
958 {
959 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
960 gimple_stmt_iterator gsi = gsi_after_labels (bb);
961 tree new_decl = unshare_expr (adjustments[j].new_decl);
962 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
963 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
964 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
965 gimple *stmt = gimple_build_assign (name, new_decl);
966 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
967 }
968 else
969 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
970 }
971 }
972 }
973
974 struct modify_stmt_info info;
975 info.adjustments = adjustments;
976
977 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
978 {
979 gimple_stmt_iterator gsi;
980
981 gsi = gsi_start_bb (bb);
982 while (!gsi_end_p (gsi))
983 {
984 gimple *stmt = gsi_stmt (gsi);
985 info.stmt = stmt;
986 struct walk_stmt_info wi;
987
988 memset (&wi, 0, sizeof (wi));
989 info.modified = false;
990 wi.info = &info;
991 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
992
993 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
994 {
995 tree retval = gimple_return_retval (return_stmt);
996 if (!retval)
997 {
998 gsi_remove (&gsi, true);
999 continue;
1000 }
1001
1002 /* Replace `return foo' with `retval_array[iter] = foo'. */
1003 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1004 retval_array, iter, NULL, NULL);
1005 stmt = gimple_build_assign (ref, retval);
1006 gsi_replace (&gsi, stmt, true);
1007 info.modified = true;
1008 }
1009
1010 if (info.modified)
1011 {
1012 update_stmt (stmt);
1013 if (maybe_clean_eh_stmt (stmt))
1014 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1015 }
1016 gsi_next (&gsi);
1017 }
1018 }
1019 }
1020
1021 /* Helper function of simd_clone_adjust, return linear step addend
1022 of Ith argument. */
1023
1024 static tree
1025 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1026 tree addtype, basic_block entry_bb)
1027 {
1028 tree ptype = NULL_TREE;
1029 switch (node->simdclone->args[i].arg_type)
1030 {
1031 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1032 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1033 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1034 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1035 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1036 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1037 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1038 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1039 break;
1040 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1041 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1042 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1043 break;
1044 default:
1045 gcc_unreachable ();
1046 }
1047
1048 unsigned int idx = node->simdclone->args[i].linear_step;
1049 tree arg = node->simdclone->args[idx].orig_arg;
1050 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1051 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1052 gimple *g;
1053 tree ret;
1054 if (is_gimple_reg (arg))
1055 ret = get_or_create_ssa_default_def (cfun, arg);
1056 else
1057 {
1058 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1059 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1060 ret = gimple_assign_lhs (g);
1061 }
1062 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1063 {
1064 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1065 build_simple_mem_ref (ret));
1066 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1067 ret = gimple_assign_lhs (g);
1068 }
1069 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1070 {
1071 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1072 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1073 ret = gimple_assign_lhs (g);
1074 }
1075 if (POINTER_TYPE_P (ptype))
1076 {
1077 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1078 if (size && TREE_CODE (size) == INTEGER_CST)
1079 {
1080 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1081 ret, fold_convert (addtype, size));
1082 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1083 ret = gimple_assign_lhs (g);
1084 }
1085 }
1086 return ret;
1087 }
1088
1089 /* Adjust the argument types in NODE to their appropriate vector
1090 counterparts. */
1091
1092 static void
1093 simd_clone_adjust (struct cgraph_node *node)
1094 {
1095 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1096
1097 targetm.simd_clone.adjust (node);
1098
1099 tree retval = simd_clone_adjust_return_type (node);
1100 ipa_parm_adjustment_vec adjustments
1101 = simd_clone_adjust_argument_types (node);
1102
1103 push_gimplify_context ();
1104
1105 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1106
1107 /* Adjust all uses of vector arguments accordingly. Adjust all
1108 return values accordingly. */
1109 tree iter = create_tmp_var (unsigned_type_node, "iter");
1110 tree iter1 = make_ssa_name (iter);
1111 tree iter2 = NULL_TREE;
1112 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1113 adjustments.release ();
1114
1115 /* Initialize the iteration variable. */
1116 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1117 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1118 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1119 /* Insert the SIMD array and iv initialization at function
1120 entry. */
1121 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1122
1123 pop_gimplify_context (NULL);
1124
1125 gimple *g;
1126 basic_block incr_bb = NULL;
1127 struct loop *loop = NULL;
1128
1129 /* Create a new BB right before the original exit BB, to hold the
1130 iteration increment and the condition/branch. */
1131 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1132 {
1133 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1134 incr_bb = create_empty_bb (orig_exit);
1135 add_bb_to_loop (incr_bb, body_bb->loop_father);
1136 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1137 flag. Set it now to be a FALLTHRU_EDGE. */
1138 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
1139 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
1140 for (unsigned i = 0;
1141 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
1142 {
1143 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
1144 redirect_edge_succ (e, incr_bb);
1145 }
1146 }
1147 else if (node->simdclone->inbranch)
1148 {
1149 incr_bb = create_empty_bb (entry_bb);
1150 add_bb_to_loop (incr_bb, body_bb->loop_father);
1151 }
1152
1153 if (incr_bb)
1154 {
1155 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1156 gsi = gsi_last_bb (incr_bb);
1157 iter2 = make_ssa_name (iter);
1158 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1159 build_int_cst (unsigned_type_node, 1));
1160 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1161
1162 /* Mostly annotate the loop for the vectorizer (the rest is done
1163 below). */
1164 loop = alloc_loop ();
1165 cfun->has_force_vectorize_loops = true;
1166 loop->safelen = node->simdclone->simdlen;
1167 loop->force_vectorize = true;
1168 loop->header = body_bb;
1169 }
1170
1171 /* Branch around the body if the mask applies. */
1172 if (node->simdclone->inbranch)
1173 {
1174 gsi = gsi_last_bb (loop->header);
1175 tree mask_array
1176 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1177 tree mask;
1178 if (node->simdclone->mask_mode != VOIDmode)
1179 {
1180 tree shift_cnt;
1181 if (mask_array == NULL_TREE)
1182 {
1183 tree arg = node->simdclone->args[node->simdclone->nargs
1184 - 1].vector_arg;
1185 mask = get_or_create_ssa_default_def (cfun, arg);
1186 shift_cnt = iter1;
1187 }
1188 else
1189 {
1190 tree maskt = TREE_TYPE (mask_array);
1191 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1192 c = node->simdclone->simdlen / (c + 1);
1193 int s = exact_log2 (c);
1194 gcc_assert (s > 0);
1195 c--;
1196 tree idx = make_ssa_name (TREE_TYPE (iter1));
1197 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1198 build_int_cst (NULL_TREE, s));
1199 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1200 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1201 tree aref = build4 (ARRAY_REF,
1202 TREE_TYPE (TREE_TYPE (mask_array)),
1203 mask_array, idx, NULL, NULL);
1204 g = gimple_build_assign (mask, aref);
1205 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1206 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1207 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1208 build_int_cst (TREE_TYPE (iter1), c));
1209 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1210 }
1211 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1212 RSHIFT_EXPR, mask, shift_cnt);
1213 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1214 mask = gimple_assign_lhs (g);
1215 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1216 BIT_AND_EXPR, mask,
1217 build_int_cst (TREE_TYPE (mask), 1));
1218 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1219 mask = gimple_assign_lhs (g);
1220 }
1221 else
1222 {
1223 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1224 tree aref = build4 (ARRAY_REF,
1225 TREE_TYPE (TREE_TYPE (mask_array)),
1226 mask_array, iter1, NULL, NULL);
1227 g = gimple_build_assign (mask, aref);
1228 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1229 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1230 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1231 {
1232 aref = build1 (VIEW_CONVERT_EXPR,
1233 build_nonstandard_integer_type (bitsize, 0),
1234 mask);
1235 mask = make_ssa_name (TREE_TYPE (aref));
1236 g = gimple_build_assign (mask, aref);
1237 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1238 }
1239 }
1240
1241 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1242 NULL, NULL);
1243 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1244 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1245 e->probability = profile_probability::unlikely ().guessed ();
1246 edge fallthru = FALLTHRU_EDGE (loop->header);
1247 fallthru->flags = EDGE_FALSE_VALUE;
1248 fallthru->probability = profile_probability::likely ().guessed ();
1249 }
1250
1251 basic_block latch_bb = NULL;
1252 basic_block new_exit_bb = NULL;
1253
1254 /* Generate the condition. */
1255 if (incr_bb)
1256 {
1257 gsi = gsi_last_bb (incr_bb);
1258 g = gimple_build_cond (LT_EXPR, iter2,
1259 build_int_cst (unsigned_type_node,
1260 node->simdclone->simdlen),
1261 NULL, NULL);
1262 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1263 edge e = split_block (incr_bb, gsi_stmt (gsi));
1264 latch_bb = e->dest;
1265 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1266 loop->latch = latch_bb;
1267
1268 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1269
1270 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1271
1272 /* FIXME: Do we need to distribute probabilities for the conditional? */
1273 new_e->probability = profile_probability::guessed_never ();
1274 /* The successor of incr_bb is already pointing to latch_bb; just
1275 change the flags.
1276 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1277 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1278 }
1279
1280 gphi *phi = create_phi_node (iter1, body_bb);
1281 edge preheader_edge = find_edge (entry_bb, body_bb);
1282 edge latch_edge = NULL;
1283 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1284 UNKNOWN_LOCATION);
1285 if (incr_bb)
1286 {
1287 latch_edge = single_succ_edge (latch_bb);
1288 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1289
1290 /* Generate the new return. */
1291 gsi = gsi_last_bb (new_exit_bb);
1292 if (retval
1293 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1294 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1295 retval = TREE_OPERAND (retval, 0);
1296 else if (retval)
1297 {
1298 retval = build1 (VIEW_CONVERT_EXPR,
1299 TREE_TYPE (TREE_TYPE (node->decl)),
1300 retval);
1301 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1302 false, GSI_CONTINUE_LINKING);
1303 }
1304 g = gimple_build_return (retval);
1305 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1306 }
1307
1308 /* Handle aligned clauses by replacing default defs of the aligned
1309 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1310 lhs. Handle linear by adding PHIs. */
1311 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1312 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1313 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1314 || !is_gimple_reg_type
1315 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1316 {
1317 tree orig_arg = node->simdclone->args[i].orig_arg;
1318 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1319 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1320 else
1321 {
1322 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1323 gimple_add_tmp_var (iter1);
1324 }
1325 gsi = gsi_after_labels (entry_bb);
1326 g = gimple_build_assign (iter1, orig_arg);
1327 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1328 gsi = gsi_after_labels (body_bb);
1329 g = gimple_build_assign (orig_arg, iter1);
1330 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1331 }
1332 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1333 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1334 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1335 == REFERENCE_TYPE
1336 && TREE_ADDRESSABLE
1337 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1338 {
1339 tree orig_arg = node->simdclone->args[i].orig_arg;
1340 tree def = ssa_default_def (cfun, orig_arg);
1341 if (def && !has_zero_uses (def))
1342 {
1343 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1344 gimple_add_tmp_var (iter1);
1345 gsi = gsi_after_labels (entry_bb);
1346 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1347 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1348 gsi = gsi_after_labels (body_bb);
1349 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1350 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1351 }
1352 }
1353 else if (node->simdclone->args[i].alignment
1354 && node->simdclone->args[i].arg_type
1355 == SIMD_CLONE_ARG_TYPE_UNIFORM
1356 && (node->simdclone->args[i].alignment
1357 & (node->simdclone->args[i].alignment - 1)) == 0
1358 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1359 == POINTER_TYPE)
1360 {
1361 unsigned int alignment = node->simdclone->args[i].alignment;
1362 tree orig_arg = node->simdclone->args[i].orig_arg;
1363 tree def = ssa_default_def (cfun, orig_arg);
1364 if (def && !has_zero_uses (def))
1365 {
1366 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1367 gimple_seq seq = NULL;
1368 bool need_cvt = false;
1369 gcall *call
1370 = gimple_build_call (fn, 2, def, size_int (alignment));
1371 g = call;
1372 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1373 ptr_type_node))
1374 need_cvt = true;
1375 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1376 gimple_call_set_lhs (g, t);
1377 gimple_seq_add_stmt_without_update (&seq, g);
1378 if (need_cvt)
1379 {
1380 t = make_ssa_name (orig_arg);
1381 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1382 gimple_seq_add_stmt_without_update (&seq, g);
1383 }
1384 gsi_insert_seq_on_edge_immediate
1385 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1386
1387 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1388 int freq = compute_call_stmt_bb_frequency (current_function_decl,
1389 entry_bb);
1390 node->create_edge (cgraph_node::get_create (fn),
1391 call, entry_bb->count, freq);
1392
1393 imm_use_iterator iter;
1394 use_operand_p use_p;
1395 gimple *use_stmt;
1396 tree repl = gimple_get_lhs (g);
1397 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1398 if (is_gimple_debug (use_stmt) || use_stmt == call)
1399 continue;
1400 else
1401 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1402 SET_USE (use_p, repl);
1403 }
1404 }
1405 else if ((node->simdclone->args[i].arg_type
1406 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1407 || (node->simdclone->args[i].arg_type
1408 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1409 || (node->simdclone->args[i].arg_type
1410 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1411 || (node->simdclone->args[i].arg_type
1412 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1413 {
1414 tree orig_arg = node->simdclone->args[i].orig_arg;
1415 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1416 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1417 tree def = NULL_TREE;
1418 if (TREE_ADDRESSABLE (orig_arg))
1419 {
1420 def = make_ssa_name (TREE_TYPE (orig_arg));
1421 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1422 if (incr_bb)
1423 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1424 gsi = gsi_after_labels (entry_bb);
1425 g = gimple_build_assign (def, orig_arg);
1426 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1427 }
1428 else
1429 {
1430 def = ssa_default_def (cfun, orig_arg);
1431 if (!def || has_zero_uses (def))
1432 def = NULL_TREE;
1433 else
1434 {
1435 iter1 = make_ssa_name (orig_arg);
1436 if (incr_bb)
1437 iter2 = make_ssa_name (orig_arg);
1438 }
1439 }
1440 if (def)
1441 {
1442 phi = create_phi_node (iter1, body_bb);
1443 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1444 if (incr_bb)
1445 {
1446 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1447 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1448 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1449 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1450 ? TREE_TYPE (orig_arg) : sizetype;
1451 tree addcst = simd_clone_linear_addend (node, i, addtype,
1452 entry_bb);
1453 gsi = gsi_last_bb (incr_bb);
1454 g = gimple_build_assign (iter2, code, iter1, addcst);
1455 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1456 }
1457
1458 imm_use_iterator iter;
1459 use_operand_p use_p;
1460 gimple *use_stmt;
1461 if (TREE_ADDRESSABLE (orig_arg))
1462 {
1463 gsi = gsi_after_labels (body_bb);
1464 g = gimple_build_assign (orig_arg, iter1);
1465 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1466 }
1467 else
1468 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1469 if (use_stmt == phi)
1470 continue;
1471 else
1472 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1473 SET_USE (use_p, iter1);
1474 }
1475 }
1476 else if (node->simdclone->args[i].arg_type
1477 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1478 || (node->simdclone->args[i].arg_type
1479 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1480 {
1481 tree orig_arg = node->simdclone->args[i].orig_arg;
1482 tree def = ssa_default_def (cfun, orig_arg);
1483 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1484 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1485 if (def && !has_zero_uses (def))
1486 {
1487 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1488 iter1 = make_ssa_name (orig_arg);
1489 if (incr_bb)
1490 iter2 = make_ssa_name (orig_arg);
1491 tree iter3 = make_ssa_name (rtype);
1492 tree iter4 = make_ssa_name (rtype);
1493 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1494 gsi = gsi_after_labels (entry_bb);
1495 gimple *load
1496 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1497 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1498
1499 tree array = node->simdclone->args[i].simd_array;
1500 TREE_ADDRESSABLE (array) = 1;
1501 tree ptr = build_fold_addr_expr (array);
1502 phi = create_phi_node (iter1, body_bb);
1503 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1504 if (incr_bb)
1505 {
1506 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1507 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1508 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1509 gsi = gsi_last_bb (incr_bb);
1510 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1511 }
1512
1513 phi = create_phi_node (iter4, body_bb);
1514 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1515 if (incr_bb)
1516 {
1517 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1518 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1519 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1520 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1521 ? TREE_TYPE (iter3) : sizetype;
1522 tree addcst = simd_clone_linear_addend (node, i, addtype,
1523 entry_bb);
1524 g = gimple_build_assign (iter5, code, iter4, addcst);
1525 gsi = gsi_last_bb (incr_bb);
1526 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1527 }
1528
1529 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1530 gsi = gsi_after_labels (body_bb);
1531 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1532
1533 imm_use_iterator iter;
1534 use_operand_p use_p;
1535 gimple *use_stmt;
1536 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1537 if (use_stmt == load)
1538 continue;
1539 else
1540 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1541 SET_USE (use_p, iter1);
1542
1543 if (!TYPE_READONLY (rtype) && incr_bb)
1544 {
1545 tree v = make_ssa_name (rtype);
1546 tree aref = build4 (ARRAY_REF, rtype, array,
1547 size_zero_node, NULL_TREE,
1548 NULL_TREE);
1549 gsi = gsi_after_labels (new_exit_bb);
1550 g = gimple_build_assign (v, aref);
1551 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1552 g = gimple_build_assign (build_simple_mem_ref (def), v);
1553 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1554 }
1555 }
1556 }
1557
1558 calculate_dominance_info (CDI_DOMINATORS);
1559 if (loop)
1560 add_loop (loop, loop->header->loop_father);
1561 update_ssa (TODO_update_ssa);
1562
1563 pop_cfun ();
1564 }
1565
1566 /* If the function in NODE is tagged as an elemental SIMD function,
1567 create the appropriate SIMD clones. */
1568
1569 static void
1570 expand_simd_clones (struct cgraph_node *node)
1571 {
1572 tree attr = lookup_attribute ("omp declare simd",
1573 DECL_ATTRIBUTES (node->decl));
1574 if (attr == NULL_TREE
1575 || node->global.inlined_to
1576 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1577 return;
1578
1579 /* Ignore
1580 #pragma omp declare simd
1581 extern int foo ();
1582 in C, there we don't know the argument types at all. */
1583 if (!node->definition
1584 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1585 return;
1586
1587 /* Call this before creating clone_info, as it might ggc_collect. */
1588 if (node->definition && node->has_gimple_body_p ())
1589 node->get_body ();
1590
1591 do
1592 {
1593 /* Start with parsing the "omp declare simd" attribute(s). */
1594 bool inbranch_clause_specified;
1595 struct cgraph_simd_clone *clone_info
1596 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1597 &inbranch_clause_specified);
1598 if (clone_info == NULL)
1599 continue;
1600
1601 int orig_simdlen = clone_info->simdlen;
1602 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1603 /* The target can return 0 (no simd clones should be created),
1604 1 (just one ISA of simd clones should be created) or higher
1605 count of ISA variants. In that case, clone_info is initialized
1606 for the first ISA variant. */
1607 int count
1608 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1609 base_type, 0);
1610 if (count == 0)
1611 continue;
1612
1613 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1614 also create one inbranch and one !inbranch clone of it. */
1615 for (int i = 0; i < count * 2; i++)
1616 {
1617 struct cgraph_simd_clone *clone = clone_info;
1618 if (inbranch_clause_specified && (i & 1) != 0)
1619 continue;
1620
1621 if (i != 0)
1622 {
1623 clone = simd_clone_struct_alloc (clone_info->nargs
1624 + ((i & 1) != 0));
1625 simd_clone_struct_copy (clone, clone_info);
1626 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1627 and simd_clone_adjust_argument_types did to the first
1628 clone's info. */
1629 clone->nargs -= clone_info->inbranch;
1630 clone->simdlen = orig_simdlen;
1631 /* And call the target hook again to get the right ISA. */
1632 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1633 base_type,
1634 i / 2);
1635 if ((i & 1) != 0)
1636 clone->inbranch = 1;
1637 }
1638
1639 /* simd_clone_mangle might fail if such a clone has been created
1640 already. */
1641 tree id = simd_clone_mangle (node, clone);
1642 if (id == NULL_TREE)
1643 continue;
1644
1645 /* Only when we are sure we want to create the clone actually
1646 clone the function (or definitions) or create another
1647 extern FUNCTION_DECL (for prototypes without definitions). */
1648 struct cgraph_node *n = simd_clone_create (node);
1649 if (n == NULL)
1650 continue;
1651
1652 n->simdclone = clone;
1653 clone->origin = node;
1654 clone->next_clone = NULL;
1655 if (node->simd_clones == NULL)
1656 {
1657 clone->prev_clone = n;
1658 node->simd_clones = n;
1659 }
1660 else
1661 {
1662 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1663 clone->prev_clone->simdclone->next_clone = n;
1664 node->simd_clones->simdclone->prev_clone = n;
1665 }
1666 symtab->change_decl_assembler_name (n->decl, id);
1667 /* And finally adjust the return type, parameters and for
1668 definitions also function body. */
1669 if (node->definition)
1670 simd_clone_adjust (n);
1671 else
1672 {
1673 simd_clone_adjust_return_type (n);
1674 simd_clone_adjust_argument_types (n);
1675 }
1676 }
1677 }
1678 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1679 }
1680
1681 /* Entry point for IPA simd clone creation pass. */
1682
1683 static unsigned int
1684 ipa_omp_simd_clone (void)
1685 {
1686 struct cgraph_node *node;
1687 FOR_EACH_FUNCTION (node)
1688 expand_simd_clones (node);
1689 return 0;
1690 }
1691
1692 namespace {
1693
1694 const pass_data pass_data_omp_simd_clone =
1695 {
1696 SIMPLE_IPA_PASS, /* type */
1697 "simdclone", /* name */
1698 OPTGROUP_OMP, /* optinfo_flags */
1699 TV_NONE, /* tv_id */
1700 ( PROP_ssa | PROP_cfg ), /* properties_required */
1701 0, /* properties_provided */
1702 0, /* properties_destroyed */
1703 0, /* todo_flags_start */
1704 0, /* todo_flags_finish */
1705 };
1706
1707 class pass_omp_simd_clone : public simple_ipa_opt_pass
1708 {
1709 public:
1710 pass_omp_simd_clone(gcc::context *ctxt)
1711 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1712 {}
1713
1714 /* opt_pass methods: */
1715 virtual bool gate (function *);
1716 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1717 };
1718
1719 bool
1720 pass_omp_simd_clone::gate (function *)
1721 {
1722 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1723 }
1724
1725 } // anon namespace
1726
1727 simple_ipa_opt_pass *
1728 make_pass_omp_simd_clone (gcc::context *ctxt)
1729 {
1730 return new pass_omp_simd_clone (ctxt);
1731 }