1026 lines
31 KiB
C
1026 lines
31 KiB
C
/* Copyright (C) 2005-2022 Free Software Foundation, Inc.
|
|
Contributed by Richard Henderson <rth@redhat.com>.
|
|
|
|
This file is part of the GNU Offloading and Multi Processing Library
|
|
(libgomp).
|
|
|
|
Libgomp is free software; you can redistribute it and/or modify it
|
|
under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 3, or (at your option)
|
|
any later version.
|
|
|
|
Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
more details.
|
|
|
|
Under Section 7 of GPL version 3, you are granted additional
|
|
permissions described in the GCC Runtime Library Exception, version
|
|
3.1, as published by the Free Software Foundation.
|
|
|
|
You should have received a copy of the GNU General Public License and
|
|
a copy of the GCC Runtime Library Exception along with this program;
|
|
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
<http://www.gnu.org/licenses/>. */
|
|
|
|
/* This file handles the LOOP (FOR/DO) construct. */
|
|
|
|
#include <limits.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include "libgomp.h"
|
|
|
|
ialias (GOMP_loop_ull_runtime_next)
|
|
ialias_redirect (GOMP_taskgroup_reduction_register)
|
|
|
|
typedef unsigned long long gomp_ull;
|
|
|
|
/* Initialize the given work share construct from the given arguments. */
|
|
|
|
static inline void
|
|
gomp_loop_ull_init (struct gomp_work_share *ws, bool up, gomp_ull start,
|
|
gomp_ull end, gomp_ull incr, enum gomp_schedule_type sched,
|
|
gomp_ull chunk_size)
|
|
{
|
|
ws->sched = sched;
|
|
ws->chunk_size_ull = chunk_size;
|
|
/* Canonicalize loops that have zero iterations to ->next == ->end. */
|
|
ws->end_ull = ((up && start > end) || (!up && start < end))
|
|
? start : end;
|
|
ws->incr_ull = incr;
|
|
ws->next_ull = start;
|
|
ws->mode = 0;
|
|
if (sched == GFS_DYNAMIC)
|
|
{
|
|
ws->chunk_size_ull *= incr;
|
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
|
{
|
|
/* For dynamic scheduling prepare things to make each iteration
|
|
faster. */
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
struct gomp_team *team = thr->ts.team;
|
|
long nthreads = team ? team->nthreads : 1;
|
|
|
|
if (__builtin_expect (up, 1))
|
|
{
|
|
/* Cheap overflow protection. */
|
|
if (__builtin_expect ((nthreads | ws->chunk_size_ull)
|
|
< 1ULL << (sizeof (gomp_ull)
|
|
* __CHAR_BIT__ / 2 - 1), 1))
|
|
ws->mode = ws->end_ull < (__LONG_LONG_MAX__ * 2ULL + 1
|
|
- (nthreads + 1) * ws->chunk_size_ull);
|
|
}
|
|
/* Cheap overflow protection. */
|
|
else if (__builtin_expect ((nthreads | -ws->chunk_size_ull)
|
|
< 1ULL << (sizeof (gomp_ull)
|
|
* __CHAR_BIT__ / 2 - 1), 1))
|
|
ws->mode = ws->end_ull > ((nthreads + 1) * -ws->chunk_size_ull
|
|
- (__LONG_LONG_MAX__ * 2ULL + 1));
|
|
}
|
|
#endif
|
|
}
|
|
if (!up)
|
|
ws->mode |= 2;
|
|
}
|
|
|
|
/* The *_start routines are called when first encountering a loop construct
|
|
that is not bound directly to a parallel construct. The first thread
|
|
that arrives will create the work-share construct; subsequent threads
|
|
will see the construct exists and allocate work from it.
|
|
|
|
START, END, INCR are the bounds of the loop; due to the restrictions of
|
|
OpenMP, these values must be the same in every thread. This is not
|
|
verified (nor is it entirely verifiable, since START is not necessarily
|
|
retained intact in the work-share data structure). CHUNK_SIZE is the
|
|
scheduling parameter; again this must be identical in all threads.
|
|
|
|
Returns true if there's any work for this thread to perform. If so,
|
|
*ISTART and *IEND are filled with the bounds of the iteration block
|
|
allocated to this thread. Returns false if all work was assigned to
|
|
other threads prior to this thread's arrival. */
|
|
|
|
static bool
|
|
gomp_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
|
thr->ts.static_trip = 0;
|
|
if (gomp_work_share_start (0))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
|
GFS_STATIC, chunk_size);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
|
|
return !gomp_iter_ull_static_next (istart, iend);
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
bool ret;
|
|
|
|
if (gomp_work_share_start (0))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
|
GFS_DYNAMIC, chunk_size);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
|
ret = gomp_iter_ull_dynamic_next (istart, iend);
|
|
#else
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
#endif
|
|
|
|
return ret;
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
bool ret;
|
|
|
|
if (gomp_work_share_start (0))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
|
GFS_GUIDED, chunk_size);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
|
ret = gomp_iter_ull_guided_next (istart, iend);
|
|
#else
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
#endif
|
|
|
|
return ret;
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_runtime_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_task_icv *icv = gomp_icv (false);
|
|
switch (icv->run_sched_var & ~GFS_MONOTONIC)
|
|
{
|
|
case GFS_STATIC:
|
|
return gomp_loop_ull_static_start (up, start, end, incr,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_DYNAMIC:
|
|
return gomp_loop_ull_dynamic_start (up, start, end, incr,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_GUIDED:
|
|
return gomp_loop_ull_guided_start (up, start, end, incr,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_AUTO:
|
|
/* For now map to schedule(static), later on we could play with feedback
|
|
driven choice. */
|
|
return gomp_loop_ull_static_start (up, start, end, incr,
|
|
0, istart, iend);
|
|
default:
|
|
abort ();
|
|
}
|
|
}
|
|
|
|
static long
|
|
gomp_adjust_sched (long sched, gomp_ull *chunk_size)
|
|
{
|
|
sched &= ~GFS_MONOTONIC;
|
|
switch (sched)
|
|
{
|
|
case GFS_STATIC:
|
|
case GFS_DYNAMIC:
|
|
case GFS_GUIDED:
|
|
return sched;
|
|
/* GFS_RUNTIME is used for runtime schedule without monotonic
|
|
or nonmonotonic modifiers on the clause.
|
|
GFS_RUNTIME|GFS_MONOTONIC for runtime schedule with monotonic
|
|
modifier. */
|
|
case GFS_RUNTIME:
|
|
/* GFS_AUTO is used for runtime schedule with nonmonotonic
|
|
modifier. */
|
|
case GFS_AUTO:
|
|
{
|
|
struct gomp_task_icv *icv = gomp_icv (false);
|
|
sched = icv->run_sched_var & ~GFS_MONOTONIC;
|
|
switch (sched)
|
|
{
|
|
case GFS_STATIC:
|
|
case GFS_DYNAMIC:
|
|
case GFS_GUIDED:
|
|
*chunk_size = icv->run_sched_chunk_size;
|
|
break;
|
|
case GFS_AUTO:
|
|
sched = GFS_STATIC;
|
|
*chunk_size = 0;
|
|
break;
|
|
default:
|
|
abort ();
|
|
}
|
|
return sched;
|
|
}
|
|
default:
|
|
abort ();
|
|
}
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, long sched, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend,
|
|
uintptr_t *reductions, void **mem)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
|
thr->ts.static_trip = 0;
|
|
if (reductions)
|
|
gomp_workshare_taskgroup_start ();
|
|
if (gomp_work_share_start (0))
|
|
{
|
|
sched = gomp_adjust_sched (sched, &chunk_size);
|
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
|
sched, chunk_size);
|
|
if (reductions)
|
|
{
|
|
GOMP_taskgroup_reduction_register (reductions);
|
|
thr->task->taskgroup->workshare = true;
|
|
thr->ts.work_share->task_reductions = reductions;
|
|
}
|
|
if (mem)
|
|
{
|
|
uintptr_t size = (uintptr_t) *mem;
|
|
#define INLINE_ORDERED_TEAM_IDS_OFF \
|
|
((offsetof (struct gomp_work_share, inline_ordered_team_ids) \
|
|
+ __alignof__ (long long) - 1) & ~(__alignof__ (long long) - 1))
|
|
if (sizeof (struct gomp_work_share)
|
|
<= INLINE_ORDERED_TEAM_IDS_OFF
|
|
|| __alignof__ (struct gomp_work_share) < __alignof__ (long long)
|
|
|| size > (sizeof (struct gomp_work_share)
|
|
- INLINE_ORDERED_TEAM_IDS_OFF))
|
|
*mem
|
|
= (void *) (thr->ts.work_share->ordered_team_ids
|
|
= gomp_malloc_cleared (size));
|
|
else
|
|
*mem = memset (((char *) thr->ts.work_share)
|
|
+ INLINE_ORDERED_TEAM_IDS_OFF, '\0', size);
|
|
}
|
|
gomp_work_share_init_done ();
|
|
}
|
|
else
|
|
{
|
|
if (reductions)
|
|
{
|
|
uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
|
|
gomp_workshare_task_reduction_register (reductions,
|
|
first_reductions);
|
|
}
|
|
if (mem)
|
|
{
|
|
if ((offsetof (struct gomp_work_share, inline_ordered_team_ids)
|
|
& (__alignof__ (long long) - 1)) == 0)
|
|
*mem = (void *) thr->ts.work_share->ordered_team_ids;
|
|
else
|
|
{
|
|
uintptr_t p = (uintptr_t) thr->ts.work_share->ordered_team_ids;
|
|
p += __alignof__ (long long) - 1;
|
|
p &= ~(__alignof__ (long long) - 1);
|
|
*mem = (void *) p;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ialias_call (GOMP_loop_ull_runtime_next) (istart, iend);
|
|
}
|
|
|
|
/* The *_ordered_*_start routines are similar. The only difference is that
|
|
this work-share construct is initialized to expect an ORDERED section. */
|
|
|
|
static bool
|
|
gomp_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
|
thr->ts.static_trip = 0;
|
|
if (gomp_work_share_start (1))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
|
GFS_STATIC, chunk_size);
|
|
gomp_ordered_static_init ();
|
|
gomp_work_share_init_done ();
|
|
}
|
|
|
|
return !gomp_iter_ull_static_next (istart, iend);
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
bool ret;
|
|
|
|
if (gomp_work_share_start (1))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
|
GFS_DYNAMIC, chunk_size);
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
else
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
|
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
|
if (ret)
|
|
gomp_ordered_first ();
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
bool ret;
|
|
|
|
if (gomp_work_share_start (1))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
|
GFS_GUIDED, chunk_size);
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
else
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
|
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
|
if (ret)
|
|
gomp_ordered_first ();
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_runtime_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
struct gomp_task_icv *icv = gomp_icv (false);
|
|
switch (icv->run_sched_var & ~GFS_MONOTONIC)
|
|
{
|
|
case GFS_STATIC:
|
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_DYNAMIC:
|
|
return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_GUIDED:
|
|
return gomp_loop_ull_ordered_guided_start (up, start, end, incr,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_AUTO:
|
|
/* For now map to schedule(static), later on we could play with feedback
|
|
driven choice. */
|
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr,
|
|
0, istart, iend);
|
|
default:
|
|
abort ();
|
|
}
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, long sched, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend,
|
|
uintptr_t *reductions, void **mem)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
size_t ordered = 1;
|
|
bool ret;
|
|
|
|
thr->ts.static_trip = 0;
|
|
if (reductions)
|
|
gomp_workshare_taskgroup_start ();
|
|
if (mem)
|
|
ordered += (uintptr_t) *mem;
|
|
if (gomp_work_share_start (ordered))
|
|
{
|
|
sched = gomp_adjust_sched (sched, &chunk_size);
|
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
|
sched, chunk_size);
|
|
if (reductions)
|
|
{
|
|
GOMP_taskgroup_reduction_register (reductions);
|
|
thr->task->taskgroup->workshare = true;
|
|
thr->ts.work_share->task_reductions = reductions;
|
|
}
|
|
if (sched == GFS_STATIC)
|
|
gomp_ordered_static_init ();
|
|
else
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
else
|
|
{
|
|
if (reductions)
|
|
{
|
|
uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
|
|
gomp_workshare_task_reduction_register (reductions,
|
|
first_reductions);
|
|
}
|
|
sched = thr->ts.work_share->sched;
|
|
if (sched != GFS_STATIC)
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
}
|
|
|
|
if (mem)
|
|
{
|
|
uintptr_t p
|
|
= (uintptr_t) (thr->ts.work_share->ordered_team_ids
|
|
+ (thr->ts.team ? thr->ts.team->nthreads : 1));
|
|
p += __alignof__ (long long) - 1;
|
|
p &= ~(__alignof__ (long long) - 1);
|
|
*mem = (void *) p;
|
|
}
|
|
|
|
switch (sched)
|
|
{
|
|
case GFS_STATIC:
|
|
case GFS_AUTO:
|
|
return !gomp_iter_ull_static_next (istart, iend);
|
|
case GFS_DYNAMIC:
|
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
|
break;
|
|
case GFS_GUIDED:
|
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
|
break;
|
|
default:
|
|
abort ();
|
|
}
|
|
|
|
if (ret)
|
|
gomp_ordered_first ();
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
return ret;
|
|
}
|
|
|
|
/* The *_doacross_*_start routines are similar. The only difference is that
|
|
this work-share construct is initialized to expect an ORDERED(N) - DOACROSS
|
|
section, and the worksharing loop iterates always from 0 to COUNTS[0] - 1
|
|
and other COUNTS array elements tell the library number of iterations
|
|
in the ordered inner loops. */
|
|
|
|
static bool
|
|
gomp_loop_ull_doacross_static_start (unsigned ncounts, gomp_ull *counts,
|
|
gomp_ull chunk_size, gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
|
thr->ts.static_trip = 0;
|
|
if (gomp_work_share_start (0))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
|
|
GFS_STATIC, chunk_size);
|
|
gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
|
|
return !gomp_iter_ull_static_next (istart, iend);
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_doacross_dynamic_start (unsigned ncounts, gomp_ull *counts,
|
|
gomp_ull chunk_size, gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
bool ret;
|
|
|
|
if (gomp_work_share_start (0))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
|
|
GFS_DYNAMIC, chunk_size);
|
|
gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
|
ret = gomp_iter_ull_dynamic_next (istart, iend);
|
|
#else
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
#endif
|
|
|
|
return ret;
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_doacross_guided_start (unsigned ncounts, gomp_ull *counts,
|
|
gomp_ull chunk_size, gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
bool ret;
|
|
|
|
if (gomp_work_share_start (0))
|
|
{
|
|
gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
|
|
GFS_GUIDED, chunk_size);
|
|
gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
|
|
gomp_work_share_init_done ();
|
|
}
|
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
|
ret = gomp_iter_ull_guided_next (istart, iend);
|
|
#else
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
#endif
|
|
|
|
return ret;
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_doacross_runtime_start (unsigned ncounts, gomp_ull *counts,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_task_icv *icv = gomp_icv (false);
|
|
switch (icv->run_sched_var & ~GFS_MONOTONIC)
|
|
{
|
|
case GFS_STATIC:
|
|
return gomp_loop_ull_doacross_static_start (ncounts, counts,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_DYNAMIC:
|
|
return gomp_loop_ull_doacross_dynamic_start (ncounts, counts,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_GUIDED:
|
|
return gomp_loop_ull_doacross_guided_start (ncounts, counts,
|
|
icv->run_sched_chunk_size,
|
|
istart, iend);
|
|
case GFS_AUTO:
|
|
/* For now map to schedule(static), later on we could play with feedback
|
|
driven choice. */
|
|
return gomp_loop_ull_doacross_static_start (ncounts, counts,
|
|
0, istart, iend);
|
|
default:
|
|
abort ();
|
|
}
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_doacross_start (unsigned ncounts, gomp_ull *counts,
|
|
long sched, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend,
|
|
uintptr_t *reductions, void **mem)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
|
thr->ts.static_trip = 0;
|
|
if (reductions)
|
|
gomp_workshare_taskgroup_start ();
|
|
if (gomp_work_share_start (0))
|
|
{
|
|
size_t extra = 0;
|
|
if (mem)
|
|
extra = (uintptr_t) *mem;
|
|
sched = gomp_adjust_sched (sched, &chunk_size);
|
|
gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
|
|
sched, chunk_size);
|
|
gomp_doacross_ull_init (ncounts, counts, chunk_size, extra);
|
|
if (reductions)
|
|
{
|
|
GOMP_taskgroup_reduction_register (reductions);
|
|
thr->task->taskgroup->workshare = true;
|
|
thr->ts.work_share->task_reductions = reductions;
|
|
}
|
|
gomp_work_share_init_done ();
|
|
}
|
|
else
|
|
{
|
|
if (reductions)
|
|
{
|
|
uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
|
|
gomp_workshare_task_reduction_register (reductions,
|
|
first_reductions);
|
|
}
|
|
sched = thr->ts.work_share->sched;
|
|
}
|
|
|
|
if (mem)
|
|
*mem = thr->ts.work_share->doacross->extra;
|
|
|
|
return ialias_call (GOMP_loop_ull_runtime_next) (istart, iend);
|
|
}
|
|
|
|
/* The *_next routines are called when the thread completes processing of
|
|
the iteration block currently assigned to it. If the work-share
|
|
construct is bound directly to a parallel construct, then the iteration
|
|
bounds may have been set up before the parallel. In which case, this
|
|
may be the first iteration for the thread.
|
|
|
|
Returns true if there is work remaining to be performed; *ISTART and
|
|
*IEND are filled with a new iteration block. Returns false if all work
|
|
has been assigned. */
|
|
|
|
static bool
|
|
gomp_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return !gomp_iter_ull_static_next (istart, iend);
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
bool ret;
|
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
|
ret = gomp_iter_ull_dynamic_next (istart, iend);
|
|
#else
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
#endif
|
|
|
|
return ret;
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
bool ret;
|
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
|
ret = gomp_iter_ull_guided_next (istart, iend);
|
|
#else
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
#endif
|
|
|
|
return ret;
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_runtime_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
|
switch (thr->ts.work_share->sched)
|
|
{
|
|
case GFS_STATIC:
|
|
case GFS_AUTO:
|
|
return gomp_loop_ull_static_next (istart, iend);
|
|
case GFS_DYNAMIC:
|
|
return gomp_loop_ull_dynamic_next (istart, iend);
|
|
case GFS_GUIDED:
|
|
return gomp_loop_ull_guided_next (istart, iend);
|
|
default:
|
|
abort ();
|
|
}
|
|
}
|
|
|
|
/* The *_ordered_*_next routines are called when the thread completes
|
|
processing of the iteration block currently assigned to it.
|
|
|
|
Returns true if there is work remaining to be performed; *ISTART and
|
|
*IEND are filled with a new iteration block. Returns false if all work
|
|
has been assigned. */
|
|
|
|
static bool
|
|
gomp_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
int test;
|
|
|
|
gomp_ordered_sync ();
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
test = gomp_iter_ull_static_next (istart, iend);
|
|
if (test >= 0)
|
|
gomp_ordered_static_next ();
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
|
return test == 0;
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
bool ret;
|
|
|
|
gomp_ordered_sync ();
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
|
if (ret)
|
|
gomp_ordered_next ();
|
|
else
|
|
gomp_ordered_last ();
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static bool
|
|
gomp_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
bool ret;
|
|
|
|
gomp_ordered_sync ();
|
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
|
if (ret)
|
|
gomp_ordered_next ();
|
|
else
|
|
gomp_ordered_last ();
|
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_runtime_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
|
switch (thr->ts.work_share->sched)
|
|
{
|
|
case GFS_STATIC:
|
|
case GFS_AUTO:
|
|
return gomp_loop_ull_ordered_static_next (istart, iend);
|
|
case GFS_DYNAMIC:
|
|
return gomp_loop_ull_ordered_dynamic_next (istart, iend);
|
|
case GFS_GUIDED:
|
|
return gomp_loop_ull_ordered_guided_next (istart, iend);
|
|
default:
|
|
abort ();
|
|
}
|
|
}
|
|
|
|
/* We use static functions above so that we're sure that the "runtime"
|
|
function can defer to the proper routine without interposition. We
|
|
export the static function with a strong alias when possible, or with
|
|
a wrapper function otherwise. */
|
|
|
|
#ifdef HAVE_ATTRIBUTE_ALIAS
|
|
extern __typeof(gomp_loop_ull_static_start) GOMP_loop_ull_static_start
|
|
__attribute__((alias ("gomp_loop_ull_static_start")));
|
|
extern __typeof(gomp_loop_ull_dynamic_start) GOMP_loop_ull_dynamic_start
|
|
__attribute__((alias ("gomp_loop_ull_dynamic_start")));
|
|
extern __typeof(gomp_loop_ull_guided_start) GOMP_loop_ull_guided_start
|
|
__attribute__((alias ("gomp_loop_ull_guided_start")));
|
|
extern __typeof(gomp_loop_ull_dynamic_start) GOMP_loop_ull_nonmonotonic_dynamic_start
|
|
__attribute__((alias ("gomp_loop_ull_dynamic_start")));
|
|
extern __typeof(gomp_loop_ull_guided_start) GOMP_loop_ull_nonmonotonic_guided_start
|
|
__attribute__((alias ("gomp_loop_ull_guided_start")));
|
|
extern __typeof(GOMP_loop_ull_runtime_start) GOMP_loop_ull_nonmonotonic_runtime_start
|
|
__attribute__((alias ("GOMP_loop_ull_runtime_start")));
|
|
extern __typeof(GOMP_loop_ull_runtime_start) GOMP_loop_ull_maybe_nonmonotonic_runtime_start
|
|
__attribute__((alias ("GOMP_loop_ull_runtime_start")));
|
|
|
|
extern __typeof(gomp_loop_ull_ordered_static_start) GOMP_loop_ull_ordered_static_start
|
|
__attribute__((alias ("gomp_loop_ull_ordered_static_start")));
|
|
extern __typeof(gomp_loop_ull_ordered_dynamic_start) GOMP_loop_ull_ordered_dynamic_start
|
|
__attribute__((alias ("gomp_loop_ull_ordered_dynamic_start")));
|
|
extern __typeof(gomp_loop_ull_ordered_guided_start) GOMP_loop_ull_ordered_guided_start
|
|
__attribute__((alias ("gomp_loop_ull_ordered_guided_start")));
|
|
|
|
extern __typeof(gomp_loop_ull_doacross_static_start) GOMP_loop_ull_doacross_static_start
|
|
__attribute__((alias ("gomp_loop_ull_doacross_static_start")));
|
|
extern __typeof(gomp_loop_ull_doacross_dynamic_start) GOMP_loop_ull_doacross_dynamic_start
|
|
__attribute__((alias ("gomp_loop_ull_doacross_dynamic_start")));
|
|
extern __typeof(gomp_loop_ull_doacross_guided_start) GOMP_loop_ull_doacross_guided_start
|
|
__attribute__((alias ("gomp_loop_ull_doacross_guided_start")));
|
|
|
|
extern __typeof(gomp_loop_ull_static_next) GOMP_loop_ull_static_next
|
|
__attribute__((alias ("gomp_loop_ull_static_next")));
|
|
extern __typeof(gomp_loop_ull_dynamic_next) GOMP_loop_ull_dynamic_next
|
|
__attribute__((alias ("gomp_loop_ull_dynamic_next")));
|
|
extern __typeof(gomp_loop_ull_guided_next) GOMP_loop_ull_guided_next
|
|
__attribute__((alias ("gomp_loop_ull_guided_next")));
|
|
extern __typeof(gomp_loop_ull_dynamic_next) GOMP_loop_ull_nonmonotonic_dynamic_next
|
|
__attribute__((alias ("gomp_loop_ull_dynamic_next")));
|
|
extern __typeof(gomp_loop_ull_guided_next) GOMP_loop_ull_nonmonotonic_guided_next
|
|
__attribute__((alias ("gomp_loop_ull_guided_next")));
|
|
extern __typeof(GOMP_loop_ull_runtime_next) GOMP_loop_ull_nonmonotonic_runtime_next
|
|
__attribute__((alias ("GOMP_loop_ull_runtime_next")));
|
|
extern __typeof(GOMP_loop_ull_runtime_next) GOMP_loop_ull_maybe_nonmonotonic_runtime_next
|
|
__attribute__((alias ("GOMP_loop_ull_runtime_next")));
|
|
|
|
extern __typeof(gomp_loop_ull_ordered_static_next) GOMP_loop_ull_ordered_static_next
|
|
__attribute__((alias ("gomp_loop_ull_ordered_static_next")));
|
|
extern __typeof(gomp_loop_ull_ordered_dynamic_next) GOMP_loop_ull_ordered_dynamic_next
|
|
__attribute__((alias ("gomp_loop_ull_ordered_dynamic_next")));
|
|
extern __typeof(gomp_loop_ull_ordered_guided_next) GOMP_loop_ull_ordered_guided_next
|
|
__attribute__((alias ("gomp_loop_ull_ordered_guided_next")));
|
|
#else
|
|
bool
|
|
GOMP_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_static_start (up, start, end, incr, chunk_size, istart,
|
|
iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_dynamic_start (up, start, end, incr, chunk_size, istart,
|
|
iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_guided_start (up, start, end, incr, chunk_size, istart,
|
|
iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_nonmonotonic_dynamic_start (bool up, gomp_ull start,
|
|
gomp_ull end, gomp_ull incr,
|
|
gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_dynamic_start (up, start, end, incr, chunk_size, istart,
|
|
iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_nonmonotonic_guided_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_guided_start (up, start, end, incr, chunk_size, istart,
|
|
iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_nonmonotonic_runtime_start (bool up, gomp_ull start,
|
|
gomp_ull end, gomp_ull incr,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return GOMP_loop_ull_runtime_start (up, start, end, incr, istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_maybe_nonmonotonic_runtime_start (bool up, gomp_ull start,
|
|
gomp_ull end, gomp_ull incr,
|
|
gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
return GOMP_loop_ull_runtime_start (up, start, end, incr, istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr, chunk_size,
|
|
istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr, chunk_size,
|
|
istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
|
|
gomp_ull incr, gomp_ull chunk_size,
|
|
gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_ordered_guided_start (up, start, end, incr, chunk_size,
|
|
istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_doacross_static_start (unsigned ncounts, gomp_ull *counts,
|
|
gomp_ull chunk_size, gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_doacross_static_start (ncounts, counts, chunk_size,
|
|
istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_doacross_dynamic_start (unsigned ncounts, gomp_ull *counts,
|
|
gomp_ull chunk_size, gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_doacross_dynamic_start (ncounts, counts, chunk_size,
|
|
istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_doacross_guided_start (unsigned ncounts, gomp_ull *counts,
|
|
gomp_ull chunk_size, gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_doacross_guided_start (ncounts, counts, chunk_size,
|
|
istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_static_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_dynamic_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_guided_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_nonmonotonic_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_dynamic_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_nonmonotonic_guided_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_guided_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_nonmonotonic_runtime_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return GOMP_loop_ull_runtime_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_maybe_nonmonotonic_runtime_next (gomp_ull *istart,
|
|
gomp_ull *iend)
|
|
{
|
|
return GOMP_loop_ull_runtime_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_ordered_static_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_ordered_dynamic_next (istart, iend);
|
|
}
|
|
|
|
bool
|
|
GOMP_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
|
|
{
|
|
return gomp_loop_ull_ordered_guided_next (istart, iend);
|
|
}
|
|
#endif
|