400 lines
11 KiB
C
400 lines
11 KiB
C
/* Copyright (C) 2015-2022 Free Software Foundation, Inc.
|
|
Contributed by Jakub Jelinek <jakub@redhat.com>.
|
|
|
|
This file is part of the GNU Offloading and Multi Processing Library
|
|
(libgomp).
|
|
|
|
Libgomp is free software; you can redistribute it and/or modify it
|
|
under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 3, or (at your option)
|
|
any later version.
|
|
|
|
Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
more details.
|
|
|
|
Under Section 7 of GPL version 3, you are granted additional
|
|
permissions described in the GCC Runtime Library Exception, version
|
|
3.1, as published by the Free Software Foundation.
|
|
|
|
You should have received a copy of the GNU General Public License and
|
|
a copy of the GCC Runtime Library Exception along with this program;
|
|
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
<http://www.gnu.org/licenses/>. */
|
|
|
|
/* This file handles the taskloop construct. It is included twice, once
|
|
for the long and once for unsigned long long variant. */
|
|
|
|
/* Called when encountering an explicit task directive. If IF_CLAUSE is
|
|
false, then we must not delay in executing the task. If UNTIED is true,
|
|
then the task may be executed by any member of the team. */
|
|
|
|
void
|
|
GOMP_taskloop (void (*fn) (void *), void *data, void (*cpyfn) (void *, void *),
|
|
long arg_size, long arg_align, unsigned flags,
|
|
unsigned long num_tasks, int priority,
|
|
TYPE start, TYPE end, TYPE step)
|
|
{
|
|
struct gomp_thread *thr = gomp_thread ();
|
|
struct gomp_team *team = thr->ts.team;
|
|
|
|
#ifdef HAVE_BROKEN_POSIX_SEMAPHORES
|
|
/* If pthread_mutex_* is used for omp_*lock*, then each task must be
|
|
tied to one thread all the time. This means UNTIED tasks must be
|
|
tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN
|
|
might be running on different thread than FN. */
|
|
if (cpyfn)
|
|
flags &= ~GOMP_TASK_FLAG_IF;
|
|
flags &= ~GOMP_TASK_FLAG_UNTIED;
|
|
#endif
|
|
|
|
/* If parallel or taskgroup has been cancelled, don't start new tasks. */
|
|
if (team && gomp_team_barrier_cancelled (&team->barrier))
|
|
{
|
|
early_return:
|
|
if ((flags & (GOMP_TASK_FLAG_NOGROUP | GOMP_TASK_FLAG_REDUCTION))
|
|
== GOMP_TASK_FLAG_REDUCTION)
|
|
{
|
|
struct gomp_data_head { TYPE t1, t2; uintptr_t *ptr; };
|
|
uintptr_t *ptr = ((struct gomp_data_head *) data)->ptr;
|
|
/* Tell callers GOMP_taskgroup_reduction_register has not been
|
|
called. */
|
|
ptr[2] = 0;
|
|
}
|
|
return;
|
|
}
|
|
|
|
#ifdef TYPE_is_long
|
|
TYPE s = step;
|
|
if (step > 0)
|
|
{
|
|
if (start >= end)
|
|
goto early_return;
|
|
s--;
|
|
}
|
|
else
|
|
{
|
|
if (start <= end)
|
|
goto early_return;
|
|
s++;
|
|
}
|
|
UTYPE n = (end - start + s) / step;
|
|
#else
|
|
UTYPE n;
|
|
if (flags & GOMP_TASK_FLAG_UP)
|
|
{
|
|
if (start >= end)
|
|
goto early_return;
|
|
n = (end - start + step - 1) / step;
|
|
}
|
|
else
|
|
{
|
|
if (start <= end)
|
|
goto early_return;
|
|
n = (start - end - step - 1) / -step;
|
|
}
|
|
#endif
|
|
|
|
TYPE task_step = step;
|
|
TYPE nfirst_task_step = step;
|
|
unsigned long nfirst = n;
|
|
if (flags & GOMP_TASK_FLAG_GRAINSIZE)
|
|
{
|
|
unsigned long grainsize = num_tasks;
|
|
#ifdef TYPE_is_long
|
|
num_tasks = n / grainsize;
|
|
#else
|
|
UTYPE ndiv = n / grainsize;
|
|
num_tasks = ndiv;
|
|
if (num_tasks != ndiv)
|
|
num_tasks = ~0UL;
|
|
#endif
|
|
if ((flags & GOMP_TASK_FLAG_STRICT)
|
|
&& num_tasks != ~0ULL)
|
|
{
|
|
UTYPE mod = n % grainsize;
|
|
task_step = (TYPE) grainsize * step;
|
|
if (mod)
|
|
{
|
|
num_tasks++;
|
|
nfirst_task_step = (TYPE) mod * step;
|
|
if (num_tasks == 1)
|
|
task_step = nfirst_task_step;
|
|
else
|
|
nfirst = num_tasks - 2;
|
|
}
|
|
}
|
|
else if (num_tasks <= 1)
|
|
{
|
|
num_tasks = 1;
|
|
task_step = end - start;
|
|
}
|
|
else if (num_tasks >= grainsize
|
|
#ifndef TYPE_is_long
|
|
&& num_tasks != ~0UL
|
|
#endif
|
|
)
|
|
{
|
|
UTYPE mul = num_tasks * grainsize;
|
|
task_step = (TYPE) grainsize * step;
|
|
if (mul != n)
|
|
{
|
|
nfirst_task_step = task_step;
|
|
task_step += step;
|
|
nfirst = n - mul - 1;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
UTYPE div = n / num_tasks;
|
|
UTYPE mod = n % num_tasks;
|
|
task_step = (TYPE) div * step;
|
|
if (mod)
|
|
{
|
|
nfirst_task_step = task_step;
|
|
task_step += step;
|
|
nfirst = mod - 1;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (num_tasks == 0)
|
|
num_tasks = team ? team->nthreads : 1;
|
|
if (num_tasks >= n)
|
|
num_tasks = n;
|
|
else
|
|
{
|
|
UTYPE div = n / num_tasks;
|
|
UTYPE mod = n % num_tasks;
|
|
task_step = (TYPE) div * step;
|
|
if (mod)
|
|
{
|
|
nfirst_task_step = task_step;
|
|
task_step += step;
|
|
nfirst = mod - 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (flags & GOMP_TASK_FLAG_NOGROUP)
|
|
{
|
|
if (__builtin_expect (gomp_cancel_var, 0)
|
|
&& thr->task
|
|
&& thr->task->taskgroup)
|
|
{
|
|
if (thr->task->taskgroup->cancelled)
|
|
return;
|
|
if (thr->task->taskgroup->workshare
|
|
&& thr->task->taskgroup->prev
|
|
&& thr->task->taskgroup->prev->cancelled)
|
|
return;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
ialias_call (GOMP_taskgroup_start) ();
|
|
if (flags & GOMP_TASK_FLAG_REDUCTION)
|
|
{
|
|
struct gomp_data_head { TYPE t1, t2; uintptr_t *ptr; };
|
|
uintptr_t *ptr = ((struct gomp_data_head *) data)->ptr;
|
|
ialias_call (GOMP_taskgroup_reduction_register) (ptr);
|
|
}
|
|
}
|
|
|
|
if (priority > gomp_max_task_priority_var)
|
|
priority = gomp_max_task_priority_var;
|
|
|
|
if ((flags & GOMP_TASK_FLAG_IF) == 0 || team == NULL
|
|
|| (thr->task && thr->task->final_task)
|
|
|| team->task_count + num_tasks > 64 * team->nthreads)
|
|
{
|
|
unsigned long i;
|
|
if (__builtin_expect (cpyfn != NULL, 0))
|
|
{
|
|
struct gomp_task task[num_tasks];
|
|
struct gomp_task *parent = thr->task;
|
|
arg_size = (arg_size + arg_align - 1) & ~(arg_align - 1);
|
|
char buf[num_tasks * arg_size + arg_align - 1];
|
|
char *arg = (char *) (((uintptr_t) buf + arg_align - 1)
|
|
& ~(uintptr_t) (arg_align - 1));
|
|
char *orig_arg = arg;
|
|
for (i = 0; i < num_tasks; i++)
|
|
{
|
|
gomp_init_task (&task[i], parent, gomp_icv (false));
|
|
task[i].priority = priority;
|
|
task[i].kind = GOMP_TASK_UNDEFERRED;
|
|
task[i].final_task = (thr->task && thr->task->final_task)
|
|
|| (flags & GOMP_TASK_FLAG_FINAL);
|
|
if (thr->task)
|
|
{
|
|
task[i].in_tied_task = thr->task->in_tied_task;
|
|
task[i].taskgroup = thr->task->taskgroup;
|
|
}
|
|
thr->task = &task[i];
|
|
cpyfn (arg, data);
|
|
arg += arg_size;
|
|
}
|
|
arg = orig_arg;
|
|
for (i = 0; i < num_tasks; i++)
|
|
{
|
|
thr->task = &task[i];
|
|
((TYPE *)arg)[0] = start;
|
|
start += task_step;
|
|
((TYPE *)arg)[1] = start;
|
|
if (i == nfirst)
|
|
task_step = nfirst_task_step;
|
|
fn (arg);
|
|
arg += arg_size;
|
|
if (!priority_queue_empty_p (&task[i].children_queue,
|
|
MEMMODEL_RELAXED))
|
|
{
|
|
gomp_mutex_lock (&team->task_lock);
|
|
gomp_clear_parent (&task[i].children_queue);
|
|
gomp_mutex_unlock (&team->task_lock);
|
|
}
|
|
gomp_end_task ();
|
|
}
|
|
}
|
|
else
|
|
for (i = 0; i < num_tasks; i++)
|
|
{
|
|
struct gomp_task task;
|
|
|
|
gomp_init_task (&task, thr->task, gomp_icv (false));
|
|
task.priority = priority;
|
|
task.kind = GOMP_TASK_UNDEFERRED;
|
|
task.final_task = (thr->task && thr->task->final_task)
|
|
|| (flags & GOMP_TASK_FLAG_FINAL);
|
|
if (thr->task)
|
|
{
|
|
task.in_tied_task = thr->task->in_tied_task;
|
|
task.taskgroup = thr->task->taskgroup;
|
|
}
|
|
thr->task = &task;
|
|
((TYPE *)data)[0] = start;
|
|
start += task_step;
|
|
((TYPE *)data)[1] = start;
|
|
if (i == nfirst)
|
|
task_step = nfirst_task_step;
|
|
fn (data);
|
|
if (!priority_queue_empty_p (&task.children_queue,
|
|
MEMMODEL_RELAXED))
|
|
{
|
|
gomp_mutex_lock (&team->task_lock);
|
|
gomp_clear_parent (&task.children_queue);
|
|
gomp_mutex_unlock (&team->task_lock);
|
|
}
|
|
gomp_end_task ();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
struct gomp_task *tasks[num_tasks];
|
|
struct gomp_task *parent = thr->task;
|
|
struct gomp_taskgroup *taskgroup = parent->taskgroup;
|
|
char *arg;
|
|
int do_wake;
|
|
unsigned long i;
|
|
|
|
for (i = 0; i < num_tasks; i++)
|
|
{
|
|
struct gomp_task *task
|
|
= gomp_malloc (sizeof (*task) + arg_size + arg_align - 1);
|
|
tasks[i] = task;
|
|
arg = (char *) (((uintptr_t) (task + 1) + arg_align - 1)
|
|
& ~(uintptr_t) (arg_align - 1));
|
|
gomp_init_task (task, parent, gomp_icv (false));
|
|
task->priority = priority;
|
|
task->kind = GOMP_TASK_UNDEFERRED;
|
|
task->in_tied_task = parent->in_tied_task;
|
|
task->taskgroup = taskgroup;
|
|
thr->task = task;
|
|
if (cpyfn)
|
|
{
|
|
cpyfn (arg, data);
|
|
task->copy_ctors_done = true;
|
|
}
|
|
else
|
|
memcpy (arg, data, arg_size);
|
|
((TYPE *)arg)[0] = start;
|
|
start += task_step;
|
|
((TYPE *)arg)[1] = start;
|
|
if (i == nfirst)
|
|
task_step = nfirst_task_step;
|
|
thr->task = parent;
|
|
task->kind = GOMP_TASK_WAITING;
|
|
task->fn = fn;
|
|
task->fn_data = arg;
|
|
task->final_task = (flags & GOMP_TASK_FLAG_FINAL) >> 1;
|
|
}
|
|
gomp_mutex_lock (&team->task_lock);
|
|
/* If parallel or taskgroup has been cancelled, don't start new
|
|
tasks. */
|
|
if (__builtin_expect (gomp_cancel_var, 0)
|
|
&& cpyfn == NULL)
|
|
{
|
|
if (gomp_team_barrier_cancelled (&team->barrier))
|
|
{
|
|
do_cancel:
|
|
gomp_mutex_unlock (&team->task_lock);
|
|
for (i = 0; i < num_tasks; i++)
|
|
{
|
|
gomp_finish_task (tasks[i]);
|
|
free (tasks[i]);
|
|
}
|
|
if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0)
|
|
ialias_call (GOMP_taskgroup_end) ();
|
|
return;
|
|
}
|
|
if (taskgroup)
|
|
{
|
|
if (taskgroup->cancelled)
|
|
goto do_cancel;
|
|
if (taskgroup->workshare
|
|
&& taskgroup->prev
|
|
&& taskgroup->prev->cancelled)
|
|
goto do_cancel;
|
|
}
|
|
}
|
|
if (taskgroup)
|
|
taskgroup->num_children += num_tasks;
|
|
for (i = 0; i < num_tasks; i++)
|
|
{
|
|
struct gomp_task *task = tasks[i];
|
|
priority_queue_insert (PQ_CHILDREN, &parent->children_queue,
|
|
task, priority,
|
|
PRIORITY_INSERT_BEGIN,
|
|
/*last_parent_depends_on=*/false,
|
|
task->parent_depends_on);
|
|
if (taskgroup)
|
|
priority_queue_insert (PQ_TASKGROUP, &taskgroup->taskgroup_queue,
|
|
task, priority, PRIORITY_INSERT_BEGIN,
|
|
/*last_parent_depends_on=*/false,
|
|
task->parent_depends_on);
|
|
priority_queue_insert (PQ_TEAM, &team->task_queue, task, priority,
|
|
PRIORITY_INSERT_END,
|
|
/*last_parent_depends_on=*/false,
|
|
task->parent_depends_on);
|
|
++team->task_count;
|
|
++team->task_queued_count;
|
|
}
|
|
gomp_team_barrier_set_task_pending (&team->barrier);
|
|
if (team->task_running_count + !parent->in_tied_task
|
|
< team->nthreads)
|
|
{
|
|
do_wake = team->nthreads - team->task_running_count
|
|
- !parent->in_tied_task;
|
|
if ((unsigned long) do_wake > num_tasks)
|
|
do_wake = num_tasks;
|
|
}
|
|
else
|
|
do_wake = 0;
|
|
gomp_mutex_unlock (&team->task_lock);
|
|
if (do_wake)
|
|
gomp_team_barrier_wake (&team->barrier, do_wake);
|
|
}
|
|
if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0)
|
|
ialias_call (GOMP_taskgroup_end) ();
|
|
}
|