490 lines
16 KiB
C
490 lines
16 KiB
C
/* Copyright (C) 2015-2022 Free Software Foundation, Inc.
|
|
Contributed by Aldy Hernandez <aldyh@redhat.com>.
|
|
|
|
This file is part of the GNU Offloading and Multi Processing Library
|
|
(libgomp).
|
|
|
|
Libgomp is free software; you can redistribute it and/or modify it
|
|
under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 3, or (at your option)
|
|
any later version.
|
|
|
|
Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
more details.
|
|
|
|
Under Section 7 of GPL version 3, you are granted additional
|
|
permissions described in the GCC Runtime Library Exception, version
|
|
3.1, as published by the Free Software Foundation.
|
|
|
|
You should have received a copy of the GNU General Public License and
|
|
a copy of the GCC Runtime Library Exception along with this program;
|
|
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
<http://www.gnu.org/licenses/>. */
|
|
|
|
/* Header file for a priority queue of GOMP tasks. */
|
|
|
|
/* ?? Perhaps all the priority_tree_* functions are complex and rare
|
|
enough to go out-of-line and be moved to priority_queue.c. ?? */
|
|
|
|
#ifndef _PRIORITY_QUEUE_H_
|
|
#define _PRIORITY_QUEUE_H_
|
|
|
|
/* One task. */
|
|
|
|
struct priority_node
|
|
{
|
|
/* Next and previous chains in a circular doubly linked list for
|
|
tasks within this task's priority. */
|
|
struct priority_node *next, *prev;
|
|
};
|
|
|
|
/* All tasks within the same priority. */
|
|
|
|
struct priority_list
|
|
{
|
|
/* Priority of the tasks in this set. */
|
|
int priority;
|
|
|
|
/* Tasks. */
|
|
struct priority_node *tasks;
|
|
|
|
/* This points to the last of the higher priority WAITING tasks.
|
|
Remember that for the children queue, we have:
|
|
|
|
parent_depends_on WAITING tasks.
|
|
!parent_depends_on WAITING tasks.
|
|
TIED tasks.
|
|
|
|
This is a pointer to the last of the parent_depends_on WAITING
|
|
tasks which are essentially, higher priority items within their
|
|
priority. */
|
|
struct priority_node *last_parent_depends_on;
|
|
};
|
|
|
|
/* Another splay tree instantiation, for priority_list's. */
|
|
typedef struct prio_splay_tree_node_s *prio_splay_tree_node;
|
|
typedef struct prio_splay_tree_s *prio_splay_tree;
|
|
typedef struct prio_splay_tree_key_s *prio_splay_tree_key;
|
|
struct prio_splay_tree_key_s {
|
|
/* This structure must only containing a priority_list, as we cast
|
|
prio_splay_tree_key to priority_list throughout. */
|
|
struct priority_list l;
|
|
};
|
|
#define splay_tree_prefix prio
|
|
#include "splay-tree.h"
|
|
|
|
/* The entry point into a priority queue of tasks.
|
|
|
|
There are two alternate implementations with which to store tasks:
|
|
as a balanced tree of sorts, or as a simple list of tasks. If
|
|
there are only priority-0 items (ROOT is NULL), we use the simple
|
|
list, otherwise (ROOT is non-NULL) we use the tree. */
|
|
|
|
struct priority_queue
|
|
{
|
|
/* If t.root != NULL, this is a splay tree of priority_lists to hold
|
|
all tasks. This is only used if multiple priorities are in play,
|
|
otherwise we use the priority_list `l' below to hold all
|
|
(priority-0) tasks. */
|
|
struct prio_splay_tree_s t;
|
|
|
|
/* If T above is NULL, only priority-0 items exist, so keep them
|
|
in a simple list. */
|
|
struct priority_list l;
|
|
};
|
|
|
|
enum priority_insert_type {
|
|
/* Insert at the beginning of a priority list. */
|
|
PRIORITY_INSERT_BEGIN,
|
|
/* Insert at the end of a priority list. */
|
|
PRIORITY_INSERT_END
|
|
};
|
|
|
|
/* Used to determine in which queue a given priority node belongs in.
|
|
See pnode field of gomp_task. */
|
|
|
|
enum priority_queue_type
|
|
{
|
|
PQ_TEAM, /* Node belongs in gomp_team's task_queue. */
|
|
PQ_CHILDREN, /* Node belongs in parent's children_queue. */
|
|
PQ_TASKGROUP, /* Node belongs in taskgroup->taskgroup_queue. */
|
|
PQ_IGNORED = 999
|
|
};
|
|
|
|
typedef bool (*priority_queue_predicate) (struct gomp_task *);
|
|
|
|
/* Priority queue implementation prototypes. */
|
|
|
|
extern bool priority_queue_task_in_queue_p (enum priority_queue_type,
|
|
struct priority_queue *,
|
|
struct gomp_task *);
|
|
extern void priority_queue_dump (enum priority_queue_type,
|
|
struct priority_queue *);
|
|
extern void priority_queue_verify (enum priority_queue_type,
|
|
struct priority_queue *, bool);
|
|
extern struct gomp_task *priority_queue_find (enum priority_queue_type,
|
|
struct priority_queue *,
|
|
priority_queue_predicate);
|
|
extern void priority_tree_remove (enum priority_queue_type,
|
|
struct priority_queue *,
|
|
struct priority_node *);
|
|
extern struct gomp_task *priority_tree_next_task (enum priority_queue_type,
|
|
struct priority_queue *,
|
|
enum priority_queue_type,
|
|
struct priority_queue *,
|
|
bool *);
|
|
|
|
/* Return TRUE if there is more than one priority in HEAD. This is
|
|
used throughout to to choose between the fast path (priority 0 only
|
|
items) and a world with multiple priorities. */
|
|
|
|
static inline bool
|
|
priority_queue_multi_p (struct priority_queue *head)
|
|
{
|
|
return __builtin_expect (head->t.root != NULL, 0);
|
|
}
|
|
|
|
/* Initialize a priority queue. */
|
|
|
|
static inline void
|
|
priority_queue_init (struct priority_queue *head)
|
|
{
|
|
head->t.root = NULL;
|
|
/* To save a few microseconds, we don't initialize head->l.priority
|
|
to 0 here. It is implied that priority will be 0 if head->t.root
|
|
== NULL.
|
|
|
|
priority_tree_insert() will fix this when we encounter multiple
|
|
priorities. */
|
|
head->l.tasks = NULL;
|
|
head->l.last_parent_depends_on = NULL;
|
|
}
|
|
|
|
static inline void
|
|
priority_queue_free (struct priority_queue *head)
|
|
{
|
|
/* There's nothing to do, as tasks were freed as they were removed
|
|
in priority_queue_remove. */
|
|
}
|
|
|
|
/* Forward declarations. */
|
|
static inline size_t priority_queue_offset (enum priority_queue_type);
|
|
static inline struct gomp_task *priority_node_to_task
|
|
(enum priority_queue_type,
|
|
struct priority_node *);
|
|
static inline struct priority_node *task_to_priority_node
|
|
(enum priority_queue_type,
|
|
struct gomp_task *);
|
|
|
|
/* Return TRUE if priority queue HEAD is empty.
|
|
|
|
MODEL IS MEMMODEL_ACQUIRE if we should use an acquire atomic to
|
|
read from the root of the queue, otherwise MEMMODEL_RELAXED if we
|
|
should use a plain load. */
|
|
|
|
static inline _Bool
|
|
priority_queue_empty_p (struct priority_queue *head, enum memmodel model)
|
|
{
|
|
/* Note: The acquire barriers on the loads here synchronize with
|
|
the write of a NULL in gomp_task_run_post_remove_parent. It is
|
|
not necessary that we synchronize with other non-NULL writes at
|
|
this point, but we must ensure that all writes to memory by a
|
|
child thread task work function are seen before we exit from
|
|
GOMP_taskwait. */
|
|
if (priority_queue_multi_p (head))
|
|
{
|
|
if (model == MEMMODEL_ACQUIRE)
|
|
return __atomic_load_n (&head->t.root, MEMMODEL_ACQUIRE) == NULL;
|
|
return head->t.root == NULL;
|
|
}
|
|
if (model == MEMMODEL_ACQUIRE)
|
|
return __atomic_load_n (&head->l.tasks, MEMMODEL_ACQUIRE) == NULL;
|
|
return head->l.tasks == NULL;
|
|
}
|
|
|
|
/* Look for a given PRIORITY in HEAD. Return it if found, otherwise
|
|
return NULL. This only applies to the tree variant in HEAD. There
|
|
is no point in searching for priorities in HEAD->L. */
|
|
|
|
static inline struct priority_list *
|
|
priority_queue_lookup_priority (struct priority_queue *head, int priority)
|
|
{
|
|
if (head->t.root == NULL)
|
|
return NULL;
|
|
struct prio_splay_tree_key_s k;
|
|
k.l.priority = priority;
|
|
return (struct priority_list *)
|
|
prio_splay_tree_lookup (&head->t, &k);
|
|
}
|
|
|
|
/* Insert task in DATA, with PRIORITY, in the priority list in LIST.
|
|
LIST contains items of type TYPE.
|
|
|
|
If POS is PRIORITY_INSERT_BEGIN, the new task is inserted at the
|
|
top of its respective priority. If POS is PRIORITY_INSERT_END, the
|
|
task is inserted at the end of its priority.
|
|
|
|
If ADJUST_PARENT_DEPENDS_ON is TRUE, LIST is a children queue, and
|
|
we must keep track of higher and lower priority WAITING tasks by
|
|
keeping the queue's last_parent_depends_on field accurate. This
|
|
only applies to the children queue, and the caller must ensure LIST
|
|
is a children queue in this case.
|
|
|
|
If ADJUST_PARENT_DEPENDS_ON is TRUE, TASK_IS_PARENT_DEPENDS_ON is
|
|
set to the task's parent_depends_on field. If
|
|
ADJUST_PARENT_DEPENDS_ON is FALSE, this field is irrelevant.
|
|
|
|
Return the new priority_node. */
|
|
|
|
static inline void
|
|
priority_list_insert (enum priority_queue_type type,
|
|
struct priority_list *list,
|
|
struct gomp_task *task,
|
|
int priority,
|
|
enum priority_insert_type pos,
|
|
bool adjust_parent_depends_on,
|
|
bool task_is_parent_depends_on)
|
|
{
|
|
struct priority_node *node = task_to_priority_node (type, task);
|
|
if (list->tasks)
|
|
{
|
|
/* If we are keeping track of higher/lower priority items,
|
|
but this is a lower priority WAITING task
|
|
(parent_depends_on != NULL), put it after all ready to
|
|
run tasks. See the comment in
|
|
priority_queue_upgrade_task for a visual on how tasks
|
|
should be organized. */
|
|
if (adjust_parent_depends_on
|
|
&& pos == PRIORITY_INSERT_BEGIN
|
|
&& list->last_parent_depends_on
|
|
&& !task_is_parent_depends_on)
|
|
{
|
|
struct priority_node *last_parent_depends_on
|
|
= list->last_parent_depends_on;
|
|
node->next = last_parent_depends_on->next;
|
|
node->prev = last_parent_depends_on;
|
|
}
|
|
/* Otherwise, put it at the top/bottom of the queue. */
|
|
else
|
|
{
|
|
node->next = list->tasks;
|
|
node->prev = list->tasks->prev;
|
|
if (pos == PRIORITY_INSERT_BEGIN)
|
|
list->tasks = node;
|
|
}
|
|
node->next->prev = node;
|
|
node->prev->next = node;
|
|
}
|
|
else
|
|
{
|
|
node->next = node;
|
|
node->prev = node;
|
|
list->tasks = node;
|
|
}
|
|
if (adjust_parent_depends_on
|
|
&& list->last_parent_depends_on == NULL
|
|
&& task_is_parent_depends_on)
|
|
list->last_parent_depends_on = node;
|
|
}
|
|
|
|
/* Tree version of priority_list_insert. */
|
|
|
|
static inline void
|
|
priority_tree_insert (enum priority_queue_type type,
|
|
struct priority_queue *head,
|
|
struct gomp_task *task,
|
|
int priority,
|
|
enum priority_insert_type pos,
|
|
bool adjust_parent_depends_on,
|
|
bool task_is_parent_depends_on)
|
|
{
|
|
if (__builtin_expect (head->t.root == NULL, 0))
|
|
{
|
|
/* The first time around, transfer any priority 0 items to the
|
|
tree. */
|
|
if (head->l.tasks != NULL)
|
|
{
|
|
prio_splay_tree_node k = gomp_malloc (sizeof (*k));
|
|
k->left = NULL;
|
|
k->right = NULL;
|
|
k->key.l.priority = 0;
|
|
k->key.l.tasks = head->l.tasks;
|
|
k->key.l.last_parent_depends_on = head->l.last_parent_depends_on;
|
|
prio_splay_tree_insert (&head->t, k);
|
|
head->l.tasks = NULL;
|
|
}
|
|
}
|
|
struct priority_list *list
|
|
= priority_queue_lookup_priority (head, priority);
|
|
if (!list)
|
|
{
|
|
prio_splay_tree_node k = gomp_malloc (sizeof (*k));
|
|
k->left = NULL;
|
|
k->right = NULL;
|
|
k->key.l.priority = priority;
|
|
k->key.l.tasks = NULL;
|
|
k->key.l.last_parent_depends_on = NULL;
|
|
prio_splay_tree_insert (&head->t, k);
|
|
list = &k->key.l;
|
|
}
|
|
priority_list_insert (type, list, task, priority, pos,
|
|
adjust_parent_depends_on,
|
|
task_is_parent_depends_on);
|
|
}
|
|
|
|
/* Generic version of priority_*_insert. */
|
|
|
|
static inline void
|
|
priority_queue_insert (enum priority_queue_type type,
|
|
struct priority_queue *head,
|
|
struct gomp_task *task,
|
|
int priority,
|
|
enum priority_insert_type pos,
|
|
bool adjust_parent_depends_on,
|
|
bool task_is_parent_depends_on)
|
|
{
|
|
#if _LIBGOMP_CHECKING_
|
|
if (priority_queue_task_in_queue_p (type, head, task))
|
|
gomp_fatal ("Attempt to insert existing task %p", task);
|
|
#endif
|
|
if (priority_queue_multi_p (head) || __builtin_expect (priority > 0, 0))
|
|
priority_tree_insert (type, head, task, priority, pos,
|
|
adjust_parent_depends_on,
|
|
task_is_parent_depends_on);
|
|
else
|
|
priority_list_insert (type, &head->l, task, priority, pos,
|
|
adjust_parent_depends_on,
|
|
task_is_parent_depends_on);
|
|
}
|
|
|
|
/* If multiple priorities are in play, return the highest priority
|
|
task from within Q1 and Q2, while giving preference to tasks from
|
|
Q1. If the returned task is chosen from Q1, *Q1_CHOSEN_P is set to
|
|
TRUE, otherwise it is set to FALSE.
|
|
|
|
If multiple priorities are not in play (only 0 priorities are
|
|
available), the next task is chosen exclusively from Q1.
|
|
|
|
As a special case, Q2 can be NULL, in which case, we just choose
|
|
the highest priority WAITING task in Q1. This is an optimization
|
|
to speed up looking through only one queue.
|
|
|
|
We assume Q1 has at least one item. */
|
|
|
|
static inline struct gomp_task *
|
|
priority_queue_next_task (enum priority_queue_type t1,
|
|
struct priority_queue *q1,
|
|
enum priority_queue_type t2,
|
|
struct priority_queue *q2,
|
|
bool *q1_chosen_p)
|
|
{
|
|
#if _LIBGOMP_CHECKING_
|
|
if (priority_queue_empty_p (q1, MEMMODEL_RELAXED))
|
|
gomp_fatal ("priority_queue_next_task: Q1 is empty");
|
|
#endif
|
|
if (priority_queue_multi_p (q1))
|
|
{
|
|
struct gomp_task *t
|
|
= priority_tree_next_task (t1, q1, t2, q2, q1_chosen_p);
|
|
/* If T is NULL, there are no WAITING tasks in Q1. In which
|
|
case, return any old (non-waiting) task which will cause the
|
|
caller to do the right thing when checking T->KIND ==
|
|
GOMP_TASK_WAITING. */
|
|
if (!t)
|
|
{
|
|
#if _LIBGOMP_CHECKING_
|
|
if (*q1_chosen_p == false)
|
|
gomp_fatal ("priority_queue_next_task inconsistency");
|
|
#endif
|
|
return priority_node_to_task (t1, q1->t.root->key.l.tasks);
|
|
}
|
|
return t;
|
|
}
|
|
else
|
|
{
|
|
*q1_chosen_p = true;
|
|
return priority_node_to_task (t1, q1->l.tasks);
|
|
}
|
|
}
|
|
|
|
/* Remove NODE from LIST.
|
|
|
|
If we are removing the one and only item in the list, and MODEL is
|
|
MEMMODEL_RELEASE, use an atomic release to clear the list.
|
|
|
|
If the list becomes empty after the remove, return TRUE. */
|
|
|
|
static inline bool
|
|
priority_list_remove (struct priority_list *list,
|
|
struct priority_node *node,
|
|
enum memmodel model)
|
|
{
|
|
bool empty = false;
|
|
node->prev->next = node->next;
|
|
node->next->prev = node->prev;
|
|
if (list->tasks == node)
|
|
{
|
|
if (node->next != node)
|
|
list->tasks = node->next;
|
|
else
|
|
{
|
|
/* We access task->children in GOMP_taskwait outside of
|
|
the task lock mutex region, so need a release barrier
|
|
here to ensure memory written by child_task->fn above
|
|
is flushed before the NULL is written. */
|
|
if (model == MEMMODEL_RELEASE)
|
|
__atomic_store_n (&list->tasks, NULL, MEMMODEL_RELEASE);
|
|
else
|
|
list->tasks = NULL;
|
|
empty = true;
|
|
goto remove_out;
|
|
}
|
|
}
|
|
remove_out:
|
|
#if _LIBGOMP_CHECKING_
|
|
memset (node, 0xaf, sizeof (*node));
|
|
#endif
|
|
return empty;
|
|
}
|
|
|
|
/* This is the generic version of priority_list_remove.
|
|
|
|
Remove NODE from priority queue HEAD. HEAD contains tasks of type TYPE.
|
|
|
|
If we are removing the one and only item in the priority queue and
|
|
MODEL is MEMMODEL_RELEASE, use an atomic release to clear the queue.
|
|
|
|
If the queue becomes empty after the remove, return TRUE. */
|
|
|
|
static inline bool
|
|
priority_queue_remove (enum priority_queue_type type,
|
|
struct priority_queue *head,
|
|
struct gomp_task *task,
|
|
enum memmodel model)
|
|
{
|
|
#if _LIBGOMP_CHECKING_
|
|
if (!priority_queue_task_in_queue_p (type, head, task))
|
|
gomp_fatal ("Attempt to remove missing task %p", task);
|
|
#endif
|
|
if (priority_queue_multi_p (head))
|
|
{
|
|
priority_tree_remove (type, head, task_to_priority_node (type, task));
|
|
if (head->t.root == NULL)
|
|
{
|
|
if (model == MEMMODEL_RELEASE)
|
|
/* Errr, we store NULL twice, the alternative would be to
|
|
use an atomic release directly in the splay tree
|
|
routines. Worth it? */
|
|
__atomic_store_n (&head->t.root, NULL, MEMMODEL_RELEASE);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
else
|
|
return priority_list_remove (&head->l,
|
|
task_to_priority_node (type, task), model);
|
|
}
|
|
|
|
#endif /* _PRIORITY_QUEUE_H_ */
|