167 lines
4.7 KiB
C
167 lines
4.7 KiB
C
|
/* Copyright (C) 2015-2022 Free Software Foundation, Inc.
|
||
|
Contributed by Alexander Monakov <amonakov@ispras.ru>
|
||
|
|
||
|
This file is part of the GNU Offloading and Multi Processing Library
|
||
|
(libgomp).
|
||
|
|
||
|
Libgomp is free software; you can redistribute it and/or modify it
|
||
|
under the terms of the GNU General Public License as published by
|
||
|
the Free Software Foundation; either version 3, or (at your option)
|
||
|
any later version.
|
||
|
|
||
|
Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
|
||
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||
|
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||
|
more details.
|
||
|
|
||
|
Under Section 7 of GPL version 3, you are granted additional
|
||
|
permissions described in the GCC Runtime Library Exception, version
|
||
|
3.1, as published by the Free Software Foundation.
|
||
|
|
||
|
You should have received a copy of the GNU General Public License and
|
||
|
a copy of the GCC Runtime Library Exception along with this program;
|
||
|
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
||
|
<http://www.gnu.org/licenses/>. */
|
||
|
|
||
|
/* This is an NVPTX specific implementation of a barrier synchronization
|
||
|
mechanism for libgomp. This type is private to the library. This
|
||
|
implementation uses atomic instructions and bar.sync instruction. */
|
||
|
|
||
|
#include <limits.h>
|
||
|
#include "libgomp.h"
|
||
|
|
||
|
/* For cpu_relax. */
|
||
|
#include "doacross.h"
|
||
|
|
||
|
/* Assuming ADDR is &bar->generation, return bar. Copied from
|
||
|
rtems/bar.c. */
|
||
|
|
||
|
static gomp_barrier_t *
|
||
|
generation_to_barrier (int *addr)
|
||
|
{
|
||
|
char *bar
|
||
|
= (char *) addr - __builtin_offsetof (gomp_barrier_t, generation);
|
||
|
return (gomp_barrier_t *)bar;
|
||
|
}
|
||
|
|
||
|
/* Implement futex_wait-like behaviour to plug into the linux/bar.c
|
||
|
implementation. Assumes ADDR is &bar->generation. */
|
||
|
|
||
|
static inline void
|
||
|
futex_wait (int *addr, int val)
|
||
|
{
|
||
|
gomp_barrier_t *bar = generation_to_barrier (addr);
|
||
|
|
||
|
if (bar->total < 2)
|
||
|
/* A barrier with less than two threads, nop. */
|
||
|
return;
|
||
|
|
||
|
gomp_mutex_lock (&bar->lock);
|
||
|
|
||
|
/* Futex semantics: only go to sleep if *addr == val. */
|
||
|
if (__builtin_expect (__atomic_load_n (addr, MEMMODEL_ACQUIRE) != val, 0))
|
||
|
{
|
||
|
gomp_mutex_unlock (&bar->lock);
|
||
|
return;
|
||
|
}
|
||
|
|
||
|
/* Register as waiter. */
|
||
|
unsigned int waiters
|
||
|
= __atomic_add_fetch (&bar->waiters, 1, MEMMODEL_ACQ_REL);
|
||
|
if (waiters == 0)
|
||
|
__builtin_abort ();
|
||
|
unsigned int waiter_id = waiters;
|
||
|
|
||
|
if (waiters > 1)
|
||
|
{
|
||
|
/* Wake other threads in bar.sync. */
|
||
|
asm volatile ("bar.sync 1, %0;" : : "r" (32 * waiters));
|
||
|
|
||
|
/* Ensure that they have updated waiters. */
|
||
|
asm volatile ("bar.sync 1, %0;" : : "r" (32 * waiters));
|
||
|
}
|
||
|
|
||
|
gomp_mutex_unlock (&bar->lock);
|
||
|
|
||
|
while (1)
|
||
|
{
|
||
|
/* Wait for next thread in barrier. */
|
||
|
asm volatile ("bar.sync 1, %0;" : : "r" (32 * (waiters + 1)));
|
||
|
|
||
|
/* Get updated waiters. */
|
||
|
unsigned int updated_waiters
|
||
|
= __atomic_load_n (&bar->waiters, MEMMODEL_ACQUIRE);
|
||
|
|
||
|
/* Notify that we have updated waiters. */
|
||
|
asm volatile ("bar.sync 1, %0;" : : "r" (32 * (waiters + 1)));
|
||
|
|
||
|
waiters = updated_waiters;
|
||
|
|
||
|
if (waiter_id > waiters)
|
||
|
/* A wake happened, and we're in the group of woken threads. */
|
||
|
break;
|
||
|
|
||
|
/* Continue waiting. */
|
||
|
}
|
||
|
}
|
||
|
|
||
|
/* Implement futex_wake-like behaviour to plug into the linux/bar.c
|
||
|
implementation. Assumes ADDR is &bar->generation. */
|
||
|
|
||
|
static inline void
|
||
|
futex_wake (int *addr, int count)
|
||
|
{
|
||
|
gomp_barrier_t *bar = generation_to_barrier (addr);
|
||
|
|
||
|
if (bar->total < 2)
|
||
|
/* A barrier with less than two threads, nop. */
|
||
|
return;
|
||
|
|
||
|
gomp_mutex_lock (&bar->lock);
|
||
|
unsigned int waiters = __atomic_load_n (&bar->waiters, MEMMODEL_ACQUIRE);
|
||
|
if (waiters == 0)
|
||
|
{
|
||
|
/* No threads to wake. */
|
||
|
gomp_mutex_unlock (&bar->lock);
|
||
|
return;
|
||
|
}
|
||
|
|
||
|
if (count == INT_MAX)
|
||
|
/* Release all threads. */
|
||
|
__atomic_store_n (&bar->waiters, 0, MEMMODEL_RELEASE);
|
||
|
else if (count < bar->total)
|
||
|
/* Release count threads. */
|
||
|
__atomic_add_fetch (&bar->waiters, -count, MEMMODEL_ACQ_REL);
|
||
|
else
|
||
|
/* Count has an illegal value. */
|
||
|
__builtin_abort ();
|
||
|
|
||
|
/* Wake other threads in bar.sync. */
|
||
|
asm volatile ("bar.sync 1, %0;" : : "r" (32 * (waiters + 1)));
|
||
|
|
||
|
/* Let them get the updated waiters. */
|
||
|
asm volatile ("bar.sync 1, %0;" : : "r" (32 * (waiters + 1)));
|
||
|
|
||
|
gomp_mutex_unlock (&bar->lock);
|
||
|
}
|
||
|
|
||
|
/* Copied from linux/wait.h. */
|
||
|
|
||
|
static inline int do_spin (int *addr, int val)
|
||
|
{
|
||
|
/* The current implementation doesn't spin. */
|
||
|
return 1;
|
||
|
}
|
||
|
|
||
|
/* Copied from linux/wait.h. */
|
||
|
|
||
|
static inline void do_wait (int *addr, int val)
|
||
|
{
|
||
|
if (do_spin (addr, val))
|
||
|
futex_wait (addr, val);
|
||
|
}
|
||
|
|
||
|
/* Reuse the linux implementation. */
|
||
|
#define GOMP_WAIT_H 1
|
||
|
#include "../linux/bar.c"
|