123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023 |
- /* Copyright (C) 2005-2022 Free Software Foundation, Inc.
- Contributed by Richard Henderson <rth@redhat.com>.
- This file is part of the GNU Offloading and Multi Processing Library
- (libgomp).
- Libgomp is free software; you can redistribute it and/or modify it
- under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3, or (at your option)
- any later version.
- Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
- Under Section 7 of GPL version 3, you are granted additional
- permissions described in the GCC Runtime Library Exception, version
- 3.1, as published by the Free Software Foundation.
- You should have received a copy of the GNU General Public License and
- a copy of the GCC Runtime Library Exception along with this program;
- see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
- <http://www.gnu.org/licenses/>. */
- /* This file handles the LOOP (FOR/DO) construct. */
- #include <limits.h>
- #include <stdlib.h>
- #include <string.h>
- #include "libgomp.h"
- ialias (GOMP_loop_ull_runtime_next)
- ialias_redirect (GOMP_taskgroup_reduction_register)
- typedef unsigned long long gomp_ull;
- /* Initialize the given work share construct from the given arguments. */
- static inline void
- gomp_loop_ull_init (struct gomp_work_share *ws, bool up, gomp_ull start,
- gomp_ull end, gomp_ull incr, enum gomp_schedule_type sched,
- gomp_ull chunk_size)
- {
- ws->sched = sched;
- ws->chunk_size_ull = chunk_size;
- /* Canonicalize loops that have zero iterations to ->next == ->end. */
- ws->end_ull = ((up && start > end) || (!up && start < end))
- ? start : end;
- ws->incr_ull = incr;
- ws->next_ull = start;
- ws->mode = 0;
- if (sched == GFS_DYNAMIC)
- {
- ws->chunk_size_ull *= incr;
- #if defined HAVE_SYNC_BUILTINS && defined __LP64__
- {
- /* For dynamic scheduling prepare things to make each iteration
- faster. */
- struct gomp_thread *thr = gomp_thread ();
- struct gomp_team *team = thr->ts.team;
- long nthreads = team ? team->nthreads : 1;
- if (__builtin_expect (up, 1))
- {
- /* Cheap overflow protection. */
- if (__builtin_expect ((nthreads | ws->chunk_size_ull)
- < 1ULL << (sizeof (gomp_ull)
- * __CHAR_BIT__ / 2 - 1), 1))
- ws->mode = ws->end_ull < (__LONG_LONG_MAX__ * 2ULL + 1
- - (nthreads + 1) * ws->chunk_size_ull);
- }
- /* Cheap overflow protection. */
- else if (__builtin_expect ((nthreads | -ws->chunk_size_ull)
- < 1ULL << (sizeof (gomp_ull)
- * __CHAR_BIT__ / 2 - 1), 1))
- ws->mode = ws->end_ull > ((nthreads + 1) * -ws->chunk_size_ull
- - (__LONG_LONG_MAX__ * 2ULL + 1));
- }
- #endif
- }
- if (!up)
- ws->mode |= 2;
- }
- /* The *_start routines are called when first encountering a loop construct
- that is not bound directly to a parallel construct. The first thread
- that arrives will create the work-share construct; subsequent threads
- will see the construct exists and allocate work from it.
- START, END, INCR are the bounds of the loop; due to the restrictions of
- OpenMP, these values must be the same in every thread. This is not
- verified (nor is it entirely verifiable, since START is not necessarily
- retained intact in the work-share data structure). CHUNK_SIZE is the
- scheduling parameter; again this must be identical in all threads.
- Returns true if there's any work for this thread to perform. If so,
- *ISTART and *IEND are filled with the bounds of the iteration block
- allocated to this thread. Returns false if all work was assigned to
- other threads prior to this thread's arrival. */
- static bool
- gomp_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (gomp_work_share_start (0))
- {
- gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
- GFS_STATIC, chunk_size);
- gomp_work_share_init_done ();
- }
- return !gomp_iter_ull_static_next (istart, iend);
- }
- static bool
- gomp_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (0))
- {
- gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
- GFS_DYNAMIC, chunk_size);
- gomp_work_share_init_done ();
- }
- #if defined HAVE_SYNC_BUILTINS && defined __LP64__
- ret = gomp_iter_ull_dynamic_next (istart, iend);
- #else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- static bool
- gomp_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (0))
- {
- gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
- GFS_GUIDED, chunk_size);
- gomp_work_share_init_done ();
- }
- #if defined HAVE_SYNC_BUILTINS && defined __LP64__
- ret = gomp_iter_ull_guided_next (istart, iend);
- #else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_guided_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- bool
- GOMP_loop_ull_runtime_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- switch (icv->run_sched_var & ~GFS_MONOTONIC)
- {
- case GFS_STATIC:
- return gomp_loop_ull_static_start (up, start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_ull_dynamic_start (up, start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_GUIDED:
- return gomp_loop_ull_guided_start (up, start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_AUTO:
- /* For now map to schedule(static), later on we could play with feedback
- driven choice. */
- return gomp_loop_ull_static_start (up, start, end, incr,
- 0, istart, iend);
- default:
- abort ();
- }
- }
- static long
- gomp_adjust_sched (long sched, gomp_ull *chunk_size)
- {
- sched &= ~GFS_MONOTONIC;
- switch (sched)
- {
- case GFS_STATIC:
- case GFS_DYNAMIC:
- case GFS_GUIDED:
- return sched;
- /* GFS_RUNTIME is used for runtime schedule without monotonic
- or nonmonotonic modifiers on the clause.
- GFS_RUNTIME|GFS_MONOTONIC for runtime schedule with monotonic
- modifier. */
- case GFS_RUNTIME:
- /* GFS_AUTO is used for runtime schedule with nonmonotonic
- modifier. */
- case GFS_AUTO:
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- sched = icv->run_sched_var & ~GFS_MONOTONIC;
- switch (sched)
- {
- case GFS_STATIC:
- case GFS_DYNAMIC:
- case GFS_GUIDED:
- *chunk_size = icv->run_sched_chunk_size;
- break;
- case GFS_AUTO:
- sched = GFS_STATIC;
- *chunk_size = 0;
- break;
- default:
- abort ();
- }
- return sched;
- }
- default:
- abort ();
- }
- }
- bool
- GOMP_loop_ull_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, long sched, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend,
- uintptr_t *reductions, void **mem)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (reductions)
- gomp_workshare_taskgroup_start ();
- if (gomp_work_share_start (0))
- {
- sched = gomp_adjust_sched (sched, &chunk_size);
- gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
- sched, chunk_size);
- if (reductions)
- {
- GOMP_taskgroup_reduction_register (reductions);
- thr->task->taskgroup->workshare = true;
- thr->ts.work_share->task_reductions = reductions;
- }
- if (mem)
- {
- uintptr_t size = (uintptr_t) *mem;
- #define INLINE_ORDERED_TEAM_IDS_OFF \
- ((offsetof (struct gomp_work_share, inline_ordered_team_ids) \
- + __alignof__ (long long) - 1) & ~(__alignof__ (long long) - 1))
- if (size > (sizeof (struct gomp_work_share)
- - INLINE_ORDERED_TEAM_IDS_OFF))
- *mem
- = (void *) (thr->ts.work_share->ordered_team_ids
- = gomp_malloc_cleared (size));
- else
- *mem = memset (((char *) thr->ts.work_share)
- + INLINE_ORDERED_TEAM_IDS_OFF, '\0', size);
- }
- gomp_work_share_init_done ();
- }
- else
- {
- if (reductions)
- {
- uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
- gomp_workshare_task_reduction_register (reductions,
- first_reductions);
- }
- if (mem)
- {
- if ((offsetof (struct gomp_work_share, inline_ordered_team_ids)
- & (__alignof__ (long long) - 1)) == 0)
- *mem = (void *) thr->ts.work_share->ordered_team_ids;
- else
- {
- uintptr_t p = (uintptr_t) thr->ts.work_share->ordered_team_ids;
- p += __alignof__ (long long) - 1;
- p &= ~(__alignof__ (long long) - 1);
- *mem = (void *) p;
- }
- }
- }
- return ialias_call (GOMP_loop_ull_runtime_next) (istart, iend);
- }
- /* The *_ordered_*_start routines are similar. The only difference is that
- this work-share construct is initialized to expect an ORDERED section. */
- static bool
- gomp_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (gomp_work_share_start (1))
- {
- gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
- GFS_STATIC, chunk_size);
- gomp_ordered_static_init ();
- gomp_work_share_init_done ();
- }
- return !gomp_iter_ull_static_next (istart, iend);
- }
- static bool
- gomp_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (1))
- {
- gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
- GFS_DYNAMIC, chunk_size);
- gomp_mutex_lock (&thr->ts.work_share->lock);
- gomp_work_share_init_done ();
- }
- else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
- if (ret)
- gomp_ordered_first ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- static bool
- gomp_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (1))
- {
- gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
- GFS_GUIDED, chunk_size);
- gomp_mutex_lock (&thr->ts.work_share->lock);
- gomp_work_share_init_done ();
- }
- else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_guided_next_locked (istart, iend);
- if (ret)
- gomp_ordered_first ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- bool
- GOMP_loop_ull_ordered_runtime_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull *istart,
- gomp_ull *iend)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- switch (icv->run_sched_var & ~GFS_MONOTONIC)
- {
- case GFS_STATIC:
- return gomp_loop_ull_ordered_static_start (up, start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_GUIDED:
- return gomp_loop_ull_ordered_guided_start (up, start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_AUTO:
- /* For now map to schedule(static), later on we could play with feedback
- driven choice. */
- return gomp_loop_ull_ordered_static_start (up, start, end, incr,
- 0, istart, iend);
- default:
- abort ();
- }
- }
- bool
- GOMP_loop_ull_ordered_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, long sched, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend,
- uintptr_t *reductions, void **mem)
- {
- struct gomp_thread *thr = gomp_thread ();
- size_t ordered = 1;
- bool ret;
- thr->ts.static_trip = 0;
- if (reductions)
- gomp_workshare_taskgroup_start ();
- if (mem)
- ordered += (uintptr_t) *mem;
- if (gomp_work_share_start (ordered))
- {
- sched = gomp_adjust_sched (sched, &chunk_size);
- gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
- sched, chunk_size);
- if (reductions)
- {
- GOMP_taskgroup_reduction_register (reductions);
- thr->task->taskgroup->workshare = true;
- thr->ts.work_share->task_reductions = reductions;
- }
- if (sched == GFS_STATIC)
- gomp_ordered_static_init ();
- else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- gomp_work_share_init_done ();
- }
- else
- {
- if (reductions)
- {
- uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
- gomp_workshare_task_reduction_register (reductions,
- first_reductions);
- }
- sched = thr->ts.work_share->sched;
- if (sched != GFS_STATIC)
- gomp_mutex_lock (&thr->ts.work_share->lock);
- }
- if (mem)
- {
- uintptr_t p
- = (uintptr_t) (thr->ts.work_share->ordered_team_ids
- + (thr->ts.team ? thr->ts.team->nthreads : 1));
- p += __alignof__ (long long) - 1;
- p &= ~(__alignof__ (long long) - 1);
- *mem = (void *) p;
- }
- switch (sched)
- {
- case GFS_STATIC:
- case GFS_AUTO:
- return !gomp_iter_ull_static_next (istart, iend);
- case GFS_DYNAMIC:
- ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
- break;
- case GFS_GUIDED:
- ret = gomp_iter_ull_guided_next_locked (istart, iend);
- break;
- default:
- abort ();
- }
- if (ret)
- gomp_ordered_first ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- /* The *_doacross_*_start routines are similar. The only difference is that
- this work-share construct is initialized to expect an ORDERED(N) - DOACROSS
- section, and the worksharing loop iterates always from 0 to COUNTS[0] - 1
- and other COUNTS array elements tell the library number of iterations
- in the ordered inner loops. */
- static bool
- gomp_loop_ull_doacross_static_start (unsigned ncounts, gomp_ull *counts,
- gomp_ull chunk_size, gomp_ull *istart,
- gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (gomp_work_share_start (0))
- {
- gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
- GFS_STATIC, chunk_size);
- gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
- gomp_work_share_init_done ();
- }
- return !gomp_iter_ull_static_next (istart, iend);
- }
- static bool
- gomp_loop_ull_doacross_dynamic_start (unsigned ncounts, gomp_ull *counts,
- gomp_ull chunk_size, gomp_ull *istart,
- gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (0))
- {
- gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
- GFS_DYNAMIC, chunk_size);
- gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
- gomp_work_share_init_done ();
- }
- #if defined HAVE_SYNC_BUILTINS && defined __LP64__
- ret = gomp_iter_ull_dynamic_next (istart, iend);
- #else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- static bool
- gomp_loop_ull_doacross_guided_start (unsigned ncounts, gomp_ull *counts,
- gomp_ull chunk_size, gomp_ull *istart,
- gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (0))
- {
- gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
- GFS_GUIDED, chunk_size);
- gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
- gomp_work_share_init_done ();
- }
- #if defined HAVE_SYNC_BUILTINS && defined __LP64__
- ret = gomp_iter_ull_guided_next (istart, iend);
- #else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_guided_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- bool
- GOMP_loop_ull_doacross_runtime_start (unsigned ncounts, gomp_ull *counts,
- gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- switch (icv->run_sched_var & ~GFS_MONOTONIC)
- {
- case GFS_STATIC:
- return gomp_loop_ull_doacross_static_start (ncounts, counts,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_ull_doacross_dynamic_start (ncounts, counts,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_GUIDED:
- return gomp_loop_ull_doacross_guided_start (ncounts, counts,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_AUTO:
- /* For now map to schedule(static), later on we could play with feedback
- driven choice. */
- return gomp_loop_ull_doacross_static_start (ncounts, counts,
- 0, istart, iend);
- default:
- abort ();
- }
- }
- bool
- GOMP_loop_ull_doacross_start (unsigned ncounts, gomp_ull *counts,
- long sched, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend,
- uintptr_t *reductions, void **mem)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (reductions)
- gomp_workshare_taskgroup_start ();
- if (gomp_work_share_start (0))
- {
- size_t extra = 0;
- if (mem)
- extra = (uintptr_t) *mem;
- sched = gomp_adjust_sched (sched, &chunk_size);
- gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
- sched, chunk_size);
- gomp_doacross_ull_init (ncounts, counts, chunk_size, extra);
- if (reductions)
- {
- GOMP_taskgroup_reduction_register (reductions);
- thr->task->taskgroup->workshare = true;
- thr->ts.work_share->task_reductions = reductions;
- }
- gomp_work_share_init_done ();
- }
- else
- {
- if (reductions)
- {
- uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
- gomp_workshare_task_reduction_register (reductions,
- first_reductions);
- }
- sched = thr->ts.work_share->sched;
- }
- if (mem)
- *mem = thr->ts.work_share->doacross->extra;
- return ialias_call (GOMP_loop_ull_runtime_next) (istart, iend);
- }
- /* The *_next routines are called when the thread completes processing of
- the iteration block currently assigned to it. If the work-share
- construct is bound directly to a parallel construct, then the iteration
- bounds may have been set up before the parallel. In which case, this
- may be the first iteration for the thread.
- Returns true if there is work remaining to be performed; *ISTART and
- *IEND are filled with a new iteration block. Returns false if all work
- has been assigned. */
- static bool
- gomp_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
- {
- return !gomp_iter_ull_static_next (istart, iend);
- }
- static bool
- gomp_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
- {
- bool ret;
- #if defined HAVE_SYNC_BUILTINS && defined __LP64__
- ret = gomp_iter_ull_dynamic_next (istart, iend);
- #else
- struct gomp_thread *thr = gomp_thread ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- static bool
- gomp_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
- {
- bool ret;
- #if defined HAVE_SYNC_BUILTINS && defined __LP64__
- ret = gomp_iter_ull_guided_next (istart, iend);
- #else
- struct gomp_thread *thr = gomp_thread ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_guided_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- bool
- GOMP_loop_ull_runtime_next (gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- switch (thr->ts.work_share->sched)
- {
- case GFS_STATIC:
- case GFS_AUTO:
- return gomp_loop_ull_static_next (istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_ull_dynamic_next (istart, iend);
- case GFS_GUIDED:
- return gomp_loop_ull_guided_next (istart, iend);
- default:
- abort ();
- }
- }
- /* The *_ordered_*_next routines are called when the thread completes
- processing of the iteration block currently assigned to it.
- Returns true if there is work remaining to be performed; *ISTART and
- *IEND are filled with a new iteration block. Returns false if all work
- has been assigned. */
- static bool
- gomp_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- int test;
- gomp_ordered_sync ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- test = gomp_iter_ull_static_next (istart, iend);
- if (test >= 0)
- gomp_ordered_static_next ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return test == 0;
- }
- static bool
- gomp_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- gomp_ordered_sync ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
- if (ret)
- gomp_ordered_next ();
- else
- gomp_ordered_last ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- static bool
- gomp_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- gomp_ordered_sync ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_ull_guided_next_locked (istart, iend);
- if (ret)
- gomp_ordered_next ();
- else
- gomp_ordered_last ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- bool
- GOMP_loop_ull_ordered_runtime_next (gomp_ull *istart, gomp_ull *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- switch (thr->ts.work_share->sched)
- {
- case GFS_STATIC:
- case GFS_AUTO:
- return gomp_loop_ull_ordered_static_next (istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_ull_ordered_dynamic_next (istart, iend);
- case GFS_GUIDED:
- return gomp_loop_ull_ordered_guided_next (istart, iend);
- default:
- abort ();
- }
- }
- /* We use static functions above so that we're sure that the "runtime"
- function can defer to the proper routine without interposition. We
- export the static function with a strong alias when possible, or with
- a wrapper function otherwise. */
- #ifdef HAVE_ATTRIBUTE_ALIAS
- extern __typeof(gomp_loop_ull_static_start) GOMP_loop_ull_static_start
- __attribute__((alias ("gomp_loop_ull_static_start")));
- extern __typeof(gomp_loop_ull_dynamic_start) GOMP_loop_ull_dynamic_start
- __attribute__((alias ("gomp_loop_ull_dynamic_start")));
- extern __typeof(gomp_loop_ull_guided_start) GOMP_loop_ull_guided_start
- __attribute__((alias ("gomp_loop_ull_guided_start")));
- extern __typeof(gomp_loop_ull_dynamic_start) GOMP_loop_ull_nonmonotonic_dynamic_start
- __attribute__((alias ("gomp_loop_ull_dynamic_start")));
- extern __typeof(gomp_loop_ull_guided_start) GOMP_loop_ull_nonmonotonic_guided_start
- __attribute__((alias ("gomp_loop_ull_guided_start")));
- extern __typeof(GOMP_loop_ull_runtime_start) GOMP_loop_ull_nonmonotonic_runtime_start
- __attribute__((alias ("GOMP_loop_ull_runtime_start")));
- extern __typeof(GOMP_loop_ull_runtime_start) GOMP_loop_ull_maybe_nonmonotonic_runtime_start
- __attribute__((alias ("GOMP_loop_ull_runtime_start")));
- extern __typeof(gomp_loop_ull_ordered_static_start) GOMP_loop_ull_ordered_static_start
- __attribute__((alias ("gomp_loop_ull_ordered_static_start")));
- extern __typeof(gomp_loop_ull_ordered_dynamic_start) GOMP_loop_ull_ordered_dynamic_start
- __attribute__((alias ("gomp_loop_ull_ordered_dynamic_start")));
- extern __typeof(gomp_loop_ull_ordered_guided_start) GOMP_loop_ull_ordered_guided_start
- __attribute__((alias ("gomp_loop_ull_ordered_guided_start")));
- extern __typeof(gomp_loop_ull_doacross_static_start) GOMP_loop_ull_doacross_static_start
- __attribute__((alias ("gomp_loop_ull_doacross_static_start")));
- extern __typeof(gomp_loop_ull_doacross_dynamic_start) GOMP_loop_ull_doacross_dynamic_start
- __attribute__((alias ("gomp_loop_ull_doacross_dynamic_start")));
- extern __typeof(gomp_loop_ull_doacross_guided_start) GOMP_loop_ull_doacross_guided_start
- __attribute__((alias ("gomp_loop_ull_doacross_guided_start")));
- extern __typeof(gomp_loop_ull_static_next) GOMP_loop_ull_static_next
- __attribute__((alias ("gomp_loop_ull_static_next")));
- extern __typeof(gomp_loop_ull_dynamic_next) GOMP_loop_ull_dynamic_next
- __attribute__((alias ("gomp_loop_ull_dynamic_next")));
- extern __typeof(gomp_loop_ull_guided_next) GOMP_loop_ull_guided_next
- __attribute__((alias ("gomp_loop_ull_guided_next")));
- extern __typeof(gomp_loop_ull_dynamic_next) GOMP_loop_ull_nonmonotonic_dynamic_next
- __attribute__((alias ("gomp_loop_ull_dynamic_next")));
- extern __typeof(gomp_loop_ull_guided_next) GOMP_loop_ull_nonmonotonic_guided_next
- __attribute__((alias ("gomp_loop_ull_guided_next")));
- extern __typeof(GOMP_loop_ull_runtime_next) GOMP_loop_ull_nonmonotonic_runtime_next
- __attribute__((alias ("GOMP_loop_ull_runtime_next")));
- extern __typeof(GOMP_loop_ull_runtime_next) GOMP_loop_ull_maybe_nonmonotonic_runtime_next
- __attribute__((alias ("GOMP_loop_ull_runtime_next")));
- extern __typeof(gomp_loop_ull_ordered_static_next) GOMP_loop_ull_ordered_static_next
- __attribute__((alias ("gomp_loop_ull_ordered_static_next")));
- extern __typeof(gomp_loop_ull_ordered_dynamic_next) GOMP_loop_ull_ordered_dynamic_next
- __attribute__((alias ("gomp_loop_ull_ordered_dynamic_next")));
- extern __typeof(gomp_loop_ull_ordered_guided_next) GOMP_loop_ull_ordered_guided_next
- __attribute__((alias ("gomp_loop_ull_ordered_guided_next")));
- #else
- bool
- GOMP_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_static_start (up, start, end, incr, chunk_size, istart,
- iend);
- }
- bool
- GOMP_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_dynamic_start (up, start, end, incr, chunk_size, istart,
- iend);
- }
- bool
- GOMP_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_guided_start (up, start, end, incr, chunk_size, istart,
- iend);
- }
- bool
- GOMP_loop_ull_nonmonotonic_dynamic_start (bool up, gomp_ull start,
- gomp_ull end, gomp_ull incr,
- gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_dynamic_start (up, start, end, incr, chunk_size, istart,
- iend);
- }
- bool
- GOMP_loop_ull_nonmonotonic_guided_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_guided_start (up, start, end, incr, chunk_size, istart,
- iend);
- }
- bool
- GOMP_loop_ull_nonmonotonic_runtime_start (bool up, gomp_ull start,
- gomp_ull end, gomp_ull incr,
- gomp_ull *istart, gomp_ull *iend)
- {
- return GOMP_loop_ull_runtime_start (up, start, end, incr, istart, iend);
- }
- bool
- GOMP_loop_ull_maybe_nonmonotonic_runtime_start (bool up, gomp_ull start,
- gomp_ull end, gomp_ull incr,
- gomp_ull *istart,
- gomp_ull *iend)
- {
- return GOMP_loop_ull_runtime_start (up, start, end, incr, istart, iend);
- }
- bool
- GOMP_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_ordered_static_start (up, start, end, incr, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
- gomp_ull incr, gomp_ull chunk_size,
- gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_ordered_guided_start (up, start, end, incr, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_ull_doacross_static_start (unsigned ncounts, gomp_ull *counts,
- gomp_ull chunk_size, gomp_ull *istart,
- gomp_ull *iend)
- {
- return gomp_loop_ull_doacross_static_start (ncounts, counts, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_ull_doacross_dynamic_start (unsigned ncounts, gomp_ull *counts,
- gomp_ull chunk_size, gomp_ull *istart,
- gomp_ull *iend)
- {
- return gomp_loop_ull_doacross_dynamic_start (ncounts, counts, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_ull_doacross_guided_start (unsigned ncounts, gomp_ull *counts,
- gomp_ull chunk_size, gomp_ull *istart,
- gomp_ull *iend)
- {
- return gomp_loop_ull_doacross_guided_start (ncounts, counts, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_static_next (istart, iend);
- }
- bool
- GOMP_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_dynamic_next (istart, iend);
- }
- bool
- GOMP_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_guided_next (istart, iend);
- }
- bool
- GOMP_loop_ull_nonmonotonic_dynamic_next (gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_dynamic_next (istart, iend);
- }
- bool
- GOMP_loop_ull_nonmonotonic_guided_next (gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_guided_next (istart, iend);
- }
- bool
- GOMP_loop_ull_nonmonotonic_runtime_next (gomp_ull *istart, gomp_ull *iend)
- {
- return GOMP_loop_ull_runtime_next (istart, iend);
- }
- bool
- GOMP_loop_ull_maybe_nonmonotonic_runtime_next (gomp_ull *istart,
- gomp_ull *iend)
- {
- return GOMP_loop_ull_runtime_next (istart, iend);
- }
- bool
- GOMP_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_ordered_static_next (istart, iend);
- }
- bool
- GOMP_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_ordered_dynamic_next (istart, iend);
- }
- bool
- GOMP_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
- {
- return gomp_loop_ull_ordered_guided_next (istart, iend);
- }
- #endif
|