123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185 |
- /* Copyright (C) 2005-2022 Free Software Foundation, Inc.
- Contributed by Richard Henderson <rth@redhat.com>.
- This file is part of the GNU Offloading and Multi Processing Library
- (libgomp).
- Libgomp is free software; you can redistribute it and/or modify it
- under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3, or (at your option)
- any later version.
- Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- more details.
- Under Section 7 of GPL version 3, you are granted additional
- permissions described in the GCC Runtime Library Exception, version
- 3.1, as published by the Free Software Foundation.
- You should have received a copy of the GNU General Public License and
- a copy of the GCC Runtime Library Exception along with this program;
- see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
- <http://www.gnu.org/licenses/>. */
- /* This file handles the LOOP (FOR/DO) construct. */
- #include <limits.h>
- #include <stdlib.h>
- #include <string.h>
- #include "libgomp.h"
- ialias (GOMP_loop_runtime_next)
- ialias_redirect (GOMP_taskgroup_reduction_register)
- /* Initialize the given work share construct from the given arguments. */
- static inline void
- gomp_loop_init (struct gomp_work_share *ws, long start, long end, long incr,
- enum gomp_schedule_type sched, long chunk_size)
- {
- ws->sched = sched;
- ws->chunk_size = chunk_size;
- /* Canonicalize loops that have zero iterations to ->next == ->end. */
- ws->end = ((incr > 0 && start > end) || (incr < 0 && start < end))
- ? start : end;
- ws->incr = incr;
- ws->next = start;
- if (sched == GFS_DYNAMIC)
- {
- ws->chunk_size *= incr;
- #ifdef HAVE_SYNC_BUILTINS
- {
- /* For dynamic scheduling prepare things to make each iteration
- faster. */
- struct gomp_thread *thr = gomp_thread ();
- struct gomp_team *team = thr->ts.team;
- long nthreads = team ? team->nthreads : 1;
- if (__builtin_expect (incr > 0, 1))
- {
- /* Cheap overflow protection. */
- if (__builtin_expect ((nthreads | ws->chunk_size)
- >= 1UL << (sizeof (long)
- * __CHAR_BIT__ / 2 - 1), 0))
- ws->mode = 0;
- else
- ws->mode = ws->end < (LONG_MAX
- - (nthreads + 1) * ws->chunk_size);
- }
- /* Cheap overflow protection. */
- else if (__builtin_expect ((nthreads | -ws->chunk_size)
- >= 1UL << (sizeof (long)
- * __CHAR_BIT__ / 2 - 1), 0))
- ws->mode = 0;
- else
- ws->mode = ws->end > (nthreads + 1) * -ws->chunk_size - LONG_MAX;
- }
- #endif
- }
- }
- /* The *_start routines are called when first encountering a loop construct
- that is not bound directly to a parallel construct. The first thread
- that arrives will create the work-share construct; subsequent threads
- will see the construct exists and allocate work from it.
- START, END, INCR are the bounds of the loop; due to the restrictions of
- OpenMP, these values must be the same in every thread. This is not
- verified (nor is it entirely verifiable, since START is not necessarily
- retained intact in the work-share data structure). CHUNK_SIZE is the
- scheduling parameter; again this must be identical in all threads.
- Returns true if there's any work for this thread to perform. If so,
- *ISTART and *IEND are filled with the bounds of the iteration block
- allocated to this thread. Returns false if all work was assigned to
- other threads prior to this thread's arrival. */
- static bool
- gomp_loop_static_start (long start, long end, long incr, long chunk_size,
- long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (gomp_work_share_start (0))
- {
- gomp_loop_init (thr->ts.work_share, start, end, incr,
- GFS_STATIC, chunk_size);
- gomp_work_share_init_done ();
- }
- return !gomp_iter_static_next (istart, iend);
- }
- /* The current dynamic implementation is always monotonic. The
- entrypoints without nonmonotonic in them have to be always monotonic,
- but the nonmonotonic ones could be changed to use work-stealing for
- improved scalability. */
- static bool
- gomp_loop_dynamic_start (long start, long end, long incr, long chunk_size,
- long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (0))
- {
- gomp_loop_init (thr->ts.work_share, start, end, incr,
- GFS_DYNAMIC, chunk_size);
- gomp_work_share_init_done ();
- }
- #ifdef HAVE_SYNC_BUILTINS
- ret = gomp_iter_dynamic_next (istart, iend);
- #else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_dynamic_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- /* Similarly as for dynamic, though the question is how can the chunk sizes
- be decreased without a central locking or atomics. */
- static bool
- gomp_loop_guided_start (long start, long end, long incr, long chunk_size,
- long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (0))
- {
- gomp_loop_init (thr->ts.work_share, start, end, incr,
- GFS_GUIDED, chunk_size);
- gomp_work_share_init_done ();
- }
- #ifdef HAVE_SYNC_BUILTINS
- ret = gomp_iter_guided_next (istart, iend);
- #else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_guided_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- bool
- GOMP_loop_runtime_start (long start, long end, long incr,
- long *istart, long *iend)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- switch (icv->run_sched_var & ~GFS_MONOTONIC)
- {
- case GFS_STATIC:
- return gomp_loop_static_start (start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_dynamic_start (start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_GUIDED:
- return gomp_loop_guided_start (start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_AUTO:
- /* For now map to schedule(static), later on we could play with feedback
- driven choice. */
- return gomp_loop_static_start (start, end, incr, 0, istart, iend);
- default:
- abort ();
- }
- }
- static long
- gomp_adjust_sched (long sched, long *chunk_size)
- {
- sched &= ~GFS_MONOTONIC;
- switch (sched)
- {
- case GFS_STATIC:
- case GFS_DYNAMIC:
- case GFS_GUIDED:
- return sched;
- /* GFS_RUNTIME is used for runtime schedule without monotonic
- or nonmonotonic modifiers on the clause.
- GFS_RUNTIME|GFS_MONOTONIC for runtime schedule with monotonic
- modifier. */
- case GFS_RUNTIME:
- /* GFS_AUTO is used for runtime schedule with nonmonotonic
- modifier. */
- case GFS_AUTO:
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- sched = icv->run_sched_var & ~GFS_MONOTONIC;
- switch (sched)
- {
- case GFS_STATIC:
- case GFS_DYNAMIC:
- case GFS_GUIDED:
- *chunk_size = icv->run_sched_chunk_size;
- break;
- case GFS_AUTO:
- sched = GFS_STATIC;
- *chunk_size = 0;
- break;
- default:
- abort ();
- }
- return sched;
- }
- default:
- abort ();
- }
- }
- bool
- GOMP_loop_start (long start, long end, long incr, long sched,
- long chunk_size, long *istart, long *iend,
- uintptr_t *reductions, void **mem)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (reductions)
- gomp_workshare_taskgroup_start ();
- if (gomp_work_share_start (0))
- {
- sched = gomp_adjust_sched (sched, &chunk_size);
- gomp_loop_init (thr->ts.work_share, start, end, incr,
- sched, chunk_size);
- if (reductions)
- {
- GOMP_taskgroup_reduction_register (reductions);
- thr->task->taskgroup->workshare = true;
- thr->ts.work_share->task_reductions = reductions;
- }
- if (mem)
- {
- uintptr_t size = (uintptr_t) *mem;
- #define INLINE_ORDERED_TEAM_IDS_OFF \
- ((offsetof (struct gomp_work_share, inline_ordered_team_ids) \
- + __alignof__ (long long) - 1) & ~(__alignof__ (long long) - 1))
- if (size > (sizeof (struct gomp_work_share)
- - INLINE_ORDERED_TEAM_IDS_OFF))
- *mem
- = (void *) (thr->ts.work_share->ordered_team_ids
- = gomp_malloc_cleared (size));
- else
- *mem = memset (((char *) thr->ts.work_share)
- + INLINE_ORDERED_TEAM_IDS_OFF, '\0', size);
- }
- gomp_work_share_init_done ();
- }
- else
- {
- if (reductions)
- {
- uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
- gomp_workshare_task_reduction_register (reductions,
- first_reductions);
- }
- if (mem)
- {
- if ((offsetof (struct gomp_work_share, inline_ordered_team_ids)
- & (__alignof__ (long long) - 1)) == 0)
- *mem = (void *) thr->ts.work_share->ordered_team_ids;
- else
- {
- uintptr_t p = (uintptr_t) thr->ts.work_share->ordered_team_ids;
- p += __alignof__ (long long) - 1;
- p &= ~(__alignof__ (long long) - 1);
- *mem = (void *) p;
- }
- }
- }
- if (!istart)
- return true;
- return ialias_call (GOMP_loop_runtime_next) (istart, iend);
- }
- /* The *_ordered_*_start routines are similar. The only difference is that
- this work-share construct is initialized to expect an ORDERED section. */
- static bool
- gomp_loop_ordered_static_start (long start, long end, long incr,
- long chunk_size, long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (gomp_work_share_start (1))
- {
- gomp_loop_init (thr->ts.work_share, start, end, incr,
- GFS_STATIC, chunk_size);
- gomp_ordered_static_init ();
- gomp_work_share_init_done ();
- }
- return !gomp_iter_static_next (istart, iend);
- }
- static bool
- gomp_loop_ordered_dynamic_start (long start, long end, long incr,
- long chunk_size, long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (1))
- {
- gomp_loop_init (thr->ts.work_share, start, end, incr,
- GFS_DYNAMIC, chunk_size);
- gomp_mutex_lock (&thr->ts.work_share->lock);
- gomp_work_share_init_done ();
- }
- else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_dynamic_next_locked (istart, iend);
- if (ret)
- gomp_ordered_first ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- static bool
- gomp_loop_ordered_guided_start (long start, long end, long incr,
- long chunk_size, long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (1))
- {
- gomp_loop_init (thr->ts.work_share, start, end, incr,
- GFS_GUIDED, chunk_size);
- gomp_mutex_lock (&thr->ts.work_share->lock);
- gomp_work_share_init_done ();
- }
- else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_guided_next_locked (istart, iend);
- if (ret)
- gomp_ordered_first ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- bool
- GOMP_loop_ordered_runtime_start (long start, long end, long incr,
- long *istart, long *iend)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- switch (icv->run_sched_var & ~GFS_MONOTONIC)
- {
- case GFS_STATIC:
- return gomp_loop_ordered_static_start (start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_ordered_dynamic_start (start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_GUIDED:
- return gomp_loop_ordered_guided_start (start, end, incr,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_AUTO:
- /* For now map to schedule(static), later on we could play with feedback
- driven choice. */
- return gomp_loop_ordered_static_start (start, end, incr,
- 0, istart, iend);
- default:
- abort ();
- }
- }
- bool
- GOMP_loop_ordered_start (long start, long end, long incr, long sched,
- long chunk_size, long *istart, long *iend,
- uintptr_t *reductions, void **mem)
- {
- struct gomp_thread *thr = gomp_thread ();
- size_t ordered = 1;
- bool ret;
- thr->ts.static_trip = 0;
- if (reductions)
- gomp_workshare_taskgroup_start ();
- if (mem)
- ordered += (uintptr_t) *mem;
- if (gomp_work_share_start (ordered))
- {
- sched = gomp_adjust_sched (sched, &chunk_size);
- gomp_loop_init (thr->ts.work_share, start, end, incr,
- sched, chunk_size);
- if (reductions)
- {
- GOMP_taskgroup_reduction_register (reductions);
- thr->task->taskgroup->workshare = true;
- thr->ts.work_share->task_reductions = reductions;
- }
- if (sched == GFS_STATIC)
- gomp_ordered_static_init ();
- else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- gomp_work_share_init_done ();
- }
- else
- {
- if (reductions)
- {
- uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
- gomp_workshare_task_reduction_register (reductions,
- first_reductions);
- }
- sched = thr->ts.work_share->sched;
- if (sched != GFS_STATIC)
- gomp_mutex_lock (&thr->ts.work_share->lock);
- }
- if (mem)
- {
- uintptr_t p
- = (uintptr_t) (thr->ts.work_share->ordered_team_ids
- + (thr->ts.team ? thr->ts.team->nthreads : 1));
- p += __alignof__ (long long) - 1;
- p &= ~(__alignof__ (long long) - 1);
- *mem = (void *) p;
- }
- switch (sched)
- {
- case GFS_STATIC:
- case GFS_AUTO:
- return !gomp_iter_static_next (istart, iend);
- case GFS_DYNAMIC:
- ret = gomp_iter_dynamic_next_locked (istart, iend);
- break;
- case GFS_GUIDED:
- ret = gomp_iter_guided_next_locked (istart, iend);
- break;
- default:
- abort ();
- }
- if (ret)
- gomp_ordered_first ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- /* The *_doacross_*_start routines are similar. The only difference is that
- this work-share construct is initialized to expect an ORDERED(N) - DOACROSS
- section, and the worksharing loop iterates always from 0 to COUNTS[0] - 1
- and other COUNTS array elements tell the library number of iterations
- in the ordered inner loops. */
- static bool
- gomp_loop_doacross_static_start (unsigned ncounts, long *counts,
- long chunk_size, long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (gomp_work_share_start (0))
- {
- gomp_loop_init (thr->ts.work_share, 0, counts[0], 1,
- GFS_STATIC, chunk_size);
- gomp_doacross_init (ncounts, counts, chunk_size, 0);
- gomp_work_share_init_done ();
- }
- return !gomp_iter_static_next (istart, iend);
- }
- static bool
- gomp_loop_doacross_dynamic_start (unsigned ncounts, long *counts,
- long chunk_size, long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (0))
- {
- gomp_loop_init (thr->ts.work_share, 0, counts[0], 1,
- GFS_DYNAMIC, chunk_size);
- gomp_doacross_init (ncounts, counts, chunk_size, 0);
- gomp_work_share_init_done ();
- }
- #ifdef HAVE_SYNC_BUILTINS
- ret = gomp_iter_dynamic_next (istart, iend);
- #else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_dynamic_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- static bool
- gomp_loop_doacross_guided_start (unsigned ncounts, long *counts,
- long chunk_size, long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- if (gomp_work_share_start (0))
- {
- gomp_loop_init (thr->ts.work_share, 0, counts[0], 1,
- GFS_GUIDED, chunk_size);
- gomp_doacross_init (ncounts, counts, chunk_size, 0);
- gomp_work_share_init_done ();
- }
- #ifdef HAVE_SYNC_BUILTINS
- ret = gomp_iter_guided_next (istart, iend);
- #else
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_guided_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- bool
- GOMP_loop_doacross_runtime_start (unsigned ncounts, long *counts,
- long *istart, long *iend)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- switch (icv->run_sched_var & ~GFS_MONOTONIC)
- {
- case GFS_STATIC:
- return gomp_loop_doacross_static_start (ncounts, counts,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_doacross_dynamic_start (ncounts, counts,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_GUIDED:
- return gomp_loop_doacross_guided_start (ncounts, counts,
- icv->run_sched_chunk_size,
- istart, iend);
- case GFS_AUTO:
- /* For now map to schedule(static), later on we could play with feedback
- driven choice. */
- return gomp_loop_doacross_static_start (ncounts, counts,
- 0, istart, iend);
- default:
- abort ();
- }
- }
- bool
- GOMP_loop_doacross_start (unsigned ncounts, long *counts, long sched,
- long chunk_size, long *istart, long *iend,
- uintptr_t *reductions, void **mem)
- {
- struct gomp_thread *thr = gomp_thread ();
- thr->ts.static_trip = 0;
- if (reductions)
- gomp_workshare_taskgroup_start ();
- if (gomp_work_share_start (0))
- {
- size_t extra = 0;
- if (mem)
- extra = (uintptr_t) *mem;
- sched = gomp_adjust_sched (sched, &chunk_size);
- gomp_loop_init (thr->ts.work_share, 0, counts[0], 1,
- sched, chunk_size);
- gomp_doacross_init (ncounts, counts, chunk_size, extra);
- if (reductions)
- {
- GOMP_taskgroup_reduction_register (reductions);
- thr->task->taskgroup->workshare = true;
- thr->ts.work_share->task_reductions = reductions;
- }
- gomp_work_share_init_done ();
- }
- else
- {
- if (reductions)
- {
- uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
- gomp_workshare_task_reduction_register (reductions,
- first_reductions);
- }
- sched = thr->ts.work_share->sched;
- }
- if (mem)
- *mem = thr->ts.work_share->doacross->extra;
- return ialias_call (GOMP_loop_runtime_next) (istart, iend);
- }
- /* The *_next routines are called when the thread completes processing of
- the iteration block currently assigned to it. If the work-share
- construct is bound directly to a parallel construct, then the iteration
- bounds may have been set up before the parallel. In which case, this
- may be the first iteration for the thread.
- Returns true if there is work remaining to be performed; *ISTART and
- *IEND are filled with a new iteration block. Returns false if all work
- has been assigned. */
- static bool
- gomp_loop_static_next (long *istart, long *iend)
- {
- return !gomp_iter_static_next (istart, iend);
- }
- static bool
- gomp_loop_dynamic_next (long *istart, long *iend)
- {
- bool ret;
- #ifdef HAVE_SYNC_BUILTINS
- ret = gomp_iter_dynamic_next (istart, iend);
- #else
- struct gomp_thread *thr = gomp_thread ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_dynamic_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- static bool
- gomp_loop_guided_next (long *istart, long *iend)
- {
- bool ret;
- #ifdef HAVE_SYNC_BUILTINS
- ret = gomp_iter_guided_next (istart, iend);
- #else
- struct gomp_thread *thr = gomp_thread ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_guided_next_locked (istart, iend);
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- #endif
- return ret;
- }
- bool
- GOMP_loop_runtime_next (long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- switch (thr->ts.work_share->sched)
- {
- case GFS_STATIC:
- case GFS_AUTO:
- return gomp_loop_static_next (istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_dynamic_next (istart, iend);
- case GFS_GUIDED:
- return gomp_loop_guided_next (istart, iend);
- default:
- abort ();
- }
- }
- /* The *_ordered_*_next routines are called when the thread completes
- processing of the iteration block currently assigned to it.
- Returns true if there is work remaining to be performed; *ISTART and
- *IEND are filled with a new iteration block. Returns false if all work
- has been assigned. */
- static bool
- gomp_loop_ordered_static_next (long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- int test;
- gomp_ordered_sync ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- test = gomp_iter_static_next (istart, iend);
- if (test >= 0)
- gomp_ordered_static_next ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return test == 0;
- }
- static bool
- gomp_loop_ordered_dynamic_next (long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- gomp_ordered_sync ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_dynamic_next_locked (istart, iend);
- if (ret)
- gomp_ordered_next ();
- else
- gomp_ordered_last ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- static bool
- gomp_loop_ordered_guided_next (long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- bool ret;
- gomp_ordered_sync ();
- gomp_mutex_lock (&thr->ts.work_share->lock);
- ret = gomp_iter_guided_next_locked (istart, iend);
- if (ret)
- gomp_ordered_next ();
- else
- gomp_ordered_last ();
- gomp_mutex_unlock (&thr->ts.work_share->lock);
- return ret;
- }
- bool
- GOMP_loop_ordered_runtime_next (long *istart, long *iend)
- {
- struct gomp_thread *thr = gomp_thread ();
- switch (thr->ts.work_share->sched)
- {
- case GFS_STATIC:
- case GFS_AUTO:
- return gomp_loop_ordered_static_next (istart, iend);
- case GFS_DYNAMIC:
- return gomp_loop_ordered_dynamic_next (istart, iend);
- case GFS_GUIDED:
- return gomp_loop_ordered_guided_next (istart, iend);
- default:
- abort ();
- }
- }
- /* The GOMP_parallel_loop_* routines pre-initialize a work-share construct
- to avoid one synchronization once we get into the loop. */
- static void
- gomp_parallel_loop_start (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr, enum gomp_schedule_type sched,
- long chunk_size, unsigned int flags)
- {
- struct gomp_team *team;
- num_threads = gomp_resolve_num_threads (num_threads, 0);
- team = gomp_new_team (num_threads);
- gomp_loop_init (&team->work_shares[0], start, end, incr, sched, chunk_size);
- gomp_team_start (fn, data, num_threads, flags, team, NULL);
- }
- void
- GOMP_parallel_loop_static_start (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr, long chunk_size)
- {
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- GFS_STATIC, chunk_size, 0);
- }
- void
- GOMP_parallel_loop_dynamic_start (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr, long chunk_size)
- {
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- GFS_DYNAMIC, chunk_size, 0);
- }
- void
- GOMP_parallel_loop_guided_start (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr, long chunk_size)
- {
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- GFS_GUIDED, chunk_size, 0);
- }
- void
- GOMP_parallel_loop_runtime_start (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- icv->run_sched_var & ~GFS_MONOTONIC,
- icv->run_sched_chunk_size, 0);
- }
- ialias_redirect (GOMP_parallel_end)
- void
- GOMP_parallel_loop_static (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr, long chunk_size, unsigned flags)
- {
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- GFS_STATIC, chunk_size, flags);
- fn (data);
- GOMP_parallel_end ();
- }
- void
- GOMP_parallel_loop_dynamic (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr, long chunk_size, unsigned flags)
- {
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- GFS_DYNAMIC, chunk_size, flags);
- fn (data);
- GOMP_parallel_end ();
- }
- void
- GOMP_parallel_loop_guided (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr, long chunk_size, unsigned flags)
- {
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- GFS_GUIDED, chunk_size, flags);
- fn (data);
- GOMP_parallel_end ();
- }
- void
- GOMP_parallel_loop_runtime (void (*fn) (void *), void *data,
- unsigned num_threads, long start, long end,
- long incr, unsigned flags)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- icv->run_sched_var & ~GFS_MONOTONIC,
- icv->run_sched_chunk_size, flags);
- fn (data);
- GOMP_parallel_end ();
- }
- #ifdef HAVE_ATTRIBUTE_ALIAS
- extern __typeof(GOMP_parallel_loop_dynamic) GOMP_parallel_loop_nonmonotonic_dynamic
- __attribute__((alias ("GOMP_parallel_loop_dynamic")));
- extern __typeof(GOMP_parallel_loop_guided) GOMP_parallel_loop_nonmonotonic_guided
- __attribute__((alias ("GOMP_parallel_loop_guided")));
- extern __typeof(GOMP_parallel_loop_runtime) GOMP_parallel_loop_nonmonotonic_runtime
- __attribute__((alias ("GOMP_parallel_loop_runtime")));
- extern __typeof(GOMP_parallel_loop_runtime) GOMP_parallel_loop_maybe_nonmonotonic_runtime
- __attribute__((alias ("GOMP_parallel_loop_runtime")));
- #else
- void
- GOMP_parallel_loop_nonmonotonic_dynamic (void (*fn) (void *), void *data,
- unsigned num_threads, long start,
- long end, long incr, long chunk_size,
- unsigned flags)
- {
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- GFS_DYNAMIC, chunk_size, flags);
- fn (data);
- GOMP_parallel_end ();
- }
- void
- GOMP_parallel_loop_nonmonotonic_guided (void (*fn) (void *), void *data,
- unsigned num_threads, long start,
- long end, long incr, long chunk_size,
- unsigned flags)
- {
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- GFS_GUIDED, chunk_size, flags);
- fn (data);
- GOMP_parallel_end ();
- }
- void
- GOMP_parallel_loop_nonmonotonic_runtime (void (*fn) (void *), void *data,
- unsigned num_threads, long start,
- long end, long incr, unsigned flags)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- icv->run_sched_var & ~GFS_MONOTONIC,
- icv->run_sched_chunk_size, flags);
- fn (data);
- GOMP_parallel_end ();
- }
- void
- GOMP_parallel_loop_maybe_nonmonotonic_runtime (void (*fn) (void *), void *data,
- unsigned num_threads, long start,
- long end, long incr,
- unsigned flags)
- {
- struct gomp_task_icv *icv = gomp_icv (false);
- gomp_parallel_loop_start (fn, data, num_threads, start, end, incr,
- icv->run_sched_var & ~GFS_MONOTONIC,
- icv->run_sched_chunk_size, flags);
- fn (data);
- GOMP_parallel_end ();
- }
- #endif
- /* The GOMP_loop_end* routines are called after the thread is told that
- all loop iterations are complete. The first two versions synchronize
- all threads; the nowait version does not. */
- void
- GOMP_loop_end (void)
- {
- gomp_work_share_end ();
- }
- bool
- GOMP_loop_end_cancel (void)
- {
- return gomp_work_share_end_cancel ();
- }
- void
- GOMP_loop_end_nowait (void)
- {
- gomp_work_share_end_nowait ();
- }
- /* We use static functions above so that we're sure that the "runtime"
- function can defer to the proper routine without interposition. We
- export the static function with a strong alias when possible, or with
- a wrapper function otherwise. */
- #ifdef HAVE_ATTRIBUTE_ALIAS
- extern __typeof(gomp_loop_static_start) GOMP_loop_static_start
- __attribute__((alias ("gomp_loop_static_start")));
- extern __typeof(gomp_loop_dynamic_start) GOMP_loop_dynamic_start
- __attribute__((alias ("gomp_loop_dynamic_start")));
- extern __typeof(gomp_loop_guided_start) GOMP_loop_guided_start
- __attribute__((alias ("gomp_loop_guided_start")));
- extern __typeof(gomp_loop_dynamic_start) GOMP_loop_nonmonotonic_dynamic_start
- __attribute__((alias ("gomp_loop_dynamic_start")));
- extern __typeof(gomp_loop_guided_start) GOMP_loop_nonmonotonic_guided_start
- __attribute__((alias ("gomp_loop_guided_start")));
- extern __typeof(GOMP_loop_runtime_start) GOMP_loop_nonmonotonic_runtime_start
- __attribute__((alias ("GOMP_loop_runtime_start")));
- extern __typeof(GOMP_loop_runtime_start) GOMP_loop_maybe_nonmonotonic_runtime_start
- __attribute__((alias ("GOMP_loop_runtime_start")));
- extern __typeof(gomp_loop_ordered_static_start) GOMP_loop_ordered_static_start
- __attribute__((alias ("gomp_loop_ordered_static_start")));
- extern __typeof(gomp_loop_ordered_dynamic_start) GOMP_loop_ordered_dynamic_start
- __attribute__((alias ("gomp_loop_ordered_dynamic_start")));
- extern __typeof(gomp_loop_ordered_guided_start) GOMP_loop_ordered_guided_start
- __attribute__((alias ("gomp_loop_ordered_guided_start")));
- extern __typeof(gomp_loop_doacross_static_start) GOMP_loop_doacross_static_start
- __attribute__((alias ("gomp_loop_doacross_static_start")));
- extern __typeof(gomp_loop_doacross_dynamic_start) GOMP_loop_doacross_dynamic_start
- __attribute__((alias ("gomp_loop_doacross_dynamic_start")));
- extern __typeof(gomp_loop_doacross_guided_start) GOMP_loop_doacross_guided_start
- __attribute__((alias ("gomp_loop_doacross_guided_start")));
- extern __typeof(gomp_loop_static_next) GOMP_loop_static_next
- __attribute__((alias ("gomp_loop_static_next")));
- extern __typeof(gomp_loop_dynamic_next) GOMP_loop_dynamic_next
- __attribute__((alias ("gomp_loop_dynamic_next")));
- extern __typeof(gomp_loop_guided_next) GOMP_loop_guided_next
- __attribute__((alias ("gomp_loop_guided_next")));
- extern __typeof(gomp_loop_dynamic_next) GOMP_loop_nonmonotonic_dynamic_next
- __attribute__((alias ("gomp_loop_dynamic_next")));
- extern __typeof(gomp_loop_guided_next) GOMP_loop_nonmonotonic_guided_next
- __attribute__((alias ("gomp_loop_guided_next")));
- extern __typeof(GOMP_loop_runtime_next) GOMP_loop_nonmonotonic_runtime_next
- __attribute__((alias ("GOMP_loop_runtime_next")));
- extern __typeof(GOMP_loop_runtime_next) GOMP_loop_maybe_nonmonotonic_runtime_next
- __attribute__((alias ("GOMP_loop_runtime_next")));
- extern __typeof(gomp_loop_ordered_static_next) GOMP_loop_ordered_static_next
- __attribute__((alias ("gomp_loop_ordered_static_next")));
- extern __typeof(gomp_loop_ordered_dynamic_next) GOMP_loop_ordered_dynamic_next
- __attribute__((alias ("gomp_loop_ordered_dynamic_next")));
- extern __typeof(gomp_loop_ordered_guided_next) GOMP_loop_ordered_guided_next
- __attribute__((alias ("gomp_loop_ordered_guided_next")));
- #else
- bool
- GOMP_loop_static_start (long start, long end, long incr, long chunk_size,
- long *istart, long *iend)
- {
- return gomp_loop_static_start (start, end, incr, chunk_size, istart, iend);
- }
- bool
- GOMP_loop_dynamic_start (long start, long end, long incr, long chunk_size,
- long *istart, long *iend)
- {
- return gomp_loop_dynamic_start (start, end, incr, chunk_size, istart, iend);
- }
- bool
- GOMP_loop_guided_start (long start, long end, long incr, long chunk_size,
- long *istart, long *iend)
- {
- return gomp_loop_guided_start (start, end, incr, chunk_size, istart, iend);
- }
- bool
- GOMP_loop_nonmonotonic_dynamic_start (long start, long end, long incr,
- long chunk_size, long *istart,
- long *iend)
- {
- return gomp_loop_dynamic_start (start, end, incr, chunk_size, istart, iend);
- }
- bool
- GOMP_loop_nonmonotonic_guided_start (long start, long end, long incr,
- long chunk_size, long *istart, long *iend)
- {
- return gomp_loop_guided_start (start, end, incr, chunk_size, istart, iend);
- }
- bool
- GOMP_loop_nonmonotonic_runtime_start (long start, long end, long incr,
- long *istart, long *iend)
- {
- return GOMP_loop_runtime_start (start, end, incr, istart, iend);
- }
- bool
- GOMP_loop_maybe_nonmonotonic_runtime_start (long start, long end, long incr,
- long *istart, long *iend)
- {
- return GOMP_loop_runtime_start (start, end, incr, istart, iend);
- }
- bool
- GOMP_loop_ordered_static_start (long start, long end, long incr,
- long chunk_size, long *istart, long *iend)
- {
- return gomp_loop_ordered_static_start (start, end, incr, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_ordered_dynamic_start (long start, long end, long incr,
- long chunk_size, long *istart, long *iend)
- {
- return gomp_loop_ordered_dynamic_start (start, end, incr, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_ordered_guided_start (long start, long end, long incr,
- long chunk_size, long *istart, long *iend)
- {
- return gomp_loop_ordered_guided_start (start, end, incr, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_doacross_static_start (unsigned ncounts, long *counts,
- long chunk_size, long *istart, long *iend)
- {
- return gomp_loop_doacross_static_start (ncounts, counts, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_doacross_dynamic_start (unsigned ncounts, long *counts,
- long chunk_size, long *istart, long *iend)
- {
- return gomp_loop_doacross_dynamic_start (ncounts, counts, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_doacross_guided_start (unsigned ncounts, long *counts,
- long chunk_size, long *istart, long *iend)
- {
- return gomp_loop_doacross_guided_start (ncounts, counts, chunk_size,
- istart, iend);
- }
- bool
- GOMP_loop_static_next (long *istart, long *iend)
- {
- return gomp_loop_static_next (istart, iend);
- }
- bool
- GOMP_loop_dynamic_next (long *istart, long *iend)
- {
- return gomp_loop_dynamic_next (istart, iend);
- }
- bool
- GOMP_loop_guided_next (long *istart, long *iend)
- {
- return gomp_loop_guided_next (istart, iend);
- }
- bool
- GOMP_loop_nonmonotonic_dynamic_next (long *istart, long *iend)
- {
- return gomp_loop_dynamic_next (istart, iend);
- }
- bool
- GOMP_loop_nonmonotonic_guided_next (long *istart, long *iend)
- {
- return gomp_loop_guided_next (istart, iend);
- }
- bool
- GOMP_loop_nonmonotonic_runtime_next (long *istart, long *iend)
- {
- return GOMP_loop_runtime_next (istart, iend);
- }
- bool
- GOMP_loop_maybe_nonmonotonic_runtime_next (long *istart, long *iend)
- {
- return GOMP_loop_runtime_next (istart, iend);
- }
- bool
- GOMP_loop_ordered_static_next (long *istart, long *iend)
- {
- return gomp_loop_ordered_static_next (istart, iend);
- }
- bool
- GOMP_loop_ordered_dynamic_next (long *istart, long *iend)
- {
- return gomp_loop_ordered_dynamic_next (istart, iend);
- }
- bool
- GOMP_loop_ordered_guided_next (long *istart, long *iend)
- {
- return gomp_loop_ordered_guided_next (istart, iend);
- }
- #endif
|