gcc/liboffloadmic/runtime/offload_target.cpp

/*
    Copyright (c) 2014-2016 Intel Corporation.  All Rights Reserved.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions
    are met:

      * Redistributions of source code must retain the above copyright
        notice, this list of conditions and the following disclaimer.
      * Redistributions in binary form must reproduce the above copyright
        notice, this list of conditions and the following disclaimer in the
        documentation and/or other materials provided with the distribution.
      * Neither the name of Intel Corporation nor the names of its
        contributors may be used to endorse or promote products derived
        from this software without specific prior written permission.

    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/


#include "offload_target.h"
#include <stdlib.h>
#include <unistd.h>
#ifdef SEP_SUPPORT
#include <fcntl.h>
#include <sys/ioctl.h>
#endif // SEP_SUPPORT
#include <omp.h>
#include <map>

// typedef offload_func_with_parms.
// Pointer to function that represents an offloaded entry point.
// The parameters are a temporary fix for parameters on the stack.
typedef void (*offload_func_with_parms)(void *);

// Target console and file logging
const char *prefix;
int console_enabled = 0;
int offload_report_level = 0;

// Trace information
static const char* vardesc_direction_as_string[] = {
    "NOCOPY",
    "IN",
    "OUT",
    "INOUT"
};
static const char* vardesc_type_as_string[] = {
    "unknown",
    "data",
    "data_ptr",
    "func_ptr",
    "void_ptr",
    "string_ptr",
    "dv",
    "dv_data",
    "dv_data_slice",
    "dv_ptr",
    "dv_ptr_data",
    "dv_ptr_data_slice",
    "cean_var",
    "cean_var_ptr",
    "c_data_ptr_array",
    "c_extended_type",
    "c_func_ptr_array",
    "c_void_ptr_array",
    "c_string_ptr_array",
    "c_data_ptr_ptr",
    "c_func_ptr_ptr",
    "c_void_ptr_ptr",
    "c_string_ptr_ptr",
    "c_cean_var_ptr_ptr",
};

int mic_index = -1;
int mic_engines_total = -1;
uint64_t mic_frequency = 0;
int offload_number = 0;
static std::map<void*, RefInfo*> ref_data;
static mutex_t add_ref_lock;

#ifdef SEP_SUPPORT
static const char*  sep_monitor_env = "SEP_MONITOR";
static bool         sep_monitor = false;
static const char*  sep_device_env = "SEP_DEVICE";
static const char*  sep_device =  "/dev/sep3.8/c";
static int          sep_counter = 0;

#define SEP_API_IOC_MAGIC   99
#define SEP_IOCTL_PAUSE     _IO (SEP_API_IOC_MAGIC, 31)
#define SEP_IOCTL_RESUME    _IO (SEP_API_IOC_MAGIC, 32)

static void add_ref_count(void * buf, bool created)
{
    mutex_locker_t locker(add_ref_lock);
    RefInfo * info = ref_data[buf];

    if (info) {
        info->count++;
    }
    else {
        info = new RefInfo((int)created,(long)1);
    }
    info->is_added |= created;
    ref_data[buf] = info;
}

static void BufReleaseRef(void * buf)
{
    mutex_locker_t locker(add_ref_lock);
    RefInfo * info = ref_data[buf];

    if (info) {
        --info->count;
        if (info->count == 0 && info->is_added) {
            OFFLOAD_TRACE(1, "Calling COIBufferReleaseRef AddRef count = %d\n",
                             ((RefInfo *) ref_data[buf])->count);
            BufferReleaseRef(buf);
            info->is_added = 0;
        }
    }
}

static int VTPauseSampling(void)
{
    int ret = -1;
    int handle = open(sep_device, O_RDWR);
    if (handle > 0) {
        ret = ioctl(handle, SEP_IOCTL_PAUSE);
        close(handle);
    }
    return ret;
}

static int VTResumeSampling(void)
{
    int ret = -1;
    int handle = open(sep_device, O_RDWR);
    if (handle > 0) {
        ret = ioctl(handle, SEP_IOCTL_RESUME);
        close(handle);
    }
    return ret;
}
#endif // SEP_SUPPORT

void OffloadDescriptor::offload(
    uint32_t  buffer_count,
    void**    buffers,
    void*     misc_data,
    uint16_t  misc_data_len,
    void*     return_data,
    uint16_t  return_data_len
)
{
    FunctionDescriptor *func = (FunctionDescriptor*) misc_data;
    const char *name = func->data;
    OffloadDescriptor ofld;
    char *in_data = 0;
    char *out_data = 0;
    char *timer_data = 0;

    console_enabled = func->console_enabled;
    timer_enabled = func->timer_enabled;
    offload_report_level = func->offload_report_level;
    offload_number = func->offload_number;
    ofld.set_offload_number(func->offload_number);

#ifdef SEP_SUPPORT
    if (sep_monitor) {
        if (__sync_fetch_and_add(&sep_counter, 1) == 0) {
            OFFLOAD_DEBUG_TRACE(2, "VTResumeSampling\n");
            VTResumeSampling();
        }
    }
#endif // SEP_SUPPORT

    OFFLOAD_DEBUG_TRACE_1(2, ofld.get_offload_number(),
                          c_offload_start_target_func,
                          "Offload \"%s\" started\n", name);

    // initialize timer data
    OFFLOAD_TIMER_INIT();

    OFFLOAD_TIMER_START(c_offload_target_total_time);

    OFFLOAD_TIMER_START(c_offload_target_descriptor_setup);

    // get input/output buffer addresses
    if (func->in_datalen > 0 || func->out_datalen > 0) {
        if (func->data_offset != 0) {
            in_data = (char*) misc_data + func->data_offset;
            out_data = (char*) return_data;
        }
        else {
            char *inout_buf = (char*) buffers[--buffer_count];
            in_data = inout_buf;
            out_data = inout_buf;
        }
    }

    // assign variable descriptors
    ofld.m_vars_total = func->vars_num;
    if (ofld.m_vars_total > 0) {
        uint64_t var_data_len = ofld.m_vars_total * sizeof(VarDesc);

        ofld.m_vars = (VarDesc*) malloc(var_data_len);
        if (ofld.m_vars == NULL)
          LIBOFFLOAD_ERROR(c_malloc);
        memcpy(ofld.m_vars, in_data, var_data_len);

        ofld.m_vars_extra =
            (VarExtra*) malloc(ofld.m_vars_total * sizeof(VarExtra));
        if (ofld.m_vars == NULL)
          LIBOFFLOAD_ERROR(c_malloc);

        in_data += var_data_len;
        func->in_datalen -= var_data_len;
    }

    // timer data
    if (func->timer_enabled) {
        uint64_t timer_data_len = OFFLOAD_TIMER_DATALEN();

        timer_data = out_data;
        out_data += timer_data_len;
        func->out_datalen -= timer_data_len;
    }

    // init Marshallers
    ofld.m_in.init_buffer(in_data, func->in_datalen);
    ofld.m_out.init_buffer(out_data, func->out_datalen);

    // copy buffers to offload descriptor
    std::copy(buffers, buffers + buffer_count,
              std::back_inserter(ofld.m_buffers));

    OFFLOAD_TIMER_STOP(c_offload_target_descriptor_setup);

    // find offload entry address
    OFFLOAD_TIMER_START(c_offload_target_func_lookup);

    offload_func_with_parms entry = (offload_func_with_parms)
        __offload_entries.find_addr(name);

    if (entry == NULL) {
#if OFFLOAD_DEBUG > 0
        if (console_enabled > 2) {
            __offload_entries.dump();
        }
#endif
        LIBOFFLOAD_ERROR(c_offload_descriptor_offload, name);
        exit(1);
    }

    OFFLOAD_TIMER_STOP(c_offload_target_func_lookup);

    OFFLOAD_TIMER_START(c_offload_target_func_time);

    // execute offload entry
    entry(&ofld);

    OFFLOAD_TIMER_STOP(c_offload_target_func_time);

    OFFLOAD_TIMER_STOP(c_offload_target_total_time);

    // copy timer data to the buffer
    OFFLOAD_TIMER_TARGET_DATA(timer_data);

    OFFLOAD_DEBUG_TRACE(2, "Offload \"%s\" finished\n", name);

#ifdef SEP_SUPPORT
    if (sep_monitor) {
        if (__sync_sub_and_fetch(&sep_counter, 1) == 0) {
            OFFLOAD_DEBUG_TRACE(2, "VTPauseSampling\n");
            VTPauseSampling();
        }
    }
#endif // SEP_SUPPORT
}

void OffloadDescriptor::merge_var_descs(
    VarDesc *vars,
    VarDesc2 *vars2,
    int vars_total
)
{
    // number of variable descriptors received from host and generated
    // locally should match
    if (m_vars_total < vars_total) {
        LIBOFFLOAD_ERROR(c_merge_var_descs1);
        exit(1);
    }

    for (int i = 0; i < m_vars_total; i++) {
        // instead of m_vars[i].type.src we will use m_vars_extra[i].type_src

        if (i < vars_total) {
            // variable type must match
            if (m_vars[i].type.bits != vars[i].type.bits) {
                OFFLOAD_TRACE(2,
                    "m_vars[%d].type.bits=%08x, vars[%d].type.bits=%08x\n",
                    i, m_vars[i].type.bits, i, vars[i].type.bits);
                LIBOFFLOAD_ERROR(c_merge_var_descs2);
                exit(1);
            }

            if (m_vars[i].type.src == c_extended_type) {
                VarDescExtendedType *etype =
                    reinterpret_cast<VarDescExtendedType*>(vars[i].ptr);
                m_vars_extra[i].type_src = etype->extended_type;
                m_vars[i].ptr            = etype->ptr;
            }
            else {
                m_vars_extra[i].type_src = m_vars[i].type.src;
                if (!(m_vars[i].flags.use_device_ptr &&
                      m_vars[i].type.src == c_dv)) {
                    m_vars[i].ptr = vars[i].ptr;
                }
            }
            // instead of m_vars[i].type.dst we will use m_vars_extra[i].type_dst
            if (i < vars_total && m_vars[i].type.dst == c_extended_type) {
                VarDescExtendedType *etype =
                    reinterpret_cast<VarDescExtendedType*>(vars[i].into);
                m_vars_extra[i].type_dst = etype->extended_type;
                m_vars[i].into           = etype->ptr;
            }
            else {
                m_vars_extra[i].type_dst = m_vars[i].type.dst;
                m_vars[i].into = vars[i].into;
            }

            const char *var_sname = "";
            if (vars2 != NULL) {
                if (vars2[i].sname != NULL) {
                    var_sname = vars2[i].sname;
                }
            }
            OFFLOAD_DEBUG_TRACE_1(2, get_offload_number(), c_offload_var,
                "   VarDesc %d, var=%s, %s, %s\n",
                i, var_sname,
                vardesc_direction_as_string[m_vars[i].direction.bits],
                vardesc_type_as_string[m_vars_extra[i].type_src]);
            if (vars2 != NULL && vars2[i].dname != NULL) {
                OFFLOAD_TRACE(2, "              into=%s, %s\n", vars2[i].dname,
                    vardesc_type_as_string[m_vars_extra[i].type_dst]);
            }
        }
        else {
            m_vars_extra[i].type_src = m_vars[i].type.src;
            m_vars_extra[i].type_dst = m_vars[i].type.dst;
        }

        OFFLOAD_TRACE(2,
            "              type_src=%d, type_dstn=%d, direction=%d, "
            "alloc_if=%d, free_if=%d, align=%d, mic_offset=%d, flags=0x%x, "
            "offset=%lld, size=%lld, count/disp=%lld, ptr=%p into=%p\n",
            m_vars_extra[i].type_src,
            m_vars_extra[i].type_dst,
            m_vars[i].direction.bits,
            m_vars[i].alloc_if,
            m_vars[i].free_if,
            m_vars[i].align,
            m_vars[i].mic_offset,
            m_vars[i].flags.bits,
            m_vars[i].offset,
            m_vars[i].size,
            m_vars[i].count,
            m_vars[i].ptr,
            m_vars[i].into);
    }
}

void OffloadDescriptor::scatter_copyin_data()
{
    OFFLOAD_TIMER_START(c_offload_target_scatter_inputs);

    OFFLOAD_DEBUG_TRACE(2, "IN  buffer @ %p size %lld\n",
                        m_in.get_buffer_start(),
                        m_in.get_buffer_size());
    OFFLOAD_DEBUG_DUMP_BYTES(2, m_in.get_buffer_start(),
                             m_in.get_buffer_size());

    // receive data
    for (int i = 0; i < m_vars_total; i++) {
        bool src_is_for_mic = (m_vars[i].direction.out ||
                               m_vars[i].into == NULL);
        void** ptr_addr = src_is_for_mic ?
                          static_cast<void**>(m_vars[i].ptr) :
                          static_cast<void**>(m_vars[i].into);
        int type = src_is_for_mic ? m_vars_extra[i].type_src :
                                    m_vars_extra[i].type_dst;
        bool is_static = src_is_for_mic ?
                         m_vars[i].flags.is_static :
                         m_vars[i].flags.is_static_dstn;
        void *ptr = NULL;

        if (m_vars[i].flags.alloc_disp) {
            int64_t offset = 0;
            m_in.receive_data(&offset, sizeof(offset));
        }
        if (VAR_TYPE_IS_DV_DATA_SLICE(type) ||
            VAR_TYPE_IS_DV_DATA(type)) {
            ArrDesc *dvp = (type == c_dv_data_slice || type == c_dv_data)?
                  reinterpret_cast<ArrDesc*>(ptr_addr) :
                  *reinterpret_cast<ArrDesc**>(ptr_addr);
            ptr_addr = reinterpret_cast<void**>(&dvp->Base);
        }
        // Set pointer values
        switch (type) {
            case c_data_ptr_array:
                {
                    int j = m_vars[i].ptr_arr_offset;
                    int max_el = j + m_vars[i].count;
                    char *dst_arr_ptr = (src_is_for_mic)?
                        *(reinterpret_cast<char**>(m_vars[i].ptr)) :
                        reinterpret_cast<char*>(m_vars[i].into);

                    // if is_pointer is 1 it means that pointer array itself
                    // is defined either via pointer or as class member.
                    // i.e. arr_ptr[0:5] or this->ARR[0:5]
                    if (m_vars[i].flags.is_pointer) {
                        int64_t offset = 0;
                        m_in.receive_data(&offset, sizeof(offset));
                        dst_arr_ptr = *((char**)dst_arr_ptr) + offset;
                    }
                    for (; j < max_el; j++) {
                        if (src_is_for_mic) {
                            m_vars[j].ptr =
                                dst_arr_ptr + m_vars[j].ptr_arr_offset;
                        }
                        else {
                            m_vars[j].into =
                                dst_arr_ptr + m_vars[j].ptr_arr_offset;
                        }
                    }
                }
                break;
            case c_data:
            case c_void_ptr:
            case c_void_ptr_ptr:
            case c_cean_var:
            case c_dv:
                break;

            case c_string_ptr:
            case c_data_ptr:
            case c_string_ptr_ptr:
            case c_data_ptr_ptr:
            case c_cean_var_ptr:
            case c_cean_var_ptr_ptr:
            case c_dv_ptr:
                // Don't need ptr_addr value for variables from stack buffer.
                // Stack buffer address is set at var_desc with #0.
                if (i != 0 && m_vars[i].flags.is_stack_buf) {
                    break;
                }
                if (TYPE_IS_PTR_TO_PTR(m_vars_extra[i].type_src) ||
                    TYPE_IS_PTR_TO_PTR(m_vars_extra[i].type_dst)) {
                    int64_t offset;

                    m_in.receive_data(&offset, sizeof(offset));
                    ptr_addr = reinterpret_cast<void**>(
                                 reinterpret_cast<char*>(*ptr_addr) + offset);

                }

                if (m_vars[i].alloc_if && !m_vars[i].flags.preallocated) {
                    void *buf = NULL;
                    if (m_vars[i].flags.sink_addr) {
                        m_in.receive_data(&buf, sizeof(buf));
                    }
                    else {
                        buf = m_buffers.front();
                        m_buffers.pop_front();
                    }
                    if (buf) {
                        if (!is_static) {
                            if (!m_vars[i].flags.sink_addr) {
                                // increment buffer reference
                                OFFLOAD_TIMER_START(c_offload_target_add_buffer_refs);
                                BufferAddRef(buf);
                                OFFLOAD_TRACE(1, "Calling COIBufferAddRef %p\n", buf);
                                OFFLOAD_TIMER_STOP(c_offload_target_add_buffer_refs);
                            }
                            add_ref_count(buf, 0 == m_vars[i].flags.sink_addr);
                            OFFLOAD_TRACE(1, "    AddRef count = %d\n",
                                              ((RefInfo *) ref_data[buf])->count);
                        }
                        ptr = static_cast<char*>(buf) +
                                  m_vars[i].mic_offset +
                                  (m_vars[i].flags.is_stack_buf ?
                                   0 : m_vars[i].offset);

                    }
                    *ptr_addr = ptr;
                }
                else if (m_vars[i].flags.sink_addr) {
                    void *buf;
                    m_in.receive_data(&buf, sizeof(buf));
                    void *ptr = static_cast<char*>(buf) +
                                    m_vars[i].mic_offset +
                                    (m_vars[i].flags.is_stack_buf ?
                                     0 : m_vars[i].offset);
                    *ptr_addr = ptr;
                }
                break;

            case c_func_ptr:
            case c_func_ptr_ptr:
                break;

            case c_dv_data:
            case c_dv_ptr_data:
            case c_dv_data_slice:
            case c_dv_ptr_data_slice:
                if (m_vars[i].alloc_if) {
                    void *buf;
                    if (m_vars[i].flags.sink_addr) {
                        m_in.receive_data(&buf, sizeof(buf));
                    }
                    else {
                        buf = m_buffers.front();
                        m_buffers.pop_front();
                    }
                    if (buf) {
                        if (!is_static) {
                            if (!m_vars[i].flags.sink_addr) {
                                // increment buffer reference
                                OFFLOAD_TIMER_START(c_offload_target_add_buffer_refs);
                                BufferAddRef(buf);
                                OFFLOAD_TIMER_STOP(c_offload_target_add_buffer_refs);
                            }
                            add_ref_count(buf, 0 == m_vars[i].flags.sink_addr);
                        }
                        ptr = static_cast<char*>(buf) +
                            m_vars[i].mic_offset + m_vars[i].offset;
                    }
                    *ptr_addr = ptr;
                }
                else if (m_vars[i].flags.sink_addr) {
                    void *buf;
                    m_in.receive_data(&buf, sizeof(buf));
                    ptr = static_cast<char*>(buf) +
                          m_vars[i].mic_offset + m_vars[i].offset;
                    *ptr_addr = ptr;
                }
                break;

            default:
                LIBOFFLOAD_ERROR(c_unknown_var_type, type);
                abort();
        }
        // Release obsolete buffers for stack of persistent objects.
        // The vardesc with i==0 and flags.is_stack_buf==TRUE is always for
        // stack buffer pointer.
        if (i == 0 &&
            m_vars[i].flags.is_stack_buf &&
            !m_vars[i].direction.bits &&
            m_vars[i].alloc_if &&
            m_vars[i].size != 0) {
                for (int j=0; j < m_vars[i].size; j++) {
                    void *buf;
                    m_in.receive_data(&buf, sizeof(buf));
                    OFFLOAD_TRACE(4, "Releasing stack buffer %p\n", buf);
                    BufferReleaseRef(buf);
                    ref_data.erase(buf);
                }
        }
        // Do copyin
        switch (m_vars_extra[i].type_dst) {
            case c_data_ptr_array:
                break;
            case c_data:
            case c_void_ptr:
            case c_void_ptr_ptr:
            case c_cean_var:
                if (m_vars[i].direction.in &&
                    !m_vars[i].flags.is_static_dstn) {
                    int64_t size;
                    int64_t disp;
                    char* ptr = m_vars[i].into ?
                                 static_cast<char*>(m_vars[i].into) :
                                 static_cast<char*>(m_vars[i].ptr);
                    if (m_vars_extra[i].type_dst == c_cean_var) {
                        m_in.receive_data((&size), sizeof(int64_t));
                        m_in.receive_data((&disp), sizeof(int64_t));
                    }
                    else {
                        size = m_vars[i].size;
                        disp = 0;
                    }
                    m_in.receive_data(ptr + disp, size);
                }
                break;

            case c_dv:
                if (m_vars[i].direction.bits ||
                    m_vars[i].alloc_if ||
                    m_vars[i].free_if) {
                    char* ptr = m_vars[i].into ?
                                 static_cast<char*>(m_vars[i].into) :
                                 static_cast<char*>(m_vars[i].ptr);
                    m_in.receive_data(ptr + sizeof(uint64_t),
                                      m_vars[i].size - sizeof(uint64_t));
                }
                break;

            case c_string_ptr:
            case c_data_ptr:
            case c_string_ptr_ptr:
            case c_data_ptr_ptr:
            case c_cean_var_ptr:
            case c_cean_var_ptr_ptr:
            case c_dv_ptr:
            case c_dv_data:
            case c_dv_ptr_data:
            case c_dv_data_slice:
            case c_dv_ptr_data_slice:
                break;

            case c_func_ptr:
            case c_func_ptr_ptr:
                if (m_vars[i].direction.in) {
                    m_in.receive_func_ptr((const void**) m_vars[i].ptr);
                }
                break;

            default:
                LIBOFFLOAD_ERROR(c_unknown_var_type, m_vars_extra[i].type_dst);
                abort();
        }
    }

    OFFLOAD_TRACE(1, "Total copyin data received from host: [%lld] bytes\n",
                  m_in.get_tfr_size());

    OFFLOAD_TIMER_STOP(c_offload_target_scatter_inputs);

    OFFLOAD_TIMER_START(c_offload_target_compute);
}

void OffloadDescriptor::gather_copyout_data()
{
    OFFLOAD_TIMER_STOP(c_offload_target_compute);

    OFFLOAD_TIMER_START(c_offload_target_gather_outputs);

    for (int i = 0; i < m_vars_total; i++) {
        bool src_is_for_mic = (m_vars[i].direction.out ||
                               m_vars[i].into == NULL);
        if (m_vars[i].flags.is_stack_buf) {
            continue;
        }
        switch (m_vars_extra[i].type_src) {
            case c_data_ptr_array:
                break;
            case c_data:
            case c_void_ptr:
            case c_void_ptr_ptr:
            case c_cean_var:
                if (m_vars[i].direction.out &&
                    !m_vars[i].flags.is_static) {
                    m_out.send_data(
                        static_cast<char*>(m_vars[i].ptr) + m_vars[i].disp,
                        m_vars[i].size);
                }
                break;

            case c_dv:
                break;

            case c_string_ptr:
            case c_data_ptr:
            case c_string_ptr_ptr:
            case c_data_ptr_ptr:
            case c_cean_var_ptr:
            case c_cean_var_ptr_ptr:
            case c_dv_ptr:
                if (m_vars[i].free_if &&
                    src_is_for_mic &&
                    !m_vars[i].flags.preallocated &&
                    !m_vars[i].flags.is_static) {
                    void *buf = *static_cast<char**>(m_vars[i].ptr) -
                                    m_vars[i].mic_offset -
                                    (m_vars[i].flags.is_stack_buf?
                                     0 : m_vars[i].offset);
                    if (buf == NULL) {
                        break;
                    }
                    // decrement buffer reference count
                    OFFLOAD_TIMER_START(c_offload_target_release_buffer_refs);
                    BufReleaseRef(buf);
                    OFFLOAD_TIMER_STOP(c_offload_target_release_buffer_refs);
                }
                if (m_vars[i].flags.preallocated && m_vars[i].alloc_if) {
                    m_out.send_data((void*) m_vars[i].ptr, sizeof(void*));
                }
                break;

            case c_func_ptr:
            case c_func_ptr_ptr:
                if (m_vars[i].direction.out) {
                    m_out.send_func_ptr(*((void**) m_vars[i].ptr));
                }
                break;

            case c_dv_data:
            case c_dv_ptr_data:
            case c_dv_data_slice:
            case c_dv_ptr_data_slice:
                if (src_is_for_mic &&
                    m_vars[i].free_if &&
                    !m_vars[i].flags.is_static) {
                    ArrDesc *dvp = (m_vars_extra[i].type_src == c_dv_data ||
                               m_vars_extra[i].type_src == c_dv_data_slice) ?
                               static_cast<ArrDesc*>(m_vars[i].ptr) :
                               *static_cast<ArrDesc**>(m_vars[i].ptr);

                    void *buf = reinterpret_cast<char*>(dvp->Base) -
                                m_vars[i].mic_offset -
                                m_vars[i].offset;

                    if (buf == NULL) {
                        break;
                    }

                    // decrement buffer reference count
                    OFFLOAD_TIMER_START(c_offload_target_release_buffer_refs);
                    BufReleaseRef(buf);
                    OFFLOAD_TIMER_STOP(c_offload_target_release_buffer_refs);
                }
                break;

            default:
                LIBOFFLOAD_ERROR(c_unknown_var_type, m_vars_extra[i].type_dst);
                abort();
        }

        if (m_vars[i].into) {
            switch (m_vars_extra[i].type_dst) {
                case c_data_ptr_array:
                    break;
                case c_data:
                case c_void_ptr:
                case c_void_ptr_ptr:
                case c_cean_var:
                case c_dv:
                    break;

                case c_string_ptr:
                case c_data_ptr:
                case c_string_ptr_ptr:
                case c_data_ptr_ptr:
                case c_cean_var_ptr:
                case c_cean_var_ptr_ptr:
                case c_dv_ptr:
                    if (m_vars[i].direction.in &&
                        m_vars[i].free_if &&
                        !m_vars[i].flags.is_static_dstn) {
                        void *buf = *static_cast<char**>(m_vars[i].into) -
                                    m_vars[i].mic_offset -
                                    (m_vars[i].flags.is_stack_buf?
                                     0 : m_vars[i].offset);

                        if (buf == NULL) {
                            break;
                        }
                        // decrement buffer reference count
                        OFFLOAD_TIMER_START(
                            c_offload_target_release_buffer_refs);
                        BufReleaseRef(buf);
                        OFFLOAD_TIMER_STOP(
                            c_offload_target_release_buffer_refs);
                    }
                    break;

                case c_func_ptr:
                case c_func_ptr_ptr:
                    break;

                case c_dv_data:
                case c_dv_ptr_data:
                case c_dv_data_slice:
                case c_dv_ptr_data_slice:
                    if (m_vars[i].free_if &&
                        m_vars[i].direction.in &&
                        !m_vars[i].flags.is_static_dstn) {
                        ArrDesc *dvp =
                            (m_vars_extra[i].type_dst == c_dv_data_slice ||
                             m_vars_extra[i].type_dst == c_dv_data) ?
                            static_cast<ArrDesc*>(m_vars[i].into) :
                            *static_cast<ArrDesc**>(m_vars[i].into);
                        void *buf = reinterpret_cast<char*>(dvp->Base) -
                              m_vars[i].mic_offset -
                              m_vars[i].offset;

                        if (buf == NULL) {
                            break;
                        }
                        // decrement buffer reference count
                        OFFLOAD_TIMER_START(
                            c_offload_target_release_buffer_refs);
                        BufReleaseRef(buf);
                        OFFLOAD_TIMER_STOP(
                            c_offload_target_release_buffer_refs);
                    }
                    break;

                default:
                    LIBOFFLOAD_ERROR(c_unknown_var_type, m_vars_extra[i].type_dst);
                    abort();
            }
        }
    }

    OFFLOAD_DEBUG_TRACE(2, "OUT buffer @ p %p size %lld\n",
                        m_out.get_buffer_start(),
                        m_out.get_buffer_size());

    OFFLOAD_DEBUG_DUMP_BYTES(2,
                             m_out.get_buffer_start(),
                             m_out.get_buffer_size());

    OFFLOAD_DEBUG_TRACE_1(1, get_offload_number(), c_offload_copyout_data,
                  "Total copyout data sent to host: [%lld] bytes\n",
                  m_out.get_tfr_size());

    OFFLOAD_TIMER_STOP(c_offload_target_gather_outputs);
}

void __offload_target_init(void)
{
#ifdef SEP_SUPPORT
    const char* env_var = getenv(sep_monitor_env);
    if (env_var != 0 && *env_var != '\0') {
        sep_monitor = atoi(env_var);
    }
    env_var = getenv(sep_device_env);
    if (env_var != 0 && *env_var != '\0') {
        sep_device = env_var;
    }
#endif // SEP_SUPPORT

    prefix = report_get_message_str(c_report_mic);

    // init frequency
    mic_frequency = COIPerfGetCycleFrequency();
}

// User-visible offload API

int _Offload_number_of_devices(void)
{
    return mic_engines_total;
}

int _Offload_get_device_number(void)
{
    return mic_index;
}

int _Offload_get_physical_device_number(void)
{
    uint32_t index;
    EngineGetIndex(&index);
    return index;
}