|
150
|
1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
|
|
|
2 //
|
|
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8 //
|
|
|
9 // Implementation of OpenMP API interface functions.
|
|
|
10 //
|
|
|
11 //===----------------------------------------------------------------------===//
|
|
|
12
|
|
|
13 #include <omptarget.h>
|
|
|
14
|
|
|
15 #include "device.h"
|
|
|
16 #include "private.h"
|
|
|
17 #include "rtl.h"
|
|
|
18
|
|
|
19 #include <climits>
|
|
|
20 #include <cstring>
|
|
|
21 #include <cstdlib>
|
|
|
22
|
|
|
23 EXTERN int omp_get_num_devices(void) {
|
|
173
|
24 RTLsMtx->lock();
|
|
150
|
25 size_t Devices_size = Devices.size();
|
|
173
|
26 RTLsMtx->unlock();
|
|
150
|
27
|
|
|
28 DP("Call to omp_get_num_devices returning %zd\n", Devices_size);
|
|
|
29
|
|
|
30 return Devices_size;
|
|
|
31 }
|
|
|
32
|
|
|
33 EXTERN int omp_get_initial_device(void) {
|
|
|
34 DP("Call to omp_get_initial_device returning %d\n", HOST_DEVICE);
|
|
|
35 return HOST_DEVICE;
|
|
|
36 }
|
|
|
37
|
|
|
38 EXTERN void *omp_target_alloc(size_t size, int device_num) {
|
|
|
39 DP("Call to omp_target_alloc for device %d requesting %zu bytes\n",
|
|
|
40 device_num, size);
|
|
|
41
|
|
|
42 if (size <= 0) {
|
|
|
43 DP("Call to omp_target_alloc with non-positive length\n");
|
|
|
44 return NULL;
|
|
|
45 }
|
|
|
46
|
|
|
47 void *rc = NULL;
|
|
|
48
|
|
|
49 if (device_num == omp_get_initial_device()) {
|
|
|
50 rc = malloc(size);
|
|
|
51 DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc));
|
|
|
52 return rc;
|
|
|
53 }
|
|
|
54
|
|
|
55 if (!device_is_ready(device_num)) {
|
|
|
56 DP("omp_target_alloc returns NULL ptr\n");
|
|
|
57 return NULL;
|
|
|
58 }
|
|
|
59
|
|
|
60 DeviceTy &Device = Devices[device_num];
|
|
|
61 rc = Device.RTL->data_alloc(Device.RTLDeviceID, size, NULL);
|
|
|
62 DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc));
|
|
|
63 return rc;
|
|
|
64 }
|
|
|
65
|
|
|
66 EXTERN void omp_target_free(void *device_ptr, int device_num) {
|
|
|
67 DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
|
|
|
68 device_num, DPxPTR(device_ptr));
|
|
|
69
|
|
|
70 if (!device_ptr) {
|
|
|
71 DP("Call to omp_target_free with NULL ptr\n");
|
|
|
72 return;
|
|
|
73 }
|
|
|
74
|
|
|
75 if (device_num == omp_get_initial_device()) {
|
|
|
76 free(device_ptr);
|
|
|
77 DP("omp_target_free deallocated host ptr\n");
|
|
|
78 return;
|
|
|
79 }
|
|
|
80
|
|
|
81 if (!device_is_ready(device_num)) {
|
|
|
82 DP("omp_target_free returns, nothing to do\n");
|
|
|
83 return;
|
|
|
84 }
|
|
|
85
|
|
|
86 DeviceTy &Device = Devices[device_num];
|
|
|
87 Device.RTL->data_delete(Device.RTLDeviceID, (void *)device_ptr);
|
|
|
88 DP("omp_target_free deallocated device ptr\n");
|
|
|
89 }
|
|
|
90
|
|
|
91 EXTERN int omp_target_is_present(void *ptr, int device_num) {
|
|
|
92 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
|
|
|
93 device_num, DPxPTR(ptr));
|
|
|
94
|
|
|
95 if (!ptr) {
|
|
|
96 DP("Call to omp_target_is_present with NULL ptr, returning false\n");
|
|
|
97 return false;
|
|
|
98 }
|
|
|
99
|
|
|
100 if (device_num == omp_get_initial_device()) {
|
|
|
101 DP("Call to omp_target_is_present on host, returning true\n");
|
|
|
102 return true;
|
|
|
103 }
|
|
|
104
|
|
173
|
105 RTLsMtx->lock();
|
|
150
|
106 size_t Devices_size = Devices.size();
|
|
173
|
107 RTLsMtx->unlock();
|
|
150
|
108 if (Devices_size <= (size_t)device_num) {
|
|
|
109 DP("Call to omp_target_is_present with invalid device ID, returning "
|
|
|
110 "false\n");
|
|
|
111 return false;
|
|
|
112 }
|
|
|
113
|
|
|
114 DeviceTy& Device = Devices[device_num];
|
|
|
115 bool IsLast; // not used
|
|
|
116 bool IsHostPtr;
|
|
|
117 void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr);
|
|
|
118 int rc = (TgtPtr != NULL);
|
|
|
119 // Under unified memory the host pointer can be returned by the
|
|
|
120 // getTgtPtrBegin() function which means that there is no device
|
|
|
121 // corresponding point for ptr. This function should return false
|
|
|
122 // in that situation.
|
|
173
|
123 if (RTLs->RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
|
|
150
|
124 rc = !IsHostPtr;
|
|
|
125 DP("Call to omp_target_is_present returns %d\n", rc);
|
|
|
126 return rc;
|
|
|
127 }
|
|
|
128
|
|
|
129 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
|
|
|
130 size_t dst_offset, size_t src_offset, int dst_device, int src_device) {
|
|
|
131 DP("Call to omp_target_memcpy, dst device %d, src device %d, "
|
|
|
132 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
|
|
|
133 "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst),
|
|
|
134 DPxPTR(src), dst_offset, src_offset, length);
|
|
|
135
|
|
|
136 if (!dst || !src || length <= 0) {
|
|
|
137 DP("Call to omp_target_memcpy with invalid arguments\n");
|
|
|
138 return OFFLOAD_FAIL;
|
|
|
139 }
|
|
|
140
|
|
|
141 if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
|
|
|
142 DP("omp_target_memcpy returns OFFLOAD_FAIL\n");
|
|
|
143 return OFFLOAD_FAIL;
|
|
|
144 }
|
|
|
145
|
|
|
146 if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
|
|
|
147 DP("omp_target_memcpy returns OFFLOAD_FAIL\n");
|
|
|
148 return OFFLOAD_FAIL;
|
|
|
149 }
|
|
|
150
|
|
|
151 int rc = OFFLOAD_SUCCESS;
|
|
|
152 void *srcAddr = (char *)src + src_offset;
|
|
|
153 void *dstAddr = (char *)dst + dst_offset;
|
|
|
154
|
|
|
155 if (src_device == omp_get_initial_device() &&
|
|
|
156 dst_device == omp_get_initial_device()) {
|
|
|
157 DP("copy from host to host\n");
|
|
|
158 const void *p = memcpy(dstAddr, srcAddr, length);
|
|
|
159 if (p == NULL)
|
|
|
160 rc = OFFLOAD_FAIL;
|
|
|
161 } else if (src_device == omp_get_initial_device()) {
|
|
|
162 DP("copy from host to device\n");
|
|
|
163 DeviceTy& DstDev = Devices[dst_device];
|
|
173
|
164 rc = DstDev.data_submit(dstAddr, srcAddr, length, nullptr);
|
|
150
|
165 } else if (dst_device == omp_get_initial_device()) {
|
|
|
166 DP("copy from device to host\n");
|
|
|
167 DeviceTy& SrcDev = Devices[src_device];
|
|
173
|
168 rc = SrcDev.data_retrieve(dstAddr, srcAddr, length, nullptr);
|
|
150
|
169 } else {
|
|
|
170 DP("copy from device to device\n");
|
|
|
171 void *buffer = malloc(length);
|
|
|
172 DeviceTy& SrcDev = Devices[src_device];
|
|
|
173 DeviceTy& DstDev = Devices[dst_device];
|
|
173
|
174 rc = SrcDev.data_retrieve(buffer, srcAddr, length, nullptr);
|
|
150
|
175 if (rc == OFFLOAD_SUCCESS)
|
|
173
|
176 rc = DstDev.data_submit(dstAddr, buffer, length, nullptr);
|
|
|
177 free(buffer);
|
|
150
|
178 }
|
|
|
179
|
|
|
180 DP("omp_target_memcpy returns %d\n", rc);
|
|
|
181 return rc;
|
|
|
182 }
|
|
|
183
|
|
|
184 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
|
|
|
185 int num_dims, const size_t *volume, const size_t *dst_offsets,
|
|
|
186 const size_t *src_offsets, const size_t *dst_dimensions,
|
|
|
187 const size_t *src_dimensions, int dst_device, int src_device) {
|
|
|
188 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
|
|
|
189 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
|
|
|
190 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
|
|
|
191 "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device,
|
|
|
192 src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
|
|
|
193 DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
|
|
|
194 DPxPTR(volume), element_size, num_dims);
|
|
|
195
|
|
|
196 if (!(dst || src)) {
|
|
|
197 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
|
|
|
198 INT_MAX);
|
|
|
199 return INT_MAX;
|
|
|
200 }
|
|
|
201
|
|
|
202 if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
|
|
|
203 !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
|
|
|
204 DP("Call to omp_target_memcpy_rect with invalid arguments\n");
|
|
|
205 return OFFLOAD_FAIL;
|
|
|
206 }
|
|
|
207
|
|
|
208 int rc;
|
|
|
209 if (num_dims == 1) {
|
|
|
210 rc = omp_target_memcpy(dst, src, element_size * volume[0],
|
|
|
211 element_size * dst_offsets[0], element_size * src_offsets[0],
|
|
|
212 dst_device, src_device);
|
|
|
213 } else {
|
|
|
214 size_t dst_slice_size = element_size;
|
|
|
215 size_t src_slice_size = element_size;
|
|
|
216 for (int i=1; i<num_dims; ++i) {
|
|
|
217 dst_slice_size *= dst_dimensions[i];
|
|
|
218 src_slice_size *= src_dimensions[i];
|
|
|
219 }
|
|
|
220
|
|
|
221 size_t dst_off = dst_offsets[0] * dst_slice_size;
|
|
|
222 size_t src_off = src_offsets[0] * src_slice_size;
|
|
|
223 for (size_t i=0; i<volume[0]; ++i) {
|
|
|
224 rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i,
|
|
|
225 (char *) src + src_off + src_slice_size * i, element_size,
|
|
|
226 num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1,
|
|
|
227 dst_dimensions + 1, src_dimensions + 1, dst_device, src_device);
|
|
|
228
|
|
|
229 if (rc) {
|
|
|
230 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
|
|
|
231 return rc;
|
|
|
232 }
|
|
|
233 }
|
|
|
234 }
|
|
|
235
|
|
|
236 DP("omp_target_memcpy_rect returns %d\n", rc);
|
|
|
237 return rc;
|
|
|
238 }
|
|
|
239
|
|
|
240 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
|
|
|
241 size_t size, size_t device_offset, int device_num) {
|
|
|
242 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
|
|
|
243 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
|
|
|
244 DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
|
|
|
245
|
|
|
246 if (!host_ptr || !device_ptr || size <= 0) {
|
|
|
247 DP("Call to omp_target_associate_ptr with invalid arguments\n");
|
|
|
248 return OFFLOAD_FAIL;
|
|
|
249 }
|
|
|
250
|
|
|
251 if (device_num == omp_get_initial_device()) {
|
|
|
252 DP("omp_target_associate_ptr: no association possible on the host\n");
|
|
|
253 return OFFLOAD_FAIL;
|
|
|
254 }
|
|
|
255
|
|
|
256 if (!device_is_ready(device_num)) {
|
|
|
257 DP("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
|
|
|
258 return OFFLOAD_FAIL;
|
|
|
259 }
|
|
|
260
|
|
|
261 DeviceTy& Device = Devices[device_num];
|
|
|
262 void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
|
|
|
263 int rc = Device.associatePtr(host_ptr, device_addr, size);
|
|
|
264 DP("omp_target_associate_ptr returns %d\n", rc);
|
|
|
265 return rc;
|
|
|
266 }
|
|
|
267
|
|
|
268 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) {
|
|
|
269 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
|
|
|
270 "device_num %d\n", DPxPTR(host_ptr), device_num);
|
|
|
271
|
|
|
272 if (!host_ptr) {
|
|
|
273 DP("Call to omp_target_associate_ptr with invalid host_ptr\n");
|
|
|
274 return OFFLOAD_FAIL;
|
|
|
275 }
|
|
|
276
|
|
|
277 if (device_num == omp_get_initial_device()) {
|
|
|
278 DP("omp_target_disassociate_ptr: no association possible on the host\n");
|
|
|
279 return OFFLOAD_FAIL;
|
|
|
280 }
|
|
|
281
|
|
|
282 if (!device_is_ready(device_num)) {
|
|
|
283 DP("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
|
|
|
284 return OFFLOAD_FAIL;
|
|
|
285 }
|
|
|
286
|
|
|
287 DeviceTy& Device = Devices[device_num];
|
|
|
288 int rc = Device.disassociatePtr(host_ptr);
|
|
|
289 DP("omp_target_disassociate_ptr returns %d\n", rc);
|
|
|
290 return rc;
|
|
|
291 }
|
