1
// SPDX-License-Identifier: GPL-2.0-only or MIT
2
/* Copyright 2025 Arm, Ltd. */
3

4
#include <linux/err.h>
5
#include <linux/slab.h>
6

7
#include <drm/ethosu_accel.h>
8

9
#include "ethosu_device.h"
10
#include "ethosu_gem.h"
11

12
static void ethosu_gem_free_object(struct drm_gem_object *obj)
13
{
14
	struct ethosu_gem_object *bo = to_ethosu_bo(obj);
15

16
	kfree(bo->info);
17
	drm_gem_free_mmap_offset(&bo->base.base);
18
	drm_gem_dma_free(&bo->base);
19
}
20

21
static int ethosu_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
22
{
23
	struct ethosu_gem_object *bo = to_ethosu_bo(obj);
24

25
	/* Don't allow mmap on objects that have the NO_MMAP flag set. */
26
	if (bo->flags & DRM_ETHOSU_BO_NO_MMAP)
27
		return -EINVAL;
28

29
	return drm_gem_dma_object_mmap(obj, vma);
30
}
31

32
static const struct drm_gem_object_funcs ethosu_gem_funcs = {
33
	.free = ethosu_gem_free_object,
34
	.print_info = drm_gem_dma_object_print_info,
35
	.get_sg_table = drm_gem_dma_object_get_sg_table,
36
	.vmap = drm_gem_dma_object_vmap,
37
	.mmap = ethosu_gem_mmap,
38
	.vm_ops = &drm_gem_dma_vm_ops,
39
};
40

41
/**
42
 * ethosu_gem_create_object - Implementation of driver->gem_create_object.
43
 * @ddev: DRM device
44
 * @size: Size in bytes of the memory the object will reference
45
 *
46
 * This lets the GEM helpers allocate object structs for us, and keep
47
 * our BO stats correct.
48
 */
49
struct drm_gem_object *ethosu_gem_create_object(struct drm_device *ddev, size_t size)
50
{
51
	struct ethosu_gem_object *obj;
52

53
	obj = kzalloc_obj(*obj);
54
	if (!obj)
55
		return ERR_PTR(-ENOMEM);
56

57
	obj->base.base.funcs = &ethosu_gem_funcs;
58
	return &obj->base.base;
59
}
60

61
/**
62
 * ethosu_gem_create_with_handle() - Create a GEM object and attach it to a handle.
63
 * @file: DRM file.
64
 * @ddev: DRM device.
65
 * @size: Size of the GEM object to allocate.
66
 * @flags: Combination of drm_ethosu_bo_flags flags.
67
 * @handle: Pointer holding the handle pointing to the new GEM object.
68
 *
69
 * Return: Zero on success
70
 */
71
int ethosu_gem_create_with_handle(struct drm_file *file,
72
				  struct drm_device *ddev,
73
				  u64 *size, u32 flags, u32 *handle)
74
{
75
	struct drm_gem_dma_object *mem;
76
	struct ethosu_gem_object *bo;
77
	int ret;
78

79
	mem = drm_gem_dma_create(ddev, *size);
80
	if (IS_ERR(mem))
81
		return PTR_ERR(mem);
82

83
	bo = to_ethosu_bo(&mem->base);
84
	bo->flags = flags;
85

86
	/*
87
	 * Allocate an id of idr table where the obj is registered
88
	 * and handle has the id what user can see.
89
	 */
90
	ret = drm_gem_handle_create(file, &mem->base, handle);
91
	if (!ret)
92
		*size = bo->base.base.size;
93

94
	/* drop reference from allocate - handle holds it now. */
95
	drm_gem_object_put(&mem->base);
96

97
	return ret;
98
}
99

100
struct dma {
101
	s8 region;
102
	u64 len;
103
	u64 offset;
104
	s64 stride[2];
105
};
106

107
struct dma_state {
108
	u16 size0;
109
	u16 size1;
110
	s8 mode;
111
	struct dma src;
112
	struct dma dst;
113
};
114

115
struct buffer {
116
	u64 base;
117
	u32 length;
118
	s8 region;
119
};
120

121
struct feat_matrix {
122
	u64 base[4];
123
	s64 stride_x;
124
	s64 stride_y;
125
	s64 stride_c;
126
	s8 region;
127
	u8 broadcast;
128
	u16 stride_kernel;
129
	u16 precision;
130
	u16 depth;
131
	u16 width;
132
	u16 width0;
133
	u16 height[3];
134
	u8 pad_top;
135
	u8 pad_left;
136
	u8 pad_bottom;
137
	u8 pad_right;
138
};
139

140
struct cmd_state {
141
	struct dma_state dma;
142
	struct buffer scale[2];
143
	struct buffer weight[4];
144
	struct feat_matrix ofm;
145
	struct feat_matrix ifm;
146
	struct feat_matrix ifm2;
147
};
148

149
static void cmd_state_init(struct cmd_state *st)
150
{
151
	/* Initialize to all 1s to detect missing setup */
152
	memset(st, 0xff, sizeof(*st));
153
}
154

155
static u64 cmd_to_addr(u32 *cmd)
156
{
157
	return (((u64)cmd[0] & 0xff0000) << 16) | cmd[1];
158
}
159

160
static u64 dma_length(struct ethosu_validated_cmdstream_info *info,
161
		      struct dma_state *dma_st, struct dma *dma)
162
{
163
	s8 mode = dma_st->mode;
164
	u64 len = dma->len;
165

166
	if (mode >= 1) {
167
		len += dma->stride[0];
168
		len *= dma_st->size0;
169
	}
170
	if (mode == 2) {
171
		len += dma->stride[1];
172
		len *= dma_st->size1;
173
	}
174
	if (dma->region >= 0)
175
		info->region_size[dma->region] = max(info->region_size[dma->region],
176
						     len + dma->offset);
177

178
	return len;
179
}
180

181
static u64 feat_matrix_length(struct ethosu_validated_cmdstream_info *info,
182
			      struct feat_matrix *fm,
183
			      u32 x, u32 y, u32 c)
184
{
185
	u32 element_size, storage = fm->precision >> 14;
186
	int tile = 0;
187
	u64 addr;
188

189
	if (fm->region < 0)
190
		return U64_MAX;
191

192
	switch (storage) {
193
	case 0:
194
		if (x >= fm->width0 + 1) {
195
			x -= fm->width0 + 1;
196
			tile += 1;
197
		}
198
		if (y >= fm->height[tile] + 1) {
199
			y -= fm->height[tile] + 1;
200
			tile += 2;
201
		}
202
		break;
203
	case 1:
204
		if (y >= fm->height[1] + 1) {
205
			y -= fm->height[1] + 1;
206
			tile = 2;
207
		} else if (y >= fm->height[0] + 1) {
208
			y -= fm->height[0] + 1;
209
			tile = 1;
210
		}
211
		break;
212
	}
213
	if (fm->base[tile] == U64_MAX)
214
		return U64_MAX;
215

216
	addr = fm->base[tile] + y * fm->stride_y;
217

218
	switch ((fm->precision >> 6) & 0x3) { // format
219
	case 0: //nhwc:
220
		addr += x * fm->stride_x + c;
221
		break;
222
	case 1: //nhcwb16:
223
		element_size = BIT((fm->precision >> 1) & 0x3);
224

225
		addr += (c / 16) * fm->stride_c + (16 * x + (c & 0xf)) * element_size;
226
		break;
227
	}
228

229
	info->region_size[fm->region] = max(info->region_size[fm->region], addr + 1);
230

231
	return addr;
232
}
233

234
static int calc_sizes(struct drm_device *ddev,
235
		      struct ethosu_validated_cmdstream_info *info,
236
		      u16 op, struct cmd_state *st,
237
		      bool ifm, bool ifm2, bool weight, bool scale)
238
{
239
	u64 len;
240

241
	if (ifm) {
242
		if (st->ifm.stride_kernel == U16_MAX)
243
			return -EINVAL;
244
		u32 stride_y = ((st->ifm.stride_kernel >> 8) & 0x2) +
245
			((st->ifm.stride_kernel >> 1) & 0x1) + 1;
246
		u32 stride_x = ((st->ifm.stride_kernel >> 5) & 0x2) +
247
			(st->ifm.stride_kernel & 0x1) + 1;
248
		s32 ifm_height = st->ofm.height[2] * stride_y +
249
			st->ifm.height[2] - (st->ifm.pad_top + st->ifm.pad_bottom);
250
		s32 ifm_width = st->ofm.width * stride_x +
251
			st->ifm.width - (st->ifm.pad_left + st->ifm.pad_right);
252

253
		if (ifm_height < 0 || ifm_width < 0)
254
			return -EINVAL;
255

256
		len = feat_matrix_length(info, &st->ifm, ifm_width,
257
					 ifm_height, st->ifm.depth);
258
		dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n",
259
			op, st->ifm.region, st->ifm.base[0], len);
260
		if (len == U64_MAX)
261
			return -EINVAL;
262
	}
263

264
	if (ifm2) {
265
		len = feat_matrix_length(info, &st->ifm2, st->ifm.depth,
266
					 0, st->ofm.depth);
267
		dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n",
268
			op, st->ifm2.region, st->ifm2.base[0], len);
269
		if (len == U64_MAX)
270
			return -EINVAL;
271
	}
272

273
	if (weight) {
274
		dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n",
275
			op, st->weight[0].region, st->weight[0].base,
276
			st->weight[0].base + st->weight[0].length - 1);
277
		if (st->weight[0].region < 0 || st->weight[0].base == U64_MAX ||
278
		    st->weight[0].length == U32_MAX)
279
			return -EINVAL;
280
		info->region_size[st->weight[0].region] =
281
			max(info->region_size[st->weight[0].region],
282
			    st->weight[0].base + st->weight[0].length);
283
	}
284

285
	if (scale) {
286
		dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n",
287
			op, st->scale[0].region, st->scale[0].base,
288
			st->scale[0].base + st->scale[0].length - 1);
289
		if (st->scale[0].region < 0 || st->scale[0].base == U64_MAX ||
290
		    st->scale[0].length == U32_MAX)
291
			return -EINVAL;
292
		info->region_size[st->scale[0].region] =
293
			max(info->region_size[st->scale[0].region],
294
			    st->scale[0].base + st->scale[0].length);
295
	}
296

297
	len = feat_matrix_length(info, &st->ofm, st->ofm.width,
298
				 st->ofm.height[2], st->ofm.depth);
299
	dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n",
300
		op, st->ofm.region, st->ofm.base[0], len);
301
	if (len == U64_MAX)
302
		return -EINVAL;
303
	info->output_region[st->ofm.region] = true;
304

305
	return 0;
306
}
307

308
static int calc_sizes_elemwise(struct drm_device *ddev,
309
			       struct ethosu_validated_cmdstream_info *info,
310
			       u16 op, struct cmd_state *st,
311
			       bool ifm, bool ifm2)
312
{
313
	u32 height, width, depth;
314
	u64 len;
315

316
	if (ifm) {
317
		height = st->ifm.broadcast & 0x1 ? 0 : st->ofm.height[2];
318
		width = st->ifm.broadcast & 0x2 ? 0 : st->ofm.width;
319
		depth = st->ifm.broadcast & 0x4 ? 0 : st->ofm.depth;
320

321
		len = feat_matrix_length(info, &st->ifm, width,
322
					 height, depth);
323
		dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n",
324
			op, st->ifm.region, st->ifm.base[0], len);
325
		if (len == U64_MAX)
326
			return -EINVAL;
327
	}
328

329
	if (ifm2) {
330
		height = st->ifm2.broadcast & 0x1 ? 0 : st->ofm.height[2];
331
		width = st->ifm2.broadcast & 0x2 ? 0 : st->ofm.width;
332
		depth = st->ifm2.broadcast & 0x4 ? 0 : st->ofm.depth;
333

334
		len = feat_matrix_length(info, &st->ifm2, width,
335
					 height, depth);
336
		dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n",
337
			op, st->ifm2.region, st->ifm2.base[0], len);
338
		if (len == U64_MAX)
339
			return -EINVAL;
340
	}
341

342
	len = feat_matrix_length(info, &st->ofm, st->ofm.width,
343
				 st->ofm.height[2], st->ofm.depth);
344
	dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n",
345
		op, st->ofm.region, st->ofm.base[0], len);
346
	if (len == U64_MAX)
347
		return -EINVAL;
348
	info->output_region[st->ofm.region] = true;
349

350
	return 0;
351
}
352

353
static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev,
354
						  u32 __user *ucmds,
355
						  struct ethosu_gem_object *bo,
356
						  u32 size)
357
{
358
	struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc_obj(*info);
359
	struct ethosu_device *edev = to_ethosu_device(ddev);
360
	u32 *bocmds = bo->base.vaddr;
361
	struct cmd_state st;
362
	int i, ret;
363

364
	if (!info)
365
		return -ENOMEM;
366
	info->cmd_size = size;
367

368
	cmd_state_init(&st);
369

370
	for (i = 0; i < size / 4; i++) {
371
		bool use_ifm, use_ifm2, use_scale;
372
		u64 dstlen, srclen;
373
		u16 cmd, param;
374
		u32 cmds[2];
375
		u64 addr;
376

377
		if (get_user(cmds[0], ucmds++))
378
			return -EFAULT;
379

380
		bocmds[i] = cmds[0];
381

382
		cmd = cmds[0];
383
		param = cmds[0] >> 16;
384

385
		if (cmd & 0x4000) {
386
			if (get_user(cmds[1], ucmds++))
387
				return -EFAULT;
388

389
			i++;
390
			bocmds[i] = cmds[1];
391
			addr = cmd_to_addr(cmds);
392
		}
393

394
		switch (cmd) {
395
		case NPU_OP_DMA_START:
396
			srclen = dma_length(info, &st.dma, &st.dma.src);
397
			dstlen = dma_length(info, &st.dma, &st.dma.dst);
398

399
			if (st.dma.dst.region >= 0)
400
				info->output_region[st.dma.dst.region] = true;
401
			dev_dbg(ddev->dev, "cmd: DMA SRC:%d:0x%llx+0x%llx DST:%d:0x%llx+0x%llx\n",
402
				st.dma.src.region, st.dma.src.offset, srclen,
403
				st.dma.dst.region, st.dma.dst.offset, dstlen);
404
			break;
405
		case NPU_OP_CONV:
406
		case NPU_OP_DEPTHWISE:
407
			use_ifm2 = param & 0x1;  // weights_ifm2
408
			use_scale = !(st.ofm.precision & 0x100);
409
			ret = calc_sizes(ddev, info, cmd, &st, true, use_ifm2,
410
					 !use_ifm2, use_scale);
411
			if (ret)
412
				return ret;
413
			break;
414
		case NPU_OP_POOL:
415
			use_ifm = param != 0x4;  // pooling mode
416
			use_scale = !(st.ofm.precision & 0x100);
417
			ret = calc_sizes(ddev, info, cmd, &st, use_ifm, false,
418
					 false, use_scale);
419
			if (ret)
420
				return ret;
421
			break;
422
		case NPU_OP_ELEMENTWISE:
423
			use_scale = ethosu_is_u65(edev) ?
424
				    (st.ifm2.broadcast & 0x80) :
425
				    (st.ifm2.broadcast == 8);
426
			use_ifm2 = !(use_scale || (param == 5) ||
427
				(param == 6) || (param == 7) || (param == 0x24));
428
			use_ifm = st.ifm.broadcast != 8;
429
			ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2);
430
			if (ret)
431
				return ret;
432
			break;
433
		case NPU_OP_RESIZE: // U85 only
434
			WARN_ON(1); // TODO
435
			break;
436
		case NPU_SET_KERNEL_WIDTH_M1:
437
			st.ifm.width = param;
438
			break;
439
		case NPU_SET_KERNEL_HEIGHT_M1:
440
			st.ifm.height[2] = param;
441
			break;
442
		case NPU_SET_KERNEL_STRIDE:
443
			st.ifm.stride_kernel = param;
444
			break;
445
		case NPU_SET_IFM_PAD_TOP:
446
			st.ifm.pad_top = param & 0x7f;
447
			break;
448
		case NPU_SET_IFM_PAD_LEFT:
449
			st.ifm.pad_left = param & 0x7f;
450
			break;
451
		case NPU_SET_IFM_PAD_RIGHT:
452
			st.ifm.pad_right = param & 0xff;
453
			break;
454
		case NPU_SET_IFM_PAD_BOTTOM:
455
			st.ifm.pad_bottom = param & 0xff;
456
			break;
457
		case NPU_SET_IFM_DEPTH_M1:
458
			st.ifm.depth = param;
459
			break;
460
		case NPU_SET_IFM_PRECISION:
461
			st.ifm.precision = param;
462
			break;
463
		case NPU_SET_IFM_BROADCAST:
464
			st.ifm.broadcast = param;
465
			break;
466
		case NPU_SET_IFM_REGION:
467
			st.ifm.region = param & 0x7f;
468
			break;
469
		case NPU_SET_IFM_WIDTH0_M1:
470
			st.ifm.width0 = param;
471
			break;
472
		case NPU_SET_IFM_HEIGHT0_M1:
473
			st.ifm.height[0] = param;
474
			break;
475
		case NPU_SET_IFM_HEIGHT1_M1:
476
			st.ifm.height[1] = param;
477
			break;
478
		case NPU_SET_IFM_BASE0:
479
		case NPU_SET_IFM_BASE1:
480
		case NPU_SET_IFM_BASE2:
481
		case NPU_SET_IFM_BASE3:
482
			st.ifm.base[cmd & 0x3] = addr;
483
			break;
484
		case NPU_SET_IFM_STRIDE_X:
485
			st.ifm.stride_x = addr;
486
			break;
487
		case NPU_SET_IFM_STRIDE_Y:
488
			st.ifm.stride_y = addr;
489
			break;
490
		case NPU_SET_IFM_STRIDE_C:
491
			st.ifm.stride_c = addr;
492
			break;
493

494
		case NPU_SET_OFM_WIDTH_M1:
495
			st.ofm.width = param;
496
			break;
497
		case NPU_SET_OFM_HEIGHT_M1:
498
			st.ofm.height[2] = param;
499
			break;
500
		case NPU_SET_OFM_DEPTH_M1:
501
			st.ofm.depth = param;
502
			break;
503
		case NPU_SET_OFM_PRECISION:
504
			st.ofm.precision = param;
505
			break;
506
		case NPU_SET_OFM_REGION:
507
			st.ofm.region = param & 0x7;
508
			break;
509
		case NPU_SET_OFM_WIDTH0_M1:
510
			st.ofm.width0 = param;
511
			break;
512
		case NPU_SET_OFM_HEIGHT0_M1:
513
			st.ofm.height[0] = param;
514
			break;
515
		case NPU_SET_OFM_HEIGHT1_M1:
516
			st.ofm.height[1] = param;
517
			break;
518
		case NPU_SET_OFM_BASE0:
519
		case NPU_SET_OFM_BASE1:
520
		case NPU_SET_OFM_BASE2:
521
		case NPU_SET_OFM_BASE3:
522
			st.ofm.base[cmd & 0x3] = addr;
523
			break;
524
		case NPU_SET_OFM_STRIDE_X:
525
			st.ofm.stride_x = addr;
526
			break;
527
		case NPU_SET_OFM_STRIDE_Y:
528
			st.ofm.stride_y = addr;
529
			break;
530
		case NPU_SET_OFM_STRIDE_C:
531
			st.ofm.stride_c = addr;
532
			break;
533

534
		case NPU_SET_IFM2_BROADCAST:
535
			st.ifm2.broadcast = param;
536
			break;
537
		case NPU_SET_IFM2_PRECISION:
538
			st.ifm2.precision = param;
539
			break;
540
		case NPU_SET_IFM2_REGION:
541
			st.ifm2.region = param & 0x7;
542
			break;
543
		case NPU_SET_IFM2_WIDTH0_M1:
544
			st.ifm2.width0 = param;
545
			break;
546
		case NPU_SET_IFM2_HEIGHT0_M1:
547
			st.ifm2.height[0] = param;
548
			break;
549
		case NPU_SET_IFM2_HEIGHT1_M1:
550
			st.ifm2.height[1] = param;
551
			break;
552
		case NPU_SET_IFM2_BASE0:
553
		case NPU_SET_IFM2_BASE1:
554
		case NPU_SET_IFM2_BASE2:
555
		case NPU_SET_IFM2_BASE3:
556
			st.ifm2.base[cmd & 0x3] = addr;
557
			break;
558
		case NPU_SET_IFM2_STRIDE_X:
559
			st.ifm2.stride_x = addr;
560
			break;
561
		case NPU_SET_IFM2_STRIDE_Y:
562
			st.ifm2.stride_y = addr;
563
			break;
564
		case NPU_SET_IFM2_STRIDE_C:
565
			st.ifm2.stride_c = addr;
566
			break;
567

568
		case NPU_SET_WEIGHT_REGION:
569
			st.weight[0].region = param & 0x7;
570
			break;
571
		case NPU_SET_SCALE_REGION:
572
			st.scale[0].region = param & 0x7;
573
			break;
574
		case NPU_SET_WEIGHT_BASE:
575
			st.weight[0].base = addr;
576
			break;
577
		case NPU_SET_WEIGHT_LENGTH:
578
			st.weight[0].length = cmds[1];
579
			break;
580
		case NPU_SET_SCALE_BASE:
581
			st.scale[0].base = addr;
582
			break;
583
		case NPU_SET_SCALE_LENGTH:
584
			st.scale[0].length = cmds[1];
585
			break;
586
		case NPU_SET_WEIGHT1_BASE:
587
			st.weight[1].base = addr;
588
			break;
589
		case NPU_SET_WEIGHT1_LENGTH:
590
			st.weight[1].length = cmds[1];
591
			break;
592
		case NPU_SET_SCALE1_BASE: // NPU_SET_WEIGHT2_BASE (U85)
593
			if (ethosu_is_u65(edev))
594
				st.scale[1].base = addr;
595
			else
596
				st.weight[2].base = addr;
597
			break;
598
		case NPU_SET_SCALE1_LENGTH: // NPU_SET_WEIGHT2_LENGTH (U85)
599
			if (ethosu_is_u65(edev))
600
				st.scale[1].length = cmds[1];
601
			else
602
				st.weight[1].length = cmds[1];
603
			break;
604
		case NPU_SET_WEIGHT3_BASE:
605
			st.weight[3].base = addr;
606
			break;
607
		case NPU_SET_WEIGHT3_LENGTH:
608
			st.weight[3].length = cmds[1];
609
			break;
610

611
		case NPU_SET_DMA0_SRC_REGION:
612
			if (param & 0x100)
613
				st.dma.src.region = -1;
614
			else
615
				st.dma.src.region = param & 0x7;
616
			st.dma.mode = (param >> 9) & 0x3;
617
			break;
618
		case NPU_SET_DMA0_DST_REGION:
619
			if (param & 0x100)
620
				st.dma.dst.region = -1;
621
			else
622
				st.dma.dst.region = param & 0x7;
623
			break;
624
		case NPU_SET_DMA0_SIZE0:
625
			st.dma.size0 = param;
626
			break;
627
		case NPU_SET_DMA0_SIZE1:
628
			st.dma.size1 = param;
629
			break;
630
		case NPU_SET_DMA0_SRC_STRIDE0:
631
			st.dma.src.stride[0] = ((s64)addr << 24) >> 24;
632
			break;
633
		case NPU_SET_DMA0_SRC_STRIDE1:
634
			st.dma.src.stride[1] = ((s64)addr << 24) >> 24;
635
			break;
636
		case NPU_SET_DMA0_DST_STRIDE0:
637
			st.dma.dst.stride[0] = ((s64)addr << 24) >> 24;
638
			break;
639
		case NPU_SET_DMA0_DST_STRIDE1:
640
			st.dma.dst.stride[1] = ((s64)addr << 24) >> 24;
641
			break;
642
		case NPU_SET_DMA0_SRC:
643
			st.dma.src.offset = addr;
644
			break;
645
		case NPU_SET_DMA0_DST:
646
			st.dma.dst.offset = addr;
647
			break;
648
		case NPU_SET_DMA0_LEN:
649
			st.dma.src.len = st.dma.dst.len = addr;
650
			break;
651
		default:
652
			break;
653
		}
654
	}
655

656
	for (i = 0; i < NPU_BASEP_REGION_MAX; i++) {
657
		if (!info->region_size[i])
658
			continue;
659
		dev_dbg(ddev->dev, "region %d max size: 0x%llx\n",
660
			i, info->region_size[i]);
661
	}
662

663
	bo->info = no_free_ptr(info);
664
	return 0;
665
}
666

667
/**
668
 * ethosu_gem_cmdstream_create() - Create a GEM object and attach it to a handle.
669
 * @file: DRM file.
670
 * @ddev: DRM device.
671
 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared.
672
 * @size: Size of the GEM object to allocate.
673
 * @flags: Combination of drm_ethosu_bo_flags flags.
674
 * @handle: Pointer holding the handle pointing to the new GEM object.
675
 *
676
 * Return: Zero on success
677
 */
678
int ethosu_gem_cmdstream_create(struct drm_file *file,
679
				struct drm_device *ddev,
680
				u32 size, u64 data, u32 flags, u32 *handle)
681
{
682
	int ret;
683
	struct drm_gem_dma_object *mem;
684
	struct ethosu_gem_object *bo;
685

686
	mem = drm_gem_dma_create(ddev, size);
687
	if (IS_ERR(mem))
688
		return PTR_ERR(mem);
689

690
	bo = to_ethosu_bo(&mem->base);
691
	bo->flags = flags;
692

693
	ret = ethosu_gem_cmdstream_copy_and_validate(ddev,
694
						     (void __user *)(uintptr_t)data,
695
						     bo, size);
696
	if (ret)
697
		goto fail;
698

699
	/*
700
	 * Allocate an id of idr table where the obj is registered
701
	 * and handle has the id what user can see.
702
	 */
703
	ret = drm_gem_handle_create(file, &mem->base, handle);
704

705
fail:
706
	/* drop reference from allocate - handle holds it now. */
707
	drm_gem_object_put(&mem->base);
708

709
	return ret;
710
}
711

712