path: root/drivers/gpu/drm/xe/xe_exec_queue.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2021 Intel Corporation
 */

#include "xe_exec_queue.h"

#include <linux/nospec.h>

#include <drm/drm_device.h>
#include <drm/drm_file.h>
#include <uapi/drm/xe_drm.h>

#include "xe_device.h"
#include "xe_gt.h"
#include "xe_hw_engine_class_sysfs.h"
#include "xe_hw_engine_group.h"
#include "xe_hw_fence.h"
#include "xe_lrc.h"
#include "xe_macros.h"
#include "xe_migrate.h"
#include "xe_pm.h"
#include "xe_ring_ops_types.h"
#include "xe_trace.h"
#include "xe_vm.h"

enum xe_exec_queue_sched_prop {
	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
	XE_EXEC_QUEUE_TIMESLICE = 1,
	XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,
	XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,
};

static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
				      u64 extensions, int ext_number);

static void __xe_exec_queue_free(struct xe_exec_queue *q)
{
	if (q->vm)
		xe_vm_put(q->vm);

	if (q->xef)
		xe_file_put(q->xef);

	kfree(q);
}

static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
						   struct xe_vm *vm,
						   u32 logical_mask,
						   u16 width, struct xe_hw_engine *hwe,
						   u32 flags, u64 extensions)
{
	struct xe_exec_queue *q;
	struct xe_gt *gt = hwe->gt;
	int err;

	/* only kernel queues can be permanent */
	XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL));

	q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL);
	if (!q)
		return ERR_PTR(-ENOMEM);

	kref_init(&q->refcount);
	q->flags = flags;
	q->hwe = hwe;
	q->gt = gt;
	q->class = hwe->class;
	q->width = width;
	q->logical_mask = logical_mask;
	q->fence_irq = &gt->fence_irq[hwe->class];
	q->ring_ops = gt->ring_ops[hwe->class];
	q->ops = gt->exec_queue_ops;
	INIT_LIST_HEAD(&q->lr.link);
	INIT_LIST_HEAD(&q->multi_gt_link);
	INIT_LIST_HEAD(&q->hw_engine_group_link);

	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
	q->sched_props.preempt_timeout_us =
				hwe->eclass->sched_props.preempt_timeout_us;
	q->sched_props.job_timeout_ms =
				hwe->eclass->sched_props.job_timeout_ms;
	if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&
	    q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)
		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;
	else
		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL;

	if (vm)
		q->vm = xe_vm_get(vm);

	if (extensions) {
		/*
		 * may set q->usm, must come before xe_lrc_create(),
		 * may overwrite q->sched_props, must come before q->ops->init()
		 */
		err = exec_queue_user_extensions(xe, q, extensions, 0);
		if (err) {
			__xe_exec_queue_free(q);
			return ERR_PTR(err);
		}
	}

	return q;
}

static int __xe_exec_queue_init(struct xe_exec_queue *q)
{
	struct xe_vm *vm = q->vm;
	int i, err;

	if (vm) {
		err = xe_vm_lock(vm, true);
		if (err)
			return err;
	}

	for (i = 0; i < q->width; ++i) {
		q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K);
		if (IS_ERR(q->lrc[i])) {
			err = PTR_ERR(q->lrc[i]);
			goto err_unlock;
		}
	}

	if (vm)
		xe_vm_unlock(vm);

	err = q->ops->init(q);
	if (err)
		goto err_lrc;

	return 0;

err_unlock:
	if (vm)
		xe_vm_unlock(vm);
err_lrc:
	for (i = i - 1; i >= 0; --i)
		xe_lrc_put(q->lrc[i]);
	return err;
}

struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
					   u32 logical_mask, u16 width,
					   struct xe_hw_engine *hwe, u32 flags,
					   u64 extensions)
{
	struct xe_exec_queue *q;
	int err;

	q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags,
				  extensions);
	if (IS_ERR(q))
		return q;

	err = __xe_exec_queue_init(q);
	if (err)
		goto err_post_alloc;

	return q;

err_post_alloc:
	__xe_exec_queue_free(q);
	return ERR_PTR(err);
}

struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,
						 struct xe_vm *vm,
						 enum xe_engine_class class,
						 u32 flags, u64 extensions)
{
	struct xe_hw_engine *hwe, *hwe0 = NULL;
	enum xe_hw_engine_id id;
	u32 logical_mask = 0;

	for_each_hw_engine(hwe, gt, id) {
		if (xe_hw_engine_is_reserved(hwe))
			continue;

		if (hwe->class == class) {
			logical_mask |= BIT(hwe->logical_instance);
			if (!hwe0)
				hwe0 = hwe;
		}
	}

	if (!logical_mask)
		return ERR_PTR(-ENODEV);

	return xe_exec_queue_create(xe, vm, logical_mask, 1