path: root/drivers/gpu/drm/xe/xe_gt.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2022 Intel Corporation
 */

#include "xe_gt.h"

#include <linux/minmax.h>

#include <drm/drm_managed.h>
#include <uapi/drm/xe_drm.h>

#include <generated/xe_wa_oob.h>

#include "instructions/xe_gfxpipe_commands.h"
#include "instructions/xe_mi_commands.h"
#include "regs/xe_gt_regs.h"
#include "xe_assert.h"
#include "xe_bb.h"
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_exec_queue.h"
#include "xe_execlist.h"
#include "xe_force_wake.h"
#include "xe_ggtt.h"
#include "xe_gsc.h"
#include "xe_gt_ccs_mode.h"
#include "xe_gt_clock.h"
#include "xe_gt_freq.h"
#include "xe_gt_idle.h"
#include "xe_gt_mcr.h"
#include "xe_gt_pagefault.h"
#include "xe_gt_printk.h"
#include "xe_gt_sriov_pf.h"
#include "xe_gt_sysfs.h"
#include "xe_gt_tlb_invalidation.h"
#include "xe_gt_topology.h"
#include "xe_guc_exec_queue_types.h"
#include "xe_guc_pc.h"
#include "xe_hw_fence.h"
#include "xe_hw_engine_class_sysfs.h"
#include "xe_irq.h"
#include "xe_lmtt.h"
#include "xe_lrc.h"
#include "xe_map.h"
#include "xe_migrate.h"
#include "xe_mmio.h"
#include "xe_pat.h"
#include "xe_pm.h"
#include "xe_mocs.h"
#include "xe_reg_sr.h"
#include "xe_ring_ops.h"
#include "xe_sa.h"
#include "xe_sched_job.h"
#include "xe_sriov.h"
#include "xe_tuning.h"
#include "xe_uc.h"
#include "xe_uc_fw.h"
#include "xe_vm.h"
#include "xe_wa.h"
#include "xe_wopcm.h"

static void gt_fini(struct drm_device *drm, void *arg)
{
	struct xe_gt *gt = arg;

	destroy_workqueue(gt->ordered_wq);
}

struct xe_gt *xe_gt_alloc(struct xe_tile *tile)
{
	struct xe_gt *gt;
	int err;

	gt = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*gt), GFP_KERNEL);
	if (!gt)
		return ERR_PTR(-ENOMEM);

	gt->tile = tile;
	gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0);

	err = drmm_add_action_or_reset(&gt_to_xe(gt)->drm, gt_fini, gt);
	if (err)
		return ERR_PTR(err);

	return gt;
}

void xe_gt_sanitize(struct xe_gt *gt)
{
	/*
	 * FIXME: if xe_uc_sanitize is called here, on TGL driver will not
	 * reload
	 */
	gt->uc.guc.submission_state.enabled = false;
}

static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
{
	u32 reg;
	int err;

	if (!XE_WA(gt, 16023588340))
		return;

	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
	if (WARN_ON(err))
		return;

	if (!xe_gt_is_media_type(gt)) {
		reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
		reg |= CG_DIS_CNTLBUS;
		xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
	}

	xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0x3);
	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
}

static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
{
	u32 reg;
	int err;

	if (!XE_WA(gt, 16023588340))
		return;

	if (xe_gt_is_media_type(gt))
		return;

	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
	if (WARN_ON(err))
		return;

	reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
	reg &= ~CG_DIS_CNTLBUS;
	xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);

	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
}

/**
 * xe_gt_remove() - Clean up the GT structures before driver removal
 * @gt: the GT object
 *
 * This function should only act on objects/structures that must be cleaned
 * before the driver removal callback is complete and therefore can't be
 * deferred to a drmm action.
 */
void xe_gt_remove(struct xe_gt *gt)
{
	int i;

	xe_uc_remove(&gt->uc);

	for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
		xe_hw_fence_irq_finish(&gt->fence_irq[i]);

	xe_gt_disable_host_l2_vram(gt);
}

static void gt_reset_worker(struct work_struct *w);

static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q)
{
	struct xe_sched_job *job;
	struct xe_bb *bb;
	struct dma_fence *fence;
	long timeout;

	bb = xe_bb_new(gt, 4, false);
	if (IS_ERR(bb))
		return PTR_ERR(bb);

	job = xe_bb_create_job(q, bb);
	if (IS_ERR(job)) {
		xe_bb_free(bb, NULL);
		return PTR_ERR(job);
	}

	xe_sched_job_arm(job);
	fence = dma_fence_get(&job->drm.s_fence->finished);
	xe_sched_job_push(job);

	timeout = dma_fence_wait_timeout(fence, false, HZ);
	dma_fence_put(fence);
	xe_bb_free(bb, NULL);
	if (timeout < 0)
		return timeout;
	else if (!timeout)
		return -ETIME;

	return 0;
}

/*
 * Convert back from encoded value to type-safe, only to be used when reg.mcr
 * is true
 */
static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)
{
	return (const struct xe_reg_mcr){.__reg.raw = reg.raw };
}

static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q)
{
	struct xe_reg_sr *sr = &q->hwe->reg_lrc;
	struct xe_reg_sr_entry *entry;
	unsigned long idx;
	struct xe_sched_job *job;
	struct xe_bb *bb;
	struct dma_fence *fence;
	long timeout;
	int count = 0;

	if (q->hwe->class == XE_ENGINE_CLASS_RENDER)
		/* Big enough to emit all of the context's 3DSTATE */
		bb = xe_bb_new(