From: Tianrui Zhao <zhaotianrui@loongson.cn> LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC1M5U CVE: NA -------------------------------- Signed-off-by: Tianrui Zhao <zhaotianrui@loongson.cn> Change-Id: I6050c8c50f24a64cee36a9d9e45f16d0cfb2dbda --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 5 + drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c | 158 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h | 6 + drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c | 4 + drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h | 3 + 5 files changed, 176 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 13c97ba7a820..d73c892c3280 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -1279,6 +1279,11 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, uint64_t seq; int r; +#ifdef CONFIG_LOONGARCH + while (amdgpu_ih_fix_is_busy(p->adev)) + msleep(10); +#endif + for (i = 0; i < p->gang_size; ++i) drm_sched_job_arm(&p->jobs[i]->base); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c index f3b0aaf3ebc6..15493d61f069 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c @@ -26,6 +26,10 @@ #include "amdgpu.h" #include "amdgpu_ih.h" +#ifdef CONFIG_LOONGARCH +static void amdgpu_ih_handle_fix_work(struct work_struct *work); +#endif + /** * amdgpu_ih_ring_init - initialize the IH state * @@ -71,6 +75,14 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, ih->wptr_cpu = &ih->ring[ih->ring_size / 4]; ih->rptr_addr = dma_addr + ih->ring_size + 4; ih->rptr_cpu = &ih->ring[(ih->ring_size / 4) + 1]; + + #ifdef CONFIG_LOONGARCH + INIT_WORK(&adev->irq.ih.fix_work, amdgpu_ih_handle_fix_work); + for (r = 0; r < (adev->irq.ih.ring_size >> 2); r++) + adev->irq.ih.ring[r] = 0xDEADBEFF; + mb(); + #endif + } else { unsigned wptr_offs, rptr_offs; @@ -98,6 +110,14 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih, ih->wptr_cpu = &adev->wb.wb[wptr_offs]; ih->rptr_addr = adev->wb.gpu_addr + rptr_offs * 4; ih->rptr_cpu = &adev->wb.wb[rptr_offs]; + + #ifdef CONFIG_LOONGARCH + INIT_WORK(&adev->irq.ih.fix_work, amdgpu_ih_handle_fix_work); + for (r = 0; r < (adev->irq.ih.ring_size >> 2); r++) + adev->irq.ih.ring[r] = 0xDEADBEFF; + mb(); + #endif + } init_waitqueue_head(&ih->wait_process); @@ -119,6 +139,10 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) if (!ih->ring) return; +#ifdef CONFIG_LOONGARCH + cancel_work_sync(&adev->irq.ih.fix_work); +#endif + if (ih->use_bus_addr) { /* add 8 bytes for the rptr/wptr shadows and @@ -135,6 +159,119 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) } } +#ifdef CONFIG_LOONGARCH + +int amdgpu_ih_fix_is_busy(struct amdgpu_device *adev) +{ + return atomic_read(&adev->irq.cs_lock); +} + +static int amdgpu_ih_fix_loongarch_pcie_order_start(struct amdgpu_ih_ring *ih, + u32 rptr, u32 wptr, + bool forever) +{ + int i; + int check_cnt = 0; + u32 ring_end = ih->ring_size >> 2; + + if (rptr == wptr) + return 0; + + rptr = rptr >> 2; + wptr = wptr >> 2; + + wptr = (rptr > wptr) ? ring_end : wptr; + +restart_check: + if (!forever && ++check_cnt > 1) + return -ENAVAIL; + + if (forever) + msleep(10); + + for (i = rptr; i < wptr; i += 1) { + if (le32_to_cpu(ih->ring[i]) == 0xDEADBEFF) + goto restart_check; + } + + if (rptr > wptr) { + for (i = 0; i < wptr; i += 1) { + if (le32_to_cpu(ih->ring[i]) == 0xDEADBEFF) + goto restart_check; + } + } + + return 0; +} + +static int amdgpu_ih_fix_loongarch_pcie_order_end(struct amdgpu_ih_ring *ih, + u32 rptr, u32 wptr) +{ + int i; + u32 ring_end = ih->ring_size >> 2; + + if (rptr == wptr) + return 0; + + rptr = rptr >> 2; + wptr = wptr >> 2; + + wptr = (rptr > wptr) ? ring_end : wptr; + + for (i = rptr; i < wptr; i += 1) + ih->ring[i] = 0xDEADBEFF; + + if (rptr > wptr) { + for (i = 0; i < wptr; i += 1) + ih->ring[i] = 0xDEADBEFF; + } + + mb(); + return 0; +} + +static void amdgpu_ih_handle_fix_work(struct work_struct *work) +{ + struct amdgpu_device *adev = + container_of(work, struct amdgpu_device, irq.ih.fix_work); + struct amdgpu_ih_ring *ih = &adev->irq.ih; + struct amdgpu_iv_entry entry; + + u32 wptr; + u32 old_rptr; + int restart_fg = 0; + +restart: + + wptr = amdgpu_ih_get_wptr(adev, ih); + /* Order reading of wptr vs. reading of IH ring data */ + rmb(); + + old_rptr = ih->rptr; + amdgpu_ih_fix_loongarch_pcie_order_start(&adev->irq.ih, old_rptr, wptr, true); + + while (adev->irq.ih.rptr != wptr) { + u32 ring_index = adev->irq.ih.rptr >> 2; + + amdgpu_irq_dispatch(adev, ih); + ih->rptr &= ih->ptr_mask; + } + + amdgpu_ih_fix_loongarch_pcie_order_end(&adev->irq.ih, old_rptr, adev->irq.ih.rptr); + + amdgpu_ih_set_rptr(adev, ih); + mb(); + + if (ih->rptr != amdgpu_ih_get_wptr(adev, ih)) { + restart_fg = 1; + goto restart; + } + + atomic_set(&adev->irq.cs_lock, 0); + return; +} +#endif + /** * amdgpu_ih_ring_write - write IV to the ring buffer * @@ -209,6 +346,10 @@ int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { unsigned int count; u32 wptr; +#ifdef CONFIG_LOONGARCH + u32 old_rptr; + int r; +#endif if (!ih->enabled || adev->shutdown) return IRQ_NONE; @@ -222,11 +363,28 @@ int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) /* Order reading of wptr vs. reading of IH ring data */ rmb(); +#ifdef CONFIG_LOONGARCH + old_rptr = adev->irq.ih.rptr; + r = amdgpu_ih_fix_loongarch_pcie_order_start(&adev->irq.ih, old_rptr, wptr, false); + if (r) { + if (old_rptr == ((wptr + 16) & adev->irq.ih.ptr_mask)) { + return IRQ_NONE; + } + atomic_xchg(&adev->irq.cs_lock, 1); + schedule_work(&adev->irq.ih.fix_work); + return IRQ_NONE; + } +#endif + while (ih->rptr != wptr && --count) { amdgpu_irq_dispatch(adev, ih); ih->rptr &= ih->ptr_mask; } +#ifdef CONFIG_LOONGARCH + amdgpu_ih_fix_loongarch_pcie_order_end(&adev->irq.ih, old_rptr, adev->irq.ih.rptr); +#endif + amdgpu_ih_set_rptr(adev, ih); wake_up_all(&ih->wait_process); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h index 508f02eb0cf8..375afd059cc5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h @@ -72,6 +72,9 @@ struct amdgpu_ih_ring { /* For waiting on IH processing at checkpoint. */ wait_queue_head_t wait_process; uint64_t processed_timestamp; +#ifdef CONFIG_LOONGARCH + struct work_struct fix_work; +#endif }; /* return true if time stamp t2 is after t1 with 48bit wrap around */ @@ -110,4 +113,7 @@ void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry); uint64_t amdgpu_ih_decode_iv_ts_helper(struct amdgpu_ih_ring *ih, u32 rptr, signed int offset); +#ifdef CONFIG_LOONGARCH +int amdgpu_ih_fix_is_busy(struct amdgpu_device *adev); +#endif #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 5978edf7ea71..1aaf3f91b148 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -275,6 +275,10 @@ int amdgpu_irq_init(struct amdgpu_device *adev) spin_lock_init(&adev->irq.lock); +#ifdef CONFIG_LOONGARCH + atomic_set(&adev->irq.cs_lock, 0); +#endif + /* Enable MSI if not disabled by module parameter */ adev->irq.msi_enabled = false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h index 04c0b4fa17a4..3cf74a4b4c35 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h @@ -100,6 +100,9 @@ struct amdgpu_irq { uint32_t srbm_soft_reset; u32 retry_cam_doorbell_index; bool retry_cam_enabled; +#ifdef CONFIG_LOONGARCH + atomic_t cs_lock; +#endif }; enum interrupt_node_id_per_aid { -- 2.33.0