blob: 96e440061bdbf5b65a6213047f62bbeb89aea77f [file] [log] [blame]
Jerome Glisse771fe6b2009-06-05 14:42:42 +02001/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
David Howells760285e2012-10-02 18:01:07 +010028#include <drm/drmP.h>
29#include <drm/radeon_drm.h>
Jerome Glisse771fe6b2009-06-05 14:42:42 +020030#include "radeon.h"
31#include "radeon_reg.h"
Christian König84d597b2013-11-25 15:42:10 +010032#include "radeon_trace.h"
Jerome Glisse771fe6b2009-06-05 14:42:42 +020033
34/*
Alex Deucher03eec932012-07-17 14:02:39 -040035 * GART
36 * The GART (Graphics Aperture Remapping Table) is an aperture
37 * in the GPU's address space. System pages can be mapped into
38 * the aperture and look like contiguous pages from the GPU's
39 * perspective. A page table maps the pages in the aperture
40 * to the actual backing pages in system memory.
41 *
42 * Radeon GPUs support both an internal GART, as described above,
43 * and AGP. AGP works similarly, but the GART table is configured
44 * and maintained by the northbridge rather than the driver.
45 * Radeon hw has a separate AGP aperture that is programmed to
46 * point to the AGP aperture provided by the northbridge and the
47 * requests are passed through to the northbridge aperture.
48 * Both AGP and internal GART can be used at the same time, however
49 * that is not currently supported by the driver.
50 *
51 * This file handles the common internal GART management.
52 */
53
54/*
Jerome Glisse771fe6b2009-06-05 14:42:42 +020055 * Common GART table functions.
56 */
Alex Deucher03eec932012-07-17 14:02:39 -040057/**
58 * radeon_gart_table_ram_alloc - allocate system ram for gart page table
59 *
60 * @rdev: radeon_device pointer
61 *
62 * Allocate system memory for GART page table
63 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the
64 * gart table to be in system memory.
65 * Returns 0 for success, -ENOMEM for failure.
66 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +020067int radeon_gart_table_ram_alloc(struct radeon_device *rdev)
68{
69 void *ptr;
70
71 ptr = pci_alloc_consistent(rdev->pdev, rdev->gart.table_size,
72 &rdev->gart.table_addr);
73 if (ptr == NULL) {
74 return -ENOMEM;
75 }
76#ifdef CONFIG_X86
77 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
78 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
79 set_memory_uc((unsigned long)ptr,
80 rdev->gart.table_size >> PAGE_SHIFT);
81 }
82#endif
Jerome Glissec9a1be92011-11-03 11:16:49 -040083 rdev->gart.ptr = ptr;
84 memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size);
Jerome Glisse771fe6b2009-06-05 14:42:42 +020085 return 0;
86}
87
Alex Deucher03eec932012-07-17 14:02:39 -040088/**
89 * radeon_gart_table_ram_free - free system ram for gart page table
90 *
91 * @rdev: radeon_device pointer
92 *
93 * Free system memory for GART page table
94 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the
95 * gart table to be in system memory.
96 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +020097void radeon_gart_table_ram_free(struct radeon_device *rdev)
98{
Jerome Glissec9a1be92011-11-03 11:16:49 -040099 if (rdev->gart.ptr == NULL) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200100 return;
101 }
102#ifdef CONFIG_X86
103 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
104 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
Jerome Glissec9a1be92011-11-03 11:16:49 -0400105 set_memory_wb((unsigned long)rdev->gart.ptr,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200106 rdev->gart.table_size >> PAGE_SHIFT);
107 }
108#endif
109 pci_free_consistent(rdev->pdev, rdev->gart.table_size,
Jerome Glissec9a1be92011-11-03 11:16:49 -0400110 (void *)rdev->gart.ptr,
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200111 rdev->gart.table_addr);
Jerome Glissec9a1be92011-11-03 11:16:49 -0400112 rdev->gart.ptr = NULL;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200113 rdev->gart.table_addr = 0;
114}
115
Alex Deucher03eec932012-07-17 14:02:39 -0400116/**
117 * radeon_gart_table_vram_alloc - allocate vram for gart page table
118 *
119 * @rdev: radeon_device pointer
120 *
121 * Allocate video memory for GART page table
122 * (pcie r4xx, r5xx+). These asics require the
123 * gart table to be in video memory.
124 * Returns 0 for success, error for failure.
125 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200126int radeon_gart_table_vram_alloc(struct radeon_device *rdev)
127{
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200128 int r;
129
Jerome Glissec9a1be92011-11-03 11:16:49 -0400130 if (rdev->gart.robj == NULL) {
Daniel Vetter441921d2011-02-18 17:59:16 +0100131 r = radeon_bo_create(rdev, rdev->gart.table_size,
Alex Deucher268b2512010-11-17 19:00:26 -0500132 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
Alex Deucher40f5cf92012-05-10 18:33:13 -0400133 NULL, &rdev->gart.robj);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200134 if (r) {
135 return r;
136 }
137 }
Jerome Glisse4aac0472009-09-14 18:29:49 +0200138 return 0;
139}
140
Alex Deucher03eec932012-07-17 14:02:39 -0400141/**
142 * radeon_gart_table_vram_pin - pin gart page table in vram
143 *
144 * @rdev: radeon_device pointer
145 *
146 * Pin the GART page table in vram so it will not be moved
147 * by the memory manager (pcie r4xx, r5xx+). These asics require the
148 * gart table to be in video memory.
149 * Returns 0 for success, error for failure.
150 */
Jerome Glisse4aac0472009-09-14 18:29:49 +0200151int radeon_gart_table_vram_pin(struct radeon_device *rdev)
152{
153 uint64_t gpu_addr;
154 int r;
155
Jerome Glissec9a1be92011-11-03 11:16:49 -0400156 r = radeon_bo_reserve(rdev->gart.robj, false);
Jerome Glisse4c788672009-11-20 14:29:23 +0100157 if (unlikely(r != 0))
158 return r;
Jerome Glissec9a1be92011-11-03 11:16:49 -0400159 r = radeon_bo_pin(rdev->gart.robj,
Jerome Glisse4c788672009-11-20 14:29:23 +0100160 RADEON_GEM_DOMAIN_VRAM, &gpu_addr);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200161 if (r) {
Jerome Glissec9a1be92011-11-03 11:16:49 -0400162 radeon_bo_unreserve(rdev->gart.robj);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200163 return r;
164 }
Jerome Glissec9a1be92011-11-03 11:16:49 -0400165 r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr);
Jerome Glisse4c788672009-11-20 14:29:23 +0100166 if (r)
Jerome Glissec9a1be92011-11-03 11:16:49 -0400167 radeon_bo_unpin(rdev->gart.robj);
168 radeon_bo_unreserve(rdev->gart.robj);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200169 rdev->gart.table_addr = gpu_addr;
Jerome Glisse4c788672009-11-20 14:29:23 +0100170 return r;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200171}
172
Alex Deucher03eec932012-07-17 14:02:39 -0400173/**
174 * radeon_gart_table_vram_unpin - unpin gart page table in vram
175 *
176 * @rdev: radeon_device pointer
177 *
178 * Unpin the GART page table in vram (pcie r4xx, r5xx+).
179 * These asics require the gart table to be in video memory.
180 */
Jerome Glissec9a1be92011-11-03 11:16:49 -0400181void radeon_gart_table_vram_unpin(struct radeon_device *rdev)
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200182{
Jerome Glisse4c788672009-11-20 14:29:23 +0100183 int r;
184
Jerome Glissec9a1be92011-11-03 11:16:49 -0400185 if (rdev->gart.robj == NULL) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200186 return;
187 }
Jerome Glissec9a1be92011-11-03 11:16:49 -0400188 r = radeon_bo_reserve(rdev->gart.robj, false);
Jerome Glisse4c788672009-11-20 14:29:23 +0100189 if (likely(r == 0)) {
Jerome Glissec9a1be92011-11-03 11:16:49 -0400190 radeon_bo_kunmap(rdev->gart.robj);
191 radeon_bo_unpin(rdev->gart.robj);
192 radeon_bo_unreserve(rdev->gart.robj);
193 rdev->gart.ptr = NULL;
Jerome Glisse4c788672009-11-20 14:29:23 +0100194 }
Jerome Glissec9a1be92011-11-03 11:16:49 -0400195}
196
Alex Deucher03eec932012-07-17 14:02:39 -0400197/**
198 * radeon_gart_table_vram_free - free gart page table vram
199 *
200 * @rdev: radeon_device pointer
201 *
202 * Free the video memory used for the GART page table
203 * (pcie r4xx, r5xx+). These asics require the gart table to
204 * be in video memory.
205 */
Jerome Glissec9a1be92011-11-03 11:16:49 -0400206void radeon_gart_table_vram_free(struct radeon_device *rdev)
207{
208 if (rdev->gart.robj == NULL) {
209 return;
210 }
Jerome Glissec9a1be92011-11-03 11:16:49 -0400211 radeon_bo_unref(&rdev->gart.robj);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200212}
213
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200214/*
215 * Common gart functions.
216 */
Alex Deucher03eec932012-07-17 14:02:39 -0400217/**
218 * radeon_gart_unbind - unbind pages from the gart page table
219 *
220 * @rdev: radeon_device pointer
221 * @offset: offset into the GPU's gart aperture
222 * @pages: number of pages to unbind
223 *
224 * Unbinds the requested pages from the gart page table and
225 * replaces them with the dummy page (all asics).
226 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200227void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
228 int pages)
229{
230 unsigned t;
231 unsigned p;
232 int i, j;
Dave Airlie82568562010-02-05 16:00:07 +1000233 u64 page_base;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200234
235 if (!rdev->gart.ready) {
Tormod Voldenfcf4de52011-08-31 21:54:07 +0000236 WARN(1, "trying to unbind memory from uninitialized GART !\n");
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200237 return;
238 }
Matt Turnera77f1712009-10-14 00:34:41 -0400239 t = offset / RADEON_GPU_PAGE_SIZE;
240 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200241 for (i = 0; i < pages; i++, p++) {
242 if (rdev->gart.pages[p]) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200243 rdev->gart.pages[p] = NULL;
Dave Airlie82568562010-02-05 16:00:07 +1000244 rdev->gart.pages_addr[p] = rdev->dummy_page.addr;
245 page_base = rdev->gart.pages_addr[p];
Matt Turnera77f1712009-10-14 00:34:41 -0400246 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
Jerome Glissec9a1be92011-11-03 11:16:49 -0400247 if (rdev->gart.ptr) {
248 radeon_gart_set_page(rdev, t, page_base);
249 }
Dave Airlie82568562010-02-05 16:00:07 +1000250 page_base += RADEON_GPU_PAGE_SIZE;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200251 }
252 }
253 }
254 mb();
255 radeon_gart_tlb_flush(rdev);
256}
257
Alex Deucher03eec932012-07-17 14:02:39 -0400258/**
259 * radeon_gart_bind - bind pages into the gart page table
260 *
261 * @rdev: radeon_device pointer
262 * @offset: offset into the GPU's gart aperture
263 * @pages: number of pages to bind
264 * @pagelist: pages to bind
265 * @dma_addr: DMA addresses of pages
266 *
267 * Binds the requested pages to the gart page table
268 * (all asics).
269 * Returns 0 for success, -EINVAL for failure.
270 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200271int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
Konrad Rzeszutek Wilkc39d3512010-12-02 11:04:29 -0500272 int pages, struct page **pagelist, dma_addr_t *dma_addr)
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200273{
274 unsigned t;
275 unsigned p;
276 uint64_t page_base;
277 int i, j;
278
279 if (!rdev->gart.ready) {
Tormod Voldenfcf4de52011-08-31 21:54:07 +0000280 WARN(1, "trying to bind memory to uninitialized GART !\n");
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200281 return -EINVAL;
282 }
Matt Turnera77f1712009-10-14 00:34:41 -0400283 t = offset / RADEON_GPU_PAGE_SIZE;
284 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200285
286 for (i = 0; i < pages; i++, p++) {
Konrad Rzeszutek Wilkc52494f2011-10-17 17:15:08 -0400287 rdev->gart.pages_addr[p] = dma_addr[i];
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200288 rdev->gart.pages[p] = pagelist[i];
Jerome Glissec9a1be92011-11-03 11:16:49 -0400289 if (rdev->gart.ptr) {
290 page_base = rdev->gart.pages_addr[p];
291 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
292 radeon_gart_set_page(rdev, t, page_base);
293 page_base += RADEON_GPU_PAGE_SIZE;
294 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200295 }
296 }
297 mb();
298 radeon_gart_tlb_flush(rdev);
299 return 0;
300}
301
Alex Deucher03eec932012-07-17 14:02:39 -0400302/**
303 * radeon_gart_restore - bind all pages in the gart page table
304 *
305 * @rdev: radeon_device pointer
306 *
307 * Binds all pages in the gart page table (all asics).
308 * Used to rebuild the gart table on device startup or resume.
309 */
Dave Airlie82568562010-02-05 16:00:07 +1000310void radeon_gart_restore(struct radeon_device *rdev)
311{
312 int i, j, t;
313 u64 page_base;
314
Jerome Glissec9a1be92011-11-03 11:16:49 -0400315 if (!rdev->gart.ptr) {
316 return;
317 }
Dave Airlie82568562010-02-05 16:00:07 +1000318 for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) {
319 page_base = rdev->gart.pages_addr[i];
320 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
321 radeon_gart_set_page(rdev, t, page_base);
322 page_base += RADEON_GPU_PAGE_SIZE;
323 }
324 }
325 mb();
326 radeon_gart_tlb_flush(rdev);
327}
328
Alex Deucher03eec932012-07-17 14:02:39 -0400329/**
330 * radeon_gart_init - init the driver info for managing the gart
331 *
332 * @rdev: radeon_device pointer
333 *
334 * Allocate the dummy page and init the gart driver info (all asics).
335 * Returns 0 for success, error for failure.
336 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200337int radeon_gart_init(struct radeon_device *rdev)
338{
Dave Airlie82568562010-02-05 16:00:07 +1000339 int r, i;
340
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200341 if (rdev->gart.pages) {
342 return 0;
343 }
Matt Turnera77f1712009-10-14 00:34:41 -0400344 /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */
345 if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) {
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200346 DRM_ERROR("Page size is smaller than GPU page size!\n");
347 return -EINVAL;
348 }
Dave Airlie82568562010-02-05 16:00:07 +1000349 r = radeon_dummy_page_init(rdev);
350 if (r)
351 return r;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200352 /* Compute table size */
353 rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE;
Matt Turnera77f1712009-10-14 00:34:41 -0400354 rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE;
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200355 DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
356 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages);
357 /* Allocate pages table */
Christian König59240ee2012-10-23 15:53:17 +0200358 rdev->gart.pages = vzalloc(sizeof(void *) * rdev->gart.num_cpu_pages);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200359 if (rdev->gart.pages == NULL) {
360 radeon_gart_fini(rdev);
361 return -ENOMEM;
362 }
Christian König59240ee2012-10-23 15:53:17 +0200363 rdev->gart.pages_addr = vzalloc(sizeof(dma_addr_t) *
364 rdev->gart.num_cpu_pages);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200365 if (rdev->gart.pages_addr == NULL) {
366 radeon_gart_fini(rdev);
367 return -ENOMEM;
368 }
Dave Airlie82568562010-02-05 16:00:07 +1000369 /* set GART entry to point to the dummy page by default */
370 for (i = 0; i < rdev->gart.num_cpu_pages; i++) {
371 rdev->gart.pages_addr[i] = rdev->dummy_page.addr;
372 }
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200373 return 0;
374}
375
Alex Deucher03eec932012-07-17 14:02:39 -0400376/**
377 * radeon_gart_fini - tear down the driver info for managing the gart
378 *
379 * @rdev: radeon_device pointer
380 *
381 * Tear down the gart driver info and free the dummy page (all asics).
382 */
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200383void radeon_gart_fini(struct radeon_device *rdev)
384{
385 if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) {
386 /* unbind pages */
387 radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages);
388 }
389 rdev->gart.ready = false;
Christian König59240ee2012-10-23 15:53:17 +0200390 vfree(rdev->gart.pages);
391 vfree(rdev->gart.pages_addr);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200392 rdev->gart.pages = NULL;
393 rdev->gart.pages_addr = NULL;
Alex Deucher92656d72011-04-12 13:32:13 -0400394
395 radeon_dummy_page_fini(rdev);
Jerome Glisse771fe6b2009-06-05 14:42:42 +0200396}
Jerome Glisse721604a2012-01-05 22:11:05 -0500397
398/*
Alex Deucher09db8642012-07-17 14:02:40 -0400399 * GPUVM
400 * GPUVM is similar to the legacy gart on older asics, however
401 * rather than there being a single global gart table
402 * for the entire GPU, there are multiple VM page tables active
403 * at any given time. The VM page tables can contain a mix
404 * vram pages and system memory pages and system memory pages
405 * can be mapped as snooped (cached system pages) or unsnooped
406 * (uncached system pages).
407 * Each VM has an ID associated with it and there is a page table
408 * associated with each VMID. When execting a command buffer,
409 * the kernel tells the the ring what VMID to use for that command
410 * buffer. VMIDs are allocated dynamically as commands are submitted.
411 * The userspace drivers maintain their own address space and the kernel
412 * sets up their pages tables accordingly when they submit their
413 * command buffers and a VMID is assigned.
414 * Cayman/Trinity support up to 8 active VMs at any given time;
415 * SI supports 16.
416 */
417
418/*
Jerome Glisse721604a2012-01-05 22:11:05 -0500419 * vm helpers
420 *
421 * TODO bind a default page at vm initialization for default address
422 */
Christian Königc6105f22012-07-05 14:32:00 +0200423
Alex Deucher09db8642012-07-17 14:02:40 -0400424/**
Christian König90a51a32012-10-09 13:31:17 +0200425 * radeon_vm_num_pde - return the number of page directory entries
426 *
427 * @rdev: radeon_device pointer
428 *
429 * Calculate the number of page directory entries (cayman+).
430 */
431static unsigned radeon_vm_num_pdes(struct radeon_device *rdev)
432{
433 return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE;
434}
435
436/**
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200437 * radeon_vm_directory_size - returns the size of the page directory in bytes
438 *
439 * @rdev: radeon_device pointer
440 *
441 * Calculate the size of the page directory in bytes (cayman+).
442 */
443static unsigned radeon_vm_directory_size(struct radeon_device *rdev)
444{
Christian König90a51a32012-10-09 13:31:17 +0200445 return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8);
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200446}
447
448/**
Alex Deucher09db8642012-07-17 14:02:40 -0400449 * radeon_vm_manager_init - init the vm manager
450 *
451 * @rdev: radeon_device pointer
452 *
453 * Init the vm manager (cayman+).
454 * Returns 0 for success, error for failure.
455 */
Jerome Glisse721604a2012-01-05 22:11:05 -0500456int radeon_vm_manager_init(struct radeon_device *rdev)
457{
Christian Königc6105f22012-07-05 14:32:00 +0200458 struct radeon_vm *vm;
459 struct radeon_bo_va *bo_va;
Jerome Glisse721604a2012-01-05 22:11:05 -0500460 int r;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200461 unsigned size;
Jerome Glisse721604a2012-01-05 22:11:05 -0500462
Christian Königc6105f22012-07-05 14:32:00 +0200463 if (!rdev->vm_manager.enabled) {
Dave Airliee6b0b6a2012-07-20 00:53:28 -0400464 /* allocate enough for 2 full VM pts */
Christian König90a51a32012-10-09 13:31:17 +0200465 size = radeon_vm_directory_size(rdev);
466 size += rdev->vm_manager.max_pfn * 8;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +0200467 size *= 2;
Christian Königc6105f22012-07-05 14:32:00 +0200468 r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager,
Alex Deucher3e3e53f2013-07-18 13:11:56 -0400469 RADEON_GPU_PAGE_ALIGN(size),
Alex Deucher1c011032013-07-12 15:56:02 -0400470 RADEON_VM_PTB_ALIGN_SIZE,
Christian Königc6105f22012-07-05 14:32:00 +0200471 RADEON_GEM_DOMAIN_VRAM);
472 if (r) {
473 dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n",
474 (rdev->vm_manager.max_pfn * 8) >> 10);
475 return r;
476 }
Alex Deucher67e915e2012-01-06 09:38:15 -0500477
Christian König05b07142012-08-06 20:21:10 +0200478 r = radeon_asic_vm_init(rdev);
Christian Königc6105f22012-07-05 14:32:00 +0200479 if (r)
480 return r;
Christian König089a7862012-08-11 11:54:05 +0200481
Alex Deucher67e915e2012-01-06 09:38:15 -0500482 rdev->vm_manager.enabled = true;
483
Christian Königc6105f22012-07-05 14:32:00 +0200484 r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager);
485 if (r)
486 return r;
487 }
488
489 /* restore page table */
490 list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) {
Christian König90a51a32012-10-09 13:31:17 +0200491 if (vm->page_directory == NULL)
Christian Königc6105f22012-07-05 14:32:00 +0200492 continue;
493
494 list_for_each_entry(bo_va, &vm->va, vm_list) {
Christian Königc6105f22012-07-05 14:32:00 +0200495 bo_va->valid = false;
Christian Königc6105f22012-07-05 14:32:00 +0200496 }
497 }
498 return 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500499}
500
Alex Deucher09db8642012-07-17 14:02:40 -0400501/**
Christian Königddf03f52012-08-09 20:02:28 +0200502 * radeon_vm_free_pt - free the page table for a specific vm
Alex Deucher09db8642012-07-17 14:02:40 -0400503 *
504 * @rdev: radeon_device pointer
505 * @vm: vm to unbind
506 *
Christian Königddf03f52012-08-09 20:02:28 +0200507 * Free the page table of a specific vm (cayman+).
508 *
509 * Global and local mutex must be lock!
Alex Deucher09db8642012-07-17 14:02:40 -0400510 */
Christian Königddf03f52012-08-09 20:02:28 +0200511static void radeon_vm_free_pt(struct radeon_device *rdev,
Jerome Glisse721604a2012-01-05 22:11:05 -0500512 struct radeon_vm *vm)
513{
514 struct radeon_bo_va *bo_va;
Christian König90a51a32012-10-09 13:31:17 +0200515 int i;
Jerome Glisse721604a2012-01-05 22:11:05 -0500516
Christian König90a51a32012-10-09 13:31:17 +0200517 if (!vm->page_directory)
Jerome Glisse721604a2012-01-05 22:11:05 -0500518 return;
Jerome Glisse721604a2012-01-05 22:11:05 -0500519
Jerome Glisse721604a2012-01-05 22:11:05 -0500520 list_del_init(&vm->list);
Christian König90a51a32012-10-09 13:31:17 +0200521 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
Jerome Glisse721604a2012-01-05 22:11:05 -0500522
523 list_for_each_entry(bo_va, &vm->va, vm_list) {
524 bo_va->valid = false;
525 }
Christian König90a51a32012-10-09 13:31:17 +0200526
527 if (vm->page_tables == NULL)
528 return;
529
530 for (i = 0; i < radeon_vm_num_pdes(rdev); i++)
531 radeon_sa_bo_free(rdev, &vm->page_tables[i], vm->fence);
532
533 kfree(vm->page_tables);
Jerome Glisse721604a2012-01-05 22:11:05 -0500534}
535
Alex Deucher09db8642012-07-17 14:02:40 -0400536/**
537 * radeon_vm_manager_fini - tear down the vm manager
538 *
539 * @rdev: radeon_device pointer
540 *
541 * Tear down the VM manager (cayman+).
542 */
Jerome Glisse721604a2012-01-05 22:11:05 -0500543void radeon_vm_manager_fini(struct radeon_device *rdev)
544{
Jerome Glisse721604a2012-01-05 22:11:05 -0500545 struct radeon_vm *vm, *tmp;
Christian Königee60e292012-08-09 16:21:08 +0200546 int i;
Jerome Glisse721604a2012-01-05 22:11:05 -0500547
Christian Königc6105f22012-07-05 14:32:00 +0200548 if (!rdev->vm_manager.enabled)
549 return;
550
Christian König36ff39c2012-05-09 10:07:08 +0200551 mutex_lock(&rdev->vm_manager.lock);
Christian Königddf03f52012-08-09 20:02:28 +0200552 /* free all allocated page tables */
Jerome Glisse721604a2012-01-05 22:11:05 -0500553 list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) {
Christian Königddf03f52012-08-09 20:02:28 +0200554 mutex_lock(&vm->mutex);
555 radeon_vm_free_pt(rdev, vm);
556 mutex_unlock(&vm->mutex);
Jerome Glisse721604a2012-01-05 22:11:05 -0500557 }
Christian Königee60e292012-08-09 16:21:08 +0200558 for (i = 0; i < RADEON_NUM_VM; ++i) {
559 radeon_fence_unref(&rdev->vm_manager.active[i]);
560 }
Christian König05b07142012-08-06 20:21:10 +0200561 radeon_asic_vm_fini(rdev);
Christian König36ff39c2012-05-09 10:07:08 +0200562 mutex_unlock(&rdev->vm_manager.lock);
Christian Königc6105f22012-07-05 14:32:00 +0200563
564 radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager);
565 radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager);
566 rdev->vm_manager.enabled = false;
Jerome Glisse721604a2012-01-05 22:11:05 -0500567}
568
Alex Deucher09db8642012-07-17 14:02:40 -0400569/**
Christian König90a51a32012-10-09 13:31:17 +0200570 * radeon_vm_evict - evict page table to make room for new one
571 *
572 * @rdev: radeon_device pointer
573 * @vm: VM we want to allocate something for
574 *
575 * Evict a VM from the lru, making sure that it isn't @vm. (cayman+).
576 * Returns 0 for success, -ENOMEM for failure.
577 *
578 * Global and local mutex must be locked!
579 */
Alex Deucher1518d7f2012-10-17 12:42:13 -0400580static int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm)
Christian König90a51a32012-10-09 13:31:17 +0200581{
582 struct radeon_vm *vm_evict;
583
584 if (list_empty(&rdev->vm_manager.lru_vm))
585 return -ENOMEM;
586
587 vm_evict = list_first_entry(&rdev->vm_manager.lru_vm,
588 struct radeon_vm, list);
589 if (vm_evict == vm)
590 return -ENOMEM;
591
592 mutex_lock(&vm_evict->mutex);
593 radeon_vm_free_pt(rdev, vm_evict);
594 mutex_unlock(&vm_evict->mutex);
595 return 0;
596}
597
598/**
Christian Königddf03f52012-08-09 20:02:28 +0200599 * radeon_vm_alloc_pt - allocates a page table for a VM
Alex Deucher09db8642012-07-17 14:02:40 -0400600 *
601 * @rdev: radeon_device pointer
602 * @vm: vm to bind
603 *
Christian Königddf03f52012-08-09 20:02:28 +0200604 * Allocate a page table for the requested vm (cayman+).
Alex Deucher09db8642012-07-17 14:02:40 -0400605 * Returns 0 for success, error for failure.
Christian Königddf03f52012-08-09 20:02:28 +0200606 *
607 * Global and local mutex must be locked!
Alex Deucher09db8642012-07-17 14:02:40 -0400608 */
Christian Königddf03f52012-08-09 20:02:28 +0200609int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm)
Jerome Glisse721604a2012-01-05 22:11:05 -0500610{
Christian Königdb96bd22013-10-29 20:14:51 +0100611 unsigned pd_size, pd_entries, pts_size;
612 struct radeon_ib ib;
Christian König90a51a32012-10-09 13:31:17 +0200613 int r;
Jerome Glisse721604a2012-01-05 22:11:05 -0500614
615 if (vm == NULL) {
616 return -EINVAL;
617 }
618
Christian König90a51a32012-10-09 13:31:17 +0200619 if (vm->page_directory != NULL) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500620 return 0;
621 }
622
Alex Deucher3e3e53f2013-07-18 13:11:56 -0400623 pd_size = radeon_vm_directory_size(rdev);
Christian Königdb96bd22013-10-29 20:14:51 +0100624 pd_entries = radeon_vm_num_pdes(rdev);
625
626retry:
Christian König90a51a32012-10-09 13:31:17 +0200627 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager,
628 &vm->page_directory, pd_size,
Alex Deucher1c011032013-07-12 15:56:02 -0400629 RADEON_VM_PTB_ALIGN_SIZE, false);
Christian Königddf03f52012-08-09 20:02:28 +0200630 if (r == -ENOMEM) {
Christian König90a51a32012-10-09 13:31:17 +0200631 r = radeon_vm_evict(rdev, vm);
632 if (r)
Jerome Glisse721604a2012-01-05 22:11:05 -0500633 return r;
Jerome Glisse721604a2012-01-05 22:11:05 -0500634 goto retry;
Jerome Glisse721604a2012-01-05 22:11:05 -0500635
Christian Königddf03f52012-08-09 20:02:28 +0200636 } else if (r) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500637 return r;
638 }
Christian Königddf03f52012-08-09 20:02:28 +0200639
Christian König90a51a32012-10-09 13:31:17 +0200640 vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory);
641
642 /* Initially clear the page directory */
Christian Königdb96bd22013-10-29 20:14:51 +0100643 r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib,
644 NULL, pd_entries * 2 + 64);
645 if (r) {
646 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
647 return r;
648 }
Christian König90a51a32012-10-09 13:31:17 +0200649
Christian Königdb96bd22013-10-29 20:14:51 +0100650 ib.length_dw = 0;
651
652 radeon_asic_vm_set_page(rdev, &ib, vm->pd_gpu_addr,
653 0, pd_entries, 0, 0);
654
Christian König1654b812013-11-12 12:58:05 +0100655 radeon_semaphore_sync_to(ib.semaphore, vm->fence);
Christian Königdb96bd22013-10-29 20:14:51 +0100656 r = radeon_ib_schedule(rdev, &ib, NULL);
657 if (r) {
658 radeon_ib_free(rdev, &ib);
659 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
660 return r;
661 }
662 radeon_fence_unref(&vm->fence);
663 vm->fence = radeon_fence_ref(ib.fence);
664 radeon_ib_free(rdev, &ib);
665 radeon_fence_unref(&vm->last_flush);
666
667 /* allocate page table array */
Christian König90a51a32012-10-09 13:31:17 +0200668 pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *);
669 vm->page_tables = kzalloc(pts_size, GFP_KERNEL);
670
671 if (vm->page_tables == NULL) {
672 DRM_ERROR("Cannot allocate memory for page table array\n");
673 radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
674 return -ENOMEM;
675 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500676
Christian Königd72d43c2012-10-09 13:31:18 +0200677 return 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500678}
679
Christian Königee60e292012-08-09 16:21:08 +0200680/**
Christian König13e55c32012-10-09 13:31:19 +0200681 * radeon_vm_add_to_lru - add VMs page table to LRU list
682 *
683 * @rdev: radeon_device pointer
684 * @vm: vm to add to LRU
685 *
686 * Add the allocated page table to the LRU list (cayman+).
687 *
688 * Global mutex must be locked!
689 */
690void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm)
691{
692 list_del_init(&vm->list);
693 list_add_tail(&vm->list, &rdev->vm_manager.lru_vm);
694}
695
696/**
Christian Königee60e292012-08-09 16:21:08 +0200697 * radeon_vm_grab_id - allocate the next free VMID
698 *
699 * @rdev: radeon_device pointer
700 * @vm: vm to allocate id for
701 * @ring: ring we want to submit job to
702 *
703 * Allocate an id for the vm (cayman+).
704 * Returns the fence we need to sync to (if any).
705 *
706 * Global and local mutex must be locked!
707 */
708struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
709 struct radeon_vm *vm, int ring)
710{
711 struct radeon_fence *best[RADEON_NUM_RINGS] = {};
712 unsigned choices[2] = {};
713 unsigned i;
714
715 /* check if the id is still valid */
716 if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id])
717 return NULL;
718
719 /* we definately need to flush */
720 radeon_fence_unref(&vm->last_flush);
721
722 /* skip over VMID 0, since it is the system VM */
723 for (i = 1; i < rdev->vm_manager.nvm; ++i) {
724 struct radeon_fence *fence = rdev->vm_manager.active[i];
725
726 if (fence == NULL) {
727 /* found a free one */
728 vm->id = i;
729 return NULL;
730 }
731
732 if (radeon_fence_is_earlier(fence, best[fence->ring])) {
733 best[fence->ring] = fence;
734 choices[fence->ring == ring ? 0 : 1] = i;
735 }
736 }
737
738 for (i = 0; i < 2; ++i) {
739 if (choices[i]) {
740 vm->id = choices[i];
Christian König84d597b2013-11-25 15:42:10 +0100741 trace_radeon_vm_grab_id(vm->id, ring);
Christian Königee60e292012-08-09 16:21:08 +0200742 return rdev->vm_manager.active[choices[i]];
743 }
744 }
745
746 /* should never happen */
747 BUG();
748 return NULL;
749}
750
751/**
752 * radeon_vm_fence - remember fence for vm
753 *
754 * @rdev: radeon_device pointer
755 * @vm: vm we want to fence
756 * @fence: fence to remember
757 *
758 * Fence the vm (cayman+).
759 * Set the fence used to protect page table and id.
760 *
761 * Global and local mutex must be locked!
762 */
763void radeon_vm_fence(struct radeon_device *rdev,
764 struct radeon_vm *vm,
765 struct radeon_fence *fence)
766{
767 radeon_fence_unref(&rdev->vm_manager.active[vm->id]);
768 rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence);
769
770 radeon_fence_unref(&vm->fence);
771 vm->fence = radeon_fence_ref(fence);
772}
773
Christian König421ca7a2012-09-11 16:10:00 +0200774/**
775 * radeon_vm_bo_find - find the bo_va for a specific vm & bo
776 *
777 * @vm: requested vm
778 * @bo: requested buffer object
779 *
780 * Find @bo inside the requested vm (cayman+).
781 * Search inside the @bos vm list for the requested vm
782 * Returns the found bo_va or NULL if none is found
783 *
784 * Object has to be reserved!
785 */
786struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,
787 struct radeon_bo *bo)
788{
789 struct radeon_bo_va *bo_va;
790
791 list_for_each_entry(bo_va, &bo->va, bo_list) {
792 if (bo_va->vm == vm) {
793 return bo_va;
794 }
795 }
796 return NULL;
797}
798
Alex Deucher09db8642012-07-17 14:02:40 -0400799/**
800 * radeon_vm_bo_add - add a bo to a specific vm
801 *
802 * @rdev: radeon_device pointer
803 * @vm: requested vm
804 * @bo: radeon buffer object
Alex Deucher09db8642012-07-17 14:02:40 -0400805 *
806 * Add @bo into the requested vm (cayman+).
Christian Könige971bd52012-09-11 16:10:04 +0200807 * Add @bo to the list of bos associated with the vm
808 * Returns newly added bo_va or NULL for failure
809 *
810 * Object has to be reserved!
Alex Deucher09db8642012-07-17 14:02:40 -0400811 */
Christian Könige971bd52012-09-11 16:10:04 +0200812struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
813 struct radeon_vm *vm,
814 struct radeon_bo *bo)
Jerome Glisse721604a2012-01-05 22:11:05 -0500815{
Christian Könige971bd52012-09-11 16:10:04 +0200816 struct radeon_bo_va *bo_va;
Jerome Glisse721604a2012-01-05 22:11:05 -0500817
818 bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL);
819 if (bo_va == NULL) {
Christian Könige971bd52012-09-11 16:10:04 +0200820 return NULL;
Jerome Glisse721604a2012-01-05 22:11:05 -0500821 }
822 bo_va->vm = vm;
823 bo_va->bo = bo;
Christian Könige971bd52012-09-11 16:10:04 +0200824 bo_va->soffset = 0;
825 bo_va->eoffset = 0;
826 bo_va->flags = 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500827 bo_va->valid = false;
Christian Könige971bd52012-09-11 16:10:04 +0200828 bo_va->ref_count = 1;
Jerome Glisse721604a2012-01-05 22:11:05 -0500829 INIT_LIST_HEAD(&bo_va->bo_list);
830 INIT_LIST_HEAD(&bo_va->vm_list);
Jerome Glisse721604a2012-01-05 22:11:05 -0500831
Christian Könige971bd52012-09-11 16:10:04 +0200832 mutex_lock(&vm->mutex);
833 list_add(&bo_va->vm_list, &vm->va);
834 list_add_tail(&bo_va->bo_list, &bo->va);
835 mutex_unlock(&vm->mutex);
836
837 return bo_va;
838}
839
840/**
841 * radeon_vm_bo_set_addr - set bos virtual address inside a vm
842 *
843 * @rdev: radeon_device pointer
844 * @bo_va: bo_va to store the address
845 * @soffset: requested offset of the buffer in the VM address space
846 * @flags: attributes of pages (read/write/valid/etc.)
847 *
848 * Set offset of @bo_va (cayman+).
849 * Validate and set the offset requested within the vm address space.
Jerome Glisse721604a2012-01-05 22:11:05 -0500850 * Returns 0 for success, error for failure.
Christian König421ca7a2012-09-11 16:10:00 +0200851 *
852 * Object has to be reserved!
Jerome Glisse721604a2012-01-05 22:11:05 -0500853 */
Christian Könige971bd52012-09-11 16:10:04 +0200854int radeon_vm_bo_set_addr(struct radeon_device *rdev,
855 struct radeon_bo_va *bo_va,
856 uint64_t soffset,
857 uint32_t flags)
Jerome Glisse721604a2012-01-05 22:11:05 -0500858{
Christian Könige971bd52012-09-11 16:10:04 +0200859 uint64_t size = radeon_bo_size(bo_va->bo);
860 uint64_t eoffset, last_offset = 0;
861 struct radeon_vm *vm = bo_va->vm;
862 struct radeon_bo_va *tmp;
Jerome Glisse721604a2012-01-05 22:11:05 -0500863 struct list_head *head;
Jerome Glisse721604a2012-01-05 22:11:05 -0500864 unsigned last_pfn;
865
Christian Könige971bd52012-09-11 16:10:04 +0200866 if (soffset) {
867 /* make sure object fit at this offset */
868 eoffset = soffset + size;
869 if (soffset >= eoffset) {
870 return -EINVAL;
871 }
Jerome Glisse721604a2012-01-05 22:11:05 -0500872
Christian Könige971bd52012-09-11 16:10:04 +0200873 last_pfn = eoffset / RADEON_GPU_PAGE_SIZE;
874 if (last_pfn > rdev->vm_manager.max_pfn) {
875 dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n",
876 last_pfn, rdev->vm_manager.max_pfn);
877 return -EINVAL;
878 }
879
880 } else {
881 eoffset = last_pfn = 0;
Jerome Glisse721604a2012-01-05 22:11:05 -0500882 }
883
884 mutex_lock(&vm->mutex);
Jerome Glisse721604a2012-01-05 22:11:05 -0500885 head = &vm->va;
886 last_offset = 0;
887 list_for_each_entry(tmp, &vm->va, vm_list) {
Christian Könige971bd52012-09-11 16:10:04 +0200888 if (bo_va == tmp) {
889 /* skip over currently modified bo */
890 continue;
891 }
892
893 if (soffset >= last_offset && eoffset <= tmp->soffset) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500894 /* bo can be added before this one */
895 break;
896 }
Christian Könige971bd52012-09-11 16:10:04 +0200897 if (eoffset > tmp->soffset && soffset < tmp->eoffset) {
Jerome Glisse721604a2012-01-05 22:11:05 -0500898 /* bo and tmp overlap, invalid offset */
Jerome Glisse721604a2012-01-05 22:11:05 -0500899 dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n",
Christian Könige971bd52012-09-11 16:10:04 +0200900 bo_va->bo, (unsigned)bo_va->soffset, tmp->bo,
Jerome Glisse721604a2012-01-05 22:11:05 -0500901 (unsigned)tmp->soffset, (unsigned)tmp->eoffset);
902 mutex_unlock(&vm->mutex);
903 return -EINVAL;
904 }
905 last_offset = tmp->eoffset;
906 head = &tmp->vm_list;
907 }
Christian Könige971bd52012-09-11 16:10:04 +0200908
909 bo_va->soffset = soffset;
910 bo_va->eoffset = eoffset;
911 bo_va->flags = flags;
912 bo_va->valid = false;
913 list_move(&bo_va->vm_list, head);
914
Jerome Glisse721604a2012-01-05 22:11:05 -0500915 mutex_unlock(&vm->mutex);
916 return 0;
917}
918
Alex Deucher09db8642012-07-17 14:02:40 -0400919/**
Christian Königdce34bf2012-09-17 19:36:18 +0200920 * radeon_vm_map_gart - get the physical address of a gart page
Alex Deucher09db8642012-07-17 14:02:40 -0400921 *
922 * @rdev: radeon_device pointer
Christian Königdce34bf2012-09-17 19:36:18 +0200923 * @addr: the unmapped addr
Alex Deucher09db8642012-07-17 14:02:40 -0400924 *
925 * Look up the physical address of the page that the pte resolves
926 * to (cayman+).
927 * Returns the physical address of the page.
928 */
Christian Königdce34bf2012-09-17 19:36:18 +0200929uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr)
Jerome Glisse721604a2012-01-05 22:11:05 -0500930{
Christian Königdce34bf2012-09-17 19:36:18 +0200931 uint64_t result;
Jerome Glisse721604a2012-01-05 22:11:05 -0500932
Christian Königdce34bf2012-09-17 19:36:18 +0200933 /* page table offset */
934 result = rdev->gart.pages_addr[addr >> PAGE_SHIFT];
935
936 /* in case cpu page size != gpu page size*/
937 result |= addr & (~PAGE_MASK);
938
939 return result;
Jerome Glisse721604a2012-01-05 22:11:05 -0500940}
941
Alex Deucher09db8642012-07-17 14:02:40 -0400942/**
Christian König24c16432013-10-30 11:51:09 -0400943 * radeon_vm_page_flags - translate page flags to what the hw uses
944 *
945 * @flags: flags comming from userspace
946 *
947 * Translate the flags the userspace ABI uses to hw flags.
948 */
949static uint32_t radeon_vm_page_flags(uint32_t flags)
950{
951 uint32_t hw_flags = 0;
952 hw_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0;
953 hw_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
954 hw_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
955 if (flags & RADEON_VM_PAGE_SYSTEM) {
956 hw_flags |= R600_PTE_SYSTEM;
957 hw_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
958 }
959 return hw_flags;
960}
961
962/**
Christian König90a51a32012-10-09 13:31:17 +0200963 * radeon_vm_update_pdes - make sure that page directory is valid
964 *
965 * @rdev: radeon_device pointer
966 * @vm: requested vm
967 * @start: start of GPU address range
968 * @end: end of GPU address range
969 *
970 * Allocates new page tables if necessary
971 * and updates the page directory (cayman+).
972 * Returns 0 for success, error for failure.
973 *
974 * Global and local mutex must be locked!
975 */
976static int radeon_vm_update_pdes(struct radeon_device *rdev,
977 struct radeon_vm *vm,
Alex Deucher43f12142013-02-01 17:32:42 +0100978 struct radeon_ib *ib,
Christian König90a51a32012-10-09 13:31:17 +0200979 uint64_t start, uint64_t end)
980{
981 static const uint32_t incr = RADEON_VM_PTE_COUNT * 8;
982
983 uint64_t last_pde = ~0, last_pt = ~0;
984 unsigned count = 0;
985 uint64_t pt_idx;
986 int r;
987
988 start = (start / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE;
989 end = (end / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE;
990
991 /* walk over the address space and update the page directory */
992 for (pt_idx = start; pt_idx <= end; ++pt_idx) {
993 uint64_t pde, pt;
994
995 if (vm->page_tables[pt_idx])
996 continue;
997
998retry:
999 r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager,
1000 &vm->page_tables[pt_idx],
Alex Deucher3e3e53f2013-07-18 13:11:56 -04001001 RADEON_VM_PTE_COUNT * 8,
1002 RADEON_GPU_PAGE_SIZE, false);
Christian König90a51a32012-10-09 13:31:17 +02001003
1004 if (r == -ENOMEM) {
1005 r = radeon_vm_evict(rdev, vm);
1006 if (r)
1007 return r;
1008 goto retry;
1009 } else if (r) {
1010 return r;
1011 }
1012
1013 pde = vm->pd_gpu_addr + pt_idx * 8;
1014
1015 pt = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]);
1016
1017 if (((last_pde + 8 * count) != pde) ||
1018 ((last_pt + incr * count) != pt)) {
1019
1020 if (count) {
Alex Deucher43f12142013-02-01 17:32:42 +01001021 radeon_asic_vm_set_page(rdev, ib, last_pde,
Christian König90a51a32012-10-09 13:31:17 +02001022 last_pt, count, incr,
Christian König24c16432013-10-30 11:51:09 -04001023 R600_PTE_VALID);
Christian König5b2906e2013-10-29 20:14:50 +01001024
1025 count *= RADEON_VM_PTE_COUNT;
1026 radeon_asic_vm_set_page(rdev, ib, last_pt, 0,
1027 count, 0, 0);
Christian König90a51a32012-10-09 13:31:17 +02001028 }
1029
1030 count = 1;
1031 last_pde = pde;
1032 last_pt = pt;
1033 } else {
1034 ++count;
1035 }
1036 }
1037
1038 if (count) {
Alex Deucher43f12142013-02-01 17:32:42 +01001039 radeon_asic_vm_set_page(rdev, ib, last_pde, last_pt, count,
Christian König24c16432013-10-30 11:51:09 -04001040 incr, R600_PTE_VALID);
Christian König90a51a32012-10-09 13:31:17 +02001041
Christian König5b2906e2013-10-29 20:14:50 +01001042 count *= RADEON_VM_PTE_COUNT;
1043 radeon_asic_vm_set_page(rdev, ib, last_pt, 0,
1044 count, 0, 0);
Christian König90a51a32012-10-09 13:31:17 +02001045 }
1046
1047 return 0;
1048}
1049
1050/**
1051 * radeon_vm_update_ptes - make sure that page tables are valid
1052 *
1053 * @rdev: radeon_device pointer
1054 * @vm: requested vm
1055 * @start: start of GPU address range
1056 * @end: end of GPU address range
1057 * @dst: destination address to map to
1058 * @flags: mapping flags
1059 *
1060 * Update the page tables in the range @start - @end (cayman+).
1061 *
1062 * Global and local mutex must be locked!
1063 */
1064static void radeon_vm_update_ptes(struct radeon_device *rdev,
1065 struct radeon_vm *vm,
Alex Deucher43f12142013-02-01 17:32:42 +01001066 struct radeon_ib *ib,
Christian König90a51a32012-10-09 13:31:17 +02001067 uint64_t start, uint64_t end,
1068 uint64_t dst, uint32_t flags)
1069{
1070 static const uint64_t mask = RADEON_VM_PTE_COUNT - 1;
1071
1072 uint64_t last_pte = ~0, last_dst = ~0;
1073 unsigned count = 0;
1074 uint64_t addr;
1075
1076 start = start / RADEON_GPU_PAGE_SIZE;
1077 end = end / RADEON_GPU_PAGE_SIZE;
1078
1079 /* walk over the address space and update the page tables */
1080 for (addr = start; addr < end; ) {
1081 uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE;
1082 unsigned nptes;
1083 uint64_t pte;
1084
1085 if ((addr & ~mask) == (end & ~mask))
1086 nptes = end - addr;
1087 else
1088 nptes = RADEON_VM_PTE_COUNT - (addr & mask);
1089
1090 pte = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]);
1091 pte += (addr & mask) * 8;
1092
Christian König204a3932012-10-22 17:42:38 +02001093 if ((last_pte + 8 * count) != pte) {
Christian König90a51a32012-10-09 13:31:17 +02001094
1095 if (count) {
Alex Deucher43f12142013-02-01 17:32:42 +01001096 radeon_asic_vm_set_page(rdev, ib, last_pte,
Christian König90a51a32012-10-09 13:31:17 +02001097 last_dst, count,
1098 RADEON_GPU_PAGE_SIZE,
1099 flags);
1100 }
1101
1102 count = nptes;
1103 last_pte = pte;
1104 last_dst = dst;
1105 } else {
1106 count += nptes;
1107 }
1108
1109 addr += nptes;
1110 dst += nptes * RADEON_GPU_PAGE_SIZE;
1111 }
1112
1113 if (count) {
Alex Deucher43f12142013-02-01 17:32:42 +01001114 radeon_asic_vm_set_page(rdev, ib, last_pte,
1115 last_dst, count,
Christian König90a51a32012-10-09 13:31:17 +02001116 RADEON_GPU_PAGE_SIZE, flags);
1117 }
1118}
1119
1120/**
Christian König9c57a6b2013-11-25 15:42:11 +01001121 * radeon_vm_bo_update - map a bo into the vm page table
Alex Deucher09db8642012-07-17 14:02:40 -04001122 *
1123 * @rdev: radeon_device pointer
1124 * @vm: requested vm
1125 * @bo: radeon buffer object
1126 * @mem: ttm mem
1127 *
1128 * Fill in the page table entries for @bo (cayman+).
1129 * Returns 0 for success, -EINVAL for failure.
Christian König2a6f1ab2012-08-11 15:00:30 +02001130 *
1131 * Object have to be reserved & global and local mutex must be locked!
Alex Deucher09db8642012-07-17 14:02:40 -04001132 */
Christian König9c57a6b2013-11-25 15:42:11 +01001133int radeon_vm_bo_update(struct radeon_device *rdev,
1134 struct radeon_vm *vm,
1135 struct radeon_bo *bo,
1136 struct ttm_mem_reg *mem)
Jerome Glisse721604a2012-01-05 22:11:05 -05001137{
Alex Deucher43f12142013-02-01 17:32:42 +01001138 struct radeon_ib ib;
Jerome Glisse721604a2012-01-05 22:11:05 -05001139 struct radeon_bo_va *bo_va;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001140 unsigned nptes, npdes, ndw;
Christian König90a51a32012-10-09 13:31:17 +02001141 uint64_t addr;
Christian König2a6f1ab2012-08-11 15:00:30 +02001142 int r;
Jerome Glisse721604a2012-01-05 22:11:05 -05001143
1144 /* nothing to do if vm isn't bound */
Christian König90a51a32012-10-09 13:31:17 +02001145 if (vm->page_directory == NULL)
Jesper Juhl04bd27a2012-02-26 23:51:53 +01001146 return 0;
Jerome Glisse721604a2012-01-05 22:11:05 -05001147
Christian König421ca7a2012-09-11 16:10:00 +02001148 bo_va = radeon_vm_bo_find(vm, bo);
Jerome Glisse721604a2012-01-05 22:11:05 -05001149 if (bo_va == NULL) {
1150 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm);
1151 return -EINVAL;
1152 }
1153
Christian Könige971bd52012-09-11 16:10:04 +02001154 if (!bo_va->soffset) {
1155 dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",
1156 bo, vm);
1157 return -EINVAL;
1158 }
1159
Christian König2a6f1ab2012-08-11 15:00:30 +02001160 if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL))
Jerome Glisse721604a2012-01-05 22:11:05 -05001161 return 0;
1162
Jerome Glisse721604a2012-01-05 22:11:05 -05001163 bo_va->flags &= ~RADEON_VM_PAGE_VALID;
1164 bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
1165 if (mem) {
Christian Königdce34bf2012-09-17 19:36:18 +02001166 addr = mem->start << PAGE_SHIFT;
Jerome Glisse721604a2012-01-05 22:11:05 -05001167 if (mem->mem_type != TTM_PL_SYSTEM) {
1168 bo_va->flags |= RADEON_VM_PAGE_VALID;
1169 bo_va->valid = true;
1170 }
1171 if (mem->mem_type == TTM_PL_TT) {
1172 bo_va->flags |= RADEON_VM_PAGE_SYSTEM;
Christian Königdce34bf2012-09-17 19:36:18 +02001173 } else {
1174 addr += rdev->vm_manager.vram_base_offset;
Christian König2a6f1ab2012-08-11 15:00:30 +02001175 }
1176 } else {
Christian Königdce34bf2012-09-17 19:36:18 +02001177 addr = 0;
Christian König2a6f1ab2012-08-11 15:00:30 +02001178 bo_va->valid = false;
Jerome Glisse721604a2012-01-05 22:11:05 -05001179 }
Christian König2a6f1ab2012-08-11 15:00:30 +02001180
Christian König9c57a6b2013-11-25 15:42:11 +01001181 trace_radeon_vm_bo_update(bo_va);
1182
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001183 nptes = radeon_bo_ngpu_pages(bo);
1184
Christian König90a51a32012-10-09 13:31:17 +02001185 /* assume two extra pdes in case the mapping overlaps the borders */
1186 npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001187
Alex Deucher43f12142013-02-01 17:32:42 +01001188 /* padding, etc. */
1189 ndw = 64;
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001190
Christian König90a51a32012-10-09 13:31:17 +02001191 if (RADEON_VM_BLOCK_SIZE > 11)
1192 /* reserve space for one header for every 2k dwords */
Christian König08eda322012-10-22 17:42:39 +02001193 ndw += (nptes >> 11) * 4;
Christian König90a51a32012-10-09 13:31:17 +02001194 else
1195 /* reserve space for one header for
1196 every (1 << BLOCK_SIZE) entries */
Christian König08eda322012-10-22 17:42:39 +02001197 ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4;
Christian König90a51a32012-10-09 13:31:17 +02001198
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001199 /* reserve space for pte addresses */
1200 ndw += nptes * 2;
1201
1202 /* reserve space for one header for every 2k dwords */
Christian König08eda322012-10-22 17:42:39 +02001203 ndw += (npdes >> 11) * 4;
Christian König90a51a32012-10-09 13:31:17 +02001204
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02001205 /* reserve space for pde addresses */
1206 ndw += npdes * 2;
Christian König2a6f1ab2012-08-11 15:00:30 +02001207
Christian König5b2906e2013-10-29 20:14:50 +01001208 /* reserve space for clearing new page tables */
1209 ndw += npdes * 2 * RADEON_VM_PTE_COUNT;
1210
Alex Deucher43f12142013-02-01 17:32:42 +01001211 /* update too big for an IB */
1212 if (ndw > 0xfffff)
1213 return -ENOMEM;
1214
Christian König24c16432013-10-30 11:51:09 -04001215 r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);
Alex Deucher4cc948b2013-11-13 15:25:35 -05001216 if (r)
1217 return r;
Alex Deucher43f12142013-02-01 17:32:42 +01001218 ib.length_dw = 0;
1219
1220 r = radeon_vm_update_pdes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset);
Christian König2a6f1ab2012-08-11 15:00:30 +02001221 if (r) {
Alex Deucher43f12142013-02-01 17:32:42 +01001222 radeon_ib_free(rdev, &ib);
Christian König2a6f1ab2012-08-11 15:00:30 +02001223 return r;
Jerome Glisse721604a2012-01-05 22:11:05 -05001224 }
Christian König2a6f1ab2012-08-11 15:00:30 +02001225
Alex Deucher43f12142013-02-01 17:32:42 +01001226 radeon_vm_update_ptes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset,
Christian König24c16432013-10-30 11:51:09 -04001227 addr, radeon_vm_page_flags(bo_va->flags));
Christian König2a6f1ab2012-08-11 15:00:30 +02001228
Christian König1654b812013-11-12 12:58:05 +01001229 radeon_semaphore_sync_to(ib.semaphore, vm->fence);
Alex Deucher43f12142013-02-01 17:32:42 +01001230 r = radeon_ib_schedule(rdev, &ib, NULL);
Christian König2a6f1ab2012-08-11 15:00:30 +02001231 if (r) {
Alex Deucher43f12142013-02-01 17:32:42 +01001232 radeon_ib_free(rdev, &ib);
Christian König2a6f1ab2012-08-11 15:00:30 +02001233 return r;
1234 }
Alex Deucher43f12142013-02-01 17:32:42 +01001235 radeon_fence_unref(&vm->fence);
1236 vm->fence = radeon_fence_ref(ib.fence);
1237 radeon_ib_free(rdev, &ib);
Christian König9b40e5d2012-08-08 12:22:43 +02001238 radeon_fence_unref(&vm->last_flush);
Christian König90a51a32012-10-09 13:31:17 +02001239
Jerome Glisse721604a2012-01-05 22:11:05 -05001240 return 0;
1241}
1242
Alex Deucher09db8642012-07-17 14:02:40 -04001243/**
1244 * radeon_vm_bo_rmv - remove a bo to a specific vm
1245 *
1246 * @rdev: radeon_device pointer
Christian Könige971bd52012-09-11 16:10:04 +02001247 * @bo_va: requested bo_va
Alex Deucher09db8642012-07-17 14:02:40 -04001248 *
Christian Könige971bd52012-09-11 16:10:04 +02001249 * Remove @bo_va->bo from the requested vm (cayman+).
1250 * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and
1251 * remove the ptes for @bo_va in the page table.
Alex Deucher09db8642012-07-17 14:02:40 -04001252 * Returns 0 for success.
Christian Königddf03f52012-08-09 20:02:28 +02001253 *
1254 * Object have to be reserved!
Alex Deucher09db8642012-07-17 14:02:40 -04001255 */
Jerome Glisse721604a2012-01-05 22:11:05 -05001256int radeon_vm_bo_rmv(struct radeon_device *rdev,
Christian Könige971bd52012-09-11 16:10:04 +02001257 struct radeon_bo_va *bo_va)
Jerome Glisse721604a2012-01-05 22:11:05 -05001258{
Jerome Glisse3813f5c2013-06-06 12:41:17 -04001259 int r = 0;
Jerome Glisse721604a2012-01-05 22:11:05 -05001260
Christian König36ff39c2012-05-09 10:07:08 +02001261 mutex_lock(&rdev->vm_manager.lock);
Christian Könige971bd52012-09-11 16:10:04 +02001262 mutex_lock(&bo_va->vm->mutex);
Jerome Glisse3813f5c2013-06-06 12:41:17 -04001263 if (bo_va->soffset) {
Christian König9c57a6b2013-11-25 15:42:11 +01001264 r = radeon_vm_bo_update(rdev, bo_va->vm, bo_va->bo, NULL);
Jerome Glisse3813f5c2013-06-06 12:41:17 -04001265 }
Christian König36ff39c2012-05-09 10:07:08 +02001266 mutex_unlock(&rdev->vm_manager.lock);
Jerome Glisse721604a2012-01-05 22:11:05 -05001267 list_del(&bo_va->vm_list);
Christian Könige971bd52012-09-11 16:10:04 +02001268 mutex_unlock(&bo_va->vm->mutex);
Sebastian Biemueller108b0d32012-02-29 11:04:52 -05001269 list_del(&bo_va->bo_list);
Jerome Glisse721604a2012-01-05 22:11:05 -05001270
1271 kfree(bo_va);
Christian König2a6f1ab2012-08-11 15:00:30 +02001272 return r;
Jerome Glisse721604a2012-01-05 22:11:05 -05001273}
1274
Alex Deucher09db8642012-07-17 14:02:40 -04001275/**
1276 * radeon_vm_bo_invalidate - mark the bo as invalid
1277 *
1278 * @rdev: radeon_device pointer
1279 * @vm: requested vm
1280 * @bo: radeon buffer object
1281 *
1282 * Mark @bo as invalid (cayman+).
1283 */
Jerome Glisse721604a2012-01-05 22:11:05 -05001284void radeon_vm_bo_invalidate(struct radeon_device *rdev,
1285 struct radeon_bo *bo)
1286{
1287 struct radeon_bo_va *bo_va;
1288
Jerome Glisse721604a2012-01-05 22:11:05 -05001289 list_for_each_entry(bo_va, &bo->va, bo_list) {
1290 bo_va->valid = false;
1291 }
1292}
1293
Alex Deucher09db8642012-07-17 14:02:40 -04001294/**
1295 * radeon_vm_init - initialize a vm instance
1296 *
1297 * @rdev: radeon_device pointer
1298 * @vm: requested vm
1299 *
Christian Königd72d43c2012-10-09 13:31:18 +02001300 * Init @vm fields (cayman+).
Alex Deucher09db8642012-07-17 14:02:40 -04001301 */
Christian Königd72d43c2012-10-09 13:31:18 +02001302void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
Jerome Glisse721604a2012-01-05 22:11:05 -05001303{
Christian Königee60e292012-08-09 16:21:08 +02001304 vm->id = 0;
Jerome Glisse721604a2012-01-05 22:11:05 -05001305 vm->fence = NULL;
1306 mutex_init(&vm->mutex);
1307 INIT_LIST_HEAD(&vm->list);
1308 INIT_LIST_HEAD(&vm->va);
Jerome Glisse721604a2012-01-05 22:11:05 -05001309}
1310
Alex Deucher09db8642012-07-17 14:02:40 -04001311/**
Dmitrii Cherkasovf59abbf2012-08-13 10:53:29 -04001312 * radeon_vm_fini - tear down a vm instance
Alex Deucher09db8642012-07-17 14:02:40 -04001313 *
1314 * @rdev: radeon_device pointer
1315 * @vm: requested vm
1316 *
1317 * Tear down @vm (cayman+).
1318 * Unbind the VM and remove all bos from the vm bo list
1319 */
Jerome Glisse721604a2012-01-05 22:11:05 -05001320void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
1321{
1322 struct radeon_bo_va *bo_va, *tmp;
1323 int r;
1324
Christian König36ff39c2012-05-09 10:07:08 +02001325 mutex_lock(&rdev->vm_manager.lock);
Christian Königbb409152012-06-03 16:09:43 +02001326 mutex_lock(&vm->mutex);
Christian Königddf03f52012-08-09 20:02:28 +02001327 radeon_vm_free_pt(rdev, vm);
Christian König36ff39c2012-05-09 10:07:08 +02001328 mutex_unlock(&rdev->vm_manager.lock);
Jerome Glisse721604a2012-01-05 22:11:05 -05001329
Jerome Glisse721604a2012-01-05 22:11:05 -05001330 if (!list_empty(&vm->va)) {
1331 dev_err(rdev->dev, "still active bo inside vm\n");
1332 }
1333 list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) {
1334 list_del_init(&bo_va->vm_list);
1335 r = radeon_bo_reserve(bo_va->bo, false);
1336 if (!r) {
1337 list_del_init(&bo_va->bo_list);
1338 radeon_bo_unreserve(bo_va->bo);
1339 kfree(bo_va);
1340 }
1341 }
Christian Königddf03f52012-08-09 20:02:28 +02001342 radeon_fence_unref(&vm->fence);
1343 radeon_fence_unref(&vm->last_flush);
Jerome Glisse721604a2012-01-05 22:11:05 -05001344 mutex_unlock(&vm->mutex);
1345}