blob: c41a5a96b558bd8d17ccaf3adc8b49e95c59e817 [file] [log] [blame]
Alexander Grafb71c9e22013-01-11 15:22:45 +01001/*
2 * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. All rights reserved.
3 *
4 * Author: Yu Liu, yu.liu@freescale.com
5 * Scott Wood, scottwood@freescale.com
6 * Ashish Kalra, ashish.kalra@freescale.com
7 * Varun Sethi, varun.sethi@freescale.com
8 * Alexander Graf, agraf@suse.de
9 *
10 * Description:
11 * This file is based on arch/powerpc/kvm/44x_tlb.c,
12 * by Hollis Blanchard <hollisb@us.ibm.com>.
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License, version 2, as
16 * published by the Free Software Foundation.
17 */
18
19#include <linux/kernel.h>
20#include <linux/types.h>
21#include <linux/slab.h>
22#include <linux/string.h>
23#include <linux/kvm.h>
24#include <linux/kvm_host.h>
25#include <linux/highmem.h>
26#include <linux/log2.h>
27#include <linux/uaccess.h>
28#include <linux/sched.h>
29#include <linux/rwsem.h>
30#include <linux/vmalloc.h>
31#include <linux/hugetlb.h>
32#include <asm/kvm_ppc.h>
33
34#include "e500.h"
35#include "trace.h"
36#include "timing.h"
37#include "e500_mmu_host.h"
38
39static inline unsigned int gtlb0_get_next_victim(
40 struct kvmppc_vcpu_e500 *vcpu_e500)
41{
42 unsigned int victim;
43
44 victim = vcpu_e500->gtlb_nv[0]++;
45 if (unlikely(vcpu_e500->gtlb_nv[0] >= vcpu_e500->gtlb_params[0].ways))
46 vcpu_e500->gtlb_nv[0] = 0;
47
48 return victim;
49}
50
51static int tlb0_set_base(gva_t addr, int sets, int ways)
52{
53 int set_base;
54
55 set_base = (addr >> PAGE_SHIFT) & (sets - 1);
56 set_base *= ways;
57
58 return set_base;
59}
60
61static int gtlb0_set_base(struct kvmppc_vcpu_e500 *vcpu_e500, gva_t addr)
62{
63 return tlb0_set_base(addr, vcpu_e500->gtlb_params[0].sets,
64 vcpu_e500->gtlb_params[0].ways);
65}
66
67static unsigned int get_tlb_esel(struct kvm_vcpu *vcpu, int tlbsel)
68{
69 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
70 int esel = get_tlb_esel_bit(vcpu);
71
72 if (tlbsel == 0) {
73 esel &= vcpu_e500->gtlb_params[0].ways - 1;
74 esel += gtlb0_set_base(vcpu_e500, vcpu->arch.shared->mas2);
75 } else {
76 esel &= vcpu_e500->gtlb_params[tlbsel].entries - 1;
77 }
78
79 return esel;
80}
81
82/* Search the guest TLB for a matching entry. */
83static int kvmppc_e500_tlb_index(struct kvmppc_vcpu_e500 *vcpu_e500,
84 gva_t eaddr, int tlbsel, unsigned int pid, int as)
85{
86 int size = vcpu_e500->gtlb_params[tlbsel].entries;
87 unsigned int set_base, offset;
88 int i;
89
90 if (tlbsel == 0) {
91 set_base = gtlb0_set_base(vcpu_e500, eaddr);
92 size = vcpu_e500->gtlb_params[0].ways;
93 } else {
94 if (eaddr < vcpu_e500->tlb1_min_eaddr ||
95 eaddr > vcpu_e500->tlb1_max_eaddr)
96 return -1;
97 set_base = 0;
98 }
99
100 offset = vcpu_e500->gtlb_offset[tlbsel];
101
102 for (i = 0; i < size; i++) {
103 struct kvm_book3e_206_tlb_entry *tlbe =
104 &vcpu_e500->gtlb_arch[offset + set_base + i];
105 unsigned int tid;
106
107 if (eaddr < get_tlb_eaddr(tlbe))
108 continue;
109
110 if (eaddr > get_tlb_end(tlbe))
111 continue;
112
113 tid = get_tlb_tid(tlbe);
114 if (tid && (tid != pid))
115 continue;
116
117 if (!get_tlb_v(tlbe))
118 continue;
119
120 if (get_tlb_ts(tlbe) != as && as != -1)
121 continue;
122
123 return set_base + i;
124 }
125
126 return -1;
127}
128
129static inline void kvmppc_e500_deliver_tlb_miss(struct kvm_vcpu *vcpu,
130 unsigned int eaddr, int as)
131{
132 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
133 unsigned int victim, tsized;
134 int tlbsel;
135
136 /* since we only have two TLBs, only lower bit is used. */
137 tlbsel = (vcpu->arch.shared->mas4 >> 28) & 0x1;
138 victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0;
139 tsized = (vcpu->arch.shared->mas4 >> 7) & 0x1f;
140
141 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(victim)
142 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
143 vcpu->arch.shared->mas1 = MAS1_VALID | (as ? MAS1_TS : 0)
144 | MAS1_TID(get_tlbmiss_tid(vcpu))
145 | MAS1_TSIZE(tsized);
146 vcpu->arch.shared->mas2 = (eaddr & MAS2_EPN)
147 | (vcpu->arch.shared->mas4 & MAS2_ATTRIB_MASK);
148 vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 | MAS3_U2 | MAS3_U3;
149 vcpu->arch.shared->mas6 = (vcpu->arch.shared->mas6 & MAS6_SPID1)
150 | (get_cur_pid(vcpu) << 16)
151 | (as ? MAS6_SAS : 0);
152}
153
154static void kvmppc_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500)
155{
156 int size = vcpu_e500->gtlb_params[1].entries;
157 unsigned int offset;
158 gva_t eaddr;
159 int i;
160
161 vcpu_e500->tlb1_min_eaddr = ~0UL;
162 vcpu_e500->tlb1_max_eaddr = 0;
163 offset = vcpu_e500->gtlb_offset[1];
164
165 for (i = 0; i < size; i++) {
166 struct kvm_book3e_206_tlb_entry *tlbe =
167 &vcpu_e500->gtlb_arch[offset + i];
168
169 if (!get_tlb_v(tlbe))
170 continue;
171
172 eaddr = get_tlb_eaddr(tlbe);
173 vcpu_e500->tlb1_min_eaddr =
174 min(vcpu_e500->tlb1_min_eaddr, eaddr);
175
176 eaddr = get_tlb_end(tlbe);
177 vcpu_e500->tlb1_max_eaddr =
178 max(vcpu_e500->tlb1_max_eaddr, eaddr);
179 }
180}
181
182static int kvmppc_need_recalc_tlb1map_range(struct kvmppc_vcpu_e500 *vcpu_e500,
183 struct kvm_book3e_206_tlb_entry *gtlbe)
184{
185 unsigned long start, end, size;
186
187 size = get_tlb_bytes(gtlbe);
188 start = get_tlb_eaddr(gtlbe) & ~(size - 1);
189 end = start + size - 1;
190
191 return vcpu_e500->tlb1_min_eaddr == start ||
192 vcpu_e500->tlb1_max_eaddr == end;
193}
194
195/* This function is supposed to be called for a adding a new valid tlb entry */
196static void kvmppc_set_tlb1map_range(struct kvm_vcpu *vcpu,
197 struct kvm_book3e_206_tlb_entry *gtlbe)
198{
199 unsigned long start, end, size;
200 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
201
202 if (!get_tlb_v(gtlbe))
203 return;
204
205 size = get_tlb_bytes(gtlbe);
206 start = get_tlb_eaddr(gtlbe) & ~(size - 1);
207 end = start + size - 1;
208
209 vcpu_e500->tlb1_min_eaddr = min(vcpu_e500->tlb1_min_eaddr, start);
210 vcpu_e500->tlb1_max_eaddr = max(vcpu_e500->tlb1_max_eaddr, end);
211}
212
213static inline int kvmppc_e500_gtlbe_invalidate(
214 struct kvmppc_vcpu_e500 *vcpu_e500,
215 int tlbsel, int esel)
216{
217 struct kvm_book3e_206_tlb_entry *gtlbe =
218 get_entry(vcpu_e500, tlbsel, esel);
219
220 if (unlikely(get_tlb_iprot(gtlbe)))
221 return -1;
222
223 if (tlbsel == 1 && kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe))
224 kvmppc_recalc_tlb1map_range(vcpu_e500);
225
226 gtlbe->mas1 = 0;
227
228 return 0;
229}
230
231int kvmppc_e500_emul_mt_mmucsr0(struct kvmppc_vcpu_e500 *vcpu_e500, ulong value)
232{
233 int esel;
234
235 if (value & MMUCSR0_TLB0FI)
236 for (esel = 0; esel < vcpu_e500->gtlb_params[0].entries; esel++)
237 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 0, esel);
238 if (value & MMUCSR0_TLB1FI)
239 for (esel = 0; esel < vcpu_e500->gtlb_params[1].entries; esel++)
240 kvmppc_e500_gtlbe_invalidate(vcpu_e500, 1, esel);
241
Alexander Grafb9e3e202013-01-18 15:22:08 +0100242 /* Invalidate all host shadow mappings */
243 kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
Alexander Grafb71c9e22013-01-11 15:22:45 +0100244
245 return EMULATE_DONE;
246}
247
248int kvmppc_e500_emul_tlbivax(struct kvm_vcpu *vcpu, gva_t ea)
249{
250 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
251 unsigned int ia;
252 int esel, tlbsel;
253
254 ia = (ea >> 2) & 0x1;
255
256 /* since we only have two TLBs, only lower bit is used. */
257 tlbsel = (ea >> 3) & 0x1;
258
259 if (ia) {
260 /* invalidate all entries */
261 for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries;
262 esel++)
263 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
264 } else {
265 ea &= 0xfffff000;
266 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel,
267 get_cur_pid(vcpu), -1);
268 if (esel >= 0)
269 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
270 }
271
Alexander Grafb9e3e202013-01-18 15:22:08 +0100272 /* Invalidate all host shadow mappings */
273 kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
Alexander Grafb71c9e22013-01-11 15:22:45 +0100274
275 return EMULATE_DONE;
276}
277
278static void tlbilx_all(struct kvmppc_vcpu_e500 *vcpu_e500, int tlbsel,
279 int pid, int type)
280{
281 struct kvm_book3e_206_tlb_entry *tlbe;
282 int tid, esel;
283
284 /* invalidate all entries */
285 for (esel = 0; esel < vcpu_e500->gtlb_params[tlbsel].entries; esel++) {
286 tlbe = get_entry(vcpu_e500, tlbsel, esel);
287 tid = get_tlb_tid(tlbe);
288 if (type == 0 || tid == pid) {
289 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
290 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
291 }
292 }
293}
294
295static void tlbilx_one(struct kvmppc_vcpu_e500 *vcpu_e500, int pid,
296 gva_t ea)
297{
298 int tlbsel, esel;
299
300 for (tlbsel = 0; tlbsel < 2; tlbsel++) {
301 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, -1);
302 if (esel >= 0) {
303 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
304 kvmppc_e500_gtlbe_invalidate(vcpu_e500, tlbsel, esel);
305 break;
306 }
307 }
308}
309
310int kvmppc_e500_emul_tlbilx(struct kvm_vcpu *vcpu, int type, gva_t ea)
311{
312 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
313 int pid = get_cur_spid(vcpu);
314
315 if (type == 0 || type == 1) {
316 tlbilx_all(vcpu_e500, 0, pid, type);
317 tlbilx_all(vcpu_e500, 1, pid, type);
318 } else if (type == 3) {
319 tlbilx_one(vcpu_e500, pid, ea);
320 }
321
322 return EMULATE_DONE;
323}
324
325int kvmppc_e500_emul_tlbre(struct kvm_vcpu *vcpu)
326{
327 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
328 int tlbsel, esel;
329 struct kvm_book3e_206_tlb_entry *gtlbe;
330
331 tlbsel = get_tlb_tlbsel(vcpu);
332 esel = get_tlb_esel(vcpu, tlbsel);
333
334 gtlbe = get_entry(vcpu_e500, tlbsel, esel);
335 vcpu->arch.shared->mas0 &= ~MAS0_NV(~0);
336 vcpu->arch.shared->mas0 |= MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
337 vcpu->arch.shared->mas1 = gtlbe->mas1;
338 vcpu->arch.shared->mas2 = gtlbe->mas2;
339 vcpu->arch.shared->mas7_3 = gtlbe->mas7_3;
340
341 return EMULATE_DONE;
342}
343
344int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, gva_t ea)
345{
346 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
347 int as = !!get_cur_sas(vcpu);
348 unsigned int pid = get_cur_spid(vcpu);
349 int esel, tlbsel;
350 struct kvm_book3e_206_tlb_entry *gtlbe = NULL;
351
352 for (tlbsel = 0; tlbsel < 2; tlbsel++) {
353 esel = kvmppc_e500_tlb_index(vcpu_e500, ea, tlbsel, pid, as);
354 if (esel >= 0) {
355 gtlbe = get_entry(vcpu_e500, tlbsel, esel);
356 break;
357 }
358 }
359
360 if (gtlbe) {
361 esel &= vcpu_e500->gtlb_params[tlbsel].ways - 1;
362
363 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel) | MAS0_ESEL(esel)
364 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
365 vcpu->arch.shared->mas1 = gtlbe->mas1;
366 vcpu->arch.shared->mas2 = gtlbe->mas2;
367 vcpu->arch.shared->mas7_3 = gtlbe->mas7_3;
368 } else {
369 int victim;
370
371 /* since we only have two TLBs, only lower bit is used. */
372 tlbsel = vcpu->arch.shared->mas4 >> 28 & 0x1;
373 victim = (tlbsel == 0) ? gtlb0_get_next_victim(vcpu_e500) : 0;
374
375 vcpu->arch.shared->mas0 = MAS0_TLBSEL(tlbsel)
376 | MAS0_ESEL(victim)
377 | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]);
378 vcpu->arch.shared->mas1 =
379 (vcpu->arch.shared->mas6 & MAS6_SPID0)
380 | (vcpu->arch.shared->mas6 & (MAS6_SAS ? MAS1_TS : 0))
381 | (vcpu->arch.shared->mas4 & MAS4_TSIZED(~0));
382 vcpu->arch.shared->mas2 &= MAS2_EPN;
383 vcpu->arch.shared->mas2 |= vcpu->arch.shared->mas4 &
384 MAS2_ATTRIB_MASK;
385 vcpu->arch.shared->mas7_3 &= MAS3_U0 | MAS3_U1 |
386 MAS3_U2 | MAS3_U3;
387 }
388
389 kvmppc_set_exit_type(vcpu, EMULATED_TLBSX_EXITS);
390 return EMULATE_DONE;
391}
392
393int kvmppc_e500_emul_tlbwe(struct kvm_vcpu *vcpu)
394{
395 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
396 struct kvm_book3e_206_tlb_entry *gtlbe;
397 int tlbsel, esel;
398 int recal = 0;
399
400 tlbsel = get_tlb_tlbsel(vcpu);
401 esel = get_tlb_esel(vcpu, tlbsel);
402
403 gtlbe = get_entry(vcpu_e500, tlbsel, esel);
404
405 if (get_tlb_v(gtlbe)) {
406 inval_gtlbe_on_host(vcpu_e500, tlbsel, esel);
407 if ((tlbsel == 1) &&
408 kvmppc_need_recalc_tlb1map_range(vcpu_e500, gtlbe))
409 recal = 1;
410 }
411
412 gtlbe->mas1 = vcpu->arch.shared->mas1;
413 gtlbe->mas2 = vcpu->arch.shared->mas2;
414 if (!(vcpu->arch.shared->msr & MSR_CM))
415 gtlbe->mas2 &= 0xffffffffUL;
416 gtlbe->mas7_3 = vcpu->arch.shared->mas7_3;
417
418 trace_kvm_booke206_gtlb_write(vcpu->arch.shared->mas0, gtlbe->mas1,
419 gtlbe->mas2, gtlbe->mas7_3);
420
421 if (tlbsel == 1) {
422 /*
423 * If a valid tlb1 entry is overwritten then recalculate the
424 * min/max TLB1 map address range otherwise no need to look
425 * in tlb1 array.
426 */
427 if (recal)
428 kvmppc_recalc_tlb1map_range(vcpu_e500);
429 else
430 kvmppc_set_tlb1map_range(vcpu, gtlbe);
431 }
432
433 /* Invalidate shadow mappings for the about-to-be-clobbered TLBE. */
434 if (tlbe_is_host_safe(vcpu, gtlbe)) {
435 u64 eaddr = get_tlb_eaddr(gtlbe);
436 u64 raddr = get_tlb_raddr(gtlbe);
437
438 if (tlbsel == 0) {
439 gtlbe->mas1 &= ~MAS1_TSIZE(~0);
440 gtlbe->mas1 |= MAS1_TSIZE(BOOK3E_PAGESZ_4K);
441 }
442
443 /* Premap the faulting page */
444 kvmppc_mmu_map(vcpu, eaddr, raddr, index_of(tlbsel, esel));
445 }
446
447 kvmppc_set_exit_type(vcpu, EMULATED_TLBWE_EXITS);
448 return EMULATE_DONE;
449}
450
451static int kvmppc_e500_tlb_search(struct kvm_vcpu *vcpu,
452 gva_t eaddr, unsigned int pid, int as)
453{
454 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
455 int esel, tlbsel;
456
457 for (tlbsel = 0; tlbsel < 2; tlbsel++) {
458 esel = kvmppc_e500_tlb_index(vcpu_e500, eaddr, tlbsel, pid, as);
459 if (esel >= 0)
460 return index_of(tlbsel, esel);
461 }
462
463 return -1;
464}
465
466/* 'linear_address' is actually an encoding of AS|PID|EADDR . */
467int kvmppc_core_vcpu_translate(struct kvm_vcpu *vcpu,
468 struct kvm_translation *tr)
469{
470 int index;
471 gva_t eaddr;
472 u8 pid;
473 u8 as;
474
475 eaddr = tr->linear_address;
476 pid = (tr->linear_address >> 32) & 0xff;
477 as = (tr->linear_address >> 40) & 0x1;
478
479 index = kvmppc_e500_tlb_search(vcpu, eaddr, pid, as);
480 if (index < 0) {
481 tr->valid = 0;
482 return 0;
483 }
484
485 tr->physical_address = kvmppc_mmu_xlate(vcpu, index, eaddr);
486 /* XXX what does "writeable" and "usermode" even mean? */
487 tr->valid = 1;
488
489 return 0;
490}
491
492
493int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)
494{
495 unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);
496
497 return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as);
498}
499
500int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr)
501{
502 unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS);
503
504 return kvmppc_e500_tlb_search(vcpu, eaddr, get_cur_pid(vcpu), as);
505}
506
507void kvmppc_mmu_itlb_miss(struct kvm_vcpu *vcpu)
508{
509 unsigned int as = !!(vcpu->arch.shared->msr & MSR_IS);
510
511 kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.pc, as);
512}
513
514void kvmppc_mmu_dtlb_miss(struct kvm_vcpu *vcpu)
515{
516 unsigned int as = !!(vcpu->arch.shared->msr & MSR_DS);
517
518 kvmppc_e500_deliver_tlb_miss(vcpu, vcpu->arch.fault_dear, as);
519}
520
521gpa_t kvmppc_mmu_xlate(struct kvm_vcpu *vcpu, unsigned int index,
522 gva_t eaddr)
523{
524 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
525 struct kvm_book3e_206_tlb_entry *gtlbe;
526 u64 pgmask;
527
528 gtlbe = get_entry(vcpu_e500, tlbsel_of(index), esel_of(index));
529 pgmask = get_tlb_bytes(gtlbe) - 1;
530
531 return get_tlb_raddr(gtlbe) | (eaddr & pgmask);
532}
533
534void kvmppc_mmu_destroy(struct kvm_vcpu *vcpu)
535{
536}
537
538/*****************************************/
539
540static void free_gtlb(struct kvmppc_vcpu_e500 *vcpu_e500)
541{
542 int i;
543
Alexander Graf483ba972013-01-18 15:13:19 +0100544 kvmppc_core_flush_tlb(&vcpu_e500->vcpu);
Alexander Grafb71c9e22013-01-11 15:22:45 +0100545 kfree(vcpu_e500->g2h_tlb1_map);
Alexander Grafb71c9e22013-01-11 15:22:45 +0100546 kfree(vcpu_e500->gtlb_priv[0]);
547 kfree(vcpu_e500->gtlb_priv[1]);
548
549 if (vcpu_e500->shared_tlb_pages) {
550 vfree((void *)(round_down((uintptr_t)vcpu_e500->gtlb_arch,
551 PAGE_SIZE)));
552
553 for (i = 0; i < vcpu_e500->num_shared_tlb_pages; i++) {
554 set_page_dirty_lock(vcpu_e500->shared_tlb_pages[i]);
555 put_page(vcpu_e500->shared_tlb_pages[i]);
556 }
557
558 vcpu_e500->num_shared_tlb_pages = 0;
559
560 kfree(vcpu_e500->shared_tlb_pages);
561 vcpu_e500->shared_tlb_pages = NULL;
562 } else {
563 kfree(vcpu_e500->gtlb_arch);
564 }
565
566 vcpu_e500->gtlb_arch = NULL;
567}
568
569void kvmppc_get_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
570{
571 sregs->u.e.mas0 = vcpu->arch.shared->mas0;
572 sregs->u.e.mas1 = vcpu->arch.shared->mas1;
573 sregs->u.e.mas2 = vcpu->arch.shared->mas2;
574 sregs->u.e.mas7_3 = vcpu->arch.shared->mas7_3;
575 sregs->u.e.mas4 = vcpu->arch.shared->mas4;
576 sregs->u.e.mas6 = vcpu->arch.shared->mas6;
577
578 sregs->u.e.mmucfg = vcpu->arch.mmucfg;
579 sregs->u.e.tlbcfg[0] = vcpu->arch.tlbcfg[0];
580 sregs->u.e.tlbcfg[1] = vcpu->arch.tlbcfg[1];
581 sregs->u.e.tlbcfg[2] = 0;
582 sregs->u.e.tlbcfg[3] = 0;
583}
584
585int kvmppc_set_sregs_e500_tlb(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
586{
587 if (sregs->u.e.features & KVM_SREGS_E_ARCH206_MMU) {
588 vcpu->arch.shared->mas0 = sregs->u.e.mas0;
589 vcpu->arch.shared->mas1 = sregs->u.e.mas1;
590 vcpu->arch.shared->mas2 = sregs->u.e.mas2;
591 vcpu->arch.shared->mas7_3 = sregs->u.e.mas7_3;
592 vcpu->arch.shared->mas4 = sregs->u.e.mas4;
593 vcpu->arch.shared->mas6 = sregs->u.e.mas6;
594 }
595
596 return 0;
597}
598
Mihai Caramana85d2aa2013-04-11 00:03:08 +0000599int kvmppc_get_one_reg_e500_tlb(struct kvm_vcpu *vcpu, u64 id,
600 union kvmppc_one_reg *val)
601{
602 int r = 0;
603 long int i;
604
605 switch (id) {
606 case KVM_REG_PPC_MAS0:
607 *val = get_reg_val(id, vcpu->arch.shared->mas0);
608 break;
609 case KVM_REG_PPC_MAS1:
610 *val = get_reg_val(id, vcpu->arch.shared->mas1);
611 break;
612 case KVM_REG_PPC_MAS2:
613 *val = get_reg_val(id, vcpu->arch.shared->mas2);
614 break;
615 case KVM_REG_PPC_MAS7_3:
616 *val = get_reg_val(id, vcpu->arch.shared->mas7_3);
617 break;
618 case KVM_REG_PPC_MAS4:
619 *val = get_reg_val(id, vcpu->arch.shared->mas4);
620 break;
621 case KVM_REG_PPC_MAS6:
622 *val = get_reg_val(id, vcpu->arch.shared->mas6);
623 break;
624 case KVM_REG_PPC_MMUCFG:
625 *val = get_reg_val(id, vcpu->arch.mmucfg);
626 break;
Mihai Caraman9a6061d2013-04-11 00:03:11 +0000627 case KVM_REG_PPC_EPTCFG:
628 *val = get_reg_val(id, vcpu->arch.eptcfg);
629 break;
Mihai Caramana85d2aa2013-04-11 00:03:08 +0000630 case KVM_REG_PPC_TLB0CFG:
631 case KVM_REG_PPC_TLB1CFG:
632 case KVM_REG_PPC_TLB2CFG:
633 case KVM_REG_PPC_TLB3CFG:
634 i = id - KVM_REG_PPC_TLB0CFG;
635 *val = get_reg_val(id, vcpu->arch.tlbcfg[i]);
636 break;
Mihai Caraman307d9002013-04-11 00:03:10 +0000637 case KVM_REG_PPC_TLB0PS:
638 case KVM_REG_PPC_TLB1PS:
639 case KVM_REG_PPC_TLB2PS:
640 case KVM_REG_PPC_TLB3PS:
641 i = id - KVM_REG_PPC_TLB0PS;
642 *val = get_reg_val(id, vcpu->arch.tlbps[i]);
643 break;
Mihai Caramana85d2aa2013-04-11 00:03:08 +0000644 default:
645 r = -EINVAL;
646 break;
647 }
648
649 return r;
650}
651
652int kvmppc_set_one_reg_e500_tlb(struct kvm_vcpu *vcpu, u64 id,
653 union kvmppc_one_reg *val)
654{
655 int r = 0;
656 long int i;
657
658 switch (id) {
659 case KVM_REG_PPC_MAS0:
660 vcpu->arch.shared->mas0 = set_reg_val(id, *val);
661 break;
662 case KVM_REG_PPC_MAS1:
663 vcpu->arch.shared->mas1 = set_reg_val(id, *val);
664 break;
665 case KVM_REG_PPC_MAS2:
666 vcpu->arch.shared->mas2 = set_reg_val(id, *val);
667 break;
668 case KVM_REG_PPC_MAS7_3:
669 vcpu->arch.shared->mas7_3 = set_reg_val(id, *val);
670 break;
671 case KVM_REG_PPC_MAS4:
672 vcpu->arch.shared->mas4 = set_reg_val(id, *val);
673 break;
674 case KVM_REG_PPC_MAS6:
675 vcpu->arch.shared->mas6 = set_reg_val(id, *val);
676 break;
677 /* Only allow MMU registers to be set to the config supported by KVM */
678 case KVM_REG_PPC_MMUCFG: {
679 u32 reg = set_reg_val(id, *val);
680 if (reg != vcpu->arch.mmucfg)
681 r = -EINVAL;
682 break;
683 }
Mihai Caraman9a6061d2013-04-11 00:03:11 +0000684 case KVM_REG_PPC_EPTCFG: {
685 u32 reg = set_reg_val(id, *val);
686 if (reg != vcpu->arch.eptcfg)
687 r = -EINVAL;
688 break;
689 }
Mihai Caramana85d2aa2013-04-11 00:03:08 +0000690 case KVM_REG_PPC_TLB0CFG:
691 case KVM_REG_PPC_TLB1CFG:
692 case KVM_REG_PPC_TLB2CFG:
693 case KVM_REG_PPC_TLB3CFG: {
694 /* MMU geometry (N_ENTRY/ASSOC) can be set only using SW_TLB */
695 u32 reg = set_reg_val(id, *val);
696 i = id - KVM_REG_PPC_TLB0CFG;
697 if (reg != vcpu->arch.tlbcfg[i])
698 r = -EINVAL;
699 break;
700 }
Mihai Caraman307d9002013-04-11 00:03:10 +0000701 case KVM_REG_PPC_TLB0PS:
702 case KVM_REG_PPC_TLB1PS:
703 case KVM_REG_PPC_TLB2PS:
704 case KVM_REG_PPC_TLB3PS: {
705 u32 reg = set_reg_val(id, *val);
706 i = id - KVM_REG_PPC_TLB0PS;
707 if (reg != vcpu->arch.tlbps[i])
708 r = -EINVAL;
709 break;
710 }
Mihai Caramana85d2aa2013-04-11 00:03:08 +0000711 default:
712 r = -EINVAL;
713 break;
714 }
715
716 return r;
717}
718
Mihai Caraman8893a182013-04-11 00:03:09 +0000719static int vcpu_mmu_geometry_update(struct kvm_vcpu *vcpu,
720 struct kvm_book3e_206_tlb_params *params)
721{
722 vcpu->arch.tlbcfg[0] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
723 if (params->tlb_sizes[0] <= 2048)
724 vcpu->arch.tlbcfg[0] |= params->tlb_sizes[0];
725 vcpu->arch.tlbcfg[0] |= params->tlb_ways[0] << TLBnCFG_ASSOC_SHIFT;
726
727 vcpu->arch.tlbcfg[1] &= ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
728 vcpu->arch.tlbcfg[1] |= params->tlb_sizes[1];
729 vcpu->arch.tlbcfg[1] |= params->tlb_ways[1] << TLBnCFG_ASSOC_SHIFT;
730 return 0;
731}
732
Alexander Grafb71c9e22013-01-11 15:22:45 +0100733int kvm_vcpu_ioctl_config_tlb(struct kvm_vcpu *vcpu,
734 struct kvm_config_tlb *cfg)
735{
736 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
737 struct kvm_book3e_206_tlb_params params;
738 char *virt;
739 struct page **pages;
740 struct tlbe_priv *privs[2] = {};
741 u64 *g2h_bitmap = NULL;
742 size_t array_len;
743 u32 sets;
744 int num_pages, ret, i;
745
746 if (cfg->mmu_type != KVM_MMU_FSL_BOOKE_NOHV)
747 return -EINVAL;
748
749 if (copy_from_user(&params, (void __user *)(uintptr_t)cfg->params,
750 sizeof(params)))
751 return -EFAULT;
752
753 if (params.tlb_sizes[1] > 64)
754 return -EINVAL;
755 if (params.tlb_ways[1] != params.tlb_sizes[1])
756 return -EINVAL;
757 if (params.tlb_sizes[2] != 0 || params.tlb_sizes[3] != 0)
758 return -EINVAL;
759 if (params.tlb_ways[2] != 0 || params.tlb_ways[3] != 0)
760 return -EINVAL;
761
762 if (!is_power_of_2(params.tlb_ways[0]))
763 return -EINVAL;
764
765 sets = params.tlb_sizes[0] >> ilog2(params.tlb_ways[0]);
766 if (!is_power_of_2(sets))
767 return -EINVAL;
768
769 array_len = params.tlb_sizes[0] + params.tlb_sizes[1];
770 array_len *= sizeof(struct kvm_book3e_206_tlb_entry);
771
772 if (cfg->array_len < array_len)
773 return -EINVAL;
774
775 num_pages = DIV_ROUND_UP(cfg->array + array_len - 1, PAGE_SIZE) -
776 cfg->array / PAGE_SIZE;
777 pages = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
778 if (!pages)
779 return -ENOMEM;
780
781 ret = get_user_pages_fast(cfg->array, num_pages, 1, pages);
782 if (ret < 0)
783 goto err_pages;
784
785 if (ret != num_pages) {
786 num_pages = ret;
787 ret = -EFAULT;
788 goto err_put_page;
789 }
790
791 virt = vmap(pages, num_pages, VM_MAP, PAGE_KERNEL);
792 if (!virt) {
793 ret = -ENOMEM;
794 goto err_put_page;
795 }
796
797 privs[0] = kzalloc(sizeof(struct tlbe_priv) * params.tlb_sizes[0],
798 GFP_KERNEL);
799 privs[1] = kzalloc(sizeof(struct tlbe_priv) * params.tlb_sizes[1],
800 GFP_KERNEL);
801
802 if (!privs[0] || !privs[1]) {
803 ret = -ENOMEM;
804 goto err_privs;
805 }
806
807 g2h_bitmap = kzalloc(sizeof(u64) * params.tlb_sizes[1],
808 GFP_KERNEL);
809 if (!g2h_bitmap) {
810 ret = -ENOMEM;
811 goto err_privs;
812 }
813
814 free_gtlb(vcpu_e500);
815
816 vcpu_e500->gtlb_priv[0] = privs[0];
817 vcpu_e500->gtlb_priv[1] = privs[1];
818 vcpu_e500->g2h_tlb1_map = g2h_bitmap;
819
820 vcpu_e500->gtlb_arch = (struct kvm_book3e_206_tlb_entry *)
821 (virt + (cfg->array & (PAGE_SIZE - 1)));
822
823 vcpu_e500->gtlb_params[0].entries = params.tlb_sizes[0];
824 vcpu_e500->gtlb_params[1].entries = params.tlb_sizes[1];
825
826 vcpu_e500->gtlb_offset[0] = 0;
827 vcpu_e500->gtlb_offset[1] = params.tlb_sizes[0];
828
Mihai Caraman8893a182013-04-11 00:03:09 +0000829 /* Update vcpu's MMU geometry based on SW_TLB input */
830 vcpu_mmu_geometry_update(vcpu, &params);
Alexander Grafb71c9e22013-01-11 15:22:45 +0100831
832 vcpu_e500->shared_tlb_pages = pages;
833 vcpu_e500->num_shared_tlb_pages = num_pages;
834
835 vcpu_e500->gtlb_params[0].ways = params.tlb_ways[0];
836 vcpu_e500->gtlb_params[0].sets = sets;
837
838 vcpu_e500->gtlb_params[1].ways = params.tlb_sizes[1];
839 vcpu_e500->gtlb_params[1].sets = 1;
840
841 kvmppc_recalc_tlb1map_range(vcpu_e500);
842 return 0;
843
844err_privs:
845 kfree(privs[0]);
846 kfree(privs[1]);
847
848err_put_page:
849 for (i = 0; i < num_pages; i++)
850 put_page(pages[i]);
851
852err_pages:
853 kfree(pages);
854 return ret;
855}
856
857int kvm_vcpu_ioctl_dirty_tlb(struct kvm_vcpu *vcpu,
858 struct kvm_dirty_tlb *dirty)
859{
860 struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
861 kvmppc_recalc_tlb1map_range(vcpu_e500);
Alexander Graf483ba972013-01-18 15:13:19 +0100862 kvmppc_core_flush_tlb(vcpu);
Alexander Grafb71c9e22013-01-11 15:22:45 +0100863 return 0;
864}
865
Mihai Caraman8893a182013-04-11 00:03:09 +0000866/* Vcpu's MMU default configuration */
867static int vcpu_mmu_init(struct kvm_vcpu *vcpu,
868 struct kvmppc_e500_tlb_params *params)
869{
870 /* Initialize RASIZE, PIDSIZE, NTLBS and MAVN fields with host values*/
871 vcpu->arch.mmucfg = mfspr(SPRN_MMUCFG) & ~MMUCFG_LPIDSIZE;
872
873 /* Initialize TLBnCFG fields with host values and SW_TLB geometry*/
874 vcpu->arch.tlbcfg[0] = mfspr(SPRN_TLB0CFG) &
875 ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
876 vcpu->arch.tlbcfg[0] |= params[0].entries;
877 vcpu->arch.tlbcfg[0] |= params[0].ways << TLBnCFG_ASSOC_SHIFT;
878
879 vcpu->arch.tlbcfg[1] = mfspr(SPRN_TLB1CFG) &
880 ~(TLBnCFG_N_ENTRY | TLBnCFG_ASSOC);
881 vcpu->arch.tlbcfg[1] |= params[1].entries;
882 vcpu->arch.tlbcfg[1] |= params[1].ways << TLBnCFG_ASSOC_SHIFT;
883
Mihai Caraman307d9002013-04-11 00:03:10 +0000884 if (has_feature(vcpu, VCPU_FTR_MMU_V2)) {
885 vcpu->arch.tlbps[0] = mfspr(SPRN_TLB0PS);
886 vcpu->arch.tlbps[1] = mfspr(SPRN_TLB1PS);
Mihai Caraman9a6061d2013-04-11 00:03:11 +0000887
Mihai Caraman5b215012013-04-11 00:03:12 +0000888 vcpu->arch.mmucfg &= ~MMUCFG_LRAT;
889
Mihai Caraman9a6061d2013-04-11 00:03:11 +0000890 /* Guest mmu emulation currently doesn't handle E.PT */
891 vcpu->arch.eptcfg = 0;
Mihai Caraman5b215012013-04-11 00:03:12 +0000892 vcpu->arch.tlbcfg[0] &= ~TLBnCFG_PT;
893 vcpu->arch.tlbcfg[1] &= ~TLBnCFG_IND;
Mihai Caraman307d9002013-04-11 00:03:10 +0000894 }
895
Mihai Caraman8893a182013-04-11 00:03:09 +0000896 return 0;
897}
898
Alexander Grafb71c9e22013-01-11 15:22:45 +0100899int kvmppc_e500_tlb_init(struct kvmppc_vcpu_e500 *vcpu_e500)
900{
901 struct kvm_vcpu *vcpu = &vcpu_e500->vcpu;
902 int entry_size = sizeof(struct kvm_book3e_206_tlb_entry);
903 int entries = KVM_E500_TLB0_SIZE + KVM_E500_TLB1_SIZE;
904
905 if (e500_mmu_host_init(vcpu_e500))
906 goto err;
907
908 vcpu_e500->gtlb_params[0].entries = KVM_E500_TLB0_SIZE;
909 vcpu_e500->gtlb_params[1].entries = KVM_E500_TLB1_SIZE;
910
911 vcpu_e500->gtlb_params[0].ways = KVM_E500_TLB0_WAY_NUM;
912 vcpu_e500->gtlb_params[0].sets =
913 KVM_E500_TLB0_SIZE / KVM_E500_TLB0_WAY_NUM;
914
915 vcpu_e500->gtlb_params[1].ways = KVM_E500_TLB1_SIZE;
916 vcpu_e500->gtlb_params[1].sets = 1;
917
918 vcpu_e500->gtlb_arch = kmalloc(entries * entry_size, GFP_KERNEL);
919 if (!vcpu_e500->gtlb_arch)
920 return -ENOMEM;
921
922 vcpu_e500->gtlb_offset[0] = 0;
923 vcpu_e500->gtlb_offset[1] = KVM_E500_TLB0_SIZE;
924
925 vcpu_e500->gtlb_priv[0] = kzalloc(sizeof(struct tlbe_ref) *
926 vcpu_e500->gtlb_params[0].entries,
927 GFP_KERNEL);
928 if (!vcpu_e500->gtlb_priv[0])
929 goto err;
930
931 vcpu_e500->gtlb_priv[1] = kzalloc(sizeof(struct tlbe_ref) *
932 vcpu_e500->gtlb_params[1].entries,
933 GFP_KERNEL);
934 if (!vcpu_e500->gtlb_priv[1])
935 goto err;
936
937 vcpu_e500->g2h_tlb1_map = kzalloc(sizeof(u64) *
938 vcpu_e500->gtlb_params[1].entries,
939 GFP_KERNEL);
940 if (!vcpu_e500->g2h_tlb1_map)
941 goto err;
942
Mihai Caraman8893a182013-04-11 00:03:09 +0000943 vcpu_mmu_init(vcpu, vcpu_e500->gtlb_params);
Alexander Grafb71c9e22013-01-11 15:22:45 +0100944
945 kvmppc_recalc_tlb1map_range(vcpu_e500);
946 return 0;
947
948err:
949 free_gtlb(vcpu_e500);
950 return -1;
951}
952
953void kvmppc_e500_tlb_uninit(struct kvmppc_vcpu_e500 *vcpu_e500)
954{
955 free_gtlb(vcpu_e500);
956 e500_mmu_host_uninit(vcpu_e500);
957}