blob: e4390dc3eb48e29d9b9a950d5a020d27af32676e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
7 */
8#include <linux/init.h>
9#include <linux/kernel.h>
10#include <linux/sched.h>
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/proc_fs.h>
14
15#include <asm/cacheops.h>
16#include <asm/inst.h>
17#include <asm/io.h>
18#include <asm/page.h>
19#include <asm/pgtable.h>
20#include <asm/prefetch.h>
21#include <asm/system.h>
22#include <asm/bootinfo.h>
23#include <asm/mipsregs.h>
24#include <asm/mmu_context.h>
25#include <asm/cpu.h>
26#include <asm/war.h>
27
Thiemo Seufer330cfe02005-09-01 18:33:58 +000028#define half_scache_line_size() (cpu_scache_line_size() >> 1)
29#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
30#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
31
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
33/*
34 * Maximum sizes:
35 *
36 * R4000 128 bytes S-cache: 0x58 bytes
37 * R4600 v1.7: 0x5c bytes
38 * R4600 v2.0: 0x60 bytes
39 * With prefetching, 16 byte strides 0xa0 bytes
40 */
41
42static unsigned int clear_page_array[0x130 / 4];
43
44void clear_page(void * page) __attribute__((alias("clear_page_array")));
45
46EXPORT_SYMBOL(clear_page);
47
48/*
49 * Maximum sizes:
50 *
51 * R4000 128 bytes S-cache: 0x11c bytes
52 * R4600 v1.7: 0x080 bytes
53 * R4600 v2.0: 0x07c bytes
54 * With prefetching, 16 byte strides 0x0b8 bytes
55 */
56static unsigned int copy_page_array[0x148 / 4];
57
58void copy_page(void *to, void *from) __attribute__((alias("copy_page_array")));
59
60EXPORT_SYMBOL(copy_page);
61
62/*
63 * This is suboptimal for 32-bit kernels; we assume that R10000 is only used
64 * with 64-bit kernels. The prefetch offsets have been experimentally tuned
65 * an Origin 200.
66 */
67static int pref_offset_clear __initdata = 512;
68static int pref_offset_copy __initdata = 256;
69
70static unsigned int pref_src_mode __initdata;
71static unsigned int pref_dst_mode __initdata;
72
73static int load_offset __initdata;
74static int store_offset __initdata;
75
76static unsigned int __initdata *dest, *epc;
77
78static unsigned int instruction_pending;
79static union mips_instruction delayed_mi;
80
81static void __init emit_instruction(union mips_instruction mi)
82{
83 if (instruction_pending)
84 *epc++ = delayed_mi.word;
85
86 instruction_pending = 1;
87 delayed_mi = mi;
88}
89
90static inline void flush_delay_slot_or_nop(void)
91{
92 if (instruction_pending) {
93 *epc++ = delayed_mi.word;
94 instruction_pending = 0;
95 return;
96 }
97
98 *epc++ = 0;
99}
100
101static inline unsigned int *label(void)
102{
103 if (instruction_pending) {
104 *epc++ = delayed_mi.word;
105 instruction_pending = 0;
106 }
107
108 return epc;
109}
110
111static inline void build_insn_word(unsigned int word)
112{
113 union mips_instruction mi;
114
115 mi.word = word;
116
117 emit_instruction(mi);
118}
119
120static inline void build_nop(void)
121{
122 build_insn_word(0); /* nop */
123}
124
125static inline void build_src_pref(int advance)
126{
Atsushi Nemotode862b42006-03-17 12:59:22 +0900127 if (!(load_offset & (cpu_dcache_line_size() - 1)) && advance) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 union mips_instruction mi;
129
130 mi.i_format.opcode = pref_op;
131 mi.i_format.rs = 5; /* $a1 */
132 mi.i_format.rt = pref_src_mode;
133 mi.i_format.simmediate = load_offset + advance;
134
135 emit_instruction(mi);
136 }
137}
138
139static inline void __build_load_reg(int reg)
140{
141 union mips_instruction mi;
142 unsigned int width;
143
144 if (cpu_has_64bit_gp_regs) {
145 mi.i_format.opcode = ld_op;
146 width = 8;
147 } else {
148 mi.i_format.opcode = lw_op;
149 width = 4;
150 }
151 mi.i_format.rs = 5; /* $a1 */
152 mi.i_format.rt = reg; /* $reg */
153 mi.i_format.simmediate = load_offset;
154
155 load_offset += width;
156 emit_instruction(mi);
157}
158
159static inline void build_load_reg(int reg)
160{
161 if (cpu_has_prefetch)
162 build_src_pref(pref_offset_copy);
163
164 __build_load_reg(reg);
165}
166
167static inline void build_dst_pref(int advance)
168{
Atsushi Nemotode862b42006-03-17 12:59:22 +0900169 if (!(store_offset & (cpu_dcache_line_size() - 1)) && advance) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 union mips_instruction mi;
171
172 mi.i_format.opcode = pref_op;
173 mi.i_format.rs = 4; /* $a0 */
174 mi.i_format.rt = pref_dst_mode;
175 mi.i_format.simmediate = store_offset + advance;
176
177 emit_instruction(mi);
178 }
179}
180
181static inline void build_cdex_s(void)
182{
183 union mips_instruction mi;
184
185 if ((store_offset & (cpu_scache_line_size() - 1)))
186 return;
187
188 mi.c_format.opcode = cache_op;
189 mi.c_format.rs = 4; /* $a0 */
190 mi.c_format.c_op = 3; /* Create Dirty Exclusive */
191 mi.c_format.cache = 3; /* Secondary Data Cache */
192 mi.c_format.simmediate = store_offset;
193
194 emit_instruction(mi);
195}
196
197static inline void build_cdex_p(void)
198{
199 union mips_instruction mi;
200
201 if (store_offset & (cpu_dcache_line_size() - 1))
202 return;
203
Thiemo Seufer330cfe02005-09-01 18:33:58 +0000204 if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 build_nop();
206 build_nop();
207 build_nop();
208 build_nop();
209 }
210
Thiemo Seufer330cfe02005-09-01 18:33:58 +0000211 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
Thiemo Seufer10a3dab2005-09-09 20:26:54 +0000212 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213
214 mi.c_format.opcode = cache_op;
215 mi.c_format.rs = 4; /* $a0 */
216 mi.c_format.c_op = 3; /* Create Dirty Exclusive */
217 mi.c_format.cache = 1; /* Data Cache */
218 mi.c_format.simmediate = store_offset;
219
220 emit_instruction(mi);
221}
222
223static void __init __build_store_reg(int reg)
224{
225 union mips_instruction mi;
226 unsigned int width;
227
228 if (cpu_has_64bit_gp_regs ||
229 (cpu_has_64bit_zero_reg && reg == 0)) {
230 mi.i_format.opcode = sd_op;
231 width = 8;
232 } else {
233 mi.i_format.opcode = sw_op;
234 width = 4;
235 }
236 mi.i_format.rs = 4; /* $a0 */
237 mi.i_format.rt = reg; /* $reg */
238 mi.i_format.simmediate = store_offset;
239
240 store_offset += width;
241 emit_instruction(mi);
242}
243
244static inline void build_store_reg(int reg)
245{
246 if (cpu_has_prefetch)
247 if (reg)
248 build_dst_pref(pref_offset_copy);
249 else
250 build_dst_pref(pref_offset_clear);
251 else if (cpu_has_cache_cdex_s)
252 build_cdex_s();
253 else if (cpu_has_cache_cdex_p)
254 build_cdex_p();
255
256 __build_store_reg(reg);
257}
258
259static inline void build_addiu_a2_a0(unsigned long offset)
260{
261 union mips_instruction mi;
262
263 BUG_ON(offset > 0x7fff);
264
265 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
266 mi.i_format.rs = 4; /* $a0 */
267 mi.i_format.rt = 6; /* $a2 */
268 mi.i_format.simmediate = offset;
269
270 emit_instruction(mi);
271}
272
273static inline void build_addiu_a1(unsigned long offset)
274{
275 union mips_instruction mi;
276
277 BUG_ON(offset > 0x7fff);
278
279 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
280 mi.i_format.rs = 5; /* $a1 */
281 mi.i_format.rt = 5; /* $a1 */
282 mi.i_format.simmediate = offset;
283
284 load_offset -= offset;
285
286 emit_instruction(mi);
287}
288
289static inline void build_addiu_a0(unsigned long offset)
290{
291 union mips_instruction mi;
292
293 BUG_ON(offset > 0x7fff);
294
295 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
296 mi.i_format.rs = 4; /* $a0 */
297 mi.i_format.rt = 4; /* $a0 */
298 mi.i_format.simmediate = offset;
299
300 store_offset -= offset;
301
302 emit_instruction(mi);
303}
304
305static inline void build_bne(unsigned int *dest)
306{
307 union mips_instruction mi;
308
309 mi.i_format.opcode = bne_op;
310 mi.i_format.rs = 6; /* $a2 */
311 mi.i_format.rt = 4; /* $a0 */
312 mi.i_format.simmediate = dest - epc - 1;
313
314 *epc++ = mi.word;
315 flush_delay_slot_or_nop();
316}
317
318static inline void build_jr_ra(void)
319{
320 union mips_instruction mi;
321
322 mi.r_format.opcode = spec_op;
323 mi.r_format.rs = 31;
324 mi.r_format.rt = 0;
325 mi.r_format.rd = 0;
326 mi.r_format.re = 0;
327 mi.r_format.func = jr_op;
328
329 *epc++ = mi.word;
330 flush_delay_slot_or_nop();
331}
332
333void __init build_clear_page(void)
334{
335 unsigned int loop_start;
336
337 epc = (unsigned int *) &clear_page_array;
338 instruction_pending = 0;
339 store_offset = 0;
340
341 if (cpu_has_prefetch) {
342 switch (current_cpu_data.cputype) {
Atsushi Nemotode862b42006-03-17 12:59:22 +0900343 case CPU_TX49XX:
344 /* TX49 supports only Pref_Load */
345 pref_offset_clear = 0;
346 pref_offset_copy = 0;
347 break;
348
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 case CPU_RM9000:
350 /*
351 * As a workaround for erratum G105 which make the
352 * PrepareForStore hint unusable we fall back to
353 * StoreRetained on the RM9000. Once it is known which
354 * versions of the RM9000 we'll be able to condition-
355 * alize this.
356 */
357
358 case CPU_R10000:
359 case CPU_R12000:
360 pref_src_mode = Pref_LoadStreamed;
361 pref_dst_mode = Pref_StoreStreamed;
362 break;
363
364 default:
365 pref_src_mode = Pref_LoadStreamed;
366 pref_dst_mode = Pref_PrepareForStore;
367 break;
368 }
369 }
370
371 build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0));
372
Thiemo Seufer330cfe02005-09-01 18:33:58 +0000373 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
375
376dest = label();
377 do {
378 build_store_reg(0);
379 build_store_reg(0);
380 build_store_reg(0);
381 build_store_reg(0);
382 } while (store_offset < half_scache_line_size());
383 build_addiu_a0(2 * store_offset);
384 loop_start = store_offset;
385 do {
386 build_store_reg(0);
387 build_store_reg(0);
388 build_store_reg(0);
389 build_store_reg(0);
390 } while ((store_offset - loop_start) < half_scache_line_size());
391 build_bne(dest);
392
393 if (cpu_has_prefetch && pref_offset_clear) {
394 build_addiu_a2_a0(pref_offset_clear);
395 dest = label();
396 loop_start = store_offset;
397 do {
398 __build_store_reg(0);
399 __build_store_reg(0);
400 __build_store_reg(0);
401 __build_store_reg(0);
402 } while ((store_offset - loop_start) < half_scache_line_size());
403 build_addiu_a0(2 * store_offset);
404 loop_start = store_offset;
405 do {
406 __build_store_reg(0);
407 __build_store_reg(0);
408 __build_store_reg(0);
409 __build_store_reg(0);
410 } while ((store_offset - loop_start) < half_scache_line_size());
411 build_bne(dest);
412 }
413
414 build_jr_ra();
415
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 BUG_ON(epc > clear_page_array + ARRAY_SIZE(clear_page_array));
417}
418
419void __init build_copy_page(void)
420{
421 unsigned int loop_start;
422
423 epc = (unsigned int *) &copy_page_array;
424 store_offset = load_offset = 0;
425 instruction_pending = 0;
426
427 build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0));
428
Thiemo Seufer330cfe02005-09-01 18:33:58 +0000429 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
431
432dest = label();
433 loop_start = store_offset;
434 do {
435 build_load_reg( 8);
436 build_load_reg( 9);
437 build_load_reg(10);
438 build_load_reg(11);
439 build_store_reg( 8);
440 build_store_reg( 9);
441 build_store_reg(10);
442 build_store_reg(11);
443 } while ((store_offset - loop_start) < half_scache_line_size());
444 build_addiu_a0(2 * store_offset);
445 build_addiu_a1(2 * load_offset);
446 loop_start = store_offset;
447 do {
448 build_load_reg( 8);
449 build_load_reg( 9);
450 build_load_reg(10);
451 build_load_reg(11);
452 build_store_reg( 8);
453 build_store_reg( 9);
454 build_store_reg(10);
455 build_store_reg(11);
456 } while ((store_offset - loop_start) < half_scache_line_size());
457 build_bne(dest);
458
459 if (cpu_has_prefetch && pref_offset_copy) {
460 build_addiu_a2_a0(pref_offset_copy);
461 dest = label();
462 loop_start = store_offset;
463 do {
464 __build_load_reg( 8);
465 __build_load_reg( 9);
466 __build_load_reg(10);
467 __build_load_reg(11);
468 __build_store_reg( 8);
469 __build_store_reg( 9);
470 __build_store_reg(10);
471 __build_store_reg(11);
472 } while ((store_offset - loop_start) < half_scache_line_size());
473 build_addiu_a0(2 * store_offset);
474 build_addiu_a1(2 * load_offset);
475 loop_start = store_offset;
476 do {
477 __build_load_reg( 8);
478 __build_load_reg( 9);
479 __build_load_reg(10);
480 __build_load_reg(11);
481 __build_store_reg( 8);
482 __build_store_reg( 9);
483 __build_store_reg(10);
484 __build_store_reg(11);
485 } while ((store_offset - loop_start) < half_scache_line_size());
486 build_bne(dest);
487 }
488
489 build_jr_ra();
490
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 BUG_ON(epc > copy_page_array + ARRAY_SIZE(copy_page_array));
492}