/* fuc microcode for nvc0 PGRAPH/HUB
 *
 * Copyright 2011 Red Hat Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Ben Skeggs
 */

/* To build:
 *    m4 nvc0_grhub.fuc | envyas -a -w -m fuc -V nva3 -o nvc0_grhub.fuc.h
 */

.section nvc0_grhub_data
include(`nvc0_graph.fuc')
gpc_count:		.b32 0
rop_count:		.b32 0
cmd_queue:		queue_init
hub_mmio_list_head:	.b32 0
hub_mmio_list_tail:	.b32 0

ctx_current:		.b32 0

chipsets:
.b8  0xc0 0 0 0
.b16 nvc0_hub_mmio_head
.b16 nvc0_hub_mmio_tail
.b8  0xc1 0 0 0
.b16 nvc0_hub_mmio_head
.b16 nvc1_hub_mmio_tail
.b8  0xc3 0 0 0
.b16 nvc0_hub_mmio_head
.b16 nvc0_hub_mmio_tail
.b8  0xc4 0 0 0
.b16 nvc0_hub_mmio_head
.b16 nvc0_hub_mmio_tail
.b8  0xc8 0 0 0
.b16 nvc0_hub_mmio_head
.b16 nvc0_hub_mmio_tail
.b8  0xce 0 0 0
.b16 nvc0_hub_mmio_head
.b16 nvc0_hub_mmio_tail
.b8  0 0 0 0

nvc0_hub_mmio_head:
mmctx_data(0x17e91c, 2)
mmctx_data(0x400204, 2)
mmctx_data(0x404004, 11)
mmctx_data(0x404044, 1)
mmctx_data(0x404094, 14)
mmctx_data(0x4040d0, 7)
mmctx_data(0x4040f8, 1)
mmctx_data(0x404130, 3)
mmctx_data(0x404150, 3)
mmctx_data(0x404164, 2)
mmctx_data(0x404174, 3)
mmctx_data(0x404200, 8)
mmctx_data(0x404404, 14)
mmctx_data(0x404460, 4)
mmctx_data(0x404480, 1)
mmctx_data(0x404498, 1)
mmctx_data(0x404604, 4)
mmctx_data(0x404618, 32)
mmctx_data(0x404698, 21)
mmctx_data(0x4046f0, 2)
mmctx_data(0x404700, 22)
mmctx_data(0x405800, 1)
mmctx_data(0x405830, 3)
mmctx_data(0x405854, 1)
mmctx_data(0x405870, 4)
mmctx_data(0x405a00, 2)
mmctx_data(0x405a18, 1)
mmctx_data(0x406020, 1)
mmctx_data(0x406028, 4)
mmctx_data(0x4064a8, 2)
mmctx_data(0x4064b4, 2)
mmctx_data(0x407804, 1)
mmctx_data(0x40780c, 6)
mmctx_data(0x4078bc, 1)
mmctx_data(0x408000, 7)
mmctx_data(0x408064, 1)
mmctx_data(0x408800, 3)
mmctx_data(0x408900, 4)
mmctx_data(0x408980, 1)
nvc0_hub_mmio_tail:
mmctx_data(0x4064c0, 2)
nvc1_hub_mmio_tail:

.align 256
chan_data:
chan_mmio_count:	.b32 0
chan_mmio_address:	.b32 0

.align 256
xfer_data: 		.b32 0

.section nvc0_grhub_code
bra init
define(`include_code')
include(`nvc0_graph.fuc')

// reports an exception to the host
//
// In: $r15 error code (see nvc0_graph.fuc)
//
error:
	push $r14
	mov $r14 0x814
	shl b32 $r14 6
	iowr I[$r14 + 0x000] $r15	// CC_SCRATCH[5] = error code
	mov $r14 0xc1c
	shl b32 $r14 6
	mov $r15 1
	iowr I[$r14 + 0x000] $r15	// INTR_UP_SET
	pop $r14
	ret

// HUB fuc initialisation, executed by triggering ucode start, will
// fall through to main loop after completion.
//
// Input:
//   CC_SCRATCH[0]: chipset (PMC_BOOT_0 read returns 0x0bad0bad... sigh)
//
// Output:
//   CC_SCRATCH[0]:
//	     31:31: set to signal completion
//   CC_SCRATCH[1]:
//	      31:0: total PGRAPH context size
//
init:
	clear b32 $r0
	mov $sp $r0
	mov $xdbase $r0

	// enable fifo access
	mov $r1 0x1200
	mov $r2 2
	iowr I[$r1 + 0x000] $r2	// FIFO_ENABLE

	// setup i0 handler, and route all interrupts to it
	mov $r1 ih
	mov $iv0 $r1
	mov $r1 0x400
	iowr I[$r1 + 0x300] $r0	// INTR_DISPATCH

	// route HUB_CHANNEL_SWITCH to fuc interrupt 8
	mov $r3 0x404
	shl b32 $r3 6
	mov $r2 0x2003		// { HUB_CHANNEL_SWITCH, ZERO } -> intr 8
	iowr I[$r3 + 0x000] $r2

	// not sure what these are, route them because NVIDIA does, and
	// the IRQ handler will signal the host if we ever get one.. we
	// may find out if/why we need to handle these if so..
	//
	mov $r2 0x2004
	iowr I[$r3 + 0x004] $r2 // { 0x04, ZERO } -> intr 9
	mov $r2 0x200b
	iowr I[$r3 + 0x008] $r2 // { 0x0b, ZERO } -> intr 10
	mov $r2 0x200c
	iowr I[$r3 + 0x01c] $r2 // { 0x0c, ZERO } -> intr 15

	// enable all INTR_UP interrupts
	mov $r2 0xc24
	shl b32 $r2 6
	not b32 $r3 $r0
	iowr I[$r2] $r3

	// enable fifo, ctxsw, 9, 10, 15 interrupts
	mov $r2 -0x78fc		// 0x8704
	sethi $r2 0
	iowr I[$r1 + 0x000] $r2	// INTR_EN_SET

	// fifo level triggered, rest edge
	sub b32 $r1 0x100
	mov $r2 4
	iowr I[$r1] $r2

	// enable interrupts
	bset $flags ie0

	// fetch enabled GPC/ROP counts
	mov $r14 -0x69fc	// 0x409604
	sethi $r14 0x400000
	call nv_rd32
	extr $r1 $r15 16:20
	st b32 D[$r0 + rop_count] $r1
	and $r15 0x1f
	st b32 D[$r0 + gpc_count] $r15

	// set BAR_REQMASK to GPC mask
	mov $r1 1
	shl b32 $r1 $r15
	sub b32 $r1 1
	mov $r2 0x40c
	shl b32 $r2 6
	iowr I[$r2 + 0x000] $r1
	iowr I[$r2 + 0x100] $r1

	// find context data for this chipset
	mov $r2 0x800
	shl b32 $r2 6
	iord $r2 I[$r2 + 0x000]		// CC_SCRATCH[0]
	mov $r15 chipsets - 8
	init_find_chipset:
		add b32 $r15 8
		ld b32 $r3 D[$r15 + 0x00]
		cmpu b32 $r3 $r2
		bra e init_context
		cmpu b32 $r3 0
		bra ne init_find_chipset
		// unknown chipset
		ret

	// context size calculation, reserve first 256 bytes for use by fuc
	init_context:
	mov $r1 256

	// calculate size of mmio context data
	ld b16 $r14 D[$r15 + 4]
	ld b16 $r15 D[$r15 + 6]
	sethi $r14 0
	st b32 D[$r0 + hub_mmio_list_head] $r14
	st b32 D[$r0 + hub_mmio_list_tail] $r15
	call mmctx_size

	// set mmctx base addresses now so we don't have to do it later,
	// they don't (currently) ever change
	mov $r3 0x700
	shl b32 $r3 6
	shr b32 $r4 $r1 8
	iowr I[$r3 + 0x000] $r4		// MMCTX_SAVE_SWBASE
	iowr I[$r3 + 0x100] $r4		// MMCTX_LOAD_SWBASE
	add b32 $r3 0x1300
	add b32 $r1 $r15
	shr b32 $r15 2
	iowr I[$r3 + 0x000] $r15	// MMCTX_LOAD_COUNT, wtf for?!?

	// strands, base offset needs to be aligned to 256 bytes
	shr b32 $r1 8
	add b32 $r1 1
	shl b32 $r1 8
	mov b32 $r15 $r1
	call strand_ctx_init
	add b32 $r1 $r15

	// initialise each GPC in sequence by passing in the offset of its
	// context data in GPCn_CC_SCRATCH[1], and starting its FUC (which
	// has previously been uploaded by the host) running.
	//
	// the GPC fuc init sequence will set GPCn_CC_SCRATCH[0] bit 31
	// when it has completed, and return the size of its context data
	// in GPCn_CC_SCRATCH[1]
	//
	ld b32 $r3 D[$r0 + gpc_count]
	mov $r4 0x2000
	sethi $r4 0x500000
	init_gpc:
		// setup, and start GPC ucode running
		add b32 $r14 $r4 0x804
		mov b32 $r15 $r1
		call nv_wr32			// CC_SCRATCH[1] = ctx offset
		add b32 $r14 $r4 0x800
		mov b32 $r15 $r2
		call nv_wr32			// CC_SCRATCH[0] = chipset
		add b32 $r14 $r4 0x10c
		clear b32 $r15
		call nv_wr32
		add b32 $r14 $r4 0x104
		call nv_wr32			// ENTRY
		add b32 $r14 $r4 0x100
		mov $r15 2			// CTRL_START_TRIGGER
		call nv_wr32			// CTRL

		// wait for it to complete, and adjust context size
		add b32 $r14 $r4 0x800
		init_gpc_wait:
			call nv_rd32
			xbit $r15 $r15 31
			bra e init_gpc_wait
		add b32 $r14 $r4 0x804
		call nv_rd32
		add b32 $r1 $r15

		// next!
		add b32 $r4 0x8000
		sub b32 $r3 1
		bra ne init_gpc

	// save context size, and tell host we're ready
	mov $r2 0x800
	shl b32 $r2 6
	iowr I[$r2 + 0x100] $r1		// CC_SCRATCH[1]  = context size
	add b32 $r2 0x800
	clear b32 $r1
	bset $r1 31
	iowr I[$r2 + 0x000] $r1		// CC_SCRATCH[0] |= 0x80000000

// Main program loop, very simple, sleeps until woken up by the interrupt
// handler, pulls a command from the queue and executes its handler
//
main:
	// sleep until we have something to do
	bset $flags $p0
	sleep $p0
	mov $r13 cmd_queue
	call queue_get
	bra $p1 main

	// context switch, requested by GPU?
	cmpu b32 $r14 0x4001
	bra ne main_not_ctx_switch
		trace_set(T_AUTO)
		mov $r1 0xb00
		shl b32 $r1 6
		iord $r2 I[$r1 + 0x100]		// CHAN_NEXT
		iord $r1 I[$r1 + 0x000]		// CHAN_CUR

		xbit $r3 $r1 31
		bra e chsw_no_prev
			xbit $r3 $r2 31
			bra e chsw_prev_no_next
				push $r2
				mov b32 $r2 $r1
				trace_set(T_SAVE)
				bclr $flags $p1
				bset $flags $p2
				call ctx_xfer
				trace_clr(T_SAVE);
				pop $r2
				trace_set(T_LOAD);
				bset $flags $p1
				call ctx_xfer
				trace_clr(T_LOAD);
				bra chsw_done
			chsw_prev_no_next:
				push $r2
				mov b32 $r2 $r1
				bclr $flags $p1
				bclr $flags $p2
				call ctx_xfer
				pop $r2
				mov $r1 0xb00
				shl b32 $r1 6
				iowr I[$r1] $r2
				bra chsw_done
		chsw_no_prev:
			xbit $r3 $r2 31
			bra e chsw_done
				bset $flags $p1
				bclr $flags $p2
				call ctx_xfer

		// ack the context switch request
		chsw_done:
		mov $r1 0xb0c
		shl b32 $r1 6
		mov $r2 1
		iowr I[$r1 + 0x000] $r2		// 0x409b0c
		trace_clr(T_AUTO)
		bra main

	// request to set current channel? (*not* a context switch)
	main_not_ctx_switch:
	cmpu b32 $r14 0x0001
	bra ne main_not_ctx_chan
		mov b32 $r2 $r15
		call ctx_chan
		bra main_done

	// request to store current channel context?
	main_not_ctx_chan:
	cmpu b32 $r14 0x0002
	bra ne main_not_ctx_save
		trace_set(T_SAVE)
		bclr $flags $p1
		bclr $flags $p2
		call ctx_xfer
		trace_clr(T_SAVE)
		bra main_done

	main_not_ctx_save:
		shl b32 $r15 $r14 16
		or $r15 E_BAD_COMMAND
		call error
		bra main

	main_done:
	mov $r1 0x820
	shl b32 $r1 6
	clear b32 $r2
	bset $r2 31
	iowr I[$r1 + 0x000] $r2		// CC_SCRATCH[0] |= 0x80000000
	bra main

// interrupt handler
ih:
	push $r8
	mov $r8 $flags
	push $r8
	push $r9
	push $r10
	push $r11
	push $r13
	push $r14
	push $r15

	// incoming fifo command?
	iord $r10 I[$r0 + 0x200]	// INTR
	and $r11 $r10 0x00000004
	bra e ih_no_fifo
		// queue incoming fifo command for later processing
		mov $r11 0x1900
		mov $r13 cmd_queue
		iord $r14 I[$r11 + 0x100]	// FIFO_CMD
		iord $r15 I[$r11 + 0x000]	// FIFO_DATA
		call queue_put
		add b32 $r11 0x400
		mov $r14 1
		iowr I[$r11 + 0x000] $r14	// FIFO_ACK

	// context switch request?
	ih_no_fifo:
	and $r11 $r10 0x00000100
	bra e ih_no_ctxsw
		// enqueue a context switch for later processing
		mov $r13 cmd_queue
		mov $r14 0x4001
		call queue_put

	// anything we didn't handle, bring it to the host's attention
	ih_no_ctxsw:
	mov $r11 0x104
	not b32 $r11
	and $r11 $r10 $r11
	bra e ih_no_other
		mov $r10 0xc1c
		shl b32 $r10 6
		iowr I[$r10] $r11	// INTR_UP_SET

	// ack, and wake up main()
	ih_no_other:
	iowr I[$r0 + 0x100] $r10	// INTR_ACK

	pop $r15
	pop $r14
	pop $r13
	pop $r11
	pop $r10
	pop $r9
	pop $r8
	mov $flags $r8
	pop $r8
	bclr $flags $p0
	iret

// Not real sure, but, MEM_CMD 7 will hang forever if this isn't done
ctx_4160s:
	mov $r14 0x4160
	sethi $r14 0x400000
	mov $r15 1
	call nv_wr32
	ctx_4160s_wait:
		call nv_rd32
		xbit $r15 $r15 4
		bra e ctx_4160s_wait
	ret

// Without clearing again at end of xfer, some things cause PGRAPH
// to hang with STATUS=0x00000007 until it's cleared.. fbcon can
// still function with it set however...
ctx_4160c:
	mov $r14 0x4160
	sethi $r14 0x400000
	clear b32 $r15
	call nv_wr32
	ret

// Again, not real sure
//
// In: $r15 value to set 0x404170 to
//
ctx_4170s:
	mov $r14 0x4170
	sethi $r14 0x400000
	or $r15 0x10
	call nv_wr32
	ret

// Waits for a ctx_4170s() call to complete
//
ctx_4170w:
	mov $r14 0x4170
	sethi $r14 0x400000
	call nv_rd32
	and $r15 0x10
	bra ne ctx_4170w
	ret

// Disables various things, waits a bit, and re-enables them..
//
// Not sure how exactly this helps, perhaps "ENABLE" is not such a
// good description for the bits we turn off?  Anyways, without this,
// funny things happen.
//
ctx_redswitch:
	mov $r14 0x614
	shl b32 $r14 6
	mov $r15 0x270
	iowr I[$r14] $r15	// HUB_RED_SWITCH = ENABLE_GPC, POWER_ALL
	mov $r15 8
	ctx_redswitch_delay:
		sub b32 $r15 1
		bra ne ctx_redswitch_delay
	mov $r15 0x770
	iowr I[$r14] $r15	// HUB_RED_SWITCH = ENABLE_ALL, POWER_ALL
	ret

// Not a clue what this is for, except that unless the value is 0x10, the
// strand context is saved (and presumably restored) incorrectly..
//
// In: $r15 value to set to (0x00/0x10 are used)
//
ctx_86c:
	mov $r14 0x86c
	shl b32 $r14 6
	iowr I[$r14] $r15	// HUB(0x86c) = val
	mov $r14 -0x75ec
	sethi $r14 0x400000
	call nv_wr32		// ROP(0xa14) = val
	mov $r14 -0x5794
	sethi $r14 0x410000
	call nv_wr32		// GPC(0x86c) = val
	ret

// ctx_load - load's a channel's ctxctl data, and selects its vm
//
// In: $r2 channel address
//
ctx_load:
	trace_set(T_CHAN)

	// switch to channel, somewhat magic in parts..
	mov $r10 12		// DONE_UNK12
	call wait_donez
	mov $r1 0xa24
	shl b32 $r1 6
	iowr I[$r1 + 0x000] $r0	// 0x409a24
	mov $r3 0xb00
	shl b32 $r3 6
	iowr I[$r3 + 0x100] $r2	// CHAN_NEXT
	mov $r1 0xa0c
	shl b32 $r1 6
	mov $r4 7
	iowr I[$r1 + 0x000] $r2 // MEM_CHAN
	iowr I[$r1 + 0x100] $r4	// MEM_CMD
	ctx_chan_wait_0:
		iord $r4 I[$r1 + 0x100]
		and $r4 0x1f
		bra ne ctx_chan_wait_0
	iowr I[$r3 + 0x000] $r2	// CHAN_CUR

	// load channel header, fetch PGRAPH context pointer
	mov $xtargets $r0
	bclr $r2 31
	shl b32 $r2 4
	add b32 $r2 2

	trace_set(T_LCHAN)
	mov $r1 0xa04
	shl b32 $r1 6
	iowr I[$r1 + 0x000] $r2		// MEM_BASE
	mov $r1 0xa20
	shl b32 $r1 6
	mov $r2 0x0002
	sethi $r2 0x80000000
	iowr I[$r1 + 0x000] $r2		// MEM_TARGET = vram
	mov $r1 0x10			// chan + 0x0210
	mov $r2 xfer_data
	sethi $r2 0x00020000		// 16 bytes
	xdld $r1 $r2
	xdwait
	trace_clr(T_LCHAN)

	// update current context
	ld b32 $r1 D[$r0 + xfer_data + 4]
	shl b32 $r1 24
	ld b32 $r2 D[$r0 + xfer_data + 0]
	shr b32 $r2 8
	or $r1 $r2
	st b32 D[$r0 + ctx_current] $r1

	// set transfer base to start of context, and fetch context header
	trace_set(T_LCTXH)
	mov $r2 0xa04
	shl b32 $r2 6
	iowr I[$r2 + 0x000] $r1		// MEM_BASE
	mov $r2 1
	mov $r1 0xa20
	shl b32 $r1 6
	iowr I[$r1 + 0x000] $r2		// MEM_TARGET = vm
	mov $r1 chan_data
	sethi $r1 0x00060000		// 256 bytes
	xdld $r0 $r1
	xdwait
	trace_clr(T_LCTXH)

	trace_clr(T_CHAN)
	ret

// ctx_chan - handler for HUB_SET_CHAN command, will set a channel as
//            the active channel for ctxctl, but not actually transfer
//            any context data.  intended for use only during initial
//            context construction.
//
// In: $r2 channel address
//
ctx_chan:
	call ctx_4160s
	call ctx_load
	mov $r10 12			// DONE_UNK12
	call wait_donez
	mov $r1 0xa10
	shl b32 $r1 6
	mov $r2 5
	iowr I[$r1 + 0x000] $r2		// MEM_CMD = 5 (???)
	ctx_chan_wait:
		iord $r2 I[$r1 + 0x000]
		or $r2 $r2
		bra ne ctx_chan_wait
	call ctx_4160c
	ret

// Execute per-context state overrides list
//
// Only executed on the first load of a channel.  Might want to look into
// removing this and having the host directly modify the channel's context
// to change this state...  The nouveau DRM already builds this list as
// it's definitely needed for NVIDIA's, so we may as well use it for now
//
// Input: $r1 mmio list length
//
ctx_mmio_exec:
	// set transfer base to be the mmio list
	ld b32 $r3 D[$r0 + chan_mmio_address]
	mov $r2 0xa04
	shl b32 $r2 6
	iowr I[$r2 + 0x000] $r3		// MEM_BASE

	clear b32 $r3
	ctx_mmio_loop:
		// fetch next 256 bytes of mmio list if necessary
		and $r4 $r3 0xff
		bra ne ctx_mmio_pull
			mov $r5 xfer_data
			sethi $r5 0x00060000	// 256 bytes
			xdld $r3 $r5
			xdwait

		// execute a single list entry
		ctx_mmio_pull:
		ld b32 $r14 D[$r4 + xfer_data + 0x00]
		ld b32 $r15 D[$r4 + xfer_data + 0x04]
		call nv_wr32

		// next!
		add b32 $r3 8
		sub b32 $r1 1
		bra ne ctx_mmio_loop

	// set transfer base back to the current context
	ctx_mmio_done:
	ld b32 $r3 D[$r0 + ctx_current]
	iowr I[$r2 + 0x000] $r3		// MEM_BASE

	// disable the mmio list now, we don't need/want to execute it again
	st b32 D[$r0 + chan_mmio_count] $r0
	mov $r1 chan_data
	sethi $r1 0x00060000		// 256 bytes
	xdst $r0 $r1
	xdwait
	ret

// Transfer HUB context data between GPU and storage area
//
// In: $r2 channel address
//     $p1 clear on save, set on load
//     $p2 set if opposite direction done/will be done, so:
//		on save it means: "a load will follow this save"
//		on load it means: "a save preceeded this load"
//
ctx_xfer:
	bra not $p1 ctx_xfer_pre
	bra $p2 ctx_xfer_pre_load
	ctx_xfer_pre:
		mov $r15 0x10
		call ctx_86c
		call ctx_4160s
		bra not $p1 ctx_xfer_exec

	ctx_xfer_pre_load:
		mov $r15 2
		call ctx_4170s
		call ctx_4170w
		call ctx_redswitch
		clear b32 $r15
		call ctx_4170s
		call ctx_load

	// fetch context pointer, and initiate xfer on all GPCs
	ctx_xfer_exec:
	ld b32 $r1 D[$r0 + ctx_current]
	mov $r2 0x414
	shl b32 $r2 6
	iowr I[$r2 + 0x000] $r0	// BAR_STATUS = reset
	mov $r14 -0x5b00
	sethi $r14 0x410000
	mov b32 $r15 $r1
	call nv_wr32		// GPC_BCAST_WRCMD_DATA = ctx pointer
	add b32 $r14 4
	xbit $r15 $flags $p1
	xbit $r2 $flags $p2
	shl b32 $r2 1
	or $r15 $r2
	call nv_wr32		// GPC_BCAST_WRCMD_CMD = GPC_XFER(type)

	// strands
	mov $r1 0x4afc
	sethi $r1 0x20000
	mov $r2 0xc
	iowr I[$r1] $r2		// STRAND_CMD(0x3f) = 0x0c
	call strand_wait
	mov $r2 0x47fc
	sethi $r2 0x20000
	iowr I[$r2] $r0		// STRAND_FIRST_GENE(0x3f) = 0x00
	xbit $r2 $flags $p1
	add b32 $r2 3
	iowr I[$r1] $r2		// STRAND_CMD(0x3f) = 0x03/0x04 (SAVE/LOAD)

	// mmio context
	xbit $r10 $flags $p1	// direction
	or $r10 6		// first, last
	mov $r11 0		// base = 0
	ld b32 $r12 D[$r0 + hub_mmio_list_head]
	ld b32 $r13 D[$r0 + hub_mmio_list_tail]
	mov $r14 0		// not multi
	call mmctx_xfer

	// wait for GPCs to all complete
	mov $r10 8		// DONE_BAR
	call wait_doneo

	// wait for strand xfer to complete
	call strand_wait

	// post-op
	bra $p1 ctx_xfer_post
		mov $r10 12		// DONE_UNK12
		call wait_donez
		mov $r1 0xa10
		shl b32 $r1 6
		mov $r2 5
		iowr I[$r1] $r2		// MEM_CMD
		ctx_xfer_post_save_wait:
			iord $r2 I[$r1]
			or $r2 $r2
			bra ne ctx_xfer_post_save_wait

	bra $p2 ctx_xfer_done
	ctx_xfer_post:
		mov $r15 2
		call ctx_4170s
		clear b32 $r15
		call ctx_86c
		call strand_post
		call ctx_4170w
		clear b32 $r15
		call ctx_4170s

		bra not $p1 ctx_xfer_no_post_mmio
		ld b32 $r1 D[$r0 + chan_mmio_count]
		or $r1 $r1
		bra e ctx_xfer_no_post_mmio
			call ctx_mmio_exec

		ctx_xfer_no_post_mmio:
		call ctx_4160c

	ctx_xfer_done:
	ret

.align 256