| /* |
| * arch/arm/mach-tegra/dma.c |
| * |
| * System DMA driver for NVIDIA Tegra SoCs |
| * |
| * Copyright (c) 2008-2009, NVIDIA Corporation. |
| * |
| * This program is free software; you can redistribute it and/or modify |
| * it under the terms of the GNU General Public License as published by |
| * the Free Software Foundation; either version 2 of the License, or |
| * (at your option) any later version. |
| * |
| * This program is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| * more details. |
| * |
| * You should have received a copy of the GNU General Public License along |
| * with this program; if not, write to the Free Software Foundation, Inc., |
| * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| */ |
| |
| #include <linux/io.h> |
| #include <linux/interrupt.h> |
| #include <linux/module.h> |
| #include <linux/spinlock.h> |
| #include <linux/err.h> |
| #include <linux/irq.h> |
| #include <linux/delay.h> |
| #include <linux/clk.h> |
| #include <mach/dma.h> |
| #include <mach/irqs.h> |
| #include <mach/iomap.h> |
| #include <mach/suspend.h> |
| |
| #include "apbio.h" |
| |
| #define APB_DMA_GEN 0x000 |
| #define GEN_ENABLE (1<<31) |
| |
| #define APB_DMA_CNTRL 0x010 |
| |
| #define APB_DMA_IRQ_MASK 0x01c |
| |
| #define APB_DMA_IRQ_MASK_SET 0x020 |
| |
| #define APB_DMA_CHAN_CSR 0x000 |
| #define CSR_ENB (1<<31) |
| #define CSR_IE_EOC (1<<30) |
| #define CSR_HOLD (1<<29) |
| #define CSR_DIR (1<<28) |
| #define CSR_ONCE (1<<27) |
| #define CSR_FLOW (1<<21) |
| #define CSR_REQ_SEL_SHIFT 16 |
| #define CSR_WCOUNT_SHIFT 2 |
| #define CSR_WCOUNT_MASK 0xFFFC |
| |
| #define APB_DMA_CHAN_STA 0x004 |
| #define STA_BUSY (1<<31) |
| #define STA_ISE_EOC (1<<30) |
| #define STA_HALT (1<<29) |
| #define STA_PING_PONG (1<<28) |
| #define STA_COUNT_SHIFT 2 |
| #define STA_COUNT_MASK 0xFFFC |
| |
| #define APB_DMA_CHAN_AHB_PTR 0x010 |
| |
| #define APB_DMA_CHAN_AHB_SEQ 0x014 |
| #define AHB_SEQ_INTR_ENB (1<<31) |
| #define AHB_SEQ_BUS_WIDTH_SHIFT 28 |
| #define AHB_SEQ_BUS_WIDTH_MASK (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT) |
| #define AHB_SEQ_BUS_WIDTH_8 (0<<AHB_SEQ_BUS_WIDTH_SHIFT) |
| #define AHB_SEQ_BUS_WIDTH_16 (1<<AHB_SEQ_BUS_WIDTH_SHIFT) |
| #define AHB_SEQ_BUS_WIDTH_32 (2<<AHB_SEQ_BUS_WIDTH_SHIFT) |
| #define AHB_SEQ_BUS_WIDTH_64 (3<<AHB_SEQ_BUS_WIDTH_SHIFT) |
| #define AHB_SEQ_BUS_WIDTH_128 (4<<AHB_SEQ_BUS_WIDTH_SHIFT) |
| #define AHB_SEQ_DATA_SWAP (1<<27) |
| #define AHB_SEQ_BURST_MASK (0x7<<24) |
| #define AHB_SEQ_BURST_1 (4<<24) |
| #define AHB_SEQ_BURST_4 (5<<24) |
| #define AHB_SEQ_BURST_8 (6<<24) |
| #define AHB_SEQ_DBL_BUF (1<<19) |
| #define AHB_SEQ_WRAP_SHIFT 16 |
| #define AHB_SEQ_WRAP_MASK (0x7<<AHB_SEQ_WRAP_SHIFT) |
| |
| #define APB_DMA_CHAN_APB_PTR 0x018 |
| |
| #define APB_DMA_CHAN_APB_SEQ 0x01c |
| #define APB_SEQ_BUS_WIDTH_SHIFT 28 |
| #define APB_SEQ_BUS_WIDTH_MASK (0x7<<APB_SEQ_BUS_WIDTH_SHIFT) |
| #define APB_SEQ_BUS_WIDTH_8 (0<<APB_SEQ_BUS_WIDTH_SHIFT) |
| #define APB_SEQ_BUS_WIDTH_16 (1<<APB_SEQ_BUS_WIDTH_SHIFT) |
| #define APB_SEQ_BUS_WIDTH_32 (2<<APB_SEQ_BUS_WIDTH_SHIFT) |
| #define APB_SEQ_BUS_WIDTH_64 (3<<APB_SEQ_BUS_WIDTH_SHIFT) |
| #define APB_SEQ_BUS_WIDTH_128 (4<<APB_SEQ_BUS_WIDTH_SHIFT) |
| #define APB_SEQ_DATA_SWAP (1<<27) |
| #define APB_SEQ_WRAP_SHIFT 16 |
| #define APB_SEQ_WRAP_MASK (0x7<<APB_SEQ_WRAP_SHIFT) |
| |
| #define TEGRA_SYSTEM_DMA_CH_NR 16 |
| #define TEGRA_SYSTEM_DMA_AVP_CH_NUM 4 |
| #define TEGRA_SYSTEM_DMA_CH_MIN 0 |
| #define TEGRA_SYSTEM_DMA_CH_MAX \ |
| (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1) |
| |
| #define NV_DMA_MAX_TRASFER_SIZE 0x10000 |
| |
| static const unsigned int ahb_addr_wrap_table[8] = { |
| 0, 32, 64, 128, 256, 512, 1024, 2048 |
| }; |
| |
| static const unsigned int apb_addr_wrap_table[8] = { |
| 0, 1, 2, 4, 8, 16, 32, 64 |
| }; |
| |
| static const unsigned int bus_width_table[5] = { |
| 8, 16, 32, 64, 128 |
| }; |
| |
| #define TEGRA_DMA_NAME_SIZE 16 |
| struct tegra_dma_channel { |
| struct list_head list; |
| int id; |
| spinlock_t lock; |
| char name[TEGRA_DMA_NAME_SIZE]; |
| void __iomem *addr; |
| int mode; |
| int irq; |
| int req_transfer_count; |
| }; |
| |
| #define NV_DMA_MAX_CHANNELS 32 |
| |
| static bool tegra_dma_initialized; |
| static DEFINE_MUTEX(tegra_dma_lock); |
| static DEFINE_SPINLOCK(enable_lock); |
| |
| static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS); |
| static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS]; |
| |
| static void tegra_dma_update_hw(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *req); |
| static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *req); |
| static void tegra_dma_stop(struct tegra_dma_channel *ch); |
| |
| void tegra_dma_flush(struct tegra_dma_channel *ch) |
| { |
| } |
| EXPORT_SYMBOL(tegra_dma_flush); |
| |
| void tegra_dma_dequeue(struct tegra_dma_channel *ch) |
| { |
| struct tegra_dma_req *req; |
| |
| if (tegra_dma_is_empty(ch)) |
| return; |
| |
| req = list_entry(ch->list.next, typeof(*req), node); |
| |
| tegra_dma_dequeue_req(ch, req); |
| return; |
| } |
| |
| static void tegra_dma_stop(struct tegra_dma_channel *ch) |
| { |
| u32 csr; |
| u32 status; |
| |
| csr = readl(ch->addr + APB_DMA_CHAN_CSR); |
| csr &= ~CSR_IE_EOC; |
| writel(csr, ch->addr + APB_DMA_CHAN_CSR); |
| |
| csr &= ~CSR_ENB; |
| writel(csr, ch->addr + APB_DMA_CHAN_CSR); |
| |
| status = readl(ch->addr + APB_DMA_CHAN_STA); |
| if (status & STA_ISE_EOC) |
| writel(status, ch->addr + APB_DMA_CHAN_STA); |
| } |
| |
| static int tegra_dma_cancel(struct tegra_dma_channel *ch) |
| { |
| unsigned long irq_flags; |
| |
| spin_lock_irqsave(&ch->lock, irq_flags); |
| while (!list_empty(&ch->list)) |
| list_del(ch->list.next); |
| |
| tegra_dma_stop(ch); |
| |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| return 0; |
| } |
| |
| static unsigned int get_channel_status(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *req, bool is_stop_dma) |
| { |
| void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE); |
| unsigned int status; |
| |
| if (is_stop_dma) { |
| /* |
| * STOP the DMA and get the transfer count. |
| * Getting the transfer count is tricky. |
| * - Globally disable DMA on all channels |
| * - Read the channel's status register to know the number |
| * of pending bytes to be transfered. |
| * - Stop the dma channel |
| * - Globally re-enable DMA to resume other transfers |
| */ |
| spin_lock(&enable_lock); |
| writel(0, addr + APB_DMA_GEN); |
| udelay(20); |
| status = readl(ch->addr + APB_DMA_CHAN_STA); |
| tegra_dma_stop(ch); |
| writel(GEN_ENABLE, addr + APB_DMA_GEN); |
| spin_unlock(&enable_lock); |
| if (status & STA_ISE_EOC) { |
| pr_err("Got Dma Int here clearing"); |
| writel(status, ch->addr + APB_DMA_CHAN_STA); |
| } |
| req->status = TEGRA_DMA_REQ_ERROR_ABORTED; |
| } else { |
| status = readl(ch->addr + APB_DMA_CHAN_STA); |
| } |
| return status; |
| } |
| |
| /* should be called with the channel lock held */ |
| static unsigned int dma_active_count(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *req, unsigned int status) |
| { |
| unsigned int to_transfer; |
| unsigned int req_transfer_count; |
| unsigned int bytes_transferred; |
| |
| to_transfer = ((status & STA_COUNT_MASK) >> STA_COUNT_SHIFT) + 1; |
| req_transfer_count = ch->req_transfer_count + 1; |
| bytes_transferred = req_transfer_count; |
| if (status & STA_BUSY) |
| bytes_transferred -= to_transfer; |
| /* |
| * In continuous transfer mode, DMA only tracks the count of the |
| * half DMA buffer. So, if the DMA already finished half the DMA |
| * then add the half buffer to the completed count. |
| */ |
| if (ch->mode & TEGRA_DMA_MODE_CONTINOUS) { |
| if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) |
| bytes_transferred += req_transfer_count; |
| if (status & STA_ISE_EOC) |
| bytes_transferred += req_transfer_count; |
| } |
| bytes_transferred *= 4; |
| return bytes_transferred; |
| } |
| |
| int tegra_dma_dequeue_req(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *_req) |
| { |
| unsigned int status; |
| struct tegra_dma_req *req = NULL; |
| int found = 0; |
| unsigned long irq_flags; |
| int stop = 0; |
| |
| spin_lock_irqsave(&ch->lock, irq_flags); |
| |
| if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req) |
| stop = 1; |
| |
| list_for_each_entry(req, &ch->list, node) { |
| if (req == _req) { |
| list_del(&req->node); |
| found = 1; |
| break; |
| } |
| } |
| if (!found) { |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| return 0; |
| } |
| |
| if (!stop) |
| goto skip_stop_dma; |
| |
| status = get_channel_status(ch, req, true); |
| req->bytes_transferred = dma_active_count(ch, req, status); |
| |
| if (!list_empty(&ch->list)) { |
| /* if the list is not empty, queue the next request */ |
| struct tegra_dma_req *next_req; |
| next_req = list_entry(ch->list.next, |
| typeof(*next_req), node); |
| tegra_dma_update_hw(ch, next_req); |
| } |
| |
| skip_stop_dma: |
| req->status = -TEGRA_DMA_REQ_ERROR_ABORTED; |
| |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| |
| /* Callback should be called without any lock */ |
| req->complete(req); |
| return 0; |
| } |
| EXPORT_SYMBOL(tegra_dma_dequeue_req); |
| |
| bool tegra_dma_is_empty(struct tegra_dma_channel *ch) |
| { |
| unsigned long irq_flags; |
| bool is_empty; |
| |
| spin_lock_irqsave(&ch->lock, irq_flags); |
| if (list_empty(&ch->list)) |
| is_empty = true; |
| else |
| is_empty = false; |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| return is_empty; |
| } |
| EXPORT_SYMBOL(tegra_dma_is_empty); |
| |
| bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *_req) |
| { |
| unsigned long irq_flags; |
| struct tegra_dma_req *req; |
| |
| spin_lock_irqsave(&ch->lock, irq_flags); |
| list_for_each_entry(req, &ch->list, node) { |
| if (req == _req) { |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| return true; |
| } |
| } |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| return false; |
| } |
| EXPORT_SYMBOL(tegra_dma_is_req_inflight); |
| |
| int tegra_dma_enqueue_req(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *req) |
| { |
| unsigned long irq_flags; |
| struct tegra_dma_req *_req; |
| int start_dma = 0; |
| |
| if (req->size > NV_DMA_MAX_TRASFER_SIZE || |
| req->source_addr & 0x3 || req->dest_addr & 0x3) { |
| pr_err("Invalid DMA request for channel %d\n", ch->id); |
| return -EINVAL; |
| } |
| |
| spin_lock_irqsave(&ch->lock, irq_flags); |
| |
| list_for_each_entry(_req, &ch->list, node) { |
| if (req == _req) { |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| return -EEXIST; |
| } |
| } |
| |
| req->bytes_transferred = 0; |
| req->status = 0; |
| req->buffer_status = 0; |
| if (list_empty(&ch->list)) |
| start_dma = 1; |
| |
| list_add_tail(&req->node, &ch->list); |
| |
| if (start_dma) |
| tegra_dma_update_hw(ch, req); |
| |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| |
| return 0; |
| } |
| EXPORT_SYMBOL(tegra_dma_enqueue_req); |
| |
| struct tegra_dma_channel *tegra_dma_allocate_channel(int mode) |
| { |
| int channel; |
| struct tegra_dma_channel *ch = NULL; |
| |
| if (!tegra_dma_initialized) |
| return NULL; |
| |
| mutex_lock(&tegra_dma_lock); |
| |
| /* first channel is the shared channel */ |
| if (mode & TEGRA_DMA_SHARED) { |
| channel = TEGRA_SYSTEM_DMA_CH_MIN; |
| } else { |
| channel = find_first_zero_bit(channel_usage, |
| ARRAY_SIZE(dma_channels)); |
| if (channel >= ARRAY_SIZE(dma_channels)) |
| goto out; |
| } |
| __set_bit(channel, channel_usage); |
| ch = &dma_channels[channel]; |
| ch->mode = mode; |
| |
| out: |
| mutex_unlock(&tegra_dma_lock); |
| return ch; |
| } |
| EXPORT_SYMBOL(tegra_dma_allocate_channel); |
| |
| void tegra_dma_free_channel(struct tegra_dma_channel *ch) |
| { |
| if (ch->mode & TEGRA_DMA_SHARED) |
| return; |
| tegra_dma_cancel(ch); |
| mutex_lock(&tegra_dma_lock); |
| __clear_bit(ch->id, channel_usage); |
| mutex_unlock(&tegra_dma_lock); |
| } |
| EXPORT_SYMBOL(tegra_dma_free_channel); |
| |
| static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *req) |
| { |
| u32 apb_ptr; |
| u32 ahb_ptr; |
| |
| if (req->to_memory) { |
| apb_ptr = req->source_addr; |
| ahb_ptr = req->dest_addr; |
| } else { |
| apb_ptr = req->dest_addr; |
| ahb_ptr = req->source_addr; |
| } |
| writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR); |
| writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR); |
| |
| req->status = TEGRA_DMA_REQ_INFLIGHT; |
| return; |
| } |
| |
| static void tegra_dma_update_hw(struct tegra_dma_channel *ch, |
| struct tegra_dma_req *req) |
| { |
| int ahb_addr_wrap; |
| int apb_addr_wrap; |
| int ahb_bus_width; |
| int apb_bus_width; |
| int index; |
| |
| u32 ahb_seq; |
| u32 apb_seq; |
| u32 ahb_ptr; |
| u32 apb_ptr; |
| u32 csr; |
| |
| csr = CSR_IE_EOC | CSR_FLOW; |
| ahb_seq = AHB_SEQ_INTR_ENB | AHB_SEQ_BURST_1; |
| apb_seq = 0; |
| |
| csr |= req->req_sel << CSR_REQ_SEL_SHIFT; |
| |
| /* One shot mode is always single buffered, |
| * continuous mode is always double buffered |
| * */ |
| if (ch->mode & TEGRA_DMA_MODE_ONESHOT) { |
| csr |= CSR_ONCE; |
| ch->req_transfer_count = (req->size >> 2) - 1; |
| } else { |
| ahb_seq |= AHB_SEQ_DBL_BUF; |
| |
| /* In double buffered mode, we set the size to half the |
| * requested size and interrupt when half the buffer |
| * is full */ |
| ch->req_transfer_count = (req->size >> 3) - 1; |
| } |
| |
| csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT; |
| |
| if (req->to_memory) { |
| apb_ptr = req->source_addr; |
| ahb_ptr = req->dest_addr; |
| |
| apb_addr_wrap = req->source_wrap; |
| ahb_addr_wrap = req->dest_wrap; |
| apb_bus_width = req->source_bus_width; |
| ahb_bus_width = req->dest_bus_width; |
| |
| } else { |
| csr |= CSR_DIR; |
| apb_ptr = req->dest_addr; |
| ahb_ptr = req->source_addr; |
| |
| apb_addr_wrap = req->dest_wrap; |
| ahb_addr_wrap = req->source_wrap; |
| apb_bus_width = req->dest_bus_width; |
| ahb_bus_width = req->source_bus_width; |
| } |
| |
| apb_addr_wrap >>= 2; |
| ahb_addr_wrap >>= 2; |
| |
| /* set address wrap for APB size */ |
| index = 0; |
| do { |
| if (apb_addr_wrap_table[index] == apb_addr_wrap) |
| break; |
| index++; |
| } while (index < ARRAY_SIZE(apb_addr_wrap_table)); |
| BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table)); |
| apb_seq |= index << APB_SEQ_WRAP_SHIFT; |
| |
| /* set address wrap for AHB size */ |
| index = 0; |
| do { |
| if (ahb_addr_wrap_table[index] == ahb_addr_wrap) |
| break; |
| index++; |
| } while (index < ARRAY_SIZE(ahb_addr_wrap_table)); |
| BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table)); |
| ahb_seq |= index << AHB_SEQ_WRAP_SHIFT; |
| |
| for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) { |
| if (bus_width_table[index] == ahb_bus_width) |
| break; |
| } |
| BUG_ON(index == ARRAY_SIZE(bus_width_table)); |
| ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT; |
| |
| for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) { |
| if (bus_width_table[index] == apb_bus_width) |
| break; |
| } |
| BUG_ON(index == ARRAY_SIZE(bus_width_table)); |
| apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT; |
| |
| writel(csr, ch->addr + APB_DMA_CHAN_CSR); |
| writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ); |
| writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR); |
| writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ); |
| writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR); |
| |
| csr |= CSR_ENB; |
| writel(csr, ch->addr + APB_DMA_CHAN_CSR); |
| |
| req->status = TEGRA_DMA_REQ_INFLIGHT; |
| } |
| |
| static void handle_oneshot_dma(struct tegra_dma_channel *ch) |
| { |
| struct tegra_dma_req *req; |
| unsigned long irq_flags; |
| |
| spin_lock_irqsave(&ch->lock, irq_flags); |
| if (list_empty(&ch->list)) { |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| return; |
| } |
| |
| req = list_entry(ch->list.next, typeof(*req), node); |
| if (req) { |
| int bytes_transferred; |
| |
| bytes_transferred = ch->req_transfer_count; |
| bytes_transferred += 1; |
| bytes_transferred <<= 2; |
| |
| list_del(&req->node); |
| req->bytes_transferred = bytes_transferred; |
| req->status = TEGRA_DMA_REQ_SUCCESS; |
| |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| /* Callback should be called without any lock */ |
| pr_debug("%s: transferred %d bytes\n", __func__, |
| req->bytes_transferred); |
| req->complete(req); |
| spin_lock_irqsave(&ch->lock, irq_flags); |
| } |
| |
| if (!list_empty(&ch->list)) { |
| req = list_entry(ch->list.next, typeof(*req), node); |
| /* the complete function we just called may have enqueued |
| another req, in which case dma has already started */ |
| if (req->status != TEGRA_DMA_REQ_INFLIGHT) |
| tegra_dma_update_hw(ch, req); |
| } |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| } |
| |
| static void handle_continuous_dma(struct tegra_dma_channel *ch) |
| { |
| struct tegra_dma_req *req; |
| unsigned long irq_flags; |
| |
| spin_lock_irqsave(&ch->lock, irq_flags); |
| if (list_empty(&ch->list)) { |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| return; |
| } |
| |
| req = list_entry(ch->list.next, typeof(*req), node); |
| if (req) { |
| if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) { |
| bool is_dma_ping_complete; |
| is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA) |
| & STA_PING_PONG) ? true : false; |
| if (req->to_memory) |
| is_dma_ping_complete = !is_dma_ping_complete; |
| /* Out of sync - Release current buffer */ |
| if (!is_dma_ping_complete) { |
| int bytes_transferred; |
| |
| bytes_transferred = ch->req_transfer_count; |
| bytes_transferred += 1; |
| bytes_transferred <<= 3; |
| req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL; |
| req->bytes_transferred = bytes_transferred; |
| req->status = TEGRA_DMA_REQ_SUCCESS; |
| tegra_dma_stop(ch); |
| |
| if (!list_is_last(&req->node, &ch->list)) { |
| struct tegra_dma_req *next_req; |
| |
| next_req = list_entry(req->node.next, |
| typeof(*next_req), node); |
| tegra_dma_update_hw(ch, next_req); |
| } |
| |
| list_del(&req->node); |
| |
| /* DMA lock is NOT held when callbak is called */ |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| req->complete(req); |
| return; |
| } |
| /* Load the next request into the hardware, if available |
| * */ |
| if (!list_is_last(&req->node, &ch->list)) { |
| struct tegra_dma_req *next_req; |
| |
| next_req = list_entry(req->node.next, |
| typeof(*next_req), node); |
| tegra_dma_update_hw_partial(ch, next_req); |
| } |
| req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL; |
| req->status = TEGRA_DMA_REQ_SUCCESS; |
| /* DMA lock is NOT held when callback is called */ |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| if (likely(req->threshold)) |
| req->threshold(req); |
| return; |
| |
| } else if (req->buffer_status == |
| TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) { |
| /* Callback when the buffer is completely full (i.e on |
| * the second interrupt */ |
| int bytes_transferred; |
| |
| bytes_transferred = ch->req_transfer_count; |
| bytes_transferred += 1; |
| bytes_transferred <<= 3; |
| |
| req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL; |
| req->bytes_transferred = bytes_transferred; |
| req->status = TEGRA_DMA_REQ_SUCCESS; |
| list_del(&req->node); |
| |
| /* DMA lock is NOT held when callbak is called */ |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| req->complete(req); |
| return; |
| |
| } else { |
| BUG(); |
| } |
| } |
| spin_unlock_irqrestore(&ch->lock, irq_flags); |
| } |
| |
| static irqreturn_t dma_isr(int irq, void *data) |
| { |
| struct tegra_dma_channel *ch = data; |
| unsigned long status; |
| |
| status = readl(ch->addr + APB_DMA_CHAN_STA); |
| if (status & STA_ISE_EOC) |
| writel(status, ch->addr + APB_DMA_CHAN_STA); |
| else { |
| pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id); |
| return IRQ_HANDLED; |
| } |
| return IRQ_WAKE_THREAD; |
| } |
| |
| static irqreturn_t dma_thread_fn(int irq, void *data) |
| { |
| struct tegra_dma_channel *ch = data; |
| |
| if (ch->mode & TEGRA_DMA_MODE_ONESHOT) |
| handle_oneshot_dma(ch); |
| else |
| handle_continuous_dma(ch); |
| |
| |
| return IRQ_HANDLED; |
| } |
| |
| int __init tegra_dma_init(void) |
| { |
| int ret = 0; |
| int i; |
| unsigned int irq; |
| void __iomem *addr; |
| struct clk *c; |
| |
| bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS); |
| |
| c = clk_get_sys("tegra-dma", NULL); |
| if (IS_ERR(c)) { |
| pr_err("Unable to get clock for APB DMA\n"); |
| ret = PTR_ERR(c); |
| goto fail; |
| } |
| ret = clk_enable(c); |
| if (ret != 0) { |
| pr_err("Unable to enable clock for APB DMA\n"); |
| goto fail; |
| } |
| |
| addr = IO_ADDRESS(TEGRA_APB_DMA_BASE); |
| writel(GEN_ENABLE, addr + APB_DMA_GEN); |
| writel(0, addr + APB_DMA_CNTRL); |
| writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX), |
| addr + APB_DMA_IRQ_MASK_SET); |
| |
| for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) { |
| struct tegra_dma_channel *ch = &dma_channels[i]; |
| |
| ch->id = i; |
| snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i); |
| |
| ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE + |
| TEGRA_APB_DMA_CH0_SIZE * i); |
| |
| spin_lock_init(&ch->lock); |
| INIT_LIST_HEAD(&ch->list); |
| |
| irq = INT_APB_DMA_CH0 + i; |
| ret = request_threaded_irq(irq, dma_isr, dma_thread_fn, 0, |
| dma_channels[i].name, ch); |
| if (ret) { |
| pr_err("Failed to register IRQ %d for DMA %d\n", |
| irq, i); |
| goto fail; |
| } |
| ch->irq = irq; |
| |
| __clear_bit(i, channel_usage); |
| } |
| /* mark the shared channel allocated */ |
| __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage); |
| |
| tegra_dma_initialized = true; |
| |
| return 0; |
| fail: |
| writel(0, addr + APB_DMA_GEN); |
| for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) { |
| struct tegra_dma_channel *ch = &dma_channels[i]; |
| if (ch->irq) |
| free_irq(ch->irq, ch); |
| } |
| return ret; |
| } |
| postcore_initcall(tegra_dma_init); |
| |
| #ifdef CONFIG_PM |
| static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3]; |
| |
| void tegra_dma_suspend(void) |
| { |
| void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE); |
| u32 *ctx = apb_dma; |
| int i; |
| |
| *ctx++ = readl(addr + APB_DMA_GEN); |
| *ctx++ = readl(addr + APB_DMA_CNTRL); |
| *ctx++ = readl(addr + APB_DMA_IRQ_MASK); |
| |
| for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) { |
| addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE + |
| TEGRA_APB_DMA_CH0_SIZE * i); |
| |
| *ctx++ = readl(addr + APB_DMA_CHAN_CSR); |
| *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR); |
| *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ); |
| *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR); |
| *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ); |
| } |
| } |
| |
| void tegra_dma_resume(void) |
| { |
| void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE); |
| u32 *ctx = apb_dma; |
| int i; |
| |
| writel(*ctx++, addr + APB_DMA_GEN); |
| writel(*ctx++, addr + APB_DMA_CNTRL); |
| writel(*ctx++, addr + APB_DMA_IRQ_MASK); |
| |
| for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) { |
| addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE + |
| TEGRA_APB_DMA_CH0_SIZE * i); |
| |
| writel(*ctx++, addr + APB_DMA_CHAN_CSR); |
| writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR); |
| writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ); |
| writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR); |
| writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ); |
| } |
| } |
| |
| #endif |