| /** |
| * Copyright (C) 2005 - 2010 ServerEngines |
| * All rights reserved. |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License version 2 |
| * as published by the Free Software Foundation. The full GNU General |
| * Public License is included in this distribution in the file called COPYING. |
| * |
| * Written by: Jayamohan Kallickal (jayamohank@serverengines.com) |
| * |
| * Contact Information: |
| * linux-drivers@serverengines.com |
| * |
| * ServerEngines |
| * 209 N. Fair Oaks Ave |
| * Sunnyvale, CA 94085 |
| * |
| */ |
| #include <linux/reboot.h> |
| #include <linux/delay.h> |
| #include <linux/slab.h> |
| #include <linux/interrupt.h> |
| #include <linux/blkdev.h> |
| #include <linux/pci.h> |
| #include <linux/string.h> |
| #include <linux/kernel.h> |
| #include <linux/semaphore.h> |
| #include <linux/iscsi_boot_sysfs.h> |
| |
| #include <scsi/libiscsi.h> |
| #include <scsi/scsi_transport_iscsi.h> |
| #include <scsi/scsi_transport.h> |
| #include <scsi/scsi_cmnd.h> |
| #include <scsi/scsi_device.h> |
| #include <scsi/scsi_host.h> |
| #include <scsi/scsi.h> |
| #include "be_main.h" |
| #include "be_iscsi.h" |
| #include "be_mgmt.h" |
| |
| static unsigned int be_iopoll_budget = 10; |
| static unsigned int be_max_phys_size = 64; |
| static unsigned int enable_msix = 1; |
| static unsigned int gcrashmode = 0; |
| static unsigned int num_hba = 0; |
| |
| MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); |
| MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR); |
| MODULE_AUTHOR("ServerEngines Corporation"); |
| MODULE_LICENSE("GPL"); |
| module_param(be_iopoll_budget, int, 0); |
| module_param(enable_msix, int, 0); |
| module_param(be_max_phys_size, uint, S_IRUGO); |
| MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically" |
| "contiguous memory that can be allocated." |
| "Range is 16 - 128"); |
| |
| static int beiscsi_slave_configure(struct scsi_device *sdev) |
| { |
| blk_queue_max_segment_size(sdev->request_queue, 65536); |
| return 0; |
| } |
| |
| static int beiscsi_eh_abort(struct scsi_cmnd *sc) |
| { |
| struct iscsi_cls_session *cls_session; |
| struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr; |
| struct beiscsi_io_task *aborted_io_task; |
| struct iscsi_conn *conn; |
| struct beiscsi_conn *beiscsi_conn; |
| struct beiscsi_hba *phba; |
| struct iscsi_session *session; |
| struct invalidate_command_table *inv_tbl; |
| struct be_dma_mem nonemb_cmd; |
| unsigned int cid, tag, num_invalidate; |
| |
| cls_session = starget_to_session(scsi_target(sc->device)); |
| session = cls_session->dd_data; |
| |
| spin_lock_bh(&session->lock); |
| if (!aborted_task || !aborted_task->sc) { |
| /* we raced */ |
| spin_unlock_bh(&session->lock); |
| return SUCCESS; |
| } |
| |
| aborted_io_task = aborted_task->dd_data; |
| if (!aborted_io_task->scsi_cmnd) { |
| /* raced or invalid command */ |
| spin_unlock_bh(&session->lock); |
| return SUCCESS; |
| } |
| spin_unlock_bh(&session->lock); |
| conn = aborted_task->conn; |
| beiscsi_conn = conn->dd_data; |
| phba = beiscsi_conn->phba; |
| |
| /* invalidate iocb */ |
| cid = beiscsi_conn->beiscsi_conn_cid; |
| inv_tbl = phba->inv_tbl; |
| memset(inv_tbl, 0x0, sizeof(*inv_tbl)); |
| inv_tbl->cid = cid; |
| inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index; |
| num_invalidate = 1; |
| nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, |
| sizeof(struct invalidate_commands_params_in), |
| &nonemb_cmd.dma); |
| if (nonemb_cmd.va == NULL) { |
| SE_DEBUG(DBG_LVL_1, |
| "Failed to allocate memory for" |
| "mgmt_invalidate_icds\n"); |
| return FAILED; |
| } |
| nonemb_cmd.size = sizeof(struct invalidate_commands_params_in); |
| |
| tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, |
| cid, &nonemb_cmd); |
| if (!tag) { |
| shost_printk(KERN_WARNING, phba->shost, |
| "mgmt_invalidate_icds could not be" |
| " submitted\n"); |
| pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, |
| nonemb_cmd.va, nonemb_cmd.dma); |
| |
| return FAILED; |
| } else { |
| wait_event_interruptible(phba->ctrl.mcc_wait[tag], |
| phba->ctrl.mcc_numtag[tag]); |
| free_mcc_tag(&phba->ctrl, tag); |
| } |
| pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, |
| nonemb_cmd.va, nonemb_cmd.dma); |
| return iscsi_eh_abort(sc); |
| } |
| |
| static int beiscsi_eh_device_reset(struct scsi_cmnd *sc) |
| { |
| struct iscsi_task *abrt_task; |
| struct beiscsi_io_task *abrt_io_task; |
| struct iscsi_conn *conn; |
| struct beiscsi_conn *beiscsi_conn; |
| struct beiscsi_hba *phba; |
| struct iscsi_session *session; |
| struct iscsi_cls_session *cls_session; |
| struct invalidate_command_table *inv_tbl; |
| struct be_dma_mem nonemb_cmd; |
| unsigned int cid, tag, i, num_invalidate; |
| int rc = FAILED; |
| |
| /* invalidate iocbs */ |
| cls_session = starget_to_session(scsi_target(sc->device)); |
| session = cls_session->dd_data; |
| spin_lock_bh(&session->lock); |
| if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) |
| goto unlock; |
| |
| conn = session->leadconn; |
| beiscsi_conn = conn->dd_data; |
| phba = beiscsi_conn->phba; |
| cid = beiscsi_conn->beiscsi_conn_cid; |
| inv_tbl = phba->inv_tbl; |
| memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN); |
| num_invalidate = 0; |
| for (i = 0; i < conn->session->cmds_max; i++) { |
| abrt_task = conn->session->cmds[i]; |
| abrt_io_task = abrt_task->dd_data; |
| if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE) |
| continue; |
| |
| if (abrt_task->sc->device->lun != abrt_task->sc->device->lun) |
| continue; |
| |
| inv_tbl->cid = cid; |
| inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index; |
| num_invalidate++; |
| inv_tbl++; |
| } |
| spin_unlock_bh(&session->lock); |
| inv_tbl = phba->inv_tbl; |
| |
| nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, |
| sizeof(struct invalidate_commands_params_in), |
| &nonemb_cmd.dma); |
| if (nonemb_cmd.va == NULL) { |
| SE_DEBUG(DBG_LVL_1, |
| "Failed to allocate memory for" |
| "mgmt_invalidate_icds\n"); |
| return FAILED; |
| } |
| nonemb_cmd.size = sizeof(struct invalidate_commands_params_in); |
| memset(nonemb_cmd.va, 0, nonemb_cmd.size); |
| tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, |
| cid, &nonemb_cmd); |
| if (!tag) { |
| shost_printk(KERN_WARNING, phba->shost, |
| "mgmt_invalidate_icds could not be" |
| " submitted\n"); |
| pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, |
| nonemb_cmd.va, nonemb_cmd.dma); |
| return FAILED; |
| } else { |
| wait_event_interruptible(phba->ctrl.mcc_wait[tag], |
| phba->ctrl.mcc_numtag[tag]); |
| free_mcc_tag(&phba->ctrl, tag); |
| } |
| pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, |
| nonemb_cmd.va, nonemb_cmd.dma); |
| return iscsi_eh_device_reset(sc); |
| unlock: |
| spin_unlock_bh(&session->lock); |
| return rc; |
| } |
| |
| static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf) |
| { |
| struct beiscsi_hba *phba = data; |
| char *str = buf; |
| int rc; |
| |
| switch (type) { |
| case ISCSI_BOOT_TGT_NAME: |
| rc = sprintf(buf, "%.*s\n", |
| (int)strlen(phba->boot_sess.target_name), |
| (char *)&phba->boot_sess.target_name); |
| break; |
| case ISCSI_BOOT_TGT_IP_ADDR: |
| if (phba->boot_sess.conn_list[0].dest_ipaddr.ip_type == 0x1) |
| rc = sprintf(buf, "%pI4\n", |
| (char *)&phba->boot_sess.conn_list[0]. |
| dest_ipaddr.ip_address); |
| else |
| rc = sprintf(str, "%pI6\n", |
| (char *)&phba->boot_sess.conn_list[0]. |
| dest_ipaddr.ip_address); |
| break; |
| case ISCSI_BOOT_TGT_PORT: |
| rc = sprintf(str, "%d\n", phba->boot_sess.conn_list[0]. |
| dest_port); |
| break; |
| |
| case ISCSI_BOOT_TGT_CHAP_NAME: |
| rc = sprintf(str, "%.*s\n", |
| phba->boot_sess.conn_list[0]. |
| negotiated_login_options.auth_data.chap. |
| target_chap_name_length, |
| (char *)&phba->boot_sess.conn_list[0]. |
| negotiated_login_options.auth_data.chap. |
| target_chap_name); |
| break; |
| case ISCSI_BOOT_TGT_CHAP_SECRET: |
| rc = sprintf(str, "%.*s\n", |
| phba->boot_sess.conn_list[0]. |
| negotiated_login_options.auth_data.chap. |
| target_secret_length, |
| (char *)&phba->boot_sess.conn_list[0]. |
| negotiated_login_options.auth_data.chap. |
| target_secret); |
| |
| break; |
| case ISCSI_BOOT_TGT_REV_CHAP_NAME: |
| rc = sprintf(str, "%.*s\n", |
| phba->boot_sess.conn_list[0]. |
| negotiated_login_options.auth_data.chap. |
| intr_chap_name_length, |
| (char *)&phba->boot_sess.conn_list[0]. |
| negotiated_login_options.auth_data.chap. |
| intr_chap_name); |
| |
| break; |
| case ISCSI_BOOT_TGT_REV_CHAP_SECRET: |
| rc = sprintf(str, "%.*s\n", |
| phba->boot_sess.conn_list[0]. |
| negotiated_login_options.auth_data.chap. |
| intr_secret_length, |
| (char *)&phba->boot_sess.conn_list[0]. |
| negotiated_login_options.auth_data.chap. |
| intr_secret); |
| break; |
| case ISCSI_BOOT_TGT_FLAGS: |
| rc = sprintf(str, "2\n"); |
| break; |
| case ISCSI_BOOT_TGT_NIC_ASSOC: |
| rc = sprintf(str, "0\n"); |
| break; |
| default: |
| rc = -ENOSYS; |
| break; |
| } |
| return rc; |
| } |
| |
| static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf) |
| { |
| struct beiscsi_hba *phba = data; |
| char *str = buf; |
| int rc; |
| |
| switch (type) { |
| case ISCSI_BOOT_INI_INITIATOR_NAME: |
| rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname); |
| break; |
| default: |
| rc = -ENOSYS; |
| break; |
| } |
| return rc; |
| } |
| |
| static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf) |
| { |
| struct beiscsi_hba *phba = data; |
| char *str = buf; |
| int rc; |
| |
| switch (type) { |
| case ISCSI_BOOT_ETH_FLAGS: |
| rc = sprintf(str, "2\n"); |
| break; |
| case ISCSI_BOOT_ETH_INDEX: |
| rc = sprintf(str, "0\n"); |
| break; |
| case ISCSI_BOOT_ETH_MAC: |
| rc = beiscsi_get_macaddr(buf, phba); |
| if (rc < 0) { |
| SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n"); |
| return rc; |
| } |
| break; |
| default: |
| rc = -ENOSYS; |
| break; |
| } |
| return rc; |
| } |
| |
| |
| static mode_t beiscsi_tgt_get_attr_visibility(void *data, int type) |
| { |
| int rc; |
| |
| switch (type) { |
| case ISCSI_BOOT_TGT_NAME: |
| case ISCSI_BOOT_TGT_IP_ADDR: |
| case ISCSI_BOOT_TGT_PORT: |
| case ISCSI_BOOT_TGT_CHAP_NAME: |
| case ISCSI_BOOT_TGT_CHAP_SECRET: |
| case ISCSI_BOOT_TGT_REV_CHAP_NAME: |
| case ISCSI_BOOT_TGT_REV_CHAP_SECRET: |
| case ISCSI_BOOT_TGT_NIC_ASSOC: |
| case ISCSI_BOOT_TGT_FLAGS: |
| rc = S_IRUGO; |
| break; |
| default: |
| rc = 0; |
| break; |
| } |
| return rc; |
| } |
| |
| static mode_t beiscsi_ini_get_attr_visibility(void *data, int type) |
| { |
| int rc; |
| |
| switch (type) { |
| case ISCSI_BOOT_INI_INITIATOR_NAME: |
| rc = S_IRUGO; |
| break; |
| default: |
| rc = 0; |
| break; |
| } |
| return rc; |
| } |
| |
| |
| static mode_t beiscsi_eth_get_attr_visibility(void *data, int type) |
| { |
| int rc; |
| |
| switch (type) { |
| case ISCSI_BOOT_ETH_FLAGS: |
| case ISCSI_BOOT_ETH_MAC: |
| case ISCSI_BOOT_ETH_INDEX: |
| rc = S_IRUGO; |
| break; |
| default: |
| rc = 0; |
| break; |
| } |
| return rc; |
| } |
| |
| static int beiscsi_setup_boot_info(struct beiscsi_hba *phba) |
| { |
| struct iscsi_boot_kobj *boot_kobj; |
| |
| phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no); |
| if (!phba->boot_kset) |
| return -ENOMEM; |
| |
| /* get boot info using mgmt cmd */ |
| boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba, |
| beiscsi_show_boot_tgt_info, |
| beiscsi_tgt_get_attr_visibility); |
| if (!boot_kobj) |
| goto free_kset; |
| |
| boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba, |
| beiscsi_show_boot_ini_info, |
| beiscsi_ini_get_attr_visibility); |
| if (!boot_kobj) |
| goto free_kset; |
| |
| boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba, |
| beiscsi_show_boot_eth_info, |
| beiscsi_eth_get_attr_visibility); |
| if (!boot_kobj) |
| goto free_kset; |
| return 0; |
| |
| free_kset: |
| iscsi_boot_destroy_kset(phba->boot_kset); |
| return -ENOMEM; |
| } |
| |
| /*------------------- PCI Driver operations and data ----------------- */ |
| static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = { |
| { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, |
| { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, |
| { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, |
| { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, |
| { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) }, |
| { 0 } |
| }; |
| MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); |
| |
| static struct scsi_host_template beiscsi_sht = { |
| .module = THIS_MODULE, |
| .name = "ServerEngines 10Gbe open-iscsi Initiator Driver", |
| .proc_name = DRV_NAME, |
| .queuecommand = iscsi_queuecommand, |
| .change_queue_depth = iscsi_change_queue_depth, |
| .slave_configure = beiscsi_slave_configure, |
| .target_alloc = iscsi_target_alloc, |
| .eh_abort_handler = beiscsi_eh_abort, |
| .eh_device_reset_handler = beiscsi_eh_device_reset, |
| .eh_target_reset_handler = iscsi_eh_session_reset, |
| .sg_tablesize = BEISCSI_SGLIST_ELEMENTS, |
| .can_queue = BE2_IO_DEPTH, |
| .this_id = -1, |
| .max_sectors = BEISCSI_MAX_SECTORS, |
| .cmd_per_lun = BEISCSI_CMD_PER_LUN, |
| .use_clustering = ENABLE_CLUSTERING, |
| }; |
| |
| static struct scsi_transport_template *beiscsi_scsi_transport; |
| |
| static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev) |
| { |
| struct beiscsi_hba *phba; |
| struct Scsi_Host *shost; |
| |
| shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0); |
| if (!shost) { |
| dev_err(&pcidev->dev, "beiscsi_hba_alloc -" |
| "iscsi_host_alloc failed\n"); |
| return NULL; |
| } |
| shost->dma_boundary = pcidev->dma_mask; |
| shost->max_id = BE2_MAX_SESSIONS; |
| shost->max_channel = 0; |
| shost->max_cmd_len = BEISCSI_MAX_CMD_LEN; |
| shost->max_lun = BEISCSI_NUM_MAX_LUN; |
| shost->transportt = beiscsi_scsi_transport; |
| phba = iscsi_host_priv(shost); |
| memset(phba, 0, sizeof(*phba)); |
| phba->shost = shost; |
| phba->pcidev = pci_dev_get(pcidev); |
| pci_set_drvdata(pcidev, phba); |
| |
| if (iscsi_host_add(shost, &phba->pcidev->dev)) |
| goto free_devices; |
| |
| if (beiscsi_setup_boot_info(phba)) |
| /* |
| * log error but continue, because we may not be using |
| * iscsi boot. |
| */ |
| shost_printk(KERN_ERR, phba->shost, "Could not set up " |
| "iSCSI boot info."); |
| |
| return phba; |
| |
| free_devices: |
| pci_dev_put(phba->pcidev); |
| iscsi_host_free(phba->shost); |
| return NULL; |
| } |
| |
| static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba) |
| { |
| if (phba->csr_va) { |
| iounmap(phba->csr_va); |
| phba->csr_va = NULL; |
| } |
| if (phba->db_va) { |
| iounmap(phba->db_va); |
| phba->db_va = NULL; |
| } |
| if (phba->pci_va) { |
| iounmap(phba->pci_va); |
| phba->pci_va = NULL; |
| } |
| } |
| |
| static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, |
| struct pci_dev *pcidev) |
| { |
| u8 __iomem *addr; |
| int pcicfg_reg; |
| |
| addr = ioremap_nocache(pci_resource_start(pcidev, 2), |
| pci_resource_len(pcidev, 2)); |
| if (addr == NULL) |
| return -ENOMEM; |
| phba->ctrl.csr = addr; |
| phba->csr_va = addr; |
| phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2); |
| |
| addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024); |
| if (addr == NULL) |
| goto pci_map_err; |
| phba->ctrl.db = addr; |
| phba->db_va = addr; |
| phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4); |
| |
| if (phba->generation == BE_GEN2) |
| pcicfg_reg = 1; |
| else |
| pcicfg_reg = 0; |
| |
| addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg), |
| pci_resource_len(pcidev, pcicfg_reg)); |
| |
| if (addr == NULL) |
| goto pci_map_err; |
| phba->ctrl.pcicfg = addr; |
| phba->pci_va = addr; |
| phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg); |
| return 0; |
| |
| pci_map_err: |
| beiscsi_unmap_pci_function(phba); |
| return -ENOMEM; |
| } |
| |
| static int beiscsi_enable_pci(struct pci_dev *pcidev) |
| { |
| int ret; |
| |
| ret = pci_enable_device(pcidev); |
| if (ret) { |
| dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device " |
| "failed. Returning -ENODEV\n"); |
| return ret; |
| } |
| |
| pci_set_master(pcidev); |
| if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) { |
| ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32)); |
| if (ret) { |
| dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n"); |
| pci_disable_device(pcidev); |
| return ret; |
| } |
| } |
| return 0; |
| } |
| |
| static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev) |
| { |
| struct be_ctrl_info *ctrl = &phba->ctrl; |
| struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced; |
| struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem; |
| int status = 0; |
| |
| ctrl->pdev = pdev; |
| status = beiscsi_map_pci_bars(phba, pdev); |
| if (status) |
| return status; |
| mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; |
| mbox_mem_alloc->va = pci_alloc_consistent(pdev, |
| mbox_mem_alloc->size, |
| &mbox_mem_alloc->dma); |
| if (!mbox_mem_alloc->va) { |
| beiscsi_unmap_pci_function(phba); |
| status = -ENOMEM; |
| return status; |
| } |
| |
| mbox_mem_align->size = sizeof(struct be_mcc_mailbox); |
| mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); |
| mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); |
| memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); |
| spin_lock_init(&ctrl->mbox_lock); |
| spin_lock_init(&phba->ctrl.mcc_lock); |
| spin_lock_init(&phba->ctrl.mcc_cq_lock); |
| |
| return status; |
| } |
| |
| static void beiscsi_get_params(struct beiscsi_hba *phba) |
| { |
| phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count |
| - (phba->fw_config.iscsi_cid_count |
| + BE2_TMFS |
| + BE2_NOPOUT_REQ)); |
| phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count; |
| phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2; |
| phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;; |
| phba->params.num_sge_per_io = BE2_SGE; |
| phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ; |
| phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ; |
| phba->params.eq_timer = 64; |
| phba->params.num_eq_entries = |
| (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2 |
| + BE2_TMFS) / 512) + 1) * 512; |
| phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024) |
| ? 1024 : phba->params.num_eq_entries; |
| SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n", |
| phba->params.num_eq_entries); |
| phba->params.num_cq_entries = |
| (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2 |
| + BE2_TMFS) / 512) + 1) * 512; |
| phba->params.wrbs_per_cxn = 256; |
| } |
| |
| static void hwi_ring_eq_db(struct beiscsi_hba *phba, |
| unsigned int id, unsigned int clr_interrupt, |
| unsigned int num_processed, |
| unsigned char rearm, unsigned char event) |
| { |
| u32 val = 0; |
| val |= id & DB_EQ_RING_ID_MASK; |
| if (rearm) |
| val |= 1 << DB_EQ_REARM_SHIFT; |
| if (clr_interrupt) |
| val |= 1 << DB_EQ_CLR_SHIFT; |
| if (event) |
| val |= 1 << DB_EQ_EVNT_SHIFT; |
| val |= num_processed << DB_EQ_NUM_POPPED_SHIFT; |
| iowrite32(val, phba->db_va + DB_EQ_OFFSET); |
| } |
| |
| /** |
| * be_isr_mcc - The isr routine of the driver. |
| * @irq: Not used |
| * @dev_id: Pointer to host adapter structure |
| */ |
| static irqreturn_t be_isr_mcc(int irq, void *dev_id) |
| { |
| struct beiscsi_hba *phba; |
| struct be_eq_entry *eqe = NULL; |
| struct be_queue_info *eq; |
| struct be_queue_info *mcc; |
| unsigned int num_eq_processed; |
| struct be_eq_obj *pbe_eq; |
| unsigned long flags; |
| |
| pbe_eq = dev_id; |
| eq = &pbe_eq->q; |
| phba = pbe_eq->phba; |
| mcc = &phba->ctrl.mcc_obj.cq; |
| eqe = queue_tail_node(eq); |
| if (!eqe) |
| SE_DEBUG(DBG_LVL_1, "eqe is NULL\n"); |
| |
| num_eq_processed = 0; |
| |
| while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] |
| & EQE_VALID_MASK) { |
| if (((eqe->dw[offsetof(struct amap_eq_entry, |
| resource_id) / 32] & |
| EQE_RESID_MASK) >> 16) == mcc->id) { |
| spin_lock_irqsave(&phba->isr_lock, flags); |
| phba->todo_mcc_cq = 1; |
| spin_unlock_irqrestore(&phba->isr_lock, flags); |
| } |
| AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); |
| queue_tail_inc(eq); |
| eqe = queue_tail_node(eq); |
| num_eq_processed++; |
| } |
| if (phba->todo_mcc_cq) |
| queue_work(phba->wq, &phba->work_cqs); |
| if (num_eq_processed) |
| hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1); |
| |
| return IRQ_HANDLED; |
| } |
| |
| /** |
| * be_isr_msix - The isr routine of the driver. |
| * @irq: Not used |
| * @dev_id: Pointer to host adapter structure |
| */ |
| static irqreturn_t be_isr_msix(int irq, void *dev_id) |
| { |
| struct beiscsi_hba *phba; |
| struct be_eq_entry *eqe = NULL; |
| struct be_queue_info *eq; |
| struct be_queue_info *cq; |
| unsigned int num_eq_processed; |
| struct be_eq_obj *pbe_eq; |
| unsigned long flags; |
| |
| pbe_eq = dev_id; |
| eq = &pbe_eq->q; |
| cq = pbe_eq->cq; |
| eqe = queue_tail_node(eq); |
| if (!eqe) |
| SE_DEBUG(DBG_LVL_1, "eqe is NULL\n"); |
| |
| phba = pbe_eq->phba; |
| num_eq_processed = 0; |
| if (blk_iopoll_enabled) { |
| while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] |
| & EQE_VALID_MASK) { |
| if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) |
| blk_iopoll_sched(&pbe_eq->iopoll); |
| |
| AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); |
| queue_tail_inc(eq); |
| eqe = queue_tail_node(eq); |
| num_eq_processed++; |
| } |
| if (num_eq_processed) |
| hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1); |
| |
| return IRQ_HANDLED; |
| } else { |
| while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] |
| & EQE_VALID_MASK) { |
| spin_lock_irqsave(&phba->isr_lock, flags); |
| phba->todo_cq = 1; |
| spin_unlock_irqrestore(&phba->isr_lock, flags); |
| AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); |
| queue_tail_inc(eq); |
| eqe = queue_tail_node(eq); |
| num_eq_processed++; |
| } |
| if (phba->todo_cq) |
| queue_work(phba->wq, &phba->work_cqs); |
| |
| if (num_eq_processed) |
| hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1); |
| |
| return IRQ_HANDLED; |
| } |
| } |
| |
| /** |
| * be_isr - The isr routine of the driver. |
| * @irq: Not used |
| * @dev_id: Pointer to host adapter structure |
| */ |
| static irqreturn_t be_isr(int irq, void *dev_id) |
| { |
| struct beiscsi_hba *phba; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| struct be_eq_entry *eqe = NULL; |
| struct be_queue_info *eq; |
| struct be_queue_info *cq; |
| struct be_queue_info *mcc; |
| unsigned long flags, index; |
| unsigned int num_mcceq_processed, num_ioeq_processed; |
| struct be_ctrl_info *ctrl; |
| struct be_eq_obj *pbe_eq; |
| int isr; |
| |
| phba = dev_id; |
| ctrl = &phba->ctrl;; |
| isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET + |
| (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE)); |
| if (!isr) |
| return IRQ_NONE; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| pbe_eq = &phwi_context->be_eq[0]; |
| |
| eq = &phwi_context->be_eq[0].q; |
| mcc = &phba->ctrl.mcc_obj.cq; |
| index = 0; |
| eqe = queue_tail_node(eq); |
| if (!eqe) |
| SE_DEBUG(DBG_LVL_1, "eqe is NULL\n"); |
| |
| num_ioeq_processed = 0; |
| num_mcceq_processed = 0; |
| if (blk_iopoll_enabled) { |
| while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] |
| & EQE_VALID_MASK) { |
| if (((eqe->dw[offsetof(struct amap_eq_entry, |
| resource_id) / 32] & |
| EQE_RESID_MASK) >> 16) == mcc->id) { |
| spin_lock_irqsave(&phba->isr_lock, flags); |
| phba->todo_mcc_cq = 1; |
| spin_unlock_irqrestore(&phba->isr_lock, flags); |
| num_mcceq_processed++; |
| } else { |
| if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) |
| blk_iopoll_sched(&pbe_eq->iopoll); |
| num_ioeq_processed++; |
| } |
| AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); |
| queue_tail_inc(eq); |
| eqe = queue_tail_node(eq); |
| } |
| if (num_ioeq_processed || num_mcceq_processed) { |
| if (phba->todo_mcc_cq) |
| queue_work(phba->wq, &phba->work_cqs); |
| |
| if ((num_mcceq_processed) && (!num_ioeq_processed)) |
| hwi_ring_eq_db(phba, eq->id, 0, |
| (num_ioeq_processed + |
| num_mcceq_processed) , 1, 1); |
| else |
| hwi_ring_eq_db(phba, eq->id, 0, |
| (num_ioeq_processed + |
| num_mcceq_processed), 0, 1); |
| |
| return IRQ_HANDLED; |
| } else |
| return IRQ_NONE; |
| } else { |
| cq = &phwi_context->be_cq[0]; |
| while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] |
| & EQE_VALID_MASK) { |
| |
| if (((eqe->dw[offsetof(struct amap_eq_entry, |
| resource_id) / 32] & |
| EQE_RESID_MASK) >> 16) != cq->id) { |
| spin_lock_irqsave(&phba->isr_lock, flags); |
| phba->todo_mcc_cq = 1; |
| spin_unlock_irqrestore(&phba->isr_lock, flags); |
| } else { |
| spin_lock_irqsave(&phba->isr_lock, flags); |
| phba->todo_cq = 1; |
| spin_unlock_irqrestore(&phba->isr_lock, flags); |
| } |
| AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); |
| queue_tail_inc(eq); |
| eqe = queue_tail_node(eq); |
| num_ioeq_processed++; |
| } |
| if (phba->todo_cq || phba->todo_mcc_cq) |
| queue_work(phba->wq, &phba->work_cqs); |
| |
| if (num_ioeq_processed) { |
| hwi_ring_eq_db(phba, eq->id, 0, |
| num_ioeq_processed, 1, 1); |
| return IRQ_HANDLED; |
| } else |
| return IRQ_NONE; |
| } |
| } |
| |
| static int beiscsi_init_irqs(struct beiscsi_hba *phba) |
| { |
| struct pci_dev *pcidev = phba->pcidev; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| int ret, msix_vec, i, j; |
| char desc[32]; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| |
| if (phba->msix_enabled) { |
| for (i = 0; i < phba->num_cpus; i++) { |
| sprintf(desc, "beiscsi_msix_%04x", i); |
| msix_vec = phba->msix_entries[i].vector; |
| ret = request_irq(msix_vec, be_isr_msix, 0, desc, |
| &phwi_context->be_eq[i]); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "beiscsi_init_irqs-Failed to" |
| "register msix for i = %d\n", i); |
| if (!i) |
| return ret; |
| goto free_msix_irqs; |
| } |
| } |
| msix_vec = phba->msix_entries[i].vector; |
| ret = request_irq(msix_vec, be_isr_mcc, 0, "beiscsi_msix_mcc", |
| &phwi_context->be_eq[i]); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-" |
| "Failed to register beiscsi_msix_mcc\n"); |
| i++; |
| goto free_msix_irqs; |
| } |
| |
| } else { |
| ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED, |
| "beiscsi", phba); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-" |
| "Failed to register irq\\n"); |
| return ret; |
| } |
| } |
| return 0; |
| free_msix_irqs: |
| for (j = i - 1; j == 0; j++) |
| free_irq(msix_vec, &phwi_context->be_eq[j]); |
| return ret; |
| } |
| |
| static void hwi_ring_cq_db(struct beiscsi_hba *phba, |
| unsigned int id, unsigned int num_processed, |
| unsigned char rearm, unsigned char event) |
| { |
| u32 val = 0; |
| val |= id & DB_CQ_RING_ID_MASK; |
| if (rearm) |
| val |= 1 << DB_CQ_REARM_SHIFT; |
| val |= num_processed << DB_CQ_NUM_POPPED_SHIFT; |
| iowrite32(val, phba->db_va + DB_CQ_OFFSET); |
| } |
| |
| static unsigned int |
| beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn, |
| struct beiscsi_hba *phba, |
| unsigned short cid, |
| struct pdu_base *ppdu, |
| unsigned long pdu_len, |
| void *pbuffer, unsigned long buf_len) |
| { |
| struct iscsi_conn *conn = beiscsi_conn->conn; |
| struct iscsi_session *session = conn->session; |
| struct iscsi_task *task; |
| struct beiscsi_io_task *io_task; |
| struct iscsi_hdr *login_hdr; |
| |
| switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] & |
| PDUBASE_OPCODE_MASK) { |
| case ISCSI_OP_NOOP_IN: |
| pbuffer = NULL; |
| buf_len = 0; |
| break; |
| case ISCSI_OP_ASYNC_EVENT: |
| break; |
| case ISCSI_OP_REJECT: |
| WARN_ON(!pbuffer); |
| WARN_ON(!(buf_len == 48)); |
| SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n"); |
| break; |
| case ISCSI_OP_LOGIN_RSP: |
| case ISCSI_OP_TEXT_RSP: |
| task = conn->login_task; |
| io_task = task->dd_data; |
| login_hdr = (struct iscsi_hdr *)ppdu; |
| login_hdr->itt = io_task->libiscsi_itt; |
| break; |
| default: |
| shost_printk(KERN_WARNING, phba->shost, |
| "Unrecognized opcode 0x%x in async msg\n", |
| (ppdu-> |
| dw[offsetof(struct amap_pdu_base, opcode) / 32] |
| & PDUBASE_OPCODE_MASK)); |
| return 1; |
| } |
| |
| spin_lock_bh(&session->lock); |
| __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len); |
| spin_unlock_bh(&session->lock); |
| return 0; |
| } |
| |
| static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba) |
| { |
| struct sgl_handle *psgl_handle; |
| |
| if (phba->io_sgl_hndl_avbl) { |
| SE_DEBUG(DBG_LVL_8, |
| "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n", |
| phba->io_sgl_alloc_index); |
| psgl_handle = phba->io_sgl_hndl_base[phba-> |
| io_sgl_alloc_index]; |
| phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL; |
| phba->io_sgl_hndl_avbl--; |
| if (phba->io_sgl_alloc_index == (phba->params. |
| ios_per_ctrl - 1)) |
| phba->io_sgl_alloc_index = 0; |
| else |
| phba->io_sgl_alloc_index++; |
| } else |
| psgl_handle = NULL; |
| return psgl_handle; |
| } |
| |
| static void |
| free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) |
| { |
| SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n", |
| phba->io_sgl_free_index); |
| if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) { |
| /* |
| * this can happen if clean_task is called on a task that |
| * failed in xmit_task or alloc_pdu. |
| */ |
| SE_DEBUG(DBG_LVL_8, |
| "Double Free in IO SGL io_sgl_free_index=%d," |
| "value there=%p\n", phba->io_sgl_free_index, |
| phba->io_sgl_hndl_base[phba->io_sgl_free_index]); |
| return; |
| } |
| phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle; |
| phba->io_sgl_hndl_avbl++; |
| if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1)) |
| phba->io_sgl_free_index = 0; |
| else |
| phba->io_sgl_free_index++; |
| } |
| |
| /** |
| * alloc_wrb_handle - To allocate a wrb handle |
| * @phba: The hba pointer |
| * @cid: The cid to use for allocation |
| * |
| * This happens under session_lock until submission to chip |
| */ |
| struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid) |
| { |
| struct hwi_wrb_context *pwrb_context; |
| struct hwi_controller *phwi_ctrlr; |
| struct wrb_handle *pwrb_handle, *pwrb_handle_tmp; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pwrb_context = &phwi_ctrlr->wrb_context[cid]; |
| if (pwrb_context->wrb_handles_available >= 2) { |
| pwrb_handle = pwrb_context->pwrb_handle_base[ |
| pwrb_context->alloc_index]; |
| pwrb_context->wrb_handles_available--; |
| if (pwrb_context->alloc_index == |
| (phba->params.wrbs_per_cxn - 1)) |
| pwrb_context->alloc_index = 0; |
| else |
| pwrb_context->alloc_index++; |
| pwrb_handle_tmp = pwrb_context->pwrb_handle_base[ |
| pwrb_context->alloc_index]; |
| pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index; |
| } else |
| pwrb_handle = NULL; |
| return pwrb_handle; |
| } |
| |
| /** |
| * free_wrb_handle - To free the wrb handle back to pool |
| * @phba: The hba pointer |
| * @pwrb_context: The context to free from |
| * @pwrb_handle: The wrb_handle to free |
| * |
| * This happens under session_lock until submission to chip |
| */ |
| static void |
| free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context, |
| struct wrb_handle *pwrb_handle) |
| { |
| pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle; |
| pwrb_context->wrb_handles_available++; |
| if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1)) |
| pwrb_context->free_index = 0; |
| else |
| pwrb_context->free_index++; |
| |
| SE_DEBUG(DBG_LVL_8, |
| "FREE WRB: pwrb_handle=%p free_index=0x%x" |
| "wrb_handles_available=%d\n", |
| pwrb_handle, pwrb_context->free_index, |
| pwrb_context->wrb_handles_available); |
| } |
| |
| static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba) |
| { |
| struct sgl_handle *psgl_handle; |
| |
| if (phba->eh_sgl_hndl_avbl) { |
| psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index]; |
| phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL; |
| SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n", |
| phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index); |
| phba->eh_sgl_hndl_avbl--; |
| if (phba->eh_sgl_alloc_index == |
| (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - |
| 1)) |
| phba->eh_sgl_alloc_index = 0; |
| else |
| phba->eh_sgl_alloc_index++; |
| } else |
| psgl_handle = NULL; |
| return psgl_handle; |
| } |
| |
| void |
| free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) |
| { |
| |
| SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n", |
| phba->eh_sgl_free_index); |
| if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) { |
| /* |
| * this can happen if clean_task is called on a task that |
| * failed in xmit_task or alloc_pdu. |
| */ |
| SE_DEBUG(DBG_LVL_8, |
| "Double Free in eh SGL ,eh_sgl_free_index=%d\n", |
| phba->eh_sgl_free_index); |
| return; |
| } |
| phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle; |
| phba->eh_sgl_hndl_avbl++; |
| if (phba->eh_sgl_free_index == |
| (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1)) |
| phba->eh_sgl_free_index = 0; |
| else |
| phba->eh_sgl_free_index++; |
| } |
| |
| static void |
| be_complete_io(struct beiscsi_conn *beiscsi_conn, |
| struct iscsi_task *task, struct sol_cqe *psol) |
| { |
| struct beiscsi_io_task *io_task = task->dd_data; |
| struct be_status_bhs *sts_bhs = |
| (struct be_status_bhs *)io_task->cmd_bhs; |
| struct iscsi_conn *conn = beiscsi_conn->conn; |
| unsigned int sense_len; |
| unsigned char *sense; |
| u32 resid = 0, exp_cmdsn, max_cmdsn; |
| u8 rsp, status, flags; |
| |
| exp_cmdsn = (psol-> |
| dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] |
| & SOL_EXP_CMD_SN_MASK); |
| max_cmdsn = ((psol-> |
| dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] |
| & SOL_EXP_CMD_SN_MASK) + |
| ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) |
| / 32] & SOL_CMD_WND_MASK) >> 24) - 1); |
| rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32] |
| & SOL_RESP_MASK) >> 16); |
| status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32] |
| & SOL_STS_MASK) >> 8); |
| flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] |
| & SOL_FLAGS_MASK) >> 24) | 0x80; |
| |
| task->sc->result = (DID_OK << 16) | status; |
| if (rsp != ISCSI_STATUS_CMD_COMPLETED) { |
| task->sc->result = DID_ERROR << 16; |
| goto unmap; |
| } |
| |
| /* bidi not initially supported */ |
| if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) { |
| resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / |
| 32] & SOL_RES_CNT_MASK); |
| |
| if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW)) |
| task->sc->result = DID_ERROR << 16; |
| |
| if (flags & ISCSI_FLAG_CMD_UNDERFLOW) { |
| scsi_set_resid(task->sc, resid); |
| if (!status && (scsi_bufflen(task->sc) - resid < |
| task->sc->underflow)) |
| task->sc->result = DID_ERROR << 16; |
| } |
| } |
| |
| if (status == SAM_STAT_CHECK_CONDITION) { |
| unsigned short *slen = (unsigned short *)sts_bhs->sense_info; |
| sense = sts_bhs->sense_info + sizeof(unsigned short); |
| sense_len = cpu_to_be16(*slen); |
| memcpy(task->sc->sense_buffer, sense, |
| min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE)); |
| } |
| |
| if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) { |
| if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32] |
| & SOL_RES_CNT_MASK) |
| conn->rxdata_octets += (psol-> |
| dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32] |
| & SOL_RES_CNT_MASK); |
| } |
| unmap: |
| scsi_dma_unmap(io_task->scsi_cmnd); |
| iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn); |
| } |
| |
| static void |
| be_complete_logout(struct beiscsi_conn *beiscsi_conn, |
| struct iscsi_task *task, struct sol_cqe *psol) |
| { |
| struct iscsi_logout_rsp *hdr; |
| struct beiscsi_io_task *io_task = task->dd_data; |
| struct iscsi_conn *conn = beiscsi_conn->conn; |
| |
| hdr = (struct iscsi_logout_rsp *)task->hdr; |
| hdr->opcode = ISCSI_OP_LOGOUT_RSP; |
| hdr->t2wait = 5; |
| hdr->t2retain = 0; |
| hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] |
| & SOL_FLAGS_MASK) >> 24) | 0x80; |
| hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) / |
| 32] & SOL_RESP_MASK); |
| hdr->exp_cmdsn = cpu_to_be32(psol-> |
| dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] |
| & SOL_EXP_CMD_SN_MASK); |
| hdr->max_cmdsn = be32_to_cpu((psol-> |
| dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32] |
| & SOL_EXP_CMD_SN_MASK) + |
| ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) |
| / 32] & SOL_CMD_WND_MASK) >> 24) - 1); |
| hdr->dlength[0] = 0; |
| hdr->dlength[1] = 0; |
| hdr->dlength[2] = 0; |
| hdr->hlength = 0; |
| hdr->itt = io_task->libiscsi_itt; |
| __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); |
| } |
| |
| static void |
| be_complete_tmf(struct beiscsi_conn *beiscsi_conn, |
| struct iscsi_task *task, struct sol_cqe *psol) |
| { |
| struct iscsi_tm_rsp *hdr; |
| struct iscsi_conn *conn = beiscsi_conn->conn; |
| struct beiscsi_io_task *io_task = task->dd_data; |
| |
| hdr = (struct iscsi_tm_rsp *)task->hdr; |
| hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; |
| hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] |
| & SOL_FLAGS_MASK) >> 24) | 0x80; |
| hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) / |
| 32] & SOL_RESP_MASK); |
| hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe, |
| i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK); |
| hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe, |
| i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) + |
| ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) |
| / 32] & SOL_CMD_WND_MASK) >> 24) - 1); |
| hdr->itt = io_task->libiscsi_itt; |
| __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); |
| } |
| |
| static void |
| hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn, |
| struct beiscsi_hba *phba, struct sol_cqe *psol) |
| { |
| struct hwi_wrb_context *pwrb_context; |
| struct wrb_handle *pwrb_handle = NULL; |
| struct hwi_controller *phwi_ctrlr; |
| struct iscsi_task *task; |
| struct beiscsi_io_task *io_task; |
| struct iscsi_conn *conn = beiscsi_conn->conn; |
| struct iscsi_session *session = conn->session; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pwrb_context = &phwi_ctrlr->wrb_context[((psol-> |
| dw[offsetof(struct amap_sol_cqe, cid) / 32] & |
| SOL_CID_MASK) >> 6) - |
| phba->fw_config.iscsi_cid_start]; |
| pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol-> |
| dw[offsetof(struct amap_sol_cqe, wrb_index) / |
| 32] & SOL_WRB_INDEX_MASK) >> 16)]; |
| task = pwrb_handle->pio_handle; |
| |
| io_task = task->dd_data; |
| spin_lock(&phba->mgmt_sgl_lock); |
| free_mgmt_sgl_handle(phba, io_task->psgl_handle); |
| spin_unlock(&phba->mgmt_sgl_lock); |
| spin_lock_bh(&session->lock); |
| free_wrb_handle(phba, pwrb_context, pwrb_handle); |
| spin_unlock_bh(&session->lock); |
| } |
| |
| static void |
| be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn, |
| struct iscsi_task *task, struct sol_cqe *psol) |
| { |
| struct iscsi_nopin *hdr; |
| struct iscsi_conn *conn = beiscsi_conn->conn; |
| struct beiscsi_io_task *io_task = task->dd_data; |
| |
| hdr = (struct iscsi_nopin *)task->hdr; |
| hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32] |
| & SOL_FLAGS_MASK) >> 24) | 0x80; |
| hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe, |
| i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK); |
| hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe, |
| i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) + |
| ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd) |
| / 32] & SOL_CMD_WND_MASK) >> 24) - 1); |
| hdr->opcode = ISCSI_OP_NOOP_IN; |
| hdr->itt = io_task->libiscsi_itt; |
| __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0); |
| } |
| |
| static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn, |
| struct beiscsi_hba *phba, struct sol_cqe *psol) |
| { |
| struct hwi_wrb_context *pwrb_context; |
| struct wrb_handle *pwrb_handle; |
| struct iscsi_wrb *pwrb = NULL; |
| struct hwi_controller *phwi_ctrlr; |
| struct iscsi_task *task; |
| unsigned int type; |
| struct iscsi_conn *conn = beiscsi_conn->conn; |
| struct iscsi_session *session = conn->session; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof |
| (struct amap_sol_cqe, cid) / 32] |
| & SOL_CID_MASK) >> 6) - |
| phba->fw_config.iscsi_cid_start]; |
| pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol-> |
| dw[offsetof(struct amap_sol_cqe, wrb_index) / |
| 32] & SOL_WRB_INDEX_MASK) >> 16)]; |
| task = pwrb_handle->pio_handle; |
| pwrb = pwrb_handle->pwrb; |
| type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] & |
| WRB_TYPE_MASK) >> 28; |
| |
| spin_lock_bh(&session->lock); |
| switch (type) { |
| case HWH_TYPE_IO: |
| case HWH_TYPE_IO_RD: |
| if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == |
| ISCSI_OP_NOOP_OUT) |
| be_complete_nopin_resp(beiscsi_conn, task, psol); |
| else |
| be_complete_io(beiscsi_conn, task, psol); |
| break; |
| |
| case HWH_TYPE_LOGOUT: |
| if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT) |
| be_complete_logout(beiscsi_conn, task, psol); |
| else |
| be_complete_tmf(beiscsi_conn, task, psol); |
| |
| break; |
| |
| case HWH_TYPE_LOGIN: |
| SE_DEBUG(DBG_LVL_1, |
| "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd" |
| "- Solicited path\n"); |
| break; |
| |
| case HWH_TYPE_NOP: |
| be_complete_nopin_resp(beiscsi_conn, task, psol); |
| break; |
| |
| default: |
| shost_printk(KERN_WARNING, phba->shost, |
| "In hwi_complete_cmd, unknown type = %d" |
| "wrb_index 0x%x CID 0x%x\n", type, |
| ((psol->dw[offsetof(struct amap_iscsi_wrb, |
| type) / 32] & SOL_WRB_INDEX_MASK) >> 16), |
| ((psol->dw[offsetof(struct amap_sol_cqe, |
| cid) / 32] & SOL_CID_MASK) >> 6)); |
| break; |
| } |
| |
| spin_unlock_bh(&session->lock); |
| } |
| |
| static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context |
| *pasync_ctx, unsigned int is_header, |
| unsigned int host_write_ptr) |
| { |
| if (is_header) |
| return &pasync_ctx->async_entry[host_write_ptr]. |
| header_busy_list; |
| else |
| return &pasync_ctx->async_entry[host_write_ptr].data_busy_list; |
| } |
| |
| static struct async_pdu_handle * |
| hwi_get_async_handle(struct beiscsi_hba *phba, |
| struct beiscsi_conn *beiscsi_conn, |
| struct hwi_async_pdu_context *pasync_ctx, |
| struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index) |
| { |
| struct be_bus_address phys_addr; |
| struct list_head *pbusy_list; |
| struct async_pdu_handle *pasync_handle = NULL; |
| int buffer_len = 0; |
| unsigned char buffer_index = -1; |
| unsigned char is_header = 0; |
| |
| phys_addr.u.a32.address_lo = |
| pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] - |
| ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32] |
| & PDUCQE_DPL_MASK) >> 16); |
| phys_addr.u.a32.address_hi = |
| pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32]; |
| |
| phys_addr.u.a64.address = |
| *((unsigned long long *)(&phys_addr.u.a64.address)); |
| |
| switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32] |
| & PDUCQE_CODE_MASK) { |
| case UNSOL_HDR_NOTIFY: |
| is_header = 1; |
| |
| pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1, |
| (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, |
| index) / 32] & PDUCQE_INDEX_MASK)); |
| |
| buffer_len = (unsigned int)(phys_addr.u.a64.address - |
| pasync_ctx->async_header.pa_base.u.a64.address); |
| |
| buffer_index = buffer_len / |
| pasync_ctx->async_header.buffer_size; |
| |
| break; |
| case UNSOL_DATA_NOTIFY: |
| pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe-> |
| dw[offsetof(struct amap_i_t_dpdu_cqe, |
| index) / 32] & PDUCQE_INDEX_MASK)); |
| buffer_len = (unsigned long)(phys_addr.u.a64.address - |
| pasync_ctx->async_data.pa_base.u. |
| a64.address); |
| buffer_index = buffer_len / pasync_ctx->async_data.buffer_size; |
| break; |
| default: |
| pbusy_list = NULL; |
| shost_printk(KERN_WARNING, phba->shost, |
| "Unexpected code=%d\n", |
| pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, |
| code) / 32] & PDUCQE_CODE_MASK); |
| return NULL; |
| } |
| |
| WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries)); |
| WARN_ON(list_empty(pbusy_list)); |
| list_for_each_entry(pasync_handle, pbusy_list, link) { |
| WARN_ON(pasync_handle->consumed); |
| if (pasync_handle->index == buffer_index) |
| break; |
| } |
| |
| WARN_ON(!pasync_handle); |
| |
| pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid - |
| phba->fw_config.iscsi_cid_start; |
| pasync_handle->is_header = is_header; |
| pasync_handle->buffer_len = ((pdpdu_cqe-> |
| dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32] |
| & PDUCQE_DPL_MASK) >> 16); |
| |
| *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, |
| index) / 32] & PDUCQE_INDEX_MASK); |
| return pasync_handle; |
| } |
| |
| static unsigned int |
| hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx, |
| unsigned int is_header, unsigned int cq_index) |
| { |
| struct list_head *pbusy_list; |
| struct async_pdu_handle *pasync_handle; |
| unsigned int num_entries, writables = 0; |
| unsigned int *pep_read_ptr, *pwritables; |
| |
| |
| if (is_header) { |
| pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr; |
| pwritables = &pasync_ctx->async_header.writables; |
| num_entries = pasync_ctx->async_header.num_entries; |
| } else { |
| pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr; |
| pwritables = &pasync_ctx->async_data.writables; |
| num_entries = pasync_ctx->async_data.num_entries; |
| } |
| |
| while ((*pep_read_ptr) != cq_index) { |
| (*pep_read_ptr)++; |
| *pep_read_ptr = (*pep_read_ptr) % num_entries; |
| |
| pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header, |
| *pep_read_ptr); |
| if (writables == 0) |
| WARN_ON(list_empty(pbusy_list)); |
| |
| if (!list_empty(pbusy_list)) { |
| pasync_handle = list_entry(pbusy_list->next, |
| struct async_pdu_handle, |
| link); |
| WARN_ON(!pasync_handle); |
| pasync_handle->consumed = 1; |
| } |
| |
| writables++; |
| } |
| |
| if (!writables) { |
| SE_DEBUG(DBG_LVL_1, |
| "Duplicate notification received - index 0x%x!!\n", |
| cq_index); |
| WARN_ON(1); |
| } |
| |
| *pwritables = *pwritables + writables; |
| return 0; |
| } |
| |
| static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba, |
| unsigned int cri) |
| { |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_async_pdu_context *pasync_ctx; |
| struct async_pdu_handle *pasync_handle, *tmp_handle; |
| struct list_head *plist; |
| unsigned int i = 0; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); |
| |
| plist = &pasync_ctx->async_entry[cri].wait_queue.list; |
| |
| list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) { |
| list_del(&pasync_handle->link); |
| |
| if (i == 0) { |
| list_add_tail(&pasync_handle->link, |
| &pasync_ctx->async_header.free_list); |
| pasync_ctx->async_header.free_entries++; |
| i++; |
| } else { |
| list_add_tail(&pasync_handle->link, |
| &pasync_ctx->async_data.free_list); |
| pasync_ctx->async_data.free_entries++; |
| i++; |
| } |
| } |
| |
| INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list); |
| pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0; |
| pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0; |
| return 0; |
| } |
| |
| static struct phys_addr * |
| hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx, |
| unsigned int is_header, unsigned int host_write_ptr) |
| { |
| struct phys_addr *pasync_sge = NULL; |
| |
| if (is_header) |
| pasync_sge = pasync_ctx->async_header.ring_base; |
| else |
| pasync_sge = pasync_ctx->async_data.ring_base; |
| |
| return pasync_sge + host_write_ptr; |
| } |
| |
| static void hwi_post_async_buffers(struct beiscsi_hba *phba, |
| unsigned int is_header) |
| { |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_async_pdu_context *pasync_ctx; |
| struct async_pdu_handle *pasync_handle; |
| struct list_head *pfree_link, *pbusy_list; |
| struct phys_addr *pasync_sge; |
| unsigned int ring_id, num_entries; |
| unsigned int host_write_num; |
| unsigned int writables; |
| unsigned int i = 0; |
| u32 doorbell = 0; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); |
| |
| if (is_header) { |
| num_entries = pasync_ctx->async_header.num_entries; |
| writables = min(pasync_ctx->async_header.writables, |
| pasync_ctx->async_header.free_entries); |
| pfree_link = pasync_ctx->async_header.free_list.next; |
| host_write_num = pasync_ctx->async_header.host_write_ptr; |
| ring_id = phwi_ctrlr->default_pdu_hdr.id; |
| } else { |
| num_entries = pasync_ctx->async_data.num_entries; |
| writables = min(pasync_ctx->async_data.writables, |
| pasync_ctx->async_data.free_entries); |
| pfree_link = pasync_ctx->async_data.free_list.next; |
| host_write_num = pasync_ctx->async_data.host_write_ptr; |
| ring_id = phwi_ctrlr->default_pdu_data.id; |
| } |
| |
| writables = (writables / 8) * 8; |
| if (writables) { |
| for (i = 0; i < writables; i++) { |
| pbusy_list = |
| hwi_get_async_busy_list(pasync_ctx, is_header, |
| host_write_num); |
| pasync_handle = |
| list_entry(pfree_link, struct async_pdu_handle, |
| link); |
| WARN_ON(!pasync_handle); |
| pasync_handle->consumed = 0; |
| |
| pfree_link = pfree_link->next; |
| |
| pasync_sge = hwi_get_ring_address(pasync_ctx, |
| is_header, host_write_num); |
| |
| pasync_sge->hi = pasync_handle->pa.u.a32.address_lo; |
| pasync_sge->lo = pasync_handle->pa.u.a32.address_hi; |
| |
| list_move(&pasync_handle->link, pbusy_list); |
| |
| host_write_num++; |
| host_write_num = host_write_num % num_entries; |
| } |
| |
| if (is_header) { |
| pasync_ctx->async_header.host_write_ptr = |
| host_write_num; |
| pasync_ctx->async_header.free_entries -= writables; |
| pasync_ctx->async_header.writables -= writables; |
| pasync_ctx->async_header.busy_entries += writables; |
| } else { |
| pasync_ctx->async_data.host_write_ptr = host_write_num; |
| pasync_ctx->async_data.free_entries -= writables; |
| pasync_ctx->async_data.writables -= writables; |
| pasync_ctx->async_data.busy_entries += writables; |
| } |
| |
| doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK; |
| doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT; |
| doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT; |
| doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK) |
| << DB_DEF_PDU_CQPROC_SHIFT; |
| |
| iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET); |
| } |
| } |
| |
| static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba, |
| struct beiscsi_conn *beiscsi_conn, |
| struct i_t_dpdu_cqe *pdpdu_cqe) |
| { |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_async_pdu_context *pasync_ctx; |
| struct async_pdu_handle *pasync_handle = NULL; |
| unsigned int cq_index = -1; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); |
| |
| pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx, |
| pdpdu_cqe, &cq_index); |
| BUG_ON(pasync_handle->is_header != 0); |
| if (pasync_handle->consumed == 0) |
| hwi_update_async_writables(pasync_ctx, pasync_handle->is_header, |
| cq_index); |
| |
| hwi_free_async_msg(phba, pasync_handle->cri); |
| hwi_post_async_buffers(phba, pasync_handle->is_header); |
| } |
| |
| static unsigned int |
| hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn, |
| struct beiscsi_hba *phba, |
| struct hwi_async_pdu_context *pasync_ctx, unsigned short cri) |
| { |
| struct list_head *plist; |
| struct async_pdu_handle *pasync_handle; |
| void *phdr = NULL; |
| unsigned int hdr_len = 0, buf_len = 0; |
| unsigned int status, index = 0, offset = 0; |
| void *pfirst_buffer = NULL; |
| unsigned int num_buf = 0; |
| |
| plist = &pasync_ctx->async_entry[cri].wait_queue.list; |
| |
| list_for_each_entry(pasync_handle, plist, link) { |
| if (index == 0) { |
| phdr = pasync_handle->pbuffer; |
| hdr_len = pasync_handle->buffer_len; |
| } else { |
| buf_len = pasync_handle->buffer_len; |
| if (!num_buf) { |
| pfirst_buffer = pasync_handle->pbuffer; |
| num_buf++; |
| } |
| memcpy(pfirst_buffer + offset, |
| pasync_handle->pbuffer, buf_len); |
| offset = buf_len; |
| } |
| index++; |
| } |
| |
| status = beiscsi_process_async_pdu(beiscsi_conn, phba, |
| (beiscsi_conn->beiscsi_conn_cid - |
| phba->fw_config.iscsi_cid_start), |
| phdr, hdr_len, pfirst_buffer, |
| buf_len); |
| |
| if (status == 0) |
| hwi_free_async_msg(phba, cri); |
| return 0; |
| } |
| |
| static unsigned int |
| hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn, |
| struct beiscsi_hba *phba, |
| struct async_pdu_handle *pasync_handle) |
| { |
| struct hwi_async_pdu_context *pasync_ctx; |
| struct hwi_controller *phwi_ctrlr; |
| unsigned int bytes_needed = 0, status = 0; |
| unsigned short cri = pasync_handle->cri; |
| struct pdu_base *ppdu; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); |
| |
| list_del(&pasync_handle->link); |
| if (pasync_handle->is_header) { |
| pasync_ctx->async_header.busy_entries--; |
| if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) { |
| hwi_free_async_msg(phba, cri); |
| BUG(); |
| } |
| |
| pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0; |
| pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1; |
| pasync_ctx->async_entry[cri].wait_queue.hdr_len = |
| (unsigned short)pasync_handle->buffer_len; |
| list_add_tail(&pasync_handle->link, |
| &pasync_ctx->async_entry[cri].wait_queue.list); |
| |
| ppdu = pasync_handle->pbuffer; |
| bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base, |
| data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) & |
| 0xFFFF0000) | ((be16_to_cpu((ppdu-> |
| dw[offsetof(struct amap_pdu_base, data_len_lo) / 32] |
| & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF)); |
| |
| if (status == 0) { |
| pasync_ctx->async_entry[cri].wait_queue.bytes_needed = |
| bytes_needed; |
| |
| if (bytes_needed == 0) |
| status = hwi_fwd_async_msg(beiscsi_conn, phba, |
| pasync_ctx, cri); |
| } |
| } else { |
| pasync_ctx->async_data.busy_entries--; |
| if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) { |
| list_add_tail(&pasync_handle->link, |
| &pasync_ctx->async_entry[cri].wait_queue. |
| list); |
| pasync_ctx->async_entry[cri].wait_queue. |
| bytes_received += |
| (unsigned short)pasync_handle->buffer_len; |
| |
| if (pasync_ctx->async_entry[cri].wait_queue. |
| bytes_received >= |
| pasync_ctx->async_entry[cri].wait_queue. |
| bytes_needed) |
| status = hwi_fwd_async_msg(beiscsi_conn, phba, |
| pasync_ctx, cri); |
| } |
| } |
| return status; |
| } |
| |
| static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn, |
| struct beiscsi_hba *phba, |
| struct i_t_dpdu_cqe *pdpdu_cqe) |
| { |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_async_pdu_context *pasync_ctx; |
| struct async_pdu_handle *pasync_handle = NULL; |
| unsigned int cq_index = -1; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr); |
| pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx, |
| pdpdu_cqe, &cq_index); |
| |
| if (pasync_handle->consumed == 0) |
| hwi_update_async_writables(pasync_ctx, pasync_handle->is_header, |
| cq_index); |
| hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle); |
| hwi_post_async_buffers(phba, pasync_handle->is_header); |
| } |
| |
| static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba) |
| { |
| struct be_queue_info *mcc_cq; |
| struct be_mcc_compl *mcc_compl; |
| unsigned int num_processed = 0; |
| |
| mcc_cq = &phba->ctrl.mcc_obj.cq; |
| mcc_compl = queue_tail_node(mcc_cq); |
| mcc_compl->flags = le32_to_cpu(mcc_compl->flags); |
| while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) { |
| |
| if (num_processed >= 32) { |
| hwi_ring_cq_db(phba, mcc_cq->id, |
| num_processed, 0, 0); |
| num_processed = 0; |
| } |
| if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) { |
| /* Interpret flags as an async trailer */ |
| if (is_link_state_evt(mcc_compl->flags)) |
| /* Interpret compl as a async link evt */ |
| beiscsi_async_link_state_process(phba, |
| (struct be_async_event_link_state *) mcc_compl); |
| else |
| SE_DEBUG(DBG_LVL_1, |
| " Unsupported Async Event, flags" |
| " = 0x%08x\n", mcc_compl->flags); |
| } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) { |
| be_mcc_compl_process_isr(&phba->ctrl, mcc_compl); |
| atomic_dec(&phba->ctrl.mcc_obj.q.used); |
| } |
| |
| mcc_compl->flags = 0; |
| queue_tail_inc(mcc_cq); |
| mcc_compl = queue_tail_node(mcc_cq); |
| mcc_compl->flags = le32_to_cpu(mcc_compl->flags); |
| num_processed++; |
| } |
| |
| if (num_processed > 0) |
| hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0); |
| |
| } |
| |
| static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq) |
| { |
| struct be_queue_info *cq; |
| struct sol_cqe *sol; |
| struct dmsg_cqe *dmsg; |
| unsigned int num_processed = 0; |
| unsigned int tot_nump = 0; |
| struct beiscsi_conn *beiscsi_conn; |
| struct beiscsi_endpoint *beiscsi_ep; |
| struct iscsi_endpoint *ep; |
| struct beiscsi_hba *phba; |
| |
| cq = pbe_eq->cq; |
| sol = queue_tail_node(cq); |
| phba = pbe_eq->phba; |
| |
| while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] & |
| CQE_VALID_MASK) { |
| be_dws_le_to_cpu(sol, sizeof(struct sol_cqe)); |
| |
| ep = phba->ep_array[(u32) ((sol-> |
| dw[offsetof(struct amap_sol_cqe, cid) / 32] & |
| SOL_CID_MASK) >> 6) - |
| phba->fw_config.iscsi_cid_start]; |
| |
| beiscsi_ep = ep->dd_data; |
| beiscsi_conn = beiscsi_ep->conn; |
| |
| if (num_processed >= 32) { |
| hwi_ring_cq_db(phba, cq->id, |
| num_processed, 0, 0); |
| tot_nump += num_processed; |
| num_processed = 0; |
| } |
| |
| switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) / |
| 32] & CQE_CODE_MASK) { |
| case SOL_CMD_COMPLETE: |
| hwi_complete_cmd(beiscsi_conn, phba, sol); |
| break; |
| case DRIVERMSG_NOTIFY: |
| SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n"); |
| dmsg = (struct dmsg_cqe *)sol; |
| hwi_complete_drvr_msgs(beiscsi_conn, phba, sol); |
| break; |
| case UNSOL_HDR_NOTIFY: |
| SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n"); |
| hwi_process_default_pdu_ring(beiscsi_conn, phba, |
| (struct i_t_dpdu_cqe *)sol); |
| break; |
| case UNSOL_DATA_NOTIFY: |
| SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n"); |
| hwi_process_default_pdu_ring(beiscsi_conn, phba, |
| (struct i_t_dpdu_cqe *)sol); |
| break; |
| case CXN_INVALIDATE_INDEX_NOTIFY: |
| case CMD_INVALIDATED_NOTIFY: |
| case CXN_INVALIDATE_NOTIFY: |
| SE_DEBUG(DBG_LVL_1, |
| "Ignoring CQ Error notification for cmd/cxn" |
| "invalidate\n"); |
| break; |
| case SOL_CMD_KILLED_DATA_DIGEST_ERR: |
| case CMD_KILLED_INVALID_STATSN_RCVD: |
| case CMD_KILLED_INVALID_R2T_RCVD: |
| case CMD_CXN_KILLED_LUN_INVALID: |
| case CMD_CXN_KILLED_ICD_INVALID: |
| case CMD_CXN_KILLED_ITT_INVALID: |
| case CMD_CXN_KILLED_SEQ_OUTOFORDER: |
| case CMD_CXN_KILLED_INVALID_DATASN_RCVD: |
| SE_DEBUG(DBG_LVL_1, |
| "CQ Error notification for cmd.. " |
| "code %d cid 0x%x\n", |
| sol->dw[offsetof(struct amap_sol_cqe, code) / |
| 32] & CQE_CODE_MASK, |
| (sol->dw[offsetof(struct amap_sol_cqe, cid) / |
| 32] & SOL_CID_MASK)); |
| break; |
| case UNSOL_DATA_DIGEST_ERROR_NOTIFY: |
| SE_DEBUG(DBG_LVL_1, |
| "Digest error on def pdu ring, dropping..\n"); |
| hwi_flush_default_pdu_buffer(phba, beiscsi_conn, |
| (struct i_t_dpdu_cqe *) sol); |
| break; |
| case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL: |
| case CXN_KILLED_BURST_LEN_MISMATCH: |
| case CXN_KILLED_AHS_RCVD: |
| case CXN_KILLED_HDR_DIGEST_ERR: |
| case CXN_KILLED_UNKNOWN_HDR: |
| case CXN_KILLED_STALE_ITT_TTT_RCVD: |
| case CXN_KILLED_INVALID_ITT_TTT_RCVD: |
| case CXN_KILLED_TIMED_OUT: |
| case CXN_KILLED_FIN_RCVD: |
| case CXN_KILLED_BAD_UNSOL_PDU_RCVD: |
| case CXN_KILLED_BAD_WRB_INDEX_ERROR: |
| case CXN_KILLED_OVER_RUN_RESIDUAL: |
| case CXN_KILLED_UNDER_RUN_RESIDUAL: |
| case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN: |
| SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID " |
| "0x%x...\n", |
| sol->dw[offsetof(struct amap_sol_cqe, code) / |
| 32] & CQE_CODE_MASK, |
| (sol->dw[offsetof(struct amap_sol_cqe, cid) / |
| 32] & CQE_CID_MASK)); |
| iscsi_conn_failure(beiscsi_conn->conn, |
| ISCSI_ERR_CONN_FAILED); |
| break; |
| case CXN_KILLED_RST_SENT: |
| case CXN_KILLED_RST_RCVD: |
| SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset" |
| "received/sent on CID 0x%x...\n", |
| sol->dw[offsetof(struct amap_sol_cqe, code) / |
| 32] & CQE_CODE_MASK, |
| (sol->dw[offsetof(struct amap_sol_cqe, cid) / |
| 32] & CQE_CID_MASK)); |
| iscsi_conn_failure(beiscsi_conn->conn, |
| ISCSI_ERR_CONN_FAILED); |
| break; |
| default: |
| SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d " |
| "received on CID 0x%x...\n", |
| sol->dw[offsetof(struct amap_sol_cqe, code) / |
| 32] & CQE_CODE_MASK, |
| (sol->dw[offsetof(struct amap_sol_cqe, cid) / |
| 32] & CQE_CID_MASK)); |
| break; |
| } |
| |
| AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0); |
| queue_tail_inc(cq); |
| sol = queue_tail_node(cq); |
| num_processed++; |
| } |
| |
| if (num_processed > 0) { |
| tot_nump += num_processed; |
| hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0); |
| } |
| return tot_nump; |
| } |
| |
| void beiscsi_process_all_cqs(struct work_struct *work) |
| { |
| unsigned long flags; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| struct be_eq_obj *pbe_eq; |
| struct beiscsi_hba *phba = |
| container_of(work, struct beiscsi_hba, work_cqs); |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| if (phba->msix_enabled) |
| pbe_eq = &phwi_context->be_eq[phba->num_cpus]; |
| else |
| pbe_eq = &phwi_context->be_eq[0]; |
| |
| if (phba->todo_mcc_cq) { |
| spin_lock_irqsave(&phba->isr_lock, flags); |
| phba->todo_mcc_cq = 0; |
| spin_unlock_irqrestore(&phba->isr_lock, flags); |
| beiscsi_process_mcc_isr(phba); |
| } |
| |
| if (phba->todo_cq) { |
| spin_lock_irqsave(&phba->isr_lock, flags); |
| phba->todo_cq = 0; |
| spin_unlock_irqrestore(&phba->isr_lock, flags); |
| beiscsi_process_cq(pbe_eq); |
| } |
| } |
| |
| static int be_iopoll(struct blk_iopoll *iop, int budget) |
| { |
| static unsigned int ret; |
| struct beiscsi_hba *phba; |
| struct be_eq_obj *pbe_eq; |
| |
| pbe_eq = container_of(iop, struct be_eq_obj, iopoll); |
| ret = beiscsi_process_cq(pbe_eq); |
| if (ret < budget) { |
| phba = pbe_eq->phba; |
| blk_iopoll_complete(iop); |
| SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id); |
| hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1); |
| } |
| return ret; |
| } |
| |
| static void |
| hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg, |
| unsigned int num_sg, struct beiscsi_io_task *io_task) |
| { |
| struct iscsi_sge *psgl; |
| unsigned int sg_len, index; |
| unsigned int sge_len = 0; |
| unsigned long long addr; |
| struct scatterlist *l_sg; |
| unsigned int offset; |
| |
| AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, |
| io_task->bhs_pa.u.a32.address_lo); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, |
| io_task->bhs_pa.u.a32.address_hi); |
| |
| l_sg = sg; |
| for (index = 0; (index < num_sg) && (index < 2); index++, |
| sg = sg_next(sg)) { |
| if (index == 0) { |
| sg_len = sg_dma_len(sg); |
| addr = (u64) sg_dma_address(sg); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, |
| ((u32)(addr & 0xFFFFFFFF))); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, |
| ((u32)(addr >> 32))); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, |
| sg_len); |
| sge_len = sg_len; |
| } else { |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset, |
| pwrb, sge_len); |
| sg_len = sg_dma_len(sg); |
| addr = (u64) sg_dma_address(sg); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb, |
| ((u32)(addr & 0xFFFFFFFF))); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb, |
| ((u32)(addr >> 32))); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb, |
| sg_len); |
| } |
| } |
| psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; |
| memset(psgl, 0, sizeof(*psgl) * BE2_SGE); |
| |
| AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2); |
| |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, |
| io_task->bhs_pa.u.a32.address_hi); |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, |
| io_task->bhs_pa.u.a32.address_lo); |
| |
| if (num_sg == 1) { |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, |
| 1); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, |
| 0); |
| } else if (num_sg == 2) { |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, |
| 0); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, |
| 1); |
| } else { |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, |
| 0); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb, |
| 0); |
| } |
| sg = l_sg; |
| psgl++; |
| psgl++; |
| offset = 0; |
| for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) { |
| sg_len = sg_dma_len(sg); |
| addr = (u64) sg_dma_address(sg); |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, |
| (addr & 0xFFFFFFFF)); |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, |
| (addr >> 32)); |
| AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len); |
| AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset); |
| AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); |
| offset += sg_len; |
| } |
| psgl--; |
| AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); |
| } |
| |
| static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task) |
| { |
| struct iscsi_sge *psgl; |
| unsigned long long addr; |
| struct beiscsi_io_task *io_task = task->dd_data; |
| struct beiscsi_conn *beiscsi_conn = io_task->conn; |
| struct beiscsi_hba *phba = beiscsi_conn->phba; |
| |
| io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2; |
| AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb, |
| io_task->bhs_pa.u.a32.address_lo); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb, |
| io_task->bhs_pa.u.a32.address_hi); |
| |
| if (task->data) { |
| if (task->data_count) { |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); |
| addr = (u64) pci_map_single(phba->pcidev, |
| task->data, |
| task->data_count, 1); |
| } else { |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); |
| addr = 0; |
| } |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb, |
| ((u32)(addr & 0xFFFFFFFF))); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb, |
| ((u32)(addr >> 32))); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb, |
| task->data_count); |
| |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1); |
| } else { |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); |
| addr = 0; |
| } |
| |
| psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag; |
| |
| AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len); |
| |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, |
| io_task->bhs_pa.u.a32.address_hi); |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, |
| io_task->bhs_pa.u.a32.address_lo); |
| if (task->data) { |
| psgl++; |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0); |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0); |
| AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0); |
| AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0); |
| AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0); |
| AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0); |
| |
| psgl++; |
| if (task->data) { |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, |
| ((u32)(addr & 0xFFFFFFFF))); |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, |
| ((u32)(addr >> 32))); |
| } |
| AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106); |
| } |
| AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1); |
| } |
| |
| static void beiscsi_find_mem_req(struct beiscsi_hba *phba) |
| { |
| unsigned int num_cq_pages, num_async_pdu_buf_pages; |
| unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn; |
| unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages; |
| |
| num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ |
| sizeof(struct sol_cqe)); |
| num_async_pdu_buf_pages = |
| PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ |
| phba->params.defpdu_hdr_sz); |
| num_async_pdu_buf_sgl_pages = |
| PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ |
| sizeof(struct phys_addr)); |
| num_async_pdu_data_pages = |
| PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ |
| phba->params.defpdu_data_sz); |
| num_async_pdu_data_sgl_pages = |
| PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \ |
| sizeof(struct phys_addr)); |
| |
| phba->params.hwi_ws_sz = sizeof(struct hwi_controller); |
| |
| phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 * |
| BE_ISCSI_PDU_HEADER_SIZE; |
| phba->mem_req[HWI_MEM_ADDN_CONTEXT] = |
| sizeof(struct hwi_context_memory); |
| |
| |
| phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb) |
| * (phba->params.wrbs_per_cxn) |
| * phba->params.cxns_per_ctrl; |
| wrb_sz_per_cxn = sizeof(struct wrb_handle) * |
| (phba->params.wrbs_per_cxn); |
| phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) * |
| phba->params.cxns_per_ctrl); |
| |
| phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) * |
| phba->params.icds_per_ctrl; |
| phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) * |
| phba->params.num_sge_per_io * phba->params.icds_per_ctrl; |
| |
| phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] = |
| num_async_pdu_buf_pages * PAGE_SIZE; |
| phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] = |
| num_async_pdu_data_pages * PAGE_SIZE; |
| phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] = |
| num_async_pdu_buf_sgl_pages * PAGE_SIZE; |
| phba->mem_req[HWI_MEM_ASYNC_DATA_RING] = |
| num_async_pdu_data_sgl_pages * PAGE_SIZE; |
| phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] = |
| phba->params.asyncpdus_per_ctrl * |
| sizeof(struct async_pdu_handle); |
| phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] = |
| phba->params.asyncpdus_per_ctrl * |
| sizeof(struct async_pdu_handle); |
| phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] = |
| sizeof(struct hwi_async_pdu_context) + |
| (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry)); |
| } |
| |
| static int beiscsi_alloc_mem(struct beiscsi_hba *phba) |
| { |
| struct be_mem_descriptor *mem_descr; |
| dma_addr_t bus_add; |
| struct mem_array *mem_arr, *mem_arr_orig; |
| unsigned int i, j, alloc_size, curr_alloc_size; |
| |
| phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL); |
| if (!phba->phwi_ctrlr) |
| return -ENOMEM; |
| |
| phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr), |
| GFP_KERNEL); |
| if (!phba->init_mem) { |
| kfree(phba->phwi_ctrlr); |
| return -ENOMEM; |
| } |
| |
| mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT, |
| GFP_KERNEL); |
| if (!mem_arr_orig) { |
| kfree(phba->init_mem); |
| kfree(phba->phwi_ctrlr); |
| return -ENOMEM; |
| } |
| |
| mem_descr = phba->init_mem; |
| for (i = 0; i < SE_MEM_MAX; i++) { |
| j = 0; |
| mem_arr = mem_arr_orig; |
| alloc_size = phba->mem_req[i]; |
| memset(mem_arr, 0, sizeof(struct mem_array) * |
| BEISCSI_MAX_FRAGS_INIT); |
| curr_alloc_size = min(be_max_phys_size * 1024, alloc_size); |
| do { |
| mem_arr->virtual_address = pci_alloc_consistent( |
| phba->pcidev, |
| curr_alloc_size, |
| &bus_add); |
| if (!mem_arr->virtual_address) { |
| if (curr_alloc_size <= BE_MIN_MEM_SIZE) |
| goto free_mem; |
| if (curr_alloc_size - |
| rounddown_pow_of_two(curr_alloc_size)) |
| curr_alloc_size = rounddown_pow_of_two |
| (curr_alloc_size); |
| else |
| curr_alloc_size = curr_alloc_size / 2; |
| } else { |
| mem_arr->bus_address.u. |
| a64.address = (__u64) bus_add; |
| mem_arr->size = curr_alloc_size; |
| alloc_size -= curr_alloc_size; |
| curr_alloc_size = min(be_max_phys_size * |
| 1024, alloc_size); |
| j++; |
| mem_arr++; |
| } |
| } while (alloc_size); |
| mem_descr->num_elements = j; |
| mem_descr->size_in_bytes = phba->mem_req[i]; |
| mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j, |
| GFP_KERNEL); |
| if (!mem_descr->mem_array) |
| goto free_mem; |
| |
| memcpy(mem_descr->mem_array, mem_arr_orig, |
| sizeof(struct mem_array) * j); |
| mem_descr++; |
| } |
| kfree(mem_arr_orig); |
| return 0; |
| free_mem: |
| mem_descr->num_elements = j; |
| while ((i) || (j)) { |
| for (j = mem_descr->num_elements; j > 0; j--) { |
| pci_free_consistent(phba->pcidev, |
| mem_descr->mem_array[j - 1].size, |
| mem_descr->mem_array[j - 1]. |
| virtual_address, |
| (unsigned long)mem_descr-> |
| mem_array[j - 1]. |
| bus_address.u.a64.address); |
| } |
| if (i) { |
| i--; |
| kfree(mem_descr->mem_array); |
| mem_descr--; |
| } |
| } |
| kfree(mem_arr_orig); |
| kfree(phba->init_mem); |
| kfree(phba->phwi_ctrlr); |
| return -ENOMEM; |
| } |
| |
| static int beiscsi_get_memory(struct beiscsi_hba *phba) |
| { |
| beiscsi_find_mem_req(phba); |
| return beiscsi_alloc_mem(phba); |
| } |
| |
| static void iscsi_init_global_templates(struct beiscsi_hba *phba) |
| { |
| struct pdu_data_out *pdata_out; |
| struct pdu_nop_out *pnop_out; |
| struct be_mem_descriptor *mem_descr; |
| |
| mem_descr = phba->init_mem; |
| mem_descr += ISCSI_MEM_GLOBAL_HEADER; |
| pdata_out = |
| (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address; |
| memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE); |
| |
| AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out, |
| IIOC_SCSI_DATA); |
| |
| pnop_out = |
| (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0]. |
| virtual_address + BE_ISCSI_PDU_HEADER_SIZE); |
| |
| memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE); |
| AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF); |
| AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1); |
| AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0); |
| } |
| |
| static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba) |
| { |
| struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb; |
| struct wrb_handle *pwrb_handle; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_wrb_context *pwrb_context; |
| struct iscsi_wrb *pwrb; |
| unsigned int num_cxn_wrbh; |
| unsigned int num_cxn_wrb, j, idx, index; |
| |
| mem_descr_wrbh = phba->init_mem; |
| mem_descr_wrbh += HWI_MEM_WRBH; |
| |
| mem_descr_wrb = phba->init_mem; |
| mem_descr_wrb += HWI_MEM_WRB; |
| |
| idx = 0; |
| pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address; |
| num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) / |
| ((sizeof(struct wrb_handle)) * |
| phba->params.wrbs_per_cxn)); |
| phwi_ctrlr = phba->phwi_ctrlr; |
| |
| for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { |
| pwrb_context = &phwi_ctrlr->wrb_context[index]; |
| pwrb_context->pwrb_handle_base = |
| kzalloc(sizeof(struct wrb_handle *) * |
| phba->params.wrbs_per_cxn, GFP_KERNEL); |
| pwrb_context->pwrb_handle_basestd = |
| kzalloc(sizeof(struct wrb_handle *) * |
| phba->params.wrbs_per_cxn, GFP_KERNEL); |
| if (num_cxn_wrbh) { |
| pwrb_context->alloc_index = 0; |
| pwrb_context->wrb_handles_available = 0; |
| for (j = 0; j < phba->params.wrbs_per_cxn; j++) { |
| pwrb_context->pwrb_handle_base[j] = pwrb_handle; |
| pwrb_context->pwrb_handle_basestd[j] = |
| pwrb_handle; |
| pwrb_context->wrb_handles_available++; |
| pwrb_handle->wrb_index = j; |
| pwrb_handle++; |
| } |
| pwrb_context->free_index = 0; |
| num_cxn_wrbh--; |
| } else { |
| idx++; |
| pwrb_handle = |
| mem_descr_wrbh->mem_array[idx].virtual_address; |
| num_cxn_wrbh = |
| ((mem_descr_wrbh->mem_array[idx].size) / |
| ((sizeof(struct wrb_handle)) * |
| phba->params.wrbs_per_cxn)); |
| pwrb_context->alloc_index = 0; |
| for (j = 0; j < phba->params.wrbs_per_cxn; j++) { |
| pwrb_context->pwrb_handle_base[j] = pwrb_handle; |
| pwrb_context->pwrb_handle_basestd[j] = |
| pwrb_handle; |
| pwrb_context->wrb_handles_available++; |
| pwrb_handle->wrb_index = j; |
| pwrb_handle++; |
| } |
| pwrb_context->free_index = 0; |
| num_cxn_wrbh--; |
| } |
| } |
| idx = 0; |
| pwrb = mem_descr_wrb->mem_array[idx].virtual_address; |
| num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) / |
| ((sizeof(struct iscsi_wrb) * |
| phba->params.wrbs_per_cxn)); |
| for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { |
| pwrb_context = &phwi_ctrlr->wrb_context[index]; |
| if (num_cxn_wrb) { |
| for (j = 0; j < phba->params.wrbs_per_cxn; j++) { |
| pwrb_handle = pwrb_context->pwrb_handle_base[j]; |
| pwrb_handle->pwrb = pwrb; |
| pwrb++; |
| } |
| num_cxn_wrb--; |
| } else { |
| idx++; |
| pwrb = mem_descr_wrb->mem_array[idx].virtual_address; |
| num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) / |
| ((sizeof(struct iscsi_wrb) * |
| phba->params.wrbs_per_cxn)); |
| for (j = 0; j < phba->params.wrbs_per_cxn; j++) { |
| pwrb_handle = pwrb_context->pwrb_handle_base[j]; |
| pwrb_handle->pwrb = pwrb; |
| pwrb++; |
| } |
| num_cxn_wrb--; |
| } |
| } |
| } |
| |
| static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba) |
| { |
| struct hwi_controller *phwi_ctrlr; |
| struct hba_parameters *p = &phba->params; |
| struct hwi_async_pdu_context *pasync_ctx; |
| struct async_pdu_handle *pasync_header_h, *pasync_data_h; |
| unsigned int index; |
| struct be_mem_descriptor *mem_descr; |
| |
| mem_descr = (struct be_mem_descriptor *)phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *) |
| mem_descr->mem_array[0].virtual_address; |
| pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx; |
| memset(pasync_ctx, 0, sizeof(*pasync_ctx)); |
| |
| pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl; |
| pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz; |
| pasync_ctx->async_data.buffer_size = p->defpdu_data_sz; |
| pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl; |
| |
| mem_descr = (struct be_mem_descriptor *)phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_HEADER_BUF; |
| if (mem_descr->mem_array[0].virtual_address) { |
| SE_DEBUG(DBG_LVL_8, |
| "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF" |
| "va=%p\n", mem_descr->mem_array[0].virtual_address); |
| } else |
| shost_printk(KERN_WARNING, phba->shost, |
| "No Virtual address\n"); |
| |
| pasync_ctx->async_header.va_base = |
| mem_descr->mem_array[0].virtual_address; |
| |
| pasync_ctx->async_header.pa_base.u.a64.address = |
| mem_descr->mem_array[0].bus_address.u.a64.address; |
| |
| mem_descr = (struct be_mem_descriptor *)phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_HEADER_RING; |
| if (mem_descr->mem_array[0].virtual_address) { |
| SE_DEBUG(DBG_LVL_8, |
| "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING" |
| "va=%p\n", mem_descr->mem_array[0].virtual_address); |
| } else |
| shost_printk(KERN_WARNING, phba->shost, |
| "No Virtual address\n"); |
| pasync_ctx->async_header.ring_base = |
| mem_descr->mem_array[0].virtual_address; |
| |
| mem_descr = (struct be_mem_descriptor *)phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE; |
| if (mem_descr->mem_array[0].virtual_address) { |
| SE_DEBUG(DBG_LVL_8, |
| "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE" |
| "va=%p\n", mem_descr->mem_array[0].virtual_address); |
| } else |
| shost_printk(KERN_WARNING, phba->shost, |
| "No Virtual address\n"); |
| |
| pasync_ctx->async_header.handle_base = |
| mem_descr->mem_array[0].virtual_address; |
| pasync_ctx->async_header.writables = 0; |
| INIT_LIST_HEAD(&pasync_ctx->async_header.free_list); |
| |
| mem_descr = (struct be_mem_descriptor *)phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_DATA_BUF; |
| if (mem_descr->mem_array[0].virtual_address) { |
| SE_DEBUG(DBG_LVL_8, |
| "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF" |
| "va=%p\n", mem_descr->mem_array[0].virtual_address); |
| } else |
| shost_printk(KERN_WARNING, phba->shost, |
| "No Virtual address\n"); |
| pasync_ctx->async_data.va_base = |
| mem_descr->mem_array[0].virtual_address; |
| pasync_ctx->async_data.pa_base.u.a64.address = |
| mem_descr->mem_array[0].bus_address.u.a64.address; |
| |
| mem_descr = (struct be_mem_descriptor *)phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_DATA_RING; |
| if (mem_descr->mem_array[0].virtual_address) { |
| SE_DEBUG(DBG_LVL_8, |
| "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING" |
| "va=%p\n", mem_descr->mem_array[0].virtual_address); |
| } else |
| shost_printk(KERN_WARNING, phba->shost, |
| "No Virtual address\n"); |
| |
| pasync_ctx->async_data.ring_base = |
| mem_descr->mem_array[0].virtual_address; |
| |
| mem_descr = (struct be_mem_descriptor *)phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_DATA_HANDLE; |
| if (!mem_descr->mem_array[0].virtual_address) |
| shost_printk(KERN_WARNING, phba->shost, |
| "No Virtual address\n"); |
| |
| pasync_ctx->async_data.handle_base = |
| mem_descr->mem_array[0].virtual_address; |
| pasync_ctx->async_data.writables = 0; |
| INIT_LIST_HEAD(&pasync_ctx->async_data.free_list); |
| |
| pasync_header_h = |
| (struct async_pdu_handle *)pasync_ctx->async_header.handle_base; |
| pasync_data_h = |
| (struct async_pdu_handle *)pasync_ctx->async_data.handle_base; |
| |
| for (index = 0; index < p->asyncpdus_per_ctrl; index++) { |
| pasync_header_h->cri = -1; |
| pasync_header_h->index = (char)index; |
| INIT_LIST_HEAD(&pasync_header_h->link); |
| pasync_header_h->pbuffer = |
| (void *)((unsigned long) |
| (pasync_ctx->async_header.va_base) + |
| (p->defpdu_hdr_sz * index)); |
| |
| pasync_header_h->pa.u.a64.address = |
| pasync_ctx->async_header.pa_base.u.a64.address + |
| (p->defpdu_hdr_sz * index); |
| |
| list_add_tail(&pasync_header_h->link, |
| &pasync_ctx->async_header.free_list); |
| pasync_header_h++; |
| pasync_ctx->async_header.free_entries++; |
| pasync_ctx->async_header.writables++; |
| |
| INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list); |
| INIT_LIST_HEAD(&pasync_ctx->async_entry[index]. |
| header_busy_list); |
| pasync_data_h->cri = -1; |
| pasync_data_h->index = (char)index; |
| INIT_LIST_HEAD(&pasync_data_h->link); |
| pasync_data_h->pbuffer = |
| (void *)((unsigned long) |
| (pasync_ctx->async_data.va_base) + |
| (p->defpdu_data_sz * index)); |
| |
| pasync_data_h->pa.u.a64.address = |
| pasync_ctx->async_data.pa_base.u.a64.address + |
| (p->defpdu_data_sz * index); |
| |
| list_add_tail(&pasync_data_h->link, |
| &pasync_ctx->async_data.free_list); |
| pasync_data_h++; |
| pasync_ctx->async_data.free_entries++; |
| pasync_ctx->async_data.writables++; |
| |
| INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list); |
| } |
| |
| pasync_ctx->async_header.host_write_ptr = 0; |
| pasync_ctx->async_header.ep_read_ptr = -1; |
| pasync_ctx->async_data.host_write_ptr = 0; |
| pasync_ctx->async_data.ep_read_ptr = -1; |
| } |
| |
| static int |
| be_sgl_create_contiguous(void *virtual_address, |
| u64 physical_address, u32 length, |
| struct be_dma_mem *sgl) |
| { |
| WARN_ON(!virtual_address); |
| WARN_ON(!physical_address); |
| WARN_ON(!length > 0); |
| WARN_ON(!sgl); |
| |
| sgl->va = virtual_address; |
| sgl->dma = (unsigned long)physical_address; |
| sgl->size = length; |
| |
| return 0; |
| } |
| |
| static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl) |
| { |
| memset(sgl, 0, sizeof(*sgl)); |
| } |
| |
| static void |
| hwi_build_be_sgl_arr(struct beiscsi_hba *phba, |
| struct mem_array *pmem, struct be_dma_mem *sgl) |
| { |
| if (sgl->va) |
| be_sgl_destroy_contiguous(sgl); |
| |
| be_sgl_create_contiguous(pmem->virtual_address, |
| pmem->bus_address.u.a64.address, |
| pmem->size, sgl); |
| } |
| |
| static void |
| hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba, |
| struct mem_array *pmem, struct be_dma_mem *sgl) |
| { |
| if (sgl->va) |
| be_sgl_destroy_contiguous(sgl); |
| |
| be_sgl_create_contiguous((unsigned char *)pmem->virtual_address, |
| pmem->bus_address.u.a64.address, |
| pmem->size, sgl); |
| } |
| |
| static int be_fill_queue(struct be_queue_info *q, |
| u16 len, u16 entry_size, void *vaddress) |
| { |
| struct be_dma_mem *mem = &q->dma_mem; |
| |
| memset(q, 0, sizeof(*q)); |
| q->len = len; |
| q->entry_size = entry_size; |
| mem->size = len * entry_size; |
| mem->va = vaddress; |
| if (!mem->va) |
| return -ENOMEM; |
| memset(mem->va, 0, mem->size); |
| return 0; |
| } |
| |
| static int beiscsi_create_eqs(struct beiscsi_hba *phba, |
| struct hwi_context_memory *phwi_context) |
| { |
| unsigned int i, num_eq_pages; |
| int ret, eq_for_mcc; |
| struct be_queue_info *eq; |
| struct be_dma_mem *mem; |
| void *eq_vaddress; |
| dma_addr_t paddr; |
| |
| num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \ |
| sizeof(struct be_eq_entry)); |
| |
| if (phba->msix_enabled) |
| eq_for_mcc = 1; |
| else |
| eq_for_mcc = 0; |
| for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) { |
| eq = &phwi_context->be_eq[i].q; |
| mem = &eq->dma_mem; |
| phwi_context->be_eq[i].phba = phba; |
| eq_vaddress = pci_alloc_consistent(phba->pcidev, |
| num_eq_pages * PAGE_SIZE, |
| &paddr); |
| if (!eq_vaddress) |
| goto create_eq_error; |
| |
| mem->va = eq_vaddress; |
| ret = be_fill_queue(eq, phba->params.num_eq_entries, |
| sizeof(struct be_eq_entry), eq_vaddress); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "be_fill_queue Failed for EQ\n"); |
| goto create_eq_error; |
| } |
| |
| mem->dma = paddr; |
| ret = beiscsi_cmd_eq_create(&phba->ctrl, eq, |
| phwi_context->cur_eqd); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "beiscsi_cmd_eq_create" |
| "Failedfor EQ\n"); |
| goto create_eq_error; |
| } |
| SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id); |
| } |
| return 0; |
| create_eq_error: |
| for (i = 0; i < (phba->num_cpus + 1); i++) { |
| eq = &phwi_context->be_eq[i].q; |
| mem = &eq->dma_mem; |
| if (mem->va) |
| pci_free_consistent(phba->pcidev, num_eq_pages |
| * PAGE_SIZE, |
| mem->va, mem->dma); |
| } |
| return ret; |
| } |
| |
| static int beiscsi_create_cqs(struct beiscsi_hba *phba, |
| struct hwi_context_memory *phwi_context) |
| { |
| unsigned int i, num_cq_pages; |
| int ret; |
| struct be_queue_info *cq, *eq; |
| struct be_dma_mem *mem; |
| struct be_eq_obj *pbe_eq; |
| void *cq_vaddress; |
| dma_addr_t paddr; |
| |
| num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \ |
| sizeof(struct sol_cqe)); |
| |
| for (i = 0; i < phba->num_cpus; i++) { |
| cq = &phwi_context->be_cq[i]; |
| eq = &phwi_context->be_eq[i].q; |
| pbe_eq = &phwi_context->be_eq[i]; |
| pbe_eq->cq = cq; |
| pbe_eq->phba = phba; |
| mem = &cq->dma_mem; |
| cq_vaddress = pci_alloc_consistent(phba->pcidev, |
| num_cq_pages * PAGE_SIZE, |
| &paddr); |
| if (!cq_vaddress) |
| goto create_cq_error; |
| ret = be_fill_queue(cq, phba->params.num_cq_entries, |
| sizeof(struct sol_cqe), cq_vaddress); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "be_fill_queue Failed for ISCSI CQ\n"); |
| goto create_cq_error; |
| } |
| |
| mem->dma = paddr; |
| ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false, |
| false, 0); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "beiscsi_cmd_eq_create" |
| "Failed for ISCSI CQ\n"); |
| goto create_cq_error; |
| } |
| SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n", |
| cq->id, eq->id); |
| SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n"); |
| } |
| return 0; |
| |
| create_cq_error: |
| for (i = 0; i < phba->num_cpus; i++) { |
| cq = &phwi_context->be_cq[i]; |
| mem = &cq->dma_mem; |
| if (mem->va) |
| pci_free_consistent(phba->pcidev, num_cq_pages |
| * PAGE_SIZE, |
| mem->va, mem->dma); |
| } |
| return ret; |
| |
| } |
| |
| static int |
| beiscsi_create_def_hdr(struct beiscsi_hba *phba, |
| struct hwi_context_memory *phwi_context, |
| struct hwi_controller *phwi_ctrlr, |
| unsigned int def_pdu_ring_sz) |
| { |
| unsigned int idx; |
| int ret; |
| struct be_queue_info *dq, *cq; |
| struct be_dma_mem *mem; |
| struct be_mem_descriptor *mem_descr; |
| void *dq_vaddress; |
| |
| idx = 0; |
| dq = &phwi_context->be_def_hdrq; |
| cq = &phwi_context->be_cq[0]; |
| mem = &dq->dma_mem; |
| mem_descr = phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_HEADER_RING; |
| dq_vaddress = mem_descr->mem_array[idx].virtual_address; |
| ret = be_fill_queue(dq, mem_descr->mem_array[0].size / |
| sizeof(struct phys_addr), |
| sizeof(struct phys_addr), dq_vaddress); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "be_fill_queue Failed for DEF PDU HDR\n"); |
| return ret; |
| } |
| mem->dma = (unsigned long)mem_descr->mem_array[idx]. |
| bus_address.u.a64.address; |
| ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq, |
| def_pdu_ring_sz, |
| phba->params.defpdu_hdr_sz); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "be_cmd_create_default_pdu_queue Failed DEFHDR\n"); |
| return ret; |
| } |
| phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id; |
| SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n", |
| phwi_context->be_def_hdrq.id); |
| hwi_post_async_buffers(phba, 1); |
| return 0; |
| } |
| |
| static int |
| beiscsi_create_def_data(struct beiscsi_hba *phba, |
| struct hwi_context_memory *phwi_context, |
| struct hwi_controller *phwi_ctrlr, |
| unsigned int def_pdu_ring_sz) |
| { |
| unsigned int idx; |
| int ret; |
| struct be_queue_info *dataq, *cq; |
| struct be_dma_mem *mem; |
| struct be_mem_descriptor *mem_descr; |
| void *dq_vaddress; |
| |
| idx = 0; |
| dataq = &phwi_context->be_def_dataq; |
| cq = &phwi_context->be_cq[0]; |
| mem = &dataq->dma_mem; |
| mem_descr = phba->init_mem; |
| mem_descr += HWI_MEM_ASYNC_DATA_RING; |
| dq_vaddress = mem_descr->mem_array[idx].virtual_address; |
| ret = be_fill_queue(dataq, mem_descr->mem_array[0].size / |
| sizeof(struct phys_addr), |
| sizeof(struct phys_addr), dq_vaddress); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "be_fill_queue Failed for DEF PDU DATA\n"); |
| return ret; |
| } |
| mem->dma = (unsigned long)mem_descr->mem_array[idx]. |
| bus_address.u.a64.address; |
| ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq, |
| def_pdu_ring_sz, |
| phba->params.defpdu_data_sz); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "be_cmd_create_default_pdu_queue Failed" |
| " for DEF PDU DATA\n"); |
| return ret; |
| } |
| phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id; |
| SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n", |
| phwi_context->be_def_dataq.id); |
| hwi_post_async_buffers(phba, 0); |
| SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n"); |
| return 0; |
| } |
| |
| static int |
| beiscsi_post_pages(struct beiscsi_hba *phba) |
| { |
| struct be_mem_descriptor *mem_descr; |
| struct mem_array *pm_arr; |
| unsigned int page_offset, i; |
| struct be_dma_mem sgl; |
| int status; |
| |
| mem_descr = phba->init_mem; |
| mem_descr += HWI_MEM_SGE; |
| pm_arr = mem_descr->mem_array; |
| |
| page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io * |
| phba->fw_config.iscsi_icd_start) / PAGE_SIZE; |
| for (i = 0; i < mem_descr->num_elements; i++) { |
| hwi_build_be_sgl_arr(phba, pm_arr, &sgl); |
| status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl, |
| page_offset, |
| (pm_arr->size / PAGE_SIZE)); |
| page_offset += pm_arr->size / PAGE_SIZE; |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "post sgl failed.\n"); |
| return status; |
| } |
| pm_arr++; |
| } |
| SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n"); |
| return 0; |
| } |
| |
| static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q) |
| { |
| struct be_dma_mem *mem = &q->dma_mem; |
| if (mem->va) |
| pci_free_consistent(phba->pcidev, mem->size, |
| mem->va, mem->dma); |
| } |
| |
| static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q, |
| u16 len, u16 entry_size) |
| { |
| struct be_dma_mem *mem = &q->dma_mem; |
| |
| memset(q, 0, sizeof(*q)); |
| q->len = len; |
| q->entry_size = entry_size; |
| mem->size = len * entry_size; |
| mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma); |
| if (!mem->va) |
| return -ENOMEM; |
| memset(mem->va, 0, mem->size); |
| return 0; |
| } |
| |
| static int |
| beiscsi_create_wrb_rings(struct beiscsi_hba *phba, |
| struct hwi_context_memory *phwi_context, |
| struct hwi_controller *phwi_ctrlr) |
| { |
| unsigned int wrb_mem_index, offset, size, num_wrb_rings; |
| u64 pa_addr_lo; |
| unsigned int idx, num, i; |
| struct mem_array *pwrb_arr; |
| void *wrb_vaddr; |
| struct be_dma_mem sgl; |
| struct be_mem_descriptor *mem_descr; |
| int status; |
| |
| idx = 0; |
| mem_descr = phba->init_mem; |
| mem_descr += HWI_MEM_WRB; |
| pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl, |
| GFP_KERNEL); |
| if (!pwrb_arr) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Memory alloc failed in create wrb ring.\n"); |
| return -ENOMEM; |
| } |
| wrb_vaddr = mem_descr->mem_array[idx].virtual_address; |
| pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address; |
| num_wrb_rings = mem_descr->mem_array[idx].size / |
| (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb)); |
| |
| for (num = 0; num < phba->params.cxns_per_ctrl; num++) { |
| if (num_wrb_rings) { |
| pwrb_arr[num].virtual_address = wrb_vaddr; |
| pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo; |
| pwrb_arr[num].size = phba->params.wrbs_per_cxn * |
| sizeof(struct iscsi_wrb); |
| wrb_vaddr += pwrb_arr[num].size; |
| pa_addr_lo += pwrb_arr[num].size; |
| num_wrb_rings--; |
| } else { |
| idx++; |
| wrb_vaddr = mem_descr->mem_array[idx].virtual_address; |
| pa_addr_lo = mem_descr->mem_array[idx].\ |
| bus_address.u.a64.address; |
| num_wrb_rings = mem_descr->mem_array[idx].size / |
| (phba->params.wrbs_per_cxn * |
| sizeof(struct iscsi_wrb)); |
| pwrb_arr[num].virtual_address = wrb_vaddr; |
| pwrb_arr[num].bus_address.u.a64.address\ |
| = pa_addr_lo; |
| pwrb_arr[num].size = phba->params.wrbs_per_cxn * |
| sizeof(struct iscsi_wrb); |
| wrb_vaddr += pwrb_arr[num].size; |
| pa_addr_lo += pwrb_arr[num].size; |
| num_wrb_rings--; |
| } |
| } |
| for (i = 0; i < phba->params.cxns_per_ctrl; i++) { |
| wrb_mem_index = 0; |
| offset = 0; |
| size = 0; |
| |
| hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl); |
| status = be_cmd_wrbq_create(&phba->ctrl, &sgl, |
| &phwi_context->be_wrbq[i]); |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "wrbq create failed."); |
| kfree(pwrb_arr); |
| return status; |
| } |
| phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i]. |
| id; |
| } |
| kfree(pwrb_arr); |
| return 0; |
| } |
| |
| static void free_wrb_handles(struct beiscsi_hba *phba) |
| { |
| unsigned int index; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_wrb_context *pwrb_context; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) { |
| pwrb_context = &phwi_ctrlr->wrb_context[index]; |
| kfree(pwrb_context->pwrb_handle_base); |
| kfree(pwrb_context->pwrb_handle_basestd); |
| } |
| } |
| |
| static void be_mcc_queues_destroy(struct beiscsi_hba *phba) |
| { |
| struct be_queue_info *q; |
| struct be_ctrl_info *ctrl = &phba->ctrl; |
| |
| q = &phba->ctrl.mcc_obj.q; |
| if (q->created) |
| beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ); |
| be_queue_free(phba, q); |
| |
| q = &phba->ctrl.mcc_obj.cq; |
| if (q->created) |
| beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); |
| be_queue_free(phba, q); |
| } |
| |
| static void hwi_cleanup(struct beiscsi_hba *phba) |
| { |
| struct be_queue_info *q; |
| struct be_ctrl_info *ctrl = &phba->ctrl; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| int i, eq_num; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| for (i = 0; i < phba->params.cxns_per_ctrl; i++) { |
| q = &phwi_context->be_wrbq[i]; |
| if (q->created) |
| beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ); |
| } |
| free_wrb_handles(phba); |
| |
| q = &phwi_context->be_def_hdrq; |
| if (q->created) |
| beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); |
| |
| q = &phwi_context->be_def_dataq; |
| if (q->created) |
| beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ); |
| |
| beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL); |
| |
| for (i = 0; i < (phba->num_cpus); i++) { |
| q = &phwi_context->be_cq[i]; |
| if (q->created) |
| beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ); |
| } |
| if (phba->msix_enabled) |
| eq_num = 1; |
| else |
| eq_num = 0; |
| for (i = 0; i < (phba->num_cpus + eq_num); i++) { |
| q = &phwi_context->be_eq[i].q; |
| if (q->created) |
| beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ); |
| } |
| be_mcc_queues_destroy(phba); |
| } |
| |
| static int be_mcc_queues_create(struct beiscsi_hba *phba, |
| struct hwi_context_memory *phwi_context) |
| { |
| struct be_queue_info *q, *cq; |
| struct be_ctrl_info *ctrl = &phba->ctrl; |
| |
| /* Alloc MCC compl queue */ |
| cq = &phba->ctrl.mcc_obj.cq; |
| if (be_queue_alloc(phba, cq, MCC_CQ_LEN, |
| sizeof(struct be_mcc_compl))) |
| goto err; |
| /* Ask BE to create MCC compl queue; */ |
| if (phba->msix_enabled) { |
| if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq |
| [phba->num_cpus].q, false, true, 0)) |
| goto mcc_cq_free; |
| } else { |
| if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q, |
| false, true, 0)) |
| goto mcc_cq_free; |
| } |
| |
| /* Alloc MCC queue */ |
| q = &phba->ctrl.mcc_obj.q; |
| if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) |
| goto mcc_cq_destroy; |
| |
| /* Ask BE to create MCC queue */ |
| if (beiscsi_cmd_mccq_create(phba, q, cq)) |
| goto mcc_q_free; |
| |
| return 0; |
| |
| mcc_q_free: |
| be_queue_free(phba, q); |
| mcc_cq_destroy: |
| beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ); |
| mcc_cq_free: |
| be_queue_free(phba, cq); |
| err: |
| return -ENOMEM; |
| } |
| |
| static int find_num_cpus(void) |
| { |
| int num_cpus = 0; |
| |
| num_cpus = num_online_cpus(); |
| if (num_cpus >= MAX_CPUS) |
| num_cpus = MAX_CPUS - 1; |
| |
| SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus); |
| return num_cpus; |
| } |
| |
| static int hwi_init_port(struct beiscsi_hba *phba) |
| { |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| unsigned int def_pdu_ring_sz; |
| struct be_ctrl_info *ctrl = &phba->ctrl; |
| int status; |
| |
| def_pdu_ring_sz = |
| phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr); |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| phwi_context->max_eqd = 0; |
| phwi_context->min_eqd = 0; |
| phwi_context->cur_eqd = 64; |
| be_cmd_fw_initialize(&phba->ctrl); |
| |
| status = beiscsi_create_eqs(phba, phwi_context); |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, "EQ not created\n"); |
| goto error; |
| } |
| |
| status = be_mcc_queues_create(phba, phwi_context); |
| if (status != 0) |
| goto error; |
| |
| status = mgmt_check_supported_fw(ctrl, phba); |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Unsupported fw version\n"); |
| goto error; |
| } |
| |
| status = beiscsi_create_cqs(phba, phwi_context); |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, "CQ not created\n"); |
| goto error; |
| } |
| |
| status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr, |
| def_pdu_ring_sz); |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Default Header not created\n"); |
| goto error; |
| } |
| |
| status = beiscsi_create_def_data(phba, phwi_context, |
| phwi_ctrlr, def_pdu_ring_sz); |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Default Data not created\n"); |
| goto error; |
| } |
| |
| status = beiscsi_post_pages(phba); |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n"); |
| goto error; |
| } |
| |
| status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr); |
| if (status != 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "WRB Rings not created\n"); |
| goto error; |
| } |
| |
| SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n"); |
| return 0; |
| |
| error: |
| shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed"); |
| hwi_cleanup(phba); |
| return -ENOMEM; |
| } |
| |
| static int hwi_init_controller(struct beiscsi_hba *phba) |
| { |
| struct hwi_controller *phwi_ctrlr; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) { |
| phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba-> |
| init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address; |
| SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n", |
| phwi_ctrlr->phwi_ctxt); |
| } else { |
| shost_printk(KERN_ERR, phba->shost, |
| "HWI_MEM_ADDN_CONTEXT is more than one element." |
| "Failing to load\n"); |
| return -ENOMEM; |
| } |
| |
| iscsi_init_global_templates(phba); |
| beiscsi_init_wrb_handle(phba); |
| hwi_init_async_pdu_ctx(phba); |
| if (hwi_init_port(phba) != 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "hwi_init_controller failed\n"); |
| return -ENOMEM; |
| } |
| return 0; |
| } |
| |
| static void beiscsi_free_mem(struct beiscsi_hba *phba) |
| { |
| struct be_mem_descriptor *mem_descr; |
| int i, j; |
| |
| mem_descr = phba->init_mem; |
| i = 0; |
| j = 0; |
| for (i = 0; i < SE_MEM_MAX; i++) { |
| for (j = mem_descr->num_elements; j > 0; j--) { |
| pci_free_consistent(phba->pcidev, |
| mem_descr->mem_array[j - 1].size, |
| mem_descr->mem_array[j - 1].virtual_address, |
| (unsigned long)mem_descr->mem_array[j - 1]. |
| bus_address.u.a64.address); |
| } |
| kfree(mem_descr->mem_array); |
| mem_descr++; |
| } |
| kfree(phba->init_mem); |
| kfree(phba->phwi_ctrlr); |
| } |
| |
| static int beiscsi_init_controller(struct beiscsi_hba *phba) |
| { |
| int ret = -ENOMEM; |
| |
| ret = beiscsi_get_memory(phba); |
| if (ret < 0) { |
| shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -" |
| "Failed in beiscsi_alloc_memory\n"); |
| return ret; |
| } |
| |
| ret = hwi_init_controller(phba); |
| if (ret) |
| goto free_init; |
| SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller"); |
| return 0; |
| |
| free_init: |
| beiscsi_free_mem(phba); |
| return -ENOMEM; |
| } |
| |
| static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba) |
| { |
| struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg; |
| struct sgl_handle *psgl_handle; |
| struct iscsi_sge *pfrag; |
| unsigned int arr_index, i, idx; |
| |
| phba->io_sgl_hndl_avbl = 0; |
| phba->eh_sgl_hndl_avbl = 0; |
| |
| mem_descr_sglh = phba->init_mem; |
| mem_descr_sglh += HWI_MEM_SGLH; |
| if (1 == mem_descr_sglh->num_elements) { |
| phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * |
| phba->params.ios_per_ctrl, |
| GFP_KERNEL); |
| if (!phba->io_sgl_hndl_base) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Mem Alloc Failed. Failing to load\n"); |
| return -ENOMEM; |
| } |
| phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) * |
| (phba->params.icds_per_ctrl - |
| phba->params.ios_per_ctrl), |
| GFP_KERNEL); |
| if (!phba->eh_sgl_hndl_base) { |
| kfree(phba->io_sgl_hndl_base); |
| shost_printk(KERN_ERR, phba->shost, |
| "Mem Alloc Failed. Failing to load\n"); |
| return -ENOMEM; |
| } |
| } else { |
| shost_printk(KERN_ERR, phba->shost, |
| "HWI_MEM_SGLH is more than one element." |
| "Failing to load\n"); |
| return -ENOMEM; |
| } |
| |
| arr_index = 0; |
| idx = 0; |
| while (idx < mem_descr_sglh->num_elements) { |
| psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address; |
| |
| for (i = 0; i < (mem_descr_sglh->mem_array[idx].size / |
| sizeof(struct sgl_handle)); i++) { |
| if (arr_index < phba->params.ios_per_ctrl) { |
| phba->io_sgl_hndl_base[arr_index] = psgl_handle; |
| phba->io_sgl_hndl_avbl++; |
| arr_index++; |
| } else { |
| phba->eh_sgl_hndl_base[arr_index - |
| phba->params.ios_per_ctrl] = |
| psgl_handle; |
| arr_index++; |
| phba->eh_sgl_hndl_avbl++; |
| } |
| psgl_handle++; |
| } |
| idx++; |
| } |
| SE_DEBUG(DBG_LVL_8, |
| "phba->io_sgl_hndl_avbl=%d" |
| "phba->eh_sgl_hndl_avbl=%d\n", |
| phba->io_sgl_hndl_avbl, |
| phba->eh_sgl_hndl_avbl); |
| mem_descr_sg = phba->init_mem; |
| mem_descr_sg += HWI_MEM_SGE; |
| SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n", |
| mem_descr_sg->num_elements); |
| arr_index = 0; |
| idx = 0; |
| while (idx < mem_descr_sg->num_elements) { |
| pfrag = mem_descr_sg->mem_array[idx].virtual_address; |
| |
| for (i = 0; |
| i < (mem_descr_sg->mem_array[idx].size) / |
| (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io); |
| i++) { |
| if (arr_index < phba->params.ios_per_ctrl) |
| psgl_handle = phba->io_sgl_hndl_base[arr_index]; |
| else |
| psgl_handle = phba->eh_sgl_hndl_base[arr_index - |
| phba->params.ios_per_ctrl]; |
| psgl_handle->pfrag = pfrag; |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0); |
| AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0); |
| pfrag += phba->params.num_sge_per_io; |
| psgl_handle->sgl_index = |
| phba->fw_config.iscsi_icd_start + arr_index++; |
| } |
| idx++; |
| } |
| phba->io_sgl_free_index = 0; |
| phba->io_sgl_alloc_index = 0; |
| phba->eh_sgl_free_index = 0; |
| phba->eh_sgl_alloc_index = 0; |
| return 0; |
| } |
| |
| static int hba_setup_cid_tbls(struct beiscsi_hba *phba) |
| { |
| int i, new_cid; |
| |
| phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl, |
| GFP_KERNEL); |
| if (!phba->cid_array) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Failed to allocate memory in " |
| "hba_setup_cid_tbls\n"); |
| return -ENOMEM; |
| } |
| phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) * |
| phba->params.cxns_per_ctrl * 2, GFP_KERNEL); |
| if (!phba->ep_array) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Failed to allocate memory in " |
| "hba_setup_cid_tbls\n"); |
| kfree(phba->cid_array); |
| return -ENOMEM; |
| } |
| new_cid = phba->fw_config.iscsi_cid_start; |
| for (i = 0; i < phba->params.cxns_per_ctrl; i++) { |
| phba->cid_array[i] = new_cid; |
| new_cid += 2; |
| } |
| phba->avlbl_cids = phba->params.cxns_per_ctrl; |
| return 0; |
| } |
| |
| static void hwi_enable_intr(struct beiscsi_hba *phba) |
| { |
| struct be_ctrl_info *ctrl = &phba->ctrl; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| struct be_queue_info *eq; |
| u8 __iomem *addr; |
| u32 reg, i; |
| u32 enabled; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| |
| addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg + |
| PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET); |
| reg = ioread32(addr); |
| SE_DEBUG(DBG_LVL_8, "reg =x%08x\n", reg); |
| |
| enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; |
| if (!enabled) { |
| reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; |
| SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr); |
| iowrite32(reg, addr); |
| if (!phba->msix_enabled) { |
| eq = &phwi_context->be_eq[0].q; |
| SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id); |
| hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); |
| } else { |
| for (i = 0; i <= phba->num_cpus; i++) { |
| eq = &phwi_context->be_eq[i].q; |
| SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id); |
| hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); |
| } |
| } |
| } |
| } |
| |
| static void hwi_disable_intr(struct beiscsi_hba *phba) |
| { |
| struct be_ctrl_info *ctrl = &phba->ctrl; |
| |
| u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET; |
| u32 reg = ioread32(addr); |
| |
| u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; |
| if (enabled) { |
| reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; |
| iowrite32(reg, addr); |
| } else |
| shost_printk(KERN_WARNING, phba->shost, |
| "In hwi_disable_intr, Already Disabled\n"); |
| } |
| |
| static int beiscsi_get_boot_info(struct beiscsi_hba *phba) |
| { |
| struct be_cmd_resp_get_boot_target *boot_resp; |
| struct be_cmd_resp_get_session *session_resp; |
| struct be_mcc_wrb *wrb; |
| struct be_dma_mem nonemb_cmd; |
| unsigned int tag, wrb_num; |
| unsigned short status, extd_status; |
| struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q; |
| |
| tag = beiscsi_get_boot_target(phba); |
| if (!tag) { |
| SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed\n"); |
| return -EAGAIN; |
| } else |
| wait_event_interruptible(phba->ctrl.mcc_wait[tag], |
| phba->ctrl.mcc_numtag[tag]); |
| |
| wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16; |
| extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8; |
| status = phba->ctrl.mcc_numtag[tag] & 0x000000FF; |
| if (status || extd_status) { |
| SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed" |
| " status = %d extd_status = %d\n", |
| status, extd_status); |
| free_mcc_tag(&phba->ctrl, tag); |
| return -EBUSY; |
| } |
| wrb = queue_get_wrb(mccq, wrb_num); |
| free_mcc_tag(&phba->ctrl, tag); |
| boot_resp = embedded_payload(wrb); |
| |
| if (boot_resp->boot_session_handle < 0) { |
| printk(KERN_ERR "No Boot Session for this pci_func," |
| "session Hndl = %d\n", boot_resp->boot_session_handle); |
| return -ENXIO; |
| } |
| |
| nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev, |
| sizeof(*session_resp), |
| &nonemb_cmd.dma); |
| if (nonemb_cmd.va == NULL) { |
| SE_DEBUG(DBG_LVL_1, |
| "Failed to allocate memory for" |
| "beiscsi_get_session_info\n"); |
| return -ENOMEM; |
| } |
| |
| memset(nonemb_cmd.va, 0, sizeof(*session_resp)); |
| tag = beiscsi_get_session_info(phba, |
| boot_resp->boot_session_handle, &nonemb_cmd); |
| if (!tag) { |
| SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info" |
| " Failed\n"); |
| goto boot_freemem; |
| } else |
| wait_event_interruptible(phba->ctrl.mcc_wait[tag], |
| phba->ctrl.mcc_numtag[tag]); |
| |
| wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16; |
| extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8; |
| status = phba->ctrl.mcc_numtag[tag] & 0x000000FF; |
| if (status || extd_status) { |
| SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info Failed" |
| " status = %d extd_status = %d\n", |
| status, extd_status); |
| free_mcc_tag(&phba->ctrl, tag); |
| goto boot_freemem; |
| } |
| wrb = queue_get_wrb(mccq, wrb_num); |
| free_mcc_tag(&phba->ctrl, tag); |
| session_resp = nonemb_cmd.va ; |
| memcpy(&phba->boot_sess, &session_resp->session_info, |
| sizeof(struct mgmt_session_info)); |
| pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, |
| nonemb_cmd.va, nonemb_cmd.dma); |
| return 0; |
| boot_freemem: |
| pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size, |
| nonemb_cmd.va, nonemb_cmd.dma); |
| return -ENOMEM; |
| } |
| |
| static int beiscsi_init_port(struct beiscsi_hba *phba) |
| { |
| int ret; |
| |
| ret = beiscsi_init_controller(phba); |
| if (ret < 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "beiscsi_dev_probe - Failed in" |
| "beiscsi_init_controller\n"); |
| return ret; |
| } |
| ret = beiscsi_init_sgl_handle(phba); |
| if (ret < 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "beiscsi_dev_probe - Failed in" |
| "beiscsi_init_sgl_handle\n"); |
| goto do_cleanup_ctrlr; |
| } |
| |
| if (hba_setup_cid_tbls(phba)) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Failed in hba_setup_cid_tbls\n"); |
| kfree(phba->io_sgl_hndl_base); |
| kfree(phba->eh_sgl_hndl_base); |
| goto do_cleanup_ctrlr; |
| } |
| |
| return ret; |
| |
| do_cleanup_ctrlr: |
| hwi_cleanup(phba); |
| return ret; |
| } |
| |
| static void hwi_purge_eq(struct beiscsi_hba *phba) |
| { |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| struct be_queue_info *eq; |
| struct be_eq_entry *eqe = NULL; |
| int i, eq_msix; |
| unsigned int num_processed; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| if (phba->msix_enabled) |
| eq_msix = 1; |
| else |
| eq_msix = 0; |
| |
| for (i = 0; i < (phba->num_cpus + eq_msix); i++) { |
| eq = &phwi_context->be_eq[i].q; |
| eqe = queue_tail_node(eq); |
| num_processed = 0; |
| while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] |
| & EQE_VALID_MASK) { |
| AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); |
| queue_tail_inc(eq); |
| eqe = queue_tail_node(eq); |
| num_processed++; |
| } |
| |
| if (num_processed) |
| hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1); |
| } |
| } |
| |
| static void beiscsi_clean_port(struct beiscsi_hba *phba) |
| { |
| int mgmt_status; |
| |
| mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0); |
| if (mgmt_status) |
| shost_printk(KERN_WARNING, phba->shost, |
| "mgmt_epfw_cleanup FAILED\n"); |
| |
| hwi_purge_eq(phba); |
| hwi_cleanup(phba); |
| kfree(phba->io_sgl_hndl_base); |
| kfree(phba->eh_sgl_hndl_base); |
| kfree(phba->cid_array); |
| kfree(phba->ep_array); |
| } |
| |
| void |
| beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn, |
| struct beiscsi_offload_params *params) |
| { |
| struct wrb_handle *pwrb_handle; |
| struct iscsi_target_context_update_wrb *pwrb = NULL; |
| struct be_mem_descriptor *mem_descr; |
| struct beiscsi_hba *phba = beiscsi_conn->phba; |
| u32 doorbell = 0; |
| |
| /* |
| * We can always use 0 here because it is reserved by libiscsi for |
| * login/startup related tasks. |
| */ |
| pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid - |
| phba->fw_config.iscsi_cid_start)); |
| pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb; |
| memset(pwrb, 0, sizeof(*pwrb)); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, |
| max_burst_length, pwrb, params->dw[offsetof |
| (struct amap_beiscsi_offload_params, |
| max_burst_length) / 32]); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, |
| max_send_data_segment_length, pwrb, |
| params->dw[offsetof(struct amap_beiscsi_offload_params, |
| max_send_data_segment_length) / 32]); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, |
| first_burst_length, |
| pwrb, |
| params->dw[offsetof(struct amap_beiscsi_offload_params, |
| first_burst_length) / 32]); |
| |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb, |
| (params->dw[offsetof(struct amap_beiscsi_offload_params, |
| erl) / 32] & OFFLD_PARAMS_ERL)); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb, |
| (params->dw[offsetof(struct amap_beiscsi_offload_params, |
| dde) / 32] & OFFLD_PARAMS_DDE) >> 2); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb, |
| (params->dw[offsetof(struct amap_beiscsi_offload_params, |
| hde) / 32] & OFFLD_PARAMS_HDE) >> 3); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb, |
| (params->dw[offsetof(struct amap_beiscsi_offload_params, |
| ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb, |
| (params->dw[offsetof(struct amap_beiscsi_offload_params, |
| imd) / 32] & OFFLD_PARAMS_IMD) >> 5); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn, |
| pwrb, |
| (params->dw[offsetof(struct amap_beiscsi_offload_params, |
| exp_statsn) / 32] + 1)); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb, |
| 0x7); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx, |
| pwrb, pwrb_handle->wrb_index); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb, |
| pwrb, pwrb_handle->nxt_wrb_index); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, |
| session_state, pwrb, 0); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack, |
| pwrb, 1); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq, |
| pwrb, 0); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb, |
| 0); |
| |
| mem_descr = phba->init_mem; |
| mem_descr += ISCSI_MEM_GLOBAL_HEADER; |
| |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, |
| pad_buffer_addr_hi, pwrb, |
| mem_descr->mem_array[0].bus_address.u.a32.address_hi); |
| AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, |
| pad_buffer_addr_lo, pwrb, |
| mem_descr->mem_array[0].bus_address.u.a32.address_lo); |
| |
| be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb)); |
| |
| doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; |
| doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK) |
| << DB_DEF_PDU_WRB_INDEX_SHIFT; |
| doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; |
| |
| iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); |
| } |
| |
| static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt, |
| int *index, int *age) |
| { |
| *index = (int)itt; |
| if (age) |
| *age = conn->session->age; |
| } |
| |
| /** |
| * beiscsi_alloc_pdu - allocates pdu and related resources |
| * @task: libiscsi task |
| * @opcode: opcode of pdu for task |
| * |
| * This is called with the session lock held. It will allocate |
| * the wrb and sgl if needed for the command. And it will prep |
| * the pdu's itt. beiscsi_parse_pdu will later translate |
| * the pdu itt to the libiscsi task itt. |
| */ |
| static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode) |
| { |
| struct beiscsi_io_task *io_task = task->dd_data; |
| struct iscsi_conn *conn = task->conn; |
| struct beiscsi_conn *beiscsi_conn = conn->dd_data; |
| struct beiscsi_hba *phba = beiscsi_conn->phba; |
| struct hwi_wrb_context *pwrb_context; |
| struct hwi_controller *phwi_ctrlr; |
| itt_t itt; |
| struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; |
| dma_addr_t paddr; |
| |
| io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool, |
| GFP_ATOMIC, &paddr); |
| if (!io_task->cmd_bhs) |
| return -ENOMEM; |
| io_task->bhs_pa.u.a64.address = paddr; |
| io_task->libiscsi_itt = (itt_t)task->itt; |
| io_task->conn = beiscsi_conn; |
| |
| task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr; |
| task->hdr_max = sizeof(struct be_cmd_bhs); |
| io_task->psgl_handle = NULL; |
| io_task->psgl_handle = NULL; |
| |
| if (task->sc) { |
| spin_lock(&phba->io_sgl_lock); |
| io_task->psgl_handle = alloc_io_sgl_handle(phba); |
| spin_unlock(&phba->io_sgl_lock); |
| if (!io_task->psgl_handle) |
| goto free_hndls; |
| io_task->pwrb_handle = alloc_wrb_handle(phba, |
| beiscsi_conn->beiscsi_conn_cid - |
| phba->fw_config.iscsi_cid_start); |
| if (!io_task->pwrb_handle) |
| goto free_io_hndls; |
| } else { |
| io_task->scsi_cmnd = NULL; |
| if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) { |
| if (!beiscsi_conn->login_in_progress) { |
| spin_lock(&phba->mgmt_sgl_lock); |
| io_task->psgl_handle = (struct sgl_handle *) |
| alloc_mgmt_sgl_handle(phba); |
| spin_unlock(&phba->mgmt_sgl_lock); |
| if (!io_task->psgl_handle) |
| goto free_hndls; |
| |
| beiscsi_conn->login_in_progress = 1; |
| beiscsi_conn->plogin_sgl_handle = |
| io_task->psgl_handle; |
| io_task->pwrb_handle = |
| alloc_wrb_handle(phba, |
| beiscsi_conn->beiscsi_conn_cid - |
| phba->fw_config.iscsi_cid_start); |
| if (!io_task->pwrb_handle) |
| goto free_io_hndls; |
| beiscsi_conn->plogin_wrb_handle = |
| io_task->pwrb_handle; |
| |
| } else { |
| io_task->psgl_handle = |
| beiscsi_conn->plogin_sgl_handle; |
| io_task->pwrb_handle = |
| beiscsi_conn->plogin_wrb_handle; |
| } |
| } else { |
| spin_lock(&phba->mgmt_sgl_lock); |
| io_task->psgl_handle = alloc_mgmt_sgl_handle(phba); |
| spin_unlock(&phba->mgmt_sgl_lock); |
| if (!io_task->psgl_handle) |
| goto free_hndls; |
| io_task->pwrb_handle = |
| alloc_wrb_handle(phba, |
| beiscsi_conn->beiscsi_conn_cid - |
| phba->fw_config.iscsi_cid_start); |
| if (!io_task->pwrb_handle) |
| goto free_mgmt_hndls; |
| |
| } |
| } |
| itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle-> |
| wrb_index << 16) | (unsigned int) |
| (io_task->psgl_handle->sgl_index)); |
| io_task->pwrb_handle->pio_handle = task; |
| |
| io_task->cmd_bhs->iscsi_hdr.itt = itt; |
| return 0; |
| |
| free_io_hndls: |
| spin_lock(&phba->io_sgl_lock); |
| free_io_sgl_handle(phba, io_task->psgl_handle); |
| spin_unlock(&phba->io_sgl_lock); |
| goto free_hndls; |
| free_mgmt_hndls: |
| spin_lock(&phba->mgmt_sgl_lock); |
| free_mgmt_sgl_handle(phba, io_task->psgl_handle); |
| spin_unlock(&phba->mgmt_sgl_lock); |
| free_hndls: |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pwrb_context = &phwi_ctrlr->wrb_context[ |
| beiscsi_conn->beiscsi_conn_cid - |
| phba->fw_config.iscsi_cid_start]; |
| if (io_task->pwrb_handle) |
| free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle); |
| io_task->pwrb_handle = NULL; |
| pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, |
| io_task->bhs_pa.u.a64.address); |
| SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n"); |
| return -ENOMEM; |
| } |
| |
| static void beiscsi_cleanup_task(struct iscsi_task *task) |
| { |
| struct beiscsi_io_task *io_task = task->dd_data; |
| struct iscsi_conn *conn = task->conn; |
| struct beiscsi_conn *beiscsi_conn = conn->dd_data; |
| struct beiscsi_hba *phba = beiscsi_conn->phba; |
| struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess; |
| struct hwi_wrb_context *pwrb_context; |
| struct hwi_controller *phwi_ctrlr; |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid |
| - phba->fw_config.iscsi_cid_start]; |
| if (io_task->pwrb_handle) { |
| free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle); |
| io_task->pwrb_handle = NULL; |
| } |
| |
| if (io_task->cmd_bhs) { |
| pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs, |
| io_task->bhs_pa.u.a64.address); |
| } |
| |
| if (task->sc) { |
| if (io_task->psgl_handle) { |
| spin_lock(&phba->io_sgl_lock); |
| free_io_sgl_handle(phba, io_task->psgl_handle); |
| spin_unlock(&phba->io_sgl_lock); |
| io_task->psgl_handle = NULL; |
| } |
| } else { |
| if (task->hdr && |
| ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN)) |
| return; |
| if (io_task->psgl_handle) { |
| spin_lock(&phba->mgmt_sgl_lock); |
| free_mgmt_sgl_handle(phba, io_task->psgl_handle); |
| spin_unlock(&phba->mgmt_sgl_lock); |
| io_task->psgl_handle = NULL; |
| } |
| } |
| } |
| |
| static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg, |
| unsigned int num_sg, unsigned int xferlen, |
| unsigned int writedir) |
| { |
| |
| struct beiscsi_io_task *io_task = task->dd_data; |
| struct iscsi_conn *conn = task->conn; |
| struct beiscsi_conn *beiscsi_conn = conn->dd_data; |
| struct beiscsi_hba *phba = beiscsi_conn->phba; |
| struct iscsi_wrb *pwrb = NULL; |
| unsigned int doorbell = 0; |
| |
| pwrb = io_task->pwrb_handle->pwrb; |
| io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0; |
| io_task->bhs_len = sizeof(struct be_cmd_bhs); |
| |
| if (writedir) { |
| memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48); |
| AMAP_SET_BITS(struct amap_pdu_data_out, itt, |
| &io_task->cmd_bhs->iscsi_data_pdu, |
| (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt); |
| AMAP_SET_BITS(struct amap_pdu_data_out, opcode, |
| &io_task->cmd_bhs->iscsi_data_pdu, |
| ISCSI_OPCODE_SCSI_DATA_OUT); |
| AMAP_SET_BITS(struct amap_pdu_data_out, final_bit, |
| &io_task->cmd_bhs->iscsi_data_pdu, 1); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, |
| INI_WR_CMD); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); |
| } else { |
| AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, |
| INI_RD_CMD); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); |
| } |
| memcpy(&io_task->cmd_bhs->iscsi_data_pdu. |
| dw[offsetof(struct amap_pdu_data_out, lun) / 32], |
| io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun)); |
| |
| AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb, |
| cpu_to_be16((unsigned short)io_task->cmd_bhs->iscsi_hdr. |
| lun[0])); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, |
| io_task->pwrb_handle->wrb_index); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, |
| be32_to_cpu(task->cmdsn)); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, |
| io_task->psgl_handle->sgl_index); |
| |
| hwi_write_sgl(pwrb, sg, num_sg, io_task); |
| |
| AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, |
| io_task->pwrb_handle->nxt_wrb_index); |
| be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); |
| |
| doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK; |
| doorbell |= (io_task->pwrb_handle->wrb_index & |
| DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; |
| doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; |
| |
| iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); |
| return 0; |
| } |
| |
| static int beiscsi_mtask(struct iscsi_task *task) |
| { |
| struct beiscsi_io_task *io_task = task->dd_data; |
| struct iscsi_conn *conn = task->conn; |
| struct beiscsi_conn *beiscsi_conn = conn->dd_data; |
| struct beiscsi_hba *phba = beiscsi_conn->phba; |
| struct iscsi_wrb *pwrb = NULL; |
| unsigned int doorbell = 0; |
| unsigned int cid; |
| |
| cid = beiscsi_conn->beiscsi_conn_cid; |
| pwrb = io_task->pwrb_handle->pwrb; |
| memset(pwrb, 0, sizeof(*pwrb)); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, |
| be32_to_cpu(task->cmdsn)); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb, |
| io_task->pwrb_handle->wrb_index); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb, |
| io_task->psgl_handle->sgl_index); |
| |
| switch (task->hdr->opcode & ISCSI_OPCODE_MASK) { |
| case ISCSI_OP_LOGIN: |
| AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, |
| TGT_DM_CMD); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1); |
| hwi_write_buffer(pwrb, task); |
| break; |
| case ISCSI_OP_NOOP_OUT: |
| AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, |
| INI_RD_CMD); |
| if (task->hdr->ttt == ISCSI_RESERVED_TAG) |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); |
| else |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1); |
| hwi_write_buffer(pwrb, task); |
| break; |
| case ISCSI_OP_TEXT: |
| AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, |
| TGT_DM_CMD); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); |
| hwi_write_buffer(pwrb, task); |
| break; |
| case ISCSI_OP_SCSI_TMFUNC: |
| AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, |
| INI_TMF_CMD); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); |
| hwi_write_buffer(pwrb, task); |
| break; |
| case ISCSI_OP_LOGOUT: |
| AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, |
| HWH_TYPE_LOGOUT); |
| hwi_write_buffer(pwrb, task); |
| break; |
| |
| default: |
| SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n", |
| task->hdr->opcode & ISCSI_OPCODE_MASK); |
| return -EINVAL; |
| } |
| |
| AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, |
| task->data_count); |
| AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb, |
| io_task->pwrb_handle->nxt_wrb_index); |
| be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb)); |
| |
| doorbell |= cid & DB_WRB_POST_CID_MASK; |
| doorbell |= (io_task->pwrb_handle->wrb_index & |
| DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT; |
| doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT; |
| iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET); |
| return 0; |
| } |
| |
| static int beiscsi_task_xmit(struct iscsi_task *task) |
| { |
| struct beiscsi_io_task *io_task = task->dd_data; |
| struct scsi_cmnd *sc = task->sc; |
| struct scatterlist *sg; |
| int num_sg; |
| unsigned int writedir = 0, xferlen = 0; |
| |
| if (!sc) |
| return beiscsi_mtask(task); |
| |
| io_task->scsi_cmnd = sc; |
| num_sg = scsi_dma_map(sc); |
| if (num_sg < 0) { |
| SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n") |
| return num_sg; |
| } |
| xferlen = scsi_bufflen(sc); |
| sg = scsi_sglist(sc); |
| if (sc->sc_data_direction == DMA_TO_DEVICE) { |
| writedir = 1; |
| SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n", |
| task->imm_count); |
| } else |
| writedir = 0; |
| return beiscsi_iotask(task, sg, num_sg, xferlen, writedir); |
| } |
| |
| static void beiscsi_remove(struct pci_dev *pcidev) |
| { |
| struct beiscsi_hba *phba = NULL; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| struct be_eq_obj *pbe_eq; |
| unsigned int i, msix_vec; |
| u8 *real_offset = 0; |
| u32 value = 0; |
| |
| phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev); |
| if (!phba) { |
| dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n"); |
| return; |
| } |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| hwi_disable_intr(phba); |
| if (phba->msix_enabled) { |
| for (i = 0; i <= phba->num_cpus; i++) { |
| msix_vec = phba->msix_entries[i].vector; |
| free_irq(msix_vec, &phwi_context->be_eq[i]); |
| } |
| } else |
| if (phba->pcidev->irq) |
| free_irq(phba->pcidev->irq, phba); |
| pci_disable_msix(phba->pcidev); |
| destroy_workqueue(phba->wq); |
| if (blk_iopoll_enabled) |
| for (i = 0; i < phba->num_cpus; i++) { |
| pbe_eq = &phwi_context->be_eq[i]; |
| blk_iopoll_disable(&pbe_eq->iopoll); |
| } |
| |
| beiscsi_clean_port(phba); |
| beiscsi_free_mem(phba); |
| real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE; |
| |
| value = readl((void *)real_offset); |
| |
| if (value & 0x00010000) { |
| value &= 0xfffeffff; |
| writel(value, (void *)real_offset); |
| } |
| beiscsi_unmap_pci_function(phba); |
| pci_free_consistent(phba->pcidev, |
| phba->ctrl.mbox_mem_alloced.size, |
| phba->ctrl.mbox_mem_alloced.va, |
| phba->ctrl.mbox_mem_alloced.dma); |
| iscsi_host_remove(phba->shost); |
| pci_dev_put(phba->pcidev); |
| iscsi_host_free(phba->shost); |
| iscsi_boot_destroy_kset(phba->boot_kset); |
| } |
| |
| static void beiscsi_msix_enable(struct beiscsi_hba *phba) |
| { |
| int i, status; |
| |
| for (i = 0; i <= phba->num_cpus; i++) |
| phba->msix_entries[i].entry = i; |
| |
| status = pci_enable_msix(phba->pcidev, phba->msix_entries, |
| (phba->num_cpus + 1)); |
| if (!status) |
| phba->msix_enabled = true; |
| |
| return; |
| } |
| |
| static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev, |
| const struct pci_device_id *id) |
| { |
| struct beiscsi_hba *phba = NULL; |
| struct hwi_controller *phwi_ctrlr; |
| struct hwi_context_memory *phwi_context; |
| struct be_eq_obj *pbe_eq; |
| int ret, num_cpus, i; |
| u8 *real_offset = 0; |
| u32 value = 0; |
| |
| ret = beiscsi_enable_pci(pcidev); |
| if (ret < 0) { |
| dev_err(&pcidev->dev, "beiscsi_dev_probe-" |
| " Failed to enable pci device\n"); |
| return ret; |
| } |
| |
| phba = beiscsi_hba_alloc(pcidev); |
| if (!phba) { |
| dev_err(&pcidev->dev, "beiscsi_dev_probe-" |
| " Failed in beiscsi_hba_alloc\n"); |
| goto disable_pci; |
| } |
| |
| switch (pcidev->device) { |
| case BE_DEVICE_ID1: |
| case OC_DEVICE_ID1: |
| case OC_DEVICE_ID2: |
| phba->generation = BE_GEN2; |
| break; |
| case BE_DEVICE_ID2: |
| case OC_DEVICE_ID3: |
| phba->generation = BE_GEN3; |
| break; |
| default: |
| phba->generation = 0; |
| } |
| |
| if (enable_msix) |
| num_cpus = find_num_cpus(); |
| else |
| num_cpus = 1; |
| phba->num_cpus = num_cpus; |
| SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus); |
| |
| if (enable_msix) |
| beiscsi_msix_enable(phba); |
| ret = be_ctrl_init(phba, pcidev); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" |
| "Failed in be_ctrl_init\n"); |
| goto hba_free; |
| } |
| |
| if (!num_hba) { |
| real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE; |
| value = readl((void *)real_offset); |
| if (value & 0x00010000) { |
| gcrashmode++; |
| shost_printk(KERN_ERR, phba->shost, |
| "Loading Driver in crashdump mode\n"); |
| ret = beiscsi_pci_soft_reset(phba); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Reset Failed. Aborting Crashdump\n"); |
| goto hba_free; |
| } |
| ret = be_chk_reset_complete(phba); |
| if (ret) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Failed to get out of reset." |
| "Aborting Crashdump\n"); |
| goto hba_free; |
| } |
| } else { |
| value |= 0x00010000; |
| writel(value, (void *)real_offset); |
| num_hba++; |
| } |
| } |
| |
| spin_lock_init(&phba->io_sgl_lock); |
| spin_lock_init(&phba->mgmt_sgl_lock); |
| spin_lock_init(&phba->isr_lock); |
| ret = mgmt_get_fw_config(&phba->ctrl, phba); |
| if (ret != 0) { |
| shost_printk(KERN_ERR, phba->shost, |
| "Error getting fw config\n"); |
| goto free_port; |
| } |
| phba->shost->max_id = phba->fw_config.iscsi_cid_count; |
| beiscsi_get_params(phba); |
| phba->shost->can_queue = phba->params.ios_per_ctrl; |
| ret = beiscsi_init_port(phba); |
| if (ret < 0) { |
| shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" |
| "Failed in beiscsi_init_port\n"); |
| goto free_port; |
| } |
| |
| for (i = 0; i < MAX_MCC_CMD ; i++) { |
| init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]); |
| phba->ctrl.mcc_tag[i] = i + 1; |
| phba->ctrl.mcc_numtag[i + 1] = 0; |
| phba->ctrl.mcc_tag_available++; |
| } |
| |
| phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0; |
| |
| snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u", |
| phba->shost->host_no); |
| phba->wq = create_workqueue(phba->wq_name); |
| if (!phba->wq) { |
| shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" |
| "Failed to allocate work queue\n"); |
| goto free_twq; |
| } |
| |
| INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs); |
| |
| phwi_ctrlr = phba->phwi_ctrlr; |
| phwi_context = phwi_ctrlr->phwi_ctxt; |
| if (blk_iopoll_enabled) { |
| for (i = 0; i < phba->num_cpus; i++) { |
| pbe_eq = &phwi_context->be_eq[i]; |
| blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget, |
| be_iopoll); |
| blk_iopoll_enable(&pbe_eq->iopoll); |
| } |
| } |
| ret = beiscsi_init_irqs(phba); |
| if (ret < 0) { |
| shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" |
| "Failed to beiscsi_init_irqs\n"); |
| goto free_blkenbld; |
| } |
| hwi_enable_intr(phba); |
| ret = beiscsi_get_boot_info(phba); |
| if (ret < 0) { |
| shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-" |
| "No Boot Devices !!!!!\n"); |
| } |
| SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n"); |
| return 0; |
| |
| free_blkenbld: |
| destroy_workqueue(phba->wq); |
| if (blk_iopoll_enabled) |
| for (i = 0; i < phba->num_cpus; i++) { |
| pbe_eq = &phwi_context->be_eq[i]; |
| blk_iopoll_disable(&pbe_eq->iopoll); |
| } |
| free_twq: |
| beiscsi_clean_port(phba); |
| beiscsi_free_mem(phba); |
| free_port: |
| real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE; |
| |
| value = readl((void *)real_offset); |
| |
| if (value & 0x00010000) { |
| value &= 0xfffeffff; |
| writel(value, (void *)real_offset); |
| } |
| |
| pci_free_consistent(phba->pcidev, |
| phba->ctrl.mbox_mem_alloced.size, |
| phba->ctrl.mbox_mem_alloced.va, |
| phba->ctrl.mbox_mem_alloced.dma); |
| beiscsi_unmap_pci_function(phba); |
| hba_free: |
| if (phba->msix_enabled) |
| pci_disable_msix(phba->pcidev); |
| iscsi_host_remove(phba->shost); |
| pci_dev_put(phba->pcidev); |
| iscsi_host_free(phba->shost); |
| disable_pci: |
| pci_disable_device(pcidev); |
| return ret; |
| } |
| |
| struct iscsi_transport beiscsi_iscsi_transport = { |
| .owner = THIS_MODULE, |
| .name = DRV_NAME, |
| .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO | |
| CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD, |
| .param_mask = ISCSI_MAX_RECV_DLENGTH | |
| ISCSI_MAX_XMIT_DLENGTH | |
| ISCSI_HDRDGST_EN | |
| ISCSI_DATADGST_EN | |
| ISCSI_INITIAL_R2T_EN | |
| ISCSI_MAX_R2T | |
| ISCSI_IMM_DATA_EN | |
| ISCSI_FIRST_BURST | |
| ISCSI_MAX_BURST | |
| ISCSI_PDU_INORDER_EN | |
| ISCSI_DATASEQ_INORDER_EN | |
| ISCSI_ERL | |
| ISCSI_CONN_PORT | |
| ISCSI_CONN_ADDRESS | |
| ISCSI_EXP_STATSN | |
| ISCSI_PERSISTENT_PORT | |
| ISCSI_PERSISTENT_ADDRESS | |
| ISCSI_TARGET_NAME | ISCSI_TPGT | |
| ISCSI_USERNAME | ISCSI_PASSWORD | |
| ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN | |
| ISCSI_FAST_ABORT | ISCSI_ABORT_TMO | |
| ISCSI_LU_RESET_TMO | |
| ISCSI_PING_TMO | ISCSI_RECV_TMO | |
| ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME, |
| .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS | |
| ISCSI_HOST_INITIATOR_NAME, |
| .create_session = beiscsi_session_create, |
| .destroy_session = beiscsi_session_destroy, |
| .create_conn = beiscsi_conn_create, |
| .bind_conn = beiscsi_conn_bind, |
| .destroy_conn = iscsi_conn_teardown, |
| .set_param = beiscsi_set_param, |
| .get_conn_param = beiscsi_conn_get_param, |
| .get_session_param = iscsi_session_get_param, |
| .get_host_param = beiscsi_get_host_param, |
| .start_conn = beiscsi_conn_start, |
| .stop_conn = iscsi_conn_stop, |
| .send_pdu = iscsi_conn_send_pdu, |
| .xmit_task = beiscsi_task_xmit, |
| .cleanup_task = beiscsi_cleanup_task, |
| .alloc_pdu = beiscsi_alloc_pdu, |
| .parse_pdu_itt = beiscsi_parse_pdu, |
| .get_stats = beiscsi_conn_get_stats, |
| .ep_connect = beiscsi_ep_connect, |
| .ep_poll = beiscsi_ep_poll, |
| .ep_disconnect = beiscsi_ep_disconnect, |
| .session_recovery_timedout = iscsi_session_recovery_timedout, |
| }; |
| |
| static struct pci_driver beiscsi_pci_driver = { |
| .name = DRV_NAME, |
| .probe = beiscsi_dev_probe, |
| .remove = beiscsi_remove, |
| .id_table = beiscsi_pci_id_table |
| }; |
| |
| |
| static int __init beiscsi_module_init(void) |
| { |
| int ret; |
| |
| beiscsi_scsi_transport = |
| iscsi_register_transport(&beiscsi_iscsi_transport); |
| if (!beiscsi_scsi_transport) { |
| SE_DEBUG(DBG_LVL_1, |
| "beiscsi_module_init - Unable to register beiscsi" |
| "transport.\n"); |
| return -ENOMEM; |
| } |
| SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n", |
| &beiscsi_iscsi_transport); |
| |
| ret = pci_register_driver(&beiscsi_pci_driver); |
| if (ret) { |
| SE_DEBUG(DBG_LVL_1, |
| "beiscsi_module_init - Unable to register" |
| "beiscsi pci driver.\n"); |
| goto unregister_iscsi_transport; |
| } |
| return 0; |
| |
| unregister_iscsi_transport: |
| iscsi_unregister_transport(&beiscsi_iscsi_transport); |
| return ret; |
| } |
| |
| static void __exit beiscsi_module_exit(void) |
| { |
| pci_unregister_driver(&beiscsi_pci_driver); |
| iscsi_unregister_transport(&beiscsi_iscsi_transport); |
| } |
| |
| module_init(beiscsi_module_init); |
| module_exit(beiscsi_module_exit); |
