Annotation of sys/dev/pci/ahci.c, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: ahci.c,v 1.128 2007/07/03 22:33:20 dlg Exp $ */
! 2:
! 3: /*
! 4: * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
! 5: *
! 6: * Permission to use, copy, modify, and distribute this software for any
! 7: * purpose with or without fee is hereby granted, provided that the above
! 8: * copyright notice and this permission notice appear in all copies.
! 9: *
! 10: * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
! 11: * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
! 12: * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
! 13: * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
! 14: * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
! 15: * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
! 16: * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
! 17: */
! 18:
! 19: #include <sys/param.h>
! 20: #include <sys/systm.h>
! 21: #include <sys/buf.h>
! 22: #include <sys/kernel.h>
! 23: #include <sys/malloc.h>
! 24: #include <sys/device.h>
! 25: #include <sys/proc.h>
! 26: #include <sys/queue.h>
! 27:
! 28: #include <machine/bus.h>
! 29:
! 30: #include <dev/pci/pcireg.h>
! 31: #include <dev/pci/pcivar.h>
! 32: #include <dev/pci/pcidevs.h>
! 33:
! 34: #include <dev/ata/atascsi.h>
! 35:
! 36: /* change to AHCI_DEBUG for dmesg spam */
! 37: #define NO_AHCI_DEBUG
! 38:
! 39: #ifdef AHCI_DEBUG
! 40: #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
! 41: while (0)
! 42: #define AHCI_D_TIMEOUT 0x00
! 43: #define AHCI_D_VERBOSE 0x01
! 44: #define AHCI_D_INTR 0x02
! 45: #define AHCI_D_XFER 0x08
! 46: int ahcidebug = AHCI_D_VERBOSE;
! 47: #else
! 48: #define DPRINTF(m, f...)
! 49: #endif
! 50:
! 51: #define AHCI_PCI_BAR 0x24
! 52: #define AHCI_PCI_INTERFACE 0x01
! 53:
! 54: #define AHCI_REG_CAP 0x000 /* HBA Capabilities */
! 55: #define AHCI_REG_CAP_NP(_r) (((_r) & 0x1f)+1) /* Number of Ports */
! 56: #define AHCI_REG_CAP_SXS (1<<5) /* External SATA */
! 57: #define AHCI_REG_CAP_EMS (1<<6) /* Enclosure Mgmt */
! 58: #define AHCI_REG_CAP_CCCS (1<<7) /* Cmd Coalescing */
! 59: #define AHCI_REG_CAP_NCS(_r) ((((_r) & 0x1f00)>>8)+1) /* NCmds*/
! 60: #define AHCI_REG_CAP_PSC (1<<13) /* Partial State Capable */
! 61: #define AHCI_REG_CAP_SSC (1<<14) /* Slumber State Capable */
! 62: #define AHCI_REG_CAP_PMD (1<<15) /* PIO Multiple DRQ Block */
! 63: #define AHCI_REG_CAP_FBSS (1<<16) /* FIS-Based Switching */
! 64: #define AHCI_REG_CAP_SPM (1<<17) /* Port Multiplier */
! 65: #define AHCI_REG_CAP_SAM (1<<18) /* AHCI Only mode */
! 66: #define AHCI_REG_CAP_SNZO (1<<19) /* Non Zero DMA Offsets */
! 67: #define AHCI_REG_CAP_ISS (0xf<<20) /* Interface Speed Support */
! 68: #define AHCI_REG_CAP_ISS_G1 (0x1<<20) /* Gen 1 (1.5 Gbps) */
! 69: #define AHCI_REG_CAP_ISS_G1_2 (0x2<<20) /* Gen 1 and 2 (3 Gbps) */
! 70: #define AHCI_REG_CAP_SCLO (1<<24) /* Cmd List Override */
! 71: #define AHCI_REG_CAP_SAL (1<<25) /* Activity LED */
! 72: #define AHCI_REG_CAP_SALP (1<<26) /* Aggressive Link Pwr Mgmt */
! 73: #define AHCI_REG_CAP_SSS (1<<27) /* Staggered Spinup */
! 74: #define AHCI_REG_CAP_SMPS (1<<28) /* Mech Presence Switch */
! 75: #define AHCI_REG_CAP_SSNTF (1<<29) /* SNotification Register */
! 76: #define AHCI_REG_CAP_SNCQ (1<<30) /* Native Cmd Queuing */
! 77: #define AHCI_REG_CAP_S64A (1<<31) /* 64bit Addressing */
! 78: #define AHCI_FMT_CAP "\020" "\040S64A" "\037NCQ" "\036SSNTF" \
! 79: "\035SMPS" "\034SSS" "\033SALP" "\032SAL" \
! 80: "\031SCLO" "\024SNZO" "\023SAM" "\022SPM" \
! 81: "\021FBSS" "\020PMD" "\017SSC" "\016PSC" \
! 82: "\010CCCS" "\007EMS" "\006SXS"
! 83: #define AHCI_REG_GHC 0x004 /* Global HBA Control */
! 84: #define AHCI_REG_GHC_HR (1<<0) /* HBA Reset */
! 85: #define AHCI_REG_GHC_IE (1<<1) /* Interrupt Enable */
! 86: #define AHCI_REG_GHC_MRSM (1<<2) /* MSI Revert to Single Msg */
! 87: #define AHCI_REG_GHC_AE (1<<31) /* AHCI Enable */
! 88: #define AHCI_FMT_GHC "\020" "\040AE" "\003MRSM" "\002IE" "\001HR"
! 89: #define AHCI_REG_IS 0x008 /* Interrupt Status */
! 90: #define AHCI_REG_PI 0x00c /* Ports Implemented */
! 91: #define AHCI_REG_VS 0x010 /* AHCI Version */
! 92: #define AHCI_REG_VS_0_95 0x00000905 /* 0.95 */
! 93: #define AHCI_REG_VS_1_0 0x00010000 /* 1.0 */
! 94: #define AHCI_REG_VS_1_1 0x00010100 /* 1.1 */
! 95: #define AHCI_REG_CCC_CTL 0x014 /* Coalescing Control */
! 96: #define AHCI_REG_CCC_CTL_INT(_r) (((_r) & 0xf8) >> 3) /* CCC INT slot */
! 97: #define AHCI_REG_CCC_PORTS 0x018 /* Coalescing Ports */
! 98: #define AHCI_REG_EM_LOC 0x01c /* Enclosure Mgmt Location */
! 99: #define AHCI_REG_EM_CTL 0x020 /* Enclosure Mgmt Control */
! 100:
! 101: #define AHCI_PORT_REGION(_p) (0x100 + ((_p) * 0x80))
! 102: #define AHCI_PORT_SIZE 0x80
! 103:
! 104: #define AHCI_PREG_CLB 0x00 /* Cmd List Base Addr */
! 105: #define AHCI_PREG_CLBU 0x04 /* Cmd List Base Hi Addr */
! 106: #define AHCI_PREG_FB 0x08 /* FIS Base Addr */
! 107: #define AHCI_PREG_FBU 0x0c /* FIS Base Hi Addr */
! 108: #define AHCI_PREG_IS 0x10 /* Interrupt Status */
! 109: #define AHCI_PREG_IS_DHRS (1<<0) /* Device to Host FIS */
! 110: #define AHCI_PREG_IS_PSS (1<<1) /* PIO Setup FIS */
! 111: #define AHCI_PREG_IS_DSS (1<<2) /* DMA Setup FIS */
! 112: #define AHCI_PREG_IS_SDBS (1<<3) /* Set Device Bits FIS */
! 113: #define AHCI_PREG_IS_UFS (1<<4) /* Unknown FIS */
! 114: #define AHCI_PREG_IS_DPS (1<<5) /* Descriptor Processed */
! 115: #define AHCI_PREG_IS_PCS (1<<6) /* Port Change */
! 116: #define AHCI_PREG_IS_DMPS (1<<7) /* Device Mechanical Presence */
! 117: #define AHCI_PREG_IS_PRCS (1<<22) /* PhyRdy Change */
! 118: #define AHCI_PREG_IS_IPMS (1<<23) /* Incorrect Port Multiplier */
! 119: #define AHCI_PREG_IS_OFS (1<<24) /* Overflow */
! 120: #define AHCI_PREG_IS_INFS (1<<26) /* Interface Non-fatal Error */
! 121: #define AHCI_PREG_IS_IFS (1<<27) /* Interface Fatal Error */
! 122: #define AHCI_PREG_IS_HBDS (1<<28) /* Host Bus Data Error */
! 123: #define AHCI_PREG_IS_HBFS (1<<29) /* Host Bus Fatal Error */
! 124: #define AHCI_PREG_IS_TFES (1<<30) /* Task File Error */
! 125: #define AHCI_PREG_IS_CPDS (1<<31) /* Cold Presence Detect */
! 126: #define AHCI_PFMT_IS "\20" "\040CPDS" "\037TFES" "\036HBFS" \
! 127: "\035HBDS" "\034IFS" "\033INFS" "\031OFS" \
! 128: "\030IPMS" "\027PRCS" "\010DMPS" "\006DPS" \
! 129: "\007PCS" "\005UFS" "\004SDBS" "\003DSS" \
! 130: "\002PSS" "\001DHRS"
! 131: #define AHCI_PREG_IE 0x14 /* Interrupt Enable */
! 132: #define AHCI_PREG_IE_DHRE (1<<0) /* Device to Host FIS */
! 133: #define AHCI_PREG_IE_PSE (1<<1) /* PIO Setup FIS */
! 134: #define AHCI_PREG_IE_DSE (1<<2) /* DMA Setup FIS */
! 135: #define AHCI_PREG_IE_SDBE (1<<3) /* Set Device Bits FIS */
! 136: #define AHCI_PREG_IE_UFE (1<<4) /* Unknown FIS */
! 137: #define AHCI_PREG_IE_DPE (1<<5) /* Descriptor Processed */
! 138: #define AHCI_PREG_IE_PCE (1<<6) /* Port Change */
! 139: #define AHCI_PREG_IE_DMPE (1<<7) /* Device Mechanical Presence */
! 140: #define AHCI_PREG_IE_PRCE (1<<22) /* PhyRdy Change */
! 141: #define AHCI_PREG_IE_IPME (1<<23) /* Incorrect Port Multiplier */
! 142: #define AHCI_PREG_IE_OFE (1<<24) /* Overflow */
! 143: #define AHCI_PREG_IE_INFE (1<<26) /* Interface Non-fatal Error */
! 144: #define AHCI_PREG_IE_IFE (1<<27) /* Interface Fatal Error */
! 145: #define AHCI_PREG_IE_HBDE (1<<28) /* Host Bus Data Error */
! 146: #define AHCI_PREG_IE_HBFE (1<<29) /* Host Bus Fatal Error */
! 147: #define AHCI_PREG_IE_TFEE (1<<30) /* Task File Error */
! 148: #define AHCI_PREG_IE_CPDE (1<<31) /* Cold Presence Detect */
! 149: #define AHCI_PFMT_IE "\20" "\040CPDE" "\037TFEE" "\036HBFE" \
! 150: "\035HBDE" "\034IFE" "\033INFE" "\031OFE" \
! 151: "\030IPME" "\027PRCE" "\010DMPE" "\007PCE" \
! 152: "\006DPE" "\005UFE" "\004SDBE" "\003DSE" \
! 153: "\002PSE" "\001DHRE"
! 154: #define AHCI_PREG_CMD 0x18 /* Command and Status */
! 155: #define AHCI_PREG_CMD_ST (1<<0) /* Start */
! 156: #define AHCI_PREG_CMD_SUD (1<<1) /* Spin Up Device */
! 157: #define AHCI_PREG_CMD_POD (1<<2) /* Power On Device */
! 158: #define AHCI_PREG_CMD_CLO (1<<3) /* Command List Override */
! 159: #define AHCI_PREG_CMD_FRE (1<<4) /* FIS Receive Enable */
! 160: #define AHCI_PREG_CMD_CCS(_r) (((_r) >> 8) & 0x1f) /* Curr CmdSlot# */
! 161: #define AHCI_PREG_CMD_MPSS (1<<13) /* Mech Presence State */
! 162: #define AHCI_PREG_CMD_FR (1<<14) /* FIS Receive Running */
! 163: #define AHCI_PREG_CMD_CR (1<<15) /* Command List Running */
! 164: #define AHCI_PREG_CMD_CPS (1<<16) /* Cold Presence State */
! 165: #define AHCI_PREG_CMD_PMA (1<<17) /* Port Multiplier Attached */
! 166: #define AHCI_PREG_CMD_HPCP (1<<18) /* Hot Plug Capable */
! 167: #define AHCI_PREG_CMD_MPSP (1<<19) /* Mech Presence Switch */
! 168: #define AHCI_PREG_CMD_CPD (1<<20) /* Cold Presence Detection */
! 169: #define AHCI_PREG_CMD_ESP (1<<21) /* External SATA Port */
! 170: #define AHCI_PREG_CMD_ATAPI (1<<24) /* Device is ATAPI */
! 171: #define AHCI_PREG_CMD_DLAE (1<<25) /* Drv LED on ATAPI Enable */
! 172: #define AHCI_PREG_CMD_ALPE (1<<26) /* Aggro Pwr Mgmt Enable */
! 173: #define AHCI_PREG_CMD_ASP (1<<27) /* Aggro Slumber/Partial */
! 174: #define AHCI_PREG_CMD_ICC 0xf0000000 /* Interface Comm Ctrl */
! 175: #define AHCI_PREG_CMD_ICC_SLUMBER 0x60000000
! 176: #define AHCI_PREG_CMD_ICC_PARTIAL 0x20000000
! 177: #define AHCI_PREG_CMD_ICC_ACTIVE 0x10000000
! 178: #define AHCI_PREG_CMD_ICC_IDLE 0x00000000
! 179: #define AHCI_PFMT_CMD "\020" "\034ASP" "\033ALPE" "\032DLAE" \
! 180: "\031ATAPI" "\026ESP" "\025CPD" "\024MPSP" \
! 181: "\023HPCP" "\022PMA" "\021CPS" "\020CR" \
! 182: "\017FR" "\016MPSS" "\005FRE" "\004CLO" \
! 183: "\003POD" "\002SUD" "\001ST"
! 184: #define AHCI_PREG_TFD 0x20 /* Task File Data*/
! 185: #define AHCI_PREG_TFD_STS 0xff
! 186: #define AHCI_PREG_TFD_STS_ERR (1<<0)
! 187: #define AHCI_PREG_TFD_STS_DRQ (1<<3)
! 188: #define AHCI_PREG_TFD_STS_BSY (1<<7)
! 189: #define AHCI_PREG_TFD_ERR 0xff00
! 190: #define AHCI_PFMT_TFD_STS "\20" "\010BSY" "\004DRQ" "\001ERR"
! 191: #define AHCI_PREG_SIG 0x24 /* Signature */
! 192: #define AHCI_PREG_SSTS 0x28 /* SATA Status */
! 193: #define AHCI_PREG_SSTS_DET 0xf /* Device Detection */
! 194: #define AHCI_PREG_SSTS_DET_NONE 0x0
! 195: #define AHCI_PREG_SSTS_DET_DEV_NE 0x1
! 196: #define AHCI_PREG_SSTS_DET_DEV 0x3
! 197: #define AHCI_PREG_SSTS_DET_PHYOFFLINE 0x4
! 198: #define AHCI_PREG_SSTS_SPD 0xf0 /* Current Interface Speed */
! 199: #define AHCI_PREG_SSTS_SPD_NONE 0x00
! 200: #define AHCI_PREG_SSTS_SPD_GEN1 0x10
! 201: #define AHCI_PREG_SSTS_SPD_GEN2 0x20
! 202: #define AHCI_PREG_SSTS_IPM 0xf00 /* Interface Power Management */
! 203: #define AHCI_PREG_SSTS_IPM_NONE 0x000
! 204: #define AHCI_PREG_SSTS_IPM_ACTIVE 0x100
! 205: #define AHCI_PREG_SSTS_IPM_PARTIAL 0x200
! 206: #define AHCI_PREG_SSTS_IPM_SLUMBER 0x600
! 207: #define AHCI_PREG_SCTL 0x2c /* SATA Control */
! 208: #define AHCI_PREG_SCTL_DET 0xf /* Device Detection */
! 209: #define AHCI_PREG_SCTL_DET_NONE 0x0
! 210: #define AHCI_PREG_SCTL_DET_INIT 0x1
! 211: #define AHCI_PREG_SCTL_DET_DISABLE 0x4
! 212: #define AHCI_PREG_SCTL_SPD 0xf0 /* Speed Allowed */
! 213: #define AHCI_PREG_SCTL_SPD_ANY 0x00
! 214: #define AHCI_PREG_SCTL_SPD_GEN1 0x10
! 215: #define AHCI_PREG_SCTL_SPD_GEN2 0x20
! 216: #define AHCI_PREG_SCTL_IPM 0xf00 /* Interface Power Management */
! 217: #define AHCI_PREG_SCTL_IPM_NONE 0x000
! 218: #define AHCI_PREG_SCTL_IPM_NOPARTIAL 0x100
! 219: #define AHCI_PREG_SCTL_IPM_NOSLUMBER 0x200
! 220: #define AHCI_PREG_SCTL_IPM_DISABLED 0x300
! 221: #define AHCI_PREG_SERR 0x30 /* SATA Error */
! 222: #define AHCI_PREG_SERR_ERR(_r) ((_r) & 0xffff)
! 223: #define AHCI_PREG_SERR_ERR_I (1<<0) /* Recovered Data Integrity */
! 224: #define AHCI_PREG_SERR_ERR_M (1<<1) /* Recovered Communications */
! 225: #define AHCI_PREG_SERR_ERR_T (1<<8) /* Transient Data Integrity */
! 226: #define AHCI_PREG_SERR_ERR_C (1<<9) /* Persistent Comm/Data */
! 227: #define AHCI_PREG_SERR_ERR_P (1<<10) /* Protocol */
! 228: #define AHCI_PREG_SERR_ERR_E (1<<11) /* Internal */
! 229: #define AHCI_PFMT_SERR_ERR "\020" "\014E" "\013P" "\012C" "\011T" "\002M" \
! 230: "\001I"
! 231: #define AHCI_PREG_SERR_DIAG(_r) (((_r) >> 16) & 0xffff)
! 232: #define AHCI_PREG_SERR_DIAG_N (1<<0) /* PhyRdy Change */
! 233: #define AHCI_PREG_SERR_DIAG_I (1<<1) /* Phy Internal Error */
! 234: #define AHCI_PREG_SERR_DIAG_W (1<<2) /* Comm Wake */
! 235: #define AHCI_PREG_SERR_DIAG_B (1<<3) /* 10B to 8B Decode Error */
! 236: #define AHCI_PREG_SERR_DIAG_D (1<<4) /* Disparity Error */
! 237: #define AHCI_PREG_SERR_DIAG_C (1<<5) /* CRC Error */
! 238: #define AHCI_PREG_SERR_DIAG_H (1<<6) /* Handshake Error */
! 239: #define AHCI_PREG_SERR_DIAG_S (1<<7) /* Link Sequence Error */
! 240: #define AHCI_PREG_SERR_DIAG_T (1<<8) /* Transport State Trans Err */
! 241: #define AHCI_PREG_SERR_DIAG_F (1<<9) /* Unknown FIS Type */
! 242: #define AHCI_PREG_SERR_DIAG_X (1<<10) /* Exchanged */
! 243: #define AHCI_PFMT_SERR_DIAG "\020" "\013X" "\012F" "\011T" "\010S" "\007H" \
! 244: "\006C" "\005D" "\004B" "\003W" "\002I" \
! 245: "\001N"
! 246: #define AHCI_PREG_SACT 0x34 /* SATA Active */
! 247: #define AHCI_PREG_CI 0x38 /* Command Issue */
! 248: #define AHCI_PREG_CI_ALL_SLOTS 0xffffffff
! 249: #define AHCI_PREG_SNTF 0x3c /* SNotification */
! 250:
! 251: struct ahci_cmd_hdr {
! 252: u_int16_t flags;
! 253: #define AHCI_CMD_LIST_FLAG_CFL 0x001f /* Command FIS Length */
! 254: #define AHCI_CMD_LIST_FLAG_A (1<<5) /* ATAPI */
! 255: #define AHCI_CMD_LIST_FLAG_W (1<<6) /* Write */
! 256: #define AHCI_CMD_LIST_FLAG_P (1<<7) /* Prefetchable */
! 257: #define AHCI_CMD_LIST_FLAG_R (1<<8) /* Reset */
! 258: #define AHCI_CMD_LIST_FLAG_B (1<<9) /* BIST */
! 259: #define AHCI_CMD_LIST_FLAG_C (1<<10) /* Clear Busy upon R_OK */
! 260: #define AHCI_CMD_LIST_FLAG_PMP 0xf000 /* Port Multiplier Port */
! 261: u_int16_t prdtl; /* sgl len */
! 262:
! 263: u_int32_t prdbc; /* transferred byte count */
! 264:
! 265: u_int32_t ctba_lo;
! 266: u_int32_t ctba_hi;
! 267:
! 268: u_int32_t reserved[4];
! 269: } __packed;
! 270:
! 271: struct ahci_rfis {
! 272: u_int8_t dsfis[28];
! 273: u_int8_t reserved1[4];
! 274: u_int8_t psfis[24];
! 275: u_int8_t reserved2[8];
! 276: u_int8_t rfis[24];
! 277: u_int8_t reserved3[4];
! 278: u_int8_t sdbfis[4];
! 279: u_int8_t ufis[64];
! 280: u_int8_t reserved4[96];
! 281: } __packed;
! 282:
! 283: struct ahci_prdt {
! 284: u_int32_t dba_lo;
! 285: u_int32_t dba_hi;
! 286: u_int32_t reserved;
! 287: u_int32_t flags;
! 288: #define AHCI_PRDT_FLAG_INTR (1<<31) /* interrupt on completion */
! 289: } __packed;
! 290:
! 291: /* this makes ahci_cmd_table 512 bytes, supporting 128-byte alignment */
! 292: #define AHCI_MAX_PRDT 24
! 293:
! 294: struct ahci_cmd_table {
! 295: u_int8_t cfis[64]; /* Command FIS */
! 296: u_int8_t acmd[16]; /* ATAPI Command */
! 297: u_int8_t reserved[48];
! 298:
! 299: struct ahci_prdt prdt[AHCI_MAX_PRDT];
! 300: } __packed;
! 301:
! 302: #define AHCI_MAX_PORTS 32
! 303:
! 304: struct ahci_dmamem {
! 305: bus_dmamap_t adm_map;
! 306: bus_dma_segment_t adm_seg;
! 307: size_t adm_size;
! 308: caddr_t adm_kva;
! 309: };
! 310: #define AHCI_DMA_MAP(_adm) ((_adm)->adm_map)
! 311: #define AHCI_DMA_DVA(_adm) ((_adm)->adm_map->dm_segs[0].ds_addr)
! 312: #define AHCI_DMA_KVA(_adm) ((void *)(_adm)->adm_kva)
! 313:
! 314: struct ahci_softc;
! 315: struct ahci_port;
! 316:
! 317: struct ahci_ccb {
! 318: /* ATA xfer associated with this CCB. Must be 1st struct member. */
! 319: struct ata_xfer ccb_xa;
! 320:
! 321: int ccb_slot;
! 322: struct ahci_port *ccb_port;
! 323:
! 324: bus_dmamap_t ccb_dmamap;
! 325: struct ahci_cmd_hdr *ccb_cmd_hdr;
! 326: struct ahci_cmd_table *ccb_cmd_table;
! 327:
! 328: void (*ccb_done)(struct ahci_ccb *);
! 329:
! 330: TAILQ_ENTRY(ahci_ccb) ccb_entry;
! 331: };
! 332:
! 333: struct ahci_port {
! 334: struct ahci_softc *ap_sc;
! 335: bus_space_handle_t ap_ioh;
! 336:
! 337: #ifdef AHCI_COALESCE
! 338: int ap_num;
! 339: #endif
! 340:
! 341: struct ahci_rfis *ap_rfis;
! 342: struct ahci_dmamem *ap_dmamem_rfis;
! 343:
! 344: struct ahci_dmamem *ap_dmamem_cmd_list;
! 345: struct ahci_dmamem *ap_dmamem_cmd_table;
! 346:
! 347: volatile u_int32_t ap_active;
! 348: volatile u_int32_t ap_active_cnt;
! 349: volatile u_int32_t ap_sactive;
! 350: struct ahci_ccb *ap_ccbs;
! 351:
! 352: TAILQ_HEAD(, ahci_ccb) ap_ccb_free;
! 353: TAILQ_HEAD(, ahci_ccb) ap_ccb_pending;
! 354:
! 355: u_int32_t ap_state;
! 356: #define AP_S_NORMAL 0
! 357: #define AP_S_FATAL_ERROR 1
! 358:
! 359: /* For error recovery. */
! 360: #ifdef DIAGNOSTIC
! 361: int ap_err_busy;
! 362: #endif
! 363: u_int32_t ap_err_saved_sactive;
! 364: u_int32_t ap_err_saved_active;
! 365: u_int32_t ap_err_saved_active_cnt;
! 366:
! 367: u_int8_t ap_err_scratch[512];
! 368:
! 369: #ifdef AHCI_DEBUG
! 370: char ap_name[16];
! 371: #define PORTNAME(_ap) ((_ap)->ap_name)
! 372: #else
! 373: #define PORTNAME(_ap) DEVNAME((_ap)->ap_sc)
! 374: #endif
! 375: };
! 376:
! 377: struct ahci_softc {
! 378: struct device sc_dev;
! 379:
! 380: void *sc_ih;
! 381:
! 382: bus_space_tag_t sc_iot;
! 383: bus_space_handle_t sc_ioh;
! 384: bus_size_t sc_ios;
! 385: bus_dma_tag_t sc_dmat;
! 386:
! 387: int sc_flags;
! 388: #define AHCI_F_NO_NCQ (1<<0)
! 389:
! 390: u_int sc_ncmds;
! 391:
! 392: struct ahci_port *sc_ports[AHCI_MAX_PORTS];
! 393:
! 394: struct atascsi *sc_atascsi;
! 395:
! 396: #ifdef AHCI_COALESCE
! 397: u_int32_t sc_ccc_mask;
! 398: u_int32_t sc_ccc_ports;
! 399: u_int32_t sc_ccc_ports_cur;
! 400: #endif
! 401: };
! 402: #define DEVNAME(_s) ((_s)->sc_dev.dv_xname)
! 403:
! 404: struct ahci_device {
! 405: pci_vendor_id_t ad_vendor;
! 406: pci_product_id_t ad_product;
! 407: int (*ad_match)(struct pci_attach_args *);
! 408: int (*ad_attach)(struct ahci_softc *,
! 409: struct pci_attach_args *);
! 410: };
! 411:
! 412: const struct ahci_device *ahci_lookup_device(struct pci_attach_args *);
! 413:
! 414: int ahci_no_match(struct pci_attach_args *);
! 415: int ahci_vt8251_attach(struct ahci_softc *,
! 416: struct pci_attach_args *);
! 417:
! 418: static const struct ahci_device ahci_devices[] = {
! 419: { PCI_VENDOR_VIATECH, PCI_PRODUCT_VIATECH_VT8251_SATA,
! 420: ahci_no_match, ahci_vt8251_attach }
! 421: };
! 422:
! 423: int ahci_pci_match(struct device *, void *, void *);
! 424: void ahci_pci_attach(struct device *, struct device *,
! 425: void *);
! 426:
! 427: struct cfattach ahci_pci_ca = {
! 428: sizeof(struct ahci_softc), ahci_pci_match, ahci_pci_attach
! 429: };
! 430:
! 431: struct cfattach ahci_jmb_ca = {
! 432: sizeof(struct ahci_softc), ahci_pci_match, ahci_pci_attach
! 433: };
! 434:
! 435: struct cfdriver ahci_cd = {
! 436: NULL, "ahci", DV_DULL
! 437: };
! 438:
! 439:
! 440: int ahci_map_regs(struct ahci_softc *,
! 441: struct pci_attach_args *);
! 442: void ahci_unmap_regs(struct ahci_softc *,
! 443: struct pci_attach_args *);
! 444: int ahci_map_intr(struct ahci_softc *,
! 445: struct pci_attach_args *, pci_intr_handle_t);
! 446: #ifdef notyet
! 447: void ahci_unmap_intr(struct ahci_softc *,
! 448: struct pci_attach_args *);
! 449: #endif
! 450:
! 451: int ahci_init(struct ahci_softc *);
! 452: int ahci_port_alloc(struct ahci_softc *, u_int);
! 453: void ahci_port_free(struct ahci_softc *, u_int);
! 454:
! 455: int ahci_port_start(struct ahci_port *, int);
! 456: int ahci_port_stop(struct ahci_port *, int);
! 457: int ahci_port_clo(struct ahci_port *);
! 458: int ahci_port_softreset(struct ahci_port *);
! 459: int ahci_port_portreset(struct ahci_port *);
! 460:
! 461: int ahci_load_prdt(struct ahci_ccb *);
! 462: void ahci_unload_prdt(struct ahci_ccb *);
! 463:
! 464: int ahci_poll(struct ahci_ccb *, int, void (*)(void *));
! 465: void ahci_start(struct ahci_ccb *);
! 466:
! 467: void ahci_issue_pending_ncq_commands(struct ahci_port *);
! 468: void ahci_issue_pending_commands(struct ahci_port *, int);
! 469:
! 470: int ahci_intr(void *);
! 471: u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t);
! 472:
! 473: struct ahci_ccb *ahci_get_ccb(struct ahci_port *);
! 474: void ahci_put_ccb(struct ahci_ccb *);
! 475:
! 476: struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *);
! 477: void ahci_put_err_ccb(struct ahci_ccb *);
! 478:
! 479: int ahci_port_read_ncq_error(struct ahci_port *, int *);
! 480:
! 481: struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t);
! 482: void ahci_dmamem_free(struct ahci_softc *,
! 483: struct ahci_dmamem *);
! 484:
! 485: u_int32_t ahci_read(struct ahci_softc *, bus_size_t);
! 486: void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
! 487: int ahci_wait_ne(struct ahci_softc *, bus_size_t,
! 488: u_int32_t, u_int32_t);
! 489:
! 490: u_int32_t ahci_pread(struct ahci_port *, bus_size_t);
! 491: void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
! 492: int ahci_pwait_eq(struct ahci_port *, bus_size_t,
! 493: u_int32_t, u_int32_t);
! 494:
! 495: /* Wait for all bits in _b to be cleared */
! 496: #define ahci_pwait_clr(_ap, _r, _b) ahci_pwait_eq((_ap), (_r), (_b), 0)
! 497:
! 498: /* Wait for all bits in _b to be set */
! 499: #define ahci_pwait_set(_ap, _r, _b) ahci_pwait_eq((_ap), (_r), (_b), (_b))
! 500:
! 501: /* provide methods for atascsi to call */
! 502: int ahci_ata_probe(void *, int);
! 503: struct ata_xfer * ahci_ata_get_xfer(void *, int);
! 504: void ahci_ata_put_xfer(struct ata_xfer *);
! 505: int ahci_ata_cmd(struct ata_xfer *);
! 506:
! 507: struct atascsi_methods ahci_atascsi_methods = {
! 508: ahci_ata_probe,
! 509: ahci_ata_get_xfer,
! 510: ahci_ata_cmd
! 511: };
! 512:
! 513: /* ccb completions */
! 514: void ahci_ata_cmd_done(struct ahci_ccb *);
! 515: void ahci_ata_cmd_timeout(void *);
! 516: void ahci_empty_done(struct ahci_ccb *);
! 517:
! 518: const struct ahci_device *
! 519: ahci_lookup_device(struct pci_attach_args *pa)
! 520: {
! 521: int i;
! 522: const struct ahci_device *ad;
! 523:
! 524: for (i = 0; i < (sizeof(ahci_devices) / sizeof(ahci_devices[0])); i++) {
! 525: ad = &ahci_devices[i];
! 526: if (ad->ad_vendor == PCI_VENDOR(pa->pa_id) &&
! 527: ad->ad_product == PCI_PRODUCT(pa->pa_id))
! 528: return (ad);
! 529: }
! 530:
! 531: return (NULL);
! 532: }
! 533:
! 534: int
! 535: ahci_no_match(struct pci_attach_args *pa)
! 536: {
! 537: return (0);
! 538: }
! 539:
! 540: int
! 541: ahci_vt8251_attach(struct ahci_softc *sc, struct pci_attach_args *pa)
! 542: {
! 543: sc->sc_flags |= AHCI_F_NO_NCQ;
! 544:
! 545: return (0);
! 546: }
! 547:
! 548: int
! 549: ahci_pci_match(struct device *parent, void *match, void *aux)
! 550: {
! 551: struct pci_attach_args *pa = aux;
! 552: const struct ahci_device *ad;
! 553:
! 554: ad = ahci_lookup_device(pa);
! 555: if (ad != NULL) {
! 556: /* the device may need special checks to see if it matches */
! 557: if (ad->ad_match != NULL)
! 558: return (ad->ad_match(pa));
! 559:
! 560: return (2); /* match higher than pciide */
! 561: }
! 562:
! 563: if (PCI_CLASS(pa->pa_class) == PCI_CLASS_MASS_STORAGE &&
! 564: PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_MASS_STORAGE_SATA &&
! 565: PCI_INTERFACE(pa->pa_class) == AHCI_PCI_INTERFACE)
! 566: return (2);
! 567:
! 568: return (0);
! 569: }
! 570:
! 571: void
! 572: ahci_pci_attach(struct device *parent, struct device *self, void *aux)
! 573: {
! 574: struct ahci_softc *sc = (struct ahci_softc *)self;
! 575: struct pci_attach_args *pa = aux;
! 576: struct atascsi_attach_args aaa;
! 577: const struct ahci_device *ad;
! 578: pci_intr_handle_t ih;
! 579: u_int32_t cap, pi;
! 580: int i;
! 581:
! 582: ad = ahci_lookup_device(pa);
! 583: if (ad != NULL && ad->ad_attach != NULL) {
! 584: if (ad->ad_attach(sc, pa) != 0) {
! 585: /* error should be printed by ad_attach */
! 586: return;
! 587: }
! 588: }
! 589:
! 590: if (pci_intr_map(pa, &ih) != 0) {
! 591: printf(": unable to map interrupt\n");
! 592: return;
! 593: }
! 594: printf(": %s,", pci_intr_string(pa->pa_pc, ih));
! 595:
! 596: if (ahci_map_regs(sc, pa) != 0) {
! 597: /* error already printed by ahci_map_regs */
! 598: return;
! 599: }
! 600:
! 601: if (ahci_init(sc) != 0) {
! 602: /* error already printed by ahci_init */
! 603: goto unmap;
! 604: }
! 605:
! 606: if (ahci_map_intr(sc, pa, ih) != 0) {
! 607: /* error already printed by ahci_map_intr */
! 608: goto unmap;
! 609: }
! 610:
! 611: printf("\n");
! 612:
! 613: sc->sc_dmat = pa->pa_dmat;
! 614:
! 615: cap = ahci_read(sc, AHCI_REG_CAP);
! 616: sc->sc_ncmds = AHCI_REG_CAP_NCS(cap);
! 617: #ifdef AHCI_DEBUG
! 618: if (ahcidebug & AHCI_D_VERBOSE) {
! 619: const char *gen;
! 620:
! 621: switch (cap & AHCI_REG_CAP_ISS) {
! 622: case AHCI_REG_CAP_ISS_G1:
! 623: gen = "1 (1.5Gbps)";
! 624: break;
! 625: case AHCI_REG_CAP_ISS_G1_2:
! 626: gen = "1 (1.5Gbps) and 2 (3Gbps)";
! 627: break;
! 628: default:
! 629: gen = "unknown";
! 630: break;
! 631: }
! 632:
! 633: printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
! 634: DEVNAME(sc), cap, AHCI_FMT_CAP,
! 635: AHCI_REG_CAP_NP(cap), sc->sc_ncmds, gen);
! 636: }
! 637: #endif
! 638:
! 639: pi = ahci_read(sc, AHCI_REG_PI);
! 640: DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
! 641: DEVNAME(sc), pi);
! 642:
! 643: #ifdef AHCI_COALESCE
! 644: /* Naive coalescing support - enable for all ports. */
! 645: if (cap & AHCI_REG_CAP_CCCS) {
! 646: u_int16_t ccc_timeout = 20;
! 647: u_int8_t ccc_numcomplete = 12;
! 648: u_int32_t ccc_ctl;
! 649:
! 650: /* disable coalescing during reconfiguration. */
! 651: ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL);
! 652: ccc_ctl &= ~0x00000001;
! 653: ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
! 654:
! 655: sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl);
! 656: if (pi & sc->sc_ccc_mask) {
! 657: /* A conflict with the implemented port list? */
! 658: printf("%s: coalescing interrupt/implemented port list "
! 659: "conflict, PI: %08x, ccc_mask: %08x\n",
! 660: DEVNAME(sc), pi, sc->sc_ccc_mask);
! 661: sc->sc_ccc_mask = 0;
! 662: goto noccc;
! 663: }
! 664:
! 665: /* ahci_port_start will enable each port when it starts. */
! 666: sc->sc_ccc_ports = pi;
! 667: sc->sc_ccc_ports_cur = 0;
! 668:
! 669: /* program thresholds and enable overall coalescing. */
! 670: ccc_ctl &= ~0xffffff00;
! 671: ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
! 672: ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
! 673: ahci_write(sc, AHCI_REG_CCC_PORTS, 0);
! 674: ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1);
! 675: }
! 676: noccc:
! 677: #endif
! 678: for (i = 0; i < AHCI_MAX_PORTS; i++) {
! 679: if (!ISSET(pi, 1 << i)) {
! 680: /* dont allocate stuff if the port isnt implemented */
! 681: continue;
! 682: }
! 683:
! 684: if (ahci_port_alloc(sc, i) == ENOMEM)
! 685: goto freeports;
! 686: }
! 687:
! 688: bzero(&aaa, sizeof(aaa));
! 689: aaa.aaa_cookie = sc;
! 690: aaa.aaa_methods = &ahci_atascsi_methods;
! 691: aaa.aaa_minphys = minphys;
! 692: aaa.aaa_nports = AHCI_MAX_PORTS;
! 693: aaa.aaa_ncmds = sc->sc_ncmds;
! 694: aaa.aaa_capability = ASAA_CAP_NEEDS_RESERVED;
! 695: if (!(sc->sc_flags & AHCI_F_NO_NCQ) && (cap & AHCI_REG_CAP_SNCQ))
! 696: aaa.aaa_capability |= ASAA_CAP_NCQ;
! 697:
! 698: sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
! 699:
! 700: /* Enable interrupts */
! 701: ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
! 702:
! 703: return;
! 704:
! 705: freeports:
! 706: for (i = 0; i < AHCI_MAX_PORTS; i++)
! 707: if (sc->sc_ports[i] != NULL)
! 708: ahci_port_free(sc, i);
! 709: unmap:
! 710: /* Disable controller */
! 711: ahci_write(sc, AHCI_REG_GHC, 0);
! 712: ahci_unmap_regs(sc, pa);
! 713: return;
! 714: }
! 715:
! 716: int
! 717: ahci_map_regs(struct ahci_softc *sc, struct pci_attach_args *pa)
! 718: {
! 719: pcireg_t maptype;
! 720:
! 721: maptype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, AHCI_PCI_BAR);
! 722: if (pci_mapreg_map(pa, AHCI_PCI_BAR, maptype, 0, &sc->sc_iot,
! 723: &sc->sc_ioh, NULL, &sc->sc_ios, 0) != 0) {
! 724: printf(" unable to map registers\n");
! 725: return (1);
! 726: }
! 727:
! 728: return (0);
! 729: }
! 730:
! 731: void
! 732: ahci_unmap_regs(struct ahci_softc *sc, struct pci_attach_args *pa)
! 733: {
! 734: bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios);
! 735: sc->sc_ios = 0;
! 736: }
! 737:
! 738: int
! 739: ahci_map_intr(struct ahci_softc *sc, struct pci_attach_args *pa,
! 740: pci_intr_handle_t ih)
! 741: {
! 742: sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_BIO,
! 743: ahci_intr, sc, DEVNAME(sc));
! 744: if (sc->sc_ih == NULL) {
! 745: printf("%s: unable to map interrupt\n", DEVNAME(sc));
! 746: return (1);
! 747: }
! 748:
! 749: return (0);
! 750: }
! 751:
! 752: #ifdef notyet
! 753: /* one day we'll have hotplug pci */
! 754: void
! 755: ahci_unmap_intr(struct ahci_softc *sc, struct pci_attach_args *pa)
! 756: {
! 757: pci_intr_disestablish(pa->pa_pc, sc->sc_ih);
! 758: }
! 759: #endif
! 760:
! 761: int
! 762: ahci_init(struct ahci_softc *sc)
! 763: {
! 764: u_int32_t reg, cap, pi;
! 765: const char *revision;
! 766:
! 767: DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
! 768: AHCI_FMT_GHC);
! 769:
! 770: /* save BIOS initialised parameters, enable staggered spin up */
! 771: cap = ahci_read(sc, AHCI_REG_CAP);
! 772: cap &= AHCI_REG_CAP_SMPS;
! 773: cap |= AHCI_REG_CAP_SSS;
! 774: pi = ahci_read(sc, AHCI_REG_PI);
! 775:
! 776: if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) {
! 777: /* reset the controller */
! 778: ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
! 779: if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR,
! 780: AHCI_REG_GHC_HR) != 0) {
! 781: printf(" unable to reset controller\n");
! 782: return (1);
! 783: }
! 784: }
! 785:
! 786: /* enable ahci (global interrupts disabled) */
! 787: ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
! 788:
! 789: /* restore parameters */
! 790: ahci_write(sc, AHCI_REG_CAP, cap);
! 791: ahci_write(sc, AHCI_REG_PI, pi);
! 792:
! 793: /* check the revision */
! 794: reg = ahci_read(sc, AHCI_REG_VS);
! 795: switch (reg) {
! 796: case AHCI_REG_VS_0_95:
! 797: revision = "0.95";
! 798: break;
! 799: case AHCI_REG_VS_1_0:
! 800: revision = "1.0";
! 801: break;
! 802: case AHCI_REG_VS_1_1:
! 803: revision = "1.1";
! 804: break;
! 805:
! 806: default:
! 807: printf(" unsupported AHCI revision 0x%08x\n", reg);
! 808: return (1);
! 809: }
! 810:
! 811: printf(" AHCI %s", revision);
! 812:
! 813: return (0);
! 814: }
! 815:
! 816: int
! 817: ahci_port_alloc(struct ahci_softc *sc, u_int port)
! 818: {
! 819: struct ahci_port *ap;
! 820: struct ahci_ccb *ccb;
! 821: u_int64_t dva;
! 822: u_int32_t cmd;
! 823: struct ahci_cmd_hdr *hdr;
! 824: struct ahci_cmd_table *table;
! 825: int i, rc = ENOMEM;
! 826:
! 827: ap = malloc(sizeof(struct ahci_port), M_DEVBUF, M_NOWAIT);
! 828: if (ap == NULL) {
! 829: printf("%s: unable to allocate memory for port %d\n",
! 830: DEVNAME(sc), port);
! 831: goto reterr;
! 832: }
! 833: bzero(ap, sizeof(struct ahci_port));
! 834:
! 835: #ifdef AHCI_DEBUG
! 836: snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
! 837: DEVNAME(sc), port);
! 838: #endif
! 839: sc->sc_ports[port] = ap;
! 840:
! 841: if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
! 842: AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
! 843: printf("%s: unable to create register window for port %d\n",
! 844: DEVNAME(sc), port);
! 845: goto freeport;
! 846: }
! 847:
! 848: ap->ap_sc = sc;
! 849: #ifdef AHCI_COALESCE
! 850: ap->ap_num = port;
! 851: #endif
! 852: TAILQ_INIT(&ap->ap_ccb_free);
! 853: TAILQ_INIT(&ap->ap_ccb_pending);
! 854:
! 855: /* Disable port interrupts */
! 856: ahci_pwrite(ap, AHCI_PREG_IE, 0);
! 857:
! 858: /* Sec 10.1.2 - deinitialise port if it is already running */
! 859: cmd = ahci_pread(ap, AHCI_PREG_CMD);
! 860: if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
! 861: AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
! 862: ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
! 863: int r;
! 864:
! 865: r = ahci_port_stop(ap, 1);
! 866: if (r) {
! 867: printf("%s: unable to disable %s, ignoring port %d\n",
! 868: DEVNAME(sc), r == 2 ? "CR" : "FR", port);
! 869: rc = ENXIO;
! 870: goto freeport;
! 871: }
! 872:
! 873: /* Write DET to zero */
! 874: ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
! 875: }
! 876:
! 877: /* Allocate RFIS */
! 878: ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
! 879: if (ap->ap_dmamem_rfis == NULL)
! 880: goto nomem;
! 881:
! 882: /* Setup RFIS base address */
! 883: ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
! 884: dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
! 885: ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
! 886: ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
! 887:
! 888: /* Enable FIS reception and activate port. */
! 889: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
! 890: cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
! 891: ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
! 892:
! 893: /* Check whether port activated. Skip it if not. */
! 894: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
! 895: if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
! 896: rc = ENXIO;
! 897: goto freeport;
! 898: }
! 899:
! 900: /* Allocate a CCB for each command slot */
! 901: ap->ap_ccbs = malloc(sizeof(struct ahci_ccb) * sc->sc_ncmds, M_DEVBUF,
! 902: M_NOWAIT);
! 903: if (ap->ap_ccbs == NULL) {
! 904: printf("%s: unable to allocate command list for port %d\n",
! 905: DEVNAME(sc), port);
! 906: goto freeport;
! 907: }
! 908: bzero(ap->ap_ccbs, sizeof(struct ahci_ccb) * sc->sc_ncmds);
! 909:
! 910: /* Command List Structures and Command Tables */
! 911: ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
! 912: sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
! 913: ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
! 914: sc->sc_ncmds * sizeof(struct ahci_cmd_table));
! 915: if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) {
! 916: nomem:
! 917: printf("%s: unable to allocate DMA memory for port %d\n",
! 918: DEVNAME(sc), port);
! 919: goto freeport;
! 920: }
! 921:
! 922: /* Setup command list base address */
! 923: dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
! 924: ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
! 925: ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
! 926:
! 927: /* Split CCB allocation into CCBs and assign to command header/table */
! 928: hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
! 929: table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
! 930: for (i = 0; i < sc->sc_ncmds; i++) {
! 931: ccb = &ap->ap_ccbs[i];
! 932:
! 933: if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,
! 934: (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
! 935: &ccb->ccb_dmamap) != 0) {
! 936: printf("%s: unable to create dmamap for port %d "
! 937: "ccb %d\n", DEVNAME(sc), port, i);
! 938: goto freeport;
! 939: }
! 940:
! 941: ccb->ccb_slot = i;
! 942: ccb->ccb_port = ap;
! 943: ccb->ccb_cmd_hdr = &hdr[i];
! 944: ccb->ccb_cmd_table = &table[i];
! 945: dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
! 946: ccb->ccb_slot * sizeof(struct ahci_cmd_table);
! 947: ccb->ccb_cmd_hdr->ctba_hi = htole32((u_int32_t)(dva >> 32));
! 948: ccb->ccb_cmd_hdr->ctba_lo = htole32((u_int32_t)dva);
! 949:
! 950: ccb->ccb_xa.fis =
! 951: (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
! 952: ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
! 953: ccb->ccb_xa.tag = i;
! 954:
! 955: ccb->ccb_xa.ata_put_xfer = ahci_ata_put_xfer;
! 956:
! 957: ccb->ccb_xa.state = ATA_S_COMPLETE;
! 958: ahci_put_ccb(ccb);
! 959: }
! 960:
! 961: /* Wait for ICC change to complete */
! 962: ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC);
! 963:
! 964: /* Reset port */
! 965: rc = ahci_port_portreset(ap);
! 966: switch (rc) {
! 967: case ENODEV:
! 968: switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
! 969: case AHCI_PREG_SSTS_DET_DEV_NE:
! 970: printf("%s: device not communicating on port %d\n",
! 971: DEVNAME(sc), port);
! 972: break;
! 973: case AHCI_PREG_SSTS_DET_PHYOFFLINE:
! 974: printf("%s: PHY offline on port %d\n", DEVNAME(sc),
! 975: port);
! 976: break;
! 977: default:
! 978: DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
! 979: "on port %d\n", DEVNAME(sc), port);
! 980: break;
! 981: }
! 982: goto freeport;
! 983:
! 984: case EBUSY:
! 985: printf("%s: device on port %d didn't come ready, "
! 986: "TFD: 0x%b\n", DEVNAME(sc), port,
! 987: ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
! 988:
! 989: /* Try a soft reset to clear busy */
! 990: rc = ahci_port_softreset(ap);
! 991: if (rc) {
! 992: printf("%s: unable to communicate "
! 993: "with device on port %d\n", DEVNAME(sc), port);
! 994: goto freeport;
! 995: }
! 996: break;
! 997:
! 998: default:
! 999: break;
! 1000: }
! 1001: DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
! 1002: DEVNAME(sc), port);
! 1003:
! 1004: /* Enable command transfers on port */
! 1005: if (ahci_port_start(ap, 0)) {
! 1006: printf("%s: failed to start command DMA on port %d, "
! 1007: "disabling\n", DEVNAME(sc), port);
! 1008: rc = ENXIO; /* couldn't start port */
! 1009: }
! 1010:
! 1011: /* Flush interrupts for port */
! 1012: ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
! 1013: ahci_write(sc, AHCI_REG_IS, 1 << port);
! 1014:
! 1015: /* Enable port interrupts */
! 1016: ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
! 1017: AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE |
! 1018: AHCI_PREG_IE_UFE |
! 1019: #ifdef AHCI_COALESCE
! 1020: ((sc->sc_ccc_ports & (1 << port)) ? 0 : (AHCI_PREG_IE_SDBE |
! 1021: AHCI_PREG_IE_DHRE))
! 1022: #else
! 1023: AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE
! 1024: #endif
! 1025: );
! 1026:
! 1027: freeport:
! 1028: if (rc != 0)
! 1029: ahci_port_free(sc, port);
! 1030: reterr:
! 1031: return (rc);
! 1032: }
! 1033:
! 1034: void
! 1035: ahci_port_free(struct ahci_softc *sc, u_int port)
! 1036: {
! 1037: struct ahci_port *ap = sc->sc_ports[port];
! 1038: struct ahci_ccb *ccb;
! 1039:
! 1040: /* Ensure port is disabled and its interrupts are flushed */
! 1041: if (ap->ap_sc) {
! 1042: ahci_pwrite(ap, AHCI_PREG_CMD, 0);
! 1043: ahci_pwrite(ap, AHCI_PREG_IE, 0);
! 1044: ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
! 1045: ahci_write(sc, AHCI_REG_IS, 1 << port);
! 1046: }
! 1047:
! 1048: if (ap->ap_ccbs) {
! 1049: while ((ccb = ahci_get_ccb(ap)) != NULL)
! 1050: bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
! 1051: free(ap->ap_ccbs, M_DEVBUF);
! 1052: }
! 1053:
! 1054: if (ap->ap_dmamem_cmd_list)
! 1055: ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
! 1056: if (ap->ap_dmamem_rfis)
! 1057: ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
! 1058: if (ap->ap_dmamem_cmd_table)
! 1059: ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
! 1060:
! 1061: /* bus_space(9) says we dont free the subregions handle */
! 1062:
! 1063: free(ap, M_DEVBUF);
! 1064: sc->sc_ports[port] = NULL;
! 1065: }
! 1066:
! 1067: int
! 1068: ahci_port_start(struct ahci_port *ap, int fre_only)
! 1069: {
! 1070: u_int32_t r;
! 1071:
! 1072: /* Turn on FRE (and ST) */
! 1073: r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
! 1074: r |= AHCI_PREG_CMD_FRE;
! 1075: if (!fre_only)
! 1076: r |= AHCI_PREG_CMD_ST;
! 1077: ahci_pwrite(ap, AHCI_PREG_CMD, r);
! 1078:
! 1079: #ifdef AHCI_COALESCE
! 1080: /* (Re-)enable coalescing on the port. */
! 1081: if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
! 1082: ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
! 1083: ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
! 1084: ap->ap_sc->sc_ccc_ports_cur);
! 1085: }
! 1086: #endif
! 1087:
! 1088: /* Wait for FR to come on */
! 1089: if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR))
! 1090: return (2);
! 1091:
! 1092: /* Wait for CR to come on */
! 1093: if (!fre_only && ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR))
! 1094: return (1);
! 1095:
! 1096: return (0);
! 1097: }
! 1098:
! 1099: int
! 1100: ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
! 1101: {
! 1102: u_int32_t r;
! 1103:
! 1104: #ifdef AHCI_COALESCE
! 1105: /* Disable coalescing on the port while it is stopped. */
! 1106: if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
! 1107: ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
! 1108: ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
! 1109: ap->ap_sc->sc_ccc_ports_cur);
! 1110: }
! 1111: #endif
! 1112:
! 1113: /* Turn off ST (and FRE) */
! 1114: r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
! 1115: r &= ~AHCI_PREG_CMD_ST;
! 1116: if (stop_fis_rx)
! 1117: r &= ~AHCI_PREG_CMD_FRE;
! 1118: ahci_pwrite(ap, AHCI_PREG_CMD, r);
! 1119:
! 1120: /* Wait for CR to go off */
! 1121: if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR))
! 1122: return (1);
! 1123:
! 1124: /* Wait for FR to go off */
! 1125: if (stop_fis_rx && ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR))
! 1126: return (2);
! 1127:
! 1128: return (0);
! 1129: }
! 1130:
! 1131: /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
! 1132: int
! 1133: ahci_port_clo(struct ahci_port *ap)
! 1134: {
! 1135: struct ahci_softc *sc = ap->ap_sc;
! 1136: u_int32_t cmd;
! 1137:
! 1138: /* Only attempt CLO if supported by controller */
! 1139: if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO))
! 1140: return (1);
! 1141:
! 1142: /* Issue CLO */
! 1143: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
! 1144: #ifdef DIAGNOSTIC
! 1145: if (ISSET(cmd, AHCI_PREG_CMD_ST))
! 1146: printf("%s: CLO requested while port running\n", PORTNAME(ap));
! 1147: #endif
! 1148: ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
! 1149:
! 1150: /* Wait for completion */
! 1151: if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO)) {
! 1152: printf("%s: CLO did not complete\n", PORTNAME(ap));
! 1153: return (1);
! 1154: }
! 1155:
! 1156: return (0);
! 1157: }
! 1158:
! 1159: /* AHCI soft reset, Section 10.4.1 */
! 1160: int
! 1161: ahci_port_softreset(struct ahci_port *ap)
! 1162: {
! 1163: struct ahci_ccb *ccb = NULL;
! 1164: struct ahci_cmd_hdr *cmd_slot;
! 1165: u_int8_t *fis;
! 1166: int s, rc = EIO;
! 1167: u_int32_t cmd;
! 1168:
! 1169: DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
! 1170:
! 1171: s = splbio();
! 1172:
! 1173: /* Save previous command register state */
! 1174: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
! 1175:
! 1176: /* Idle port */
! 1177: if (ahci_port_stop(ap, 0)) {
! 1178: printf("%s: failed to stop port, cannot softreset\n",
! 1179: PORTNAME(ap));
! 1180: goto err;
! 1181: }
! 1182:
! 1183: /* Request CLO if device appears hung */
! 1184: if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |
! 1185: AHCI_PREG_TFD_STS_DRQ))
! 1186: ahci_port_clo(ap);
! 1187:
! 1188: /* Clear port errors to permit TFD transfer */
! 1189: ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
! 1190:
! 1191: /* Restart port */
! 1192: if (ahci_port_start(ap, 0)) {
! 1193: printf("%s: failed to start port, cannot softreset\n",
! 1194: PORTNAME(ap));
! 1195: goto err;
! 1196: }
! 1197:
! 1198: /* Check whether CLO worked */
! 1199: if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
! 1200: AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
! 1201: printf("%s: CLO %s, need port reset\n", PORTNAME(ap),
! 1202: ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)
! 1203: ? "failed" : "unsupported");
! 1204: rc = EBUSY;
! 1205: goto err;
! 1206: }
! 1207:
! 1208: /* Prep first D2H command with SRST feature & clear busy/reset flags */
! 1209: ccb = ahci_get_err_ccb(ap);
! 1210: cmd_slot = ccb->ccb_cmd_hdr;
! 1211: bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table));
! 1212:
! 1213: fis = ccb->ccb_cmd_table->cfis;
! 1214: fis[0] = 0x27; /* Host to device */
! 1215: fis[15] = 0x04; /* SRST DEVCTL */
! 1216:
! 1217: cmd_slot->prdtl = 0;
! 1218: cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
! 1219: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
! 1220: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
! 1221: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W); /* Write */
! 1222:
! 1223: ccb->ccb_xa.state = ATA_S_PENDING;
! 1224: if (ahci_poll(ccb, 1000, NULL) != 0)
! 1225: goto err;
! 1226:
! 1227: /* Prep second D2H command to read status and complete reset sequence */
! 1228: fis[0] = 0x27; /* Host to device */
! 1229: fis[15] = 0;
! 1230:
! 1231: cmd_slot->prdtl = 0;
! 1232: cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
! 1233: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W);
! 1234:
! 1235: ccb->ccb_xa.state = ATA_S_PENDING;
! 1236: if (ahci_poll(ccb, 1000, NULL) != 0)
! 1237: goto err;
! 1238:
! 1239: if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
! 1240: AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR)) {
! 1241: printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
! 1242: PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD),
! 1243: AHCI_PFMT_TFD_STS);
! 1244: rc = EBUSY;
! 1245: goto err;
! 1246: }
! 1247:
! 1248: rc = 0;
! 1249: err:
! 1250: if (ccb != NULL) {
! 1251: /* Abort our command, if it failed, by stopping command DMA. */
! 1252: if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
! 1253: printf("%s: stopping the port, softreset slot %d was "
! 1254: "still active.\n", PORTNAME(ap), ccb->ccb_slot);
! 1255: ahci_port_stop(ap, 0);
! 1256: }
! 1257: ccb->ccb_xa.state = ATA_S_ERROR;
! 1258: ahci_put_err_ccb(ccb);
! 1259: }
! 1260:
! 1261: /* Restore saved CMD register state */
! 1262: ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
! 1263:
! 1264: splx(s);
! 1265:
! 1266: return (rc);
! 1267: }
! 1268:
! 1269: /* AHCI port reset, Section 10.4.2 */
! 1270: int
! 1271: ahci_port_portreset(struct ahci_port *ap)
! 1272: {
! 1273: u_int32_t cmd, r;
! 1274: int rc;
! 1275:
! 1276: DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
! 1277:
! 1278: /* Save previous command register state */
! 1279: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
! 1280:
! 1281: /* Clear ST, ignoring failure */
! 1282: ahci_port_stop(ap, 0);
! 1283:
! 1284: /* Perform device detection */
! 1285: ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
! 1286: delay(10000);
! 1287: r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_SPD_ANY |
! 1288: AHCI_PREG_SCTL_DET_INIT;
! 1289: ahci_pwrite(ap, AHCI_PREG_SCTL, r);
! 1290: delay(10000); /* wait at least 1ms for COMRESET to be sent */
! 1291: r &= ~AHCI_PREG_SCTL_DET_INIT;
! 1292: r |= AHCI_PREG_SCTL_DET_NONE;
! 1293: ahci_pwrite(ap, AHCI_PREG_SCTL, r);
! 1294: delay(10000);
! 1295:
! 1296: /* Wait for device to be detected and communications established */
! 1297: if (ahci_pwait_eq(ap, AHCI_PREG_SSTS, AHCI_PREG_SSTS_DET,
! 1298: AHCI_PREG_SSTS_DET_DEV)) {
! 1299: rc = ENODEV;
! 1300: goto err;
! 1301: }
! 1302:
! 1303: /* Clear SERR (incl X bit), so TFD can update */
! 1304: ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
! 1305:
! 1306: /* Wait for device to become ready */
! 1307: /* XXX maybe more than the default wait is appropriate here? */
! 1308: if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
! 1309: AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR)) {
! 1310: rc = EBUSY;
! 1311: goto err;
! 1312: }
! 1313:
! 1314: rc = 0;
! 1315: err:
! 1316: /* Restore preserved port state */
! 1317: ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
! 1318:
! 1319: return (rc);
! 1320: }
! 1321:
! 1322: int
! 1323: ahci_load_prdt(struct ahci_ccb *ccb)
! 1324: {
! 1325: struct ahci_port *ap = ccb->ccb_port;
! 1326: struct ahci_softc *sc = ap->ap_sc;
! 1327: struct ata_xfer *xa = &ccb->ccb_xa;
! 1328: struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt, *prd;
! 1329: bus_dmamap_t dmap = ccb->ccb_dmamap;
! 1330: struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
! 1331: u_int64_t addr;
! 1332: int i, error;
! 1333:
! 1334: if (xa->datalen == 0) {
! 1335: ccb->ccb_cmd_hdr->prdtl = 0;
! 1336: return (0);
! 1337: }
! 1338:
! 1339: error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
! 1340: (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
! 1341: if (error != 0) {
! 1342: printf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
! 1343: return (1);
! 1344: }
! 1345:
! 1346: for (i = 0; i < dmap->dm_nsegs; i++) {
! 1347: prd = &prdt[i];
! 1348:
! 1349: addr = dmap->dm_segs[i].ds_addr;
! 1350: prd->dba_hi = htole32((u_int32_t)(addr >> 32));
! 1351: prd->dba_lo = htole32((u_int32_t)addr);
! 1352: #ifdef DIAGNOSTIC
! 1353: if (addr & 1) {
! 1354: printf("%s: requested DMA at an odd address %llx\n",
! 1355: PORTNAME(ap), (unsigned long long)addr);
! 1356: return (1);
! 1357: }
! 1358: if (dmap->dm_segs[i].ds_len & 1) {
! 1359: printf("%s: requested DMA length %d is not even\n",
! 1360: PORTNAME(ap), (int)dmap->dm_segs[i].ds_len);
! 1361: return (1);
! 1362: }
! 1363: #endif
! 1364: prd->flags = htole32(dmap->dm_segs[i].ds_len - 1);
! 1365: }
! 1366: if (xa->flags & ATA_F_PIO)
! 1367: prd->flags |= htole32(AHCI_PRDT_FLAG_INTR);
! 1368:
! 1369: cmd_slot->prdtl = htole16(dmap->dm_nsegs);
! 1370:
! 1371: bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
! 1372: (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
! 1373: BUS_DMASYNC_PREWRITE);
! 1374:
! 1375: return (0);
! 1376: }
! 1377:
! 1378: void
! 1379: ahci_unload_prdt(struct ahci_ccb *ccb)
! 1380: {
! 1381: struct ahci_port *ap = ccb->ccb_port;
! 1382: struct ahci_softc *sc = ap->ap_sc;
! 1383: struct ata_xfer *xa = &ccb->ccb_xa;
! 1384: bus_dmamap_t dmap = ccb->ccb_dmamap;
! 1385:
! 1386: if (xa->datalen != 0) {
! 1387: bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
! 1388: (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
! 1389: BUS_DMASYNC_POSTWRITE);
! 1390:
! 1391: bus_dmamap_unload(sc->sc_dmat, dmap);
! 1392:
! 1393: if (ccb->ccb_xa.flags & ATA_F_NCQ)
! 1394: xa->resid = 0;
! 1395: else
! 1396: xa->resid = xa->datalen -
! 1397: letoh32(ccb->ccb_cmd_hdr->prdbc);
! 1398: }
! 1399: }
! 1400:
! 1401: int
! 1402: ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
! 1403: {
! 1404: struct ahci_port *ap = ccb->ccb_port;
! 1405: int s;
! 1406:
! 1407: s = splbio();
! 1408: ahci_start(ccb);
! 1409: do {
! 1410: if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),
! 1411: 1 << ccb->ccb_slot)) {
! 1412: splx(s);
! 1413: return (0);
! 1414: }
! 1415:
! 1416: delay(1000);
! 1417: } while (--timeout > 0);
! 1418:
! 1419: /* Run timeout while at splbio, otherwise ahci_intr could interfere. */
! 1420: if (timeout_fn != NULL)
! 1421: timeout_fn(ccb);
! 1422:
! 1423: splx(s);
! 1424:
! 1425: return (1);
! 1426: }
! 1427:
! 1428: void
! 1429: ahci_start(struct ahci_ccb *ccb)
! 1430: {
! 1431: struct ahci_port *ap = ccb->ccb_port;
! 1432: struct ahci_softc *sc = ap->ap_sc;
! 1433:
! 1434: KASSERT(ccb->ccb_xa.state == ATA_S_PENDING);
! 1435:
! 1436: /* Zero transferred byte count before transfer */
! 1437: ccb->ccb_cmd_hdr->prdbc = 0;
! 1438:
! 1439: /* Sync command list entry and corresponding command table entry */
! 1440: bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
! 1441: ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
! 1442: sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE);
! 1443: bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
! 1444: ccb->ccb_slot * sizeof(struct ahci_cmd_table),
! 1445: sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE);
! 1446:
! 1447: /* Prepare RFIS area for write by controller */
! 1448: bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
! 1449: sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD);
! 1450:
! 1451: if (ccb->ccb_xa.flags & ATA_F_NCQ) {
! 1452: /* Issue NCQ commands only when there are no outstanding
! 1453: * standard commands. */
! 1454: if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending))
! 1455: TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
! 1456: else {
! 1457: KASSERT(ap->ap_active_cnt == 0);
! 1458: ap->ap_sactive |= (1 << ccb->ccb_slot);
! 1459: ccb->ccb_xa.state = ATA_S_ONCHIP;
! 1460: ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
! 1461: ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
! 1462: }
! 1463: } else {
! 1464: /* Wait for all NCQ commands to finish before issuing standard
! 1465: * command. */
! 1466: if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
! 1467: TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
! 1468: else if (ap->ap_active_cnt < 2) {
! 1469: ap->ap_active |= 1 << ccb->ccb_slot;
! 1470: ccb->ccb_xa.state = ATA_S_ONCHIP;
! 1471: ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
! 1472: ap->ap_active_cnt++;
! 1473: }
! 1474: }
! 1475: }
! 1476:
! 1477: void
! 1478: ahci_issue_pending_ncq_commands(struct ahci_port *ap)
! 1479: {
! 1480: struct ahci_ccb *nextccb;
! 1481: u_int32_t sact_change = 0;
! 1482:
! 1483: KASSERT(ap->ap_active_cnt == 0);
! 1484:
! 1485: nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
! 1486: if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
! 1487: return;
! 1488:
! 1489: /* Start all the NCQ commands at the head of the pending list. */
! 1490: do {
! 1491: TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
! 1492: sact_change |= 1 << nextccb->ccb_slot;
! 1493: nextccb->ccb_xa.state = ATA_S_ONCHIP;
! 1494: nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
! 1495: } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ));
! 1496:
! 1497: ap->ap_sactive |= sact_change;
! 1498: ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
! 1499: ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
! 1500:
! 1501: return;
! 1502: }
! 1503:
! 1504: void
! 1505: ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
! 1506: {
! 1507: struct ahci_ccb *nextccb;
! 1508:
! 1509: nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
! 1510: if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) {
! 1511: KASSERT(last_was_ncq == 0); /* otherwise it should have
! 1512: * been started already. */
! 1513:
! 1514: /* Issue NCQ commands only when there are no outstanding
! 1515: * standard commands. */
! 1516: ap->ap_active_cnt--;
! 1517: if (ap->ap_active == 0)
! 1518: ahci_issue_pending_ncq_commands(ap);
! 1519: else
! 1520: KASSERT(ap->ap_active_cnt == 1);
! 1521: } else if (nextccb) {
! 1522: if (ap->ap_sactive != 0 || last_was_ncq)
! 1523: KASSERT(ap->ap_active_cnt == 0);
! 1524:
! 1525: /* Wait for all NCQ commands to finish before issuing standard
! 1526: * command. */
! 1527: if (ap->ap_sactive != 0)
! 1528: return;
! 1529:
! 1530: /* Keep up to 2 standard commands on-chip at a time. */
! 1531: do {
! 1532: TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
! 1533: ap->ap_active |= 1 << nextccb->ccb_slot;
! 1534: nextccb->ccb_xa.state = ATA_S_ONCHIP;
! 1535: ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
! 1536: if (last_was_ncq)
! 1537: ap->ap_active_cnt++;
! 1538: if (ap->ap_active_cnt == 2)
! 1539: break;
! 1540: KASSERT(ap->ap_active_cnt == 1);
! 1541: nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
! 1542: } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ));
! 1543: } else if (!last_was_ncq) {
! 1544: KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2);
! 1545:
! 1546: /* Standard command finished, none waiting to start. */
! 1547: ap->ap_active_cnt--;
! 1548: } else {
! 1549: KASSERT(ap->ap_active_cnt == 0);
! 1550:
! 1551: /* NCQ command finished. */
! 1552: }
! 1553: }
! 1554:
! 1555: int
! 1556: ahci_intr(void *arg)
! 1557: {
! 1558: struct ahci_softc *sc = arg;
! 1559: u_int32_t is, ack = 0;
! 1560: int port;
! 1561:
! 1562: /* Read global interrupt status */
! 1563: is = ahci_read(sc, AHCI_REG_IS);
! 1564: if (is == 0)
! 1565: return (0);
! 1566: ack = is;
! 1567:
! 1568: #ifdef AHCI_COALESCE
! 1569: /* Check coalescing interrupt first */
! 1570: if (is & sc->sc_ccc_mask) {
! 1571: DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
! 1572: DEVNAME(sc));
! 1573: is &= ~sc->sc_ccc_mask;
! 1574: is |= sc->sc_ccc_ports_cur;
! 1575: }
! 1576: #endif
! 1577:
! 1578: /* Process interrupts for each port */
! 1579: while (is) {
! 1580: port = ffs(is) - 1;
! 1581: if (sc->sc_ports[port])
! 1582: ahci_port_intr(sc->sc_ports[port],
! 1583: AHCI_PREG_CI_ALL_SLOTS);
! 1584: is &= ~(1 << port);
! 1585: }
! 1586:
! 1587: /* Finally, acknowledge global interrupt */
! 1588: ahci_write(sc, AHCI_REG_IS, ack);
! 1589:
! 1590: return (1);
! 1591: }
! 1592:
! 1593: u_int32_t
! 1594: ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
! 1595: {
! 1596: struct ahci_softc *sc = ap->ap_sc;
! 1597: u_int32_t is, ci_saved, ci_masked, processed = 0;
! 1598: int slot, need_restart = 0;
! 1599: struct ahci_ccb *ccb;
! 1600: volatile u_int32_t *active;
! 1601: #ifdef DIAGNOSTIC
! 1602: u_int32_t tmp;
! 1603: #endif
! 1604:
! 1605: is = ahci_pread(ap, AHCI_PREG_IS);
! 1606:
! 1607: /* Ack port interrupt only if checking all command slots. */
! 1608: if (ci_mask == AHCI_PREG_CI_ALL_SLOTS)
! 1609: ahci_pwrite(ap, AHCI_PREG_IS, is);
! 1610:
! 1611: if (is)
! 1612: DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
! 1613: is, AHCI_PFMT_IS);
! 1614:
! 1615: if (ap->ap_sactive) {
! 1616: /* Active NCQ commands - use SActive instead of CI */
! 1617: KASSERT(ap->ap_active == 0);
! 1618: KASSERT(ap->ap_active_cnt == 0);
! 1619: ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
! 1620: active = &ap->ap_sactive;
! 1621: } else {
! 1622: /* Save CI */
! 1623: ci_saved = ahci_pread(ap, AHCI_PREG_CI);
! 1624: active = &ap->ap_active;
! 1625: }
! 1626:
! 1627: /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
! 1628: if (is & AHCI_PREG_IS_TFES) {
! 1629: u_int32_t tfd, serr;
! 1630: int err_slot;
! 1631:
! 1632: tfd = ahci_pread(ap, AHCI_PREG_TFD);
! 1633: serr = ahci_pread(ap, AHCI_PREG_SERR);
! 1634:
! 1635: if (ap->ap_sactive == 0) {
! 1636: /* Errored slot is easy to determine from CMD. */
! 1637: err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,
! 1638: AHCI_PREG_CMD));
! 1639: ccb = &ap->ap_ccbs[err_slot];
! 1640:
! 1641: /* Preserve received taskfile data from the RFIS. */
! 1642: memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
! 1643: sizeof(struct ata_fis_d2h));
! 1644: } else
! 1645: err_slot = -1; /* Must extract error from log page */
! 1646:
! 1647: DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
! 1648: " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
! 1649: AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
! 1650: AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
! 1651: AHCI_PFMT_SERR_DIAG);
! 1652:
! 1653: /* Turn off ST to clear CI and SACT. */
! 1654: ahci_port_stop(ap, 0);
! 1655: need_restart = 1;
! 1656:
! 1657: /* Clear SERR to enable capturing new errors. */
! 1658: ahci_pwrite(ap, AHCI_PREG_SERR, serr);
! 1659:
! 1660: /* Acknowledge the interrupts we can recover from. */
! 1661: ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES |
! 1662: AHCI_PREG_IS_IFS);
! 1663: is = ahci_pread(ap, AHCI_PREG_IS);
! 1664:
! 1665: /* If device hasn't cleared its busy status, try to idle it. */
! 1666: if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
! 1667: printf("%s: attempting to idle device\n", PORTNAME(ap));
! 1668: if (ahci_port_softreset(ap)) {
! 1669: printf("%s: failed to soft reset device\n",
! 1670: PORTNAME(ap));
! 1671: if (ahci_port_portreset(ap)) {
! 1672: printf("%s: failed to port reset "
! 1673: "device, give up on it\n",
! 1674: PORTNAME(ap));
! 1675: goto fatal;
! 1676: }
! 1677: }
! 1678:
! 1679: /* Had to reset device, can't gather extended info. */
! 1680: } else if (ap->ap_sactive) {
! 1681: /* Recover the NCQ error from log page 10h. */
! 1682: ahci_port_read_ncq_error(ap, &err_slot);
! 1683: if (err_slot < 0)
! 1684: goto failall;
! 1685:
! 1686: DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
! 1687: PORTNAME(ap), err_slot);
! 1688:
! 1689: ccb = &ap->ap_ccbs[err_slot];
! 1690: } else {
! 1691: /* Didn't reset, could gather extended info from log. */
! 1692: }
! 1693:
! 1694: /*
! 1695: * If we couldn't determine the errored slot, reset the port
! 1696: * and fail all the active slots.
! 1697: */
! 1698: if (err_slot == -1) {
! 1699: if (ahci_port_softreset(ap) != 0 &&
! 1700: ahci_port_portreset(ap) != 0) {
! 1701: printf("%s: couldn't reset after NCQ error, "
! 1702: "disabling device.\n", PORTNAME(ap));
! 1703: goto fatal;
! 1704: }
! 1705: printf("%s: couldn't recover NCQ error, failing "
! 1706: "all outstanding commands.\n", PORTNAME(ap));
! 1707: goto failall;
! 1708: }
! 1709:
! 1710: /* Clear the failed command in saved CI so completion runs. */
! 1711: ci_saved &= ~(1 << err_slot);
! 1712:
! 1713: /* Note the error in the ata_xfer. */
! 1714: KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
! 1715: ccb->ccb_xa.state = ATA_S_ERROR;
! 1716:
! 1717: #ifdef DIAGNOSTIC
! 1718: /* There may only be one outstanding standard command now. */
! 1719: if (ap->ap_sactive == 0) {
! 1720: tmp = ci_saved;
! 1721: if (tmp) {
! 1722: slot = ffs(tmp) - 1;
! 1723: tmp &= ~(1 << slot);
! 1724: KASSERT(tmp == 0);
! 1725: }
! 1726: }
! 1727: #endif
! 1728: }
! 1729:
! 1730: /* Check for remaining errors - they are fatal. */
! 1731: if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
! 1732: AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
! 1733: printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
! 1734: PORTNAME(ap), is, AHCI_PFMT_IS);
! 1735:
! 1736: /* XXX try recovery first */
! 1737: goto fatal;
! 1738: }
! 1739:
! 1740: /* Fail all outstanding commands if we know the port won't recover. */
! 1741: if (ap->ap_state == AP_S_FATAL_ERROR) {
! 1742: fatal:
! 1743: ap->ap_state = AP_S_FATAL_ERROR;
! 1744: failall:
! 1745:
! 1746: /* Ensure port is shut down. */
! 1747: ahci_port_stop(ap, 1);
! 1748:
! 1749: /* Error all the active slots. */
! 1750: ci_masked = ci_saved & *active;
! 1751: while (ci_masked) {
! 1752: slot = ffs(ci_masked) - 1;
! 1753: ccb = &ap->ap_ccbs[slot];
! 1754: ci_masked &= ~(1 << slot);
! 1755: ccb->ccb_xa.state = ATA_S_ERROR;
! 1756: }
! 1757:
! 1758: /* Run completion for all active slots. */
! 1759: ci_saved &= ~*active;
! 1760:
! 1761: /* Don't restart the port if our problems were deemed fatal. */
! 1762: if (ap->ap_state == AP_S_FATAL_ERROR)
! 1763: need_restart = 0;
! 1764: }
! 1765:
! 1766: /*
! 1767: * CCB completion is detected by noticing its slot's bit in CI has
! 1768: * changed to zero some time after we activated it.
! 1769: * If we are polling, we may only be interested in particular slot(s).
! 1770: */
! 1771: ci_masked = ~ci_saved & *active & ci_mask;
! 1772: while (ci_masked) {
! 1773: slot = ffs(ci_masked) - 1;
! 1774: ccb = &ap->ap_ccbs[slot];
! 1775: ci_masked &= ~(1 << slot);
! 1776:
! 1777: DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
! 1778: PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
! 1779: " (error)" : "");
! 1780:
! 1781: bus_dmamap_sync(sc->sc_dmat,
! 1782: AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
! 1783: ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
! 1784: sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE);
! 1785:
! 1786: bus_dmamap_sync(sc->sc_dmat,
! 1787: AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
! 1788: ccb->ccb_slot * sizeof(struct ahci_cmd_table),
! 1789: sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE);
! 1790:
! 1791: bus_dmamap_sync(sc->sc_dmat,
! 1792: AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
! 1793: sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD);
! 1794:
! 1795: *active &= ~(1 << ccb->ccb_slot);
! 1796: ccb->ccb_done(ccb);
! 1797:
! 1798: processed |= 1 << ccb->ccb_slot;
! 1799: }
! 1800:
! 1801: if (need_restart) {
! 1802: /* Restart command DMA on the port */
! 1803: ahci_port_start(ap, 0);
! 1804:
! 1805: /* Re-enable outstanding commands on port. */
! 1806: if (ci_saved) {
! 1807: #ifdef DIAGNOSTIC
! 1808: tmp = ci_saved;
! 1809: while (tmp) {
! 1810: slot = ffs(tmp) - 1;
! 1811: tmp &= ~(1 << slot);
! 1812: ccb = &ap->ap_ccbs[slot];
! 1813: KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
! 1814: KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
! 1815: (!!ap->ap_sactive));
! 1816: }
! 1817: #endif
! 1818: DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
! 1819: "re-enabling%s slots %08x\n", PORTNAME(ap),
! 1820: ap->ap_sactive ? " NCQ" : "", ci_saved);
! 1821:
! 1822: if (ap->ap_sactive)
! 1823: ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
! 1824: ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
! 1825: }
! 1826: }
! 1827:
! 1828: return (processed);
! 1829: }
! 1830:
! 1831: struct ahci_ccb *
! 1832: ahci_get_ccb(struct ahci_port *ap)
! 1833: {
! 1834: struct ahci_ccb *ccb;
! 1835:
! 1836: ccb = TAILQ_FIRST(&ap->ap_ccb_free);
! 1837: if (ccb != NULL) {
! 1838: KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
! 1839: TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
! 1840: ccb->ccb_xa.state = ATA_S_SETUP;
! 1841: }
! 1842:
! 1843: return (ccb);
! 1844: }
! 1845:
! 1846: void
! 1847: ahci_put_ccb(struct ahci_ccb *ccb)
! 1848: {
! 1849: struct ahci_port *ap = ccb->ccb_port;
! 1850:
! 1851: #ifdef DIAGNOSTIC
! 1852: if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
! 1853: ccb->ccb_xa.state != ATA_S_TIMEOUT &&
! 1854: ccb->ccb_xa.state != ATA_S_ERROR) {
! 1855: printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
! 1856: "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
! 1857: ccb->ccb_slot);
! 1858: }
! 1859: #endif
! 1860:
! 1861: ccb->ccb_xa.state = ATA_S_PUT;
! 1862: TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
! 1863: }
! 1864:
! 1865: struct ahci_ccb *
! 1866: ahci_get_err_ccb(struct ahci_port *ap)
! 1867: {
! 1868: struct ahci_ccb *err_ccb;
! 1869: u_int32_t sact;
! 1870:
! 1871: splassert(IPL_BIO);
! 1872:
! 1873: /* No commands may be active on the chip. */
! 1874: sact = ahci_pread(ap, AHCI_PREG_SACT);
! 1875: if (sact != 0)
! 1876: printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
! 1877: KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
! 1878:
! 1879: #ifdef DIAGNOSTIC
! 1880: KASSERT(ap->ap_err_busy == 0);
! 1881: ap->ap_err_busy = 1;
! 1882: #endif
! 1883: /* Save outstanding command state. */
! 1884: ap->ap_err_saved_active = ap->ap_active;
! 1885: ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
! 1886: ap->ap_err_saved_sactive = ap->ap_sactive;
! 1887:
! 1888: /*
! 1889: * Pretend we have no commands outstanding, so that completions won't
! 1890: * run prematurely.
! 1891: */
! 1892: ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
! 1893:
! 1894: /*
! 1895: * Grab a CCB to use for error recovery. This should never fail, as
! 1896: * we ask atascsi to reserve one for us at init time.
! 1897: */
! 1898: err_ccb = ahci_get_ccb(ap);
! 1899: KASSERT(err_ccb != NULL);
! 1900: err_ccb->ccb_xa.flags = 0;
! 1901: err_ccb->ccb_done = ahci_empty_done;
! 1902:
! 1903: return err_ccb;
! 1904: }
! 1905:
! 1906: void
! 1907: ahci_put_err_ccb(struct ahci_ccb *ccb)
! 1908: {
! 1909: struct ahci_port *ap = ccb->ccb_port;
! 1910: u_int32_t sact;
! 1911:
! 1912: splassert(IPL_BIO);
! 1913:
! 1914: #ifdef DIAGNOSTIC
! 1915: KASSERT(ap->ap_err_busy);
! 1916: #endif
! 1917: /* No commands may be active on the chip */
! 1918: sact = ahci_pread(ap, AHCI_PREG_SACT);
! 1919: if (sact != 0)
! 1920: printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact);
! 1921: KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
! 1922:
! 1923: /* Done with the CCB */
! 1924: ahci_put_ccb(ccb);
! 1925:
! 1926: /* Restore outstanding command state */
! 1927: ap->ap_sactive = ap->ap_err_saved_sactive;
! 1928: ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
! 1929: ap->ap_active = ap->ap_err_saved_active;
! 1930:
! 1931: #ifdef DIAGNOSTIC
! 1932: ap->ap_err_busy = 0;
! 1933: #endif
! 1934: }
! 1935:
! 1936: int
! 1937: ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp)
! 1938: {
! 1939: struct ahci_ccb *ccb;
! 1940: struct ahci_cmd_hdr *cmd_slot;
! 1941: u_int32_t cmd;
! 1942: struct ata_fis_h2d *fis;
! 1943: int rc = EIO;
! 1944:
! 1945: DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
! 1946:
! 1947: /* Save command register state. */
! 1948: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
! 1949:
! 1950: /* Port should have been idled already. Start it. */
! 1951: KASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
! 1952: ahci_port_start(ap, 0);
! 1953:
! 1954: /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
! 1955: ccb = ahci_get_err_ccb(ap);
! 1956: ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
! 1957: ccb->ccb_xa.data = ap->ap_err_scratch;
! 1958: ccb->ccb_xa.datalen = 512;
! 1959: cmd_slot = ccb->ccb_cmd_hdr;
! 1960: bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table));
! 1961:
! 1962: fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
! 1963: fis->type = ATA_FIS_TYPE_H2D;
! 1964: fis->flags = ATA_H2D_FLAGS_CMD;
! 1965: fis->command = ATA_C_READ_LOG_EXT;
! 1966: fis->lba_low = 0x10; /* queued error log page (10h) */
! 1967: fis->sector_count = 1; /* number of sectors (1) */
! 1968: fis->sector_count_exp = 0;
! 1969: fis->lba_mid = 0; /* starting offset */
! 1970: fis->lba_mid_exp = 0;
! 1971: fis->device = 0;
! 1972:
! 1973: cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
! 1974:
! 1975: if (ahci_load_prdt(ccb) != 0) {
! 1976: rc = ENOMEM; /* XXX caller must abort all commands */
! 1977: goto err;
! 1978: }
! 1979:
! 1980: ccb->ccb_xa.state = ATA_S_PENDING;
! 1981: if (ahci_poll(ccb, 1000, NULL) != 0)
! 1982: goto err;
! 1983:
! 1984: rc = 0;
! 1985: err:
! 1986: /* Abort our command, if it failed, by stopping command DMA. */
! 1987: if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
! 1988: printf("%s: log page read failed, slot %d was still active.\n",
! 1989: PORTNAME(ap), ccb->ccb_slot);
! 1990: ahci_port_stop(ap, 0);
! 1991: }
! 1992:
! 1993: /* Done with the error CCB now. */
! 1994: ahci_unload_prdt(ccb);
! 1995: ahci_put_err_ccb(ccb);
! 1996:
! 1997: /* Extract failed register set and tags from the scratch space. */
! 1998: if (rc == 0) {
! 1999: struct ata_log_page_10h *log;
! 2000: int err_slot;
! 2001:
! 2002: log = (struct ata_log_page_10h *)ap->ap_err_scratch;
! 2003: if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
! 2004: /* Not queued bit was set - wasn't an NCQ error? */
! 2005: printf("%s: read NCQ error page, but not an NCQ "
! 2006: "error?\n", PORTNAME(ap));
! 2007: rc = ESRCH;
! 2008: } else {
! 2009: /* Copy back the log record as a D2H register FIS. */
! 2010: *err_slotp = err_slot = log->err_regs.type &
! 2011: ATA_LOG_10H_TYPE_TAG_MASK;
! 2012:
! 2013: ccb = &ap->ap_ccbs[err_slot];
! 2014: memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
! 2015: sizeof(struct ata_fis_d2h));
! 2016: ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
! 2017: ccb->ccb_xa.rfis.flags = 0;
! 2018: }
! 2019: }
! 2020:
! 2021: /* Restore saved CMD register state */
! 2022: ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
! 2023:
! 2024: return (rc);
! 2025: }
! 2026:
! 2027: struct ahci_dmamem *
! 2028: ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
! 2029: {
! 2030: struct ahci_dmamem *adm;
! 2031: int nsegs;
! 2032:
! 2033: adm = malloc(sizeof(struct ahci_dmamem), M_DEVBUF, M_NOWAIT);
! 2034: if (adm == NULL)
! 2035: return (NULL);
! 2036:
! 2037: bzero(adm, sizeof(struct ahci_dmamem));
! 2038: adm->adm_size = size;
! 2039:
! 2040: if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
! 2041: BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
! 2042: goto admfree;
! 2043:
! 2044: if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,
! 2045: 1, &nsegs, BUS_DMA_NOWAIT) != 0)
! 2046: goto destroy;
! 2047:
! 2048: if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,
! 2049: &adm->adm_kva, BUS_DMA_NOWAIT) != 0)
! 2050: goto free;
! 2051:
! 2052: if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,
! 2053: NULL, BUS_DMA_NOWAIT) != 0)
! 2054: goto unmap;
! 2055:
! 2056: bzero(adm->adm_kva, size);
! 2057:
! 2058: return (adm);
! 2059:
! 2060: unmap:
! 2061: bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size);
! 2062: free:
! 2063: bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
! 2064: destroy:
! 2065: bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
! 2066: admfree:
! 2067: free(adm, M_DEVBUF);
! 2068:
! 2069: return (NULL);
! 2070: }
! 2071:
! 2072: void
! 2073: ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
! 2074: {
! 2075: bus_dmamap_unload(sc->sc_dmat, adm->adm_map);
! 2076: bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size);
! 2077: bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
! 2078: bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
! 2079: free(adm, M_DEVBUF);
! 2080: }
! 2081:
! 2082: u_int32_t
! 2083: ahci_read(struct ahci_softc *sc, bus_size_t r)
! 2084: {
! 2085: bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
! 2086: BUS_SPACE_BARRIER_READ);
! 2087: return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
! 2088: }
! 2089:
! 2090: void
! 2091: ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
! 2092: {
! 2093: bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
! 2094: bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
! 2095: BUS_SPACE_BARRIER_WRITE);
! 2096: }
! 2097:
! 2098: int
! 2099: ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
! 2100: u_int32_t target)
! 2101: {
! 2102: int i;
! 2103:
! 2104: for (i = 0; i < 1000; i++) {
! 2105: if ((ahci_read(sc, r) & mask) != target)
! 2106: return (0);
! 2107: delay(1000);
! 2108: }
! 2109:
! 2110: return (1);
! 2111: }
! 2112:
! 2113: u_int32_t
! 2114: ahci_pread(struct ahci_port *ap, bus_size_t r)
! 2115: {
! 2116: bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
! 2117: BUS_SPACE_BARRIER_READ);
! 2118: return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
! 2119: }
! 2120:
! 2121: void
! 2122: ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
! 2123: {
! 2124: bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
! 2125: bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
! 2126: BUS_SPACE_BARRIER_WRITE);
! 2127: }
! 2128:
! 2129: int
! 2130: ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
! 2131: u_int32_t target)
! 2132: {
! 2133: int i;
! 2134:
! 2135: for (i = 0; i < 1000; i++) {
! 2136: if ((ahci_pread(ap, r) & mask) == target)
! 2137: return (0);
! 2138: delay(1000);
! 2139: }
! 2140:
! 2141: return (1);
! 2142: }
! 2143:
! 2144: int
! 2145: ahci_ata_probe(void *xsc, int port)
! 2146: {
! 2147: struct ahci_softc *sc = xsc;
! 2148: struct ahci_port *ap = sc->sc_ports[port];
! 2149: u_int32_t sig;
! 2150:
! 2151: if (ap == NULL)
! 2152: return (ATA_PORT_T_NONE);
! 2153:
! 2154: sig = ahci_pread(ap, AHCI_PREG_SIG);
! 2155: if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000))
! 2156: return (ATA_PORT_T_ATAPI);
! 2157: else
! 2158: return (ATA_PORT_T_DISK);
! 2159: }
! 2160:
! 2161: struct ata_xfer *
! 2162: ahci_ata_get_xfer(void *aaa_cookie, int port)
! 2163: {
! 2164: struct ahci_softc *sc = aaa_cookie;
! 2165: struct ahci_port *ap = sc->sc_ports[port];
! 2166: struct ahci_ccb *ccb;
! 2167:
! 2168: splassert(IPL_BIO);
! 2169:
! 2170: ccb = ahci_get_ccb(ap);
! 2171: if (ccb == NULL) {
! 2172: DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
! 2173: PORTNAME(ap));
! 2174: return (NULL);
! 2175: }
! 2176:
! 2177: DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
! 2178: PORTNAME(ap), ccb->ccb_slot);
! 2179:
! 2180: return ((struct ata_xfer *)ccb);
! 2181: }
! 2182:
! 2183: void
! 2184: ahci_ata_put_xfer(struct ata_xfer *xa)
! 2185: {
! 2186: struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
! 2187:
! 2188: splassert(IPL_BIO);
! 2189:
! 2190: DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
! 2191:
! 2192: ahci_put_ccb(ccb);
! 2193: }
! 2194:
! 2195: int
! 2196: ahci_ata_cmd(struct ata_xfer *xa)
! 2197: {
! 2198: struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
! 2199: struct ahci_cmd_hdr *cmd_slot;
! 2200: int s;
! 2201:
! 2202: KASSERT(xa->state == ATA_S_SETUP);
! 2203:
! 2204: if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
! 2205: goto failcmd;
! 2206:
! 2207: ccb->ccb_done = ahci_ata_cmd_done;
! 2208:
! 2209: cmd_slot = ccb->ccb_cmd_hdr;
! 2210: cmd_slot->flags = htole16(5); /* FIS length (in DWORDs) */
! 2211:
! 2212: if (xa->flags & ATA_F_WRITE)
! 2213: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W);
! 2214:
! 2215: if (xa->flags & ATA_F_PACKET)
! 2216: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_A);
! 2217:
! 2218: if (ahci_load_prdt(ccb) != 0)
! 2219: goto failcmd;
! 2220:
! 2221: timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
! 2222:
! 2223: xa->state = ATA_S_PENDING;
! 2224:
! 2225: if (xa->flags & ATA_F_POLL) {
! 2226: ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
! 2227: return (ATA_COMPLETE);
! 2228: }
! 2229:
! 2230: timeout_add(&xa->stimeout, (xa->timeout * hz) / 1000);
! 2231:
! 2232: s = splbio();
! 2233: ahci_start(ccb);
! 2234: splx(s);
! 2235: return (ATA_QUEUED);
! 2236:
! 2237: failcmd:
! 2238: s = splbio();
! 2239: xa->state = ATA_S_ERROR;
! 2240: xa->complete(xa);
! 2241: splx(s);
! 2242: return (ATA_ERROR);
! 2243: }
! 2244:
! 2245: void
! 2246: ahci_ata_cmd_done(struct ahci_ccb *ccb)
! 2247: {
! 2248: struct ata_xfer *xa = &ccb->ccb_xa;
! 2249:
! 2250: timeout_del(&xa->stimeout);
! 2251:
! 2252: if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
! 2253: ahci_issue_pending_commands(ccb->ccb_port,
! 2254: xa->flags & ATA_F_NCQ);
! 2255:
! 2256: ahci_unload_prdt(ccb);
! 2257:
! 2258: if (xa->state == ATA_S_ONCHIP)
! 2259: xa->state = ATA_S_COMPLETE;
! 2260: #ifdef DIAGNOSTIC
! 2261: else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
! 2262: printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
! 2263: "slot %d\n", PORTNAME(ccb->ccb_port), xa->state,
! 2264: ccb->ccb_slot);
! 2265: #endif
! 2266: if (xa->state != ATA_S_TIMEOUT)
! 2267: xa->complete(xa);
! 2268: }
! 2269:
! 2270: void
! 2271: ahci_ata_cmd_timeout(void *arg)
! 2272: {
! 2273: struct ahci_ccb *ccb = arg;
! 2274: struct ata_xfer *xa = &ccb->ccb_xa;
! 2275: struct ahci_port *ap = ccb->ccb_port;
! 2276: int s, ccb_was_started, ncq_cmd;
! 2277: volatile u_int32_t *active;
! 2278:
! 2279: s = splbio();
! 2280:
! 2281: ncq_cmd = (xa->flags & ATA_F_NCQ);
! 2282: active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
! 2283:
! 2284: if (ccb->ccb_xa.state == ATA_S_PENDING) {
! 2285: DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
! 2286: "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
! 2287: TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
! 2288: ccb_was_started = 0;
! 2289: } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap,
! 2290: 1 << ccb->ccb_slot)) {
! 2291: DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
! 2292: "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
! 2293: goto ret;
! 2294: } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
! 2295: DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
! 2296: "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
! 2297: ISSET(*active, 1 << ccb->ccb_slot) ?
! 2298: " but slot is still active?" : ".");
! 2299: goto ret;
! 2300: } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :
! 2301: AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active,
! 2302: 1 << ccb->ccb_slot)) {
! 2303: DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
! 2304: "IRQ handler didn't detect it. Why?\n", PORTNAME(ap),
! 2305: ccb->ccb_slot);
! 2306: *active &= ~(1 << ccb->ccb_slot);
! 2307: ccb->ccb_done(ccb);
! 2308: goto ret;
! 2309: } else {
! 2310: ccb_was_started = 1;
! 2311: }
! 2312:
! 2313: /* Complete the slot with a timeout error. */
! 2314: ccb->ccb_xa.state = ATA_S_TIMEOUT;
! 2315: *active &= ~(1 << ccb->ccb_slot);
! 2316: DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
! 2317: ccb->ccb_done(ccb); /* This won't issue pending commands or run the
! 2318: atascsi completion. */
! 2319:
! 2320: /* Reset port to abort running command. */
! 2321: if (ccb_was_started) {
! 2322: DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
! 2323: "in slot %d, active %08x\n", PORTNAME(ap), ncq_cmd ? " NCQ"
! 2324: : "", ccb->ccb_slot, *active);
! 2325: if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap)
! 2326: != 0) {
! 2327: printf("%s: failed to reset port during timeout "
! 2328: "handling, disabling it\n", PORTNAME(ap));
! 2329: ap->ap_state = AP_S_FATAL_ERROR;
! 2330: }
! 2331:
! 2332: /* Restart any other commands that were aborted by the reset. */
! 2333: if (*active) {
! 2334: DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
! 2335: "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
! 2336: *active);
! 2337: if (ncq_cmd)
! 2338: ahci_pwrite(ap, AHCI_PREG_SACT, *active);
! 2339: ahci_pwrite(ap, AHCI_PREG_CI, *active);
! 2340: }
! 2341: }
! 2342:
! 2343: /* Issue any pending commands now. */
! 2344: DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
! 2345: if (ccb_was_started)
! 2346: ahci_issue_pending_commands(ap, ncq_cmd);
! 2347: else if (ap->ap_active == 0)
! 2348: ahci_issue_pending_ncq_commands(ap);
! 2349:
! 2350: /* Complete the timed out ata_xfer I/O (may generate new I/O). */
! 2351: DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
! 2352: xa->complete(xa);
! 2353:
! 2354: DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
! 2355: ret:
! 2356: splx(s);
! 2357: }
! 2358:
! 2359: void
! 2360: ahci_empty_done(struct ahci_ccb *ccb)
! 2361: {
! 2362: ccb->ccb_xa.state = ATA_S_COMPLETE;
! 2363: }
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to ahci.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / dev / pci