Annotation of sys/dev/pci/ahci.c, Revision 1.1.1.1
1.1 nbrk 1: /* $OpenBSD: ahci.c,v 1.128 2007/07/03 22:33:20 dlg Exp $ */
2:
3: /*
4: * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
5: *
6: * Permission to use, copy, modify, and distribute this software for any
7: * purpose with or without fee is hereby granted, provided that the above
8: * copyright notice and this permission notice appear in all copies.
9: *
10: * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11: * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12: * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13: * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14: * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15: * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16: * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17: */
18:
19: #include <sys/param.h>
20: #include <sys/systm.h>
21: #include <sys/buf.h>
22: #include <sys/kernel.h>
23: #include <sys/malloc.h>
24: #include <sys/device.h>
25: #include <sys/proc.h>
26: #include <sys/queue.h>
27:
28: #include <machine/bus.h>
29:
30: #include <dev/pci/pcireg.h>
31: #include <dev/pci/pcivar.h>
32: #include <dev/pci/pcidevs.h>
33:
34: #include <dev/ata/atascsi.h>
35:
36: /* change to AHCI_DEBUG for dmesg spam */
37: #define NO_AHCI_DEBUG
38:
39: #ifdef AHCI_DEBUG
40: #define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) printf(f); } \
41: while (0)
42: #define AHCI_D_TIMEOUT 0x00
43: #define AHCI_D_VERBOSE 0x01
44: #define AHCI_D_INTR 0x02
45: #define AHCI_D_XFER 0x08
46: int ahcidebug = AHCI_D_VERBOSE;
47: #else
48: #define DPRINTF(m, f...)
49: #endif
50:
51: #define AHCI_PCI_BAR 0x24
52: #define AHCI_PCI_INTERFACE 0x01
53:
54: #define AHCI_REG_CAP 0x000 /* HBA Capabilities */
55: #define AHCI_REG_CAP_NP(_r) (((_r) & 0x1f)+1) /* Number of Ports */
56: #define AHCI_REG_CAP_SXS (1<<5) /* External SATA */
57: #define AHCI_REG_CAP_EMS (1<<6) /* Enclosure Mgmt */
58: #define AHCI_REG_CAP_CCCS (1<<7) /* Cmd Coalescing */
59: #define AHCI_REG_CAP_NCS(_r) ((((_r) & 0x1f00)>>8)+1) /* NCmds*/
60: #define AHCI_REG_CAP_PSC (1<<13) /* Partial State Capable */
61: #define AHCI_REG_CAP_SSC (1<<14) /* Slumber State Capable */
62: #define AHCI_REG_CAP_PMD (1<<15) /* PIO Multiple DRQ Block */
63: #define AHCI_REG_CAP_FBSS (1<<16) /* FIS-Based Switching */
64: #define AHCI_REG_CAP_SPM (1<<17) /* Port Multiplier */
65: #define AHCI_REG_CAP_SAM (1<<18) /* AHCI Only mode */
66: #define AHCI_REG_CAP_SNZO (1<<19) /* Non Zero DMA Offsets */
67: #define AHCI_REG_CAP_ISS (0xf<<20) /* Interface Speed Support */
68: #define AHCI_REG_CAP_ISS_G1 (0x1<<20) /* Gen 1 (1.5 Gbps) */
69: #define AHCI_REG_CAP_ISS_G1_2 (0x2<<20) /* Gen 1 and 2 (3 Gbps) */
70: #define AHCI_REG_CAP_SCLO (1<<24) /* Cmd List Override */
71: #define AHCI_REG_CAP_SAL (1<<25) /* Activity LED */
72: #define AHCI_REG_CAP_SALP (1<<26) /* Aggressive Link Pwr Mgmt */
73: #define AHCI_REG_CAP_SSS (1<<27) /* Staggered Spinup */
74: #define AHCI_REG_CAP_SMPS (1<<28) /* Mech Presence Switch */
75: #define AHCI_REG_CAP_SSNTF (1<<29) /* SNotification Register */
76: #define AHCI_REG_CAP_SNCQ (1<<30) /* Native Cmd Queuing */
77: #define AHCI_REG_CAP_S64A (1<<31) /* 64bit Addressing */
78: #define AHCI_FMT_CAP "\020" "\040S64A" "\037NCQ" "\036SSNTF" \
79: "\035SMPS" "\034SSS" "\033SALP" "\032SAL" \
80: "\031SCLO" "\024SNZO" "\023SAM" "\022SPM" \
81: "\021FBSS" "\020PMD" "\017SSC" "\016PSC" \
82: "\010CCCS" "\007EMS" "\006SXS"
83: #define AHCI_REG_GHC 0x004 /* Global HBA Control */
84: #define AHCI_REG_GHC_HR (1<<0) /* HBA Reset */
85: #define AHCI_REG_GHC_IE (1<<1) /* Interrupt Enable */
86: #define AHCI_REG_GHC_MRSM (1<<2) /* MSI Revert to Single Msg */
87: #define AHCI_REG_GHC_AE (1<<31) /* AHCI Enable */
88: #define AHCI_FMT_GHC "\020" "\040AE" "\003MRSM" "\002IE" "\001HR"
89: #define AHCI_REG_IS 0x008 /* Interrupt Status */
90: #define AHCI_REG_PI 0x00c /* Ports Implemented */
91: #define AHCI_REG_VS 0x010 /* AHCI Version */
92: #define AHCI_REG_VS_0_95 0x00000905 /* 0.95 */
93: #define AHCI_REG_VS_1_0 0x00010000 /* 1.0 */
94: #define AHCI_REG_VS_1_1 0x00010100 /* 1.1 */
95: #define AHCI_REG_CCC_CTL 0x014 /* Coalescing Control */
96: #define AHCI_REG_CCC_CTL_INT(_r) (((_r) & 0xf8) >> 3) /* CCC INT slot */
97: #define AHCI_REG_CCC_PORTS 0x018 /* Coalescing Ports */
98: #define AHCI_REG_EM_LOC 0x01c /* Enclosure Mgmt Location */
99: #define AHCI_REG_EM_CTL 0x020 /* Enclosure Mgmt Control */
100:
101: #define AHCI_PORT_REGION(_p) (0x100 + ((_p) * 0x80))
102: #define AHCI_PORT_SIZE 0x80
103:
104: #define AHCI_PREG_CLB 0x00 /* Cmd List Base Addr */
105: #define AHCI_PREG_CLBU 0x04 /* Cmd List Base Hi Addr */
106: #define AHCI_PREG_FB 0x08 /* FIS Base Addr */
107: #define AHCI_PREG_FBU 0x0c /* FIS Base Hi Addr */
108: #define AHCI_PREG_IS 0x10 /* Interrupt Status */
109: #define AHCI_PREG_IS_DHRS (1<<0) /* Device to Host FIS */
110: #define AHCI_PREG_IS_PSS (1<<1) /* PIO Setup FIS */
111: #define AHCI_PREG_IS_DSS (1<<2) /* DMA Setup FIS */
112: #define AHCI_PREG_IS_SDBS (1<<3) /* Set Device Bits FIS */
113: #define AHCI_PREG_IS_UFS (1<<4) /* Unknown FIS */
114: #define AHCI_PREG_IS_DPS (1<<5) /* Descriptor Processed */
115: #define AHCI_PREG_IS_PCS (1<<6) /* Port Change */
116: #define AHCI_PREG_IS_DMPS (1<<7) /* Device Mechanical Presence */
117: #define AHCI_PREG_IS_PRCS (1<<22) /* PhyRdy Change */
118: #define AHCI_PREG_IS_IPMS (1<<23) /* Incorrect Port Multiplier */
119: #define AHCI_PREG_IS_OFS (1<<24) /* Overflow */
120: #define AHCI_PREG_IS_INFS (1<<26) /* Interface Non-fatal Error */
121: #define AHCI_PREG_IS_IFS (1<<27) /* Interface Fatal Error */
122: #define AHCI_PREG_IS_HBDS (1<<28) /* Host Bus Data Error */
123: #define AHCI_PREG_IS_HBFS (1<<29) /* Host Bus Fatal Error */
124: #define AHCI_PREG_IS_TFES (1<<30) /* Task File Error */
125: #define AHCI_PREG_IS_CPDS (1<<31) /* Cold Presence Detect */
126: #define AHCI_PFMT_IS "\20" "\040CPDS" "\037TFES" "\036HBFS" \
127: "\035HBDS" "\034IFS" "\033INFS" "\031OFS" \
128: "\030IPMS" "\027PRCS" "\010DMPS" "\006DPS" \
129: "\007PCS" "\005UFS" "\004SDBS" "\003DSS" \
130: "\002PSS" "\001DHRS"
131: #define AHCI_PREG_IE 0x14 /* Interrupt Enable */
132: #define AHCI_PREG_IE_DHRE (1<<0) /* Device to Host FIS */
133: #define AHCI_PREG_IE_PSE (1<<1) /* PIO Setup FIS */
134: #define AHCI_PREG_IE_DSE (1<<2) /* DMA Setup FIS */
135: #define AHCI_PREG_IE_SDBE (1<<3) /* Set Device Bits FIS */
136: #define AHCI_PREG_IE_UFE (1<<4) /* Unknown FIS */
137: #define AHCI_PREG_IE_DPE (1<<5) /* Descriptor Processed */
138: #define AHCI_PREG_IE_PCE (1<<6) /* Port Change */
139: #define AHCI_PREG_IE_DMPE (1<<7) /* Device Mechanical Presence */
140: #define AHCI_PREG_IE_PRCE (1<<22) /* PhyRdy Change */
141: #define AHCI_PREG_IE_IPME (1<<23) /* Incorrect Port Multiplier */
142: #define AHCI_PREG_IE_OFE (1<<24) /* Overflow */
143: #define AHCI_PREG_IE_INFE (1<<26) /* Interface Non-fatal Error */
144: #define AHCI_PREG_IE_IFE (1<<27) /* Interface Fatal Error */
145: #define AHCI_PREG_IE_HBDE (1<<28) /* Host Bus Data Error */
146: #define AHCI_PREG_IE_HBFE (1<<29) /* Host Bus Fatal Error */
147: #define AHCI_PREG_IE_TFEE (1<<30) /* Task File Error */
148: #define AHCI_PREG_IE_CPDE (1<<31) /* Cold Presence Detect */
149: #define AHCI_PFMT_IE "\20" "\040CPDE" "\037TFEE" "\036HBFE" \
150: "\035HBDE" "\034IFE" "\033INFE" "\031OFE" \
151: "\030IPME" "\027PRCE" "\010DMPE" "\007PCE" \
152: "\006DPE" "\005UFE" "\004SDBE" "\003DSE" \
153: "\002PSE" "\001DHRE"
154: #define AHCI_PREG_CMD 0x18 /* Command and Status */
155: #define AHCI_PREG_CMD_ST (1<<0) /* Start */
156: #define AHCI_PREG_CMD_SUD (1<<1) /* Spin Up Device */
157: #define AHCI_PREG_CMD_POD (1<<2) /* Power On Device */
158: #define AHCI_PREG_CMD_CLO (1<<3) /* Command List Override */
159: #define AHCI_PREG_CMD_FRE (1<<4) /* FIS Receive Enable */
160: #define AHCI_PREG_CMD_CCS(_r) (((_r) >> 8) & 0x1f) /* Curr CmdSlot# */
161: #define AHCI_PREG_CMD_MPSS (1<<13) /* Mech Presence State */
162: #define AHCI_PREG_CMD_FR (1<<14) /* FIS Receive Running */
163: #define AHCI_PREG_CMD_CR (1<<15) /* Command List Running */
164: #define AHCI_PREG_CMD_CPS (1<<16) /* Cold Presence State */
165: #define AHCI_PREG_CMD_PMA (1<<17) /* Port Multiplier Attached */
166: #define AHCI_PREG_CMD_HPCP (1<<18) /* Hot Plug Capable */
167: #define AHCI_PREG_CMD_MPSP (1<<19) /* Mech Presence Switch */
168: #define AHCI_PREG_CMD_CPD (1<<20) /* Cold Presence Detection */
169: #define AHCI_PREG_CMD_ESP (1<<21) /* External SATA Port */
170: #define AHCI_PREG_CMD_ATAPI (1<<24) /* Device is ATAPI */
171: #define AHCI_PREG_CMD_DLAE (1<<25) /* Drv LED on ATAPI Enable */
172: #define AHCI_PREG_CMD_ALPE (1<<26) /* Aggro Pwr Mgmt Enable */
173: #define AHCI_PREG_CMD_ASP (1<<27) /* Aggro Slumber/Partial */
174: #define AHCI_PREG_CMD_ICC 0xf0000000 /* Interface Comm Ctrl */
175: #define AHCI_PREG_CMD_ICC_SLUMBER 0x60000000
176: #define AHCI_PREG_CMD_ICC_PARTIAL 0x20000000
177: #define AHCI_PREG_CMD_ICC_ACTIVE 0x10000000
178: #define AHCI_PREG_CMD_ICC_IDLE 0x00000000
179: #define AHCI_PFMT_CMD "\020" "\034ASP" "\033ALPE" "\032DLAE" \
180: "\031ATAPI" "\026ESP" "\025CPD" "\024MPSP" \
181: "\023HPCP" "\022PMA" "\021CPS" "\020CR" \
182: "\017FR" "\016MPSS" "\005FRE" "\004CLO" \
183: "\003POD" "\002SUD" "\001ST"
184: #define AHCI_PREG_TFD 0x20 /* Task File Data*/
185: #define AHCI_PREG_TFD_STS 0xff
186: #define AHCI_PREG_TFD_STS_ERR (1<<0)
187: #define AHCI_PREG_TFD_STS_DRQ (1<<3)
188: #define AHCI_PREG_TFD_STS_BSY (1<<7)
189: #define AHCI_PREG_TFD_ERR 0xff00
190: #define AHCI_PFMT_TFD_STS "\20" "\010BSY" "\004DRQ" "\001ERR"
191: #define AHCI_PREG_SIG 0x24 /* Signature */
192: #define AHCI_PREG_SSTS 0x28 /* SATA Status */
193: #define AHCI_PREG_SSTS_DET 0xf /* Device Detection */
194: #define AHCI_PREG_SSTS_DET_NONE 0x0
195: #define AHCI_PREG_SSTS_DET_DEV_NE 0x1
196: #define AHCI_PREG_SSTS_DET_DEV 0x3
197: #define AHCI_PREG_SSTS_DET_PHYOFFLINE 0x4
198: #define AHCI_PREG_SSTS_SPD 0xf0 /* Current Interface Speed */
199: #define AHCI_PREG_SSTS_SPD_NONE 0x00
200: #define AHCI_PREG_SSTS_SPD_GEN1 0x10
201: #define AHCI_PREG_SSTS_SPD_GEN2 0x20
202: #define AHCI_PREG_SSTS_IPM 0xf00 /* Interface Power Management */
203: #define AHCI_PREG_SSTS_IPM_NONE 0x000
204: #define AHCI_PREG_SSTS_IPM_ACTIVE 0x100
205: #define AHCI_PREG_SSTS_IPM_PARTIAL 0x200
206: #define AHCI_PREG_SSTS_IPM_SLUMBER 0x600
207: #define AHCI_PREG_SCTL 0x2c /* SATA Control */
208: #define AHCI_PREG_SCTL_DET 0xf /* Device Detection */
209: #define AHCI_PREG_SCTL_DET_NONE 0x0
210: #define AHCI_PREG_SCTL_DET_INIT 0x1
211: #define AHCI_PREG_SCTL_DET_DISABLE 0x4
212: #define AHCI_PREG_SCTL_SPD 0xf0 /* Speed Allowed */
213: #define AHCI_PREG_SCTL_SPD_ANY 0x00
214: #define AHCI_PREG_SCTL_SPD_GEN1 0x10
215: #define AHCI_PREG_SCTL_SPD_GEN2 0x20
216: #define AHCI_PREG_SCTL_IPM 0xf00 /* Interface Power Management */
217: #define AHCI_PREG_SCTL_IPM_NONE 0x000
218: #define AHCI_PREG_SCTL_IPM_NOPARTIAL 0x100
219: #define AHCI_PREG_SCTL_IPM_NOSLUMBER 0x200
220: #define AHCI_PREG_SCTL_IPM_DISABLED 0x300
221: #define AHCI_PREG_SERR 0x30 /* SATA Error */
222: #define AHCI_PREG_SERR_ERR(_r) ((_r) & 0xffff)
223: #define AHCI_PREG_SERR_ERR_I (1<<0) /* Recovered Data Integrity */
224: #define AHCI_PREG_SERR_ERR_M (1<<1) /* Recovered Communications */
225: #define AHCI_PREG_SERR_ERR_T (1<<8) /* Transient Data Integrity */
226: #define AHCI_PREG_SERR_ERR_C (1<<9) /* Persistent Comm/Data */
227: #define AHCI_PREG_SERR_ERR_P (1<<10) /* Protocol */
228: #define AHCI_PREG_SERR_ERR_E (1<<11) /* Internal */
229: #define AHCI_PFMT_SERR_ERR "\020" "\014E" "\013P" "\012C" "\011T" "\002M" \
230: "\001I"
231: #define AHCI_PREG_SERR_DIAG(_r) (((_r) >> 16) & 0xffff)
232: #define AHCI_PREG_SERR_DIAG_N (1<<0) /* PhyRdy Change */
233: #define AHCI_PREG_SERR_DIAG_I (1<<1) /* Phy Internal Error */
234: #define AHCI_PREG_SERR_DIAG_W (1<<2) /* Comm Wake */
235: #define AHCI_PREG_SERR_DIAG_B (1<<3) /* 10B to 8B Decode Error */
236: #define AHCI_PREG_SERR_DIAG_D (1<<4) /* Disparity Error */
237: #define AHCI_PREG_SERR_DIAG_C (1<<5) /* CRC Error */
238: #define AHCI_PREG_SERR_DIAG_H (1<<6) /* Handshake Error */
239: #define AHCI_PREG_SERR_DIAG_S (1<<7) /* Link Sequence Error */
240: #define AHCI_PREG_SERR_DIAG_T (1<<8) /* Transport State Trans Err */
241: #define AHCI_PREG_SERR_DIAG_F (1<<9) /* Unknown FIS Type */
242: #define AHCI_PREG_SERR_DIAG_X (1<<10) /* Exchanged */
243: #define AHCI_PFMT_SERR_DIAG "\020" "\013X" "\012F" "\011T" "\010S" "\007H" \
244: "\006C" "\005D" "\004B" "\003W" "\002I" \
245: "\001N"
246: #define AHCI_PREG_SACT 0x34 /* SATA Active */
247: #define AHCI_PREG_CI 0x38 /* Command Issue */
248: #define AHCI_PREG_CI_ALL_SLOTS 0xffffffff
249: #define AHCI_PREG_SNTF 0x3c /* SNotification */
250:
251: struct ahci_cmd_hdr {
252: u_int16_t flags;
253: #define AHCI_CMD_LIST_FLAG_CFL 0x001f /* Command FIS Length */
254: #define AHCI_CMD_LIST_FLAG_A (1<<5) /* ATAPI */
255: #define AHCI_CMD_LIST_FLAG_W (1<<6) /* Write */
256: #define AHCI_CMD_LIST_FLAG_P (1<<7) /* Prefetchable */
257: #define AHCI_CMD_LIST_FLAG_R (1<<8) /* Reset */
258: #define AHCI_CMD_LIST_FLAG_B (1<<9) /* BIST */
259: #define AHCI_CMD_LIST_FLAG_C (1<<10) /* Clear Busy upon R_OK */
260: #define AHCI_CMD_LIST_FLAG_PMP 0xf000 /* Port Multiplier Port */
261: u_int16_t prdtl; /* sgl len */
262:
263: u_int32_t prdbc; /* transferred byte count */
264:
265: u_int32_t ctba_lo;
266: u_int32_t ctba_hi;
267:
268: u_int32_t reserved[4];
269: } __packed;
270:
271: struct ahci_rfis {
272: u_int8_t dsfis[28];
273: u_int8_t reserved1[4];
274: u_int8_t psfis[24];
275: u_int8_t reserved2[8];
276: u_int8_t rfis[24];
277: u_int8_t reserved3[4];
278: u_int8_t sdbfis[4];
279: u_int8_t ufis[64];
280: u_int8_t reserved4[96];
281: } __packed;
282:
283: struct ahci_prdt {
284: u_int32_t dba_lo;
285: u_int32_t dba_hi;
286: u_int32_t reserved;
287: u_int32_t flags;
288: #define AHCI_PRDT_FLAG_INTR (1<<31) /* interrupt on completion */
289: } __packed;
290:
291: /* this makes ahci_cmd_table 512 bytes, supporting 128-byte alignment */
292: #define AHCI_MAX_PRDT 24
293:
294: struct ahci_cmd_table {
295: u_int8_t cfis[64]; /* Command FIS */
296: u_int8_t acmd[16]; /* ATAPI Command */
297: u_int8_t reserved[48];
298:
299: struct ahci_prdt prdt[AHCI_MAX_PRDT];
300: } __packed;
301:
302: #define AHCI_MAX_PORTS 32
303:
304: struct ahci_dmamem {
305: bus_dmamap_t adm_map;
306: bus_dma_segment_t adm_seg;
307: size_t adm_size;
308: caddr_t adm_kva;
309: };
310: #define AHCI_DMA_MAP(_adm) ((_adm)->adm_map)
311: #define AHCI_DMA_DVA(_adm) ((_adm)->adm_map->dm_segs[0].ds_addr)
312: #define AHCI_DMA_KVA(_adm) ((void *)(_adm)->adm_kva)
313:
314: struct ahci_softc;
315: struct ahci_port;
316:
317: struct ahci_ccb {
318: /* ATA xfer associated with this CCB. Must be 1st struct member. */
319: struct ata_xfer ccb_xa;
320:
321: int ccb_slot;
322: struct ahci_port *ccb_port;
323:
324: bus_dmamap_t ccb_dmamap;
325: struct ahci_cmd_hdr *ccb_cmd_hdr;
326: struct ahci_cmd_table *ccb_cmd_table;
327:
328: void (*ccb_done)(struct ahci_ccb *);
329:
330: TAILQ_ENTRY(ahci_ccb) ccb_entry;
331: };
332:
333: struct ahci_port {
334: struct ahci_softc *ap_sc;
335: bus_space_handle_t ap_ioh;
336:
337: #ifdef AHCI_COALESCE
338: int ap_num;
339: #endif
340:
341: struct ahci_rfis *ap_rfis;
342: struct ahci_dmamem *ap_dmamem_rfis;
343:
344: struct ahci_dmamem *ap_dmamem_cmd_list;
345: struct ahci_dmamem *ap_dmamem_cmd_table;
346:
347: volatile u_int32_t ap_active;
348: volatile u_int32_t ap_active_cnt;
349: volatile u_int32_t ap_sactive;
350: struct ahci_ccb *ap_ccbs;
351:
352: TAILQ_HEAD(, ahci_ccb) ap_ccb_free;
353: TAILQ_HEAD(, ahci_ccb) ap_ccb_pending;
354:
355: u_int32_t ap_state;
356: #define AP_S_NORMAL 0
357: #define AP_S_FATAL_ERROR 1
358:
359: /* For error recovery. */
360: #ifdef DIAGNOSTIC
361: int ap_err_busy;
362: #endif
363: u_int32_t ap_err_saved_sactive;
364: u_int32_t ap_err_saved_active;
365: u_int32_t ap_err_saved_active_cnt;
366:
367: u_int8_t ap_err_scratch[512];
368:
369: #ifdef AHCI_DEBUG
370: char ap_name[16];
371: #define PORTNAME(_ap) ((_ap)->ap_name)
372: #else
373: #define PORTNAME(_ap) DEVNAME((_ap)->ap_sc)
374: #endif
375: };
376:
377: struct ahci_softc {
378: struct device sc_dev;
379:
380: void *sc_ih;
381:
382: bus_space_tag_t sc_iot;
383: bus_space_handle_t sc_ioh;
384: bus_size_t sc_ios;
385: bus_dma_tag_t sc_dmat;
386:
387: int sc_flags;
388: #define AHCI_F_NO_NCQ (1<<0)
389:
390: u_int sc_ncmds;
391:
392: struct ahci_port *sc_ports[AHCI_MAX_PORTS];
393:
394: struct atascsi *sc_atascsi;
395:
396: #ifdef AHCI_COALESCE
397: u_int32_t sc_ccc_mask;
398: u_int32_t sc_ccc_ports;
399: u_int32_t sc_ccc_ports_cur;
400: #endif
401: };
402: #define DEVNAME(_s) ((_s)->sc_dev.dv_xname)
403:
404: struct ahci_device {
405: pci_vendor_id_t ad_vendor;
406: pci_product_id_t ad_product;
407: int (*ad_match)(struct pci_attach_args *);
408: int (*ad_attach)(struct ahci_softc *,
409: struct pci_attach_args *);
410: };
411:
412: const struct ahci_device *ahci_lookup_device(struct pci_attach_args *);
413:
414: int ahci_no_match(struct pci_attach_args *);
415: int ahci_vt8251_attach(struct ahci_softc *,
416: struct pci_attach_args *);
417:
418: static const struct ahci_device ahci_devices[] = {
419: { PCI_VENDOR_VIATECH, PCI_PRODUCT_VIATECH_VT8251_SATA,
420: ahci_no_match, ahci_vt8251_attach }
421: };
422:
423: int ahci_pci_match(struct device *, void *, void *);
424: void ahci_pci_attach(struct device *, struct device *,
425: void *);
426:
427: struct cfattach ahci_pci_ca = {
428: sizeof(struct ahci_softc), ahci_pci_match, ahci_pci_attach
429: };
430:
431: struct cfattach ahci_jmb_ca = {
432: sizeof(struct ahci_softc), ahci_pci_match, ahci_pci_attach
433: };
434:
435: struct cfdriver ahci_cd = {
436: NULL, "ahci", DV_DULL
437: };
438:
439:
440: int ahci_map_regs(struct ahci_softc *,
441: struct pci_attach_args *);
442: void ahci_unmap_regs(struct ahci_softc *,
443: struct pci_attach_args *);
444: int ahci_map_intr(struct ahci_softc *,
445: struct pci_attach_args *, pci_intr_handle_t);
446: #ifdef notyet
447: void ahci_unmap_intr(struct ahci_softc *,
448: struct pci_attach_args *);
449: #endif
450:
451: int ahci_init(struct ahci_softc *);
452: int ahci_port_alloc(struct ahci_softc *, u_int);
453: void ahci_port_free(struct ahci_softc *, u_int);
454:
455: int ahci_port_start(struct ahci_port *, int);
456: int ahci_port_stop(struct ahci_port *, int);
457: int ahci_port_clo(struct ahci_port *);
458: int ahci_port_softreset(struct ahci_port *);
459: int ahci_port_portreset(struct ahci_port *);
460:
461: int ahci_load_prdt(struct ahci_ccb *);
462: void ahci_unload_prdt(struct ahci_ccb *);
463:
464: int ahci_poll(struct ahci_ccb *, int, void (*)(void *));
465: void ahci_start(struct ahci_ccb *);
466:
467: void ahci_issue_pending_ncq_commands(struct ahci_port *);
468: void ahci_issue_pending_commands(struct ahci_port *, int);
469:
470: int ahci_intr(void *);
471: u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t);
472:
473: struct ahci_ccb *ahci_get_ccb(struct ahci_port *);
474: void ahci_put_ccb(struct ahci_ccb *);
475:
476: struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *);
477: void ahci_put_err_ccb(struct ahci_ccb *);
478:
479: int ahci_port_read_ncq_error(struct ahci_port *, int *);
480:
481: struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, size_t);
482: void ahci_dmamem_free(struct ahci_softc *,
483: struct ahci_dmamem *);
484:
485: u_int32_t ahci_read(struct ahci_softc *, bus_size_t);
486: void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t);
487: int ahci_wait_ne(struct ahci_softc *, bus_size_t,
488: u_int32_t, u_int32_t);
489:
490: u_int32_t ahci_pread(struct ahci_port *, bus_size_t);
491: void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t);
492: int ahci_pwait_eq(struct ahci_port *, bus_size_t,
493: u_int32_t, u_int32_t);
494:
495: /* Wait for all bits in _b to be cleared */
496: #define ahci_pwait_clr(_ap, _r, _b) ahci_pwait_eq((_ap), (_r), (_b), 0)
497:
498: /* Wait for all bits in _b to be set */
499: #define ahci_pwait_set(_ap, _r, _b) ahci_pwait_eq((_ap), (_r), (_b), (_b))
500:
501: /* provide methods for atascsi to call */
502: int ahci_ata_probe(void *, int);
503: struct ata_xfer * ahci_ata_get_xfer(void *, int);
504: void ahci_ata_put_xfer(struct ata_xfer *);
505: int ahci_ata_cmd(struct ata_xfer *);
506:
507: struct atascsi_methods ahci_atascsi_methods = {
508: ahci_ata_probe,
509: ahci_ata_get_xfer,
510: ahci_ata_cmd
511: };
512:
513: /* ccb completions */
514: void ahci_ata_cmd_done(struct ahci_ccb *);
515: void ahci_ata_cmd_timeout(void *);
516: void ahci_empty_done(struct ahci_ccb *);
517:
518: const struct ahci_device *
519: ahci_lookup_device(struct pci_attach_args *pa)
520: {
521: int i;
522: const struct ahci_device *ad;
523:
524: for (i = 0; i < (sizeof(ahci_devices) / sizeof(ahci_devices[0])); i++) {
525: ad = &ahci_devices[i];
526: if (ad->ad_vendor == PCI_VENDOR(pa->pa_id) &&
527: ad->ad_product == PCI_PRODUCT(pa->pa_id))
528: return (ad);
529: }
530:
531: return (NULL);
532: }
533:
534: int
535: ahci_no_match(struct pci_attach_args *pa)
536: {
537: return (0);
538: }
539:
540: int
541: ahci_vt8251_attach(struct ahci_softc *sc, struct pci_attach_args *pa)
542: {
543: sc->sc_flags |= AHCI_F_NO_NCQ;
544:
545: return (0);
546: }
547:
548: int
549: ahci_pci_match(struct device *parent, void *match, void *aux)
550: {
551: struct pci_attach_args *pa = aux;
552: const struct ahci_device *ad;
553:
554: ad = ahci_lookup_device(pa);
555: if (ad != NULL) {
556: /* the device may need special checks to see if it matches */
557: if (ad->ad_match != NULL)
558: return (ad->ad_match(pa));
559:
560: return (2); /* match higher than pciide */
561: }
562:
563: if (PCI_CLASS(pa->pa_class) == PCI_CLASS_MASS_STORAGE &&
564: PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_MASS_STORAGE_SATA &&
565: PCI_INTERFACE(pa->pa_class) == AHCI_PCI_INTERFACE)
566: return (2);
567:
568: return (0);
569: }
570:
571: void
572: ahci_pci_attach(struct device *parent, struct device *self, void *aux)
573: {
574: struct ahci_softc *sc = (struct ahci_softc *)self;
575: struct pci_attach_args *pa = aux;
576: struct atascsi_attach_args aaa;
577: const struct ahci_device *ad;
578: pci_intr_handle_t ih;
579: u_int32_t cap, pi;
580: int i;
581:
582: ad = ahci_lookup_device(pa);
583: if (ad != NULL && ad->ad_attach != NULL) {
584: if (ad->ad_attach(sc, pa) != 0) {
585: /* error should be printed by ad_attach */
586: return;
587: }
588: }
589:
590: if (pci_intr_map(pa, &ih) != 0) {
591: printf(": unable to map interrupt\n");
592: return;
593: }
594: printf(": %s,", pci_intr_string(pa->pa_pc, ih));
595:
596: if (ahci_map_regs(sc, pa) != 0) {
597: /* error already printed by ahci_map_regs */
598: return;
599: }
600:
601: if (ahci_init(sc) != 0) {
602: /* error already printed by ahci_init */
603: goto unmap;
604: }
605:
606: if (ahci_map_intr(sc, pa, ih) != 0) {
607: /* error already printed by ahci_map_intr */
608: goto unmap;
609: }
610:
611: printf("\n");
612:
613: sc->sc_dmat = pa->pa_dmat;
614:
615: cap = ahci_read(sc, AHCI_REG_CAP);
616: sc->sc_ncmds = AHCI_REG_CAP_NCS(cap);
617: #ifdef AHCI_DEBUG
618: if (ahcidebug & AHCI_D_VERBOSE) {
619: const char *gen;
620:
621: switch (cap & AHCI_REG_CAP_ISS) {
622: case AHCI_REG_CAP_ISS_G1:
623: gen = "1 (1.5Gbps)";
624: break;
625: case AHCI_REG_CAP_ISS_G1_2:
626: gen = "1 (1.5Gbps) and 2 (3Gbps)";
627: break;
628: default:
629: gen = "unknown";
630: break;
631: }
632:
633: printf("%s: capabilities 0x%b, %d ports, %d cmds, gen %s\n",
634: DEVNAME(sc), cap, AHCI_FMT_CAP,
635: AHCI_REG_CAP_NP(cap), sc->sc_ncmds, gen);
636: }
637: #endif
638:
639: pi = ahci_read(sc, AHCI_REG_PI);
640: DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n",
641: DEVNAME(sc), pi);
642:
643: #ifdef AHCI_COALESCE
644: /* Naive coalescing support - enable for all ports. */
645: if (cap & AHCI_REG_CAP_CCCS) {
646: u_int16_t ccc_timeout = 20;
647: u_int8_t ccc_numcomplete = 12;
648: u_int32_t ccc_ctl;
649:
650: /* disable coalescing during reconfiguration. */
651: ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL);
652: ccc_ctl &= ~0x00000001;
653: ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
654:
655: sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl);
656: if (pi & sc->sc_ccc_mask) {
657: /* A conflict with the implemented port list? */
658: printf("%s: coalescing interrupt/implemented port list "
659: "conflict, PI: %08x, ccc_mask: %08x\n",
660: DEVNAME(sc), pi, sc->sc_ccc_mask);
661: sc->sc_ccc_mask = 0;
662: goto noccc;
663: }
664:
665: /* ahci_port_start will enable each port when it starts. */
666: sc->sc_ccc_ports = pi;
667: sc->sc_ccc_ports_cur = 0;
668:
669: /* program thresholds and enable overall coalescing. */
670: ccc_ctl &= ~0xffffff00;
671: ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8);
672: ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl);
673: ahci_write(sc, AHCI_REG_CCC_PORTS, 0);
674: ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1);
675: }
676: noccc:
677: #endif
678: for (i = 0; i < AHCI_MAX_PORTS; i++) {
679: if (!ISSET(pi, 1 << i)) {
680: /* dont allocate stuff if the port isnt implemented */
681: continue;
682: }
683:
684: if (ahci_port_alloc(sc, i) == ENOMEM)
685: goto freeports;
686: }
687:
688: bzero(&aaa, sizeof(aaa));
689: aaa.aaa_cookie = sc;
690: aaa.aaa_methods = &ahci_atascsi_methods;
691: aaa.aaa_minphys = minphys;
692: aaa.aaa_nports = AHCI_MAX_PORTS;
693: aaa.aaa_ncmds = sc->sc_ncmds;
694: aaa.aaa_capability = ASAA_CAP_NEEDS_RESERVED;
695: if (!(sc->sc_flags & AHCI_F_NO_NCQ) && (cap & AHCI_REG_CAP_SNCQ))
696: aaa.aaa_capability |= ASAA_CAP_NCQ;
697:
698: sc->sc_atascsi = atascsi_attach(&sc->sc_dev, &aaa);
699:
700: /* Enable interrupts */
701: ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE);
702:
703: return;
704:
705: freeports:
706: for (i = 0; i < AHCI_MAX_PORTS; i++)
707: if (sc->sc_ports[i] != NULL)
708: ahci_port_free(sc, i);
709: unmap:
710: /* Disable controller */
711: ahci_write(sc, AHCI_REG_GHC, 0);
712: ahci_unmap_regs(sc, pa);
713: return;
714: }
715:
716: int
717: ahci_map_regs(struct ahci_softc *sc, struct pci_attach_args *pa)
718: {
719: pcireg_t maptype;
720:
721: maptype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, AHCI_PCI_BAR);
722: if (pci_mapreg_map(pa, AHCI_PCI_BAR, maptype, 0, &sc->sc_iot,
723: &sc->sc_ioh, NULL, &sc->sc_ios, 0) != 0) {
724: printf(" unable to map registers\n");
725: return (1);
726: }
727:
728: return (0);
729: }
730:
731: void
732: ahci_unmap_regs(struct ahci_softc *sc, struct pci_attach_args *pa)
733: {
734: bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios);
735: sc->sc_ios = 0;
736: }
737:
738: int
739: ahci_map_intr(struct ahci_softc *sc, struct pci_attach_args *pa,
740: pci_intr_handle_t ih)
741: {
742: sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_BIO,
743: ahci_intr, sc, DEVNAME(sc));
744: if (sc->sc_ih == NULL) {
745: printf("%s: unable to map interrupt\n", DEVNAME(sc));
746: return (1);
747: }
748:
749: return (0);
750: }
751:
752: #ifdef notyet
753: /* one day we'll have hotplug pci */
754: void
755: ahci_unmap_intr(struct ahci_softc *sc, struct pci_attach_args *pa)
756: {
757: pci_intr_disestablish(pa->pa_pc, sc->sc_ih);
758: }
759: #endif
760:
761: int
762: ahci_init(struct ahci_softc *sc)
763: {
764: u_int32_t reg, cap, pi;
765: const char *revision;
766:
767: DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", ahci_read(sc, AHCI_REG_GHC),
768: AHCI_FMT_GHC);
769:
770: /* save BIOS initialised parameters, enable staggered spin up */
771: cap = ahci_read(sc, AHCI_REG_CAP);
772: cap &= AHCI_REG_CAP_SMPS;
773: cap |= AHCI_REG_CAP_SSS;
774: pi = ahci_read(sc, AHCI_REG_PI);
775:
776: if (ISSET(AHCI_REG_GHC_AE, ahci_read(sc, AHCI_REG_GHC))) {
777: /* reset the controller */
778: ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
779: if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR,
780: AHCI_REG_GHC_HR) != 0) {
781: printf(" unable to reset controller\n");
782: return (1);
783: }
784: }
785:
786: /* enable ahci (global interrupts disabled) */
787: ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
788:
789: /* restore parameters */
790: ahci_write(sc, AHCI_REG_CAP, cap);
791: ahci_write(sc, AHCI_REG_PI, pi);
792:
793: /* check the revision */
794: reg = ahci_read(sc, AHCI_REG_VS);
795: switch (reg) {
796: case AHCI_REG_VS_0_95:
797: revision = "0.95";
798: break;
799: case AHCI_REG_VS_1_0:
800: revision = "1.0";
801: break;
802: case AHCI_REG_VS_1_1:
803: revision = "1.1";
804: break;
805:
806: default:
807: printf(" unsupported AHCI revision 0x%08x\n", reg);
808: return (1);
809: }
810:
811: printf(" AHCI %s", revision);
812:
813: return (0);
814: }
815:
816: int
817: ahci_port_alloc(struct ahci_softc *sc, u_int port)
818: {
819: struct ahci_port *ap;
820: struct ahci_ccb *ccb;
821: u_int64_t dva;
822: u_int32_t cmd;
823: struct ahci_cmd_hdr *hdr;
824: struct ahci_cmd_table *table;
825: int i, rc = ENOMEM;
826:
827: ap = malloc(sizeof(struct ahci_port), M_DEVBUF, M_NOWAIT);
828: if (ap == NULL) {
829: printf("%s: unable to allocate memory for port %d\n",
830: DEVNAME(sc), port);
831: goto reterr;
832: }
833: bzero(ap, sizeof(struct ahci_port));
834:
835: #ifdef AHCI_DEBUG
836: snprintf(ap->ap_name, sizeof(ap->ap_name), "%s.%d",
837: DEVNAME(sc), port);
838: #endif
839: sc->sc_ports[port] = ap;
840:
841: if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
842: AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
843: printf("%s: unable to create register window for port %d\n",
844: DEVNAME(sc), port);
845: goto freeport;
846: }
847:
848: ap->ap_sc = sc;
849: #ifdef AHCI_COALESCE
850: ap->ap_num = port;
851: #endif
852: TAILQ_INIT(&ap->ap_ccb_free);
853: TAILQ_INIT(&ap->ap_ccb_pending);
854:
855: /* Disable port interrupts */
856: ahci_pwrite(ap, AHCI_PREG_IE, 0);
857:
858: /* Sec 10.1.2 - deinitialise port if it is already running */
859: cmd = ahci_pread(ap, AHCI_PREG_CMD);
860: if (ISSET(cmd, (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
861: AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
862: ISSET(ahci_pread(ap, AHCI_PREG_SCTL), AHCI_PREG_SCTL_DET)) {
863: int r;
864:
865: r = ahci_port_stop(ap, 1);
866: if (r) {
867: printf("%s: unable to disable %s, ignoring port %d\n",
868: DEVNAME(sc), r == 2 ? "CR" : "FR", port);
869: rc = ENXIO;
870: goto freeport;
871: }
872:
873: /* Write DET to zero */
874: ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
875: }
876:
877: /* Allocate RFIS */
878: ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sizeof(struct ahci_rfis));
879: if (ap->ap_dmamem_rfis == NULL)
880: goto nomem;
881:
882: /* Setup RFIS base address */
883: ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
884: dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
885: ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
886: ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
887:
888: /* Enable FIS reception and activate port. */
889: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
890: cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
891: ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
892:
893: /* Check whether port activated. Skip it if not. */
894: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
895: if (!ISSET(cmd, AHCI_PREG_CMD_FRE)) {
896: rc = ENXIO;
897: goto freeport;
898: }
899:
900: /* Allocate a CCB for each command slot */
901: ap->ap_ccbs = malloc(sizeof(struct ahci_ccb) * sc->sc_ncmds, M_DEVBUF,
902: M_NOWAIT);
903: if (ap->ap_ccbs == NULL) {
904: printf("%s: unable to allocate command list for port %d\n",
905: DEVNAME(sc), port);
906: goto freeport;
907: }
908: bzero(ap->ap_ccbs, sizeof(struct ahci_ccb) * sc->sc_ncmds);
909:
910: /* Command List Structures and Command Tables */
911: ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc,
912: sc->sc_ncmds * sizeof(struct ahci_cmd_hdr));
913: ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc,
914: sc->sc_ncmds * sizeof(struct ahci_cmd_table));
915: if (ap->ap_dmamem_cmd_table == NULL || ap->ap_dmamem_cmd_list == NULL) {
916: nomem:
917: printf("%s: unable to allocate DMA memory for port %d\n",
918: DEVNAME(sc), port);
919: goto freeport;
920: }
921:
922: /* Setup command list base address */
923: dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
924: ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
925: ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
926:
927: /* Split CCB allocation into CCBs and assign to command header/table */
928: hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
929: table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
930: for (i = 0; i < sc->sc_ncmds; i++) {
931: ccb = &ap->ap_ccbs[i];
932:
933: if (bus_dmamap_create(sc->sc_dmat, MAXPHYS, AHCI_MAX_PRDT,
934: (4 * 1024 * 1024), 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
935: &ccb->ccb_dmamap) != 0) {
936: printf("%s: unable to create dmamap for port %d "
937: "ccb %d\n", DEVNAME(sc), port, i);
938: goto freeport;
939: }
940:
941: ccb->ccb_slot = i;
942: ccb->ccb_port = ap;
943: ccb->ccb_cmd_hdr = &hdr[i];
944: ccb->ccb_cmd_table = &table[i];
945: dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
946: ccb->ccb_slot * sizeof(struct ahci_cmd_table);
947: ccb->ccb_cmd_hdr->ctba_hi = htole32((u_int32_t)(dva >> 32));
948: ccb->ccb_cmd_hdr->ctba_lo = htole32((u_int32_t)dva);
949:
950: ccb->ccb_xa.fis =
951: (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
952: ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
953: ccb->ccb_xa.tag = i;
954:
955: ccb->ccb_xa.ata_put_xfer = ahci_ata_put_xfer;
956:
957: ccb->ccb_xa.state = ATA_S_COMPLETE;
958: ahci_put_ccb(ccb);
959: }
960:
961: /* Wait for ICC change to complete */
962: ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC);
963:
964: /* Reset port */
965: rc = ahci_port_portreset(ap);
966: switch (rc) {
967: case ENODEV:
968: switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
969: case AHCI_PREG_SSTS_DET_DEV_NE:
970: printf("%s: device not communicating on port %d\n",
971: DEVNAME(sc), port);
972: break;
973: case AHCI_PREG_SSTS_DET_PHYOFFLINE:
974: printf("%s: PHY offline on port %d\n", DEVNAME(sc),
975: port);
976: break;
977: default:
978: DPRINTF(AHCI_D_VERBOSE, "%s: no device detected "
979: "on port %d\n", DEVNAME(sc), port);
980: break;
981: }
982: goto freeport;
983:
984: case EBUSY:
985: printf("%s: device on port %d didn't come ready, "
986: "TFD: 0x%b\n", DEVNAME(sc), port,
987: ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
988:
989: /* Try a soft reset to clear busy */
990: rc = ahci_port_softreset(ap);
991: if (rc) {
992: printf("%s: unable to communicate "
993: "with device on port %d\n", DEVNAME(sc), port);
994: goto freeport;
995: }
996: break;
997:
998: default:
999: break;
1000: }
1001: DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n",
1002: DEVNAME(sc), port);
1003:
1004: /* Enable command transfers on port */
1005: if (ahci_port_start(ap, 0)) {
1006: printf("%s: failed to start command DMA on port %d, "
1007: "disabling\n", DEVNAME(sc), port);
1008: rc = ENXIO; /* couldn't start port */
1009: }
1010:
1011: /* Flush interrupts for port */
1012: ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
1013: ahci_write(sc, AHCI_REG_IS, 1 << port);
1014:
1015: /* Enable port interrupts */
1016: ahci_pwrite(ap, AHCI_PREG_IE, AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
1017: AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | AHCI_PREG_IE_DPE |
1018: AHCI_PREG_IE_UFE |
1019: #ifdef AHCI_COALESCE
1020: ((sc->sc_ccc_ports & (1 << port)) ? 0 : (AHCI_PREG_IE_SDBE |
1021: AHCI_PREG_IE_DHRE))
1022: #else
1023: AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE
1024: #endif
1025: );
1026:
1027: freeport:
1028: if (rc != 0)
1029: ahci_port_free(sc, port);
1030: reterr:
1031: return (rc);
1032: }
1033:
1034: void
1035: ahci_port_free(struct ahci_softc *sc, u_int port)
1036: {
1037: struct ahci_port *ap = sc->sc_ports[port];
1038: struct ahci_ccb *ccb;
1039:
1040: /* Ensure port is disabled and its interrupts are flushed */
1041: if (ap->ap_sc) {
1042: ahci_pwrite(ap, AHCI_PREG_CMD, 0);
1043: ahci_pwrite(ap, AHCI_PREG_IE, 0);
1044: ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
1045: ahci_write(sc, AHCI_REG_IS, 1 << port);
1046: }
1047:
1048: if (ap->ap_ccbs) {
1049: while ((ccb = ahci_get_ccb(ap)) != NULL)
1050: bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
1051: free(ap->ap_ccbs, M_DEVBUF);
1052: }
1053:
1054: if (ap->ap_dmamem_cmd_list)
1055: ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
1056: if (ap->ap_dmamem_rfis)
1057: ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
1058: if (ap->ap_dmamem_cmd_table)
1059: ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
1060:
1061: /* bus_space(9) says we dont free the subregions handle */
1062:
1063: free(ap, M_DEVBUF);
1064: sc->sc_ports[port] = NULL;
1065: }
1066:
1067: int
1068: ahci_port_start(struct ahci_port *ap, int fre_only)
1069: {
1070: u_int32_t r;
1071:
1072: /* Turn on FRE (and ST) */
1073: r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1074: r |= AHCI_PREG_CMD_FRE;
1075: if (!fre_only)
1076: r |= AHCI_PREG_CMD_ST;
1077: ahci_pwrite(ap, AHCI_PREG_CMD, r);
1078:
1079: #ifdef AHCI_COALESCE
1080: /* (Re-)enable coalescing on the port. */
1081: if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
1082: ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
1083: ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
1084: ap->ap_sc->sc_ccc_ports_cur);
1085: }
1086: #endif
1087:
1088: /* Wait for FR to come on */
1089: if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR))
1090: return (2);
1091:
1092: /* Wait for CR to come on */
1093: if (!fre_only && ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR))
1094: return (1);
1095:
1096: return (0);
1097: }
1098:
1099: int
1100: ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
1101: {
1102: u_int32_t r;
1103:
1104: #ifdef AHCI_COALESCE
1105: /* Disable coalescing on the port while it is stopped. */
1106: if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
1107: ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
1108: ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
1109: ap->ap_sc->sc_ccc_ports_cur);
1110: }
1111: #endif
1112:
1113: /* Turn off ST (and FRE) */
1114: r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1115: r &= ~AHCI_PREG_CMD_ST;
1116: if (stop_fis_rx)
1117: r &= ~AHCI_PREG_CMD_FRE;
1118: ahci_pwrite(ap, AHCI_PREG_CMD, r);
1119:
1120: /* Wait for CR to go off */
1121: if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR))
1122: return (1);
1123:
1124: /* Wait for FR to go off */
1125: if (stop_fis_rx && ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR))
1126: return (2);
1127:
1128: return (0);
1129: }
1130:
1131: /* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */
1132: int
1133: ahci_port_clo(struct ahci_port *ap)
1134: {
1135: struct ahci_softc *sc = ap->ap_sc;
1136: u_int32_t cmd;
1137:
1138: /* Only attempt CLO if supported by controller */
1139: if (!ISSET(ahci_read(sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO))
1140: return (1);
1141:
1142: /* Issue CLO */
1143: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1144: #ifdef DIAGNOSTIC
1145: if (ISSET(cmd, AHCI_PREG_CMD_ST))
1146: printf("%s: CLO requested while port running\n", PORTNAME(ap));
1147: #endif
1148: ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
1149:
1150: /* Wait for completion */
1151: if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO)) {
1152: printf("%s: CLO did not complete\n", PORTNAME(ap));
1153: return (1);
1154: }
1155:
1156: return (0);
1157: }
1158:
1159: /* AHCI soft reset, Section 10.4.1 */
1160: int
1161: ahci_port_softreset(struct ahci_port *ap)
1162: {
1163: struct ahci_ccb *ccb = NULL;
1164: struct ahci_cmd_hdr *cmd_slot;
1165: u_int8_t *fis;
1166: int s, rc = EIO;
1167: u_int32_t cmd;
1168:
1169: DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
1170:
1171: s = splbio();
1172:
1173: /* Save previous command register state */
1174: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1175:
1176: /* Idle port */
1177: if (ahci_port_stop(ap, 0)) {
1178: printf("%s: failed to stop port, cannot softreset\n",
1179: PORTNAME(ap));
1180: goto err;
1181: }
1182:
1183: /* Request CLO if device appears hung */
1184: if (ISSET(ahci_pread(ap, AHCI_PREG_TFD), AHCI_PREG_TFD_STS_BSY |
1185: AHCI_PREG_TFD_STS_DRQ))
1186: ahci_port_clo(ap);
1187:
1188: /* Clear port errors to permit TFD transfer */
1189: ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
1190:
1191: /* Restart port */
1192: if (ahci_port_start(ap, 0)) {
1193: printf("%s: failed to start port, cannot softreset\n",
1194: PORTNAME(ap));
1195: goto err;
1196: }
1197:
1198: /* Check whether CLO worked */
1199: if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1200: AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1201: printf("%s: CLO %s, need port reset\n", PORTNAME(ap),
1202: ISSET(ahci_read(ap->ap_sc, AHCI_REG_CAP), AHCI_REG_CAP_SCLO)
1203: ? "failed" : "unsupported");
1204: rc = EBUSY;
1205: goto err;
1206: }
1207:
1208: /* Prep first D2H command with SRST feature & clear busy/reset flags */
1209: ccb = ahci_get_err_ccb(ap);
1210: cmd_slot = ccb->ccb_cmd_hdr;
1211: bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table));
1212:
1213: fis = ccb->ccb_cmd_table->cfis;
1214: fis[0] = 0x27; /* Host to device */
1215: fis[15] = 0x04; /* SRST DEVCTL */
1216:
1217: cmd_slot->prdtl = 0;
1218: cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1219: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
1220: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
1221: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W); /* Write */
1222:
1223: ccb->ccb_xa.state = ATA_S_PENDING;
1224: if (ahci_poll(ccb, 1000, NULL) != 0)
1225: goto err;
1226:
1227: /* Prep second D2H command to read status and complete reset sequence */
1228: fis[0] = 0x27; /* Host to device */
1229: fis[15] = 0;
1230:
1231: cmd_slot->prdtl = 0;
1232: cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1233: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W);
1234:
1235: ccb->ccb_xa.state = ATA_S_PENDING;
1236: if (ahci_poll(ccb, 1000, NULL) != 0)
1237: goto err;
1238:
1239: if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
1240: AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR)) {
1241: printf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1242: PORTNAME(ap), ahci_pread(ap, AHCI_PREG_TFD),
1243: AHCI_PFMT_TFD_STS);
1244: rc = EBUSY;
1245: goto err;
1246: }
1247:
1248: rc = 0;
1249: err:
1250: if (ccb != NULL) {
1251: /* Abort our command, if it failed, by stopping command DMA. */
1252: if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
1253: printf("%s: stopping the port, softreset slot %d was "
1254: "still active.\n", PORTNAME(ap), ccb->ccb_slot);
1255: ahci_port_stop(ap, 0);
1256: }
1257: ccb->ccb_xa.state = ATA_S_ERROR;
1258: ahci_put_err_ccb(ccb);
1259: }
1260:
1261: /* Restore saved CMD register state */
1262: ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1263:
1264: splx(s);
1265:
1266: return (rc);
1267: }
1268:
1269: /* AHCI port reset, Section 10.4.2 */
1270: int
1271: ahci_port_portreset(struct ahci_port *ap)
1272: {
1273: u_int32_t cmd, r;
1274: int rc;
1275:
1276: DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1277:
1278: /* Save previous command register state */
1279: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1280:
1281: /* Clear ST, ignoring failure */
1282: ahci_port_stop(ap, 0);
1283:
1284: /* Perform device detection */
1285: ahci_pwrite(ap, AHCI_PREG_SCTL, 0);
1286: delay(10000);
1287: r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_SPD_ANY |
1288: AHCI_PREG_SCTL_DET_INIT;
1289: ahci_pwrite(ap, AHCI_PREG_SCTL, r);
1290: delay(10000); /* wait at least 1ms for COMRESET to be sent */
1291: r &= ~AHCI_PREG_SCTL_DET_INIT;
1292: r |= AHCI_PREG_SCTL_DET_NONE;
1293: ahci_pwrite(ap, AHCI_PREG_SCTL, r);
1294: delay(10000);
1295:
1296: /* Wait for device to be detected and communications established */
1297: if (ahci_pwait_eq(ap, AHCI_PREG_SSTS, AHCI_PREG_SSTS_DET,
1298: AHCI_PREG_SSTS_DET_DEV)) {
1299: rc = ENODEV;
1300: goto err;
1301: }
1302:
1303: /* Clear SERR (incl X bit), so TFD can update */
1304: ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR));
1305:
1306: /* Wait for device to become ready */
1307: /* XXX maybe more than the default wait is appropriate here? */
1308: if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY |
1309: AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR)) {
1310: rc = EBUSY;
1311: goto err;
1312: }
1313:
1314: rc = 0;
1315: err:
1316: /* Restore preserved port state */
1317: ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1318:
1319: return (rc);
1320: }
1321:
1322: int
1323: ahci_load_prdt(struct ahci_ccb *ccb)
1324: {
1325: struct ahci_port *ap = ccb->ccb_port;
1326: struct ahci_softc *sc = ap->ap_sc;
1327: struct ata_xfer *xa = &ccb->ccb_xa;
1328: struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt, *prd;
1329: bus_dmamap_t dmap = ccb->ccb_dmamap;
1330: struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
1331: u_int64_t addr;
1332: int i, error;
1333:
1334: if (xa->datalen == 0) {
1335: ccb->ccb_cmd_hdr->prdtl = 0;
1336: return (0);
1337: }
1338:
1339: error = bus_dmamap_load(sc->sc_dmat, dmap, xa->data, xa->datalen, NULL,
1340: (xa->flags & ATA_F_NOWAIT) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK);
1341: if (error != 0) {
1342: printf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
1343: return (1);
1344: }
1345:
1346: for (i = 0; i < dmap->dm_nsegs; i++) {
1347: prd = &prdt[i];
1348:
1349: addr = dmap->dm_segs[i].ds_addr;
1350: prd->dba_hi = htole32((u_int32_t)(addr >> 32));
1351: prd->dba_lo = htole32((u_int32_t)addr);
1352: #ifdef DIAGNOSTIC
1353: if (addr & 1) {
1354: printf("%s: requested DMA at an odd address %llx\n",
1355: PORTNAME(ap), (unsigned long long)addr);
1356: return (1);
1357: }
1358: if (dmap->dm_segs[i].ds_len & 1) {
1359: printf("%s: requested DMA length %d is not even\n",
1360: PORTNAME(ap), (int)dmap->dm_segs[i].ds_len);
1361: return (1);
1362: }
1363: #endif
1364: prd->flags = htole32(dmap->dm_segs[i].ds_len - 1);
1365: }
1366: if (xa->flags & ATA_F_PIO)
1367: prd->flags |= htole32(AHCI_PRDT_FLAG_INTR);
1368:
1369: cmd_slot->prdtl = htole16(dmap->dm_nsegs);
1370:
1371: bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1372: (xa->flags & ATA_F_READ) ? BUS_DMASYNC_PREREAD :
1373: BUS_DMASYNC_PREWRITE);
1374:
1375: return (0);
1376: }
1377:
1378: void
1379: ahci_unload_prdt(struct ahci_ccb *ccb)
1380: {
1381: struct ahci_port *ap = ccb->ccb_port;
1382: struct ahci_softc *sc = ap->ap_sc;
1383: struct ata_xfer *xa = &ccb->ccb_xa;
1384: bus_dmamap_t dmap = ccb->ccb_dmamap;
1385:
1386: if (xa->datalen != 0) {
1387: bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
1388: (xa->flags & ATA_F_READ) ? BUS_DMASYNC_POSTREAD :
1389: BUS_DMASYNC_POSTWRITE);
1390:
1391: bus_dmamap_unload(sc->sc_dmat, dmap);
1392:
1393: if (ccb->ccb_xa.flags & ATA_F_NCQ)
1394: xa->resid = 0;
1395: else
1396: xa->resid = xa->datalen -
1397: letoh32(ccb->ccb_cmd_hdr->prdbc);
1398: }
1399: }
1400:
1401: int
1402: ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *))
1403: {
1404: struct ahci_port *ap = ccb->ccb_port;
1405: int s;
1406:
1407: s = splbio();
1408: ahci_start(ccb);
1409: do {
1410: if (ISSET(ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS),
1411: 1 << ccb->ccb_slot)) {
1412: splx(s);
1413: return (0);
1414: }
1415:
1416: delay(1000);
1417: } while (--timeout > 0);
1418:
1419: /* Run timeout while at splbio, otherwise ahci_intr could interfere. */
1420: if (timeout_fn != NULL)
1421: timeout_fn(ccb);
1422:
1423: splx(s);
1424:
1425: return (1);
1426: }
1427:
1428: void
1429: ahci_start(struct ahci_ccb *ccb)
1430: {
1431: struct ahci_port *ap = ccb->ccb_port;
1432: struct ahci_softc *sc = ap->ap_sc;
1433:
1434: KASSERT(ccb->ccb_xa.state == ATA_S_PENDING);
1435:
1436: /* Zero transferred byte count before transfer */
1437: ccb->ccb_cmd_hdr->prdbc = 0;
1438:
1439: /* Sync command list entry and corresponding command table entry */
1440: bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
1441: ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
1442: sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_PREWRITE);
1443: bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
1444: ccb->ccb_slot * sizeof(struct ahci_cmd_table),
1445: sizeof(struct ahci_cmd_table), BUS_DMASYNC_PREWRITE);
1446:
1447: /* Prepare RFIS area for write by controller */
1448: bus_dmamap_sync(sc->sc_dmat, AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
1449: sizeof(struct ahci_rfis), BUS_DMASYNC_PREREAD);
1450:
1451: if (ccb->ccb_xa.flags & ATA_F_NCQ) {
1452: /* Issue NCQ commands only when there are no outstanding
1453: * standard commands. */
1454: if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending))
1455: TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1456: else {
1457: KASSERT(ap->ap_active_cnt == 0);
1458: ap->ap_sactive |= (1 << ccb->ccb_slot);
1459: ccb->ccb_xa.state = ATA_S_ONCHIP;
1460: ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
1461: ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1462: }
1463: } else {
1464: /* Wait for all NCQ commands to finish before issuing standard
1465: * command. */
1466: if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2)
1467: TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1468: else if (ap->ap_active_cnt < 2) {
1469: ap->ap_active |= 1 << ccb->ccb_slot;
1470: ccb->ccb_xa.state = ATA_S_ONCHIP;
1471: ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1472: ap->ap_active_cnt++;
1473: }
1474: }
1475: }
1476:
1477: void
1478: ahci_issue_pending_ncq_commands(struct ahci_port *ap)
1479: {
1480: struct ahci_ccb *nextccb;
1481: u_int32_t sact_change = 0;
1482:
1483: KASSERT(ap->ap_active_cnt == 0);
1484:
1485: nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1486: if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
1487: return;
1488:
1489: /* Start all the NCQ commands at the head of the pending list. */
1490: do {
1491: TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
1492: sact_change |= 1 << nextccb->ccb_slot;
1493: nextccb->ccb_xa.state = ATA_S_ONCHIP;
1494: nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1495: } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ));
1496:
1497: ap->ap_sactive |= sact_change;
1498: ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
1499: ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
1500:
1501: return;
1502: }
1503:
1504: void
1505: ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
1506: {
1507: struct ahci_ccb *nextccb;
1508:
1509: nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1510: if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) {
1511: KASSERT(last_was_ncq == 0); /* otherwise it should have
1512: * been started already. */
1513:
1514: /* Issue NCQ commands only when there are no outstanding
1515: * standard commands. */
1516: ap->ap_active_cnt--;
1517: if (ap->ap_active == 0)
1518: ahci_issue_pending_ncq_commands(ap);
1519: else
1520: KASSERT(ap->ap_active_cnt == 1);
1521: } else if (nextccb) {
1522: if (ap->ap_sactive != 0 || last_was_ncq)
1523: KASSERT(ap->ap_active_cnt == 0);
1524:
1525: /* Wait for all NCQ commands to finish before issuing standard
1526: * command. */
1527: if (ap->ap_sactive != 0)
1528: return;
1529:
1530: /* Keep up to 2 standard commands on-chip at a time. */
1531: do {
1532: TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
1533: ap->ap_active |= 1 << nextccb->ccb_slot;
1534: nextccb->ccb_xa.state = ATA_S_ONCHIP;
1535: ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
1536: if (last_was_ncq)
1537: ap->ap_active_cnt++;
1538: if (ap->ap_active_cnt == 2)
1539: break;
1540: KASSERT(ap->ap_active_cnt == 1);
1541: nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1542: } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ));
1543: } else if (!last_was_ncq) {
1544: KASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2);
1545:
1546: /* Standard command finished, none waiting to start. */
1547: ap->ap_active_cnt--;
1548: } else {
1549: KASSERT(ap->ap_active_cnt == 0);
1550:
1551: /* NCQ command finished. */
1552: }
1553: }
1554:
1555: int
1556: ahci_intr(void *arg)
1557: {
1558: struct ahci_softc *sc = arg;
1559: u_int32_t is, ack = 0;
1560: int port;
1561:
1562: /* Read global interrupt status */
1563: is = ahci_read(sc, AHCI_REG_IS);
1564: if (is == 0)
1565: return (0);
1566: ack = is;
1567:
1568: #ifdef AHCI_COALESCE
1569: /* Check coalescing interrupt first */
1570: if (is & sc->sc_ccc_mask) {
1571: DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
1572: DEVNAME(sc));
1573: is &= ~sc->sc_ccc_mask;
1574: is |= sc->sc_ccc_ports_cur;
1575: }
1576: #endif
1577:
1578: /* Process interrupts for each port */
1579: while (is) {
1580: port = ffs(is) - 1;
1581: if (sc->sc_ports[port])
1582: ahci_port_intr(sc->sc_ports[port],
1583: AHCI_PREG_CI_ALL_SLOTS);
1584: is &= ~(1 << port);
1585: }
1586:
1587: /* Finally, acknowledge global interrupt */
1588: ahci_write(sc, AHCI_REG_IS, ack);
1589:
1590: return (1);
1591: }
1592:
1593: u_int32_t
1594: ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask)
1595: {
1596: struct ahci_softc *sc = ap->ap_sc;
1597: u_int32_t is, ci_saved, ci_masked, processed = 0;
1598: int slot, need_restart = 0;
1599: struct ahci_ccb *ccb;
1600: volatile u_int32_t *active;
1601: #ifdef DIAGNOSTIC
1602: u_int32_t tmp;
1603: #endif
1604:
1605: is = ahci_pread(ap, AHCI_PREG_IS);
1606:
1607: /* Ack port interrupt only if checking all command slots. */
1608: if (ci_mask == AHCI_PREG_CI_ALL_SLOTS)
1609: ahci_pwrite(ap, AHCI_PREG_IS, is);
1610:
1611: if (is)
1612: DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap),
1613: is, AHCI_PFMT_IS);
1614:
1615: if (ap->ap_sactive) {
1616: /* Active NCQ commands - use SActive instead of CI */
1617: KASSERT(ap->ap_active == 0);
1618: KASSERT(ap->ap_active_cnt == 0);
1619: ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
1620: active = &ap->ap_sactive;
1621: } else {
1622: /* Save CI */
1623: ci_saved = ahci_pread(ap, AHCI_PREG_CI);
1624: active = &ap->ap_active;
1625: }
1626:
1627: /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */
1628: if (is & AHCI_PREG_IS_TFES) {
1629: u_int32_t tfd, serr;
1630: int err_slot;
1631:
1632: tfd = ahci_pread(ap, AHCI_PREG_TFD);
1633: serr = ahci_pread(ap, AHCI_PREG_SERR);
1634:
1635: if (ap->ap_sactive == 0) {
1636: /* Errored slot is easy to determine from CMD. */
1637: err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap,
1638: AHCI_PREG_CMD));
1639: ccb = &ap->ap_ccbs[err_slot];
1640:
1641: /* Preserve received taskfile data from the RFIS. */
1642: memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
1643: sizeof(struct ata_fis_d2h));
1644: } else
1645: err_slot = -1; /* Must extract error from log page */
1646:
1647: DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:"
1648: " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd,
1649: AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr),
1650: AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr),
1651: AHCI_PFMT_SERR_DIAG);
1652:
1653: /* Turn off ST to clear CI and SACT. */
1654: ahci_port_stop(ap, 0);
1655: need_restart = 1;
1656:
1657: /* Clear SERR to enable capturing new errors. */
1658: ahci_pwrite(ap, AHCI_PREG_SERR, serr);
1659:
1660: /* Acknowledge the interrupts we can recover from. */
1661: ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES |
1662: AHCI_PREG_IS_IFS);
1663: is = ahci_pread(ap, AHCI_PREG_IS);
1664:
1665: /* If device hasn't cleared its busy status, try to idle it. */
1666: if (ISSET(tfd, AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1667: printf("%s: attempting to idle device\n", PORTNAME(ap));
1668: if (ahci_port_softreset(ap)) {
1669: printf("%s: failed to soft reset device\n",
1670: PORTNAME(ap));
1671: if (ahci_port_portreset(ap)) {
1672: printf("%s: failed to port reset "
1673: "device, give up on it\n",
1674: PORTNAME(ap));
1675: goto fatal;
1676: }
1677: }
1678:
1679: /* Had to reset device, can't gather extended info. */
1680: } else if (ap->ap_sactive) {
1681: /* Recover the NCQ error from log page 10h. */
1682: ahci_port_read_ncq_error(ap, &err_slot);
1683: if (err_slot < 0)
1684: goto failall;
1685:
1686: DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
1687: PORTNAME(ap), err_slot);
1688:
1689: ccb = &ap->ap_ccbs[err_slot];
1690: } else {
1691: /* Didn't reset, could gather extended info from log. */
1692: }
1693:
1694: /*
1695: * If we couldn't determine the errored slot, reset the port
1696: * and fail all the active slots.
1697: */
1698: if (err_slot == -1) {
1699: if (ahci_port_softreset(ap) != 0 &&
1700: ahci_port_portreset(ap) != 0) {
1701: printf("%s: couldn't reset after NCQ error, "
1702: "disabling device.\n", PORTNAME(ap));
1703: goto fatal;
1704: }
1705: printf("%s: couldn't recover NCQ error, failing "
1706: "all outstanding commands.\n", PORTNAME(ap));
1707: goto failall;
1708: }
1709:
1710: /* Clear the failed command in saved CI so completion runs. */
1711: ci_saved &= ~(1 << err_slot);
1712:
1713: /* Note the error in the ata_xfer. */
1714: KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
1715: ccb->ccb_xa.state = ATA_S_ERROR;
1716:
1717: #ifdef DIAGNOSTIC
1718: /* There may only be one outstanding standard command now. */
1719: if (ap->ap_sactive == 0) {
1720: tmp = ci_saved;
1721: if (tmp) {
1722: slot = ffs(tmp) - 1;
1723: tmp &= ~(1 << slot);
1724: KASSERT(tmp == 0);
1725: }
1726: }
1727: #endif
1728: }
1729:
1730: /* Check for remaining errors - they are fatal. */
1731: if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
1732: AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
1733: printf("%s: unrecoverable errors (IS: %b), disabling port.\n",
1734: PORTNAME(ap), is, AHCI_PFMT_IS);
1735:
1736: /* XXX try recovery first */
1737: goto fatal;
1738: }
1739:
1740: /* Fail all outstanding commands if we know the port won't recover. */
1741: if (ap->ap_state == AP_S_FATAL_ERROR) {
1742: fatal:
1743: ap->ap_state = AP_S_FATAL_ERROR;
1744: failall:
1745:
1746: /* Ensure port is shut down. */
1747: ahci_port_stop(ap, 1);
1748:
1749: /* Error all the active slots. */
1750: ci_masked = ci_saved & *active;
1751: while (ci_masked) {
1752: slot = ffs(ci_masked) - 1;
1753: ccb = &ap->ap_ccbs[slot];
1754: ci_masked &= ~(1 << slot);
1755: ccb->ccb_xa.state = ATA_S_ERROR;
1756: }
1757:
1758: /* Run completion for all active slots. */
1759: ci_saved &= ~*active;
1760:
1761: /* Don't restart the port if our problems were deemed fatal. */
1762: if (ap->ap_state == AP_S_FATAL_ERROR)
1763: need_restart = 0;
1764: }
1765:
1766: /*
1767: * CCB completion is detected by noticing its slot's bit in CI has
1768: * changed to zero some time after we activated it.
1769: * If we are polling, we may only be interested in particular slot(s).
1770: */
1771: ci_masked = ~ci_saved & *active & ci_mask;
1772: while (ci_masked) {
1773: slot = ffs(ci_masked) - 1;
1774: ccb = &ap->ap_ccbs[slot];
1775: ci_masked &= ~(1 << slot);
1776:
1777: DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
1778: PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
1779: " (error)" : "");
1780:
1781: bus_dmamap_sync(sc->sc_dmat,
1782: AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
1783: ccb->ccb_slot * sizeof(struct ahci_cmd_hdr),
1784: sizeof(struct ahci_cmd_hdr), BUS_DMASYNC_POSTWRITE);
1785:
1786: bus_dmamap_sync(sc->sc_dmat,
1787: AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
1788: ccb->ccb_slot * sizeof(struct ahci_cmd_table),
1789: sizeof(struct ahci_cmd_table), BUS_DMASYNC_POSTWRITE);
1790:
1791: bus_dmamap_sync(sc->sc_dmat,
1792: AHCI_DMA_MAP(ap->ap_dmamem_rfis), 0,
1793: sizeof(struct ahci_rfis), BUS_DMASYNC_POSTREAD);
1794:
1795: *active &= ~(1 << ccb->ccb_slot);
1796: ccb->ccb_done(ccb);
1797:
1798: processed |= 1 << ccb->ccb_slot;
1799: }
1800:
1801: if (need_restart) {
1802: /* Restart command DMA on the port */
1803: ahci_port_start(ap, 0);
1804:
1805: /* Re-enable outstanding commands on port. */
1806: if (ci_saved) {
1807: #ifdef DIAGNOSTIC
1808: tmp = ci_saved;
1809: while (tmp) {
1810: slot = ffs(tmp) - 1;
1811: tmp &= ~(1 << slot);
1812: ccb = &ap->ap_ccbs[slot];
1813: KASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
1814: KASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
1815: (!!ap->ap_sactive));
1816: }
1817: #endif
1818: DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
1819: "re-enabling%s slots %08x\n", PORTNAME(ap),
1820: ap->ap_sactive ? " NCQ" : "", ci_saved);
1821:
1822: if (ap->ap_sactive)
1823: ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
1824: ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
1825: }
1826: }
1827:
1828: return (processed);
1829: }
1830:
1831: struct ahci_ccb *
1832: ahci_get_ccb(struct ahci_port *ap)
1833: {
1834: struct ahci_ccb *ccb;
1835:
1836: ccb = TAILQ_FIRST(&ap->ap_ccb_free);
1837: if (ccb != NULL) {
1838: KASSERT(ccb->ccb_xa.state == ATA_S_PUT);
1839: TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
1840: ccb->ccb_xa.state = ATA_S_SETUP;
1841: }
1842:
1843: return (ccb);
1844: }
1845:
1846: void
1847: ahci_put_ccb(struct ahci_ccb *ccb)
1848: {
1849: struct ahci_port *ap = ccb->ccb_port;
1850:
1851: #ifdef DIAGNOSTIC
1852: if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
1853: ccb->ccb_xa.state != ATA_S_TIMEOUT &&
1854: ccb->ccb_xa.state != ATA_S_ERROR) {
1855: printf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
1856: "slot %d\n", PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
1857: ccb->ccb_slot);
1858: }
1859: #endif
1860:
1861: ccb->ccb_xa.state = ATA_S_PUT;
1862: TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
1863: }
1864:
1865: struct ahci_ccb *
1866: ahci_get_err_ccb(struct ahci_port *ap)
1867: {
1868: struct ahci_ccb *err_ccb;
1869: u_int32_t sact;
1870:
1871: splassert(IPL_BIO);
1872:
1873: /* No commands may be active on the chip. */
1874: sact = ahci_pread(ap, AHCI_PREG_SACT);
1875: if (sact != 0)
1876: printf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
1877: KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
1878:
1879: #ifdef DIAGNOSTIC
1880: KASSERT(ap->ap_err_busy == 0);
1881: ap->ap_err_busy = 1;
1882: #endif
1883: /* Save outstanding command state. */
1884: ap->ap_err_saved_active = ap->ap_active;
1885: ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
1886: ap->ap_err_saved_sactive = ap->ap_sactive;
1887:
1888: /*
1889: * Pretend we have no commands outstanding, so that completions won't
1890: * run prematurely.
1891: */
1892: ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
1893:
1894: /*
1895: * Grab a CCB to use for error recovery. This should never fail, as
1896: * we ask atascsi to reserve one for us at init time.
1897: */
1898: err_ccb = ahci_get_ccb(ap);
1899: KASSERT(err_ccb != NULL);
1900: err_ccb->ccb_xa.flags = 0;
1901: err_ccb->ccb_done = ahci_empty_done;
1902:
1903: return err_ccb;
1904: }
1905:
1906: void
1907: ahci_put_err_ccb(struct ahci_ccb *ccb)
1908: {
1909: struct ahci_port *ap = ccb->ccb_port;
1910: u_int32_t sact;
1911:
1912: splassert(IPL_BIO);
1913:
1914: #ifdef DIAGNOSTIC
1915: KASSERT(ap->ap_err_busy);
1916: #endif
1917: /* No commands may be active on the chip */
1918: sact = ahci_pread(ap, AHCI_PREG_SACT);
1919: if (sact != 0)
1920: printf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", sact);
1921: KASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
1922:
1923: /* Done with the CCB */
1924: ahci_put_ccb(ccb);
1925:
1926: /* Restore outstanding command state */
1927: ap->ap_sactive = ap->ap_err_saved_sactive;
1928: ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
1929: ap->ap_active = ap->ap_err_saved_active;
1930:
1931: #ifdef DIAGNOSTIC
1932: ap->ap_err_busy = 0;
1933: #endif
1934: }
1935:
1936: int
1937: ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp)
1938: {
1939: struct ahci_ccb *ccb;
1940: struct ahci_cmd_hdr *cmd_slot;
1941: u_int32_t cmd;
1942: struct ata_fis_h2d *fis;
1943: int rc = EIO;
1944:
1945: DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
1946:
1947: /* Save command register state. */
1948: cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1949:
1950: /* Port should have been idled already. Start it. */
1951: KASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
1952: ahci_port_start(ap, 0);
1953:
1954: /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
1955: ccb = ahci_get_err_ccb(ap);
1956: ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
1957: ccb->ccb_xa.data = ap->ap_err_scratch;
1958: ccb->ccb_xa.datalen = 512;
1959: cmd_slot = ccb->ccb_cmd_hdr;
1960: bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table));
1961:
1962: fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
1963: fis->type = ATA_FIS_TYPE_H2D;
1964: fis->flags = ATA_H2D_FLAGS_CMD;
1965: fis->command = ATA_C_READ_LOG_EXT;
1966: fis->lba_low = 0x10; /* queued error log page (10h) */
1967: fis->sector_count = 1; /* number of sectors (1) */
1968: fis->sector_count_exp = 0;
1969: fis->lba_mid = 0; /* starting offset */
1970: fis->lba_mid_exp = 0;
1971: fis->device = 0;
1972:
1973: cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1974:
1975: if (ahci_load_prdt(ccb) != 0) {
1976: rc = ENOMEM; /* XXX caller must abort all commands */
1977: goto err;
1978: }
1979:
1980: ccb->ccb_xa.state = ATA_S_PENDING;
1981: if (ahci_poll(ccb, 1000, NULL) != 0)
1982: goto err;
1983:
1984: rc = 0;
1985: err:
1986: /* Abort our command, if it failed, by stopping command DMA. */
1987: if (rc != 0 && ISSET(ap->ap_active, 1 << ccb->ccb_slot)) {
1988: printf("%s: log page read failed, slot %d was still active.\n",
1989: PORTNAME(ap), ccb->ccb_slot);
1990: ahci_port_stop(ap, 0);
1991: }
1992:
1993: /* Done with the error CCB now. */
1994: ahci_unload_prdt(ccb);
1995: ahci_put_err_ccb(ccb);
1996:
1997: /* Extract failed register set and tags from the scratch space. */
1998: if (rc == 0) {
1999: struct ata_log_page_10h *log;
2000: int err_slot;
2001:
2002: log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2003: if (ISSET(log->err_regs.type, ATA_LOG_10H_TYPE_NOTQUEUED)) {
2004: /* Not queued bit was set - wasn't an NCQ error? */
2005: printf("%s: read NCQ error page, but not an NCQ "
2006: "error?\n", PORTNAME(ap));
2007: rc = ESRCH;
2008: } else {
2009: /* Copy back the log record as a D2H register FIS. */
2010: *err_slotp = err_slot = log->err_regs.type &
2011: ATA_LOG_10H_TYPE_TAG_MASK;
2012:
2013: ccb = &ap->ap_ccbs[err_slot];
2014: memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
2015: sizeof(struct ata_fis_d2h));
2016: ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
2017: ccb->ccb_xa.rfis.flags = 0;
2018: }
2019: }
2020:
2021: /* Restore saved CMD register state */
2022: ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
2023:
2024: return (rc);
2025: }
2026:
2027: struct ahci_dmamem *
2028: ahci_dmamem_alloc(struct ahci_softc *sc, size_t size)
2029: {
2030: struct ahci_dmamem *adm;
2031: int nsegs;
2032:
2033: adm = malloc(sizeof(struct ahci_dmamem), M_DEVBUF, M_NOWAIT);
2034: if (adm == NULL)
2035: return (NULL);
2036:
2037: bzero(adm, sizeof(struct ahci_dmamem));
2038: adm->adm_size = size;
2039:
2040: if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
2041: BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0)
2042: goto admfree;
2043:
2044: if (bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &adm->adm_seg,
2045: 1, &nsegs, BUS_DMA_NOWAIT) != 0)
2046: goto destroy;
2047:
2048: if (bus_dmamem_map(sc->sc_dmat, &adm->adm_seg, nsegs, size,
2049: &adm->adm_kva, BUS_DMA_NOWAIT) != 0)
2050: goto free;
2051:
2052: if (bus_dmamap_load(sc->sc_dmat, adm->adm_map, adm->adm_kva, size,
2053: NULL, BUS_DMA_NOWAIT) != 0)
2054: goto unmap;
2055:
2056: bzero(adm->adm_kva, size);
2057:
2058: return (adm);
2059:
2060: unmap:
2061: bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, size);
2062: free:
2063: bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
2064: destroy:
2065: bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
2066: admfree:
2067: free(adm, M_DEVBUF);
2068:
2069: return (NULL);
2070: }
2071:
2072: void
2073: ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
2074: {
2075: bus_dmamap_unload(sc->sc_dmat, adm->adm_map);
2076: bus_dmamem_unmap(sc->sc_dmat, adm->adm_kva, adm->adm_size);
2077: bus_dmamem_free(sc->sc_dmat, &adm->adm_seg, 1);
2078: bus_dmamap_destroy(sc->sc_dmat, adm->adm_map);
2079: free(adm, M_DEVBUF);
2080: }
2081:
2082: u_int32_t
2083: ahci_read(struct ahci_softc *sc, bus_size_t r)
2084: {
2085: bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2086: BUS_SPACE_BARRIER_READ);
2087: return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
2088: }
2089:
2090: void
2091: ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2092: {
2093: bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
2094: bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2095: BUS_SPACE_BARRIER_WRITE);
2096: }
2097:
2098: int
2099: ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
2100: u_int32_t target)
2101: {
2102: int i;
2103:
2104: for (i = 0; i < 1000; i++) {
2105: if ((ahci_read(sc, r) & mask) != target)
2106: return (0);
2107: delay(1000);
2108: }
2109:
2110: return (1);
2111: }
2112:
2113: u_int32_t
2114: ahci_pread(struct ahci_port *ap, bus_size_t r)
2115: {
2116: bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2117: BUS_SPACE_BARRIER_READ);
2118: return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
2119: }
2120:
2121: void
2122: ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
2123: {
2124: bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
2125: bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2126: BUS_SPACE_BARRIER_WRITE);
2127: }
2128:
2129: int
2130: ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask,
2131: u_int32_t target)
2132: {
2133: int i;
2134:
2135: for (i = 0; i < 1000; i++) {
2136: if ((ahci_pread(ap, r) & mask) == target)
2137: return (0);
2138: delay(1000);
2139: }
2140:
2141: return (1);
2142: }
2143:
2144: int
2145: ahci_ata_probe(void *xsc, int port)
2146: {
2147: struct ahci_softc *sc = xsc;
2148: struct ahci_port *ap = sc->sc_ports[port];
2149: u_int32_t sig;
2150:
2151: if (ap == NULL)
2152: return (ATA_PORT_T_NONE);
2153:
2154: sig = ahci_pread(ap, AHCI_PREG_SIG);
2155: if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000))
2156: return (ATA_PORT_T_ATAPI);
2157: else
2158: return (ATA_PORT_T_DISK);
2159: }
2160:
2161: struct ata_xfer *
2162: ahci_ata_get_xfer(void *aaa_cookie, int port)
2163: {
2164: struct ahci_softc *sc = aaa_cookie;
2165: struct ahci_port *ap = sc->sc_ports[port];
2166: struct ahci_ccb *ccb;
2167:
2168: splassert(IPL_BIO);
2169:
2170: ccb = ahci_get_ccb(ap);
2171: if (ccb == NULL) {
2172: DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
2173: PORTNAME(ap));
2174: return (NULL);
2175: }
2176:
2177: DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
2178: PORTNAME(ap), ccb->ccb_slot);
2179:
2180: return ((struct ata_xfer *)ccb);
2181: }
2182:
2183: void
2184: ahci_ata_put_xfer(struct ata_xfer *xa)
2185: {
2186: struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
2187:
2188: splassert(IPL_BIO);
2189:
2190: DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
2191:
2192: ahci_put_ccb(ccb);
2193: }
2194:
2195: int
2196: ahci_ata_cmd(struct ata_xfer *xa)
2197: {
2198: struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
2199: struct ahci_cmd_hdr *cmd_slot;
2200: int s;
2201:
2202: KASSERT(xa->state == ATA_S_SETUP);
2203:
2204: if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
2205: goto failcmd;
2206:
2207: ccb->ccb_done = ahci_ata_cmd_done;
2208:
2209: cmd_slot = ccb->ccb_cmd_hdr;
2210: cmd_slot->flags = htole16(5); /* FIS length (in DWORDs) */
2211:
2212: if (xa->flags & ATA_F_WRITE)
2213: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W);
2214:
2215: if (xa->flags & ATA_F_PACKET)
2216: cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_A);
2217:
2218: if (ahci_load_prdt(ccb) != 0)
2219: goto failcmd;
2220:
2221: timeout_set(&xa->stimeout, ahci_ata_cmd_timeout, ccb);
2222:
2223: xa->state = ATA_S_PENDING;
2224:
2225: if (xa->flags & ATA_F_POLL) {
2226: ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout);
2227: return (ATA_COMPLETE);
2228: }
2229:
2230: timeout_add(&xa->stimeout, (xa->timeout * hz) / 1000);
2231:
2232: s = splbio();
2233: ahci_start(ccb);
2234: splx(s);
2235: return (ATA_QUEUED);
2236:
2237: failcmd:
2238: s = splbio();
2239: xa->state = ATA_S_ERROR;
2240: xa->complete(xa);
2241: splx(s);
2242: return (ATA_ERROR);
2243: }
2244:
2245: void
2246: ahci_ata_cmd_done(struct ahci_ccb *ccb)
2247: {
2248: struct ata_xfer *xa = &ccb->ccb_xa;
2249:
2250: timeout_del(&xa->stimeout);
2251:
2252: if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR)
2253: ahci_issue_pending_commands(ccb->ccb_port,
2254: xa->flags & ATA_F_NCQ);
2255:
2256: ahci_unload_prdt(ccb);
2257:
2258: if (xa->state == ATA_S_ONCHIP)
2259: xa->state = ATA_S_COMPLETE;
2260: #ifdef DIAGNOSTIC
2261: else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
2262: printf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
2263: "slot %d\n", PORTNAME(ccb->ccb_port), xa->state,
2264: ccb->ccb_slot);
2265: #endif
2266: if (xa->state != ATA_S_TIMEOUT)
2267: xa->complete(xa);
2268: }
2269:
2270: void
2271: ahci_ata_cmd_timeout(void *arg)
2272: {
2273: struct ahci_ccb *ccb = arg;
2274: struct ata_xfer *xa = &ccb->ccb_xa;
2275: struct ahci_port *ap = ccb->ccb_port;
2276: int s, ccb_was_started, ncq_cmd;
2277: volatile u_int32_t *active;
2278:
2279: s = splbio();
2280:
2281: ncq_cmd = (xa->flags & ATA_F_NCQ);
2282: active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
2283:
2284: if (ccb->ccb_xa.state == ATA_S_PENDING) {
2285: DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
2286: "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
2287: TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
2288: ccb_was_started = 0;
2289: } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap,
2290: 1 << ccb->ccb_slot)) {
2291: DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed "
2292: "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot);
2293: goto ret;
2294: } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
2295: DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
2296: "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
2297: ISSET(*active, 1 << ccb->ccb_slot) ?
2298: " but slot is still active?" : ".");
2299: goto ret;
2300: } else if (!ISSET(ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT :
2301: AHCI_PREG_CI), 1 << ccb->ccb_slot) && ISSET(*active,
2302: 1 << ccb->ccb_slot)) {
2303: DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but "
2304: "IRQ handler didn't detect it. Why?\n", PORTNAME(ap),
2305: ccb->ccb_slot);
2306: *active &= ~(1 << ccb->ccb_slot);
2307: ccb->ccb_done(ccb);
2308: goto ret;
2309: } else {
2310: ccb_was_started = 1;
2311: }
2312:
2313: /* Complete the slot with a timeout error. */
2314: ccb->ccb_xa.state = ATA_S_TIMEOUT;
2315: *active &= ~(1 << ccb->ccb_slot);
2316: DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
2317: ccb->ccb_done(ccb); /* This won't issue pending commands or run the
2318: atascsi completion. */
2319:
2320: /* Reset port to abort running command. */
2321: if (ccb_was_started) {
2322: DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
2323: "in slot %d, active %08x\n", PORTNAME(ap), ncq_cmd ? " NCQ"
2324: : "", ccb->ccb_slot, *active);
2325: if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap)
2326: != 0) {
2327: printf("%s: failed to reset port during timeout "
2328: "handling, disabling it\n", PORTNAME(ap));
2329: ap->ap_state = AP_S_FATAL_ERROR;
2330: }
2331:
2332: /* Restart any other commands that were aborted by the reset. */
2333: if (*active) {
2334: DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
2335: "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
2336: *active);
2337: if (ncq_cmd)
2338: ahci_pwrite(ap, AHCI_PREG_SACT, *active);
2339: ahci_pwrite(ap, AHCI_PREG_CI, *active);
2340: }
2341: }
2342:
2343: /* Issue any pending commands now. */
2344: DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
2345: if (ccb_was_started)
2346: ahci_issue_pending_commands(ap, ncq_cmd);
2347: else if (ap->ap_active == 0)
2348: ahci_issue_pending_ncq_commands(ap);
2349:
2350: /* Complete the timed out ata_xfer I/O (may generate new I/O). */
2351: DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
2352: xa->complete(xa);
2353:
2354: DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
2355: ret:
2356: splx(s);
2357: }
2358:
2359: void
2360: ahci_empty_done(struct ahci_ccb *ccb)
2361: {
2362: ccb->ccb_xa.state = ATA_S_COMPLETE;
2363: }
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to ahci.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / dev / pci