qp.c 54 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015
  1. /*
  2. * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #include <linux/module.h>
  33. #include "iw_cxgb4.h"
  34. static int db_delay_usecs = 1;
  35. module_param(db_delay_usecs, int, 0644);
  36. MODULE_PARM_DESC(db_delay_usecs, "Usecs to delay awaiting db fifo to drain");
  37. static int ocqp_support = 1;
  38. module_param(ocqp_support, int, 0644);
  39. MODULE_PARM_DESC(ocqp_support, "Support on-chip SQs (default=1)");
  40. int db_fc_threshold = 1000;
  41. module_param(db_fc_threshold, int, 0644);
  42. MODULE_PARM_DESC(db_fc_threshold,
  43. "QP count/threshold that triggers"
  44. " automatic db flow control mode (default = 1000)");
  45. int db_coalescing_threshold;
  46. module_param(db_coalescing_threshold, int, 0644);
  47. MODULE_PARM_DESC(db_coalescing_threshold,
  48. "QP count/threshold that triggers"
  49. " disabling db coalescing (default = 0)");
  50. static int max_fr_immd = T4_MAX_FR_IMMD;
  51. module_param(max_fr_immd, int, 0644);
  52. MODULE_PARM_DESC(max_fr_immd, "fastreg threshold for using DSGL instead of immedate");
  53. static int alloc_ird(struct c4iw_dev *dev, u32 ird)
  54. {
  55. int ret = 0;
  56. spin_lock_irq(&dev->lock);
  57. if (ird <= dev->avail_ird)
  58. dev->avail_ird -= ird;
  59. else
  60. ret = -ENOMEM;
  61. spin_unlock_irq(&dev->lock);
  62. if (ret)
  63. dev_warn(&dev->rdev.lldi.pdev->dev,
  64. "device IRD resources exhausted\n");
  65. return ret;
  66. }
  67. static void free_ird(struct c4iw_dev *dev, int ird)
  68. {
  69. spin_lock_irq(&dev->lock);
  70. dev->avail_ird += ird;
  71. spin_unlock_irq(&dev->lock);
  72. }
  73. static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state)
  74. {
  75. unsigned long flag;
  76. spin_lock_irqsave(&qhp->lock, flag);
  77. qhp->attr.state = state;
  78. spin_unlock_irqrestore(&qhp->lock, flag);
  79. }
  80. static void dealloc_oc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
  81. {
  82. c4iw_ocqp_pool_free(rdev, sq->dma_addr, sq->memsize);
  83. }
  84. static void dealloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
  85. {
  86. dma_free_coherent(&(rdev->lldi.pdev->dev), sq->memsize, sq->queue,
  87. pci_unmap_addr(sq, mapping));
  88. }
  89. static void dealloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
  90. {
  91. if (t4_sq_onchip(sq))
  92. dealloc_oc_sq(rdev, sq);
  93. else
  94. dealloc_host_sq(rdev, sq);
  95. }
  96. static int alloc_oc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
  97. {
  98. if (!ocqp_support || !ocqp_supported(&rdev->lldi))
  99. return -ENOSYS;
  100. sq->dma_addr = c4iw_ocqp_pool_alloc(rdev, sq->memsize);
  101. if (!sq->dma_addr)
  102. return -ENOMEM;
  103. sq->phys_addr = rdev->oc_mw_pa + sq->dma_addr -
  104. rdev->lldi.vr->ocq.start;
  105. sq->queue = (__force union t4_wr *)(rdev->oc_mw_kva + sq->dma_addr -
  106. rdev->lldi.vr->ocq.start);
  107. sq->flags |= T4_SQ_ONCHIP;
  108. return 0;
  109. }
  110. static int alloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq)
  111. {
  112. sq->queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev), sq->memsize,
  113. &(sq->dma_addr), GFP_KERNEL);
  114. if (!sq->queue)
  115. return -ENOMEM;
  116. sq->phys_addr = virt_to_phys(sq->queue);
  117. pci_unmap_addr_set(sq, mapping, sq->dma_addr);
  118. return 0;
  119. }
  120. static int alloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq, int user)
  121. {
  122. int ret = -ENOSYS;
  123. if (user)
  124. ret = alloc_oc_sq(rdev, sq);
  125. if (ret)
  126. ret = alloc_host_sq(rdev, sq);
  127. return ret;
  128. }
  129. static int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
  130. struct c4iw_dev_ucontext *uctx)
  131. {
  132. /*
  133. * uP clears EQ contexts when the connection exits rdma mode,
  134. * so no need to post a RESET WR for these EQs.
  135. */
  136. dma_free_coherent(&(rdev->lldi.pdev->dev),
  137. wq->rq.memsize, wq->rq.queue,
  138. dma_unmap_addr(&wq->rq, mapping));
  139. dealloc_sq(rdev, &wq->sq);
  140. c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
  141. kfree(wq->rq.sw_rq);
  142. kfree(wq->sq.sw_sq);
  143. c4iw_put_qpid(rdev, wq->rq.qid, uctx);
  144. c4iw_put_qpid(rdev, wq->sq.qid, uctx);
  145. return 0;
  146. }
  147. /*
  148. * Determine the BAR2 virtual address and qid. If pbar2_pa is not NULL,
  149. * then this is a user mapping so compute the page-aligned physical address
  150. * for mapping.
  151. */
  152. void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,
  153. enum cxgb4_bar2_qtype qtype,
  154. unsigned int *pbar2_qid, u64 *pbar2_pa)
  155. {
  156. u64 bar2_qoffset;
  157. int ret;
  158. ret = cxgb4_bar2_sge_qregs(rdev->lldi.ports[0], qid, qtype,
  159. pbar2_pa ? 1 : 0,
  160. &bar2_qoffset, pbar2_qid);
  161. if (ret)
  162. return NULL;
  163. if (pbar2_pa)
  164. *pbar2_pa = (rdev->bar2_pa + bar2_qoffset) & PAGE_MASK;
  165. if (is_t4(rdev->lldi.adapter_type))
  166. return NULL;
  167. return rdev->bar2_kva + bar2_qoffset;
  168. }
  169. static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
  170. struct t4_cq *rcq, struct t4_cq *scq,
  171. struct c4iw_dev_ucontext *uctx)
  172. {
  173. int user = (uctx != &rdev->uctx);
  174. struct fw_ri_res_wr *res_wr;
  175. struct fw_ri_res *res;
  176. int wr_len;
  177. struct c4iw_wr_wait wr_wait;
  178. struct sk_buff *skb;
  179. int ret = 0;
  180. int eqsize;
  181. wq->sq.qid = c4iw_get_qpid(rdev, uctx);
  182. if (!wq->sq.qid)
  183. return -ENOMEM;
  184. wq->rq.qid = c4iw_get_qpid(rdev, uctx);
  185. if (!wq->rq.qid) {
  186. ret = -ENOMEM;
  187. goto free_sq_qid;
  188. }
  189. if (!user) {
  190. wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq,
  191. GFP_KERNEL);
  192. if (!wq->sq.sw_sq) {
  193. ret = -ENOMEM;
  194. goto free_rq_qid;
  195. }
  196. wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq,
  197. GFP_KERNEL);
  198. if (!wq->rq.sw_rq) {
  199. ret = -ENOMEM;
  200. goto free_sw_sq;
  201. }
  202. }
  203. /*
  204. * RQT must be a power of 2 and at least 16 deep.
  205. */
  206. wq->rq.rqt_size = roundup_pow_of_two(max_t(u16, wq->rq.size, 16));
  207. wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size);
  208. if (!wq->rq.rqt_hwaddr) {
  209. ret = -ENOMEM;
  210. goto free_sw_rq;
  211. }
  212. ret = alloc_sq(rdev, &wq->sq, user);
  213. if (ret)
  214. goto free_hwaddr;
  215. memset(wq->sq.queue, 0, wq->sq.memsize);
  216. dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr);
  217. wq->rq.queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev),
  218. wq->rq.memsize, &(wq->rq.dma_addr),
  219. GFP_KERNEL);
  220. if (!wq->rq.queue) {
  221. ret = -ENOMEM;
  222. goto free_sq;
  223. }
  224. PDBG("%s sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx\n",
  225. __func__, wq->sq.queue,
  226. (unsigned long long)virt_to_phys(wq->sq.queue),
  227. wq->rq.queue,
  228. (unsigned long long)virt_to_phys(wq->rq.queue));
  229. memset(wq->rq.queue, 0, wq->rq.memsize);
  230. dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr);
  231. wq->db = rdev->lldi.db_reg;
  232. wq->sq.bar2_va = c4iw_bar2_addrs(rdev, wq->sq.qid, T4_BAR2_QTYPE_EGRESS,
  233. &wq->sq.bar2_qid,
  234. user ? &wq->sq.bar2_pa : NULL);
  235. wq->rq.bar2_va = c4iw_bar2_addrs(rdev, wq->rq.qid, T4_BAR2_QTYPE_EGRESS,
  236. &wq->rq.bar2_qid,
  237. user ? &wq->rq.bar2_pa : NULL);
  238. /*
  239. * User mode must have bar2 access.
  240. */
  241. if (user && (!wq->sq.bar2_pa || !wq->rq.bar2_pa)) {
  242. pr_warn(MOD "%s: sqid %u or rqid %u not in BAR2 range.\n",
  243. pci_name(rdev->lldi.pdev), wq->sq.qid, wq->rq.qid);
  244. goto free_dma;
  245. }
  246. wq->rdev = rdev;
  247. wq->rq.msn = 1;
  248. /* build fw_ri_res_wr */
  249. wr_len = sizeof *res_wr + 2 * sizeof *res;
  250. skb = alloc_skb(wr_len, GFP_KERNEL);
  251. if (!skb) {
  252. ret = -ENOMEM;
  253. goto free_dma;
  254. }
  255. set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
  256. res_wr = (struct fw_ri_res_wr *)__skb_put(skb, wr_len);
  257. memset(res_wr, 0, wr_len);
  258. res_wr->op_nres = cpu_to_be32(
  259. FW_WR_OP_V(FW_RI_RES_WR) |
  260. FW_RI_RES_WR_NRES_V(2) |
  261. FW_WR_COMPL_F);
  262. res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
  263. res_wr->cookie = (uintptr_t)&wr_wait;
  264. res = res_wr->res;
  265. res->u.sqrq.restype = FW_RI_RES_TYPE_SQ;
  266. res->u.sqrq.op = FW_RI_RES_OP_WRITE;
  267. /*
  268. * eqsize is the number of 64B entries plus the status page size.
  269. */
  270. eqsize = wq->sq.size * T4_SQ_NUM_SLOTS +
  271. rdev->hw_queue.t4_eq_status_entries;
  272. res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
  273. FW_RI_RES_WR_HOSTFCMODE_V(0) | /* no host cidx updates */
  274. FW_RI_RES_WR_CPRIO_V(0) | /* don't keep in chip cache */
  275. FW_RI_RES_WR_PCIECHN_V(0) | /* set by uP at ri_init time */
  276. (t4_sq_onchip(&wq->sq) ? FW_RI_RES_WR_ONCHIP_F : 0) |
  277. FW_RI_RES_WR_IQID_V(scq->cqid));
  278. res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
  279. FW_RI_RES_WR_DCAEN_V(0) |
  280. FW_RI_RES_WR_DCACPU_V(0) |
  281. FW_RI_RES_WR_FBMIN_V(2) |
  282. (t4_sq_onchip(&wq->sq) ? FW_RI_RES_WR_FBMAX_V(2) :
  283. FW_RI_RES_WR_FBMAX_V(3)) |
  284. FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
  285. FW_RI_RES_WR_CIDXFTHRESH_V(0) |
  286. FW_RI_RES_WR_EQSIZE_V(eqsize));
  287. res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid);
  288. res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr);
  289. res++;
  290. res->u.sqrq.restype = FW_RI_RES_TYPE_RQ;
  291. res->u.sqrq.op = FW_RI_RES_OP_WRITE;
  292. /*
  293. * eqsize is the number of 64B entries plus the status page size.
  294. */
  295. eqsize = wq->rq.size * T4_RQ_NUM_SLOTS +
  296. rdev->hw_queue.t4_eq_status_entries;
  297. res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
  298. FW_RI_RES_WR_HOSTFCMODE_V(0) | /* no host cidx updates */
  299. FW_RI_RES_WR_CPRIO_V(0) | /* don't keep in chip cache */
  300. FW_RI_RES_WR_PCIECHN_V(0) | /* set by uP at ri_init time */
  301. FW_RI_RES_WR_IQID_V(rcq->cqid));
  302. res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
  303. FW_RI_RES_WR_DCAEN_V(0) |
  304. FW_RI_RES_WR_DCACPU_V(0) |
  305. FW_RI_RES_WR_FBMIN_V(2) |
  306. FW_RI_RES_WR_FBMAX_V(3) |
  307. FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
  308. FW_RI_RES_WR_CIDXFTHRESH_V(0) |
  309. FW_RI_RES_WR_EQSIZE_V(eqsize));
  310. res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid);
  311. res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr);
  312. c4iw_init_wr_wait(&wr_wait);
  313. ret = c4iw_ofld_send(rdev, skb);
  314. if (ret)
  315. goto free_dma;
  316. ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, __func__);
  317. if (ret)
  318. goto free_dma;
  319. PDBG("%s sqid 0x%x rqid 0x%x kdb 0x%p sq_bar2_addr %p rq_bar2_addr %p\n",
  320. __func__, wq->sq.qid, wq->rq.qid, wq->db,
  321. wq->sq.bar2_va, wq->rq.bar2_va);
  322. return 0;
  323. free_dma:
  324. dma_free_coherent(&(rdev->lldi.pdev->dev),
  325. wq->rq.memsize, wq->rq.queue,
  326. dma_unmap_addr(&wq->rq, mapping));
  327. free_sq:
  328. dealloc_sq(rdev, &wq->sq);
  329. free_hwaddr:
  330. c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
  331. free_sw_rq:
  332. kfree(wq->rq.sw_rq);
  333. free_sw_sq:
  334. kfree(wq->sq.sw_sq);
  335. free_rq_qid:
  336. c4iw_put_qpid(rdev, wq->rq.qid, uctx);
  337. free_sq_qid:
  338. c4iw_put_qpid(rdev, wq->sq.qid, uctx);
  339. return ret;
  340. }
  341. static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp,
  342. struct ib_send_wr *wr, int max, u32 *plenp)
  343. {
  344. u8 *dstp, *srcp;
  345. u32 plen = 0;
  346. int i;
  347. int rem, len;
  348. dstp = (u8 *)immdp->data;
  349. for (i = 0; i < wr->num_sge; i++) {
  350. if ((plen + wr->sg_list[i].length) > max)
  351. return -EMSGSIZE;
  352. srcp = (u8 *)(unsigned long)wr->sg_list[i].addr;
  353. plen += wr->sg_list[i].length;
  354. rem = wr->sg_list[i].length;
  355. while (rem) {
  356. if (dstp == (u8 *)&sq->queue[sq->size])
  357. dstp = (u8 *)sq->queue;
  358. if (rem <= (u8 *)&sq->queue[sq->size] - dstp)
  359. len = rem;
  360. else
  361. len = (u8 *)&sq->queue[sq->size] - dstp;
  362. memcpy(dstp, srcp, len);
  363. dstp += len;
  364. srcp += len;
  365. rem -= len;
  366. }
  367. }
  368. len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp);
  369. if (len)
  370. memset(dstp, 0, len);
  371. immdp->op = FW_RI_DATA_IMMD;
  372. immdp->r1 = 0;
  373. immdp->r2 = 0;
  374. immdp->immdlen = cpu_to_be32(plen);
  375. *plenp = plen;
  376. return 0;
  377. }
  378. static int build_isgl(__be64 *queue_start, __be64 *queue_end,
  379. struct fw_ri_isgl *isglp, struct ib_sge *sg_list,
  380. int num_sge, u32 *plenp)
  381. {
  382. int i;
  383. u32 plen = 0;
  384. __be64 *flitp = (__be64 *)isglp->sge;
  385. for (i = 0; i < num_sge; i++) {
  386. if ((plen + sg_list[i].length) < plen)
  387. return -EMSGSIZE;
  388. plen += sg_list[i].length;
  389. *flitp = cpu_to_be64(((u64)sg_list[i].lkey << 32) |
  390. sg_list[i].length);
  391. if (++flitp == queue_end)
  392. flitp = queue_start;
  393. *flitp = cpu_to_be64(sg_list[i].addr);
  394. if (++flitp == queue_end)
  395. flitp = queue_start;
  396. }
  397. *flitp = (__force __be64)0;
  398. isglp->op = FW_RI_DATA_ISGL;
  399. isglp->r1 = 0;
  400. isglp->nsge = cpu_to_be16(num_sge);
  401. isglp->r2 = 0;
  402. if (plenp)
  403. *plenp = plen;
  404. return 0;
  405. }
  406. static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
  407. struct ib_send_wr *wr, u8 *len16)
  408. {
  409. u32 plen;
  410. int size;
  411. int ret;
  412. if (wr->num_sge > T4_MAX_SEND_SGE)
  413. return -EINVAL;
  414. switch (wr->opcode) {
  415. case IB_WR_SEND:
  416. if (wr->send_flags & IB_SEND_SOLICITED)
  417. wqe->send.sendop_pkd = cpu_to_be32(
  418. FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE));
  419. else
  420. wqe->send.sendop_pkd = cpu_to_be32(
  421. FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND));
  422. wqe->send.stag_inv = 0;
  423. break;
  424. case IB_WR_SEND_WITH_INV:
  425. if (wr->send_flags & IB_SEND_SOLICITED)
  426. wqe->send.sendop_pkd = cpu_to_be32(
  427. FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE_INV));
  428. else
  429. wqe->send.sendop_pkd = cpu_to_be32(
  430. FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_INV));
  431. wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
  432. break;
  433. default:
  434. return -EINVAL;
  435. }
  436. wqe->send.r3 = 0;
  437. wqe->send.r4 = 0;
  438. plen = 0;
  439. if (wr->num_sge) {
  440. if (wr->send_flags & IB_SEND_INLINE) {
  441. ret = build_immd(sq, wqe->send.u.immd_src, wr,
  442. T4_MAX_SEND_INLINE, &plen);
  443. if (ret)
  444. return ret;
  445. size = sizeof wqe->send + sizeof(struct fw_ri_immd) +
  446. plen;
  447. } else {
  448. ret = build_isgl((__be64 *)sq->queue,
  449. (__be64 *)&sq->queue[sq->size],
  450. wqe->send.u.isgl_src,
  451. wr->sg_list, wr->num_sge, &plen);
  452. if (ret)
  453. return ret;
  454. size = sizeof wqe->send + sizeof(struct fw_ri_isgl) +
  455. wr->num_sge * sizeof(struct fw_ri_sge);
  456. }
  457. } else {
  458. wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD;
  459. wqe->send.u.immd_src[0].r1 = 0;
  460. wqe->send.u.immd_src[0].r2 = 0;
  461. wqe->send.u.immd_src[0].immdlen = 0;
  462. size = sizeof wqe->send + sizeof(struct fw_ri_immd);
  463. plen = 0;
  464. }
  465. *len16 = DIV_ROUND_UP(size, 16);
  466. wqe->send.plen = cpu_to_be32(plen);
  467. return 0;
  468. }
  469. static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
  470. struct ib_send_wr *wr, u8 *len16)
  471. {
  472. u32 plen;
  473. int size;
  474. int ret;
  475. if (wr->num_sge > T4_MAX_SEND_SGE)
  476. return -EINVAL;
  477. wqe->write.r2 = 0;
  478. wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey);
  479. wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr);
  480. if (wr->num_sge) {
  481. if (wr->send_flags & IB_SEND_INLINE) {
  482. ret = build_immd(sq, wqe->write.u.immd_src, wr,
  483. T4_MAX_WRITE_INLINE, &plen);
  484. if (ret)
  485. return ret;
  486. size = sizeof wqe->write + sizeof(struct fw_ri_immd) +
  487. plen;
  488. } else {
  489. ret = build_isgl((__be64 *)sq->queue,
  490. (__be64 *)&sq->queue[sq->size],
  491. wqe->write.u.isgl_src,
  492. wr->sg_list, wr->num_sge, &plen);
  493. if (ret)
  494. return ret;
  495. size = sizeof wqe->write + sizeof(struct fw_ri_isgl) +
  496. wr->num_sge * sizeof(struct fw_ri_sge);
  497. }
  498. } else {
  499. wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD;
  500. wqe->write.u.immd_src[0].r1 = 0;
  501. wqe->write.u.immd_src[0].r2 = 0;
  502. wqe->write.u.immd_src[0].immdlen = 0;
  503. size = sizeof wqe->write + sizeof(struct fw_ri_immd);
  504. plen = 0;
  505. }
  506. *len16 = DIV_ROUND_UP(size, 16);
  507. wqe->write.plen = cpu_to_be32(plen);
  508. return 0;
  509. }
  510. static int build_rdma_read(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16)
  511. {
  512. if (wr->num_sge > 1)
  513. return -EINVAL;
  514. if (wr->num_sge) {
  515. wqe->read.stag_src = cpu_to_be32(rdma_wr(wr)->rkey);
  516. wqe->read.to_src_hi = cpu_to_be32((u32)(rdma_wr(wr)->remote_addr
  517. >> 32));
  518. wqe->read.to_src_lo = cpu_to_be32((u32)rdma_wr(wr)->remote_addr);
  519. wqe->read.stag_sink = cpu_to_be32(wr->sg_list[0].lkey);
  520. wqe->read.plen = cpu_to_be32(wr->sg_list[0].length);
  521. wqe->read.to_sink_hi = cpu_to_be32((u32)(wr->sg_list[0].addr
  522. >> 32));
  523. wqe->read.to_sink_lo = cpu_to_be32((u32)(wr->sg_list[0].addr));
  524. } else {
  525. wqe->read.stag_src = cpu_to_be32(2);
  526. wqe->read.to_src_hi = 0;
  527. wqe->read.to_src_lo = 0;
  528. wqe->read.stag_sink = cpu_to_be32(2);
  529. wqe->read.plen = 0;
  530. wqe->read.to_sink_hi = 0;
  531. wqe->read.to_sink_lo = 0;
  532. }
  533. wqe->read.r2 = 0;
  534. wqe->read.r5 = 0;
  535. *len16 = DIV_ROUND_UP(sizeof wqe->read, 16);
  536. return 0;
  537. }
  538. static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,
  539. struct ib_recv_wr *wr, u8 *len16)
  540. {
  541. int ret;
  542. ret = build_isgl((__be64 *)qhp->wq.rq.queue,
  543. (__be64 *)&qhp->wq.rq.queue[qhp->wq.rq.size],
  544. &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL);
  545. if (ret)
  546. return ret;
  547. *len16 = DIV_ROUND_UP(sizeof wqe->recv +
  548. wr->num_sge * sizeof(struct fw_ri_sge), 16);
  549. return 0;
  550. }
  551. static void build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr *fr,
  552. struct ib_reg_wr *wr, struct c4iw_mr *mhp,
  553. u8 *len16)
  554. {
  555. __be64 *p = (__be64 *)fr->pbl;
  556. fr->r2 = cpu_to_be32(0);
  557. fr->stag = cpu_to_be32(mhp->ibmr.rkey);
  558. fr->tpte.valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F |
  559. FW_RI_TPTE_STAGKEY_V((mhp->ibmr.rkey & FW_RI_TPTE_STAGKEY_M)) |
  560. FW_RI_TPTE_STAGSTATE_V(1) |
  561. FW_RI_TPTE_STAGTYPE_V(FW_RI_STAG_NSMR) |
  562. FW_RI_TPTE_PDID_V(mhp->attr.pdid));
  563. fr->tpte.locread_to_qpid = cpu_to_be32(
  564. FW_RI_TPTE_PERM_V(c4iw_ib_to_tpt_access(wr->access)) |
  565. FW_RI_TPTE_ADDRTYPE_V(FW_RI_VA_BASED_TO) |
  566. FW_RI_TPTE_PS_V(ilog2(wr->mr->page_size) - 12));
  567. fr->tpte.nosnoop_pbladdr = cpu_to_be32(FW_RI_TPTE_PBLADDR_V(
  568. PBL_OFF(&mhp->rhp->rdev, mhp->attr.pbl_addr)>>3));
  569. fr->tpte.dca_mwbcnt_pstag = cpu_to_be32(0);
  570. fr->tpte.len_hi = cpu_to_be32(0);
  571. fr->tpte.len_lo = cpu_to_be32(mhp->ibmr.length);
  572. fr->tpte.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
  573. fr->tpte.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 0xffffffff);
  574. p[0] = cpu_to_be64((u64)mhp->mpl[0]);
  575. p[1] = cpu_to_be64((u64)mhp->mpl[1]);
  576. *len16 = DIV_ROUND_UP(sizeof(*fr), 16);
  577. }
  578. static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
  579. struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16,
  580. bool dsgl_supported)
  581. {
  582. struct fw_ri_immd *imdp;
  583. __be64 *p;
  584. int i;
  585. int pbllen = roundup(mhp->mpl_len * sizeof(u64), 32);
  586. int rem;
  587. if (mhp->mpl_len > t4_max_fr_depth(dsgl_supported && use_dsgl))
  588. return -EINVAL;
  589. wqe->fr.qpbinde_to_dcacpu = 0;
  590. wqe->fr.pgsz_shift = ilog2(wr->mr->page_size) - 12;
  591. wqe->fr.addr_type = FW_RI_VA_BASED_TO;
  592. wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->access);
  593. wqe->fr.len_hi = 0;
  594. wqe->fr.len_lo = cpu_to_be32(mhp->ibmr.length);
  595. wqe->fr.stag = cpu_to_be32(wr->key);
  596. wqe->fr.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
  597. wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova &
  598. 0xffffffff);
  599. if (dsgl_supported && use_dsgl && (pbllen > max_fr_immd)) {
  600. struct fw_ri_dsgl *sglp;
  601. for (i = 0; i < mhp->mpl_len; i++)
  602. mhp->mpl[i] = (__force u64)cpu_to_be64((u64)mhp->mpl[i]);
  603. sglp = (struct fw_ri_dsgl *)(&wqe->fr + 1);
  604. sglp->op = FW_RI_DATA_DSGL;
  605. sglp->r1 = 0;
  606. sglp->nsge = cpu_to_be16(1);
  607. sglp->addr0 = cpu_to_be64(mhp->mpl_addr);
  608. sglp->len0 = cpu_to_be32(pbllen);
  609. *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*sglp), 16);
  610. } else {
  611. imdp = (struct fw_ri_immd *)(&wqe->fr + 1);
  612. imdp->op = FW_RI_DATA_IMMD;
  613. imdp->r1 = 0;
  614. imdp->r2 = 0;
  615. imdp->immdlen = cpu_to_be32(pbllen);
  616. p = (__be64 *)(imdp + 1);
  617. rem = pbllen;
  618. for (i = 0; i < mhp->mpl_len; i++) {
  619. *p = cpu_to_be64((u64)mhp->mpl[i]);
  620. rem -= sizeof(*p);
  621. if (++p == (__be64 *)&sq->queue[sq->size])
  622. p = (__be64 *)sq->queue;
  623. }
  624. BUG_ON(rem < 0);
  625. while (rem) {
  626. *p = 0;
  627. rem -= sizeof(*p);
  628. if (++p == (__be64 *)&sq->queue[sq->size])
  629. p = (__be64 *)sq->queue;
  630. }
  631. *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*imdp)
  632. + pbllen, 16);
  633. }
  634. return 0;
  635. }
  636. static int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16)
  637. {
  638. wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
  639. wqe->inv.r2 = 0;
  640. *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16);
  641. return 0;
  642. }
  643. static void free_qp_work(struct work_struct *work)
  644. {
  645. struct c4iw_ucontext *ucontext;
  646. struct c4iw_qp *qhp;
  647. struct c4iw_dev *rhp;
  648. qhp = container_of(work, struct c4iw_qp, free_work);
  649. ucontext = qhp->ucontext;
  650. rhp = qhp->rhp;
  651. PDBG("%s qhp %p ucontext %p\n", __func__, qhp, ucontext);
  652. destroy_qp(&rhp->rdev, &qhp->wq,
  653. ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
  654. if (ucontext)
  655. c4iw_put_ucontext(ucontext);
  656. kfree(qhp);
  657. }
  658. static void queue_qp_free(struct kref *kref)
  659. {
  660. struct c4iw_qp *qhp;
  661. qhp = container_of(kref, struct c4iw_qp, kref);
  662. PDBG("%s qhp %p\n", __func__, qhp);
  663. queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work);
  664. }
  665. void c4iw_qp_add_ref(struct ib_qp *qp)
  666. {
  667. PDBG("%s ib_qp %p\n", __func__, qp);
  668. kref_get(&to_c4iw_qp(qp)->kref);
  669. }
  670. void c4iw_qp_rem_ref(struct ib_qp *qp)
  671. {
  672. PDBG("%s ib_qp %p\n", __func__, qp);
  673. kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free);
  674. }
  675. static void add_to_fc_list(struct list_head *head, struct list_head *entry)
  676. {
  677. if (list_empty(entry))
  678. list_add_tail(entry, head);
  679. }
  680. static int ring_kernel_sq_db(struct c4iw_qp *qhp, u16 inc)
  681. {
  682. unsigned long flags;
  683. spin_lock_irqsave(&qhp->rhp->lock, flags);
  684. spin_lock(&qhp->lock);
  685. if (qhp->rhp->db_state == NORMAL)
  686. t4_ring_sq_db(&qhp->wq, inc, NULL);
  687. else {
  688. add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
  689. qhp->wq.sq.wq_pidx_inc += inc;
  690. }
  691. spin_unlock(&qhp->lock);
  692. spin_unlock_irqrestore(&qhp->rhp->lock, flags);
  693. return 0;
  694. }
  695. static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc)
  696. {
  697. unsigned long flags;
  698. spin_lock_irqsave(&qhp->rhp->lock, flags);
  699. spin_lock(&qhp->lock);
  700. if (qhp->rhp->db_state == NORMAL)
  701. t4_ring_rq_db(&qhp->wq, inc, NULL);
  702. else {
  703. add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
  704. qhp->wq.rq.wq_pidx_inc += inc;
  705. }
  706. spin_unlock(&qhp->lock);
  707. spin_unlock_irqrestore(&qhp->rhp->lock, flags);
  708. return 0;
  709. }
  710. int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
  711. struct ib_send_wr **bad_wr)
  712. {
  713. int err = 0;
  714. u8 len16 = 0;
  715. enum fw_wr_opcodes fw_opcode = 0;
  716. enum fw_ri_wr_flags fw_flags;
  717. struct c4iw_qp *qhp;
  718. union t4_wr *wqe = NULL;
  719. u32 num_wrs;
  720. struct t4_swsqe *swsqe;
  721. unsigned long flag;
  722. u16 idx = 0;
  723. qhp = to_c4iw_qp(ibqp);
  724. spin_lock_irqsave(&qhp->lock, flag);
  725. if (t4_wq_in_error(&qhp->wq)) {
  726. spin_unlock_irqrestore(&qhp->lock, flag);
  727. *bad_wr = wr;
  728. return -EINVAL;
  729. }
  730. num_wrs = t4_sq_avail(&qhp->wq);
  731. if (num_wrs == 0) {
  732. spin_unlock_irqrestore(&qhp->lock, flag);
  733. *bad_wr = wr;
  734. return -ENOMEM;
  735. }
  736. while (wr) {
  737. if (num_wrs == 0) {
  738. err = -ENOMEM;
  739. *bad_wr = wr;
  740. break;
  741. }
  742. wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue +
  743. qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE);
  744. fw_flags = 0;
  745. if (wr->send_flags & IB_SEND_SOLICITED)
  746. fw_flags |= FW_RI_SOLICITED_EVENT_FLAG;
  747. if (wr->send_flags & IB_SEND_SIGNALED || qhp->sq_sig_all)
  748. fw_flags |= FW_RI_COMPLETION_FLAG;
  749. swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx];
  750. switch (wr->opcode) {
  751. case IB_WR_SEND_WITH_INV:
  752. case IB_WR_SEND:
  753. if (wr->send_flags & IB_SEND_FENCE)
  754. fw_flags |= FW_RI_READ_FENCE_FLAG;
  755. fw_opcode = FW_RI_SEND_WR;
  756. if (wr->opcode == IB_WR_SEND)
  757. swsqe->opcode = FW_RI_SEND;
  758. else
  759. swsqe->opcode = FW_RI_SEND_WITH_INV;
  760. err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16);
  761. break;
  762. case IB_WR_RDMA_WRITE:
  763. fw_opcode = FW_RI_RDMA_WRITE_WR;
  764. swsqe->opcode = FW_RI_RDMA_WRITE;
  765. err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16);
  766. break;
  767. case IB_WR_RDMA_READ:
  768. case IB_WR_RDMA_READ_WITH_INV:
  769. fw_opcode = FW_RI_RDMA_READ_WR;
  770. swsqe->opcode = FW_RI_READ_REQ;
  771. if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) {
  772. c4iw_invalidate_mr(qhp->rhp,
  773. wr->sg_list[0].lkey);
  774. fw_flags = FW_RI_RDMA_READ_INVALIDATE;
  775. } else {
  776. fw_flags = 0;
  777. }
  778. err = build_rdma_read(wqe, wr, &len16);
  779. if (err)
  780. break;
  781. swsqe->read_len = wr->sg_list[0].length;
  782. if (!qhp->wq.sq.oldest_read)
  783. qhp->wq.sq.oldest_read = swsqe;
  784. break;
  785. case IB_WR_REG_MR: {
  786. struct c4iw_mr *mhp = to_c4iw_mr(reg_wr(wr)->mr);
  787. swsqe->opcode = FW_RI_FAST_REGISTER;
  788. if (qhp->rhp->rdev.lldi.fr_nsmr_tpte_wr_support &&
  789. !mhp->attr.state && mhp->mpl_len <= 2) {
  790. fw_opcode = FW_RI_FR_NSMR_TPTE_WR;
  791. build_tpte_memreg(&wqe->fr_tpte, reg_wr(wr),
  792. mhp, &len16);
  793. } else {
  794. fw_opcode = FW_RI_FR_NSMR_WR;
  795. err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr),
  796. mhp, &len16,
  797. qhp->rhp->rdev.lldi.ulptx_memwrite_dsgl);
  798. if (err)
  799. break;
  800. }
  801. mhp->attr.state = 1;
  802. break;
  803. }
  804. case IB_WR_LOCAL_INV:
  805. if (wr->send_flags & IB_SEND_FENCE)
  806. fw_flags |= FW_RI_LOCAL_FENCE_FLAG;
  807. fw_opcode = FW_RI_INV_LSTAG_WR;
  808. swsqe->opcode = FW_RI_LOCAL_INV;
  809. err = build_inv_stag(wqe, wr, &len16);
  810. c4iw_invalidate_mr(qhp->rhp, wr->ex.invalidate_rkey);
  811. break;
  812. default:
  813. PDBG("%s post of type=%d TBD!\n", __func__,
  814. wr->opcode);
  815. err = -EINVAL;
  816. }
  817. if (err) {
  818. *bad_wr = wr;
  819. break;
  820. }
  821. swsqe->idx = qhp->wq.sq.pidx;
  822. swsqe->complete = 0;
  823. swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED) ||
  824. qhp->sq_sig_all;
  825. swsqe->flushed = 0;
  826. swsqe->wr_id = wr->wr_id;
  827. if (c4iw_wr_log) {
  828. swsqe->sge_ts = cxgb4_read_sge_timestamp(
  829. qhp->rhp->rdev.lldi.ports[0]);
  830. getnstimeofday(&swsqe->host_ts);
  831. }
  832. init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16);
  833. PDBG("%s cookie 0x%llx pidx 0x%x opcode 0x%x read_len %u\n",
  834. __func__, (unsigned long long)wr->wr_id, qhp->wq.sq.pidx,
  835. swsqe->opcode, swsqe->read_len);
  836. wr = wr->next;
  837. num_wrs--;
  838. t4_sq_produce(&qhp->wq, len16);
  839. idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
  840. }
  841. if (!qhp->rhp->rdev.status_page->db_off) {
  842. t4_ring_sq_db(&qhp->wq, idx, wqe);
  843. spin_unlock_irqrestore(&qhp->lock, flag);
  844. } else {
  845. spin_unlock_irqrestore(&qhp->lock, flag);
  846. ring_kernel_sq_db(qhp, idx);
  847. }
  848. return err;
  849. }
  850. int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
  851. struct ib_recv_wr **bad_wr)
  852. {
  853. int err = 0;
  854. struct c4iw_qp *qhp;
  855. union t4_recv_wr *wqe = NULL;
  856. u32 num_wrs;
  857. u8 len16 = 0;
  858. unsigned long flag;
  859. u16 idx = 0;
  860. qhp = to_c4iw_qp(ibqp);
  861. spin_lock_irqsave(&qhp->lock, flag);
  862. if (t4_wq_in_error(&qhp->wq)) {
  863. spin_unlock_irqrestore(&qhp->lock, flag);
  864. *bad_wr = wr;
  865. return -EINVAL;
  866. }
  867. num_wrs = t4_rq_avail(&qhp->wq);
  868. if (num_wrs == 0) {
  869. spin_unlock_irqrestore(&qhp->lock, flag);
  870. *bad_wr = wr;
  871. return -ENOMEM;
  872. }
  873. while (wr) {
  874. if (wr->num_sge > T4_MAX_RECV_SGE) {
  875. err = -EINVAL;
  876. *bad_wr = wr;
  877. break;
  878. }
  879. wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue +
  880. qhp->wq.rq.wq_pidx *
  881. T4_EQ_ENTRY_SIZE);
  882. if (num_wrs)
  883. err = build_rdma_recv(qhp, wqe, wr, &len16);
  884. else
  885. err = -ENOMEM;
  886. if (err) {
  887. *bad_wr = wr;
  888. break;
  889. }
  890. qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].wr_id = wr->wr_id;
  891. if (c4iw_wr_log) {
  892. qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].sge_ts =
  893. cxgb4_read_sge_timestamp(
  894. qhp->rhp->rdev.lldi.ports[0]);
  895. getnstimeofday(
  896. &qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].host_ts);
  897. }
  898. wqe->recv.opcode = FW_RI_RECV_WR;
  899. wqe->recv.r1 = 0;
  900. wqe->recv.wrid = qhp->wq.rq.pidx;
  901. wqe->recv.r2[0] = 0;
  902. wqe->recv.r2[1] = 0;
  903. wqe->recv.r2[2] = 0;
  904. wqe->recv.len16 = len16;
  905. PDBG("%s cookie 0x%llx pidx %u\n", __func__,
  906. (unsigned long long) wr->wr_id, qhp->wq.rq.pidx);
  907. t4_rq_produce(&qhp->wq, len16);
  908. idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
  909. wr = wr->next;
  910. num_wrs--;
  911. }
  912. if (!qhp->rhp->rdev.status_page->db_off) {
  913. t4_ring_rq_db(&qhp->wq, idx, wqe);
  914. spin_unlock_irqrestore(&qhp->lock, flag);
  915. } else {
  916. spin_unlock_irqrestore(&qhp->lock, flag);
  917. ring_kernel_rq_db(qhp, idx);
  918. }
  919. return err;
  920. }
  921. static inline void build_term_codes(struct t4_cqe *err_cqe, u8 *layer_type,
  922. u8 *ecode)
  923. {
  924. int status;
  925. int tagged;
  926. int opcode;
  927. int rqtype;
  928. int send_inv;
  929. if (!err_cqe) {
  930. *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;
  931. *ecode = 0;
  932. return;
  933. }
  934. status = CQE_STATUS(err_cqe);
  935. opcode = CQE_OPCODE(err_cqe);
  936. rqtype = RQ_TYPE(err_cqe);
  937. send_inv = (opcode == FW_RI_SEND_WITH_INV) ||
  938. (opcode == FW_RI_SEND_WITH_SE_INV);
  939. tagged = (opcode == FW_RI_RDMA_WRITE) ||
  940. (rqtype && (opcode == FW_RI_READ_RESP));
  941. switch (status) {
  942. case T4_ERR_STAG:
  943. if (send_inv) {
  944. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
  945. *ecode = RDMAP_CANT_INV_STAG;
  946. } else {
  947. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
  948. *ecode = RDMAP_INV_STAG;
  949. }
  950. break;
  951. case T4_ERR_PDID:
  952. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
  953. if ((opcode == FW_RI_SEND_WITH_INV) ||
  954. (opcode == FW_RI_SEND_WITH_SE_INV))
  955. *ecode = RDMAP_CANT_INV_STAG;
  956. else
  957. *ecode = RDMAP_STAG_NOT_ASSOC;
  958. break;
  959. case T4_ERR_QPID:
  960. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
  961. *ecode = RDMAP_STAG_NOT_ASSOC;
  962. break;
  963. case T4_ERR_ACCESS:
  964. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
  965. *ecode = RDMAP_ACC_VIOL;
  966. break;
  967. case T4_ERR_WRAP:
  968. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
  969. *ecode = RDMAP_TO_WRAP;
  970. break;
  971. case T4_ERR_BOUND:
  972. if (tagged) {
  973. *layer_type = LAYER_DDP|DDP_TAGGED_ERR;
  974. *ecode = DDPT_BASE_BOUNDS;
  975. } else {
  976. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
  977. *ecode = RDMAP_BASE_BOUNDS;
  978. }
  979. break;
  980. case T4_ERR_INVALIDATE_SHARED_MR:
  981. case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
  982. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
  983. *ecode = RDMAP_CANT_INV_STAG;
  984. break;
  985. case T4_ERR_ECC:
  986. case T4_ERR_ECC_PSTAG:
  987. case T4_ERR_INTERNAL_ERR:
  988. *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA;
  989. *ecode = 0;
  990. break;
  991. case T4_ERR_OUT_OF_RQE:
  992. *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
  993. *ecode = DDPU_INV_MSN_NOBUF;
  994. break;
  995. case T4_ERR_PBL_ADDR_BOUND:
  996. *layer_type = LAYER_DDP|DDP_TAGGED_ERR;
  997. *ecode = DDPT_BASE_BOUNDS;
  998. break;
  999. case T4_ERR_CRC:
  1000. *layer_type = LAYER_MPA|DDP_LLP;
  1001. *ecode = MPA_CRC_ERR;
  1002. break;
  1003. case T4_ERR_MARKER:
  1004. *layer_type = LAYER_MPA|DDP_LLP;
  1005. *ecode = MPA_MARKER_ERR;
  1006. break;
  1007. case T4_ERR_PDU_LEN_ERR:
  1008. *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
  1009. *ecode = DDPU_MSG_TOOBIG;
  1010. break;
  1011. case T4_ERR_DDP_VERSION:
  1012. if (tagged) {
  1013. *layer_type = LAYER_DDP|DDP_TAGGED_ERR;
  1014. *ecode = DDPT_INV_VERS;
  1015. } else {
  1016. *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
  1017. *ecode = DDPU_INV_VERS;
  1018. }
  1019. break;
  1020. case T4_ERR_RDMA_VERSION:
  1021. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
  1022. *ecode = RDMAP_INV_VERS;
  1023. break;
  1024. case T4_ERR_OPCODE:
  1025. *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
  1026. *ecode = RDMAP_INV_OPCODE;
  1027. break;
  1028. case T4_ERR_DDP_QUEUE_NUM:
  1029. *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
  1030. *ecode = DDPU_INV_QN;
  1031. break;
  1032. case T4_ERR_MSN:
  1033. case T4_ERR_MSN_GAP:
  1034. case T4_ERR_MSN_RANGE:
  1035. case T4_ERR_IRD_OVERFLOW:
  1036. *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
  1037. *ecode = DDPU_INV_MSN_RANGE;
  1038. break;
  1039. case T4_ERR_TBIT:
  1040. *layer_type = LAYER_DDP|DDP_LOCAL_CATA;
  1041. *ecode = 0;
  1042. break;
  1043. case T4_ERR_MO:
  1044. *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
  1045. *ecode = DDPU_INV_MO;
  1046. break;
  1047. default:
  1048. *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;
  1049. *ecode = 0;
  1050. break;
  1051. }
  1052. }
  1053. static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,
  1054. gfp_t gfp)
  1055. {
  1056. struct fw_ri_wr *wqe;
  1057. struct sk_buff *skb;
  1058. struct terminate_message *term;
  1059. PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid,
  1060. qhp->ep->hwtid);
  1061. skb = skb_dequeue(&qhp->ep->com.ep_skb_list);
  1062. if (WARN_ON(!skb))
  1063. return;
  1064. set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);
  1065. wqe = (struct fw_ri_wr *)__skb_put(skb, sizeof(*wqe));
  1066. memset(wqe, 0, sizeof *wqe);
  1067. wqe->op_compl = cpu_to_be32(FW_WR_OP_V(FW_RI_INIT_WR));
  1068. wqe->flowid_len16 = cpu_to_be32(
  1069. FW_WR_FLOWID_V(qhp->ep->hwtid) |
  1070. FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
  1071. wqe->u.terminate.type = FW_RI_TYPE_TERMINATE;
  1072. wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term);
  1073. term = (struct terminate_message *)wqe->u.terminate.termmsg;
  1074. if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) {
  1075. term->layer_etype = qhp->attr.layer_etype;
  1076. term->ecode = qhp->attr.ecode;
  1077. } else
  1078. build_term_codes(err_cqe, &term->layer_etype, &term->ecode);
  1079. c4iw_ofld_send(&qhp->rhp->rdev, skb);
  1080. }
  1081. /*
  1082. * Assumes qhp lock is held.
  1083. */
  1084. static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp,
  1085. struct c4iw_cq *schp)
  1086. {
  1087. int count;
  1088. int rq_flushed, sq_flushed;
  1089. unsigned long flag;
  1090. PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp);
  1091. /* locking hierarchy: cq lock first, then qp lock. */
  1092. spin_lock_irqsave(&rchp->lock, flag);
  1093. spin_lock(&qhp->lock);
  1094. if (qhp->wq.flushed) {
  1095. spin_unlock(&qhp->lock);
  1096. spin_unlock_irqrestore(&rchp->lock, flag);
  1097. return;
  1098. }
  1099. qhp->wq.flushed = 1;
  1100. c4iw_flush_hw_cq(rchp);
  1101. c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count);
  1102. rq_flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count);
  1103. spin_unlock(&qhp->lock);
  1104. spin_unlock_irqrestore(&rchp->lock, flag);
  1105. /* locking hierarchy: cq lock first, then qp lock. */
  1106. spin_lock_irqsave(&schp->lock, flag);
  1107. spin_lock(&qhp->lock);
  1108. if (schp != rchp)
  1109. c4iw_flush_hw_cq(schp);
  1110. sq_flushed = c4iw_flush_sq(qhp);
  1111. spin_unlock(&qhp->lock);
  1112. spin_unlock_irqrestore(&schp->lock, flag);
  1113. if (schp == rchp) {
  1114. if (t4_clear_cq_armed(&rchp->cq) &&
  1115. (rq_flushed || sq_flushed)) {
  1116. spin_lock_irqsave(&rchp->comp_handler_lock, flag);
  1117. (*rchp->ibcq.comp_handler)(&rchp->ibcq,
  1118. rchp->ibcq.cq_context);
  1119. spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
  1120. }
  1121. } else {
  1122. if (t4_clear_cq_armed(&rchp->cq) && rq_flushed) {
  1123. spin_lock_irqsave(&rchp->comp_handler_lock, flag);
  1124. (*rchp->ibcq.comp_handler)(&rchp->ibcq,
  1125. rchp->ibcq.cq_context);
  1126. spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
  1127. }
  1128. if (t4_clear_cq_armed(&schp->cq) && sq_flushed) {
  1129. spin_lock_irqsave(&schp->comp_handler_lock, flag);
  1130. (*schp->ibcq.comp_handler)(&schp->ibcq,
  1131. schp->ibcq.cq_context);
  1132. spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
  1133. }
  1134. }
  1135. }
  1136. static void flush_qp(struct c4iw_qp *qhp)
  1137. {
  1138. struct c4iw_cq *rchp, *schp;
  1139. unsigned long flag;
  1140. rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
  1141. schp = to_c4iw_cq(qhp->ibqp.send_cq);
  1142. t4_set_wq_in_error(&qhp->wq);
  1143. if (qhp->ibqp.uobject) {
  1144. t4_set_cq_in_error(&rchp->cq);
  1145. spin_lock_irqsave(&rchp->comp_handler_lock, flag);
  1146. (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context);
  1147. spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
  1148. if (schp != rchp) {
  1149. t4_set_cq_in_error(&schp->cq);
  1150. spin_lock_irqsave(&schp->comp_handler_lock, flag);
  1151. (*schp->ibcq.comp_handler)(&schp->ibcq,
  1152. schp->ibcq.cq_context);
  1153. spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
  1154. }
  1155. return;
  1156. }
  1157. __flush_qp(qhp, rchp, schp);
  1158. }
  1159. static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
  1160. struct c4iw_ep *ep)
  1161. {
  1162. struct fw_ri_wr *wqe;
  1163. int ret;
  1164. struct sk_buff *skb;
  1165. PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid,
  1166. ep->hwtid);
  1167. skb = skb_dequeue(&ep->com.ep_skb_list);
  1168. if (WARN_ON(!skb))
  1169. return -ENOMEM;
  1170. set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx);
  1171. wqe = (struct fw_ri_wr *)__skb_put(skb, sizeof(*wqe));
  1172. memset(wqe, 0, sizeof *wqe);
  1173. wqe->op_compl = cpu_to_be32(
  1174. FW_WR_OP_V(FW_RI_INIT_WR) |
  1175. FW_WR_COMPL_F);
  1176. wqe->flowid_len16 = cpu_to_be32(
  1177. FW_WR_FLOWID_V(ep->hwtid) |
  1178. FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
  1179. wqe->cookie = (uintptr_t)&ep->com.wr_wait;
  1180. wqe->u.fini.type = FW_RI_TYPE_FINI;
  1181. ret = c4iw_ofld_send(&rhp->rdev, skb);
  1182. if (ret)
  1183. goto out;
  1184. ret = c4iw_wait_for_reply(&rhp->rdev, &ep->com.wr_wait, qhp->ep->hwtid,
  1185. qhp->wq.sq.qid, __func__);
  1186. out:
  1187. PDBG("%s ret %d\n", __func__, ret);
  1188. return ret;
  1189. }
  1190. static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)
  1191. {
  1192. PDBG("%s p2p_type = %d\n", __func__, p2p_type);
  1193. memset(&init->u, 0, sizeof init->u);
  1194. switch (p2p_type) {
  1195. case FW_RI_INIT_P2PTYPE_RDMA_WRITE:
  1196. init->u.write.opcode = FW_RI_RDMA_WRITE_WR;
  1197. init->u.write.stag_sink = cpu_to_be32(1);
  1198. init->u.write.to_sink = cpu_to_be64(1);
  1199. init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD;
  1200. init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write +
  1201. sizeof(struct fw_ri_immd),
  1202. 16);
  1203. break;
  1204. case FW_RI_INIT_P2PTYPE_READ_REQ:
  1205. init->u.write.opcode = FW_RI_RDMA_READ_WR;
  1206. init->u.read.stag_src = cpu_to_be32(1);
  1207. init->u.read.to_src_lo = cpu_to_be32(1);
  1208. init->u.read.stag_sink = cpu_to_be32(1);
  1209. init->u.read.to_sink_lo = cpu_to_be32(1);
  1210. init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16);
  1211. break;
  1212. }
  1213. }
  1214. static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
  1215. {
  1216. struct fw_ri_wr *wqe;
  1217. int ret;
  1218. struct sk_buff *skb;
  1219. PDBG("%s qhp %p qid 0x%x tid %u ird %u ord %u\n", __func__, qhp,
  1220. qhp->wq.sq.qid, qhp->ep->hwtid, qhp->ep->ird, qhp->ep->ord);
  1221. skb = alloc_skb(sizeof *wqe, GFP_KERNEL);
  1222. if (!skb) {
  1223. ret = -ENOMEM;
  1224. goto out;
  1225. }
  1226. ret = alloc_ird(rhp, qhp->attr.max_ird);
  1227. if (ret) {
  1228. qhp->attr.max_ird = 0;
  1229. kfree_skb(skb);
  1230. goto out;
  1231. }
  1232. set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx);
  1233. wqe = (struct fw_ri_wr *)__skb_put(skb, sizeof(*wqe));
  1234. memset(wqe, 0, sizeof *wqe);
  1235. wqe->op_compl = cpu_to_be32(
  1236. FW_WR_OP_V(FW_RI_INIT_WR) |
  1237. FW_WR_COMPL_F);
  1238. wqe->flowid_len16 = cpu_to_be32(
  1239. FW_WR_FLOWID_V(qhp->ep->hwtid) |
  1240. FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16)));
  1241. wqe->cookie = (uintptr_t)&qhp->ep->com.wr_wait;
  1242. wqe->u.init.type = FW_RI_TYPE_INIT;
  1243. wqe->u.init.mpareqbit_p2ptype =
  1244. FW_RI_WR_MPAREQBIT_V(qhp->attr.mpa_attr.initiator) |
  1245. FW_RI_WR_P2PTYPE_V(qhp->attr.mpa_attr.p2p_type);
  1246. wqe->u.init.mpa_attrs = FW_RI_MPA_IETF_ENABLE;
  1247. if (qhp->attr.mpa_attr.recv_marker_enabled)
  1248. wqe->u.init.mpa_attrs |= FW_RI_MPA_RX_MARKER_ENABLE;
  1249. if (qhp->attr.mpa_attr.xmit_marker_enabled)
  1250. wqe->u.init.mpa_attrs |= FW_RI_MPA_TX_MARKER_ENABLE;
  1251. if (qhp->attr.mpa_attr.crc_enabled)
  1252. wqe->u.init.mpa_attrs |= FW_RI_MPA_CRC_ENABLE;
  1253. wqe->u.init.qp_caps = FW_RI_QP_RDMA_READ_ENABLE |
  1254. FW_RI_QP_RDMA_WRITE_ENABLE |
  1255. FW_RI_QP_BIND_ENABLE;
  1256. if (!qhp->ibqp.uobject)
  1257. wqe->u.init.qp_caps |= FW_RI_QP_FAST_REGISTER_ENABLE |
  1258. FW_RI_QP_STAG0_ENABLE;
  1259. wqe->u.init.nrqe = cpu_to_be16(t4_rqes_posted(&qhp->wq));
  1260. wqe->u.init.pdid = cpu_to_be32(qhp->attr.pd);
  1261. wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid);
  1262. wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid);
  1263. wqe->u.init.rq_eqid = cpu_to_be32(qhp->wq.rq.qid);
  1264. wqe->u.init.scqid = cpu_to_be32(qhp->attr.scq);
  1265. wqe->u.init.rcqid = cpu_to_be32(qhp->attr.rcq);
  1266. wqe->u.init.ord_max = cpu_to_be32(qhp->attr.max_ord);
  1267. wqe->u.init.ird_max = cpu_to_be32(qhp->attr.max_ird);
  1268. wqe->u.init.iss = cpu_to_be32(qhp->ep->snd_seq);
  1269. wqe->u.init.irs = cpu_to_be32(qhp->ep->rcv_seq);
  1270. wqe->u.init.hwrqsize = cpu_to_be32(qhp->wq.rq.rqt_size);
  1271. wqe->u.init.hwrqaddr = cpu_to_be32(qhp->wq.rq.rqt_hwaddr -
  1272. rhp->rdev.lldi.vr->rq.start);
  1273. if (qhp->attr.mpa_attr.initiator)
  1274. build_rtr_msg(qhp->attr.mpa_attr.p2p_type, &wqe->u.init);
  1275. ret = c4iw_ofld_send(&rhp->rdev, skb);
  1276. if (ret)
  1277. goto err1;
  1278. ret = c4iw_wait_for_reply(&rhp->rdev, &qhp->ep->com.wr_wait,
  1279. qhp->ep->hwtid, qhp->wq.sq.qid, __func__);
  1280. if (!ret)
  1281. goto out;
  1282. err1:
  1283. free_ird(rhp, qhp->attr.max_ird);
  1284. out:
  1285. PDBG("%s ret %d\n", __func__, ret);
  1286. return ret;
  1287. }
  1288. int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
  1289. enum c4iw_qp_attr_mask mask,
  1290. struct c4iw_qp_attributes *attrs,
  1291. int internal)
  1292. {
  1293. int ret = 0;
  1294. struct c4iw_qp_attributes newattr = qhp->attr;
  1295. int disconnect = 0;
  1296. int terminate = 0;
  1297. int abort = 0;
  1298. int free = 0;
  1299. struct c4iw_ep *ep = NULL;
  1300. PDBG("%s qhp %p sqid 0x%x rqid 0x%x ep %p state %d -> %d\n", __func__,
  1301. qhp, qhp->wq.sq.qid, qhp->wq.rq.qid, qhp->ep, qhp->attr.state,
  1302. (mask & C4IW_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1);
  1303. mutex_lock(&qhp->mutex);
  1304. /* Process attr changes if in IDLE */
  1305. if (mask & C4IW_QP_ATTR_VALID_MODIFY) {
  1306. if (qhp->attr.state != C4IW_QP_STATE_IDLE) {
  1307. ret = -EIO;
  1308. goto out;
  1309. }
  1310. if (mask & C4IW_QP_ATTR_ENABLE_RDMA_READ)
  1311. newattr.enable_rdma_read = attrs->enable_rdma_read;
  1312. if (mask & C4IW_QP_ATTR_ENABLE_RDMA_WRITE)
  1313. newattr.enable_rdma_write = attrs->enable_rdma_write;
  1314. if (mask & C4IW_QP_ATTR_ENABLE_RDMA_BIND)
  1315. newattr.enable_bind = attrs->enable_bind;
  1316. if (mask & C4IW_QP_ATTR_MAX_ORD) {
  1317. if (attrs->max_ord > c4iw_max_read_depth) {
  1318. ret = -EINVAL;
  1319. goto out;
  1320. }
  1321. newattr.max_ord = attrs->max_ord;
  1322. }
  1323. if (mask & C4IW_QP_ATTR_MAX_IRD) {
  1324. if (attrs->max_ird > cur_max_read_depth(rhp)) {
  1325. ret = -EINVAL;
  1326. goto out;
  1327. }
  1328. newattr.max_ird = attrs->max_ird;
  1329. }
  1330. qhp->attr = newattr;
  1331. }
  1332. if (mask & C4IW_QP_ATTR_SQ_DB) {
  1333. ret = ring_kernel_sq_db(qhp, attrs->sq_db_inc);
  1334. goto out;
  1335. }
  1336. if (mask & C4IW_QP_ATTR_RQ_DB) {
  1337. ret = ring_kernel_rq_db(qhp, attrs->rq_db_inc);
  1338. goto out;
  1339. }
  1340. if (!(mask & C4IW_QP_ATTR_NEXT_STATE))
  1341. goto out;
  1342. if (qhp->attr.state == attrs->next_state)
  1343. goto out;
  1344. switch (qhp->attr.state) {
  1345. case C4IW_QP_STATE_IDLE:
  1346. switch (attrs->next_state) {
  1347. case C4IW_QP_STATE_RTS:
  1348. if (!(mask & C4IW_QP_ATTR_LLP_STREAM_HANDLE)) {
  1349. ret = -EINVAL;
  1350. goto out;
  1351. }
  1352. if (!(mask & C4IW_QP_ATTR_MPA_ATTR)) {
  1353. ret = -EINVAL;
  1354. goto out;
  1355. }
  1356. qhp->attr.mpa_attr = attrs->mpa_attr;
  1357. qhp->attr.llp_stream_handle = attrs->llp_stream_handle;
  1358. qhp->ep = qhp->attr.llp_stream_handle;
  1359. set_state(qhp, C4IW_QP_STATE_RTS);
  1360. /*
  1361. * Ref the endpoint here and deref when we
  1362. * disassociate the endpoint from the QP. This
  1363. * happens in CLOSING->IDLE transition or *->ERROR
  1364. * transition.
  1365. */
  1366. c4iw_get_ep(&qhp->ep->com);
  1367. ret = rdma_init(rhp, qhp);
  1368. if (ret)
  1369. goto err;
  1370. break;
  1371. case C4IW_QP_STATE_ERROR:
  1372. set_state(qhp, C4IW_QP_STATE_ERROR);
  1373. flush_qp(qhp);
  1374. break;
  1375. default:
  1376. ret = -EINVAL;
  1377. goto out;
  1378. }
  1379. break;
  1380. case C4IW_QP_STATE_RTS:
  1381. switch (attrs->next_state) {
  1382. case C4IW_QP_STATE_CLOSING:
  1383. BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2);
  1384. t4_set_wq_in_error(&qhp->wq);
  1385. set_state(qhp, C4IW_QP_STATE_CLOSING);
  1386. ep = qhp->ep;
  1387. if (!internal) {
  1388. abort = 0;
  1389. disconnect = 1;
  1390. c4iw_get_ep(&qhp->ep->com);
  1391. }
  1392. ret = rdma_fini(rhp, qhp, ep);
  1393. if (ret)
  1394. goto err;
  1395. break;
  1396. case C4IW_QP_STATE_TERMINATE:
  1397. t4_set_wq_in_error(&qhp->wq);
  1398. set_state(qhp, C4IW_QP_STATE_TERMINATE);
  1399. qhp->attr.layer_etype = attrs->layer_etype;
  1400. qhp->attr.ecode = attrs->ecode;
  1401. ep = qhp->ep;
  1402. if (!internal) {
  1403. c4iw_get_ep(&qhp->ep->com);
  1404. terminate = 1;
  1405. disconnect = 1;
  1406. } else {
  1407. terminate = qhp->attr.send_term;
  1408. ret = rdma_fini(rhp, qhp, ep);
  1409. if (ret)
  1410. goto err;
  1411. }
  1412. break;
  1413. case C4IW_QP_STATE_ERROR:
  1414. t4_set_wq_in_error(&qhp->wq);
  1415. set_state(qhp, C4IW_QP_STATE_ERROR);
  1416. if (!internal) {
  1417. abort = 1;
  1418. disconnect = 1;
  1419. ep = qhp->ep;
  1420. c4iw_get_ep(&qhp->ep->com);
  1421. }
  1422. goto err;
  1423. break;
  1424. default:
  1425. ret = -EINVAL;
  1426. goto out;
  1427. }
  1428. break;
  1429. case C4IW_QP_STATE_CLOSING:
  1430. if (!internal) {
  1431. ret = -EINVAL;
  1432. goto out;
  1433. }
  1434. switch (attrs->next_state) {
  1435. case C4IW_QP_STATE_IDLE:
  1436. flush_qp(qhp);
  1437. set_state(qhp, C4IW_QP_STATE_IDLE);
  1438. qhp->attr.llp_stream_handle = NULL;
  1439. c4iw_put_ep(&qhp->ep->com);
  1440. qhp->ep = NULL;
  1441. wake_up(&qhp->wait);
  1442. break;
  1443. case C4IW_QP_STATE_ERROR:
  1444. goto err;
  1445. default:
  1446. ret = -EINVAL;
  1447. goto err;
  1448. }
  1449. break;
  1450. case C4IW_QP_STATE_ERROR:
  1451. if (attrs->next_state != C4IW_QP_STATE_IDLE) {
  1452. ret = -EINVAL;
  1453. goto out;
  1454. }
  1455. if (!t4_sq_empty(&qhp->wq) || !t4_rq_empty(&qhp->wq)) {
  1456. ret = -EINVAL;
  1457. goto out;
  1458. }
  1459. set_state(qhp, C4IW_QP_STATE_IDLE);
  1460. break;
  1461. case C4IW_QP_STATE_TERMINATE:
  1462. if (!internal) {
  1463. ret = -EINVAL;
  1464. goto out;
  1465. }
  1466. goto err;
  1467. break;
  1468. default:
  1469. printk(KERN_ERR "%s in a bad state %d\n",
  1470. __func__, qhp->attr.state);
  1471. ret = -EINVAL;
  1472. goto err;
  1473. break;
  1474. }
  1475. goto out;
  1476. err:
  1477. PDBG("%s disassociating ep %p qpid 0x%x\n", __func__, qhp->ep,
  1478. qhp->wq.sq.qid);
  1479. /* disassociate the LLP connection */
  1480. qhp->attr.llp_stream_handle = NULL;
  1481. if (!ep)
  1482. ep = qhp->ep;
  1483. qhp->ep = NULL;
  1484. set_state(qhp, C4IW_QP_STATE_ERROR);
  1485. free = 1;
  1486. abort = 1;
  1487. BUG_ON(!ep);
  1488. flush_qp(qhp);
  1489. wake_up(&qhp->wait);
  1490. out:
  1491. mutex_unlock(&qhp->mutex);
  1492. if (terminate)
  1493. post_terminate(qhp, NULL, internal ? GFP_ATOMIC : GFP_KERNEL);
  1494. /*
  1495. * If disconnect is 1, then we need to initiate a disconnect
  1496. * on the EP. This can be a normal close (RTS->CLOSING) or
  1497. * an abnormal close (RTS/CLOSING->ERROR).
  1498. */
  1499. if (disconnect) {
  1500. c4iw_ep_disconnect(ep, abort, internal ? GFP_ATOMIC :
  1501. GFP_KERNEL);
  1502. c4iw_put_ep(&ep->com);
  1503. }
  1504. /*
  1505. * If free is 1, then we've disassociated the EP from the QP
  1506. * and we need to dereference the EP.
  1507. */
  1508. if (free)
  1509. c4iw_put_ep(&ep->com);
  1510. PDBG("%s exit state %d\n", __func__, qhp->attr.state);
  1511. return ret;
  1512. }
  1513. int c4iw_destroy_qp(struct ib_qp *ib_qp)
  1514. {
  1515. struct c4iw_dev *rhp;
  1516. struct c4iw_qp *qhp;
  1517. struct c4iw_qp_attributes attrs;
  1518. qhp = to_c4iw_qp(ib_qp);
  1519. rhp = qhp->rhp;
  1520. attrs.next_state = C4IW_QP_STATE_ERROR;
  1521. if (qhp->attr.state == C4IW_QP_STATE_TERMINATE)
  1522. c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
  1523. else
  1524. c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0);
  1525. wait_event(qhp->wait, !qhp->ep);
  1526. remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
  1527. spin_lock_irq(&rhp->lock);
  1528. if (!list_empty(&qhp->db_fc_entry))
  1529. list_del_init(&qhp->db_fc_entry);
  1530. spin_unlock_irq(&rhp->lock);
  1531. free_ird(rhp, qhp->attr.max_ird);
  1532. c4iw_qp_rem_ref(ib_qp);
  1533. PDBG("%s ib_qp %p qpid 0x%0x\n", __func__, ib_qp, qhp->wq.sq.qid);
  1534. return 0;
  1535. }
  1536. struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
  1537. struct ib_udata *udata)
  1538. {
  1539. struct c4iw_dev *rhp;
  1540. struct c4iw_qp *qhp;
  1541. struct c4iw_pd *php;
  1542. struct c4iw_cq *schp;
  1543. struct c4iw_cq *rchp;
  1544. struct c4iw_create_qp_resp uresp;
  1545. unsigned int sqsize, rqsize;
  1546. struct c4iw_ucontext *ucontext;
  1547. int ret;
  1548. struct c4iw_mm_entry *sq_key_mm, *rq_key_mm = NULL, *sq_db_key_mm;
  1549. struct c4iw_mm_entry *rq_db_key_mm = NULL, *ma_sync_key_mm = NULL;
  1550. PDBG("%s ib_pd %p\n", __func__, pd);
  1551. if (attrs->qp_type != IB_QPT_RC)
  1552. return ERR_PTR(-EINVAL);
  1553. php = to_c4iw_pd(pd);
  1554. rhp = php->rhp;
  1555. schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid);
  1556. rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid);
  1557. if (!schp || !rchp)
  1558. return ERR_PTR(-EINVAL);
  1559. if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE)
  1560. return ERR_PTR(-EINVAL);
  1561. if (attrs->cap.max_recv_wr > rhp->rdev.hw_queue.t4_max_rq_size)
  1562. return ERR_PTR(-E2BIG);
  1563. rqsize = attrs->cap.max_recv_wr + 1;
  1564. if (rqsize < 8)
  1565. rqsize = 8;
  1566. if (attrs->cap.max_send_wr > rhp->rdev.hw_queue.t4_max_sq_size)
  1567. return ERR_PTR(-E2BIG);
  1568. sqsize = attrs->cap.max_send_wr + 1;
  1569. if (sqsize < 8)
  1570. sqsize = 8;
  1571. ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL;
  1572. qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
  1573. if (!qhp)
  1574. return ERR_PTR(-ENOMEM);
  1575. qhp->wq.sq.size = sqsize;
  1576. qhp->wq.sq.memsize =
  1577. (sqsize + rhp->rdev.hw_queue.t4_eq_status_entries) *
  1578. sizeof(*qhp->wq.sq.queue) + 16 * sizeof(__be64);
  1579. qhp->wq.sq.flush_cidx = -1;
  1580. qhp->wq.rq.size = rqsize;
  1581. qhp->wq.rq.memsize =
  1582. (rqsize + rhp->rdev.hw_queue.t4_eq_status_entries) *
  1583. sizeof(*qhp->wq.rq.queue);
  1584. if (ucontext) {
  1585. qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE);
  1586. qhp->wq.rq.memsize = roundup(qhp->wq.rq.memsize, PAGE_SIZE);
  1587. }
  1588. ret = create_qp(&rhp->rdev, &qhp->wq, &schp->cq, &rchp->cq,
  1589. ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
  1590. if (ret)
  1591. goto err1;
  1592. attrs->cap.max_recv_wr = rqsize - 1;
  1593. attrs->cap.max_send_wr = sqsize - 1;
  1594. attrs->cap.max_inline_data = T4_MAX_SEND_INLINE;
  1595. qhp->rhp = rhp;
  1596. qhp->attr.pd = php->pdid;
  1597. qhp->attr.scq = ((struct c4iw_cq *) attrs->send_cq)->cq.cqid;
  1598. qhp->attr.rcq = ((struct c4iw_cq *) attrs->recv_cq)->cq.cqid;
  1599. qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
  1600. qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
  1601. qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
  1602. qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
  1603. qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
  1604. qhp->attr.state = C4IW_QP_STATE_IDLE;
  1605. qhp->attr.next_state = C4IW_QP_STATE_IDLE;
  1606. qhp->attr.enable_rdma_read = 1;
  1607. qhp->attr.enable_rdma_write = 1;
  1608. qhp->attr.enable_bind = 1;
  1609. qhp->attr.max_ord = 0;
  1610. qhp->attr.max_ird = 0;
  1611. qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
  1612. spin_lock_init(&qhp->lock);
  1613. init_completion(&qhp->sq_drained);
  1614. init_completion(&qhp->rq_drained);
  1615. mutex_init(&qhp->mutex);
  1616. init_waitqueue_head(&qhp->wait);
  1617. kref_init(&qhp->kref);
  1618. INIT_WORK(&qhp->free_work, free_qp_work);
  1619. ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid);
  1620. if (ret)
  1621. goto err2;
  1622. if (udata) {
  1623. sq_key_mm = kmalloc(sizeof(*sq_key_mm), GFP_KERNEL);
  1624. if (!sq_key_mm) {
  1625. ret = -ENOMEM;
  1626. goto err3;
  1627. }
  1628. rq_key_mm = kmalloc(sizeof(*rq_key_mm), GFP_KERNEL);
  1629. if (!rq_key_mm) {
  1630. ret = -ENOMEM;
  1631. goto err4;
  1632. }
  1633. sq_db_key_mm = kmalloc(sizeof(*sq_db_key_mm), GFP_KERNEL);
  1634. if (!sq_db_key_mm) {
  1635. ret = -ENOMEM;
  1636. goto err5;
  1637. }
  1638. rq_db_key_mm = kmalloc(sizeof(*rq_db_key_mm), GFP_KERNEL);
  1639. if (!rq_db_key_mm) {
  1640. ret = -ENOMEM;
  1641. goto err6;
  1642. }
  1643. if (t4_sq_onchip(&qhp->wq.sq)) {
  1644. ma_sync_key_mm = kmalloc(sizeof(*ma_sync_key_mm),
  1645. GFP_KERNEL);
  1646. if (!ma_sync_key_mm) {
  1647. ret = -ENOMEM;
  1648. goto err7;
  1649. }
  1650. uresp.flags = C4IW_QPF_ONCHIP;
  1651. } else
  1652. uresp.flags = 0;
  1653. uresp.qid_mask = rhp->rdev.qpmask;
  1654. uresp.sqid = qhp->wq.sq.qid;
  1655. uresp.sq_size = qhp->wq.sq.size;
  1656. uresp.sq_memsize = qhp->wq.sq.memsize;
  1657. uresp.rqid = qhp->wq.rq.qid;
  1658. uresp.rq_size = qhp->wq.rq.size;
  1659. uresp.rq_memsize = qhp->wq.rq.memsize;
  1660. spin_lock(&ucontext->mmap_lock);
  1661. if (ma_sync_key_mm) {
  1662. uresp.ma_sync_key = ucontext->key;
  1663. ucontext->key += PAGE_SIZE;
  1664. } else {
  1665. uresp.ma_sync_key = 0;
  1666. }
  1667. uresp.sq_key = ucontext->key;
  1668. ucontext->key += PAGE_SIZE;
  1669. uresp.rq_key = ucontext->key;
  1670. ucontext->key += PAGE_SIZE;
  1671. uresp.sq_db_gts_key = ucontext->key;
  1672. ucontext->key += PAGE_SIZE;
  1673. uresp.rq_db_gts_key = ucontext->key;
  1674. ucontext->key += PAGE_SIZE;
  1675. spin_unlock(&ucontext->mmap_lock);
  1676. ret = ib_copy_to_udata(udata, &uresp, sizeof uresp);
  1677. if (ret)
  1678. goto err8;
  1679. sq_key_mm->key = uresp.sq_key;
  1680. sq_key_mm->addr = qhp->wq.sq.phys_addr;
  1681. sq_key_mm->len = PAGE_ALIGN(qhp->wq.sq.memsize);
  1682. insert_mmap(ucontext, sq_key_mm);
  1683. rq_key_mm->key = uresp.rq_key;
  1684. rq_key_mm->addr = virt_to_phys(qhp->wq.rq.queue);
  1685. rq_key_mm->len = PAGE_ALIGN(qhp->wq.rq.memsize);
  1686. insert_mmap(ucontext, rq_key_mm);
  1687. sq_db_key_mm->key = uresp.sq_db_gts_key;
  1688. sq_db_key_mm->addr = (u64)(unsigned long)qhp->wq.sq.bar2_pa;
  1689. sq_db_key_mm->len = PAGE_SIZE;
  1690. insert_mmap(ucontext, sq_db_key_mm);
  1691. rq_db_key_mm->key = uresp.rq_db_gts_key;
  1692. rq_db_key_mm->addr = (u64)(unsigned long)qhp->wq.rq.bar2_pa;
  1693. rq_db_key_mm->len = PAGE_SIZE;
  1694. insert_mmap(ucontext, rq_db_key_mm);
  1695. if (ma_sync_key_mm) {
  1696. ma_sync_key_mm->key = uresp.ma_sync_key;
  1697. ma_sync_key_mm->addr =
  1698. (pci_resource_start(rhp->rdev.lldi.pdev, 0) +
  1699. PCIE_MA_SYNC_A) & PAGE_MASK;
  1700. ma_sync_key_mm->len = PAGE_SIZE;
  1701. insert_mmap(ucontext, ma_sync_key_mm);
  1702. }
  1703. c4iw_get_ucontext(ucontext);
  1704. qhp->ucontext = ucontext;
  1705. }
  1706. qhp->ibqp.qp_num = qhp->wq.sq.qid;
  1707. init_timer(&(qhp->timer));
  1708. INIT_LIST_HEAD(&qhp->db_fc_entry);
  1709. PDBG("%s sq id %u size %u memsize %zu num_entries %u "
  1710. "rq id %u size %u memsize %zu num_entries %u\n", __func__,
  1711. qhp->wq.sq.qid, qhp->wq.sq.size, qhp->wq.sq.memsize,
  1712. attrs->cap.max_send_wr, qhp->wq.rq.qid, qhp->wq.rq.size,
  1713. qhp->wq.rq.memsize, attrs->cap.max_recv_wr);
  1714. return &qhp->ibqp;
  1715. err8:
  1716. kfree(ma_sync_key_mm);
  1717. err7:
  1718. kfree(rq_db_key_mm);
  1719. err6:
  1720. kfree(sq_db_key_mm);
  1721. err5:
  1722. kfree(rq_key_mm);
  1723. err4:
  1724. kfree(sq_key_mm);
  1725. err3:
  1726. remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
  1727. err2:
  1728. destroy_qp(&rhp->rdev, &qhp->wq,
  1729. ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
  1730. err1:
  1731. kfree(qhp);
  1732. return ERR_PTR(ret);
  1733. }
  1734. int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
  1735. int attr_mask, struct ib_udata *udata)
  1736. {
  1737. struct c4iw_dev *rhp;
  1738. struct c4iw_qp *qhp;
  1739. enum c4iw_qp_attr_mask mask = 0;
  1740. struct c4iw_qp_attributes attrs;
  1741. PDBG("%s ib_qp %p\n", __func__, ibqp);
  1742. /* iwarp does not support the RTR state */
  1743. if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
  1744. attr_mask &= ~IB_QP_STATE;
  1745. /* Make sure we still have something left to do */
  1746. if (!attr_mask)
  1747. return 0;
  1748. memset(&attrs, 0, sizeof attrs);
  1749. qhp = to_c4iw_qp(ibqp);
  1750. rhp = qhp->rhp;
  1751. attrs.next_state = c4iw_convert_state(attr->qp_state);
  1752. attrs.enable_rdma_read = (attr->qp_access_flags &
  1753. IB_ACCESS_REMOTE_READ) ? 1 : 0;
  1754. attrs.enable_rdma_write = (attr->qp_access_flags &
  1755. IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
  1756. attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
  1757. mask |= (attr_mask & IB_QP_STATE) ? C4IW_QP_ATTR_NEXT_STATE : 0;
  1758. mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
  1759. (C4IW_QP_ATTR_ENABLE_RDMA_READ |
  1760. C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
  1761. C4IW_QP_ATTR_ENABLE_RDMA_BIND) : 0;
  1762. /*
  1763. * Use SQ_PSN and RQ_PSN to pass in IDX_INC values for
  1764. * ringing the queue db when we're in DB_FULL mode.
  1765. * Only allow this on T4 devices.
  1766. */
  1767. attrs.sq_db_inc = attr->sq_psn;
  1768. attrs.rq_db_inc = attr->rq_psn;
  1769. mask |= (attr_mask & IB_QP_SQ_PSN) ? C4IW_QP_ATTR_SQ_DB : 0;
  1770. mask |= (attr_mask & IB_QP_RQ_PSN) ? C4IW_QP_ATTR_RQ_DB : 0;
  1771. if (!is_t4(to_c4iw_qp(ibqp)->rhp->rdev.lldi.adapter_type) &&
  1772. (mask & (C4IW_QP_ATTR_SQ_DB|C4IW_QP_ATTR_RQ_DB)))
  1773. return -EINVAL;
  1774. return c4iw_modify_qp(rhp, qhp, mask, &attrs, 0);
  1775. }
  1776. struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn)
  1777. {
  1778. PDBG("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
  1779. return (struct ib_qp *)get_qhp(to_c4iw_dev(dev), qpn);
  1780. }
  1781. int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
  1782. int attr_mask, struct ib_qp_init_attr *init_attr)
  1783. {
  1784. struct c4iw_qp *qhp = to_c4iw_qp(ibqp);
  1785. memset(attr, 0, sizeof *attr);
  1786. memset(init_attr, 0, sizeof *init_attr);
  1787. attr->qp_state = to_ib_qp_state(qhp->attr.state);
  1788. init_attr->cap.max_send_wr = qhp->attr.sq_num_entries;
  1789. init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries;
  1790. init_attr->cap.max_send_sge = qhp->attr.sq_max_sges;
  1791. init_attr->cap.max_recv_sge = qhp->attr.sq_max_sges;
  1792. init_attr->cap.max_inline_data = T4_MAX_SEND_INLINE;
  1793. init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0;
  1794. return 0;
  1795. }
  1796. static void move_qp_to_err(struct c4iw_qp *qp)
  1797. {
  1798. struct c4iw_qp_attributes attrs = { .next_state = C4IW_QP_STATE_ERROR };
  1799. (void)c4iw_modify_qp(qp->rhp, qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
  1800. }
  1801. void c4iw_drain_sq(struct ib_qp *ibqp)
  1802. {
  1803. struct c4iw_qp *qp = to_c4iw_qp(ibqp);
  1804. unsigned long flag;
  1805. bool need_to_wait;
  1806. move_qp_to_err(qp);
  1807. spin_lock_irqsave(&qp->lock, flag);
  1808. need_to_wait = !t4_sq_empty(&qp->wq);
  1809. spin_unlock_irqrestore(&qp->lock, flag);
  1810. if (need_to_wait)
  1811. wait_for_completion(&qp->sq_drained);
  1812. }
  1813. void c4iw_drain_rq(struct ib_qp *ibqp)
  1814. {
  1815. struct c4iw_qp *qp = to_c4iw_qp(ibqp);
  1816. unsigned long flag;
  1817. bool need_to_wait;
  1818. move_qp_to_err(qp);
  1819. spin_lock_irqsave(&qp->lock, flag);
  1820. need_to_wait = !t4_rq_empty(&qp->wq);
  1821. spin_unlock_irqrestore(&qp->lock, flag);
  1822. if (need_to_wait)
  1823. wait_for_completion(&qp->rq_drained);
  1824. }