ipath_user_sdma.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876
  1. /*
  2. * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #include <linux/mm.h>
  33. #include <linux/types.h>
  34. #include <linux/device.h>
  35. #include <linux/dmapool.h>
  36. #include <linux/sched.h>
  37. #include <linux/slab.h>
  38. #include <linux/list.h>
  39. #include <linux/highmem.h>
  40. #include <linux/io.h>
  41. #include <linux/uio.h>
  42. #include <linux/rbtree.h>
  43. #include <linux/spinlock.h>
  44. #include <linux/delay.h>
  45. #include "ipath_kernel.h"
  46. #include "ipath_user_sdma.h"
  47. /* minimum size of header */
  48. #define IPATH_USER_SDMA_MIN_HEADER_LENGTH 64
  49. /* expected size of headers (for dma_pool) */
  50. #define IPATH_USER_SDMA_EXP_HEADER_LENGTH 64
  51. /* length mask in PBC (lower 11 bits) */
  52. #define IPATH_PBC_LENGTH_MASK ((1 << 11) - 1)
  53. struct ipath_user_sdma_pkt {
  54. u8 naddr; /* dimension of addr (1..3) ... */
  55. u32 counter; /* sdma pkts queued counter for this entry */
  56. u64 added; /* global descq number of entries */
  57. struct {
  58. u32 offset; /* offset for kvaddr, addr */
  59. u32 length; /* length in page */
  60. u8 put_page; /* should we put_page? */
  61. u8 dma_mapped; /* is page dma_mapped? */
  62. struct page *page; /* may be NULL (coherent mem) */
  63. void *kvaddr; /* FIXME: only for pio hack */
  64. dma_addr_t addr;
  65. } addr[4]; /* max pages, any more and we coalesce */
  66. struct list_head list; /* list element */
  67. };
  68. struct ipath_user_sdma_queue {
  69. /*
  70. * pkts sent to dma engine are queued on this
  71. * list head. the type of the elements of this
  72. * list are struct ipath_user_sdma_pkt...
  73. */
  74. struct list_head sent;
  75. /* headers with expected length are allocated from here... */
  76. char header_cache_name[64];
  77. struct dma_pool *header_cache;
  78. /* packets are allocated from the slab cache... */
  79. char pkt_slab_name[64];
  80. struct kmem_cache *pkt_slab;
  81. /* as packets go on the queued queue, they are counted... */
  82. u32 counter;
  83. u32 sent_counter;
  84. /* dma page table */
  85. struct rb_root dma_pages_root;
  86. /* protect everything above... */
  87. struct mutex lock;
  88. };
  89. struct ipath_user_sdma_queue *
  90. ipath_user_sdma_queue_create(struct device *dev, int unit, int port, int sport)
  91. {
  92. struct ipath_user_sdma_queue *pq =
  93. kmalloc(sizeof(struct ipath_user_sdma_queue), GFP_KERNEL);
  94. if (!pq)
  95. goto done;
  96. pq->counter = 0;
  97. pq->sent_counter = 0;
  98. INIT_LIST_HEAD(&pq->sent);
  99. mutex_init(&pq->lock);
  100. snprintf(pq->pkt_slab_name, sizeof(pq->pkt_slab_name),
  101. "ipath-user-sdma-pkts-%u-%02u.%02u", unit, port, sport);
  102. pq->pkt_slab = kmem_cache_create(pq->pkt_slab_name,
  103. sizeof(struct ipath_user_sdma_pkt),
  104. 0, 0, NULL);
  105. if (!pq->pkt_slab)
  106. goto err_kfree;
  107. snprintf(pq->header_cache_name, sizeof(pq->header_cache_name),
  108. "ipath-user-sdma-headers-%u-%02u.%02u", unit, port, sport);
  109. pq->header_cache = dma_pool_create(pq->header_cache_name,
  110. dev,
  111. IPATH_USER_SDMA_EXP_HEADER_LENGTH,
  112. 4, 0);
  113. if (!pq->header_cache)
  114. goto err_slab;
  115. pq->dma_pages_root = RB_ROOT;
  116. goto done;
  117. err_slab:
  118. kmem_cache_destroy(pq->pkt_slab);
  119. err_kfree:
  120. kfree(pq);
  121. pq = NULL;
  122. done:
  123. return pq;
  124. }
  125. static void ipath_user_sdma_init_frag(struct ipath_user_sdma_pkt *pkt,
  126. int i, size_t offset, size_t len,
  127. int put_page, int dma_mapped,
  128. struct page *page,
  129. void *kvaddr, dma_addr_t dma_addr)
  130. {
  131. pkt->addr[i].offset = offset;
  132. pkt->addr[i].length = len;
  133. pkt->addr[i].put_page = put_page;
  134. pkt->addr[i].dma_mapped = dma_mapped;
  135. pkt->addr[i].page = page;
  136. pkt->addr[i].kvaddr = kvaddr;
  137. pkt->addr[i].addr = dma_addr;
  138. }
  139. static void ipath_user_sdma_init_header(struct ipath_user_sdma_pkt *pkt,
  140. u32 counter, size_t offset,
  141. size_t len, int dma_mapped,
  142. struct page *page,
  143. void *kvaddr, dma_addr_t dma_addr)
  144. {
  145. pkt->naddr = 1;
  146. pkt->counter = counter;
  147. ipath_user_sdma_init_frag(pkt, 0, offset, len, 0, dma_mapped, page,
  148. kvaddr, dma_addr);
  149. }
  150. /* we've too many pages in the iovec, coalesce to a single page */
  151. static int ipath_user_sdma_coalesce(const struct ipath_devdata *dd,
  152. struct ipath_user_sdma_pkt *pkt,
  153. const struct iovec *iov,
  154. unsigned long niov) {
  155. int ret = 0;
  156. struct page *page = alloc_page(GFP_KERNEL);
  157. void *mpage_save;
  158. char *mpage;
  159. int i;
  160. int len = 0;
  161. dma_addr_t dma_addr;
  162. if (!page) {
  163. ret = -ENOMEM;
  164. goto done;
  165. }
  166. mpage = kmap(page);
  167. mpage_save = mpage;
  168. for (i = 0; i < niov; i++) {
  169. int cfur;
  170. cfur = copy_from_user(mpage,
  171. iov[i].iov_base, iov[i].iov_len);
  172. if (cfur) {
  173. ret = -EFAULT;
  174. goto free_unmap;
  175. }
  176. mpage += iov[i].iov_len;
  177. len += iov[i].iov_len;
  178. }
  179. dma_addr = dma_map_page(&dd->pcidev->dev, page, 0, len,
  180. DMA_TO_DEVICE);
  181. if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
  182. ret = -ENOMEM;
  183. goto free_unmap;
  184. }
  185. ipath_user_sdma_init_frag(pkt, 1, 0, len, 0, 1, page, mpage_save,
  186. dma_addr);
  187. pkt->naddr = 2;
  188. goto done;
  189. free_unmap:
  190. kunmap(page);
  191. __free_page(page);
  192. done:
  193. return ret;
  194. }
  195. /* how many pages in this iovec element? */
  196. static int ipath_user_sdma_num_pages(const struct iovec *iov)
  197. {
  198. const unsigned long addr = (unsigned long) iov->iov_base;
  199. const unsigned long len = iov->iov_len;
  200. const unsigned long spage = addr & PAGE_MASK;
  201. const unsigned long epage = (addr + len - 1) & PAGE_MASK;
  202. return 1 + ((epage - spage) >> PAGE_SHIFT);
  203. }
  204. /* truncate length to page boundary */
  205. static int ipath_user_sdma_page_length(unsigned long addr, unsigned long len)
  206. {
  207. const unsigned long offset = addr & ~PAGE_MASK;
  208. return ((offset + len) > PAGE_SIZE) ? (PAGE_SIZE - offset) : len;
  209. }
  210. static void ipath_user_sdma_free_pkt_frag(struct device *dev,
  211. struct ipath_user_sdma_queue *pq,
  212. struct ipath_user_sdma_pkt *pkt,
  213. int frag)
  214. {
  215. const int i = frag;
  216. if (pkt->addr[i].page) {
  217. if (pkt->addr[i].dma_mapped)
  218. dma_unmap_page(dev,
  219. pkt->addr[i].addr,
  220. pkt->addr[i].length,
  221. DMA_TO_DEVICE);
  222. if (pkt->addr[i].kvaddr)
  223. kunmap(pkt->addr[i].page);
  224. if (pkt->addr[i].put_page)
  225. put_page(pkt->addr[i].page);
  226. else
  227. __free_page(pkt->addr[i].page);
  228. } else if (pkt->addr[i].kvaddr)
  229. /* free coherent mem from cache... */
  230. dma_pool_free(pq->header_cache,
  231. pkt->addr[i].kvaddr, pkt->addr[i].addr);
  232. }
  233. /* return number of pages pinned... */
  234. static int ipath_user_sdma_pin_pages(const struct ipath_devdata *dd,
  235. struct ipath_user_sdma_pkt *pkt,
  236. unsigned long addr, int tlen, int npages)
  237. {
  238. struct page *pages[2];
  239. int j;
  240. int ret;
  241. ret = get_user_pages_fast(addr, npages, 0, pages);
  242. if (ret != npages) {
  243. int i;
  244. for (i = 0; i < ret; i++)
  245. put_page(pages[i]);
  246. ret = -ENOMEM;
  247. goto done;
  248. }
  249. for (j = 0; j < npages; j++) {
  250. /* map the pages... */
  251. const int flen =
  252. ipath_user_sdma_page_length(addr, tlen);
  253. dma_addr_t dma_addr =
  254. dma_map_page(&dd->pcidev->dev,
  255. pages[j], 0, flen, DMA_TO_DEVICE);
  256. unsigned long fofs = addr & ~PAGE_MASK;
  257. if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
  258. ret = -ENOMEM;
  259. goto done;
  260. }
  261. ipath_user_sdma_init_frag(pkt, pkt->naddr, fofs, flen, 1, 1,
  262. pages[j], kmap(pages[j]),
  263. dma_addr);
  264. pkt->naddr++;
  265. addr += flen;
  266. tlen -= flen;
  267. }
  268. done:
  269. return ret;
  270. }
  271. static int ipath_user_sdma_pin_pkt(const struct ipath_devdata *dd,
  272. struct ipath_user_sdma_queue *pq,
  273. struct ipath_user_sdma_pkt *pkt,
  274. const struct iovec *iov,
  275. unsigned long niov)
  276. {
  277. int ret = 0;
  278. unsigned long idx;
  279. for (idx = 0; idx < niov; idx++) {
  280. const int npages = ipath_user_sdma_num_pages(iov + idx);
  281. const unsigned long addr = (unsigned long) iov[idx].iov_base;
  282. ret = ipath_user_sdma_pin_pages(dd, pkt,
  283. addr, iov[idx].iov_len,
  284. npages);
  285. if (ret < 0)
  286. goto free_pkt;
  287. }
  288. goto done;
  289. free_pkt:
  290. for (idx = 0; idx < pkt->naddr; idx++)
  291. ipath_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx);
  292. done:
  293. return ret;
  294. }
  295. static int ipath_user_sdma_init_payload(const struct ipath_devdata *dd,
  296. struct ipath_user_sdma_queue *pq,
  297. struct ipath_user_sdma_pkt *pkt,
  298. const struct iovec *iov,
  299. unsigned long niov, int npages)
  300. {
  301. int ret = 0;
  302. if (npages >= ARRAY_SIZE(pkt->addr))
  303. ret = ipath_user_sdma_coalesce(dd, pkt, iov, niov);
  304. else
  305. ret = ipath_user_sdma_pin_pkt(dd, pq, pkt, iov, niov);
  306. return ret;
  307. }
  308. /* free a packet list -- return counter value of last packet */
  309. static void ipath_user_sdma_free_pkt_list(struct device *dev,
  310. struct ipath_user_sdma_queue *pq,
  311. struct list_head *list)
  312. {
  313. struct ipath_user_sdma_pkt *pkt, *pkt_next;
  314. list_for_each_entry_safe(pkt, pkt_next, list, list) {
  315. int i;
  316. for (i = 0; i < pkt->naddr; i++)
  317. ipath_user_sdma_free_pkt_frag(dev, pq, pkt, i);
  318. kmem_cache_free(pq->pkt_slab, pkt);
  319. }
  320. }
  321. /*
  322. * copy headers, coalesce etc -- pq->lock must be held
  323. *
  324. * we queue all the packets to list, returning the
  325. * number of bytes total. list must be empty initially,
  326. * as, if there is an error we clean it...
  327. */
  328. static int ipath_user_sdma_queue_pkts(const struct ipath_devdata *dd,
  329. struct ipath_user_sdma_queue *pq,
  330. struct list_head *list,
  331. const struct iovec *iov,
  332. unsigned long niov,
  333. int maxpkts)
  334. {
  335. unsigned long idx = 0;
  336. int ret = 0;
  337. int npkts = 0;
  338. struct page *page = NULL;
  339. __le32 *pbc;
  340. dma_addr_t dma_addr;
  341. struct ipath_user_sdma_pkt *pkt = NULL;
  342. size_t len;
  343. size_t nw;
  344. u32 counter = pq->counter;
  345. int dma_mapped = 0;
  346. while (idx < niov && npkts < maxpkts) {
  347. const unsigned long addr = (unsigned long) iov[idx].iov_base;
  348. const unsigned long idx_save = idx;
  349. unsigned pktnw;
  350. unsigned pktnwc;
  351. int nfrags = 0;
  352. int npages = 0;
  353. int cfur;
  354. dma_mapped = 0;
  355. len = iov[idx].iov_len;
  356. nw = len >> 2;
  357. page = NULL;
  358. pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL);
  359. if (!pkt) {
  360. ret = -ENOMEM;
  361. goto free_list;
  362. }
  363. if (len < IPATH_USER_SDMA_MIN_HEADER_LENGTH ||
  364. len > PAGE_SIZE || len & 3 || addr & 3) {
  365. ret = -EINVAL;
  366. goto free_pkt;
  367. }
  368. if (len == IPATH_USER_SDMA_EXP_HEADER_LENGTH)
  369. pbc = dma_pool_alloc(pq->header_cache, GFP_KERNEL,
  370. &dma_addr);
  371. else
  372. pbc = NULL;
  373. if (!pbc) {
  374. page = alloc_page(GFP_KERNEL);
  375. if (!page) {
  376. ret = -ENOMEM;
  377. goto free_pkt;
  378. }
  379. pbc = kmap(page);
  380. }
  381. cfur = copy_from_user(pbc, iov[idx].iov_base, len);
  382. if (cfur) {
  383. ret = -EFAULT;
  384. goto free_pbc;
  385. }
  386. /*
  387. * this assignment is a bit strange. it's because the
  388. * the pbc counts the number of 32 bit words in the full
  389. * packet _except_ the first word of the pbc itself...
  390. */
  391. pktnwc = nw - 1;
  392. /*
  393. * pktnw computation yields the number of 32 bit words
  394. * that the caller has indicated in the PBC. note that
  395. * this is one less than the total number of words that
  396. * goes to the send DMA engine as the first 32 bit word
  397. * of the PBC itself is not counted. Armed with this count,
  398. * we can verify that the packet is consistent with the
  399. * iovec lengths.
  400. */
  401. pktnw = le32_to_cpu(*pbc) & IPATH_PBC_LENGTH_MASK;
  402. if (pktnw < pktnwc || pktnw > pktnwc + (PAGE_SIZE >> 2)) {
  403. ret = -EINVAL;
  404. goto free_pbc;
  405. }
  406. idx++;
  407. while (pktnwc < pktnw && idx < niov) {
  408. const size_t slen = iov[idx].iov_len;
  409. const unsigned long faddr =
  410. (unsigned long) iov[idx].iov_base;
  411. if (slen & 3 || faddr & 3 || !slen ||
  412. slen > PAGE_SIZE) {
  413. ret = -EINVAL;
  414. goto free_pbc;
  415. }
  416. npages++;
  417. if ((faddr & PAGE_MASK) !=
  418. ((faddr + slen - 1) & PAGE_MASK))
  419. npages++;
  420. pktnwc += slen >> 2;
  421. idx++;
  422. nfrags++;
  423. }
  424. if (pktnwc != pktnw) {
  425. ret = -EINVAL;
  426. goto free_pbc;
  427. }
  428. if (page) {
  429. dma_addr = dma_map_page(&dd->pcidev->dev,
  430. page, 0, len, DMA_TO_DEVICE);
  431. if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
  432. ret = -ENOMEM;
  433. goto free_pbc;
  434. }
  435. dma_mapped = 1;
  436. }
  437. ipath_user_sdma_init_header(pkt, counter, 0, len, dma_mapped,
  438. page, pbc, dma_addr);
  439. if (nfrags) {
  440. ret = ipath_user_sdma_init_payload(dd, pq, pkt,
  441. iov + idx_save + 1,
  442. nfrags, npages);
  443. if (ret < 0)
  444. goto free_pbc_dma;
  445. }
  446. counter++;
  447. npkts++;
  448. list_add_tail(&pkt->list, list);
  449. }
  450. ret = idx;
  451. goto done;
  452. free_pbc_dma:
  453. if (dma_mapped)
  454. dma_unmap_page(&dd->pcidev->dev, dma_addr, len, DMA_TO_DEVICE);
  455. free_pbc:
  456. if (page) {
  457. kunmap(page);
  458. __free_page(page);
  459. } else
  460. dma_pool_free(pq->header_cache, pbc, dma_addr);
  461. free_pkt:
  462. kmem_cache_free(pq->pkt_slab, pkt);
  463. free_list:
  464. ipath_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, list);
  465. done:
  466. return ret;
  467. }
  468. static void ipath_user_sdma_set_complete_counter(struct ipath_user_sdma_queue *pq,
  469. u32 c)
  470. {
  471. pq->sent_counter = c;
  472. }
  473. /* try to clean out queue -- needs pq->lock */
  474. static int ipath_user_sdma_queue_clean(const struct ipath_devdata *dd,
  475. struct ipath_user_sdma_queue *pq)
  476. {
  477. struct list_head free_list;
  478. struct ipath_user_sdma_pkt *pkt;
  479. struct ipath_user_sdma_pkt *pkt_prev;
  480. int ret = 0;
  481. INIT_LIST_HEAD(&free_list);
  482. list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) {
  483. s64 descd = dd->ipath_sdma_descq_removed - pkt->added;
  484. if (descd < 0)
  485. break;
  486. list_move_tail(&pkt->list, &free_list);
  487. /* one more packet cleaned */
  488. ret++;
  489. }
  490. if (!list_empty(&free_list)) {
  491. u32 counter;
  492. pkt = list_entry(free_list.prev,
  493. struct ipath_user_sdma_pkt, list);
  494. counter = pkt->counter;
  495. ipath_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
  496. ipath_user_sdma_set_complete_counter(pq, counter);
  497. }
  498. return ret;
  499. }
  500. void ipath_user_sdma_queue_destroy(struct ipath_user_sdma_queue *pq)
  501. {
  502. if (!pq)
  503. return;
  504. kmem_cache_destroy(pq->pkt_slab);
  505. dma_pool_destroy(pq->header_cache);
  506. kfree(pq);
  507. }
  508. /* clean descriptor queue, returns > 0 if some elements cleaned */
  509. static int ipath_user_sdma_hwqueue_clean(struct ipath_devdata *dd)
  510. {
  511. int ret;
  512. unsigned long flags;
  513. spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
  514. ret = ipath_sdma_make_progress(dd);
  515. spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
  516. return ret;
  517. }
  518. /* we're in close, drain packets so that we can cleanup successfully... */
  519. void ipath_user_sdma_queue_drain(struct ipath_devdata *dd,
  520. struct ipath_user_sdma_queue *pq)
  521. {
  522. int i;
  523. if (!pq)
  524. return;
  525. for (i = 0; i < 100; i++) {
  526. mutex_lock(&pq->lock);
  527. if (list_empty(&pq->sent)) {
  528. mutex_unlock(&pq->lock);
  529. break;
  530. }
  531. ipath_user_sdma_hwqueue_clean(dd);
  532. ipath_user_sdma_queue_clean(dd, pq);
  533. mutex_unlock(&pq->lock);
  534. msleep(10);
  535. }
  536. if (!list_empty(&pq->sent)) {
  537. struct list_head free_list;
  538. printk(KERN_INFO "drain: lists not empty: forcing!\n");
  539. INIT_LIST_HEAD(&free_list);
  540. mutex_lock(&pq->lock);
  541. list_splice_init(&pq->sent, &free_list);
  542. ipath_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
  543. mutex_unlock(&pq->lock);
  544. }
  545. }
  546. static inline __le64 ipath_sdma_make_desc0(struct ipath_devdata *dd,
  547. u64 addr, u64 dwlen, u64 dwoffset)
  548. {
  549. return cpu_to_le64(/* SDmaPhyAddr[31:0] */
  550. ((addr & 0xfffffffcULL) << 32) |
  551. /* SDmaGeneration[1:0] */
  552. ((dd->ipath_sdma_generation & 3ULL) << 30) |
  553. /* SDmaDwordCount[10:0] */
  554. ((dwlen & 0x7ffULL) << 16) |
  555. /* SDmaBufOffset[12:2] */
  556. (dwoffset & 0x7ffULL));
  557. }
  558. static inline __le64 ipath_sdma_make_first_desc0(__le64 descq)
  559. {
  560. return descq | cpu_to_le64(1ULL << 12);
  561. }
  562. static inline __le64 ipath_sdma_make_last_desc0(__le64 descq)
  563. {
  564. /* last */ /* dma head */
  565. return descq | cpu_to_le64(1ULL << 11 | 1ULL << 13);
  566. }
  567. static inline __le64 ipath_sdma_make_desc1(u64 addr)
  568. {
  569. /* SDmaPhyAddr[47:32] */
  570. return cpu_to_le64(addr >> 32);
  571. }
  572. static void ipath_user_sdma_send_frag(struct ipath_devdata *dd,
  573. struct ipath_user_sdma_pkt *pkt, int idx,
  574. unsigned ofs, u16 tail)
  575. {
  576. const u64 addr = (u64) pkt->addr[idx].addr +
  577. (u64) pkt->addr[idx].offset;
  578. const u64 dwlen = (u64) pkt->addr[idx].length / 4;
  579. __le64 *descqp;
  580. __le64 descq0;
  581. descqp = &dd->ipath_sdma_descq[tail].qw[0];
  582. descq0 = ipath_sdma_make_desc0(dd, addr, dwlen, ofs);
  583. if (idx == 0)
  584. descq0 = ipath_sdma_make_first_desc0(descq0);
  585. if (idx == pkt->naddr - 1)
  586. descq0 = ipath_sdma_make_last_desc0(descq0);
  587. descqp[0] = descq0;
  588. descqp[1] = ipath_sdma_make_desc1(addr);
  589. }
  590. /* pq->lock must be held, get packets on the wire... */
  591. static int ipath_user_sdma_push_pkts(struct ipath_devdata *dd,
  592. struct ipath_user_sdma_queue *pq,
  593. struct list_head *pktlist)
  594. {
  595. int ret = 0;
  596. unsigned long flags;
  597. u16 tail;
  598. if (list_empty(pktlist))
  599. return 0;
  600. if (unlikely(!(dd->ipath_flags & IPATH_LINKACTIVE)))
  601. return -ECOMM;
  602. spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
  603. if (unlikely(dd->ipath_sdma_status & IPATH_SDMA_ABORT_MASK)) {
  604. ret = -ECOMM;
  605. goto unlock;
  606. }
  607. tail = dd->ipath_sdma_descq_tail;
  608. while (!list_empty(pktlist)) {
  609. struct ipath_user_sdma_pkt *pkt =
  610. list_entry(pktlist->next, struct ipath_user_sdma_pkt,
  611. list);
  612. int i;
  613. unsigned ofs = 0;
  614. u16 dtail = tail;
  615. if (pkt->naddr > ipath_sdma_descq_freecnt(dd))
  616. goto unlock_check_tail;
  617. for (i = 0; i < pkt->naddr; i++) {
  618. ipath_user_sdma_send_frag(dd, pkt, i, ofs, tail);
  619. ofs += pkt->addr[i].length >> 2;
  620. if (++tail == dd->ipath_sdma_descq_cnt) {
  621. tail = 0;
  622. ++dd->ipath_sdma_generation;
  623. }
  624. }
  625. if ((ofs<<2) > dd->ipath_ibmaxlen) {
  626. ipath_dbg("packet size %X > ibmax %X, fail\n",
  627. ofs<<2, dd->ipath_ibmaxlen);
  628. ret = -EMSGSIZE;
  629. goto unlock;
  630. }
  631. /*
  632. * if the packet is >= 2KB mtu equivalent, we have to use
  633. * the large buffers, and have to mark each descriptor as
  634. * part of a large buffer packet.
  635. */
  636. if (ofs >= IPATH_SMALLBUF_DWORDS) {
  637. for (i = 0; i < pkt->naddr; i++) {
  638. dd->ipath_sdma_descq[dtail].qw[0] |=
  639. cpu_to_le64(1ULL << 14);
  640. if (++dtail == dd->ipath_sdma_descq_cnt)
  641. dtail = 0;
  642. }
  643. }
  644. dd->ipath_sdma_descq_added += pkt->naddr;
  645. pkt->added = dd->ipath_sdma_descq_added;
  646. list_move_tail(&pkt->list, &pq->sent);
  647. ret++;
  648. }
  649. unlock_check_tail:
  650. /* advance the tail on the chip if necessary */
  651. if (dd->ipath_sdma_descq_tail != tail) {
  652. wmb();
  653. ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, tail);
  654. dd->ipath_sdma_descq_tail = tail;
  655. }
  656. unlock:
  657. spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
  658. return ret;
  659. }
  660. int ipath_user_sdma_writev(struct ipath_devdata *dd,
  661. struct ipath_user_sdma_queue *pq,
  662. const struct iovec *iov,
  663. unsigned long dim)
  664. {
  665. int ret = 0;
  666. struct list_head list;
  667. int npkts = 0;
  668. INIT_LIST_HEAD(&list);
  669. mutex_lock(&pq->lock);
  670. if (dd->ipath_sdma_descq_added != dd->ipath_sdma_descq_removed) {
  671. ipath_user_sdma_hwqueue_clean(dd);
  672. ipath_user_sdma_queue_clean(dd, pq);
  673. }
  674. while (dim) {
  675. const int mxp = 8;
  676. ret = ipath_user_sdma_queue_pkts(dd, pq, &list, iov, dim, mxp);
  677. if (ret <= 0)
  678. goto done_unlock;
  679. else {
  680. dim -= ret;
  681. iov += ret;
  682. }
  683. /* force packets onto the sdma hw queue... */
  684. if (!list_empty(&list)) {
  685. /*
  686. * lazily clean hw queue. the 4 is a guess of about
  687. * how many sdma descriptors a packet will take (it
  688. * doesn't have to be perfect).
  689. */
  690. if (ipath_sdma_descq_freecnt(dd) < ret * 4) {
  691. ipath_user_sdma_hwqueue_clean(dd);
  692. ipath_user_sdma_queue_clean(dd, pq);
  693. }
  694. ret = ipath_user_sdma_push_pkts(dd, pq, &list);
  695. if (ret < 0)
  696. goto done_unlock;
  697. else {
  698. npkts += ret;
  699. pq->counter += ret;
  700. if (!list_empty(&list))
  701. goto done_unlock;
  702. }
  703. }
  704. }
  705. done_unlock:
  706. if (!list_empty(&list))
  707. ipath_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &list);
  708. mutex_unlock(&pq->lock);
  709. return (ret < 0) ? ret : npkts;
  710. }
  711. int ipath_user_sdma_make_progress(struct ipath_devdata *dd,
  712. struct ipath_user_sdma_queue *pq)
  713. {
  714. int ret = 0;
  715. mutex_lock(&pq->lock);
  716. ipath_user_sdma_hwqueue_clean(dd);
  717. ret = ipath_user_sdma_queue_clean(dd, pq);
  718. mutex_unlock(&pq->lock);
  719. return ret;
  720. }
  721. u32 ipath_user_sdma_complete_counter(const struct ipath_user_sdma_queue *pq)
  722. {
  723. return pq->sent_counter;
  724. }
  725. u32 ipath_user_sdma_inflight_counter(struct ipath_user_sdma_queue *pq)
  726. {
  727. return pq->counter;
  728. }