ipmmu-vmsa.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887
  1. /*
  2. * IPMMU VMSA
  3. *
  4. * Copyright (C) 2014 Renesas Electronics Corporation
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License as published by
  8. * the Free Software Foundation; version 2 of the License.
  9. */
  10. #include <linux/delay.h>
  11. #include <linux/dma-mapping.h>
  12. #include <linux/err.h>
  13. #include <linux/export.h>
  14. #include <linux/interrupt.h>
  15. #include <linux/io.h>
  16. #include <linux/iommu.h>
  17. #include <linux/module.h>
  18. #include <linux/of.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/sizes.h>
  21. #include <linux/slab.h>
  22. #include <asm/dma-iommu.h>
  23. #include <asm/pgalloc.h>
  24. #include "io-pgtable.h"
  25. struct ipmmu_vmsa_device {
  26. struct device *dev;
  27. void __iomem *base;
  28. struct list_head list;
  29. unsigned int num_utlbs;
  30. struct dma_iommu_mapping *mapping;
  31. };
  32. struct ipmmu_vmsa_domain {
  33. struct ipmmu_vmsa_device *mmu;
  34. struct iommu_domain io_domain;
  35. struct io_pgtable_cfg cfg;
  36. struct io_pgtable_ops *iop;
  37. unsigned int context_id;
  38. spinlock_t lock; /* Protects mappings */
  39. };
  40. struct ipmmu_vmsa_archdata {
  41. struct ipmmu_vmsa_device *mmu;
  42. unsigned int *utlbs;
  43. unsigned int num_utlbs;
  44. };
  45. static DEFINE_SPINLOCK(ipmmu_devices_lock);
  46. static LIST_HEAD(ipmmu_devices);
  47. static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
  48. {
  49. return container_of(dom, struct ipmmu_vmsa_domain, io_domain);
  50. }
  51. #define TLB_LOOP_TIMEOUT 100 /* 100us */
  52. /* -----------------------------------------------------------------------------
  53. * Registers Definition
  54. */
  55. #define IM_NS_ALIAS_OFFSET 0x800
  56. #define IM_CTX_SIZE 0x40
  57. #define IMCTR 0x0000
  58. #define IMCTR_TRE (1 << 17)
  59. #define IMCTR_AFE (1 << 16)
  60. #define IMCTR_RTSEL_MASK (3 << 4)
  61. #define IMCTR_RTSEL_SHIFT 4
  62. #define IMCTR_TREN (1 << 3)
  63. #define IMCTR_INTEN (1 << 2)
  64. #define IMCTR_FLUSH (1 << 1)
  65. #define IMCTR_MMUEN (1 << 0)
  66. #define IMCAAR 0x0004
  67. #define IMTTBCR 0x0008
  68. #define IMTTBCR_EAE (1 << 31)
  69. #define IMTTBCR_PMB (1 << 30)
  70. #define IMTTBCR_SH1_NON_SHAREABLE (0 << 28)
  71. #define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28)
  72. #define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28)
  73. #define IMTTBCR_SH1_MASK (3 << 28)
  74. #define IMTTBCR_ORGN1_NC (0 << 26)
  75. #define IMTTBCR_ORGN1_WB_WA (1 << 26)
  76. #define IMTTBCR_ORGN1_WT (2 << 26)
  77. #define IMTTBCR_ORGN1_WB (3 << 26)
  78. #define IMTTBCR_ORGN1_MASK (3 << 26)
  79. #define IMTTBCR_IRGN1_NC (0 << 24)
  80. #define IMTTBCR_IRGN1_WB_WA (1 << 24)
  81. #define IMTTBCR_IRGN1_WT (2 << 24)
  82. #define IMTTBCR_IRGN1_WB (3 << 24)
  83. #define IMTTBCR_IRGN1_MASK (3 << 24)
  84. #define IMTTBCR_TSZ1_MASK (7 << 16)
  85. #define IMTTBCR_TSZ1_SHIFT 16
  86. #define IMTTBCR_SH0_NON_SHAREABLE (0 << 12)
  87. #define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12)
  88. #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12)
  89. #define IMTTBCR_SH0_MASK (3 << 12)
  90. #define IMTTBCR_ORGN0_NC (0 << 10)
  91. #define IMTTBCR_ORGN0_WB_WA (1 << 10)
  92. #define IMTTBCR_ORGN0_WT (2 << 10)
  93. #define IMTTBCR_ORGN0_WB (3 << 10)
  94. #define IMTTBCR_ORGN0_MASK (3 << 10)
  95. #define IMTTBCR_IRGN0_NC (0 << 8)
  96. #define IMTTBCR_IRGN0_WB_WA (1 << 8)
  97. #define IMTTBCR_IRGN0_WT (2 << 8)
  98. #define IMTTBCR_IRGN0_WB (3 << 8)
  99. #define IMTTBCR_IRGN0_MASK (3 << 8)
  100. #define IMTTBCR_SL0_LVL_2 (0 << 4)
  101. #define IMTTBCR_SL0_LVL_1 (1 << 4)
  102. #define IMTTBCR_TSZ0_MASK (7 << 0)
  103. #define IMTTBCR_TSZ0_SHIFT O
  104. #define IMBUSCR 0x000c
  105. #define IMBUSCR_DVM (1 << 2)
  106. #define IMBUSCR_BUSSEL_SYS (0 << 0)
  107. #define IMBUSCR_BUSSEL_CCI (1 << 0)
  108. #define IMBUSCR_BUSSEL_IMCAAR (2 << 0)
  109. #define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0)
  110. #define IMBUSCR_BUSSEL_MASK (3 << 0)
  111. #define IMTTLBR0 0x0010
  112. #define IMTTUBR0 0x0014
  113. #define IMTTLBR1 0x0018
  114. #define IMTTUBR1 0x001c
  115. #define IMSTR 0x0020
  116. #define IMSTR_ERRLVL_MASK (3 << 12)
  117. #define IMSTR_ERRLVL_SHIFT 12
  118. #define IMSTR_ERRCODE_TLB_FORMAT (1 << 8)
  119. #define IMSTR_ERRCODE_ACCESS_PERM (4 << 8)
  120. #define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8)
  121. #define IMSTR_ERRCODE_MASK (7 << 8)
  122. #define IMSTR_MHIT (1 << 4)
  123. #define IMSTR_ABORT (1 << 2)
  124. #define IMSTR_PF (1 << 1)
  125. #define IMSTR_TF (1 << 0)
  126. #define IMMAIR0 0x0028
  127. #define IMMAIR1 0x002c
  128. #define IMMAIR_ATTR_MASK 0xff
  129. #define IMMAIR_ATTR_DEVICE 0x04
  130. #define IMMAIR_ATTR_NC 0x44
  131. #define IMMAIR_ATTR_WBRWA 0xff
  132. #define IMMAIR_ATTR_SHIFT(n) ((n) << 3)
  133. #define IMMAIR_ATTR_IDX_NC 0
  134. #define IMMAIR_ATTR_IDX_WBRWA 1
  135. #define IMMAIR_ATTR_IDX_DEV 2
  136. #define IMEAR 0x0030
  137. #define IMPCTR 0x0200
  138. #define IMPSTR 0x0208
  139. #define IMPEAR 0x020c
  140. #define IMPMBA(n) (0x0280 + ((n) * 4))
  141. #define IMPMBD(n) (0x02c0 + ((n) * 4))
  142. #define IMUCTR(n) (0x0300 + ((n) * 16))
  143. #define IMUCTR_FIXADDEN (1 << 31)
  144. #define IMUCTR_FIXADD_MASK (0xff << 16)
  145. #define IMUCTR_FIXADD_SHIFT 16
  146. #define IMUCTR_TTSEL_MMU(n) ((n) << 4)
  147. #define IMUCTR_TTSEL_PMB (8 << 4)
  148. #define IMUCTR_TTSEL_MASK (15 << 4)
  149. #define IMUCTR_FLUSH (1 << 1)
  150. #define IMUCTR_MMUEN (1 << 0)
  151. #define IMUASID(n) (0x0308 + ((n) * 16))
  152. #define IMUASID_ASID8_MASK (0xff << 8)
  153. #define IMUASID_ASID8_SHIFT 8
  154. #define IMUASID_ASID0_MASK (0xff << 0)
  155. #define IMUASID_ASID0_SHIFT 0
  156. /* -----------------------------------------------------------------------------
  157. * Read/Write Access
  158. */
  159. static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)
  160. {
  161. return ioread32(mmu->base + offset);
  162. }
  163. static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,
  164. u32 data)
  165. {
  166. iowrite32(data, mmu->base + offset);
  167. }
  168. static u32 ipmmu_ctx_read(struct ipmmu_vmsa_domain *domain, unsigned int reg)
  169. {
  170. return ipmmu_read(domain->mmu, domain->context_id * IM_CTX_SIZE + reg);
  171. }
  172. static void ipmmu_ctx_write(struct ipmmu_vmsa_domain *domain, unsigned int reg,
  173. u32 data)
  174. {
  175. ipmmu_write(domain->mmu, domain->context_id * IM_CTX_SIZE + reg, data);
  176. }
  177. /* -----------------------------------------------------------------------------
  178. * TLB and microTLB Management
  179. */
  180. /* Wait for any pending TLB invalidations to complete */
  181. static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain)
  182. {
  183. unsigned int count = 0;
  184. while (ipmmu_ctx_read(domain, IMCTR) & IMCTR_FLUSH) {
  185. cpu_relax();
  186. if (++count == TLB_LOOP_TIMEOUT) {
  187. dev_err_ratelimited(domain->mmu->dev,
  188. "TLB sync timed out -- MMU may be deadlocked\n");
  189. return;
  190. }
  191. udelay(1);
  192. }
  193. }
  194. static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain)
  195. {
  196. u32 reg;
  197. reg = ipmmu_ctx_read(domain, IMCTR);
  198. reg |= IMCTR_FLUSH;
  199. ipmmu_ctx_write(domain, IMCTR, reg);
  200. ipmmu_tlb_sync(domain);
  201. }
  202. /*
  203. * Enable MMU translation for the microTLB.
  204. */
  205. static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain,
  206. unsigned int utlb)
  207. {
  208. struct ipmmu_vmsa_device *mmu = domain->mmu;
  209. /*
  210. * TODO: Reference-count the microTLB as several bus masters can be
  211. * connected to the same microTLB.
  212. */
  213. /* TODO: What should we set the ASID to ? */
  214. ipmmu_write(mmu, IMUASID(utlb), 0);
  215. /* TODO: Do we need to flush the microTLB ? */
  216. ipmmu_write(mmu, IMUCTR(utlb),
  217. IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH |
  218. IMUCTR_MMUEN);
  219. }
  220. /*
  221. * Disable MMU translation for the microTLB.
  222. */
  223. static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain,
  224. unsigned int utlb)
  225. {
  226. struct ipmmu_vmsa_device *mmu = domain->mmu;
  227. ipmmu_write(mmu, IMUCTR(utlb), 0);
  228. }
  229. static void ipmmu_tlb_flush_all(void *cookie)
  230. {
  231. struct ipmmu_vmsa_domain *domain = cookie;
  232. ipmmu_tlb_invalidate(domain);
  233. }
  234. static void ipmmu_tlb_add_flush(unsigned long iova, size_t size,
  235. size_t granule, bool leaf, void *cookie)
  236. {
  237. /* The hardware doesn't support selective TLB flush. */
  238. }
  239. static struct iommu_gather_ops ipmmu_gather_ops = {
  240. .tlb_flush_all = ipmmu_tlb_flush_all,
  241. .tlb_add_flush = ipmmu_tlb_add_flush,
  242. .tlb_sync = ipmmu_tlb_flush_all,
  243. };
  244. /* -----------------------------------------------------------------------------
  245. * Domain/Context Management
  246. */
  247. static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
  248. {
  249. u64 ttbr;
  250. /*
  251. * Allocate the page table operations.
  252. *
  253. * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory
  254. * access, Long-descriptor format" that the NStable bit being set in a
  255. * table descriptor will result in the NStable and NS bits of all child
  256. * entries being ignored and considered as being set. The IPMMU seems
  257. * not to comply with this, as it generates a secure access page fault
  258. * if any of the NStable and NS bits isn't set when running in
  259. * non-secure mode.
  260. */
  261. domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS;
  262. domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
  263. domain->cfg.ias = 32;
  264. domain->cfg.oas = 40;
  265. domain->cfg.tlb = &ipmmu_gather_ops;
  266. /*
  267. * TODO: Add support for coherent walk through CCI with DVM and remove
  268. * cache handling. For now, delegate it to the io-pgtable code.
  269. */
  270. domain->cfg.iommu_dev = domain->mmu->dev;
  271. domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
  272. domain);
  273. if (!domain->iop)
  274. return -EINVAL;
  275. /*
  276. * TODO: When adding support for multiple contexts, find an unused
  277. * context.
  278. */
  279. domain->context_id = 0;
  280. /* TTBR0 */
  281. ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0];
  282. ipmmu_ctx_write(domain, IMTTLBR0, ttbr);
  283. ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32);
  284. /*
  285. * TTBCR
  286. * We use long descriptors with inner-shareable WBWA tables and allocate
  287. * the whole 32-bit VA space to TTBR0.
  288. */
  289. ipmmu_ctx_write(domain, IMTTBCR, IMTTBCR_EAE |
  290. IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA |
  291. IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1);
  292. /* MAIR0 */
  293. ipmmu_ctx_write(domain, IMMAIR0, domain->cfg.arm_lpae_s1_cfg.mair[0]);
  294. /* IMBUSCR */
  295. ipmmu_ctx_write(domain, IMBUSCR,
  296. ipmmu_ctx_read(domain, IMBUSCR) &
  297. ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK));
  298. /*
  299. * IMSTR
  300. * Clear all interrupt flags.
  301. */
  302. ipmmu_ctx_write(domain, IMSTR, ipmmu_ctx_read(domain, IMSTR));
  303. /*
  304. * IMCTR
  305. * Enable the MMU and interrupt generation. The long-descriptor
  306. * translation table format doesn't use TEX remapping. Don't enable AF
  307. * software management as we have no use for it. Flush the TLB as
  308. * required when modifying the context registers.
  309. */
  310. ipmmu_ctx_write(domain, IMCTR, IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN);
  311. return 0;
  312. }
  313. static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
  314. {
  315. /*
  316. * Disable the context. Flush the TLB as required when modifying the
  317. * context registers.
  318. *
  319. * TODO: Is TLB flush really needed ?
  320. */
  321. ipmmu_ctx_write(domain, IMCTR, IMCTR_FLUSH);
  322. ipmmu_tlb_sync(domain);
  323. }
  324. /* -----------------------------------------------------------------------------
  325. * Fault Handling
  326. */
  327. static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain)
  328. {
  329. const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF;
  330. struct ipmmu_vmsa_device *mmu = domain->mmu;
  331. u32 status;
  332. u32 iova;
  333. status = ipmmu_ctx_read(domain, IMSTR);
  334. if (!(status & err_mask))
  335. return IRQ_NONE;
  336. iova = ipmmu_ctx_read(domain, IMEAR);
  337. /*
  338. * Clear the error status flags. Unlike traditional interrupt flag
  339. * registers that must be cleared by writing 1, this status register
  340. * seems to require 0. The error address register must be read before,
  341. * otherwise its value will be 0.
  342. */
  343. ipmmu_ctx_write(domain, IMSTR, 0);
  344. /* Log fatal errors. */
  345. if (status & IMSTR_MHIT)
  346. dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%08x\n",
  347. iova);
  348. if (status & IMSTR_ABORT)
  349. dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%08x\n",
  350. iova);
  351. if (!(status & (IMSTR_PF | IMSTR_TF)))
  352. return IRQ_NONE;
  353. /*
  354. * Try to handle page faults and translation faults.
  355. *
  356. * TODO: We need to look up the faulty device based on the I/O VA. Use
  357. * the IOMMU device for now.
  358. */
  359. if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))
  360. return IRQ_HANDLED;
  361. dev_err_ratelimited(mmu->dev,
  362. "Unhandled fault: status 0x%08x iova 0x%08x\n",
  363. status, iova);
  364. return IRQ_HANDLED;
  365. }
  366. static irqreturn_t ipmmu_irq(int irq, void *dev)
  367. {
  368. struct ipmmu_vmsa_device *mmu = dev;
  369. struct iommu_domain *io_domain;
  370. struct ipmmu_vmsa_domain *domain;
  371. if (!mmu->mapping)
  372. return IRQ_NONE;
  373. io_domain = mmu->mapping->domain;
  374. domain = to_vmsa_domain(io_domain);
  375. return ipmmu_domain_irq(domain);
  376. }
  377. /* -----------------------------------------------------------------------------
  378. * IOMMU Operations
  379. */
  380. static struct iommu_domain *ipmmu_domain_alloc(unsigned type)
  381. {
  382. struct ipmmu_vmsa_domain *domain;
  383. if (type != IOMMU_DOMAIN_UNMANAGED)
  384. return NULL;
  385. domain = kzalloc(sizeof(*domain), GFP_KERNEL);
  386. if (!domain)
  387. return NULL;
  388. spin_lock_init(&domain->lock);
  389. return &domain->io_domain;
  390. }
  391. static void ipmmu_domain_free(struct iommu_domain *io_domain)
  392. {
  393. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  394. /*
  395. * Free the domain resources. We assume that all devices have already
  396. * been detached.
  397. */
  398. ipmmu_domain_destroy_context(domain);
  399. free_io_pgtable_ops(domain->iop);
  400. kfree(domain);
  401. }
  402. static int ipmmu_attach_device(struct iommu_domain *io_domain,
  403. struct device *dev)
  404. {
  405. struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
  406. struct ipmmu_vmsa_device *mmu = archdata->mmu;
  407. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  408. unsigned long flags;
  409. unsigned int i;
  410. int ret = 0;
  411. if (!mmu) {
  412. dev_err(dev, "Cannot attach to IPMMU\n");
  413. return -ENXIO;
  414. }
  415. spin_lock_irqsave(&domain->lock, flags);
  416. if (!domain->mmu) {
  417. /* The domain hasn't been used yet, initialize it. */
  418. domain->mmu = mmu;
  419. ret = ipmmu_domain_init_context(domain);
  420. } else if (domain->mmu != mmu) {
  421. /*
  422. * Something is wrong, we can't attach two devices using
  423. * different IOMMUs to the same domain.
  424. */
  425. dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
  426. dev_name(mmu->dev), dev_name(domain->mmu->dev));
  427. ret = -EINVAL;
  428. }
  429. spin_unlock_irqrestore(&domain->lock, flags);
  430. if (ret < 0)
  431. return ret;
  432. for (i = 0; i < archdata->num_utlbs; ++i)
  433. ipmmu_utlb_enable(domain, archdata->utlbs[i]);
  434. return 0;
  435. }
  436. static void ipmmu_detach_device(struct iommu_domain *io_domain,
  437. struct device *dev)
  438. {
  439. struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
  440. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  441. unsigned int i;
  442. for (i = 0; i < archdata->num_utlbs; ++i)
  443. ipmmu_utlb_disable(domain, archdata->utlbs[i]);
  444. /*
  445. * TODO: Optimize by disabling the context when no device is attached.
  446. */
  447. }
  448. static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
  449. phys_addr_t paddr, size_t size, int prot)
  450. {
  451. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  452. if (!domain)
  453. return -ENODEV;
  454. return domain->iop->map(domain->iop, iova, paddr, size, prot);
  455. }
  456. static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
  457. size_t size)
  458. {
  459. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  460. return domain->iop->unmap(domain->iop, iova, size);
  461. }
  462. static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
  463. dma_addr_t iova)
  464. {
  465. struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
  466. /* TODO: Is locking needed ? */
  467. return domain->iop->iova_to_phys(domain->iop, iova);
  468. }
  469. static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev,
  470. unsigned int *utlbs, unsigned int num_utlbs)
  471. {
  472. unsigned int i;
  473. for (i = 0; i < num_utlbs; ++i) {
  474. struct of_phandle_args args;
  475. int ret;
  476. ret = of_parse_phandle_with_args(dev->of_node, "iommus",
  477. "#iommu-cells", i, &args);
  478. if (ret < 0)
  479. return ret;
  480. of_node_put(args.np);
  481. if (args.np != mmu->dev->of_node || args.args_count != 1)
  482. return -EINVAL;
  483. utlbs[i] = args.args[0];
  484. }
  485. return 0;
  486. }
  487. static int ipmmu_add_device(struct device *dev)
  488. {
  489. struct ipmmu_vmsa_archdata *archdata;
  490. struct ipmmu_vmsa_device *mmu;
  491. struct iommu_group *group = NULL;
  492. unsigned int *utlbs;
  493. unsigned int i;
  494. int num_utlbs;
  495. int ret = -ENODEV;
  496. if (dev->archdata.iommu) {
  497. dev_warn(dev, "IOMMU driver already assigned to device %s\n",
  498. dev_name(dev));
  499. return -EINVAL;
  500. }
  501. /* Find the master corresponding to the device. */
  502. num_utlbs = of_count_phandle_with_args(dev->of_node, "iommus",
  503. "#iommu-cells");
  504. if (num_utlbs < 0)
  505. return -ENODEV;
  506. utlbs = kcalloc(num_utlbs, sizeof(*utlbs), GFP_KERNEL);
  507. if (!utlbs)
  508. return -ENOMEM;
  509. spin_lock(&ipmmu_devices_lock);
  510. list_for_each_entry(mmu, &ipmmu_devices, list) {
  511. ret = ipmmu_find_utlbs(mmu, dev, utlbs, num_utlbs);
  512. if (!ret) {
  513. /*
  514. * TODO Take a reference to the MMU to protect
  515. * against device removal.
  516. */
  517. break;
  518. }
  519. }
  520. spin_unlock(&ipmmu_devices_lock);
  521. if (ret < 0)
  522. goto error;
  523. for (i = 0; i < num_utlbs; ++i) {
  524. if (utlbs[i] >= mmu->num_utlbs) {
  525. ret = -EINVAL;
  526. goto error;
  527. }
  528. }
  529. /* Create a device group and add the device to it. */
  530. group = iommu_group_alloc();
  531. if (IS_ERR(group)) {
  532. dev_err(dev, "Failed to allocate IOMMU group\n");
  533. ret = PTR_ERR(group);
  534. goto error;
  535. }
  536. ret = iommu_group_add_device(group, dev);
  537. iommu_group_put(group);
  538. if (ret < 0) {
  539. dev_err(dev, "Failed to add device to IPMMU group\n");
  540. group = NULL;
  541. goto error;
  542. }
  543. archdata = kzalloc(sizeof(*archdata), GFP_KERNEL);
  544. if (!archdata) {
  545. ret = -ENOMEM;
  546. goto error;
  547. }
  548. archdata->mmu = mmu;
  549. archdata->utlbs = utlbs;
  550. archdata->num_utlbs = num_utlbs;
  551. dev->archdata.iommu = archdata;
  552. /*
  553. * Create the ARM mapping, used by the ARM DMA mapping core to allocate
  554. * VAs. This will allocate a corresponding IOMMU domain.
  555. *
  556. * TODO:
  557. * - Create one mapping per context (TLB).
  558. * - Make the mapping size configurable ? We currently use a 2GB mapping
  559. * at a 1GB offset to ensure that NULL VAs will fault.
  560. */
  561. if (!mmu->mapping) {
  562. struct dma_iommu_mapping *mapping;
  563. mapping = arm_iommu_create_mapping(&platform_bus_type,
  564. SZ_1G, SZ_2G);
  565. if (IS_ERR(mapping)) {
  566. dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");
  567. ret = PTR_ERR(mapping);
  568. goto error;
  569. }
  570. mmu->mapping = mapping;
  571. }
  572. /* Attach the ARM VA mapping to the device. */
  573. ret = arm_iommu_attach_device(dev, mmu->mapping);
  574. if (ret < 0) {
  575. dev_err(dev, "Failed to attach device to VA mapping\n");
  576. goto error;
  577. }
  578. return 0;
  579. error:
  580. arm_iommu_release_mapping(mmu->mapping);
  581. kfree(dev->archdata.iommu);
  582. kfree(utlbs);
  583. dev->archdata.iommu = NULL;
  584. if (!IS_ERR_OR_NULL(group))
  585. iommu_group_remove_device(dev);
  586. return ret;
  587. }
  588. static void ipmmu_remove_device(struct device *dev)
  589. {
  590. struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
  591. arm_iommu_detach_device(dev);
  592. iommu_group_remove_device(dev);
  593. kfree(archdata->utlbs);
  594. kfree(archdata);
  595. dev->archdata.iommu = NULL;
  596. }
  597. static const struct iommu_ops ipmmu_ops = {
  598. .domain_alloc = ipmmu_domain_alloc,
  599. .domain_free = ipmmu_domain_free,
  600. .attach_dev = ipmmu_attach_device,
  601. .detach_dev = ipmmu_detach_device,
  602. .map = ipmmu_map,
  603. .unmap = ipmmu_unmap,
  604. .map_sg = default_iommu_map_sg,
  605. .iova_to_phys = ipmmu_iova_to_phys,
  606. .add_device = ipmmu_add_device,
  607. .remove_device = ipmmu_remove_device,
  608. .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
  609. };
  610. /* -----------------------------------------------------------------------------
  611. * Probe/remove and init
  612. */
  613. static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)
  614. {
  615. unsigned int i;
  616. /* Disable all contexts. */
  617. for (i = 0; i < 4; ++i)
  618. ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0);
  619. }
  620. static int ipmmu_probe(struct platform_device *pdev)
  621. {
  622. struct ipmmu_vmsa_device *mmu;
  623. struct resource *res;
  624. int irq;
  625. int ret;
  626. if (!IS_ENABLED(CONFIG_OF) && !pdev->dev.platform_data) {
  627. dev_err(&pdev->dev, "missing platform data\n");
  628. return -EINVAL;
  629. }
  630. mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);
  631. if (!mmu) {
  632. dev_err(&pdev->dev, "cannot allocate device data\n");
  633. return -ENOMEM;
  634. }
  635. mmu->dev = &pdev->dev;
  636. mmu->num_utlbs = 32;
  637. /* Map I/O memory and request IRQ. */
  638. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  639. mmu->base = devm_ioremap_resource(&pdev->dev, res);
  640. if (IS_ERR(mmu->base))
  641. return PTR_ERR(mmu->base);
  642. /*
  643. * The IPMMU has two register banks, for secure and non-secure modes.
  644. * The bank mapped at the beginning of the IPMMU address space
  645. * corresponds to the running mode of the CPU. When running in secure
  646. * mode the non-secure register bank is also available at an offset.
  647. *
  648. * Secure mode operation isn't clearly documented and is thus currently
  649. * not implemented in the driver. Furthermore, preliminary tests of
  650. * non-secure operation with the main register bank were not successful.
  651. * Offset the registers base unconditionally to point to the non-secure
  652. * alias space for now.
  653. */
  654. mmu->base += IM_NS_ALIAS_OFFSET;
  655. irq = platform_get_irq(pdev, 0);
  656. if (irq < 0) {
  657. dev_err(&pdev->dev, "no IRQ found\n");
  658. return irq;
  659. }
  660. ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0,
  661. dev_name(&pdev->dev), mmu);
  662. if (ret < 0) {
  663. dev_err(&pdev->dev, "failed to request IRQ %d\n", irq);
  664. return ret;
  665. }
  666. ipmmu_device_reset(mmu);
  667. /*
  668. * We can't create the ARM mapping here as it requires the bus to have
  669. * an IOMMU, which only happens when bus_set_iommu() is called in
  670. * ipmmu_init() after the probe function returns.
  671. */
  672. spin_lock(&ipmmu_devices_lock);
  673. list_add(&mmu->list, &ipmmu_devices);
  674. spin_unlock(&ipmmu_devices_lock);
  675. platform_set_drvdata(pdev, mmu);
  676. return 0;
  677. }
  678. static int ipmmu_remove(struct platform_device *pdev)
  679. {
  680. struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
  681. spin_lock(&ipmmu_devices_lock);
  682. list_del(&mmu->list);
  683. spin_unlock(&ipmmu_devices_lock);
  684. arm_iommu_release_mapping(mmu->mapping);
  685. ipmmu_device_reset(mmu);
  686. return 0;
  687. }
  688. static const struct of_device_id ipmmu_of_ids[] = {
  689. { .compatible = "renesas,ipmmu-vmsa", },
  690. { }
  691. };
  692. static struct platform_driver ipmmu_driver = {
  693. .driver = {
  694. .name = "ipmmu-vmsa",
  695. .of_match_table = of_match_ptr(ipmmu_of_ids),
  696. },
  697. .probe = ipmmu_probe,
  698. .remove = ipmmu_remove,
  699. };
  700. static int __init ipmmu_init(void)
  701. {
  702. int ret;
  703. ret = platform_driver_register(&ipmmu_driver);
  704. if (ret < 0)
  705. return ret;
  706. if (!iommu_present(&platform_bus_type))
  707. bus_set_iommu(&platform_bus_type, &ipmmu_ops);
  708. return 0;
  709. }
  710. static void __exit ipmmu_exit(void)
  711. {
  712. return platform_driver_unregister(&ipmmu_driver);
  713. }
  714. subsys_initcall(ipmmu_init);
  715. module_exit(ipmmu_exit);
  716. MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU");
  717. MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
  718. MODULE_LICENSE("GPL v2");