assigned-dev.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059
  1. /*
  2. * Kernel-based Virtual Machine - device assignment support
  3. *
  4. * Copyright (C) 2010 Red Hat, Inc. and/or its affiliates.
  5. *
  6. * This work is licensed under the terms of the GNU GPL, version 2. See
  7. * the COPYING file in the top-level directory.
  8. *
  9. */
  10. #include <linux/kvm_host.h>
  11. #include <linux/kvm.h>
  12. #include <linux/uaccess.h>
  13. #include <linux/vmalloc.h>
  14. #include <linux/errno.h>
  15. #include <linux/spinlock.h>
  16. #include <linux/pci.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/slab.h>
  19. #include <linux/namei.h>
  20. #include <linux/fs.h>
  21. #include "irq.h"
  22. #include "assigned-dev.h"
  23. #include "trace/events/kvm.h"
  24. struct kvm_assigned_dev_kernel {
  25. struct kvm_irq_ack_notifier ack_notifier;
  26. struct list_head list;
  27. int assigned_dev_id;
  28. int host_segnr;
  29. int host_busnr;
  30. int host_devfn;
  31. unsigned int entries_nr;
  32. int host_irq;
  33. bool host_irq_disabled;
  34. bool pci_2_3;
  35. struct msix_entry *host_msix_entries;
  36. int guest_irq;
  37. struct msix_entry *guest_msix_entries;
  38. unsigned long irq_requested_type;
  39. int irq_source_id;
  40. int flags;
  41. struct pci_dev *dev;
  42. struct kvm *kvm;
  43. spinlock_t intx_lock;
  44. spinlock_t intx_mask_lock;
  45. char irq_name[32];
  46. struct pci_saved_state *pci_saved_state;
  47. };
  48. static struct kvm_assigned_dev_kernel *kvm_find_assigned_dev(struct list_head *head,
  49. int assigned_dev_id)
  50. {
  51. struct kvm_assigned_dev_kernel *match;
  52. list_for_each_entry(match, head, list) {
  53. if (match->assigned_dev_id == assigned_dev_id)
  54. return match;
  55. }
  56. return NULL;
  57. }
  58. static int find_index_from_host_irq(struct kvm_assigned_dev_kernel
  59. *assigned_dev, int irq)
  60. {
  61. int i, index;
  62. struct msix_entry *host_msix_entries;
  63. host_msix_entries = assigned_dev->host_msix_entries;
  64. index = -1;
  65. for (i = 0; i < assigned_dev->entries_nr; i++)
  66. if (irq == host_msix_entries[i].vector) {
  67. index = i;
  68. break;
  69. }
  70. if (index < 0)
  71. printk(KERN_WARNING "Fail to find correlated MSI-X entry!\n");
  72. return index;
  73. }
  74. static irqreturn_t kvm_assigned_dev_intx(int irq, void *dev_id)
  75. {
  76. struct kvm_assigned_dev_kernel *assigned_dev = dev_id;
  77. int ret;
  78. spin_lock(&assigned_dev->intx_lock);
  79. if (pci_check_and_mask_intx(assigned_dev->dev)) {
  80. assigned_dev->host_irq_disabled = true;
  81. ret = IRQ_WAKE_THREAD;
  82. } else
  83. ret = IRQ_NONE;
  84. spin_unlock(&assigned_dev->intx_lock);
  85. return ret;
  86. }
  87. static void
  88. kvm_assigned_dev_raise_guest_irq(struct kvm_assigned_dev_kernel *assigned_dev,
  89. int vector)
  90. {
  91. if (unlikely(assigned_dev->irq_requested_type &
  92. KVM_DEV_IRQ_GUEST_INTX)) {
  93. spin_lock(&assigned_dev->intx_mask_lock);
  94. if (!(assigned_dev->flags & KVM_DEV_ASSIGN_MASK_INTX))
  95. kvm_set_irq(assigned_dev->kvm,
  96. assigned_dev->irq_source_id, vector, 1,
  97. false);
  98. spin_unlock(&assigned_dev->intx_mask_lock);
  99. } else
  100. kvm_set_irq(assigned_dev->kvm, assigned_dev->irq_source_id,
  101. vector, 1, false);
  102. }
  103. static irqreturn_t kvm_assigned_dev_thread_intx(int irq, void *dev_id)
  104. {
  105. struct kvm_assigned_dev_kernel *assigned_dev = dev_id;
  106. if (!(assigned_dev->flags & KVM_DEV_ASSIGN_PCI_2_3)) {
  107. spin_lock_irq(&assigned_dev->intx_lock);
  108. disable_irq_nosync(irq);
  109. assigned_dev->host_irq_disabled = true;
  110. spin_unlock_irq(&assigned_dev->intx_lock);
  111. }
  112. kvm_assigned_dev_raise_guest_irq(assigned_dev,
  113. assigned_dev->guest_irq);
  114. return IRQ_HANDLED;
  115. }
  116. /*
  117. * Deliver an IRQ in an atomic context if we can, or return a failure,
  118. * user can retry in a process context.
  119. * Return value:
  120. * -EWOULDBLOCK - Can't deliver in atomic context: retry in a process context.
  121. * Other values - No need to retry.
  122. */
  123. static int kvm_set_irq_inatomic(struct kvm *kvm, int irq_source_id, u32 irq,
  124. int level)
  125. {
  126. struct kvm_kernel_irq_routing_entry entries[KVM_NR_IRQCHIPS];
  127. struct kvm_kernel_irq_routing_entry *e;
  128. int ret = -EINVAL;
  129. int idx;
  130. trace_kvm_set_irq(irq, level, irq_source_id);
  131. /*
  132. * Injection into either PIC or IOAPIC might need to scan all CPUs,
  133. * which would need to be retried from thread context; when same GSI
  134. * is connected to both PIC and IOAPIC, we'd have to report a
  135. * partial failure here.
  136. * Since there's no easy way to do this, we only support injecting MSI
  137. * which is limited to 1:1 GSI mapping.
  138. */
  139. idx = srcu_read_lock(&kvm->irq_srcu);
  140. if (kvm_irq_map_gsi(kvm, entries, irq) > 0) {
  141. e = &entries[0];
  142. ret = kvm_arch_set_irq_inatomic(e, kvm, irq_source_id,
  143. irq, level);
  144. }
  145. srcu_read_unlock(&kvm->irq_srcu, idx);
  146. return ret;
  147. }
  148. static irqreturn_t kvm_assigned_dev_msi(int irq, void *dev_id)
  149. {
  150. struct kvm_assigned_dev_kernel *assigned_dev = dev_id;
  151. int ret = kvm_set_irq_inatomic(assigned_dev->kvm,
  152. assigned_dev->irq_source_id,
  153. assigned_dev->guest_irq, 1);
  154. return unlikely(ret == -EWOULDBLOCK) ? IRQ_WAKE_THREAD : IRQ_HANDLED;
  155. }
  156. static irqreturn_t kvm_assigned_dev_thread_msi(int irq, void *dev_id)
  157. {
  158. struct kvm_assigned_dev_kernel *assigned_dev = dev_id;
  159. kvm_assigned_dev_raise_guest_irq(assigned_dev,
  160. assigned_dev->guest_irq);
  161. return IRQ_HANDLED;
  162. }
  163. static irqreturn_t kvm_assigned_dev_msix(int irq, void *dev_id)
  164. {
  165. struct kvm_assigned_dev_kernel *assigned_dev = dev_id;
  166. int index = find_index_from_host_irq(assigned_dev, irq);
  167. u32 vector;
  168. int ret = 0;
  169. if (index >= 0) {
  170. vector = assigned_dev->guest_msix_entries[index].vector;
  171. ret = kvm_set_irq_inatomic(assigned_dev->kvm,
  172. assigned_dev->irq_source_id,
  173. vector, 1);
  174. }
  175. return unlikely(ret == -EWOULDBLOCK) ? IRQ_WAKE_THREAD : IRQ_HANDLED;
  176. }
  177. static irqreturn_t kvm_assigned_dev_thread_msix(int irq, void *dev_id)
  178. {
  179. struct kvm_assigned_dev_kernel *assigned_dev = dev_id;
  180. int index = find_index_from_host_irq(assigned_dev, irq);
  181. u32 vector;
  182. if (index >= 0) {
  183. vector = assigned_dev->guest_msix_entries[index].vector;
  184. kvm_assigned_dev_raise_guest_irq(assigned_dev, vector);
  185. }
  186. return IRQ_HANDLED;
  187. }
  188. /* Ack the irq line for an assigned device */
  189. static void kvm_assigned_dev_ack_irq(struct kvm_irq_ack_notifier *kian)
  190. {
  191. struct kvm_assigned_dev_kernel *dev =
  192. container_of(kian, struct kvm_assigned_dev_kernel,
  193. ack_notifier);
  194. kvm_set_irq(dev->kvm, dev->irq_source_id, dev->guest_irq, 0, false);
  195. spin_lock(&dev->intx_mask_lock);
  196. if (!(dev->flags & KVM_DEV_ASSIGN_MASK_INTX)) {
  197. bool reassert = false;
  198. spin_lock_irq(&dev->intx_lock);
  199. /*
  200. * The guest IRQ may be shared so this ack can come from an
  201. * IRQ for another guest device.
  202. */
  203. if (dev->host_irq_disabled) {
  204. if (!(dev->flags & KVM_DEV_ASSIGN_PCI_2_3))
  205. enable_irq(dev->host_irq);
  206. else if (!pci_check_and_unmask_intx(dev->dev))
  207. reassert = true;
  208. dev->host_irq_disabled = reassert;
  209. }
  210. spin_unlock_irq(&dev->intx_lock);
  211. if (reassert)
  212. kvm_set_irq(dev->kvm, dev->irq_source_id,
  213. dev->guest_irq, 1, false);
  214. }
  215. spin_unlock(&dev->intx_mask_lock);
  216. }
  217. static void deassign_guest_irq(struct kvm *kvm,
  218. struct kvm_assigned_dev_kernel *assigned_dev)
  219. {
  220. if (assigned_dev->ack_notifier.gsi != -1)
  221. kvm_unregister_irq_ack_notifier(kvm,
  222. &assigned_dev->ack_notifier);
  223. kvm_set_irq(assigned_dev->kvm, assigned_dev->irq_source_id,
  224. assigned_dev->guest_irq, 0, false);
  225. if (assigned_dev->irq_source_id != -1)
  226. kvm_free_irq_source_id(kvm, assigned_dev->irq_source_id);
  227. assigned_dev->irq_source_id = -1;
  228. assigned_dev->irq_requested_type &= ~(KVM_DEV_IRQ_GUEST_MASK);
  229. }
  230. /* The function implicit hold kvm->lock mutex due to cancel_work_sync() */
  231. static void deassign_host_irq(struct kvm *kvm,
  232. struct kvm_assigned_dev_kernel *assigned_dev)
  233. {
  234. /*
  235. * We disable irq here to prevent further events.
  236. *
  237. * Notice this maybe result in nested disable if the interrupt type is
  238. * INTx, but it's OK for we are going to free it.
  239. *
  240. * If this function is a part of VM destroy, please ensure that till
  241. * now, the kvm state is still legal for probably we also have to wait
  242. * on a currently running IRQ handler.
  243. */
  244. if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX) {
  245. int i;
  246. for (i = 0; i < assigned_dev->entries_nr; i++)
  247. disable_irq(assigned_dev->host_msix_entries[i].vector);
  248. for (i = 0; i < assigned_dev->entries_nr; i++)
  249. free_irq(assigned_dev->host_msix_entries[i].vector,
  250. assigned_dev);
  251. assigned_dev->entries_nr = 0;
  252. kfree(assigned_dev->host_msix_entries);
  253. kfree(assigned_dev->guest_msix_entries);
  254. pci_disable_msix(assigned_dev->dev);
  255. } else {
  256. /* Deal with MSI and INTx */
  257. if ((assigned_dev->irq_requested_type &
  258. KVM_DEV_IRQ_HOST_INTX) &&
  259. (assigned_dev->flags & KVM_DEV_ASSIGN_PCI_2_3)) {
  260. spin_lock_irq(&assigned_dev->intx_lock);
  261. pci_intx(assigned_dev->dev, false);
  262. spin_unlock_irq(&assigned_dev->intx_lock);
  263. synchronize_irq(assigned_dev->host_irq);
  264. } else
  265. disable_irq(assigned_dev->host_irq);
  266. free_irq(assigned_dev->host_irq, assigned_dev);
  267. if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSI)
  268. pci_disable_msi(assigned_dev->dev);
  269. }
  270. assigned_dev->irq_requested_type &= ~(KVM_DEV_IRQ_HOST_MASK);
  271. }
  272. static int kvm_deassign_irq(struct kvm *kvm,
  273. struct kvm_assigned_dev_kernel *assigned_dev,
  274. unsigned long irq_requested_type)
  275. {
  276. unsigned long guest_irq_type, host_irq_type;
  277. if (!irqchip_in_kernel(kvm))
  278. return -EINVAL;
  279. /* no irq assignment to deassign */
  280. if (!assigned_dev->irq_requested_type)
  281. return -ENXIO;
  282. host_irq_type = irq_requested_type & KVM_DEV_IRQ_HOST_MASK;
  283. guest_irq_type = irq_requested_type & KVM_DEV_IRQ_GUEST_MASK;
  284. if (host_irq_type)
  285. deassign_host_irq(kvm, assigned_dev);
  286. if (guest_irq_type)
  287. deassign_guest_irq(kvm, assigned_dev);
  288. return 0;
  289. }
  290. static void kvm_free_assigned_irq(struct kvm *kvm,
  291. struct kvm_assigned_dev_kernel *assigned_dev)
  292. {
  293. kvm_deassign_irq(kvm, assigned_dev, assigned_dev->irq_requested_type);
  294. }
  295. static void kvm_free_assigned_device(struct kvm *kvm,
  296. struct kvm_assigned_dev_kernel
  297. *assigned_dev)
  298. {
  299. kvm_free_assigned_irq(kvm, assigned_dev);
  300. pci_reset_function(assigned_dev->dev);
  301. if (pci_load_and_free_saved_state(assigned_dev->dev,
  302. &assigned_dev->pci_saved_state))
  303. printk(KERN_INFO "%s: Couldn't reload %s saved state\n",
  304. __func__, dev_name(&assigned_dev->dev->dev));
  305. else
  306. pci_restore_state(assigned_dev->dev);
  307. pci_clear_dev_assigned(assigned_dev->dev);
  308. pci_release_regions(assigned_dev->dev);
  309. pci_disable_device(assigned_dev->dev);
  310. pci_dev_put(assigned_dev->dev);
  311. list_del(&assigned_dev->list);
  312. kfree(assigned_dev);
  313. }
  314. void kvm_free_all_assigned_devices(struct kvm *kvm)
  315. {
  316. struct kvm_assigned_dev_kernel *assigned_dev, *tmp;
  317. list_for_each_entry_safe(assigned_dev, tmp,
  318. &kvm->arch.assigned_dev_head, list) {
  319. kvm_free_assigned_device(kvm, assigned_dev);
  320. }
  321. }
  322. static int assigned_device_enable_host_intx(struct kvm *kvm,
  323. struct kvm_assigned_dev_kernel *dev)
  324. {
  325. irq_handler_t irq_handler;
  326. unsigned long flags;
  327. dev->host_irq = dev->dev->irq;
  328. /*
  329. * We can only share the IRQ line with other host devices if we are
  330. * able to disable the IRQ source at device-level - independently of
  331. * the guest driver. Otherwise host devices may suffer from unbounded
  332. * IRQ latencies when the guest keeps the line asserted.
  333. */
  334. if (dev->flags & KVM_DEV_ASSIGN_PCI_2_3) {
  335. irq_handler = kvm_assigned_dev_intx;
  336. flags = IRQF_SHARED;
  337. } else {
  338. irq_handler = NULL;
  339. flags = IRQF_ONESHOT;
  340. }
  341. if (request_threaded_irq(dev->host_irq, irq_handler,
  342. kvm_assigned_dev_thread_intx, flags,
  343. dev->irq_name, dev))
  344. return -EIO;
  345. if (dev->flags & KVM_DEV_ASSIGN_PCI_2_3) {
  346. spin_lock_irq(&dev->intx_lock);
  347. pci_intx(dev->dev, true);
  348. spin_unlock_irq(&dev->intx_lock);
  349. }
  350. return 0;
  351. }
  352. static int assigned_device_enable_host_msi(struct kvm *kvm,
  353. struct kvm_assigned_dev_kernel *dev)
  354. {
  355. int r;
  356. if (!dev->dev->msi_enabled) {
  357. r = pci_enable_msi(dev->dev);
  358. if (r)
  359. return r;
  360. }
  361. dev->host_irq = dev->dev->irq;
  362. if (request_threaded_irq(dev->host_irq, kvm_assigned_dev_msi,
  363. kvm_assigned_dev_thread_msi, 0,
  364. dev->irq_name, dev)) {
  365. pci_disable_msi(dev->dev);
  366. return -EIO;
  367. }
  368. return 0;
  369. }
  370. static int assigned_device_enable_host_msix(struct kvm *kvm,
  371. struct kvm_assigned_dev_kernel *dev)
  372. {
  373. int i, r = -EINVAL;
  374. /* host_msix_entries and guest_msix_entries should have been
  375. * initialized */
  376. if (dev->entries_nr == 0)
  377. return r;
  378. r = pci_enable_msix_exact(dev->dev,
  379. dev->host_msix_entries, dev->entries_nr);
  380. if (r)
  381. return r;
  382. for (i = 0; i < dev->entries_nr; i++) {
  383. r = request_threaded_irq(dev->host_msix_entries[i].vector,
  384. kvm_assigned_dev_msix,
  385. kvm_assigned_dev_thread_msix,
  386. 0, dev->irq_name, dev);
  387. if (r)
  388. goto err;
  389. }
  390. return 0;
  391. err:
  392. for (i -= 1; i >= 0; i--)
  393. free_irq(dev->host_msix_entries[i].vector, dev);
  394. pci_disable_msix(dev->dev);
  395. return r;
  396. }
  397. static int assigned_device_enable_guest_intx(struct kvm *kvm,
  398. struct kvm_assigned_dev_kernel *dev,
  399. struct kvm_assigned_irq *irq)
  400. {
  401. dev->guest_irq = irq->guest_irq;
  402. dev->ack_notifier.gsi = irq->guest_irq;
  403. return 0;
  404. }
  405. static int assigned_device_enable_guest_msi(struct kvm *kvm,
  406. struct kvm_assigned_dev_kernel *dev,
  407. struct kvm_assigned_irq *irq)
  408. {
  409. dev->guest_irq = irq->guest_irq;
  410. dev->ack_notifier.gsi = -1;
  411. return 0;
  412. }
  413. static int assigned_device_enable_guest_msix(struct kvm *kvm,
  414. struct kvm_assigned_dev_kernel *dev,
  415. struct kvm_assigned_irq *irq)
  416. {
  417. dev->guest_irq = irq->guest_irq;
  418. dev->ack_notifier.gsi = -1;
  419. return 0;
  420. }
  421. static int assign_host_irq(struct kvm *kvm,
  422. struct kvm_assigned_dev_kernel *dev,
  423. __u32 host_irq_type)
  424. {
  425. int r = -EEXIST;
  426. if (dev->irq_requested_type & KVM_DEV_IRQ_HOST_MASK)
  427. return r;
  428. snprintf(dev->irq_name, sizeof(dev->irq_name), "kvm:%s",
  429. pci_name(dev->dev));
  430. switch (host_irq_type) {
  431. case KVM_DEV_IRQ_HOST_INTX:
  432. r = assigned_device_enable_host_intx(kvm, dev);
  433. break;
  434. case KVM_DEV_IRQ_HOST_MSI:
  435. r = assigned_device_enable_host_msi(kvm, dev);
  436. break;
  437. case KVM_DEV_IRQ_HOST_MSIX:
  438. r = assigned_device_enable_host_msix(kvm, dev);
  439. break;
  440. default:
  441. r = -EINVAL;
  442. }
  443. dev->host_irq_disabled = false;
  444. if (!r)
  445. dev->irq_requested_type |= host_irq_type;
  446. return r;
  447. }
  448. static int assign_guest_irq(struct kvm *kvm,
  449. struct kvm_assigned_dev_kernel *dev,
  450. struct kvm_assigned_irq *irq,
  451. unsigned long guest_irq_type)
  452. {
  453. int id;
  454. int r = -EEXIST;
  455. if (dev->irq_requested_type & KVM_DEV_IRQ_GUEST_MASK)
  456. return r;
  457. id = kvm_request_irq_source_id(kvm);
  458. if (id < 0)
  459. return id;
  460. dev->irq_source_id = id;
  461. switch (guest_irq_type) {
  462. case KVM_DEV_IRQ_GUEST_INTX:
  463. r = assigned_device_enable_guest_intx(kvm, dev, irq);
  464. break;
  465. case KVM_DEV_IRQ_GUEST_MSI:
  466. r = assigned_device_enable_guest_msi(kvm, dev, irq);
  467. break;
  468. case KVM_DEV_IRQ_GUEST_MSIX:
  469. r = assigned_device_enable_guest_msix(kvm, dev, irq);
  470. break;
  471. default:
  472. r = -EINVAL;
  473. }
  474. if (!r) {
  475. dev->irq_requested_type |= guest_irq_type;
  476. if (dev->ack_notifier.gsi != -1)
  477. kvm_register_irq_ack_notifier(kvm, &dev->ack_notifier);
  478. } else {
  479. kvm_free_irq_source_id(kvm, dev->irq_source_id);
  480. dev->irq_source_id = -1;
  481. }
  482. return r;
  483. }
  484. /* TODO Deal with KVM_DEV_IRQ_ASSIGNED_MASK_MSIX */
  485. static int kvm_vm_ioctl_assign_irq(struct kvm *kvm,
  486. struct kvm_assigned_irq *assigned_irq)
  487. {
  488. int r = -EINVAL;
  489. struct kvm_assigned_dev_kernel *match;
  490. unsigned long host_irq_type, guest_irq_type;
  491. if (!irqchip_in_kernel(kvm))
  492. return r;
  493. mutex_lock(&kvm->lock);
  494. r = -ENODEV;
  495. match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
  496. assigned_irq->assigned_dev_id);
  497. if (!match)
  498. goto out;
  499. host_irq_type = (assigned_irq->flags & KVM_DEV_IRQ_HOST_MASK);
  500. guest_irq_type = (assigned_irq->flags & KVM_DEV_IRQ_GUEST_MASK);
  501. r = -EINVAL;
  502. /* can only assign one type at a time */
  503. if (hweight_long(host_irq_type) > 1)
  504. goto out;
  505. if (hweight_long(guest_irq_type) > 1)
  506. goto out;
  507. if (host_irq_type == 0 && guest_irq_type == 0)
  508. goto out;
  509. r = 0;
  510. if (host_irq_type)
  511. r = assign_host_irq(kvm, match, host_irq_type);
  512. if (r)
  513. goto out;
  514. if (guest_irq_type)
  515. r = assign_guest_irq(kvm, match, assigned_irq, guest_irq_type);
  516. out:
  517. mutex_unlock(&kvm->lock);
  518. return r;
  519. }
  520. static int kvm_vm_ioctl_deassign_dev_irq(struct kvm *kvm,
  521. struct kvm_assigned_irq
  522. *assigned_irq)
  523. {
  524. int r = -ENODEV;
  525. struct kvm_assigned_dev_kernel *match;
  526. unsigned long irq_type;
  527. mutex_lock(&kvm->lock);
  528. match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
  529. assigned_irq->assigned_dev_id);
  530. if (!match)
  531. goto out;
  532. irq_type = assigned_irq->flags & (KVM_DEV_IRQ_HOST_MASK |
  533. KVM_DEV_IRQ_GUEST_MASK);
  534. r = kvm_deassign_irq(kvm, match, irq_type);
  535. out:
  536. mutex_unlock(&kvm->lock);
  537. return r;
  538. }
  539. /*
  540. * We want to test whether the caller has been granted permissions to
  541. * use this device. To be able to configure and control the device,
  542. * the user needs access to PCI configuration space and BAR resources.
  543. * These are accessed through PCI sysfs. PCI config space is often
  544. * passed to the process calling this ioctl via file descriptor, so we
  545. * can't rely on access to that file. We can check for permissions
  546. * on each of the BAR resource files, which is a pretty clear
  547. * indicator that the user has been granted access to the device.
  548. */
  549. static int probe_sysfs_permissions(struct pci_dev *dev)
  550. {
  551. #ifdef CONFIG_SYSFS
  552. int i;
  553. bool bar_found = false;
  554. for (i = PCI_STD_RESOURCES; i <= PCI_STD_RESOURCE_END; i++) {
  555. char *kpath, *syspath;
  556. struct path path;
  557. struct inode *inode;
  558. int r;
  559. if (!pci_resource_len(dev, i))
  560. continue;
  561. kpath = kobject_get_path(&dev->dev.kobj, GFP_KERNEL);
  562. if (!kpath)
  563. return -ENOMEM;
  564. /* Per sysfs-rules, sysfs is always at /sys */
  565. syspath = kasprintf(GFP_KERNEL, "/sys%s/resource%d", kpath, i);
  566. kfree(kpath);
  567. if (!syspath)
  568. return -ENOMEM;
  569. r = kern_path(syspath, LOOKUP_FOLLOW, &path);
  570. kfree(syspath);
  571. if (r)
  572. return r;
  573. inode = d_backing_inode(path.dentry);
  574. r = inode_permission(inode, MAY_READ | MAY_WRITE | MAY_ACCESS);
  575. path_put(&path);
  576. if (r)
  577. return r;
  578. bar_found = true;
  579. }
  580. /* If no resources, probably something special */
  581. if (!bar_found)
  582. return -EPERM;
  583. return 0;
  584. #else
  585. return -EINVAL; /* No way to control the device without sysfs */
  586. #endif
  587. }
  588. static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
  589. struct kvm_assigned_pci_dev *assigned_dev)
  590. {
  591. int r = 0, idx;
  592. struct kvm_assigned_dev_kernel *match;
  593. struct pci_dev *dev;
  594. if (!(assigned_dev->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU))
  595. return -EINVAL;
  596. mutex_lock(&kvm->lock);
  597. idx = srcu_read_lock(&kvm->srcu);
  598. match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
  599. assigned_dev->assigned_dev_id);
  600. if (match) {
  601. /* device already assigned */
  602. r = -EEXIST;
  603. goto out;
  604. }
  605. match = kzalloc(sizeof(struct kvm_assigned_dev_kernel), GFP_KERNEL);
  606. if (match == NULL) {
  607. printk(KERN_INFO "%s: Couldn't allocate memory\n",
  608. __func__);
  609. r = -ENOMEM;
  610. goto out;
  611. }
  612. dev = pci_get_domain_bus_and_slot(assigned_dev->segnr,
  613. assigned_dev->busnr,
  614. assigned_dev->devfn);
  615. if (!dev) {
  616. printk(KERN_INFO "%s: host device not found\n", __func__);
  617. r = -EINVAL;
  618. goto out_free;
  619. }
  620. /* Don't allow bridges to be assigned */
  621. if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL) {
  622. r = -EPERM;
  623. goto out_put;
  624. }
  625. r = probe_sysfs_permissions(dev);
  626. if (r)
  627. goto out_put;
  628. if (pci_enable_device(dev)) {
  629. printk(KERN_INFO "%s: Could not enable PCI device\n", __func__);
  630. r = -EBUSY;
  631. goto out_put;
  632. }
  633. r = pci_request_regions(dev, "kvm_assigned_device");
  634. if (r) {
  635. printk(KERN_INFO "%s: Could not get access to device regions\n",
  636. __func__);
  637. goto out_disable;
  638. }
  639. pci_reset_function(dev);
  640. pci_save_state(dev);
  641. match->pci_saved_state = pci_store_saved_state(dev);
  642. if (!match->pci_saved_state)
  643. printk(KERN_DEBUG "%s: Couldn't store %s saved state\n",
  644. __func__, dev_name(&dev->dev));
  645. if (!pci_intx_mask_supported(dev))
  646. assigned_dev->flags &= ~KVM_DEV_ASSIGN_PCI_2_3;
  647. match->assigned_dev_id = assigned_dev->assigned_dev_id;
  648. match->host_segnr = assigned_dev->segnr;
  649. match->host_busnr = assigned_dev->busnr;
  650. match->host_devfn = assigned_dev->devfn;
  651. match->flags = assigned_dev->flags;
  652. match->dev = dev;
  653. spin_lock_init(&match->intx_lock);
  654. spin_lock_init(&match->intx_mask_lock);
  655. match->irq_source_id = -1;
  656. match->kvm = kvm;
  657. match->ack_notifier.irq_acked = kvm_assigned_dev_ack_irq;
  658. list_add(&match->list, &kvm->arch.assigned_dev_head);
  659. if (!kvm->arch.iommu_domain) {
  660. r = kvm_iommu_map_guest(kvm);
  661. if (r)
  662. goto out_list_del;
  663. }
  664. r = kvm_assign_device(kvm, match->dev);
  665. if (r)
  666. goto out_list_del;
  667. out:
  668. srcu_read_unlock(&kvm->srcu, idx);
  669. mutex_unlock(&kvm->lock);
  670. return r;
  671. out_list_del:
  672. if (pci_load_and_free_saved_state(dev, &match->pci_saved_state))
  673. printk(KERN_INFO "%s: Couldn't reload %s saved state\n",
  674. __func__, dev_name(&dev->dev));
  675. list_del(&match->list);
  676. pci_release_regions(dev);
  677. out_disable:
  678. pci_disable_device(dev);
  679. out_put:
  680. pci_dev_put(dev);
  681. out_free:
  682. kfree(match);
  683. srcu_read_unlock(&kvm->srcu, idx);
  684. mutex_unlock(&kvm->lock);
  685. return r;
  686. }
  687. static int kvm_vm_ioctl_deassign_device(struct kvm *kvm,
  688. struct kvm_assigned_pci_dev *assigned_dev)
  689. {
  690. int r = 0;
  691. struct kvm_assigned_dev_kernel *match;
  692. mutex_lock(&kvm->lock);
  693. match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
  694. assigned_dev->assigned_dev_id);
  695. if (!match) {
  696. printk(KERN_INFO "%s: device hasn't been assigned before, "
  697. "so cannot be deassigned\n", __func__);
  698. r = -EINVAL;
  699. goto out;
  700. }
  701. kvm_deassign_device(kvm, match->dev);
  702. kvm_free_assigned_device(kvm, match);
  703. out:
  704. mutex_unlock(&kvm->lock);
  705. return r;
  706. }
  707. static int kvm_vm_ioctl_set_msix_nr(struct kvm *kvm,
  708. struct kvm_assigned_msix_nr *entry_nr)
  709. {
  710. int r = 0;
  711. struct kvm_assigned_dev_kernel *adev;
  712. mutex_lock(&kvm->lock);
  713. adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
  714. entry_nr->assigned_dev_id);
  715. if (!adev) {
  716. r = -EINVAL;
  717. goto msix_nr_out;
  718. }
  719. if (adev->entries_nr == 0) {
  720. adev->entries_nr = entry_nr->entry_nr;
  721. if (adev->entries_nr == 0 ||
  722. adev->entries_nr > KVM_MAX_MSIX_PER_DEV) {
  723. r = -EINVAL;
  724. goto msix_nr_out;
  725. }
  726. adev->host_msix_entries = kzalloc(sizeof(struct msix_entry) *
  727. entry_nr->entry_nr,
  728. GFP_KERNEL);
  729. if (!adev->host_msix_entries) {
  730. r = -ENOMEM;
  731. goto msix_nr_out;
  732. }
  733. adev->guest_msix_entries =
  734. kzalloc(sizeof(struct msix_entry) * entry_nr->entry_nr,
  735. GFP_KERNEL);
  736. if (!adev->guest_msix_entries) {
  737. kfree(adev->host_msix_entries);
  738. r = -ENOMEM;
  739. goto msix_nr_out;
  740. }
  741. } else /* Not allowed set MSI-X number twice */
  742. r = -EINVAL;
  743. msix_nr_out:
  744. mutex_unlock(&kvm->lock);
  745. return r;
  746. }
  747. static int kvm_vm_ioctl_set_msix_entry(struct kvm *kvm,
  748. struct kvm_assigned_msix_entry *entry)
  749. {
  750. int r = 0, i;
  751. struct kvm_assigned_dev_kernel *adev;
  752. mutex_lock(&kvm->lock);
  753. adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
  754. entry->assigned_dev_id);
  755. if (!adev) {
  756. r = -EINVAL;
  757. goto msix_entry_out;
  758. }
  759. for (i = 0; i < adev->entries_nr; i++)
  760. if (adev->guest_msix_entries[i].vector == 0 ||
  761. adev->guest_msix_entries[i].entry == entry->entry) {
  762. adev->guest_msix_entries[i].entry = entry->entry;
  763. adev->guest_msix_entries[i].vector = entry->gsi;
  764. adev->host_msix_entries[i].entry = entry->entry;
  765. break;
  766. }
  767. if (i == adev->entries_nr) {
  768. r = -ENOSPC;
  769. goto msix_entry_out;
  770. }
  771. msix_entry_out:
  772. mutex_unlock(&kvm->lock);
  773. return r;
  774. }
  775. static int kvm_vm_ioctl_set_pci_irq_mask(struct kvm *kvm,
  776. struct kvm_assigned_pci_dev *assigned_dev)
  777. {
  778. int r = 0;
  779. struct kvm_assigned_dev_kernel *match;
  780. mutex_lock(&kvm->lock);
  781. match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
  782. assigned_dev->assigned_dev_id);
  783. if (!match) {
  784. r = -ENODEV;
  785. goto out;
  786. }
  787. spin_lock(&match->intx_mask_lock);
  788. match->flags &= ~KVM_DEV_ASSIGN_MASK_INTX;
  789. match->flags |= assigned_dev->flags & KVM_DEV_ASSIGN_MASK_INTX;
  790. if (match->irq_requested_type & KVM_DEV_IRQ_GUEST_INTX) {
  791. if (assigned_dev->flags & KVM_DEV_ASSIGN_MASK_INTX) {
  792. kvm_set_irq(match->kvm, match->irq_source_id,
  793. match->guest_irq, 0, false);
  794. /*
  795. * Masking at hardware-level is performed on demand,
  796. * i.e. when an IRQ actually arrives at the host.
  797. */
  798. } else if (!(assigned_dev->flags & KVM_DEV_ASSIGN_PCI_2_3)) {
  799. /*
  800. * Unmask the IRQ line if required. Unmasking at
  801. * device level will be performed by user space.
  802. */
  803. spin_lock_irq(&match->intx_lock);
  804. if (match->host_irq_disabled) {
  805. enable_irq(match->host_irq);
  806. match->host_irq_disabled = false;
  807. }
  808. spin_unlock_irq(&match->intx_lock);
  809. }
  810. }
  811. spin_unlock(&match->intx_mask_lock);
  812. out:
  813. mutex_unlock(&kvm->lock);
  814. return r;
  815. }
  816. long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
  817. unsigned long arg)
  818. {
  819. void __user *argp = (void __user *)arg;
  820. int r;
  821. switch (ioctl) {
  822. case KVM_ASSIGN_PCI_DEVICE: {
  823. struct kvm_assigned_pci_dev assigned_dev;
  824. r = -EFAULT;
  825. if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
  826. goto out;
  827. r = kvm_vm_ioctl_assign_device(kvm, &assigned_dev);
  828. if (r)
  829. goto out;
  830. break;
  831. }
  832. case KVM_ASSIGN_IRQ: {
  833. r = -EOPNOTSUPP;
  834. break;
  835. }
  836. case KVM_ASSIGN_DEV_IRQ: {
  837. struct kvm_assigned_irq assigned_irq;
  838. r = -EFAULT;
  839. if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
  840. goto out;
  841. r = kvm_vm_ioctl_assign_irq(kvm, &assigned_irq);
  842. if (r)
  843. goto out;
  844. break;
  845. }
  846. case KVM_DEASSIGN_DEV_IRQ: {
  847. struct kvm_assigned_irq assigned_irq;
  848. r = -EFAULT;
  849. if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
  850. goto out;
  851. r = kvm_vm_ioctl_deassign_dev_irq(kvm, &assigned_irq);
  852. if (r)
  853. goto out;
  854. break;
  855. }
  856. case KVM_DEASSIGN_PCI_DEVICE: {
  857. struct kvm_assigned_pci_dev assigned_dev;
  858. r = -EFAULT;
  859. if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
  860. goto out;
  861. r = kvm_vm_ioctl_deassign_device(kvm, &assigned_dev);
  862. if (r)
  863. goto out;
  864. break;
  865. }
  866. case KVM_ASSIGN_SET_MSIX_NR: {
  867. struct kvm_assigned_msix_nr entry_nr;
  868. r = -EFAULT;
  869. if (copy_from_user(&entry_nr, argp, sizeof entry_nr))
  870. goto out;
  871. r = kvm_vm_ioctl_set_msix_nr(kvm, &entry_nr);
  872. if (r)
  873. goto out;
  874. break;
  875. }
  876. case KVM_ASSIGN_SET_MSIX_ENTRY: {
  877. struct kvm_assigned_msix_entry entry;
  878. r = -EFAULT;
  879. if (copy_from_user(&entry, argp, sizeof entry))
  880. goto out;
  881. r = kvm_vm_ioctl_set_msix_entry(kvm, &entry);
  882. if (r)
  883. goto out;
  884. break;
  885. }
  886. case KVM_ASSIGN_SET_INTX_MASK: {
  887. struct kvm_assigned_pci_dev assigned_dev;
  888. r = -EFAULT;
  889. if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
  890. goto out;
  891. r = kvm_vm_ioctl_set_pci_irq_mask(kvm, &assigned_dev);
  892. break;
  893. }
  894. default:
  895. r = -ENOTTY;
  896. break;
  897. }
  898. out:
  899. return r;
  900. }