amdgpu_vm.c 70 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732
  1. /*
  2. * Copyright 2008 Advanced Micro Devices, Inc.
  3. * Copyright 2008 Red Hat Inc.
  4. * Copyright 2009 Jerome Glisse.
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the "Software"),
  8. * to deal in the Software without restriction, including without limitation
  9. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10. * and/or sell copies of the Software, and to permit persons to whom the
  11. * Software is furnished to do so, subject to the following conditions:
  12. *
  13. * The above copyright notice and this permission notice shall be included in
  14. * all copies or substantial portions of the Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22. * OTHER DEALINGS IN THE SOFTWARE.
  23. *
  24. * Authors: Dave Airlie
  25. * Alex Deucher
  26. * Jerome Glisse
  27. */
  28. #include <linux/dma-fence-array.h>
  29. #include <linux/interval_tree_generic.h>
  30. #include <drm/drmP.h>
  31. #include <drm/amdgpu_drm.h>
  32. #include "amdgpu.h"
  33. #include "amdgpu_trace.h"
  34. /*
  35. * GPUVM
  36. * GPUVM is similar to the legacy gart on older asics, however
  37. * rather than there being a single global gart table
  38. * for the entire GPU, there are multiple VM page tables active
  39. * at any given time. The VM page tables can contain a mix
  40. * vram pages and system memory pages and system memory pages
  41. * can be mapped as snooped (cached system pages) or unsnooped
  42. * (uncached system pages).
  43. * Each VM has an ID associated with it and there is a page table
  44. * associated with each VMID. When execting a command buffer,
  45. * the kernel tells the the ring what VMID to use for that command
  46. * buffer. VMIDs are allocated dynamically as commands are submitted.
  47. * The userspace drivers maintain their own address space and the kernel
  48. * sets up their pages tables accordingly when they submit their
  49. * command buffers and a VMID is assigned.
  50. * Cayman/Trinity support up to 8 active VMs at any given time;
  51. * SI supports 16.
  52. */
  53. #define START(node) ((node)->start)
  54. #define LAST(node) ((node)->last)
  55. INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  56. START, LAST, static, amdgpu_vm_it)
  57. #undef START
  58. #undef LAST
  59. /* Local structure. Encapsulate some VM table update parameters to reduce
  60. * the number of function parameters
  61. */
  62. struct amdgpu_pte_update_params {
  63. /* amdgpu device we do this update for */
  64. struct amdgpu_device *adev;
  65. /* optional amdgpu_vm we do this update for */
  66. struct amdgpu_vm *vm;
  67. /* address where to copy page table entries from */
  68. uint64_t src;
  69. /* indirect buffer to fill with commands */
  70. struct amdgpu_ib *ib;
  71. /* Function which actually does the update */
  72. void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
  73. uint64_t addr, unsigned count, uint32_t incr,
  74. uint64_t flags);
  75. /* The next two are used during VM update by CPU
  76. * DMA addresses to use for mapping
  77. * Kernel pointer of PD/PT BO that needs to be updated
  78. */
  79. dma_addr_t *pages_addr;
  80. void *kptr;
  81. };
  82. /* Helper to disable partial resident texture feature from a fence callback */
  83. struct amdgpu_prt_cb {
  84. struct amdgpu_device *adev;
  85. struct dma_fence_cb cb;
  86. };
  87. /**
  88. * amdgpu_vm_num_entries - return the number of entries in a PD/PT
  89. *
  90. * @adev: amdgpu_device pointer
  91. *
  92. * Calculate the number of entries in a page directory or page table.
  93. */
  94. static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
  95. unsigned level)
  96. {
  97. if (level == 0)
  98. /* For the root directory */
  99. return adev->vm_manager.max_pfn >>
  100. (adev->vm_manager.block_size *
  101. adev->vm_manager.num_level);
  102. else if (level == adev->vm_manager.num_level)
  103. /* For the page tables on the leaves */
  104. return AMDGPU_VM_PTE_COUNT(adev);
  105. else
  106. /* Everything in between */
  107. return 1 << adev->vm_manager.block_size;
  108. }
  109. /**
  110. * amdgpu_vm_bo_size - returns the size of the BOs in bytes
  111. *
  112. * @adev: amdgpu_device pointer
  113. *
  114. * Calculate the size of the BO for a page directory or page table in bytes.
  115. */
  116. static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
  117. {
  118. return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
  119. }
  120. /**
  121. * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
  122. *
  123. * @vm: vm providing the BOs
  124. * @validated: head of validation list
  125. * @entry: entry to add
  126. *
  127. * Add the page directory to the list of BOs to
  128. * validate for command submission.
  129. */
  130. void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
  131. struct list_head *validated,
  132. struct amdgpu_bo_list_entry *entry)
  133. {
  134. entry->robj = vm->root.bo;
  135. entry->priority = 0;
  136. entry->tv.bo = &entry->robj->tbo;
  137. entry->tv.shared = true;
  138. entry->user_pages = NULL;
  139. list_add(&entry->tv.head, validated);
  140. }
  141. /**
  142. * amdgpu_vm_validate_layer - validate a single page table level
  143. *
  144. * @parent: parent page table level
  145. * @validate: callback to do the validation
  146. * @param: parameter for the validation callback
  147. *
  148. * Validate the page table BOs on command submission if neccessary.
  149. */
  150. static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
  151. int (*validate)(void *, struct amdgpu_bo *),
  152. void *param, bool use_cpu_for_update,
  153. struct ttm_bo_global *glob)
  154. {
  155. unsigned i;
  156. int r;
  157. if (use_cpu_for_update) {
  158. r = amdgpu_bo_kmap(parent->bo, NULL);
  159. if (r)
  160. return r;
  161. }
  162. if (!parent->entries)
  163. return 0;
  164. for (i = 0; i <= parent->last_entry_used; ++i) {
  165. struct amdgpu_vm_pt *entry = &parent->entries[i];
  166. if (!entry->bo)
  167. continue;
  168. r = validate(param, entry->bo);
  169. if (r)
  170. return r;
  171. spin_lock(&glob->lru_lock);
  172. ttm_bo_move_to_lru_tail(&entry->bo->tbo);
  173. if (entry->bo->shadow)
  174. ttm_bo_move_to_lru_tail(&entry->bo->shadow->tbo);
  175. spin_unlock(&glob->lru_lock);
  176. /*
  177. * Recurse into the sub directory. This is harmless because we
  178. * have only a maximum of 5 layers.
  179. */
  180. r = amdgpu_vm_validate_level(entry, validate, param,
  181. use_cpu_for_update, glob);
  182. if (r)
  183. return r;
  184. }
  185. return r;
  186. }
  187. /**
  188. * amdgpu_vm_validate_pt_bos - validate the page table BOs
  189. *
  190. * @adev: amdgpu device pointer
  191. * @vm: vm providing the BOs
  192. * @validate: callback to do the validation
  193. * @param: parameter for the validation callback
  194. *
  195. * Validate the page table BOs on command submission if neccessary.
  196. */
  197. int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  198. int (*validate)(void *p, struct amdgpu_bo *bo),
  199. void *param)
  200. {
  201. uint64_t num_evictions;
  202. /* We only need to validate the page tables
  203. * if they aren't already valid.
  204. */
  205. num_evictions = atomic64_read(&adev->num_evictions);
  206. if (num_evictions == vm->last_eviction_counter)
  207. return 0;
  208. return amdgpu_vm_validate_level(&vm->root, validate, param,
  209. vm->use_cpu_for_update,
  210. adev->mman.bdev.glob);
  211. }
  212. /**
  213. * amdgpu_vm_alloc_levels - allocate the PD/PT levels
  214. *
  215. * @adev: amdgpu_device pointer
  216. * @vm: requested vm
  217. * @saddr: start of the address range
  218. * @eaddr: end of the address range
  219. *
  220. * Make sure the page directories and page tables are allocated
  221. */
  222. static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
  223. struct amdgpu_vm *vm,
  224. struct amdgpu_vm_pt *parent,
  225. uint64_t saddr, uint64_t eaddr,
  226. unsigned level)
  227. {
  228. unsigned shift = (adev->vm_manager.num_level - level) *
  229. adev->vm_manager.block_size;
  230. unsigned pt_idx, from, to;
  231. int r;
  232. u64 flags;
  233. uint64_t init_value = 0;
  234. if (!parent->entries) {
  235. unsigned num_entries = amdgpu_vm_num_entries(adev, level);
  236. parent->entries = kvmalloc_array(num_entries,
  237. sizeof(struct amdgpu_vm_pt),
  238. GFP_KERNEL | __GFP_ZERO);
  239. if (!parent->entries)
  240. return -ENOMEM;
  241. memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt));
  242. }
  243. from = saddr >> shift;
  244. to = eaddr >> shift;
  245. if (from >= amdgpu_vm_num_entries(adev, level) ||
  246. to >= amdgpu_vm_num_entries(adev, level))
  247. return -EINVAL;
  248. if (to > parent->last_entry_used)
  249. parent->last_entry_used = to;
  250. ++level;
  251. saddr = saddr & ((1 << shift) - 1);
  252. eaddr = eaddr & ((1 << shift) - 1);
  253. flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
  254. AMDGPU_GEM_CREATE_VRAM_CLEARED;
  255. if (vm->use_cpu_for_update)
  256. flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
  257. else
  258. flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
  259. AMDGPU_GEM_CREATE_SHADOW);
  260. if (vm->pte_support_ats) {
  261. init_value = AMDGPU_PTE_SYSTEM;
  262. if (level != adev->vm_manager.num_level - 1)
  263. init_value |= AMDGPU_PDE_PTE;
  264. }
  265. /* walk over the address space and allocate the page tables */
  266. for (pt_idx = from; pt_idx <= to; ++pt_idx) {
  267. struct reservation_object *resv = vm->root.bo->tbo.resv;
  268. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  269. struct amdgpu_bo *pt;
  270. if (!entry->bo) {
  271. r = amdgpu_bo_create(adev,
  272. amdgpu_vm_bo_size(adev, level),
  273. AMDGPU_GPU_PAGE_SIZE, true,
  274. AMDGPU_GEM_DOMAIN_VRAM,
  275. flags,
  276. NULL, resv, init_value, &pt);
  277. if (r)
  278. return r;
  279. if (vm->use_cpu_for_update) {
  280. r = amdgpu_bo_kmap(pt, NULL);
  281. if (r) {
  282. amdgpu_bo_unref(&pt);
  283. return r;
  284. }
  285. }
  286. /* Keep a reference to the root directory to avoid
  287. * freeing them up in the wrong order.
  288. */
  289. pt->parent = amdgpu_bo_ref(vm->root.bo);
  290. entry->bo = pt;
  291. entry->addr = 0;
  292. }
  293. if (level < adev->vm_manager.num_level) {
  294. uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
  295. uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
  296. ((1 << shift) - 1);
  297. r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
  298. sub_eaddr, level);
  299. if (r)
  300. return r;
  301. }
  302. }
  303. return 0;
  304. }
  305. /**
  306. * amdgpu_vm_alloc_pts - Allocate page tables.
  307. *
  308. * @adev: amdgpu_device pointer
  309. * @vm: VM to allocate page tables for
  310. * @saddr: Start address which needs to be allocated
  311. * @size: Size from start address we need.
  312. *
  313. * Make sure the page tables are allocated.
  314. */
  315. int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
  316. struct amdgpu_vm *vm,
  317. uint64_t saddr, uint64_t size)
  318. {
  319. uint64_t last_pfn;
  320. uint64_t eaddr;
  321. /* validate the parameters */
  322. if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
  323. return -EINVAL;
  324. eaddr = saddr + size - 1;
  325. last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
  326. if (last_pfn >= adev->vm_manager.max_pfn) {
  327. dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
  328. last_pfn, adev->vm_manager.max_pfn);
  329. return -EINVAL;
  330. }
  331. saddr /= AMDGPU_GPU_PAGE_SIZE;
  332. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  333. return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);
  334. }
  335. /**
  336. * amdgpu_vm_had_gpu_reset - check if reset occured since last use
  337. *
  338. * @adev: amdgpu_device pointer
  339. * @id: VMID structure
  340. *
  341. * Check if GPU reset occured since last use of the VMID.
  342. */
  343. static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev,
  344. struct amdgpu_vm_id *id)
  345. {
  346. return id->current_gpu_reset_count !=
  347. atomic_read(&adev->gpu_reset_counter);
  348. }
  349. static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub)
  350. {
  351. return !!vm->reserved_vmid[vmhub];
  352. }
  353. /* idr_mgr->lock must be held */
  354. static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
  355. struct amdgpu_ring *ring,
  356. struct amdgpu_sync *sync,
  357. struct dma_fence *fence,
  358. struct amdgpu_job *job)
  359. {
  360. struct amdgpu_device *adev = ring->adev;
  361. unsigned vmhub = ring->funcs->vmhub;
  362. uint64_t fence_context = adev->fence_context + ring->idx;
  363. struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub];
  364. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  365. struct dma_fence *updates = sync->last_vm_update;
  366. int r = 0;
  367. struct dma_fence *flushed, *tmp;
  368. bool needs_flush = vm->use_cpu_for_update;
  369. flushed = id->flushed_updates;
  370. if ((amdgpu_vm_had_gpu_reset(adev, id)) ||
  371. (atomic64_read(&id->owner) != vm->client_id) ||
  372. (job->vm_pd_addr != id->pd_gpu_addr) ||
  373. (updates && (!flushed || updates->context != flushed->context ||
  374. dma_fence_is_later(updates, flushed))) ||
  375. (!id->last_flush || (id->last_flush->context != fence_context &&
  376. !dma_fence_is_signaled(id->last_flush)))) {
  377. needs_flush = true;
  378. /* to prevent one context starved by another context */
  379. id->pd_gpu_addr = 0;
  380. tmp = amdgpu_sync_peek_fence(&id->active, ring);
  381. if (tmp) {
  382. r = amdgpu_sync_fence(adev, sync, tmp);
  383. return r;
  384. }
  385. }
  386. /* Good we can use this VMID. Remember this submission as
  387. * user of the VMID.
  388. */
  389. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  390. if (r)
  391. goto out;
  392. if (updates && (!flushed || updates->context != flushed->context ||
  393. dma_fence_is_later(updates, flushed))) {
  394. dma_fence_put(id->flushed_updates);
  395. id->flushed_updates = dma_fence_get(updates);
  396. }
  397. id->pd_gpu_addr = job->vm_pd_addr;
  398. atomic64_set(&id->owner, vm->client_id);
  399. job->vm_needs_flush = needs_flush;
  400. if (needs_flush) {
  401. dma_fence_put(id->last_flush);
  402. id->last_flush = NULL;
  403. }
  404. job->vm_id = id - id_mgr->ids;
  405. trace_amdgpu_vm_grab_id(vm, ring, job);
  406. out:
  407. return r;
  408. }
  409. /**
  410. * amdgpu_vm_grab_id - allocate the next free VMID
  411. *
  412. * @vm: vm to allocate id for
  413. * @ring: ring we want to submit job to
  414. * @sync: sync object where we add dependencies
  415. * @fence: fence protecting ID from reuse
  416. *
  417. * Allocate an id for the vm, adding fences to the sync obj as necessary.
  418. */
  419. int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
  420. struct amdgpu_sync *sync, struct dma_fence *fence,
  421. struct amdgpu_job *job)
  422. {
  423. struct amdgpu_device *adev = ring->adev;
  424. unsigned vmhub = ring->funcs->vmhub;
  425. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  426. uint64_t fence_context = adev->fence_context + ring->idx;
  427. struct dma_fence *updates = sync->last_vm_update;
  428. struct amdgpu_vm_id *id, *idle;
  429. struct dma_fence **fences;
  430. unsigned i;
  431. int r = 0;
  432. mutex_lock(&id_mgr->lock);
  433. if (amdgpu_vm_reserved_vmid_ready(vm, vmhub)) {
  434. r = amdgpu_vm_grab_reserved_vmid_locked(vm, ring, sync, fence, job);
  435. mutex_unlock(&id_mgr->lock);
  436. return r;
  437. }
  438. fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL);
  439. if (!fences) {
  440. mutex_unlock(&id_mgr->lock);
  441. return -ENOMEM;
  442. }
  443. /* Check if we have an idle VMID */
  444. i = 0;
  445. list_for_each_entry(idle, &id_mgr->ids_lru, list) {
  446. fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
  447. if (!fences[i])
  448. break;
  449. ++i;
  450. }
  451. /* If we can't find a idle VMID to use, wait till one becomes available */
  452. if (&idle->list == &id_mgr->ids_lru) {
  453. u64 fence_context = adev->vm_manager.fence_context + ring->idx;
  454. unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
  455. struct dma_fence_array *array;
  456. unsigned j;
  457. for (j = 0; j < i; ++j)
  458. dma_fence_get(fences[j]);
  459. array = dma_fence_array_create(i, fences, fence_context,
  460. seqno, true);
  461. if (!array) {
  462. for (j = 0; j < i; ++j)
  463. dma_fence_put(fences[j]);
  464. kfree(fences);
  465. r = -ENOMEM;
  466. goto error;
  467. }
  468. r = amdgpu_sync_fence(ring->adev, sync, &array->base);
  469. dma_fence_put(&array->base);
  470. if (r)
  471. goto error;
  472. mutex_unlock(&id_mgr->lock);
  473. return 0;
  474. }
  475. kfree(fences);
  476. job->vm_needs_flush = vm->use_cpu_for_update;
  477. /* Check if we can use a VMID already assigned to this VM */
  478. list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
  479. struct dma_fence *flushed;
  480. bool needs_flush = vm->use_cpu_for_update;
  481. /* Check all the prerequisites to using this VMID */
  482. if (amdgpu_vm_had_gpu_reset(adev, id))
  483. continue;
  484. if (atomic64_read(&id->owner) != vm->client_id)
  485. continue;
  486. if (job->vm_pd_addr != id->pd_gpu_addr)
  487. continue;
  488. if (!id->last_flush ||
  489. (id->last_flush->context != fence_context &&
  490. !dma_fence_is_signaled(id->last_flush)))
  491. needs_flush = true;
  492. flushed = id->flushed_updates;
  493. if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
  494. needs_flush = true;
  495. /* Concurrent flushes are only possible starting with Vega10 */
  496. if (adev->asic_type < CHIP_VEGA10 && needs_flush)
  497. continue;
  498. /* Good we can use this VMID. Remember this submission as
  499. * user of the VMID.
  500. */
  501. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  502. if (r)
  503. goto error;
  504. if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
  505. dma_fence_put(id->flushed_updates);
  506. id->flushed_updates = dma_fence_get(updates);
  507. }
  508. if (needs_flush)
  509. goto needs_flush;
  510. else
  511. goto no_flush_needed;
  512. };
  513. /* Still no ID to use? Then use the idle one found earlier */
  514. id = idle;
  515. /* Remember this submission as user of the VMID */
  516. r = amdgpu_sync_fence(ring->adev, &id->active, fence);
  517. if (r)
  518. goto error;
  519. id->pd_gpu_addr = job->vm_pd_addr;
  520. dma_fence_put(id->flushed_updates);
  521. id->flushed_updates = dma_fence_get(updates);
  522. atomic64_set(&id->owner, vm->client_id);
  523. needs_flush:
  524. job->vm_needs_flush = true;
  525. dma_fence_put(id->last_flush);
  526. id->last_flush = NULL;
  527. no_flush_needed:
  528. list_move_tail(&id->list, &id_mgr->ids_lru);
  529. job->vm_id = id - id_mgr->ids;
  530. trace_amdgpu_vm_grab_id(vm, ring, job);
  531. error:
  532. mutex_unlock(&id_mgr->lock);
  533. return r;
  534. }
  535. static void amdgpu_vm_free_reserved_vmid(struct amdgpu_device *adev,
  536. struct amdgpu_vm *vm,
  537. unsigned vmhub)
  538. {
  539. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  540. mutex_lock(&id_mgr->lock);
  541. if (vm->reserved_vmid[vmhub]) {
  542. list_add(&vm->reserved_vmid[vmhub]->list,
  543. &id_mgr->ids_lru);
  544. vm->reserved_vmid[vmhub] = NULL;
  545. atomic_dec(&id_mgr->reserved_vmid_num);
  546. }
  547. mutex_unlock(&id_mgr->lock);
  548. }
  549. static int amdgpu_vm_alloc_reserved_vmid(struct amdgpu_device *adev,
  550. struct amdgpu_vm *vm,
  551. unsigned vmhub)
  552. {
  553. struct amdgpu_vm_id_manager *id_mgr;
  554. struct amdgpu_vm_id *idle;
  555. int r = 0;
  556. id_mgr = &adev->vm_manager.id_mgr[vmhub];
  557. mutex_lock(&id_mgr->lock);
  558. if (vm->reserved_vmid[vmhub])
  559. goto unlock;
  560. if (atomic_inc_return(&id_mgr->reserved_vmid_num) >
  561. AMDGPU_VM_MAX_RESERVED_VMID) {
  562. DRM_ERROR("Over limitation of reserved vmid\n");
  563. atomic_dec(&id_mgr->reserved_vmid_num);
  564. r = -EINVAL;
  565. goto unlock;
  566. }
  567. /* Select the first entry VMID */
  568. idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vm_id, list);
  569. list_del_init(&idle->list);
  570. vm->reserved_vmid[vmhub] = idle;
  571. mutex_unlock(&id_mgr->lock);
  572. return 0;
  573. unlock:
  574. mutex_unlock(&id_mgr->lock);
  575. return r;
  576. }
  577. /**
  578. * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
  579. *
  580. * @adev: amdgpu_device pointer
  581. */
  582. void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
  583. {
  584. const struct amdgpu_ip_block *ip_block;
  585. bool has_compute_vm_bug;
  586. struct amdgpu_ring *ring;
  587. int i;
  588. has_compute_vm_bug = false;
  589. ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
  590. if (ip_block) {
  591. /* Compute has a VM bug for GFX version < 7.
  592. Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
  593. if (ip_block->version->major <= 7)
  594. has_compute_vm_bug = true;
  595. else if (ip_block->version->major == 8)
  596. if (adev->gfx.mec_fw_version < 673)
  597. has_compute_vm_bug = true;
  598. }
  599. for (i = 0; i < adev->num_rings; i++) {
  600. ring = adev->rings[i];
  601. if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
  602. /* only compute rings */
  603. ring->has_compute_vm_bug = has_compute_vm_bug;
  604. else
  605. ring->has_compute_vm_bug = false;
  606. }
  607. }
  608. bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
  609. struct amdgpu_job *job)
  610. {
  611. struct amdgpu_device *adev = ring->adev;
  612. unsigned vmhub = ring->funcs->vmhub;
  613. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  614. struct amdgpu_vm_id *id;
  615. bool gds_switch_needed;
  616. bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
  617. if (job->vm_id == 0)
  618. return false;
  619. id = &id_mgr->ids[job->vm_id];
  620. gds_switch_needed = ring->funcs->emit_gds_switch && (
  621. id->gds_base != job->gds_base ||
  622. id->gds_size != job->gds_size ||
  623. id->gws_base != job->gws_base ||
  624. id->gws_size != job->gws_size ||
  625. id->oa_base != job->oa_base ||
  626. id->oa_size != job->oa_size);
  627. if (amdgpu_vm_had_gpu_reset(adev, id))
  628. return true;
  629. return vm_flush_needed || gds_switch_needed;
  630. }
  631. static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
  632. {
  633. return (adev->mc.real_vram_size == adev->mc.visible_vram_size);
  634. }
  635. /**
  636. * amdgpu_vm_flush - hardware flush the vm
  637. *
  638. * @ring: ring to use for flush
  639. * @vm_id: vmid number to use
  640. * @pd_addr: address of the page directory
  641. *
  642. * Emit a VM flush when it is necessary.
  643. */
  644. int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
  645. {
  646. struct amdgpu_device *adev = ring->adev;
  647. unsigned vmhub = ring->funcs->vmhub;
  648. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  649. struct amdgpu_vm_id *id = &id_mgr->ids[job->vm_id];
  650. bool gds_switch_needed = ring->funcs->emit_gds_switch && (
  651. id->gds_base != job->gds_base ||
  652. id->gds_size != job->gds_size ||
  653. id->gws_base != job->gws_base ||
  654. id->gws_size != job->gws_size ||
  655. id->oa_base != job->oa_base ||
  656. id->oa_size != job->oa_size);
  657. bool vm_flush_needed = job->vm_needs_flush;
  658. unsigned patch_offset = 0;
  659. int r;
  660. if (amdgpu_vm_had_gpu_reset(adev, id)) {
  661. gds_switch_needed = true;
  662. vm_flush_needed = true;
  663. }
  664. if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
  665. return 0;
  666. if (ring->funcs->init_cond_exec)
  667. patch_offset = amdgpu_ring_init_cond_exec(ring);
  668. if (need_pipe_sync)
  669. amdgpu_ring_emit_pipeline_sync(ring);
  670. if (ring->funcs->emit_vm_flush && vm_flush_needed) {
  671. struct dma_fence *fence;
  672. trace_amdgpu_vm_flush(ring, job->vm_id, job->vm_pd_addr);
  673. amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
  674. r = amdgpu_fence_emit(ring, &fence);
  675. if (r)
  676. return r;
  677. mutex_lock(&id_mgr->lock);
  678. dma_fence_put(id->last_flush);
  679. id->last_flush = fence;
  680. id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
  681. mutex_unlock(&id_mgr->lock);
  682. }
  683. if (ring->funcs->emit_gds_switch && gds_switch_needed) {
  684. id->gds_base = job->gds_base;
  685. id->gds_size = job->gds_size;
  686. id->gws_base = job->gws_base;
  687. id->gws_size = job->gws_size;
  688. id->oa_base = job->oa_base;
  689. id->oa_size = job->oa_size;
  690. amdgpu_ring_emit_gds_switch(ring, job->vm_id, job->gds_base,
  691. job->gds_size, job->gws_base,
  692. job->gws_size, job->oa_base,
  693. job->oa_size);
  694. }
  695. if (ring->funcs->patch_cond_exec)
  696. amdgpu_ring_patch_cond_exec(ring, patch_offset);
  697. /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
  698. if (ring->funcs->emit_switch_buffer) {
  699. amdgpu_ring_emit_switch_buffer(ring);
  700. amdgpu_ring_emit_switch_buffer(ring);
  701. }
  702. return 0;
  703. }
  704. /**
  705. * amdgpu_vm_reset_id - reset VMID to zero
  706. *
  707. * @adev: amdgpu device structure
  708. * @vm_id: vmid number to use
  709. *
  710. * Reset saved GDW, GWS and OA to force switch on next flush.
  711. */
  712. void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
  713. unsigned vmid)
  714. {
  715. struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
  716. struct amdgpu_vm_id *id = &id_mgr->ids[vmid];
  717. atomic64_set(&id->owner, 0);
  718. id->gds_base = 0;
  719. id->gds_size = 0;
  720. id->gws_base = 0;
  721. id->gws_size = 0;
  722. id->oa_base = 0;
  723. id->oa_size = 0;
  724. }
  725. /**
  726. * amdgpu_vm_reset_all_id - reset VMID to zero
  727. *
  728. * @adev: amdgpu device structure
  729. *
  730. * Reset VMID to force flush on next use
  731. */
  732. void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev)
  733. {
  734. unsigned i, j;
  735. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  736. struct amdgpu_vm_id_manager *id_mgr =
  737. &adev->vm_manager.id_mgr[i];
  738. for (j = 1; j < id_mgr->num_ids; ++j)
  739. amdgpu_vm_reset_id(adev, i, j);
  740. }
  741. }
  742. /**
  743. * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
  744. *
  745. * @vm: requested vm
  746. * @bo: requested buffer object
  747. *
  748. * Find @bo inside the requested vm.
  749. * Search inside the @bos vm list for the requested vm
  750. * Returns the found bo_va or NULL if none is found
  751. *
  752. * Object has to be reserved!
  753. */
  754. struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
  755. struct amdgpu_bo *bo)
  756. {
  757. struct amdgpu_bo_va *bo_va;
  758. list_for_each_entry(bo_va, &bo->va, base.bo_list) {
  759. if (bo_va->base.vm == vm) {
  760. return bo_va;
  761. }
  762. }
  763. return NULL;
  764. }
  765. /**
  766. * amdgpu_vm_do_set_ptes - helper to call the right asic function
  767. *
  768. * @params: see amdgpu_pte_update_params definition
  769. * @pe: addr of the page entry
  770. * @addr: dst addr to write into pe
  771. * @count: number of page entries to update
  772. * @incr: increase next addr by incr bytes
  773. * @flags: hw access flags
  774. *
  775. * Traces the parameters and calls the right asic functions
  776. * to setup the page table using the DMA.
  777. */
  778. static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
  779. uint64_t pe, uint64_t addr,
  780. unsigned count, uint32_t incr,
  781. uint64_t flags)
  782. {
  783. trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
  784. if (count < 3) {
  785. amdgpu_vm_write_pte(params->adev, params->ib, pe,
  786. addr | flags, count, incr);
  787. } else {
  788. amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
  789. count, incr, flags);
  790. }
  791. }
  792. /**
  793. * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
  794. *
  795. * @params: see amdgpu_pte_update_params definition
  796. * @pe: addr of the page entry
  797. * @addr: dst addr to write into pe
  798. * @count: number of page entries to update
  799. * @incr: increase next addr by incr bytes
  800. * @flags: hw access flags
  801. *
  802. * Traces the parameters and calls the DMA function to copy the PTEs.
  803. */
  804. static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
  805. uint64_t pe, uint64_t addr,
  806. unsigned count, uint32_t incr,
  807. uint64_t flags)
  808. {
  809. uint64_t src = (params->src + (addr >> 12) * 8);
  810. trace_amdgpu_vm_copy_ptes(pe, src, count);
  811. amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
  812. }
  813. /**
  814. * amdgpu_vm_map_gart - Resolve gart mapping of addr
  815. *
  816. * @pages_addr: optional DMA address to use for lookup
  817. * @addr: the unmapped addr
  818. *
  819. * Look up the physical address of the page that the pte resolves
  820. * to and return the pointer for the page table entry.
  821. */
  822. static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
  823. {
  824. uint64_t result;
  825. /* page table offset */
  826. result = pages_addr[addr >> PAGE_SHIFT];
  827. /* in case cpu page size != gpu page size*/
  828. result |= addr & (~PAGE_MASK);
  829. result &= 0xFFFFFFFFFFFFF000ULL;
  830. return result;
  831. }
  832. /**
  833. * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
  834. *
  835. * @params: see amdgpu_pte_update_params definition
  836. * @pe: kmap addr of the page entry
  837. * @addr: dst addr to write into pe
  838. * @count: number of page entries to update
  839. * @incr: increase next addr by incr bytes
  840. * @flags: hw access flags
  841. *
  842. * Write count number of PT/PD entries directly.
  843. */
  844. static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
  845. uint64_t pe, uint64_t addr,
  846. unsigned count, uint32_t incr,
  847. uint64_t flags)
  848. {
  849. unsigned int i;
  850. uint64_t value;
  851. trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
  852. for (i = 0; i < count; i++) {
  853. value = params->pages_addr ?
  854. amdgpu_vm_map_gart(params->pages_addr, addr) :
  855. addr;
  856. amdgpu_gart_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
  857. i, value, flags);
  858. addr += incr;
  859. }
  860. }
  861. static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  862. void *owner)
  863. {
  864. struct amdgpu_sync sync;
  865. int r;
  866. amdgpu_sync_create(&sync);
  867. amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.resv, owner);
  868. r = amdgpu_sync_wait(&sync, true);
  869. amdgpu_sync_free(&sync);
  870. return r;
  871. }
  872. /*
  873. * amdgpu_vm_update_level - update a single level in the hierarchy
  874. *
  875. * @adev: amdgpu_device pointer
  876. * @vm: requested vm
  877. * @parent: parent directory
  878. *
  879. * Makes sure all entries in @parent are up to date.
  880. * Returns 0 for success, error for failure.
  881. */
  882. static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  883. struct amdgpu_vm *vm,
  884. struct amdgpu_vm_pt *parent,
  885. unsigned level)
  886. {
  887. struct amdgpu_bo *shadow;
  888. struct amdgpu_ring *ring = NULL;
  889. uint64_t pd_addr, shadow_addr = 0;
  890. uint32_t incr = amdgpu_vm_bo_size(adev, level + 1);
  891. uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
  892. unsigned count = 0, pt_idx, ndw = 0;
  893. struct amdgpu_job *job;
  894. struct amdgpu_pte_update_params params;
  895. struct dma_fence *fence = NULL;
  896. int r;
  897. if (!parent->entries)
  898. return 0;
  899. memset(&params, 0, sizeof(params));
  900. params.adev = adev;
  901. shadow = parent->bo->shadow;
  902. if (vm->use_cpu_for_update) {
  903. pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
  904. r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
  905. if (unlikely(r))
  906. return r;
  907. params.func = amdgpu_vm_cpu_set_ptes;
  908. } else {
  909. ring = container_of(vm->entity.sched, struct amdgpu_ring,
  910. sched);
  911. /* padding, etc. */
  912. ndw = 64;
  913. /* assume the worst case */
  914. ndw += parent->last_entry_used * 6;
  915. pd_addr = amdgpu_bo_gpu_offset(parent->bo);
  916. if (shadow) {
  917. shadow_addr = amdgpu_bo_gpu_offset(shadow);
  918. ndw *= 2;
  919. } else {
  920. shadow_addr = 0;
  921. }
  922. r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
  923. if (r)
  924. return r;
  925. params.ib = &job->ibs[0];
  926. params.func = amdgpu_vm_do_set_ptes;
  927. }
  928. /* walk over the address space and update the directory */
  929. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  930. struct amdgpu_bo *bo = parent->entries[pt_idx].bo;
  931. uint64_t pde, pt;
  932. if (bo == NULL)
  933. continue;
  934. pt = amdgpu_bo_gpu_offset(bo);
  935. pt = amdgpu_gart_get_vm_pde(adev, pt);
  936. /* Don't update huge pages here */
  937. if ((parent->entries[pt_idx].addr & AMDGPU_PDE_PTE) ||
  938. parent->entries[pt_idx].addr == (pt | AMDGPU_PTE_VALID))
  939. continue;
  940. parent->entries[pt_idx].addr = pt | AMDGPU_PTE_VALID;
  941. pde = pd_addr + pt_idx * 8;
  942. if (((last_pde + 8 * count) != pde) ||
  943. ((last_pt + incr * count) != pt) ||
  944. (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
  945. if (count) {
  946. if (shadow)
  947. params.func(&params,
  948. last_shadow,
  949. last_pt, count,
  950. incr,
  951. AMDGPU_PTE_VALID);
  952. params.func(&params, last_pde,
  953. last_pt, count, incr,
  954. AMDGPU_PTE_VALID);
  955. }
  956. count = 1;
  957. last_pde = pde;
  958. last_shadow = shadow_addr + pt_idx * 8;
  959. last_pt = pt;
  960. } else {
  961. ++count;
  962. }
  963. }
  964. if (count) {
  965. if (vm->root.bo->shadow)
  966. params.func(&params, last_shadow, last_pt,
  967. count, incr, AMDGPU_PTE_VALID);
  968. params.func(&params, last_pde, last_pt,
  969. count, incr, AMDGPU_PTE_VALID);
  970. }
  971. if (!vm->use_cpu_for_update) {
  972. if (params.ib->length_dw == 0) {
  973. amdgpu_job_free(job);
  974. } else {
  975. amdgpu_ring_pad_ib(ring, params.ib);
  976. amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
  977. AMDGPU_FENCE_OWNER_VM);
  978. if (shadow)
  979. amdgpu_sync_resv(adev, &job->sync,
  980. shadow->tbo.resv,
  981. AMDGPU_FENCE_OWNER_VM);
  982. WARN_ON(params.ib->length_dw > ndw);
  983. r = amdgpu_job_submit(job, ring, &vm->entity,
  984. AMDGPU_FENCE_OWNER_VM, &fence);
  985. if (r)
  986. goto error_free;
  987. amdgpu_bo_fence(parent->bo, fence, true);
  988. dma_fence_put(vm->last_dir_update);
  989. vm->last_dir_update = dma_fence_get(fence);
  990. dma_fence_put(fence);
  991. }
  992. }
  993. /*
  994. * Recurse into the subdirectories. This recursion is harmless because
  995. * we only have a maximum of 5 layers.
  996. */
  997. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  998. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  999. if (!entry->bo)
  1000. continue;
  1001. r = amdgpu_vm_update_level(adev, vm, entry, level + 1);
  1002. if (r)
  1003. return r;
  1004. }
  1005. return 0;
  1006. error_free:
  1007. amdgpu_job_free(job);
  1008. return r;
  1009. }
  1010. /*
  1011. * amdgpu_vm_invalidate_level - mark all PD levels as invalid
  1012. *
  1013. * @parent: parent PD
  1014. *
  1015. * Mark all PD level as invalid after an error.
  1016. */
  1017. static void amdgpu_vm_invalidate_level(struct amdgpu_vm_pt *parent)
  1018. {
  1019. unsigned pt_idx;
  1020. /*
  1021. * Recurse into the subdirectories. This recursion is harmless because
  1022. * we only have a maximum of 5 layers.
  1023. */
  1024. for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
  1025. struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
  1026. if (!entry->bo)
  1027. continue;
  1028. entry->addr = ~0ULL;
  1029. amdgpu_vm_invalidate_level(entry);
  1030. }
  1031. }
  1032. /*
  1033. * amdgpu_vm_update_directories - make sure that all directories are valid
  1034. *
  1035. * @adev: amdgpu_device pointer
  1036. * @vm: requested vm
  1037. *
  1038. * Makes sure all directories are up to date.
  1039. * Returns 0 for success, error for failure.
  1040. */
  1041. int amdgpu_vm_update_directories(struct amdgpu_device *adev,
  1042. struct amdgpu_vm *vm)
  1043. {
  1044. int r = 0;
  1045. r = amdgpu_vm_update_level(adev, vm, &vm->root, 0);
  1046. if (r)
  1047. amdgpu_vm_invalidate_level(&vm->root);
  1048. if (vm->use_cpu_for_update) {
  1049. /* Flush HDP */
  1050. mb();
  1051. amdgpu_gart_flush_gpu_tlb(adev, 0);
  1052. }
  1053. return r;
  1054. }
  1055. /**
  1056. * amdgpu_vm_find_entry - find the entry for an address
  1057. *
  1058. * @p: see amdgpu_pte_update_params definition
  1059. * @addr: virtual address in question
  1060. * @entry: resulting entry or NULL
  1061. * @parent: parent entry
  1062. *
  1063. * Find the vm_pt entry and it's parent for the given address.
  1064. */
  1065. void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
  1066. struct amdgpu_vm_pt **entry,
  1067. struct amdgpu_vm_pt **parent)
  1068. {
  1069. unsigned idx, level = p->adev->vm_manager.num_level;
  1070. *parent = NULL;
  1071. *entry = &p->vm->root;
  1072. while ((*entry)->entries) {
  1073. idx = addr >> (p->adev->vm_manager.block_size * level--);
  1074. idx %= amdgpu_bo_size((*entry)->bo) / 8;
  1075. *parent = *entry;
  1076. *entry = &(*entry)->entries[idx];
  1077. }
  1078. if (level)
  1079. *entry = NULL;
  1080. }
  1081. /**
  1082. * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages
  1083. *
  1084. * @p: see amdgpu_pte_update_params definition
  1085. * @entry: vm_pt entry to check
  1086. * @parent: parent entry
  1087. * @nptes: number of PTEs updated with this operation
  1088. * @dst: destination address where the PTEs should point to
  1089. * @flags: access flags fro the PTEs
  1090. *
  1091. * Check if we can update the PD with a huge page.
  1092. */
  1093. static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
  1094. struct amdgpu_vm_pt *entry,
  1095. struct amdgpu_vm_pt *parent,
  1096. unsigned nptes, uint64_t dst,
  1097. uint64_t flags)
  1098. {
  1099. bool use_cpu_update = (p->func == amdgpu_vm_cpu_set_ptes);
  1100. uint64_t pd_addr, pde;
  1101. /* In the case of a mixed PT the PDE must point to it*/
  1102. if (p->adev->asic_type < CHIP_VEGA10 ||
  1103. nptes != AMDGPU_VM_PTE_COUNT(p->adev) ||
  1104. p->src ||
  1105. !(flags & AMDGPU_PTE_VALID)) {
  1106. dst = amdgpu_bo_gpu_offset(entry->bo);
  1107. dst = amdgpu_gart_get_vm_pde(p->adev, dst);
  1108. flags = AMDGPU_PTE_VALID;
  1109. } else {
  1110. /* Set the huge page flag to stop scanning at this PDE */
  1111. flags |= AMDGPU_PDE_PTE;
  1112. }
  1113. if (entry->addr == (dst | flags))
  1114. return;
  1115. entry->addr = (dst | flags);
  1116. if (use_cpu_update) {
  1117. /* In case a huge page is replaced with a system
  1118. * memory mapping, p->pages_addr != NULL and
  1119. * amdgpu_vm_cpu_set_ptes would try to translate dst
  1120. * through amdgpu_vm_map_gart. But dst is already a
  1121. * GPU address (of the page table). Disable
  1122. * amdgpu_vm_map_gart temporarily.
  1123. */
  1124. dma_addr_t *tmp;
  1125. tmp = p->pages_addr;
  1126. p->pages_addr = NULL;
  1127. pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
  1128. pde = pd_addr + (entry - parent->entries) * 8;
  1129. amdgpu_vm_cpu_set_ptes(p, pde, dst, 1, 0, flags);
  1130. p->pages_addr = tmp;
  1131. } else {
  1132. if (parent->bo->shadow) {
  1133. pd_addr = amdgpu_bo_gpu_offset(parent->bo->shadow);
  1134. pde = pd_addr + (entry - parent->entries) * 8;
  1135. amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
  1136. }
  1137. pd_addr = amdgpu_bo_gpu_offset(parent->bo);
  1138. pde = pd_addr + (entry - parent->entries) * 8;
  1139. amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
  1140. }
  1141. }
  1142. /**
  1143. * amdgpu_vm_update_ptes - make sure that page tables are valid
  1144. *
  1145. * @params: see amdgpu_pte_update_params definition
  1146. * @vm: requested vm
  1147. * @start: start of GPU address range
  1148. * @end: end of GPU address range
  1149. * @dst: destination address to map to, the next dst inside the function
  1150. * @flags: mapping flags
  1151. *
  1152. * Update the page tables in the range @start - @end.
  1153. * Returns 0 for success, -EINVAL for failure.
  1154. */
  1155. static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
  1156. uint64_t start, uint64_t end,
  1157. uint64_t dst, uint64_t flags)
  1158. {
  1159. struct amdgpu_device *adev = params->adev;
  1160. const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
  1161. uint64_t addr, pe_start;
  1162. struct amdgpu_bo *pt;
  1163. unsigned nptes;
  1164. bool use_cpu_update = (params->func == amdgpu_vm_cpu_set_ptes);
  1165. /* walk over the address space and update the page tables */
  1166. for (addr = start; addr < end; addr += nptes,
  1167. dst += nptes * AMDGPU_GPU_PAGE_SIZE) {
  1168. struct amdgpu_vm_pt *entry, *parent;
  1169. amdgpu_vm_get_entry(params, addr, &entry, &parent);
  1170. if (!entry)
  1171. return -ENOENT;
  1172. if ((addr & ~mask) == (end & ~mask))
  1173. nptes = end - addr;
  1174. else
  1175. nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
  1176. amdgpu_vm_handle_huge_pages(params, entry, parent,
  1177. nptes, dst, flags);
  1178. /* We don't need to update PTEs for huge pages */
  1179. if (entry->addr & AMDGPU_PDE_PTE)
  1180. continue;
  1181. pt = entry->bo;
  1182. if (use_cpu_update) {
  1183. pe_start = (unsigned long)amdgpu_bo_kptr(pt);
  1184. } else {
  1185. if (pt->shadow) {
  1186. pe_start = amdgpu_bo_gpu_offset(pt->shadow);
  1187. pe_start += (addr & mask) * 8;
  1188. params->func(params, pe_start, dst, nptes,
  1189. AMDGPU_GPU_PAGE_SIZE, flags);
  1190. }
  1191. pe_start = amdgpu_bo_gpu_offset(pt);
  1192. }
  1193. pe_start += (addr & mask) * 8;
  1194. params->func(params, pe_start, dst, nptes,
  1195. AMDGPU_GPU_PAGE_SIZE, flags);
  1196. }
  1197. return 0;
  1198. }
  1199. /*
  1200. * amdgpu_vm_frag_ptes - add fragment information to PTEs
  1201. *
  1202. * @params: see amdgpu_pte_update_params definition
  1203. * @vm: requested vm
  1204. * @start: first PTE to handle
  1205. * @end: last PTE to handle
  1206. * @dst: addr those PTEs should point to
  1207. * @flags: hw mapping flags
  1208. * Returns 0 for success, -EINVAL for failure.
  1209. */
  1210. static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params *params,
  1211. uint64_t start, uint64_t end,
  1212. uint64_t dst, uint64_t flags)
  1213. {
  1214. int r;
  1215. /**
  1216. * The MC L1 TLB supports variable sized pages, based on a fragment
  1217. * field in the PTE. When this field is set to a non-zero value, page
  1218. * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
  1219. * flags are considered valid for all PTEs within the fragment range
  1220. * and corresponding mappings are assumed to be physically contiguous.
  1221. *
  1222. * The L1 TLB can store a single PTE for the whole fragment,
  1223. * significantly increasing the space available for translation
  1224. * caching. This leads to large improvements in throughput when the
  1225. * TLB is under pressure.
  1226. *
  1227. * The L2 TLB distributes small and large fragments into two
  1228. * asymmetric partitions. The large fragment cache is significantly
  1229. * larger. Thus, we try to use large fragments wherever possible.
  1230. * Userspace can support this by aligning virtual base address and
  1231. * allocation size to the fragment size.
  1232. */
  1233. unsigned pages_per_frag = params->adev->vm_manager.fragment_size;
  1234. uint64_t frag_flags = AMDGPU_PTE_FRAG(pages_per_frag);
  1235. uint64_t frag_align = 1 << pages_per_frag;
  1236. uint64_t frag_start = ALIGN(start, frag_align);
  1237. uint64_t frag_end = end & ~(frag_align - 1);
  1238. /* system pages are non continuously */
  1239. if (params->src || !(flags & AMDGPU_PTE_VALID) ||
  1240. (frag_start >= frag_end))
  1241. return amdgpu_vm_update_ptes(params, start, end, dst, flags);
  1242. /* handle the 4K area at the beginning */
  1243. if (start != frag_start) {
  1244. r = amdgpu_vm_update_ptes(params, start, frag_start,
  1245. dst, flags);
  1246. if (r)
  1247. return r;
  1248. dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
  1249. }
  1250. /* handle the area in the middle */
  1251. r = amdgpu_vm_update_ptes(params, frag_start, frag_end, dst,
  1252. flags | frag_flags);
  1253. if (r)
  1254. return r;
  1255. /* handle the 4K area at the end */
  1256. if (frag_end != end) {
  1257. dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
  1258. r = amdgpu_vm_update_ptes(params, frag_end, end, dst, flags);
  1259. }
  1260. return r;
  1261. }
  1262. /**
  1263. * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
  1264. *
  1265. * @adev: amdgpu_device pointer
  1266. * @exclusive: fence we need to sync to
  1267. * @src: address where to copy page table entries from
  1268. * @pages_addr: DMA addresses to use for mapping
  1269. * @vm: requested vm
  1270. * @start: start of mapped range
  1271. * @last: last mapped entry
  1272. * @flags: flags for the entries
  1273. * @addr: addr to set the area to
  1274. * @fence: optional resulting fence
  1275. *
  1276. * Fill in the page table entries between @start and @last.
  1277. * Returns 0 for success, -EINVAL for failure.
  1278. */
  1279. static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
  1280. struct dma_fence *exclusive,
  1281. uint64_t src,
  1282. dma_addr_t *pages_addr,
  1283. struct amdgpu_vm *vm,
  1284. uint64_t start, uint64_t last,
  1285. uint64_t flags, uint64_t addr,
  1286. struct dma_fence **fence)
  1287. {
  1288. struct amdgpu_ring *ring;
  1289. void *owner = AMDGPU_FENCE_OWNER_VM;
  1290. unsigned nptes, ncmds, ndw;
  1291. struct amdgpu_job *job;
  1292. struct amdgpu_pte_update_params params;
  1293. struct dma_fence *f = NULL;
  1294. int r;
  1295. memset(&params, 0, sizeof(params));
  1296. params.adev = adev;
  1297. params.vm = vm;
  1298. params.src = src;
  1299. /* sync to everything on unmapping */
  1300. if (!(flags & AMDGPU_PTE_VALID))
  1301. owner = AMDGPU_FENCE_OWNER_UNDEFINED;
  1302. if (vm->use_cpu_for_update) {
  1303. /* params.src is used as flag to indicate system Memory */
  1304. if (pages_addr)
  1305. params.src = ~0;
  1306. /* Wait for PT BOs to be free. PTs share the same resv. object
  1307. * as the root PD BO
  1308. */
  1309. r = amdgpu_vm_wait_pd(adev, vm, owner);
  1310. if (unlikely(r))
  1311. return r;
  1312. params.func = amdgpu_vm_cpu_set_ptes;
  1313. params.pages_addr = pages_addr;
  1314. return amdgpu_vm_frag_ptes(&params, start, last + 1,
  1315. addr, flags);
  1316. }
  1317. ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
  1318. nptes = last - start + 1;
  1319. /*
  1320. * reserve space for one command every (1 << BLOCK_SIZE)
  1321. * entries or 2k dwords (whatever is smaller)
  1322. */
  1323. ncmds = (nptes >> min(adev->vm_manager.block_size, 11u)) + 1;
  1324. /* padding, etc. */
  1325. ndw = 64;
  1326. /* one PDE write for each huge page */
  1327. ndw += ((nptes >> adev->vm_manager.block_size) + 1) * 6;
  1328. if (src) {
  1329. /* only copy commands needed */
  1330. ndw += ncmds * 7;
  1331. params.func = amdgpu_vm_do_copy_ptes;
  1332. } else if (pages_addr) {
  1333. /* copy commands needed */
  1334. ndw += ncmds * 7;
  1335. /* and also PTEs */
  1336. ndw += nptes * 2;
  1337. params.func = amdgpu_vm_do_copy_ptes;
  1338. } else {
  1339. /* set page commands needed */
  1340. ndw += ncmds * 10;
  1341. /* two extra commands for begin/end of fragment */
  1342. ndw += 2 * 10;
  1343. params.func = amdgpu_vm_do_set_ptes;
  1344. }
  1345. r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
  1346. if (r)
  1347. return r;
  1348. params.ib = &job->ibs[0];
  1349. if (!src && pages_addr) {
  1350. uint64_t *pte;
  1351. unsigned i;
  1352. /* Put the PTEs at the end of the IB. */
  1353. i = ndw - nptes * 2;
  1354. pte= (uint64_t *)&(job->ibs->ptr[i]);
  1355. params.src = job->ibs->gpu_addr + i * 4;
  1356. for (i = 0; i < nptes; ++i) {
  1357. pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
  1358. AMDGPU_GPU_PAGE_SIZE);
  1359. pte[i] |= flags;
  1360. }
  1361. addr = 0;
  1362. }
  1363. r = amdgpu_sync_fence(adev, &job->sync, exclusive);
  1364. if (r)
  1365. goto error_free;
  1366. r = amdgpu_sync_resv(adev, &job->sync, vm->root.bo->tbo.resv,
  1367. owner);
  1368. if (r)
  1369. goto error_free;
  1370. r = reservation_object_reserve_shared(vm->root.bo->tbo.resv);
  1371. if (r)
  1372. goto error_free;
  1373. r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
  1374. if (r)
  1375. goto error_free;
  1376. amdgpu_ring_pad_ib(ring, params.ib);
  1377. WARN_ON(params.ib->length_dw > ndw);
  1378. r = amdgpu_job_submit(job, ring, &vm->entity,
  1379. AMDGPU_FENCE_OWNER_VM, &f);
  1380. if (r)
  1381. goto error_free;
  1382. amdgpu_bo_fence(vm->root.bo, f, true);
  1383. dma_fence_put(*fence);
  1384. *fence = f;
  1385. return 0;
  1386. error_free:
  1387. amdgpu_job_free(job);
  1388. amdgpu_vm_invalidate_level(&vm->root);
  1389. return r;
  1390. }
  1391. /**
  1392. * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
  1393. *
  1394. * @adev: amdgpu_device pointer
  1395. * @exclusive: fence we need to sync to
  1396. * @pages_addr: DMA addresses to use for mapping
  1397. * @vm: requested vm
  1398. * @mapping: mapped range and flags to use for the update
  1399. * @flags: HW flags for the mapping
  1400. * @nodes: array of drm_mm_nodes with the MC addresses
  1401. * @fence: optional resulting fence
  1402. *
  1403. * Split the mapping into smaller chunks so that each update fits
  1404. * into a SDMA IB.
  1405. * Returns 0 for success, -EINVAL for failure.
  1406. */
  1407. static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
  1408. struct dma_fence *exclusive,
  1409. dma_addr_t *pages_addr,
  1410. struct amdgpu_vm *vm,
  1411. struct amdgpu_bo_va_mapping *mapping,
  1412. uint64_t flags,
  1413. struct drm_mm_node *nodes,
  1414. struct dma_fence **fence)
  1415. {
  1416. uint64_t pfn, src = 0, start = mapping->start;
  1417. int r;
  1418. /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
  1419. * but in case of something, we filter the flags in first place
  1420. */
  1421. if (!(mapping->flags & AMDGPU_PTE_READABLE))
  1422. flags &= ~AMDGPU_PTE_READABLE;
  1423. if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
  1424. flags &= ~AMDGPU_PTE_WRITEABLE;
  1425. flags &= ~AMDGPU_PTE_EXECUTABLE;
  1426. flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
  1427. flags &= ~AMDGPU_PTE_MTYPE_MASK;
  1428. flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);
  1429. if ((mapping->flags & AMDGPU_PTE_PRT) &&
  1430. (adev->asic_type >= CHIP_VEGA10)) {
  1431. flags |= AMDGPU_PTE_PRT;
  1432. flags &= ~AMDGPU_PTE_VALID;
  1433. }
  1434. trace_amdgpu_vm_bo_update(mapping);
  1435. pfn = mapping->offset >> PAGE_SHIFT;
  1436. if (nodes) {
  1437. while (pfn >= nodes->size) {
  1438. pfn -= nodes->size;
  1439. ++nodes;
  1440. }
  1441. }
  1442. do {
  1443. uint64_t max_entries;
  1444. uint64_t addr, last;
  1445. if (nodes) {
  1446. addr = nodes->start << PAGE_SHIFT;
  1447. max_entries = (nodes->size - pfn) *
  1448. (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
  1449. } else {
  1450. addr = 0;
  1451. max_entries = S64_MAX;
  1452. }
  1453. if (pages_addr) {
  1454. max_entries = min(max_entries, 16ull * 1024ull);
  1455. addr = 0;
  1456. } else if (flags & AMDGPU_PTE_VALID) {
  1457. addr += adev->vm_manager.vram_base_offset;
  1458. }
  1459. addr += pfn << PAGE_SHIFT;
  1460. last = min((uint64_t)mapping->last, start + max_entries - 1);
  1461. r = amdgpu_vm_bo_update_mapping(adev, exclusive,
  1462. src, pages_addr, vm,
  1463. start, last, flags, addr,
  1464. fence);
  1465. if (r)
  1466. return r;
  1467. pfn += last - start + 1;
  1468. if (nodes && nodes->size == pfn) {
  1469. pfn = 0;
  1470. ++nodes;
  1471. }
  1472. start = last + 1;
  1473. } while (unlikely(start != mapping->last + 1));
  1474. return 0;
  1475. }
  1476. /**
  1477. * amdgpu_vm_bo_update - update all BO mappings in the vm page table
  1478. *
  1479. * @adev: amdgpu_device pointer
  1480. * @bo_va: requested BO and VM object
  1481. * @clear: if true clear the entries
  1482. *
  1483. * Fill in the page table entries for @bo_va.
  1484. * Returns 0 for success, -EINVAL for failure.
  1485. */
  1486. int amdgpu_vm_bo_update(struct amdgpu_device *adev,
  1487. struct amdgpu_bo_va *bo_va,
  1488. bool clear)
  1489. {
  1490. struct amdgpu_bo *bo = bo_va->base.bo;
  1491. struct amdgpu_vm *vm = bo_va->base.vm;
  1492. struct amdgpu_bo_va_mapping *mapping;
  1493. dma_addr_t *pages_addr = NULL;
  1494. struct ttm_mem_reg *mem;
  1495. struct drm_mm_node *nodes;
  1496. struct dma_fence *exclusive;
  1497. uint64_t flags;
  1498. int r;
  1499. if (clear || !bo_va->base.bo) {
  1500. mem = NULL;
  1501. nodes = NULL;
  1502. exclusive = NULL;
  1503. } else {
  1504. struct ttm_dma_tt *ttm;
  1505. mem = &bo_va->base.bo->tbo.mem;
  1506. nodes = mem->mm_node;
  1507. if (mem->mem_type == TTM_PL_TT) {
  1508. ttm = container_of(bo_va->base.bo->tbo.ttm,
  1509. struct ttm_dma_tt, ttm);
  1510. pages_addr = ttm->dma_address;
  1511. }
  1512. exclusive = reservation_object_get_excl(bo->tbo.resv);
  1513. }
  1514. if (bo)
  1515. flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
  1516. else
  1517. flags = 0x0;
  1518. spin_lock(&vm->status_lock);
  1519. if (!list_empty(&bo_va->base.vm_status))
  1520. list_splice_init(&bo_va->valids, &bo_va->invalids);
  1521. spin_unlock(&vm->status_lock);
  1522. list_for_each_entry(mapping, &bo_va->invalids, list) {
  1523. r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
  1524. mapping, flags, nodes,
  1525. &bo_va->last_pt_update);
  1526. if (r)
  1527. return r;
  1528. }
  1529. if (trace_amdgpu_vm_bo_mapping_enabled()) {
  1530. list_for_each_entry(mapping, &bo_va->valids, list)
  1531. trace_amdgpu_vm_bo_mapping(mapping);
  1532. list_for_each_entry(mapping, &bo_va->invalids, list)
  1533. trace_amdgpu_vm_bo_mapping(mapping);
  1534. }
  1535. spin_lock(&vm->status_lock);
  1536. list_splice_init(&bo_va->invalids, &bo_va->valids);
  1537. list_del_init(&bo_va->base.vm_status);
  1538. if (clear)
  1539. list_add(&bo_va->base.vm_status, &vm->cleared);
  1540. spin_unlock(&vm->status_lock);
  1541. if (vm->use_cpu_for_update) {
  1542. /* Flush HDP */
  1543. mb();
  1544. amdgpu_gart_flush_gpu_tlb(adev, 0);
  1545. }
  1546. return 0;
  1547. }
  1548. /**
  1549. * amdgpu_vm_update_prt_state - update the global PRT state
  1550. */
  1551. static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
  1552. {
  1553. unsigned long flags;
  1554. bool enable;
  1555. spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
  1556. enable = !!atomic_read(&adev->vm_manager.num_prt_users);
  1557. adev->gart.gart_funcs->set_prt(adev, enable);
  1558. spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
  1559. }
  1560. /**
  1561. * amdgpu_vm_prt_get - add a PRT user
  1562. */
  1563. static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
  1564. {
  1565. if (!adev->gart.gart_funcs->set_prt)
  1566. return;
  1567. if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
  1568. amdgpu_vm_update_prt_state(adev);
  1569. }
  1570. /**
  1571. * amdgpu_vm_prt_put - drop a PRT user
  1572. */
  1573. static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
  1574. {
  1575. if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
  1576. amdgpu_vm_update_prt_state(adev);
  1577. }
  1578. /**
  1579. * amdgpu_vm_prt_cb - callback for updating the PRT status
  1580. */
  1581. static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
  1582. {
  1583. struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
  1584. amdgpu_vm_prt_put(cb->adev);
  1585. kfree(cb);
  1586. }
  1587. /**
  1588. * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
  1589. */
  1590. static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
  1591. struct dma_fence *fence)
  1592. {
  1593. struct amdgpu_prt_cb *cb;
  1594. if (!adev->gart.gart_funcs->set_prt)
  1595. return;
  1596. cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
  1597. if (!cb) {
  1598. /* Last resort when we are OOM */
  1599. if (fence)
  1600. dma_fence_wait(fence, false);
  1601. amdgpu_vm_prt_put(adev);
  1602. } else {
  1603. cb->adev = adev;
  1604. if (!fence || dma_fence_add_callback(fence, &cb->cb,
  1605. amdgpu_vm_prt_cb))
  1606. amdgpu_vm_prt_cb(fence, &cb->cb);
  1607. }
  1608. }
  1609. /**
  1610. * amdgpu_vm_free_mapping - free a mapping
  1611. *
  1612. * @adev: amdgpu_device pointer
  1613. * @vm: requested vm
  1614. * @mapping: mapping to be freed
  1615. * @fence: fence of the unmap operation
  1616. *
  1617. * Free a mapping and make sure we decrease the PRT usage count if applicable.
  1618. */
  1619. static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
  1620. struct amdgpu_vm *vm,
  1621. struct amdgpu_bo_va_mapping *mapping,
  1622. struct dma_fence *fence)
  1623. {
  1624. if (mapping->flags & AMDGPU_PTE_PRT)
  1625. amdgpu_vm_add_prt_cb(adev, fence);
  1626. kfree(mapping);
  1627. }
  1628. /**
  1629. * amdgpu_vm_prt_fini - finish all prt mappings
  1630. *
  1631. * @adev: amdgpu_device pointer
  1632. * @vm: requested vm
  1633. *
  1634. * Register a cleanup callback to disable PRT support after VM dies.
  1635. */
  1636. static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  1637. {
  1638. struct reservation_object *resv = vm->root.bo->tbo.resv;
  1639. struct dma_fence *excl, **shared;
  1640. unsigned i, shared_count;
  1641. int r;
  1642. r = reservation_object_get_fences_rcu(resv, &excl,
  1643. &shared_count, &shared);
  1644. if (r) {
  1645. /* Not enough memory to grab the fence list, as last resort
  1646. * block for all the fences to complete.
  1647. */
  1648. reservation_object_wait_timeout_rcu(resv, true, false,
  1649. MAX_SCHEDULE_TIMEOUT);
  1650. return;
  1651. }
  1652. /* Add a callback for each fence in the reservation object */
  1653. amdgpu_vm_prt_get(adev);
  1654. amdgpu_vm_add_prt_cb(adev, excl);
  1655. for (i = 0; i < shared_count; ++i) {
  1656. amdgpu_vm_prt_get(adev);
  1657. amdgpu_vm_add_prt_cb(adev, shared[i]);
  1658. }
  1659. kfree(shared);
  1660. }
  1661. /**
  1662. * amdgpu_vm_clear_freed - clear freed BOs in the PT
  1663. *
  1664. * @adev: amdgpu_device pointer
  1665. * @vm: requested vm
  1666. * @fence: optional resulting fence (unchanged if no work needed to be done
  1667. * or if an error occurred)
  1668. *
  1669. * Make sure all freed BOs are cleared in the PT.
  1670. * Returns 0 for success.
  1671. *
  1672. * PTs have to be reserved and mutex must be locked!
  1673. */
  1674. int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
  1675. struct amdgpu_vm *vm,
  1676. struct dma_fence **fence)
  1677. {
  1678. struct amdgpu_bo_va_mapping *mapping;
  1679. struct dma_fence *f = NULL;
  1680. int r;
  1681. uint64_t init_pte_value = 0;
  1682. while (!list_empty(&vm->freed)) {
  1683. mapping = list_first_entry(&vm->freed,
  1684. struct amdgpu_bo_va_mapping, list);
  1685. list_del(&mapping->list);
  1686. if (vm->pte_support_ats)
  1687. init_pte_value = AMDGPU_PTE_SYSTEM;
  1688. r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
  1689. mapping->start, mapping->last,
  1690. init_pte_value, 0, &f);
  1691. amdgpu_vm_free_mapping(adev, vm, mapping, f);
  1692. if (r) {
  1693. dma_fence_put(f);
  1694. return r;
  1695. }
  1696. }
  1697. if (fence && f) {
  1698. dma_fence_put(*fence);
  1699. *fence = f;
  1700. } else {
  1701. dma_fence_put(f);
  1702. }
  1703. return 0;
  1704. }
  1705. /**
  1706. * amdgpu_vm_clear_moved - clear moved BOs in the PT
  1707. *
  1708. * @adev: amdgpu_device pointer
  1709. * @vm: requested vm
  1710. *
  1711. * Make sure all moved BOs are cleared in the PT.
  1712. * Returns 0 for success.
  1713. *
  1714. * PTs have to be reserved and mutex must be locked!
  1715. */
  1716. int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  1717. struct amdgpu_sync *sync)
  1718. {
  1719. struct amdgpu_bo_va *bo_va = NULL;
  1720. int r = 0;
  1721. spin_lock(&vm->status_lock);
  1722. while (!list_empty(&vm->moved)) {
  1723. bo_va = list_first_entry(&vm->moved,
  1724. struct amdgpu_bo_va, base.vm_status);
  1725. spin_unlock(&vm->status_lock);
  1726. r = amdgpu_vm_bo_update(adev, bo_va, true);
  1727. if (r)
  1728. return r;
  1729. spin_lock(&vm->status_lock);
  1730. }
  1731. spin_unlock(&vm->status_lock);
  1732. if (bo_va)
  1733. r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
  1734. return r;
  1735. }
  1736. /**
  1737. * amdgpu_vm_bo_add - add a bo to a specific vm
  1738. *
  1739. * @adev: amdgpu_device pointer
  1740. * @vm: requested vm
  1741. * @bo: amdgpu buffer object
  1742. *
  1743. * Add @bo into the requested vm.
  1744. * Add @bo to the list of bos associated with the vm
  1745. * Returns newly added bo_va or NULL for failure
  1746. *
  1747. * Object has to be reserved!
  1748. */
  1749. struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
  1750. struct amdgpu_vm *vm,
  1751. struct amdgpu_bo *bo)
  1752. {
  1753. struct amdgpu_bo_va *bo_va;
  1754. bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
  1755. if (bo_va == NULL) {
  1756. return NULL;
  1757. }
  1758. bo_va->base.vm = vm;
  1759. bo_va->base.bo = bo;
  1760. INIT_LIST_HEAD(&bo_va->base.bo_list);
  1761. INIT_LIST_HEAD(&bo_va->base.vm_status);
  1762. bo_va->ref_count = 1;
  1763. INIT_LIST_HEAD(&bo_va->valids);
  1764. INIT_LIST_HEAD(&bo_va->invalids);
  1765. if (bo)
  1766. list_add_tail(&bo_va->base.bo_list, &bo->va);
  1767. return bo_va;
  1768. }
  1769. /**
  1770. * amdgpu_vm_bo_map - map bo inside a vm
  1771. *
  1772. * @adev: amdgpu_device pointer
  1773. * @bo_va: bo_va to store the address
  1774. * @saddr: where to map the BO
  1775. * @offset: requested offset in the BO
  1776. * @flags: attributes of pages (read/write/valid/etc.)
  1777. *
  1778. * Add a mapping of the BO at the specefied addr into the VM.
  1779. * Returns 0 for success, error for failure.
  1780. *
  1781. * Object has to be reserved and unreserved outside!
  1782. */
  1783. int amdgpu_vm_bo_map(struct amdgpu_device *adev,
  1784. struct amdgpu_bo_va *bo_va,
  1785. uint64_t saddr, uint64_t offset,
  1786. uint64_t size, uint64_t flags)
  1787. {
  1788. struct amdgpu_bo_va_mapping *mapping, *tmp;
  1789. struct amdgpu_bo *bo = bo_va->base.bo;
  1790. struct amdgpu_vm *vm = bo_va->base.vm;
  1791. uint64_t eaddr;
  1792. /* validate the parameters */
  1793. if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
  1794. size == 0 || size & ~PAGE_MASK)
  1795. return -EINVAL;
  1796. /* make sure object fit at this offset */
  1797. eaddr = saddr + size - 1;
  1798. if (saddr >= eaddr ||
  1799. (bo && offset + size > amdgpu_bo_size(bo)))
  1800. return -EINVAL;
  1801. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1802. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1803. tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
  1804. if (tmp) {
  1805. /* bo and tmp overlap, invalid addr */
  1806. dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
  1807. "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
  1808. tmp->start, tmp->last + 1);
  1809. return -EINVAL;
  1810. }
  1811. mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
  1812. if (!mapping)
  1813. return -ENOMEM;
  1814. INIT_LIST_HEAD(&mapping->list);
  1815. mapping->start = saddr;
  1816. mapping->last = eaddr;
  1817. mapping->offset = offset;
  1818. mapping->flags = flags;
  1819. list_add(&mapping->list, &bo_va->invalids);
  1820. amdgpu_vm_it_insert(mapping, &vm->va);
  1821. if (flags & AMDGPU_PTE_PRT)
  1822. amdgpu_vm_prt_get(adev);
  1823. return 0;
  1824. }
  1825. /**
  1826. * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
  1827. *
  1828. * @adev: amdgpu_device pointer
  1829. * @bo_va: bo_va to store the address
  1830. * @saddr: where to map the BO
  1831. * @offset: requested offset in the BO
  1832. * @flags: attributes of pages (read/write/valid/etc.)
  1833. *
  1834. * Add a mapping of the BO at the specefied addr into the VM. Replace existing
  1835. * mappings as we do so.
  1836. * Returns 0 for success, error for failure.
  1837. *
  1838. * Object has to be reserved and unreserved outside!
  1839. */
  1840. int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
  1841. struct amdgpu_bo_va *bo_va,
  1842. uint64_t saddr, uint64_t offset,
  1843. uint64_t size, uint64_t flags)
  1844. {
  1845. struct amdgpu_bo_va_mapping *mapping;
  1846. struct amdgpu_bo *bo = bo_va->base.bo;
  1847. struct amdgpu_vm *vm = bo_va->base.vm;
  1848. uint64_t eaddr;
  1849. int r;
  1850. /* validate the parameters */
  1851. if (saddr & ~PAGE_MASK || offset & ~PAGE_MASK ||
  1852. size == 0 || size & ~PAGE_MASK)
  1853. return -EINVAL;
  1854. /* make sure object fit at this offset */
  1855. eaddr = saddr + size - 1;
  1856. if (saddr >= eaddr ||
  1857. (bo && offset + size > amdgpu_bo_size(bo)))
  1858. return -EINVAL;
  1859. /* Allocate all the needed memory */
  1860. mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
  1861. if (!mapping)
  1862. return -ENOMEM;
  1863. r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
  1864. if (r) {
  1865. kfree(mapping);
  1866. return r;
  1867. }
  1868. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1869. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1870. mapping->start = saddr;
  1871. mapping->last = eaddr;
  1872. mapping->offset = offset;
  1873. mapping->flags = flags;
  1874. list_add(&mapping->list, &bo_va->invalids);
  1875. amdgpu_vm_it_insert(mapping, &vm->va);
  1876. if (flags & AMDGPU_PTE_PRT)
  1877. amdgpu_vm_prt_get(adev);
  1878. return 0;
  1879. }
  1880. /**
  1881. * amdgpu_vm_bo_unmap - remove bo mapping from vm
  1882. *
  1883. * @adev: amdgpu_device pointer
  1884. * @bo_va: bo_va to remove the address from
  1885. * @saddr: where to the BO is mapped
  1886. *
  1887. * Remove a mapping of the BO at the specefied addr from the VM.
  1888. * Returns 0 for success, error for failure.
  1889. *
  1890. * Object has to be reserved and unreserved outside!
  1891. */
  1892. int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
  1893. struct amdgpu_bo_va *bo_va,
  1894. uint64_t saddr)
  1895. {
  1896. struct amdgpu_bo_va_mapping *mapping;
  1897. struct amdgpu_vm *vm = bo_va->base.vm;
  1898. bool valid = true;
  1899. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1900. list_for_each_entry(mapping, &bo_va->valids, list) {
  1901. if (mapping->start == saddr)
  1902. break;
  1903. }
  1904. if (&mapping->list == &bo_va->valids) {
  1905. valid = false;
  1906. list_for_each_entry(mapping, &bo_va->invalids, list) {
  1907. if (mapping->start == saddr)
  1908. break;
  1909. }
  1910. if (&mapping->list == &bo_va->invalids)
  1911. return -ENOENT;
  1912. }
  1913. list_del(&mapping->list);
  1914. amdgpu_vm_it_remove(mapping, &vm->va);
  1915. trace_amdgpu_vm_bo_unmap(bo_va, mapping);
  1916. if (valid)
  1917. list_add(&mapping->list, &vm->freed);
  1918. else
  1919. amdgpu_vm_free_mapping(adev, vm, mapping,
  1920. bo_va->last_pt_update);
  1921. return 0;
  1922. }
  1923. /**
  1924. * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
  1925. *
  1926. * @adev: amdgpu_device pointer
  1927. * @vm: VM structure to use
  1928. * @saddr: start of the range
  1929. * @size: size of the range
  1930. *
  1931. * Remove all mappings in a range, split them as appropriate.
  1932. * Returns 0 for success, error for failure.
  1933. */
  1934. int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
  1935. struct amdgpu_vm *vm,
  1936. uint64_t saddr, uint64_t size)
  1937. {
  1938. struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
  1939. LIST_HEAD(removed);
  1940. uint64_t eaddr;
  1941. eaddr = saddr + size - 1;
  1942. saddr /= AMDGPU_GPU_PAGE_SIZE;
  1943. eaddr /= AMDGPU_GPU_PAGE_SIZE;
  1944. /* Allocate all the needed memory */
  1945. before = kzalloc(sizeof(*before), GFP_KERNEL);
  1946. if (!before)
  1947. return -ENOMEM;
  1948. INIT_LIST_HEAD(&before->list);
  1949. after = kzalloc(sizeof(*after), GFP_KERNEL);
  1950. if (!after) {
  1951. kfree(before);
  1952. return -ENOMEM;
  1953. }
  1954. INIT_LIST_HEAD(&after->list);
  1955. /* Now gather all removed mappings */
  1956. tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
  1957. while (tmp) {
  1958. /* Remember mapping split at the start */
  1959. if (tmp->start < saddr) {
  1960. before->start = tmp->start;
  1961. before->last = saddr - 1;
  1962. before->offset = tmp->offset;
  1963. before->flags = tmp->flags;
  1964. list_add(&before->list, &tmp->list);
  1965. }
  1966. /* Remember mapping split at the end */
  1967. if (tmp->last > eaddr) {
  1968. after->start = eaddr + 1;
  1969. after->last = tmp->last;
  1970. after->offset = tmp->offset;
  1971. after->offset += (after->start - tmp->start) << PAGE_SHIFT;
  1972. after->flags = tmp->flags;
  1973. list_add(&after->list, &tmp->list);
  1974. }
  1975. list_del(&tmp->list);
  1976. list_add(&tmp->list, &removed);
  1977. tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
  1978. }
  1979. /* And free them up */
  1980. list_for_each_entry_safe(tmp, next, &removed, list) {
  1981. amdgpu_vm_it_remove(tmp, &vm->va);
  1982. list_del(&tmp->list);
  1983. if (tmp->start < saddr)
  1984. tmp->start = saddr;
  1985. if (tmp->last > eaddr)
  1986. tmp->last = eaddr;
  1987. list_add(&tmp->list, &vm->freed);
  1988. trace_amdgpu_vm_bo_unmap(NULL, tmp);
  1989. }
  1990. /* Insert partial mapping before the range */
  1991. if (!list_empty(&before->list)) {
  1992. amdgpu_vm_it_insert(before, &vm->va);
  1993. if (before->flags & AMDGPU_PTE_PRT)
  1994. amdgpu_vm_prt_get(adev);
  1995. } else {
  1996. kfree(before);
  1997. }
  1998. /* Insert partial mapping after the range */
  1999. if (!list_empty(&after->list)) {
  2000. amdgpu_vm_it_insert(after, &vm->va);
  2001. if (after->flags & AMDGPU_PTE_PRT)
  2002. amdgpu_vm_prt_get(adev);
  2003. } else {
  2004. kfree(after);
  2005. }
  2006. return 0;
  2007. }
  2008. /**
  2009. * amdgpu_vm_bo_rmv - remove a bo to a specific vm
  2010. *
  2011. * @adev: amdgpu_device pointer
  2012. * @bo_va: requested bo_va
  2013. *
  2014. * Remove @bo_va->bo from the requested vm.
  2015. *
  2016. * Object have to be reserved!
  2017. */
  2018. void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
  2019. struct amdgpu_bo_va *bo_va)
  2020. {
  2021. struct amdgpu_bo_va_mapping *mapping, *next;
  2022. struct amdgpu_vm *vm = bo_va->base.vm;
  2023. list_del(&bo_va->base.bo_list);
  2024. spin_lock(&vm->status_lock);
  2025. list_del(&bo_va->base.vm_status);
  2026. spin_unlock(&vm->status_lock);
  2027. list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
  2028. list_del(&mapping->list);
  2029. amdgpu_vm_it_remove(mapping, &vm->va);
  2030. trace_amdgpu_vm_bo_unmap(bo_va, mapping);
  2031. list_add(&mapping->list, &vm->freed);
  2032. }
  2033. list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
  2034. list_del(&mapping->list);
  2035. amdgpu_vm_it_remove(mapping, &vm->va);
  2036. amdgpu_vm_free_mapping(adev, vm, mapping,
  2037. bo_va->last_pt_update);
  2038. }
  2039. dma_fence_put(bo_va->last_pt_update);
  2040. kfree(bo_va);
  2041. }
  2042. /**
  2043. * amdgpu_vm_bo_invalidate - mark the bo as invalid
  2044. *
  2045. * @adev: amdgpu_device pointer
  2046. * @vm: requested vm
  2047. * @bo: amdgpu buffer object
  2048. *
  2049. * Mark @bo as invalid.
  2050. */
  2051. void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
  2052. struct amdgpu_bo *bo)
  2053. {
  2054. struct amdgpu_vm_bo_base *bo_base;
  2055. list_for_each_entry(bo_base, &bo->va, bo_list) {
  2056. spin_lock(&bo_base->vm->status_lock);
  2057. if (list_empty(&bo_base->vm_status))
  2058. list_add(&bo_base->vm_status,
  2059. &bo_base->vm->moved);
  2060. spin_unlock(&bo_base->vm->status_lock);
  2061. }
  2062. }
  2063. static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
  2064. {
  2065. /* Total bits covered by PD + PTs */
  2066. unsigned bits = ilog2(vm_size) + 18;
  2067. /* Make sure the PD is 4K in size up to 8GB address space.
  2068. Above that split equal between PD and PTs */
  2069. if (vm_size <= 8)
  2070. return (bits - 9);
  2071. else
  2072. return ((bits + 3) / 2);
  2073. }
  2074. /**
  2075. * amdgpu_vm_set_fragment_size - adjust fragment size in PTE
  2076. *
  2077. * @adev: amdgpu_device pointer
  2078. * @fragment_size_default: the default fragment size if it's set auto
  2079. */
  2080. void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_size_default)
  2081. {
  2082. if (amdgpu_vm_fragment_size == -1)
  2083. adev->vm_manager.fragment_size = fragment_size_default;
  2084. else
  2085. adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
  2086. }
  2087. /**
  2088. * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
  2089. *
  2090. * @adev: amdgpu_device pointer
  2091. * @vm_size: the default vm size if it's set auto
  2092. */
  2093. void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_t fragment_size_default)
  2094. {
  2095. /* adjust vm size firstly */
  2096. if (amdgpu_vm_size == -1)
  2097. adev->vm_manager.vm_size = vm_size;
  2098. else
  2099. adev->vm_manager.vm_size = amdgpu_vm_size;
  2100. /* block size depends on vm size */
  2101. if (amdgpu_vm_block_size == -1)
  2102. adev->vm_manager.block_size =
  2103. amdgpu_vm_get_block_size(adev->vm_manager.vm_size);
  2104. else
  2105. adev->vm_manager.block_size = amdgpu_vm_block_size;
  2106. amdgpu_vm_set_fragment_size(adev, fragment_size_default);
  2107. DRM_INFO("vm size is %llu GB, block size is %u-bit, fragment size is %u-bit\n",
  2108. adev->vm_manager.vm_size, adev->vm_manager.block_size,
  2109. adev->vm_manager.fragment_size);
  2110. }
  2111. /**
  2112. * amdgpu_vm_init - initialize a vm instance
  2113. *
  2114. * @adev: amdgpu_device pointer
  2115. * @vm: requested vm
  2116. * @vm_context: Indicates if it GFX or Compute context
  2117. *
  2118. * Init @vm fields.
  2119. */
  2120. int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  2121. int vm_context)
  2122. {
  2123. const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
  2124. AMDGPU_VM_PTE_COUNT(adev) * 8);
  2125. unsigned ring_instance;
  2126. struct amdgpu_ring *ring;
  2127. struct amd_sched_rq *rq;
  2128. int r, i;
  2129. u64 flags;
  2130. uint64_t init_pde_value = 0;
  2131. vm->va = RB_ROOT_CACHED;
  2132. vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
  2133. for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
  2134. vm->reserved_vmid[i] = NULL;
  2135. spin_lock_init(&vm->status_lock);
  2136. INIT_LIST_HEAD(&vm->moved);
  2137. INIT_LIST_HEAD(&vm->cleared);
  2138. INIT_LIST_HEAD(&vm->freed);
  2139. /* create scheduler entity for page table updates */
  2140. ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
  2141. ring_instance %= adev->vm_manager.vm_pte_num_rings;
  2142. ring = adev->vm_manager.vm_pte_rings[ring_instance];
  2143. rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
  2144. r = amd_sched_entity_init(&ring->sched, &vm->entity,
  2145. rq, amdgpu_sched_jobs);
  2146. if (r)
  2147. return r;
  2148. vm->pte_support_ats = false;
  2149. if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
  2150. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2151. AMDGPU_VM_USE_CPU_FOR_COMPUTE);
  2152. if (adev->asic_type == CHIP_RAVEN) {
  2153. vm->pte_support_ats = true;
  2154. init_pde_value = AMDGPU_PTE_SYSTEM | AMDGPU_PDE_PTE;
  2155. }
  2156. } else
  2157. vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
  2158. AMDGPU_VM_USE_CPU_FOR_GFX);
  2159. DRM_DEBUG_DRIVER("VM update mode is %s\n",
  2160. vm->use_cpu_for_update ? "CPU" : "SDMA");
  2161. WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
  2162. "CPU update of VM recommended only for large BAR system\n");
  2163. vm->last_dir_update = NULL;
  2164. flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
  2165. AMDGPU_GEM_CREATE_VRAM_CLEARED;
  2166. if (vm->use_cpu_for_update)
  2167. flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
  2168. else
  2169. flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
  2170. AMDGPU_GEM_CREATE_SHADOW);
  2171. r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
  2172. AMDGPU_GEM_DOMAIN_VRAM,
  2173. flags,
  2174. NULL, NULL, init_pde_value, &vm->root.bo);
  2175. if (r)
  2176. goto error_free_sched_entity;
  2177. r = amdgpu_bo_reserve(vm->root.bo, false);
  2178. if (r)
  2179. goto error_free_root;
  2180. vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
  2181. if (vm->use_cpu_for_update) {
  2182. r = amdgpu_bo_kmap(vm->root.bo, NULL);
  2183. if (r)
  2184. goto error_free_root;
  2185. }
  2186. amdgpu_bo_unreserve(vm->root.bo);
  2187. return 0;
  2188. error_free_root:
  2189. amdgpu_bo_unref(&vm->root.bo->shadow);
  2190. amdgpu_bo_unref(&vm->root.bo);
  2191. vm->root.bo = NULL;
  2192. error_free_sched_entity:
  2193. amd_sched_entity_fini(&ring->sched, &vm->entity);
  2194. return r;
  2195. }
  2196. /**
  2197. * amdgpu_vm_free_levels - free PD/PT levels
  2198. *
  2199. * @level: PD/PT starting level to free
  2200. *
  2201. * Free the page directory or page table level and all sub levels.
  2202. */
  2203. static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
  2204. {
  2205. unsigned i;
  2206. if (level->bo) {
  2207. amdgpu_bo_unref(&level->bo->shadow);
  2208. amdgpu_bo_unref(&level->bo);
  2209. }
  2210. if (level->entries)
  2211. for (i = 0; i <= level->last_entry_used; i++)
  2212. amdgpu_vm_free_levels(&level->entries[i]);
  2213. kvfree(level->entries);
  2214. }
  2215. /**
  2216. * amdgpu_vm_fini - tear down a vm instance
  2217. *
  2218. * @adev: amdgpu_device pointer
  2219. * @vm: requested vm
  2220. *
  2221. * Tear down @vm.
  2222. * Unbind the VM and remove all bos from the vm bo list
  2223. */
  2224. void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
  2225. {
  2226. struct amdgpu_bo_va_mapping *mapping, *tmp;
  2227. bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
  2228. struct amdgpu_bo *root;
  2229. int i, r;
  2230. amd_sched_entity_fini(vm->entity.sched, &vm->entity);
  2231. if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
  2232. dev_err(adev->dev, "still active bo inside vm\n");
  2233. }
  2234. rbtree_postorder_for_each_entry_safe(mapping, tmp,
  2235. &vm->va.rb_root, rb) {
  2236. list_del(&mapping->list);
  2237. amdgpu_vm_it_remove(mapping, &vm->va);
  2238. kfree(mapping);
  2239. }
  2240. list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
  2241. if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
  2242. amdgpu_vm_prt_fini(adev, vm);
  2243. prt_fini_needed = false;
  2244. }
  2245. list_del(&mapping->list);
  2246. amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
  2247. }
  2248. root = amdgpu_bo_ref(vm->root.bo);
  2249. r = amdgpu_bo_reserve(root, true);
  2250. if (r) {
  2251. dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
  2252. } else {
  2253. amdgpu_vm_free_levels(&vm->root);
  2254. amdgpu_bo_unreserve(root);
  2255. }
  2256. amdgpu_bo_unref(&root);
  2257. dma_fence_put(vm->last_dir_update);
  2258. for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
  2259. amdgpu_vm_free_reserved_vmid(adev, vm, i);
  2260. }
  2261. /**
  2262. * amdgpu_vm_manager_init - init the VM manager
  2263. *
  2264. * @adev: amdgpu_device pointer
  2265. *
  2266. * Initialize the VM manager structures
  2267. */
  2268. void amdgpu_vm_manager_init(struct amdgpu_device *adev)
  2269. {
  2270. unsigned i, j;
  2271. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  2272. struct amdgpu_vm_id_manager *id_mgr =
  2273. &adev->vm_manager.id_mgr[i];
  2274. mutex_init(&id_mgr->lock);
  2275. INIT_LIST_HEAD(&id_mgr->ids_lru);
  2276. atomic_set(&id_mgr->reserved_vmid_num, 0);
  2277. /* skip over VMID 0, since it is the system VM */
  2278. for (j = 1; j < id_mgr->num_ids; ++j) {
  2279. amdgpu_vm_reset_id(adev, i, j);
  2280. amdgpu_sync_create(&id_mgr->ids[i].active);
  2281. list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
  2282. }
  2283. }
  2284. adev->vm_manager.fence_context =
  2285. dma_fence_context_alloc(AMDGPU_MAX_RINGS);
  2286. for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
  2287. adev->vm_manager.seqno[i] = 0;
  2288. atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
  2289. atomic64_set(&adev->vm_manager.client_counter, 0);
  2290. spin_lock_init(&adev->vm_manager.prt_lock);
  2291. atomic_set(&adev->vm_manager.num_prt_users, 0);
  2292. /* If not overridden by the user, by default, only in large BAR systems
  2293. * Compute VM tables will be updated by CPU
  2294. */
  2295. #ifdef CONFIG_X86_64
  2296. if (amdgpu_vm_update_mode == -1) {
  2297. if (amdgpu_vm_is_large_bar(adev))
  2298. adev->vm_manager.vm_update_mode =
  2299. AMDGPU_VM_USE_CPU_FOR_COMPUTE;
  2300. else
  2301. adev->vm_manager.vm_update_mode = 0;
  2302. } else
  2303. adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
  2304. #else
  2305. adev->vm_manager.vm_update_mode = 0;
  2306. #endif
  2307. }
  2308. /**
  2309. * amdgpu_vm_manager_fini - cleanup VM manager
  2310. *
  2311. * @adev: amdgpu_device pointer
  2312. *
  2313. * Cleanup the VM manager and free resources.
  2314. */
  2315. void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
  2316. {
  2317. unsigned i, j;
  2318. for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
  2319. struct amdgpu_vm_id_manager *id_mgr =
  2320. &adev->vm_manager.id_mgr[i];
  2321. mutex_destroy(&id_mgr->lock);
  2322. for (j = 0; j < AMDGPU_NUM_VM; ++j) {
  2323. struct amdgpu_vm_id *id = &id_mgr->ids[j];
  2324. amdgpu_sync_free(&id->active);
  2325. dma_fence_put(id->flushed_updates);
  2326. dma_fence_put(id->last_flush);
  2327. }
  2328. }
  2329. }
  2330. int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
  2331. {
  2332. union drm_amdgpu_vm *args = data;
  2333. struct amdgpu_device *adev = dev->dev_private;
  2334. struct amdgpu_fpriv *fpriv = filp->driver_priv;
  2335. int r;
  2336. switch (args->in.op) {
  2337. case AMDGPU_VM_OP_RESERVE_VMID:
  2338. /* current, we only have requirement to reserve vmid from gfxhub */
  2339. r = amdgpu_vm_alloc_reserved_vmid(adev, &fpriv->vm,
  2340. AMDGPU_GFXHUB);
  2341. if (r)
  2342. return r;
  2343. break;
  2344. case AMDGPU_VM_OP_UNRESERVE_VMID:
  2345. amdgpu_vm_free_reserved_vmid(adev, &fpriv->vm, AMDGPU_GFXHUB);
  2346. break;
  2347. default:
  2348. return -EINVAL;
  2349. }
  2350. return 0;
  2351. }