file.c 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684
  1. /*
  2. * Copyright (C) 2007 Oracle. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public
  6. * License v2 as published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful,
  9. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11. * General Public License for more details.
  12. *
  13. * You should have received a copy of the GNU General Public
  14. * License along with this program; if not, write to the
  15. * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16. * Boston, MA 021110-1307, USA.
  17. */
  18. #include <linux/fs.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/highmem.h>
  21. #include <linux/time.h>
  22. #include <linux/init.h>
  23. #include <linux/string.h>
  24. #include <linux/backing-dev.h>
  25. #include <linux/mpage.h>
  26. #include <linux/falloc.h>
  27. #include <linux/swap.h>
  28. #include <linux/writeback.h>
  29. #include <linux/statfs.h>
  30. #include <linux/compat.h>
  31. #include <linux/slab.h>
  32. #include "ctree.h"
  33. #include "disk-io.h"
  34. #include "transaction.h"
  35. #include "btrfs_inode.h"
  36. #include "ioctl.h"
  37. #include "print-tree.h"
  38. #include "tree-log.h"
  39. #include "locking.h"
  40. #include "compat.h"
  41. /*
  42. * when auto defrag is enabled we
  43. * queue up these defrag structs to remember which
  44. * inodes need defragging passes
  45. */
  46. struct inode_defrag {
  47. struct rb_node rb_node;
  48. /* objectid */
  49. u64 ino;
  50. /*
  51. * transid where the defrag was added, we search for
  52. * extents newer than this
  53. */
  54. u64 transid;
  55. /* root objectid */
  56. u64 root;
  57. /* last offset we were able to defrag */
  58. u64 last_offset;
  59. /* if we've wrapped around back to zero once already */
  60. int cycled;
  61. };
  62. /* pop a record for an inode into the defrag tree. The lock
  63. * must be held already
  64. *
  65. * If you're inserting a record for an older transid than an
  66. * existing record, the transid already in the tree is lowered
  67. *
  68. * If an existing record is found the defrag item you
  69. * pass in is freed
  70. */
  71. static int __btrfs_add_inode_defrag(struct inode *inode,
  72. struct inode_defrag *defrag)
  73. {
  74. struct btrfs_root *root = BTRFS_I(inode)->root;
  75. struct inode_defrag *entry;
  76. struct rb_node **p;
  77. struct rb_node *parent = NULL;
  78. p = &root->fs_info->defrag_inodes.rb_node;
  79. while (*p) {
  80. parent = *p;
  81. entry = rb_entry(parent, struct inode_defrag, rb_node);
  82. if (defrag->ino < entry->ino)
  83. p = &parent->rb_left;
  84. else if (defrag->ino > entry->ino)
  85. p = &parent->rb_right;
  86. else {
  87. /* if we're reinserting an entry for
  88. * an old defrag run, make sure to
  89. * lower the transid of our existing record
  90. */
  91. if (defrag->transid < entry->transid)
  92. entry->transid = defrag->transid;
  93. if (defrag->last_offset > entry->last_offset)
  94. entry->last_offset = defrag->last_offset;
  95. goto exists;
  96. }
  97. }
  98. BTRFS_I(inode)->in_defrag = 1;
  99. rb_link_node(&defrag->rb_node, parent, p);
  100. rb_insert_color(&defrag->rb_node, &root->fs_info->defrag_inodes);
  101. return 0;
  102. exists:
  103. kfree(defrag);
  104. return 0;
  105. }
  106. /*
  107. * insert a defrag record for this inode if auto defrag is
  108. * enabled
  109. */
  110. int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
  111. struct inode *inode)
  112. {
  113. struct btrfs_root *root = BTRFS_I(inode)->root;
  114. struct inode_defrag *defrag;
  115. int ret = 0;
  116. u64 transid;
  117. if (!btrfs_test_opt(root, AUTO_DEFRAG))
  118. return 0;
  119. if (btrfs_fs_closing(root->fs_info))
  120. return 0;
  121. if (BTRFS_I(inode)->in_defrag)
  122. return 0;
  123. if (trans)
  124. transid = trans->transid;
  125. else
  126. transid = BTRFS_I(inode)->root->last_trans;
  127. defrag = kzalloc(sizeof(*defrag), GFP_NOFS);
  128. if (!defrag)
  129. return -ENOMEM;
  130. defrag->ino = btrfs_ino(inode);
  131. defrag->transid = transid;
  132. defrag->root = root->root_key.objectid;
  133. spin_lock(&root->fs_info->defrag_inodes_lock);
  134. if (!BTRFS_I(inode)->in_defrag)
  135. ret = __btrfs_add_inode_defrag(inode, defrag);
  136. spin_unlock(&root->fs_info->defrag_inodes_lock);
  137. return ret;
  138. }
  139. /*
  140. * must be called with the defrag_inodes lock held
  141. */
  142. struct inode_defrag *btrfs_find_defrag_inode(struct btrfs_fs_info *info, u64 ino,
  143. struct rb_node **next)
  144. {
  145. struct inode_defrag *entry = NULL;
  146. struct rb_node *p;
  147. struct rb_node *parent = NULL;
  148. p = info->defrag_inodes.rb_node;
  149. while (p) {
  150. parent = p;
  151. entry = rb_entry(parent, struct inode_defrag, rb_node);
  152. if (ino < entry->ino)
  153. p = parent->rb_left;
  154. else if (ino > entry->ino)
  155. p = parent->rb_right;
  156. else
  157. return entry;
  158. }
  159. if (next) {
  160. while (parent && ino > entry->ino) {
  161. parent = rb_next(parent);
  162. entry = rb_entry(parent, struct inode_defrag, rb_node);
  163. }
  164. *next = parent;
  165. }
  166. return NULL;
  167. }
  168. /*
  169. * run through the list of inodes in the FS that need
  170. * defragging
  171. */
  172. int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
  173. {
  174. struct inode_defrag *defrag;
  175. struct btrfs_root *inode_root;
  176. struct inode *inode;
  177. struct rb_node *n;
  178. struct btrfs_key key;
  179. struct btrfs_ioctl_defrag_range_args range;
  180. u64 first_ino = 0;
  181. int num_defrag;
  182. int defrag_batch = 1024;
  183. memset(&range, 0, sizeof(range));
  184. range.len = (u64)-1;
  185. atomic_inc(&fs_info->defrag_running);
  186. spin_lock(&fs_info->defrag_inodes_lock);
  187. while(1) {
  188. n = NULL;
  189. /* find an inode to defrag */
  190. defrag = btrfs_find_defrag_inode(fs_info, first_ino, &n);
  191. if (!defrag) {
  192. if (n)
  193. defrag = rb_entry(n, struct inode_defrag, rb_node);
  194. else if (first_ino) {
  195. first_ino = 0;
  196. continue;
  197. } else {
  198. break;
  199. }
  200. }
  201. /* remove it from the rbtree */
  202. first_ino = defrag->ino + 1;
  203. rb_erase(&defrag->rb_node, &fs_info->defrag_inodes);
  204. if (btrfs_fs_closing(fs_info))
  205. goto next_free;
  206. spin_unlock(&fs_info->defrag_inodes_lock);
  207. /* get the inode */
  208. key.objectid = defrag->root;
  209. btrfs_set_key_type(&key, BTRFS_ROOT_ITEM_KEY);
  210. key.offset = (u64)-1;
  211. inode_root = btrfs_read_fs_root_no_name(fs_info, &key);
  212. if (IS_ERR(inode_root))
  213. goto next;
  214. key.objectid = defrag->ino;
  215. btrfs_set_key_type(&key, BTRFS_INODE_ITEM_KEY);
  216. key.offset = 0;
  217. inode = btrfs_iget(fs_info->sb, &key, inode_root, NULL);
  218. if (IS_ERR(inode))
  219. goto next;
  220. /* do a chunk of defrag */
  221. BTRFS_I(inode)->in_defrag = 0;
  222. range.start = defrag->last_offset;
  223. num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
  224. defrag_batch);
  225. /*
  226. * if we filled the whole defrag batch, there
  227. * must be more work to do. Queue this defrag
  228. * again
  229. */
  230. if (num_defrag == defrag_batch) {
  231. defrag->last_offset = range.start;
  232. __btrfs_add_inode_defrag(inode, defrag);
  233. /*
  234. * we don't want to kfree defrag, we added it back to
  235. * the rbtree
  236. */
  237. defrag = NULL;
  238. } else if (defrag->last_offset && !defrag->cycled) {
  239. /*
  240. * we didn't fill our defrag batch, but
  241. * we didn't start at zero. Make sure we loop
  242. * around to the start of the file.
  243. */
  244. defrag->last_offset = 0;
  245. defrag->cycled = 1;
  246. __btrfs_add_inode_defrag(inode, defrag);
  247. defrag = NULL;
  248. }
  249. iput(inode);
  250. next:
  251. spin_lock(&fs_info->defrag_inodes_lock);
  252. next_free:
  253. kfree(defrag);
  254. }
  255. spin_unlock(&fs_info->defrag_inodes_lock);
  256. atomic_dec(&fs_info->defrag_running);
  257. /*
  258. * during unmount, we use the transaction_wait queue to
  259. * wait for the defragger to stop
  260. */
  261. wake_up(&fs_info->transaction_wait);
  262. return 0;
  263. }
  264. /* simple helper to fault in pages and copy. This should go away
  265. * and be replaced with calls into generic code.
  266. */
  267. static noinline int btrfs_copy_from_user(loff_t pos, int num_pages,
  268. size_t write_bytes,
  269. struct page **prepared_pages,
  270. struct iov_iter *i)
  271. {
  272. size_t copied = 0;
  273. size_t total_copied = 0;
  274. int pg = 0;
  275. int offset = pos & (PAGE_CACHE_SIZE - 1);
  276. while (write_bytes > 0) {
  277. size_t count = min_t(size_t,
  278. PAGE_CACHE_SIZE - offset, write_bytes);
  279. struct page *page = prepared_pages[pg];
  280. /*
  281. * Copy data from userspace to the current page
  282. *
  283. * Disable pagefault to avoid recursive lock since
  284. * the pages are already locked
  285. */
  286. pagefault_disable();
  287. copied = iov_iter_copy_from_user_atomic(page, i, offset, count);
  288. pagefault_enable();
  289. /* Flush processor's dcache for this page */
  290. flush_dcache_page(page);
  291. /*
  292. * if we get a partial write, we can end up with
  293. * partially up to date pages. These add
  294. * a lot of complexity, so make sure they don't
  295. * happen by forcing this copy to be retried.
  296. *
  297. * The rest of the btrfs_file_write code will fall
  298. * back to page at a time copies after we return 0.
  299. */
  300. if (!PageUptodate(page) && copied < count)
  301. copied = 0;
  302. iov_iter_advance(i, copied);
  303. write_bytes -= copied;
  304. total_copied += copied;
  305. /* Return to btrfs_file_aio_write to fault page */
  306. if (unlikely(copied == 0))
  307. break;
  308. if (unlikely(copied < PAGE_CACHE_SIZE - offset)) {
  309. offset += copied;
  310. } else {
  311. pg++;
  312. offset = 0;
  313. }
  314. }
  315. return total_copied;
  316. }
  317. /*
  318. * unlocks pages after btrfs_file_write is done with them
  319. */
  320. void btrfs_drop_pages(struct page **pages, size_t num_pages)
  321. {
  322. size_t i;
  323. for (i = 0; i < num_pages; i++) {
  324. /* page checked is some magic around finding pages that
  325. * have been modified without going through btrfs_set_page_dirty
  326. * clear it here
  327. */
  328. ClearPageChecked(pages[i]);
  329. unlock_page(pages[i]);
  330. mark_page_accessed(pages[i]);
  331. page_cache_release(pages[i]);
  332. }
  333. }
  334. /*
  335. * after copy_from_user, pages need to be dirtied and we need to make
  336. * sure holes are created between the current EOF and the start of
  337. * any next extents (if required).
  338. *
  339. * this also makes the decision about creating an inline extent vs
  340. * doing real data extents, marking pages dirty and delalloc as required.
  341. */
  342. int btrfs_dirty_pages(struct btrfs_root *root, struct inode *inode,
  343. struct page **pages, size_t num_pages,
  344. loff_t pos, size_t write_bytes,
  345. struct extent_state **cached)
  346. {
  347. int err = 0;
  348. int i;
  349. u64 num_bytes;
  350. u64 start_pos;
  351. u64 end_of_last_block;
  352. u64 end_pos = pos + write_bytes;
  353. loff_t isize = i_size_read(inode);
  354. start_pos = pos & ~((u64)root->sectorsize - 1);
  355. num_bytes = (write_bytes + pos - start_pos +
  356. root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
  357. end_of_last_block = start_pos + num_bytes - 1;
  358. err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
  359. cached);
  360. if (err)
  361. return err;
  362. for (i = 0; i < num_pages; i++) {
  363. struct page *p = pages[i];
  364. SetPageUptodate(p);
  365. ClearPageChecked(p);
  366. set_page_dirty(p);
  367. }
  368. /*
  369. * we've only changed i_size in ram, and we haven't updated
  370. * the disk i_size. There is no need to log the inode
  371. * at this time.
  372. */
  373. if (end_pos > isize)
  374. i_size_write(inode, end_pos);
  375. return 0;
  376. }
  377. /*
  378. * this drops all the extents in the cache that intersect the range
  379. * [start, end]. Existing extents are split as required.
  380. */
  381. int btrfs_drop_extent_cache(struct inode *inode, u64 start, u64 end,
  382. int skip_pinned)
  383. {
  384. struct extent_map *em;
  385. struct extent_map *split = NULL;
  386. struct extent_map *split2 = NULL;
  387. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  388. u64 len = end - start + 1;
  389. int ret;
  390. int testend = 1;
  391. unsigned long flags;
  392. int compressed = 0;
  393. WARN_ON(end < start);
  394. if (end == (u64)-1) {
  395. len = (u64)-1;
  396. testend = 0;
  397. }
  398. while (1) {
  399. if (!split)
  400. split = alloc_extent_map();
  401. if (!split2)
  402. split2 = alloc_extent_map();
  403. BUG_ON(!split || !split2);
  404. write_lock(&em_tree->lock);
  405. em = lookup_extent_mapping(em_tree, start, len);
  406. if (!em) {
  407. write_unlock(&em_tree->lock);
  408. break;
  409. }
  410. flags = em->flags;
  411. if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
  412. if (testend && em->start + em->len >= start + len) {
  413. free_extent_map(em);
  414. write_unlock(&em_tree->lock);
  415. break;
  416. }
  417. start = em->start + em->len;
  418. if (testend)
  419. len = start + len - (em->start + em->len);
  420. free_extent_map(em);
  421. write_unlock(&em_tree->lock);
  422. continue;
  423. }
  424. compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  425. clear_bit(EXTENT_FLAG_PINNED, &em->flags);
  426. remove_extent_mapping(em_tree, em);
  427. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  428. em->start < start) {
  429. split->start = em->start;
  430. split->len = start - em->start;
  431. split->orig_start = em->orig_start;
  432. split->block_start = em->block_start;
  433. if (compressed)
  434. split->block_len = em->block_len;
  435. else
  436. split->block_len = split->len;
  437. split->bdev = em->bdev;
  438. split->flags = flags;
  439. split->compress_type = em->compress_type;
  440. ret = add_extent_mapping(em_tree, split);
  441. BUG_ON(ret);
  442. free_extent_map(split);
  443. split = split2;
  444. split2 = NULL;
  445. }
  446. if (em->block_start < EXTENT_MAP_LAST_BYTE &&
  447. testend && em->start + em->len > start + len) {
  448. u64 diff = start + len - em->start;
  449. split->start = start + len;
  450. split->len = em->start + em->len - (start + len);
  451. split->bdev = em->bdev;
  452. split->flags = flags;
  453. split->compress_type = em->compress_type;
  454. if (compressed) {
  455. split->block_len = em->block_len;
  456. split->block_start = em->block_start;
  457. split->orig_start = em->orig_start;
  458. } else {
  459. split->block_len = split->len;
  460. split->block_start = em->block_start + diff;
  461. split->orig_start = split->start;
  462. }
  463. ret = add_extent_mapping(em_tree, split);
  464. BUG_ON(ret);
  465. free_extent_map(split);
  466. split = NULL;
  467. }
  468. write_unlock(&em_tree->lock);
  469. /* once for us */
  470. free_extent_map(em);
  471. /* once for the tree*/
  472. free_extent_map(em);
  473. }
  474. if (split)
  475. free_extent_map(split);
  476. if (split2)
  477. free_extent_map(split2);
  478. return 0;
  479. }
  480. /*
  481. * this is very complex, but the basic idea is to drop all extents
  482. * in the range start - end. hint_block is filled in with a block number
  483. * that would be a good hint to the block allocator for this file.
  484. *
  485. * If an extent intersects the range but is not entirely inside the range
  486. * it is either truncated or split. Anything entirely inside the range
  487. * is deleted from the tree.
  488. */
  489. int btrfs_drop_extents(struct btrfs_trans_handle *trans, struct inode *inode,
  490. u64 start, u64 end, u64 *hint_byte, int drop_cache)
  491. {
  492. struct btrfs_root *root = BTRFS_I(inode)->root;
  493. struct extent_buffer *leaf;
  494. struct btrfs_file_extent_item *fi;
  495. struct btrfs_path *path;
  496. struct btrfs_key key;
  497. struct btrfs_key new_key;
  498. u64 ino = btrfs_ino(inode);
  499. u64 search_start = start;
  500. u64 disk_bytenr = 0;
  501. u64 num_bytes = 0;
  502. u64 extent_offset = 0;
  503. u64 extent_end = 0;
  504. int del_nr = 0;
  505. int del_slot = 0;
  506. int extent_type;
  507. int recow;
  508. int ret;
  509. if (drop_cache)
  510. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  511. path = btrfs_alloc_path();
  512. if (!path)
  513. return -ENOMEM;
  514. while (1) {
  515. recow = 0;
  516. ret = btrfs_lookup_file_extent(trans, root, path, ino,
  517. search_start, -1);
  518. if (ret < 0)
  519. break;
  520. if (ret > 0 && path->slots[0] > 0 && search_start == start) {
  521. leaf = path->nodes[0];
  522. btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
  523. if (key.objectid == ino &&
  524. key.type == BTRFS_EXTENT_DATA_KEY)
  525. path->slots[0]--;
  526. }
  527. ret = 0;
  528. next_slot:
  529. leaf = path->nodes[0];
  530. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  531. BUG_ON(del_nr > 0);
  532. ret = btrfs_next_leaf(root, path);
  533. if (ret < 0)
  534. break;
  535. if (ret > 0) {
  536. ret = 0;
  537. break;
  538. }
  539. leaf = path->nodes[0];
  540. recow = 1;
  541. }
  542. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  543. if (key.objectid > ino ||
  544. key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= end)
  545. break;
  546. fi = btrfs_item_ptr(leaf, path->slots[0],
  547. struct btrfs_file_extent_item);
  548. extent_type = btrfs_file_extent_type(leaf, fi);
  549. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  550. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  551. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  552. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  553. extent_offset = btrfs_file_extent_offset(leaf, fi);
  554. extent_end = key.offset +
  555. btrfs_file_extent_num_bytes(leaf, fi);
  556. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  557. extent_end = key.offset +
  558. btrfs_file_extent_inline_len(leaf, fi);
  559. } else {
  560. WARN_ON(1);
  561. extent_end = search_start;
  562. }
  563. if (extent_end <= search_start) {
  564. path->slots[0]++;
  565. goto next_slot;
  566. }
  567. search_start = max(key.offset, start);
  568. if (recow) {
  569. btrfs_release_path(path);
  570. continue;
  571. }
  572. /*
  573. * | - range to drop - |
  574. * | -------- extent -------- |
  575. */
  576. if (start > key.offset && end < extent_end) {
  577. BUG_ON(del_nr > 0);
  578. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  579. memcpy(&new_key, &key, sizeof(new_key));
  580. new_key.offset = start;
  581. ret = btrfs_duplicate_item(trans, root, path,
  582. &new_key);
  583. if (ret == -EAGAIN) {
  584. btrfs_release_path(path);
  585. continue;
  586. }
  587. if (ret < 0)
  588. break;
  589. leaf = path->nodes[0];
  590. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  591. struct btrfs_file_extent_item);
  592. btrfs_set_file_extent_num_bytes(leaf, fi,
  593. start - key.offset);
  594. fi = btrfs_item_ptr(leaf, path->slots[0],
  595. struct btrfs_file_extent_item);
  596. extent_offset += start - key.offset;
  597. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  598. btrfs_set_file_extent_num_bytes(leaf, fi,
  599. extent_end - start);
  600. btrfs_mark_buffer_dirty(leaf);
  601. if (disk_bytenr > 0) {
  602. ret = btrfs_inc_extent_ref(trans, root,
  603. disk_bytenr, num_bytes, 0,
  604. root->root_key.objectid,
  605. new_key.objectid,
  606. start - extent_offset);
  607. BUG_ON(ret);
  608. *hint_byte = disk_bytenr;
  609. }
  610. key.offset = start;
  611. }
  612. /*
  613. * | ---- range to drop ----- |
  614. * | -------- extent -------- |
  615. */
  616. if (start <= key.offset && end < extent_end) {
  617. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  618. memcpy(&new_key, &key, sizeof(new_key));
  619. new_key.offset = end;
  620. btrfs_set_item_key_safe(trans, root, path, &new_key);
  621. extent_offset += end - key.offset;
  622. btrfs_set_file_extent_offset(leaf, fi, extent_offset);
  623. btrfs_set_file_extent_num_bytes(leaf, fi,
  624. extent_end - end);
  625. btrfs_mark_buffer_dirty(leaf);
  626. if (disk_bytenr > 0) {
  627. inode_sub_bytes(inode, end - key.offset);
  628. *hint_byte = disk_bytenr;
  629. }
  630. break;
  631. }
  632. search_start = extent_end;
  633. /*
  634. * | ---- range to drop ----- |
  635. * | -------- extent -------- |
  636. */
  637. if (start > key.offset && end >= extent_end) {
  638. BUG_ON(del_nr > 0);
  639. BUG_ON(extent_type == BTRFS_FILE_EXTENT_INLINE);
  640. btrfs_set_file_extent_num_bytes(leaf, fi,
  641. start - key.offset);
  642. btrfs_mark_buffer_dirty(leaf);
  643. if (disk_bytenr > 0) {
  644. inode_sub_bytes(inode, extent_end - start);
  645. *hint_byte = disk_bytenr;
  646. }
  647. if (end == extent_end)
  648. break;
  649. path->slots[0]++;
  650. goto next_slot;
  651. }
  652. /*
  653. * | ---- range to drop ----- |
  654. * | ------ extent ------ |
  655. */
  656. if (start <= key.offset && end >= extent_end) {
  657. if (del_nr == 0) {
  658. del_slot = path->slots[0];
  659. del_nr = 1;
  660. } else {
  661. BUG_ON(del_slot + del_nr != path->slots[0]);
  662. del_nr++;
  663. }
  664. if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  665. inode_sub_bytes(inode,
  666. extent_end - key.offset);
  667. extent_end = ALIGN(extent_end,
  668. root->sectorsize);
  669. } else if (disk_bytenr > 0) {
  670. ret = btrfs_free_extent(trans, root,
  671. disk_bytenr, num_bytes, 0,
  672. root->root_key.objectid,
  673. key.objectid, key.offset -
  674. extent_offset);
  675. BUG_ON(ret);
  676. inode_sub_bytes(inode,
  677. extent_end - key.offset);
  678. *hint_byte = disk_bytenr;
  679. }
  680. if (end == extent_end)
  681. break;
  682. if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
  683. path->slots[0]++;
  684. goto next_slot;
  685. }
  686. ret = btrfs_del_items(trans, root, path, del_slot,
  687. del_nr);
  688. BUG_ON(ret);
  689. del_nr = 0;
  690. del_slot = 0;
  691. btrfs_release_path(path);
  692. continue;
  693. }
  694. BUG_ON(1);
  695. }
  696. if (del_nr > 0) {
  697. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  698. BUG_ON(ret);
  699. }
  700. btrfs_free_path(path);
  701. return ret;
  702. }
  703. static int extent_mergeable(struct extent_buffer *leaf, int slot,
  704. u64 objectid, u64 bytenr, u64 orig_offset,
  705. u64 *start, u64 *end)
  706. {
  707. struct btrfs_file_extent_item *fi;
  708. struct btrfs_key key;
  709. u64 extent_end;
  710. if (slot < 0 || slot >= btrfs_header_nritems(leaf))
  711. return 0;
  712. btrfs_item_key_to_cpu(leaf, &key, slot);
  713. if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
  714. return 0;
  715. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  716. if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
  717. btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
  718. btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
  719. btrfs_file_extent_compression(leaf, fi) ||
  720. btrfs_file_extent_encryption(leaf, fi) ||
  721. btrfs_file_extent_other_encoding(leaf, fi))
  722. return 0;
  723. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  724. if ((*start && *start != key.offset) || (*end && *end != extent_end))
  725. return 0;
  726. *start = key.offset;
  727. *end = extent_end;
  728. return 1;
  729. }
  730. /*
  731. * Mark extent in the range start - end as written.
  732. *
  733. * This changes extent type from 'pre-allocated' to 'regular'. If only
  734. * part of extent is marked as written, the extent will be split into
  735. * two or three.
  736. */
  737. int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
  738. struct inode *inode, u64 start, u64 end)
  739. {
  740. struct btrfs_root *root = BTRFS_I(inode)->root;
  741. struct extent_buffer *leaf;
  742. struct btrfs_path *path;
  743. struct btrfs_file_extent_item *fi;
  744. struct btrfs_key key;
  745. struct btrfs_key new_key;
  746. u64 bytenr;
  747. u64 num_bytes;
  748. u64 extent_end;
  749. u64 orig_offset;
  750. u64 other_start;
  751. u64 other_end;
  752. u64 split;
  753. int del_nr = 0;
  754. int del_slot = 0;
  755. int recow;
  756. int ret;
  757. u64 ino = btrfs_ino(inode);
  758. btrfs_drop_extent_cache(inode, start, end - 1, 0);
  759. path = btrfs_alloc_path();
  760. BUG_ON(!path);
  761. again:
  762. recow = 0;
  763. split = start;
  764. key.objectid = ino;
  765. key.type = BTRFS_EXTENT_DATA_KEY;
  766. key.offset = split;
  767. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  768. if (ret < 0)
  769. goto out;
  770. if (ret > 0 && path->slots[0] > 0)
  771. path->slots[0]--;
  772. leaf = path->nodes[0];
  773. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  774. BUG_ON(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY);
  775. fi = btrfs_item_ptr(leaf, path->slots[0],
  776. struct btrfs_file_extent_item);
  777. BUG_ON(btrfs_file_extent_type(leaf, fi) !=
  778. BTRFS_FILE_EXTENT_PREALLOC);
  779. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  780. BUG_ON(key.offset > start || extent_end < end);
  781. bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  782. num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
  783. orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
  784. memcpy(&new_key, &key, sizeof(new_key));
  785. if (start == key.offset && end < extent_end) {
  786. other_start = 0;
  787. other_end = start;
  788. if (extent_mergeable(leaf, path->slots[0] - 1,
  789. ino, bytenr, orig_offset,
  790. &other_start, &other_end)) {
  791. new_key.offset = end;
  792. btrfs_set_item_key_safe(trans, root, path, &new_key);
  793. fi = btrfs_item_ptr(leaf, path->slots[0],
  794. struct btrfs_file_extent_item);
  795. btrfs_set_file_extent_num_bytes(leaf, fi,
  796. extent_end - end);
  797. btrfs_set_file_extent_offset(leaf, fi,
  798. end - orig_offset);
  799. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  800. struct btrfs_file_extent_item);
  801. btrfs_set_file_extent_num_bytes(leaf, fi,
  802. end - other_start);
  803. btrfs_mark_buffer_dirty(leaf);
  804. goto out;
  805. }
  806. }
  807. if (start > key.offset && end == extent_end) {
  808. other_start = end;
  809. other_end = 0;
  810. if (extent_mergeable(leaf, path->slots[0] + 1,
  811. ino, bytenr, orig_offset,
  812. &other_start, &other_end)) {
  813. fi = btrfs_item_ptr(leaf, path->slots[0],
  814. struct btrfs_file_extent_item);
  815. btrfs_set_file_extent_num_bytes(leaf, fi,
  816. start - key.offset);
  817. path->slots[0]++;
  818. new_key.offset = start;
  819. btrfs_set_item_key_safe(trans, root, path, &new_key);
  820. fi = btrfs_item_ptr(leaf, path->slots[0],
  821. struct btrfs_file_extent_item);
  822. btrfs_set_file_extent_num_bytes(leaf, fi,
  823. other_end - start);
  824. btrfs_set_file_extent_offset(leaf, fi,
  825. start - orig_offset);
  826. btrfs_mark_buffer_dirty(leaf);
  827. goto out;
  828. }
  829. }
  830. while (start > key.offset || end < extent_end) {
  831. if (key.offset == start)
  832. split = end;
  833. new_key.offset = split;
  834. ret = btrfs_duplicate_item(trans, root, path, &new_key);
  835. if (ret == -EAGAIN) {
  836. btrfs_release_path(path);
  837. goto again;
  838. }
  839. BUG_ON(ret < 0);
  840. leaf = path->nodes[0];
  841. fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
  842. struct btrfs_file_extent_item);
  843. btrfs_set_file_extent_num_bytes(leaf, fi,
  844. split - key.offset);
  845. fi = btrfs_item_ptr(leaf, path->slots[0],
  846. struct btrfs_file_extent_item);
  847. btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
  848. btrfs_set_file_extent_num_bytes(leaf, fi,
  849. extent_end - split);
  850. btrfs_mark_buffer_dirty(leaf);
  851. ret = btrfs_inc_extent_ref(trans, root, bytenr, num_bytes, 0,
  852. root->root_key.objectid,
  853. ino, orig_offset);
  854. BUG_ON(ret);
  855. if (split == start) {
  856. key.offset = start;
  857. } else {
  858. BUG_ON(start != key.offset);
  859. path->slots[0]--;
  860. extent_end = end;
  861. }
  862. recow = 1;
  863. }
  864. other_start = end;
  865. other_end = 0;
  866. if (extent_mergeable(leaf, path->slots[0] + 1,
  867. ino, bytenr, orig_offset,
  868. &other_start, &other_end)) {
  869. if (recow) {
  870. btrfs_release_path(path);
  871. goto again;
  872. }
  873. extent_end = other_end;
  874. del_slot = path->slots[0] + 1;
  875. del_nr++;
  876. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  877. 0, root->root_key.objectid,
  878. ino, orig_offset);
  879. BUG_ON(ret);
  880. }
  881. other_start = 0;
  882. other_end = start;
  883. if (extent_mergeable(leaf, path->slots[0] - 1,
  884. ino, bytenr, orig_offset,
  885. &other_start, &other_end)) {
  886. if (recow) {
  887. btrfs_release_path(path);
  888. goto again;
  889. }
  890. key.offset = other_start;
  891. del_slot = path->slots[0];
  892. del_nr++;
  893. ret = btrfs_free_extent(trans, root, bytenr, num_bytes,
  894. 0, root->root_key.objectid,
  895. ino, orig_offset);
  896. BUG_ON(ret);
  897. }
  898. if (del_nr == 0) {
  899. fi = btrfs_item_ptr(leaf, path->slots[0],
  900. struct btrfs_file_extent_item);
  901. btrfs_set_file_extent_type(leaf, fi,
  902. BTRFS_FILE_EXTENT_REG);
  903. btrfs_mark_buffer_dirty(leaf);
  904. } else {
  905. fi = btrfs_item_ptr(leaf, del_slot - 1,
  906. struct btrfs_file_extent_item);
  907. btrfs_set_file_extent_type(leaf, fi,
  908. BTRFS_FILE_EXTENT_REG);
  909. btrfs_set_file_extent_num_bytes(leaf, fi,
  910. extent_end - key.offset);
  911. btrfs_mark_buffer_dirty(leaf);
  912. ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
  913. BUG_ON(ret);
  914. }
  915. out:
  916. btrfs_free_path(path);
  917. return 0;
  918. }
  919. /*
  920. * on error we return an unlocked page and the error value
  921. * on success we return a locked page and 0
  922. */
  923. static int prepare_uptodate_page(struct page *page, u64 pos)
  924. {
  925. int ret = 0;
  926. if ((pos & (PAGE_CACHE_SIZE - 1)) && !PageUptodate(page)) {
  927. ret = btrfs_readpage(NULL, page);
  928. if (ret)
  929. return ret;
  930. lock_page(page);
  931. if (!PageUptodate(page)) {
  932. unlock_page(page);
  933. return -EIO;
  934. }
  935. }
  936. return 0;
  937. }
  938. /*
  939. * this gets pages into the page cache and locks them down, it also properly
  940. * waits for data=ordered extents to finish before allowing the pages to be
  941. * modified.
  942. */
  943. static noinline int prepare_pages(struct btrfs_root *root, struct file *file,
  944. struct page **pages, size_t num_pages,
  945. loff_t pos, unsigned long first_index,
  946. unsigned long last_index, size_t write_bytes)
  947. {
  948. struct extent_state *cached_state = NULL;
  949. int i;
  950. unsigned long index = pos >> PAGE_CACHE_SHIFT;
  951. struct inode *inode = fdentry(file)->d_inode;
  952. int err = 0;
  953. int faili = 0;
  954. u64 start_pos;
  955. u64 last_pos;
  956. start_pos = pos & ~((u64)root->sectorsize - 1);
  957. last_pos = ((u64)index + num_pages) << PAGE_CACHE_SHIFT;
  958. if (start_pos > inode->i_size) {
  959. err = btrfs_cont_expand(inode, i_size_read(inode), start_pos);
  960. if (err)
  961. return err;
  962. }
  963. again:
  964. for (i = 0; i < num_pages; i++) {
  965. pages[i] = grab_cache_page(inode->i_mapping, index + i);
  966. if (!pages[i]) {
  967. faili = i - 1;
  968. err = -ENOMEM;
  969. goto fail;
  970. }
  971. if (i == 0)
  972. err = prepare_uptodate_page(pages[i], pos);
  973. if (i == num_pages - 1)
  974. err = prepare_uptodate_page(pages[i],
  975. pos + write_bytes);
  976. if (err) {
  977. page_cache_release(pages[i]);
  978. faili = i - 1;
  979. goto fail;
  980. }
  981. wait_on_page_writeback(pages[i]);
  982. }
  983. err = 0;
  984. if (start_pos < inode->i_size) {
  985. struct btrfs_ordered_extent *ordered;
  986. lock_extent_bits(&BTRFS_I(inode)->io_tree,
  987. start_pos, last_pos - 1, 0, &cached_state,
  988. GFP_NOFS);
  989. ordered = btrfs_lookup_first_ordered_extent(inode,
  990. last_pos - 1);
  991. if (ordered &&
  992. ordered->file_offset + ordered->len > start_pos &&
  993. ordered->file_offset < last_pos) {
  994. btrfs_put_ordered_extent(ordered);
  995. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  996. start_pos, last_pos - 1,
  997. &cached_state, GFP_NOFS);
  998. for (i = 0; i < num_pages; i++) {
  999. unlock_page(pages[i]);
  1000. page_cache_release(pages[i]);
  1001. }
  1002. btrfs_wait_ordered_range(inode, start_pos,
  1003. last_pos - start_pos);
  1004. goto again;
  1005. }
  1006. if (ordered)
  1007. btrfs_put_ordered_extent(ordered);
  1008. clear_extent_bit(&BTRFS_I(inode)->io_tree, start_pos,
  1009. last_pos - 1, EXTENT_DIRTY | EXTENT_DELALLOC |
  1010. EXTENT_DO_ACCOUNTING, 0, 0, &cached_state,
  1011. GFP_NOFS);
  1012. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1013. start_pos, last_pos - 1, &cached_state,
  1014. GFP_NOFS);
  1015. }
  1016. for (i = 0; i < num_pages; i++) {
  1017. clear_page_dirty_for_io(pages[i]);
  1018. set_page_extent_mapped(pages[i]);
  1019. WARN_ON(!PageLocked(pages[i]));
  1020. }
  1021. return 0;
  1022. fail:
  1023. while (faili >= 0) {
  1024. unlock_page(pages[faili]);
  1025. page_cache_release(pages[faili]);
  1026. faili--;
  1027. }
  1028. return err;
  1029. }
  1030. static noinline ssize_t __btrfs_buffered_write(struct file *file,
  1031. struct iov_iter *i,
  1032. loff_t pos)
  1033. {
  1034. struct inode *inode = fdentry(file)->d_inode;
  1035. struct btrfs_root *root = BTRFS_I(inode)->root;
  1036. struct page **pages = NULL;
  1037. unsigned long first_index;
  1038. unsigned long last_index;
  1039. size_t num_written = 0;
  1040. int nrptrs;
  1041. int ret = 0;
  1042. nrptrs = min((iov_iter_count(i) + PAGE_CACHE_SIZE - 1) /
  1043. PAGE_CACHE_SIZE, PAGE_CACHE_SIZE /
  1044. (sizeof(struct page *)));
  1045. pages = kmalloc(nrptrs * sizeof(struct page *), GFP_KERNEL);
  1046. if (!pages)
  1047. return -ENOMEM;
  1048. first_index = pos >> PAGE_CACHE_SHIFT;
  1049. last_index = (pos + iov_iter_count(i)) >> PAGE_CACHE_SHIFT;
  1050. while (iov_iter_count(i) > 0) {
  1051. size_t offset = pos & (PAGE_CACHE_SIZE - 1);
  1052. size_t write_bytes = min(iov_iter_count(i),
  1053. nrptrs * (size_t)PAGE_CACHE_SIZE -
  1054. offset);
  1055. size_t num_pages = (write_bytes + offset +
  1056. PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1057. size_t dirty_pages;
  1058. size_t copied;
  1059. WARN_ON(num_pages > nrptrs);
  1060. /*
  1061. * Fault pages before locking them in prepare_pages
  1062. * to avoid recursive lock
  1063. */
  1064. if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
  1065. ret = -EFAULT;
  1066. break;
  1067. }
  1068. ret = btrfs_delalloc_reserve_space(inode,
  1069. num_pages << PAGE_CACHE_SHIFT);
  1070. if (ret)
  1071. break;
  1072. /*
  1073. * This is going to setup the pages array with the number of
  1074. * pages we want, so we don't really need to worry about the
  1075. * contents of pages from loop to loop
  1076. */
  1077. ret = prepare_pages(root, file, pages, num_pages,
  1078. pos, first_index, last_index,
  1079. write_bytes);
  1080. if (ret) {
  1081. btrfs_delalloc_release_space(inode,
  1082. num_pages << PAGE_CACHE_SHIFT);
  1083. break;
  1084. }
  1085. copied = btrfs_copy_from_user(pos, num_pages,
  1086. write_bytes, pages, i);
  1087. /*
  1088. * if we have trouble faulting in the pages, fall
  1089. * back to one page at a time
  1090. */
  1091. if (copied < write_bytes)
  1092. nrptrs = 1;
  1093. if (copied == 0)
  1094. dirty_pages = 0;
  1095. else
  1096. dirty_pages = (copied + offset +
  1097. PAGE_CACHE_SIZE - 1) >>
  1098. PAGE_CACHE_SHIFT;
  1099. /*
  1100. * If we had a short copy we need to release the excess delaloc
  1101. * bytes we reserved. We need to increment outstanding_extents
  1102. * because btrfs_delalloc_release_space will decrement it, but
  1103. * we still have an outstanding extent for the chunk we actually
  1104. * managed to copy.
  1105. */
  1106. if (num_pages > dirty_pages) {
  1107. if (copied > 0)
  1108. atomic_inc(
  1109. &BTRFS_I(inode)->outstanding_extents);
  1110. btrfs_delalloc_release_space(inode,
  1111. (num_pages - dirty_pages) <<
  1112. PAGE_CACHE_SHIFT);
  1113. }
  1114. if (copied > 0) {
  1115. ret = btrfs_dirty_pages(root, inode, pages,
  1116. dirty_pages, pos, copied,
  1117. NULL);
  1118. if (ret) {
  1119. btrfs_delalloc_release_space(inode,
  1120. dirty_pages << PAGE_CACHE_SHIFT);
  1121. btrfs_drop_pages(pages, num_pages);
  1122. break;
  1123. }
  1124. }
  1125. btrfs_drop_pages(pages, num_pages);
  1126. cond_resched();
  1127. balance_dirty_pages_ratelimited_nr(inode->i_mapping,
  1128. dirty_pages);
  1129. if (dirty_pages < (root->leafsize >> PAGE_CACHE_SHIFT) + 1)
  1130. btrfs_btree_balance_dirty(root, 1);
  1131. btrfs_throttle(root);
  1132. pos += copied;
  1133. num_written += copied;
  1134. }
  1135. kfree(pages);
  1136. return num_written ? num_written : ret;
  1137. }
  1138. static ssize_t __btrfs_direct_write(struct kiocb *iocb,
  1139. const struct iovec *iov,
  1140. unsigned long nr_segs, loff_t pos,
  1141. loff_t *ppos, size_t count, size_t ocount)
  1142. {
  1143. struct file *file = iocb->ki_filp;
  1144. struct inode *inode = fdentry(file)->d_inode;
  1145. struct iov_iter i;
  1146. ssize_t written;
  1147. ssize_t written_buffered;
  1148. loff_t endbyte;
  1149. int err;
  1150. written = generic_file_direct_write(iocb, iov, &nr_segs, pos, ppos,
  1151. count, ocount);
  1152. /*
  1153. * the generic O_DIRECT will update in-memory i_size after the
  1154. * DIOs are done. But our endio handlers that update the on
  1155. * disk i_size never update past the in memory i_size. So we
  1156. * need one more update here to catch any additions to the
  1157. * file
  1158. */
  1159. if (inode->i_size != BTRFS_I(inode)->disk_i_size) {
  1160. btrfs_ordered_update_i_size(inode, inode->i_size, NULL);
  1161. mark_inode_dirty(inode);
  1162. }
  1163. if (written < 0 || written == count)
  1164. return written;
  1165. pos += written;
  1166. count -= written;
  1167. iov_iter_init(&i, iov, nr_segs, count, written);
  1168. written_buffered = __btrfs_buffered_write(file, &i, pos);
  1169. if (written_buffered < 0) {
  1170. err = written_buffered;
  1171. goto out;
  1172. }
  1173. endbyte = pos + written_buffered - 1;
  1174. err = filemap_write_and_wait_range(file->f_mapping, pos, endbyte);
  1175. if (err)
  1176. goto out;
  1177. written += written_buffered;
  1178. *ppos = pos + written_buffered;
  1179. invalidate_mapping_pages(file->f_mapping, pos >> PAGE_CACHE_SHIFT,
  1180. endbyte >> PAGE_CACHE_SHIFT);
  1181. out:
  1182. return written ? written : err;
  1183. }
  1184. static ssize_t btrfs_file_aio_write(struct kiocb *iocb,
  1185. const struct iovec *iov,
  1186. unsigned long nr_segs, loff_t pos)
  1187. {
  1188. struct file *file = iocb->ki_filp;
  1189. struct inode *inode = fdentry(file)->d_inode;
  1190. struct btrfs_root *root = BTRFS_I(inode)->root;
  1191. loff_t *ppos = &iocb->ki_pos;
  1192. ssize_t num_written = 0;
  1193. ssize_t err = 0;
  1194. size_t count, ocount;
  1195. vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
  1196. mutex_lock(&inode->i_mutex);
  1197. err = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ);
  1198. if (err) {
  1199. mutex_unlock(&inode->i_mutex);
  1200. goto out;
  1201. }
  1202. count = ocount;
  1203. current->backing_dev_info = inode->i_mapping->backing_dev_info;
  1204. err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
  1205. if (err) {
  1206. mutex_unlock(&inode->i_mutex);
  1207. goto out;
  1208. }
  1209. if (count == 0) {
  1210. mutex_unlock(&inode->i_mutex);
  1211. goto out;
  1212. }
  1213. err = file_remove_suid(file);
  1214. if (err) {
  1215. mutex_unlock(&inode->i_mutex);
  1216. goto out;
  1217. }
  1218. /*
  1219. * If BTRFS flips readonly due to some impossible error
  1220. * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
  1221. * although we have opened a file as writable, we have
  1222. * to stop this write operation to ensure FS consistency.
  1223. */
  1224. if (root->fs_info->fs_state & BTRFS_SUPER_FLAG_ERROR) {
  1225. mutex_unlock(&inode->i_mutex);
  1226. err = -EROFS;
  1227. goto out;
  1228. }
  1229. file_update_time(file);
  1230. BTRFS_I(inode)->sequence++;
  1231. if (unlikely(file->f_flags & O_DIRECT)) {
  1232. num_written = __btrfs_direct_write(iocb, iov, nr_segs,
  1233. pos, ppos, count, ocount);
  1234. } else {
  1235. struct iov_iter i;
  1236. iov_iter_init(&i, iov, nr_segs, count, num_written);
  1237. num_written = __btrfs_buffered_write(file, &i, pos);
  1238. if (num_written > 0)
  1239. *ppos = pos + num_written;
  1240. }
  1241. mutex_unlock(&inode->i_mutex);
  1242. /*
  1243. * we want to make sure fsync finds this change
  1244. * but we haven't joined a transaction running right now.
  1245. *
  1246. * Later on, someone is sure to update the inode and get the
  1247. * real transid recorded.
  1248. *
  1249. * We set last_trans now to the fs_info generation + 1,
  1250. * this will either be one more than the running transaction
  1251. * or the generation used for the next transaction if there isn't
  1252. * one running right now.
  1253. */
  1254. BTRFS_I(inode)->last_trans = root->fs_info->generation + 1;
  1255. if (num_written > 0 || num_written == -EIOCBQUEUED) {
  1256. err = generic_write_sync(file, pos, num_written);
  1257. if (err < 0 && num_written > 0)
  1258. num_written = err;
  1259. }
  1260. out:
  1261. current->backing_dev_info = NULL;
  1262. return num_written ? num_written : err;
  1263. }
  1264. int btrfs_release_file(struct inode *inode, struct file *filp)
  1265. {
  1266. /*
  1267. * ordered_data_close is set by settattr when we are about to truncate
  1268. * a file from a non-zero size to a zero size. This tries to
  1269. * flush down new bytes that may have been written if the
  1270. * application were using truncate to replace a file in place.
  1271. */
  1272. if (BTRFS_I(inode)->ordered_data_close) {
  1273. BTRFS_I(inode)->ordered_data_close = 0;
  1274. btrfs_add_ordered_operation(NULL, BTRFS_I(inode)->root, inode);
  1275. if (inode->i_size > BTRFS_ORDERED_OPERATIONS_FLUSH_LIMIT)
  1276. filemap_flush(inode->i_mapping);
  1277. }
  1278. if (filp->private_data)
  1279. btrfs_ioctl_trans_end(filp);
  1280. return 0;
  1281. }
  1282. /*
  1283. * fsync call for both files and directories. This logs the inode into
  1284. * the tree log instead of forcing full commits whenever possible.
  1285. *
  1286. * It needs to call filemap_fdatawait so that all ordered extent updates are
  1287. * in the metadata btree are up to date for copying to the log.
  1288. *
  1289. * It drops the inode mutex before doing the tree log commit. This is an
  1290. * important optimization for directories because holding the mutex prevents
  1291. * new operations on the dir while we write to disk.
  1292. */
  1293. int btrfs_sync_file(struct file *file, int datasync)
  1294. {
  1295. struct dentry *dentry = file->f_path.dentry;
  1296. struct inode *inode = dentry->d_inode;
  1297. struct btrfs_root *root = BTRFS_I(inode)->root;
  1298. int ret = 0;
  1299. struct btrfs_trans_handle *trans;
  1300. trace_btrfs_sync_file(file, datasync);
  1301. /* we wait first, since the writeback may change the inode */
  1302. root->log_batch++;
  1303. /* the VFS called filemap_fdatawrite for us */
  1304. btrfs_wait_ordered_range(inode, 0, (u64)-1);
  1305. root->log_batch++;
  1306. /*
  1307. * check the transaction that last modified this inode
  1308. * and see if its already been committed
  1309. */
  1310. if (!BTRFS_I(inode)->last_trans)
  1311. goto out;
  1312. /*
  1313. * if the last transaction that changed this file was before
  1314. * the current transaction, we can bail out now without any
  1315. * syncing
  1316. */
  1317. smp_mb();
  1318. if (BTRFS_I(inode)->last_trans <=
  1319. root->fs_info->last_trans_committed) {
  1320. BTRFS_I(inode)->last_trans = 0;
  1321. goto out;
  1322. }
  1323. /*
  1324. * ok we haven't committed the transaction yet, lets do a commit
  1325. */
  1326. if (file->private_data)
  1327. btrfs_ioctl_trans_end(file);
  1328. trans = btrfs_start_transaction(root, 0);
  1329. if (IS_ERR(trans)) {
  1330. ret = PTR_ERR(trans);
  1331. goto out;
  1332. }
  1333. ret = btrfs_log_dentry_safe(trans, root, dentry);
  1334. if (ret < 0)
  1335. goto out;
  1336. /* we've logged all the items and now have a consistent
  1337. * version of the file in the log. It is possible that
  1338. * someone will come in and modify the file, but that's
  1339. * fine because the log is consistent on disk, and we
  1340. * have references to all of the file's extents
  1341. *
  1342. * It is possible that someone will come in and log the
  1343. * file again, but that will end up using the synchronization
  1344. * inside btrfs_sync_log to keep things safe.
  1345. */
  1346. mutex_unlock(&dentry->d_inode->i_mutex);
  1347. if (ret != BTRFS_NO_LOG_SYNC) {
  1348. if (ret > 0) {
  1349. ret = btrfs_commit_transaction(trans, root);
  1350. } else {
  1351. ret = btrfs_sync_log(trans, root);
  1352. if (ret == 0)
  1353. ret = btrfs_end_transaction(trans, root);
  1354. else
  1355. ret = btrfs_commit_transaction(trans, root);
  1356. }
  1357. } else {
  1358. ret = btrfs_end_transaction(trans, root);
  1359. }
  1360. mutex_lock(&dentry->d_inode->i_mutex);
  1361. out:
  1362. return ret > 0 ? -EIO : ret;
  1363. }
  1364. static const struct vm_operations_struct btrfs_file_vm_ops = {
  1365. .fault = filemap_fault,
  1366. .page_mkwrite = btrfs_page_mkwrite,
  1367. };
  1368. static int btrfs_file_mmap(struct file *filp, struct vm_area_struct *vma)
  1369. {
  1370. struct address_space *mapping = filp->f_mapping;
  1371. if (!mapping->a_ops->readpage)
  1372. return -ENOEXEC;
  1373. file_accessed(filp);
  1374. vma->vm_ops = &btrfs_file_vm_ops;
  1375. vma->vm_flags |= VM_CAN_NONLINEAR;
  1376. return 0;
  1377. }
  1378. static long btrfs_fallocate(struct file *file, int mode,
  1379. loff_t offset, loff_t len)
  1380. {
  1381. struct inode *inode = file->f_path.dentry->d_inode;
  1382. struct extent_state *cached_state = NULL;
  1383. u64 cur_offset;
  1384. u64 last_byte;
  1385. u64 alloc_start;
  1386. u64 alloc_end;
  1387. u64 alloc_hint = 0;
  1388. u64 locked_end;
  1389. u64 mask = BTRFS_I(inode)->root->sectorsize - 1;
  1390. struct extent_map *em;
  1391. int ret;
  1392. alloc_start = offset & ~mask;
  1393. alloc_end = (offset + len + mask) & ~mask;
  1394. /* We only support the FALLOC_FL_KEEP_SIZE mode */
  1395. if (mode & ~FALLOC_FL_KEEP_SIZE)
  1396. return -EOPNOTSUPP;
  1397. /*
  1398. * wait for ordered IO before we have any locks. We'll loop again
  1399. * below with the locks held.
  1400. */
  1401. btrfs_wait_ordered_range(inode, alloc_start, alloc_end - alloc_start);
  1402. mutex_lock(&inode->i_mutex);
  1403. ret = inode_newsize_ok(inode, alloc_end);
  1404. if (ret)
  1405. goto out;
  1406. if (alloc_start > inode->i_size) {
  1407. ret = btrfs_cont_expand(inode, i_size_read(inode),
  1408. alloc_start);
  1409. if (ret)
  1410. goto out;
  1411. }
  1412. ret = btrfs_check_data_free_space(inode, alloc_end - alloc_start);
  1413. if (ret)
  1414. goto out;
  1415. locked_end = alloc_end - 1;
  1416. while (1) {
  1417. struct btrfs_ordered_extent *ordered;
  1418. /* the extent lock is ordered inside the running
  1419. * transaction
  1420. */
  1421. lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
  1422. locked_end, 0, &cached_state, GFP_NOFS);
  1423. ordered = btrfs_lookup_first_ordered_extent(inode,
  1424. alloc_end - 1);
  1425. if (ordered &&
  1426. ordered->file_offset + ordered->len > alloc_start &&
  1427. ordered->file_offset < alloc_end) {
  1428. btrfs_put_ordered_extent(ordered);
  1429. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  1430. alloc_start, locked_end,
  1431. &cached_state, GFP_NOFS);
  1432. /*
  1433. * we can't wait on the range with the transaction
  1434. * running or with the extent lock held
  1435. */
  1436. btrfs_wait_ordered_range(inode, alloc_start,
  1437. alloc_end - alloc_start);
  1438. } else {
  1439. if (ordered)
  1440. btrfs_put_ordered_extent(ordered);
  1441. break;
  1442. }
  1443. }
  1444. cur_offset = alloc_start;
  1445. while (1) {
  1446. em = btrfs_get_extent(inode, NULL, 0, cur_offset,
  1447. alloc_end - cur_offset, 0);
  1448. BUG_ON(IS_ERR_OR_NULL(em));
  1449. last_byte = min(extent_map_end(em), alloc_end);
  1450. last_byte = (last_byte + mask) & ~mask;
  1451. if (em->block_start == EXTENT_MAP_HOLE ||
  1452. (cur_offset >= inode->i_size &&
  1453. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
  1454. ret = btrfs_prealloc_file_range(inode, mode, cur_offset,
  1455. last_byte - cur_offset,
  1456. 1 << inode->i_blkbits,
  1457. offset + len,
  1458. &alloc_hint);
  1459. if (ret < 0) {
  1460. free_extent_map(em);
  1461. break;
  1462. }
  1463. }
  1464. free_extent_map(em);
  1465. cur_offset = last_byte;
  1466. if (cur_offset >= alloc_end) {
  1467. ret = 0;
  1468. break;
  1469. }
  1470. }
  1471. unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
  1472. &cached_state, GFP_NOFS);
  1473. btrfs_free_reserved_data_space(inode, alloc_end - alloc_start);
  1474. out:
  1475. mutex_unlock(&inode->i_mutex);
  1476. return ret;
  1477. }
  1478. const struct file_operations btrfs_file_operations = {
  1479. .llseek = generic_file_llseek,
  1480. .read = do_sync_read,
  1481. .write = do_sync_write,
  1482. .aio_read = generic_file_aio_read,
  1483. .splice_read = generic_file_splice_read,
  1484. .aio_write = btrfs_file_aio_write,
  1485. .mmap = btrfs_file_mmap,
  1486. .open = generic_file_open,
  1487. .release = btrfs_release_file,
  1488. .fsync = btrfs_sync_file,
  1489. .fallocate = btrfs_fallocate,
  1490. .unlocked_ioctl = btrfs_ioctl,
  1491. #ifdef CONFIG_COMPAT
  1492. .compat_ioctl = btrfs_ioctl,
  1493. #endif
  1494. };