mtdswap.c 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569
  1. /*
  2. * Swap block device support for MTDs
  3. * Turns an MTD device into a swap device with block wear leveling
  4. *
  5. * Copyright © 2007,2011 Nokia Corporation. All rights reserved.
  6. *
  7. * Authors: Jarkko Lavinen <jarkko.lavinen@nokia.com>
  8. *
  9. * Based on Richard Purdie's earlier implementation in 2007. Background
  10. * support and lock-less operation written by Adrian Hunter.
  11. *
  12. * This program is free software; you can redistribute it and/or
  13. * modify it under the terms of the GNU General Public License
  14. * version 2 as published by the Free Software Foundation.
  15. *
  16. * This program is distributed in the hope that it will be useful, but
  17. * WITHOUT ANY WARRANTY; without even the implied warranty of
  18. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  19. * General Public License for more details.
  20. *
  21. * You should have received a copy of the GNU General Public License
  22. * along with this program; if not, write to the Free Software
  23. * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  24. * 02110-1301 USA
  25. */
  26. #include <linux/kernel.h>
  27. #include <linux/module.h>
  28. #include <linux/mtd/mtd.h>
  29. #include <linux/mtd/blktrans.h>
  30. #include <linux/rbtree.h>
  31. #include <linux/sched.h>
  32. #include <linux/slab.h>
  33. #include <linux/vmalloc.h>
  34. #include <linux/genhd.h>
  35. #include <linux/swap.h>
  36. #include <linux/debugfs.h>
  37. #include <linux/seq_file.h>
  38. #include <linux/device.h>
  39. #include <linux/math64.h>
  40. #define MTDSWAP_PREFIX "mtdswap"
  41. /*
  42. * The number of free eraseblocks when GC should stop
  43. */
  44. #define CLEAN_BLOCK_THRESHOLD 20
  45. /*
  46. * Number of free eraseblocks below which GC can also collect low frag
  47. * blocks.
  48. */
  49. #define LOW_FRAG_GC_TRESHOLD 5
  50. /*
  51. * Wear level cost amortization. We want to do wear leveling on the background
  52. * without disturbing gc too much. This is made by defining max GC frequency.
  53. * Frequency value 6 means 1/6 of the GC passes will pick an erase block based
  54. * on the biggest wear difference rather than the biggest dirtiness.
  55. *
  56. * The lower freq2 should be chosen so that it makes sure the maximum erase
  57. * difference will decrease even if a malicious application is deliberately
  58. * trying to make erase differences large.
  59. */
  60. #define MAX_ERASE_DIFF 4000
  61. #define COLLECT_NONDIRTY_BASE MAX_ERASE_DIFF
  62. #define COLLECT_NONDIRTY_FREQ1 6
  63. #define COLLECT_NONDIRTY_FREQ2 4
  64. #define PAGE_UNDEF UINT_MAX
  65. #define BLOCK_UNDEF UINT_MAX
  66. #define BLOCK_ERROR (UINT_MAX - 1)
  67. #define BLOCK_MAX (UINT_MAX - 2)
  68. #define EBLOCK_BAD (1 << 0)
  69. #define EBLOCK_NOMAGIC (1 << 1)
  70. #define EBLOCK_BITFLIP (1 << 2)
  71. #define EBLOCK_FAILED (1 << 3)
  72. #define EBLOCK_READERR (1 << 4)
  73. #define EBLOCK_IDX_SHIFT 5
  74. struct swap_eb {
  75. struct rb_node rb;
  76. struct rb_root *root;
  77. unsigned int flags;
  78. unsigned int active_count;
  79. unsigned int erase_count;
  80. unsigned int pad; /* speeds up pointer decrement */
  81. };
  82. #define MTDSWAP_ECNT_MIN(rbroot) (rb_entry(rb_first(rbroot), struct swap_eb, \
  83. rb)->erase_count)
  84. #define MTDSWAP_ECNT_MAX(rbroot) (rb_entry(rb_last(rbroot), struct swap_eb, \
  85. rb)->erase_count)
  86. struct mtdswap_tree {
  87. struct rb_root root;
  88. unsigned int count;
  89. };
  90. enum {
  91. MTDSWAP_CLEAN,
  92. MTDSWAP_USED,
  93. MTDSWAP_LOWFRAG,
  94. MTDSWAP_HIFRAG,
  95. MTDSWAP_DIRTY,
  96. MTDSWAP_BITFLIP,
  97. MTDSWAP_FAILING,
  98. MTDSWAP_TREE_CNT,
  99. };
  100. struct mtdswap_dev {
  101. struct mtd_blktrans_dev *mbd_dev;
  102. struct mtd_info *mtd;
  103. struct device *dev;
  104. unsigned int *page_data;
  105. unsigned int *revmap;
  106. unsigned int eblks;
  107. unsigned int spare_eblks;
  108. unsigned int pages_per_eblk;
  109. unsigned int max_erase_count;
  110. struct swap_eb *eb_data;
  111. struct mtdswap_tree trees[MTDSWAP_TREE_CNT];
  112. unsigned long long sect_read_count;
  113. unsigned long long sect_write_count;
  114. unsigned long long mtd_write_count;
  115. unsigned long long mtd_read_count;
  116. unsigned long long discard_count;
  117. unsigned long long discard_page_count;
  118. unsigned int curr_write_pos;
  119. struct swap_eb *curr_write;
  120. char *page_buf;
  121. char *oob_buf;
  122. };
  123. struct mtdswap_oobdata {
  124. __le16 magic;
  125. __le32 count;
  126. } __packed;
  127. #define MTDSWAP_MAGIC_CLEAN 0x2095
  128. #define MTDSWAP_MAGIC_DIRTY (MTDSWAP_MAGIC_CLEAN + 1)
  129. #define MTDSWAP_TYPE_CLEAN 0
  130. #define MTDSWAP_TYPE_DIRTY 1
  131. #define MTDSWAP_OOBSIZE sizeof(struct mtdswap_oobdata)
  132. #define MTDSWAP_ERASE_RETRIES 3 /* Before marking erase block bad */
  133. #define MTDSWAP_IO_RETRIES 3
  134. enum {
  135. MTDSWAP_SCANNED_CLEAN,
  136. MTDSWAP_SCANNED_DIRTY,
  137. MTDSWAP_SCANNED_BITFLIP,
  138. MTDSWAP_SCANNED_BAD,
  139. };
  140. /*
  141. * In the worst case mtdswap_writesect() has allocated the last clean
  142. * page from the current block and is then pre-empted by the GC
  143. * thread. The thread can consume a full erase block when moving a
  144. * block.
  145. */
  146. #define MIN_SPARE_EBLOCKS 2
  147. #define MIN_ERASE_BLOCKS (MIN_SPARE_EBLOCKS + 1)
  148. #define TREE_ROOT(d, name) (&d->trees[MTDSWAP_ ## name].root)
  149. #define TREE_EMPTY(d, name) (TREE_ROOT(d, name)->rb_node == NULL)
  150. #define TREE_NONEMPTY(d, name) (!TREE_EMPTY(d, name))
  151. #define TREE_COUNT(d, name) (d->trees[MTDSWAP_ ## name].count)
  152. #define MTDSWAP_MBD_TO_MTDSWAP(dev) ((struct mtdswap_dev *)dev->priv)
  153. static char partitions[128] = "";
  154. module_param_string(partitions, partitions, sizeof(partitions), 0444);
  155. MODULE_PARM_DESC(partitions, "MTD partition numbers to use as swap "
  156. "partitions=\"1,3,5\"");
  157. static unsigned int spare_eblocks = 10;
  158. module_param(spare_eblocks, uint, 0444);
  159. MODULE_PARM_DESC(spare_eblocks, "Percentage of spare erase blocks for "
  160. "garbage collection (default 10%)");
  161. static bool header; /* false */
  162. module_param(header, bool, 0444);
  163. MODULE_PARM_DESC(header,
  164. "Include builtin swap header (default 0, without header)");
  165. static int mtdswap_gc(struct mtdswap_dev *d, unsigned int background);
  166. static loff_t mtdswap_eb_offset(struct mtdswap_dev *d, struct swap_eb *eb)
  167. {
  168. return (loff_t)(eb - d->eb_data) * d->mtd->erasesize;
  169. }
  170. static void mtdswap_eb_detach(struct mtdswap_dev *d, struct swap_eb *eb)
  171. {
  172. unsigned int oldidx;
  173. struct mtdswap_tree *tp;
  174. if (eb->root) {
  175. tp = container_of(eb->root, struct mtdswap_tree, root);
  176. oldidx = tp - &d->trees[0];
  177. d->trees[oldidx].count--;
  178. rb_erase(&eb->rb, eb->root);
  179. }
  180. }
  181. static void __mtdswap_rb_add(struct rb_root *root, struct swap_eb *eb)
  182. {
  183. struct rb_node **p, *parent = NULL;
  184. struct swap_eb *cur;
  185. p = &root->rb_node;
  186. while (*p) {
  187. parent = *p;
  188. cur = rb_entry(parent, struct swap_eb, rb);
  189. if (eb->erase_count > cur->erase_count)
  190. p = &(*p)->rb_right;
  191. else
  192. p = &(*p)->rb_left;
  193. }
  194. rb_link_node(&eb->rb, parent, p);
  195. rb_insert_color(&eb->rb, root);
  196. }
  197. static void mtdswap_rb_add(struct mtdswap_dev *d, struct swap_eb *eb, int idx)
  198. {
  199. struct rb_root *root;
  200. if (eb->root == &d->trees[idx].root)
  201. return;
  202. mtdswap_eb_detach(d, eb);
  203. root = &d->trees[idx].root;
  204. __mtdswap_rb_add(root, eb);
  205. eb->root = root;
  206. d->trees[idx].count++;
  207. }
  208. static struct rb_node *mtdswap_rb_index(struct rb_root *root, unsigned int idx)
  209. {
  210. struct rb_node *p;
  211. unsigned int i;
  212. p = rb_first(root);
  213. i = 0;
  214. while (i < idx && p) {
  215. p = rb_next(p);
  216. i++;
  217. }
  218. return p;
  219. }
  220. static int mtdswap_handle_badblock(struct mtdswap_dev *d, struct swap_eb *eb)
  221. {
  222. int ret;
  223. loff_t offset;
  224. d->spare_eblks--;
  225. eb->flags |= EBLOCK_BAD;
  226. mtdswap_eb_detach(d, eb);
  227. eb->root = NULL;
  228. /* badblocks not supported */
  229. if (!mtd_can_have_bb(d->mtd))
  230. return 1;
  231. offset = mtdswap_eb_offset(d, eb);
  232. dev_warn(d->dev, "Marking bad block at %08llx\n", offset);
  233. ret = mtd_block_markbad(d->mtd, offset);
  234. if (ret) {
  235. dev_warn(d->dev, "Mark block bad failed for block at %08llx "
  236. "error %d\n", offset, ret);
  237. return ret;
  238. }
  239. return 1;
  240. }
  241. static int mtdswap_handle_write_error(struct mtdswap_dev *d, struct swap_eb *eb)
  242. {
  243. unsigned int marked = eb->flags & EBLOCK_FAILED;
  244. struct swap_eb *curr_write = d->curr_write;
  245. eb->flags |= EBLOCK_FAILED;
  246. if (curr_write == eb) {
  247. d->curr_write = NULL;
  248. if (!marked && d->curr_write_pos != 0) {
  249. mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
  250. return 0;
  251. }
  252. }
  253. return mtdswap_handle_badblock(d, eb);
  254. }
  255. static int mtdswap_read_oob(struct mtdswap_dev *d, loff_t from,
  256. struct mtd_oob_ops *ops)
  257. {
  258. int ret = mtd_read_oob(d->mtd, from, ops);
  259. if (mtd_is_bitflip(ret))
  260. return ret;
  261. if (ret) {
  262. dev_warn(d->dev, "Read OOB failed %d for block at %08llx\n",
  263. ret, from);
  264. return ret;
  265. }
  266. if (ops->oobretlen < ops->ooblen) {
  267. dev_warn(d->dev, "Read OOB return short read (%zd bytes not "
  268. "%zd) for block at %08llx\n",
  269. ops->oobretlen, ops->ooblen, from);
  270. return -EIO;
  271. }
  272. return 0;
  273. }
  274. static int mtdswap_read_markers(struct mtdswap_dev *d, struct swap_eb *eb)
  275. {
  276. struct mtdswap_oobdata *data, *data2;
  277. int ret;
  278. loff_t offset;
  279. struct mtd_oob_ops ops;
  280. offset = mtdswap_eb_offset(d, eb);
  281. /* Check first if the block is bad. */
  282. if (mtd_can_have_bb(d->mtd) && mtd_block_isbad(d->mtd, offset))
  283. return MTDSWAP_SCANNED_BAD;
  284. ops.ooblen = 2 * d->mtd->oobavail;
  285. ops.oobbuf = d->oob_buf;
  286. ops.ooboffs = 0;
  287. ops.datbuf = NULL;
  288. ops.mode = MTD_OPS_AUTO_OOB;
  289. ret = mtdswap_read_oob(d, offset, &ops);
  290. if (ret && !mtd_is_bitflip(ret))
  291. return ret;
  292. data = (struct mtdswap_oobdata *)d->oob_buf;
  293. data2 = (struct mtdswap_oobdata *)
  294. (d->oob_buf + d->mtd->oobavail);
  295. if (le16_to_cpu(data->magic) == MTDSWAP_MAGIC_CLEAN) {
  296. eb->erase_count = le32_to_cpu(data->count);
  297. if (mtd_is_bitflip(ret))
  298. ret = MTDSWAP_SCANNED_BITFLIP;
  299. else {
  300. if (le16_to_cpu(data2->magic) == MTDSWAP_MAGIC_DIRTY)
  301. ret = MTDSWAP_SCANNED_DIRTY;
  302. else
  303. ret = MTDSWAP_SCANNED_CLEAN;
  304. }
  305. } else {
  306. eb->flags |= EBLOCK_NOMAGIC;
  307. ret = MTDSWAP_SCANNED_DIRTY;
  308. }
  309. return ret;
  310. }
  311. static int mtdswap_write_marker(struct mtdswap_dev *d, struct swap_eb *eb,
  312. u16 marker)
  313. {
  314. struct mtdswap_oobdata n;
  315. int ret;
  316. loff_t offset;
  317. struct mtd_oob_ops ops;
  318. ops.ooboffs = 0;
  319. ops.oobbuf = (uint8_t *)&n;
  320. ops.mode = MTD_OPS_AUTO_OOB;
  321. ops.datbuf = NULL;
  322. if (marker == MTDSWAP_TYPE_CLEAN) {
  323. n.magic = cpu_to_le16(MTDSWAP_MAGIC_CLEAN);
  324. n.count = cpu_to_le32(eb->erase_count);
  325. ops.ooblen = MTDSWAP_OOBSIZE;
  326. offset = mtdswap_eb_offset(d, eb);
  327. } else {
  328. n.magic = cpu_to_le16(MTDSWAP_MAGIC_DIRTY);
  329. ops.ooblen = sizeof(n.magic);
  330. offset = mtdswap_eb_offset(d, eb) + d->mtd->writesize;
  331. }
  332. ret = mtd_write_oob(d->mtd, offset, &ops);
  333. if (ret) {
  334. dev_warn(d->dev, "Write OOB failed for block at %08llx "
  335. "error %d\n", offset, ret);
  336. if (ret == -EIO || mtd_is_eccerr(ret))
  337. mtdswap_handle_write_error(d, eb);
  338. return ret;
  339. }
  340. if (ops.oobretlen != ops.ooblen) {
  341. dev_warn(d->dev, "Short OOB write for block at %08llx: "
  342. "%zd not %zd\n",
  343. offset, ops.oobretlen, ops.ooblen);
  344. return ret;
  345. }
  346. return 0;
  347. }
  348. /*
  349. * Are there any erase blocks without MAGIC_CLEAN header, presumably
  350. * because power was cut off after erase but before header write? We
  351. * need to guestimate the erase count.
  352. */
  353. static void mtdswap_check_counts(struct mtdswap_dev *d)
  354. {
  355. struct rb_root hist_root = RB_ROOT;
  356. struct rb_node *medrb;
  357. struct swap_eb *eb;
  358. unsigned int i, cnt, median;
  359. cnt = 0;
  360. for (i = 0; i < d->eblks; i++) {
  361. eb = d->eb_data + i;
  362. if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_BAD | EBLOCK_READERR))
  363. continue;
  364. __mtdswap_rb_add(&hist_root, eb);
  365. cnt++;
  366. }
  367. if (cnt == 0)
  368. return;
  369. medrb = mtdswap_rb_index(&hist_root, cnt / 2);
  370. median = rb_entry(medrb, struct swap_eb, rb)->erase_count;
  371. d->max_erase_count = MTDSWAP_ECNT_MAX(&hist_root);
  372. for (i = 0; i < d->eblks; i++) {
  373. eb = d->eb_data + i;
  374. if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_READERR))
  375. eb->erase_count = median;
  376. if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_BAD | EBLOCK_READERR))
  377. continue;
  378. rb_erase(&eb->rb, &hist_root);
  379. }
  380. }
  381. static void mtdswap_scan_eblks(struct mtdswap_dev *d)
  382. {
  383. int status;
  384. unsigned int i, idx;
  385. struct swap_eb *eb;
  386. for (i = 0; i < d->eblks; i++) {
  387. eb = d->eb_data + i;
  388. status = mtdswap_read_markers(d, eb);
  389. if (status < 0)
  390. eb->flags |= EBLOCK_READERR;
  391. else if (status == MTDSWAP_SCANNED_BAD) {
  392. eb->flags |= EBLOCK_BAD;
  393. continue;
  394. }
  395. switch (status) {
  396. case MTDSWAP_SCANNED_CLEAN:
  397. idx = MTDSWAP_CLEAN;
  398. break;
  399. case MTDSWAP_SCANNED_DIRTY:
  400. case MTDSWAP_SCANNED_BITFLIP:
  401. idx = MTDSWAP_DIRTY;
  402. break;
  403. default:
  404. idx = MTDSWAP_FAILING;
  405. }
  406. eb->flags |= (idx << EBLOCK_IDX_SHIFT);
  407. }
  408. mtdswap_check_counts(d);
  409. for (i = 0; i < d->eblks; i++) {
  410. eb = d->eb_data + i;
  411. if (eb->flags & EBLOCK_BAD)
  412. continue;
  413. idx = eb->flags >> EBLOCK_IDX_SHIFT;
  414. mtdswap_rb_add(d, eb, idx);
  415. }
  416. }
  417. /*
  418. * Place eblk into a tree corresponding to its number of active blocks
  419. * it contains.
  420. */
  421. static void mtdswap_store_eb(struct mtdswap_dev *d, struct swap_eb *eb)
  422. {
  423. unsigned int weight = eb->active_count;
  424. unsigned int maxweight = d->pages_per_eblk;
  425. if (eb == d->curr_write)
  426. return;
  427. if (eb->flags & EBLOCK_BITFLIP)
  428. mtdswap_rb_add(d, eb, MTDSWAP_BITFLIP);
  429. else if (eb->flags & (EBLOCK_READERR | EBLOCK_FAILED))
  430. mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
  431. if (weight == maxweight)
  432. mtdswap_rb_add(d, eb, MTDSWAP_USED);
  433. else if (weight == 0)
  434. mtdswap_rb_add(d, eb, MTDSWAP_DIRTY);
  435. else if (weight > (maxweight/2))
  436. mtdswap_rb_add(d, eb, MTDSWAP_LOWFRAG);
  437. else
  438. mtdswap_rb_add(d, eb, MTDSWAP_HIFRAG);
  439. }
  440. static void mtdswap_erase_callback(struct erase_info *done)
  441. {
  442. wait_queue_head_t *wait_q = (wait_queue_head_t *)done->priv;
  443. wake_up(wait_q);
  444. }
  445. static int mtdswap_erase_block(struct mtdswap_dev *d, struct swap_eb *eb)
  446. {
  447. struct mtd_info *mtd = d->mtd;
  448. struct erase_info erase;
  449. wait_queue_head_t wq;
  450. unsigned int retries = 0;
  451. int ret;
  452. eb->erase_count++;
  453. if (eb->erase_count > d->max_erase_count)
  454. d->max_erase_count = eb->erase_count;
  455. retry:
  456. init_waitqueue_head(&wq);
  457. memset(&erase, 0, sizeof(struct erase_info));
  458. erase.mtd = mtd;
  459. erase.callback = mtdswap_erase_callback;
  460. erase.addr = mtdswap_eb_offset(d, eb);
  461. erase.len = mtd->erasesize;
  462. erase.priv = (u_long)&wq;
  463. ret = mtd_erase(mtd, &erase);
  464. if (ret) {
  465. if (retries++ < MTDSWAP_ERASE_RETRIES) {
  466. dev_warn(d->dev,
  467. "erase of erase block %#llx on %s failed",
  468. erase.addr, mtd->name);
  469. yield();
  470. goto retry;
  471. }
  472. dev_err(d->dev, "Cannot erase erase block %#llx on %s\n",
  473. erase.addr, mtd->name);
  474. mtdswap_handle_badblock(d, eb);
  475. return -EIO;
  476. }
  477. ret = wait_event_interruptible(wq, erase.state == MTD_ERASE_DONE ||
  478. erase.state == MTD_ERASE_FAILED);
  479. if (ret) {
  480. dev_err(d->dev, "Interrupted erase block %#llx erasure on %s\n",
  481. erase.addr, mtd->name);
  482. return -EINTR;
  483. }
  484. if (erase.state == MTD_ERASE_FAILED) {
  485. if (retries++ < MTDSWAP_ERASE_RETRIES) {
  486. dev_warn(d->dev,
  487. "erase of erase block %#llx on %s failed",
  488. erase.addr, mtd->name);
  489. yield();
  490. goto retry;
  491. }
  492. mtdswap_handle_badblock(d, eb);
  493. return -EIO;
  494. }
  495. return 0;
  496. }
  497. static int mtdswap_map_free_block(struct mtdswap_dev *d, unsigned int page,
  498. unsigned int *block)
  499. {
  500. int ret;
  501. struct swap_eb *old_eb = d->curr_write;
  502. struct rb_root *clean_root;
  503. struct swap_eb *eb;
  504. if (old_eb == NULL || d->curr_write_pos >= d->pages_per_eblk) {
  505. do {
  506. if (TREE_EMPTY(d, CLEAN))
  507. return -ENOSPC;
  508. clean_root = TREE_ROOT(d, CLEAN);
  509. eb = rb_entry(rb_first(clean_root), struct swap_eb, rb);
  510. rb_erase(&eb->rb, clean_root);
  511. eb->root = NULL;
  512. TREE_COUNT(d, CLEAN)--;
  513. ret = mtdswap_write_marker(d, eb, MTDSWAP_TYPE_DIRTY);
  514. } while (ret == -EIO || mtd_is_eccerr(ret));
  515. if (ret)
  516. return ret;
  517. d->curr_write_pos = 0;
  518. d->curr_write = eb;
  519. if (old_eb)
  520. mtdswap_store_eb(d, old_eb);
  521. }
  522. *block = (d->curr_write - d->eb_data) * d->pages_per_eblk +
  523. d->curr_write_pos;
  524. d->curr_write->active_count++;
  525. d->revmap[*block] = page;
  526. d->curr_write_pos++;
  527. return 0;
  528. }
  529. static unsigned int mtdswap_free_page_cnt(struct mtdswap_dev *d)
  530. {
  531. return TREE_COUNT(d, CLEAN) * d->pages_per_eblk +
  532. d->pages_per_eblk - d->curr_write_pos;
  533. }
  534. static unsigned int mtdswap_enough_free_pages(struct mtdswap_dev *d)
  535. {
  536. return mtdswap_free_page_cnt(d) > d->pages_per_eblk;
  537. }
  538. static int mtdswap_write_block(struct mtdswap_dev *d, char *buf,
  539. unsigned int page, unsigned int *bp, int gc_context)
  540. {
  541. struct mtd_info *mtd = d->mtd;
  542. struct swap_eb *eb;
  543. size_t retlen;
  544. loff_t writepos;
  545. int ret;
  546. retry:
  547. if (!gc_context)
  548. while (!mtdswap_enough_free_pages(d))
  549. if (mtdswap_gc(d, 0) > 0)
  550. return -ENOSPC;
  551. ret = mtdswap_map_free_block(d, page, bp);
  552. eb = d->eb_data + (*bp / d->pages_per_eblk);
  553. if (ret == -EIO || mtd_is_eccerr(ret)) {
  554. d->curr_write = NULL;
  555. eb->active_count--;
  556. d->revmap[*bp] = PAGE_UNDEF;
  557. goto retry;
  558. }
  559. if (ret < 0)
  560. return ret;
  561. writepos = (loff_t)*bp << PAGE_SHIFT;
  562. ret = mtd_write(mtd, writepos, PAGE_SIZE, &retlen, buf);
  563. if (ret == -EIO || mtd_is_eccerr(ret)) {
  564. d->curr_write_pos--;
  565. eb->active_count--;
  566. d->revmap[*bp] = PAGE_UNDEF;
  567. mtdswap_handle_write_error(d, eb);
  568. goto retry;
  569. }
  570. if (ret < 0) {
  571. dev_err(d->dev, "Write to MTD device failed: %d (%zd written)",
  572. ret, retlen);
  573. goto err;
  574. }
  575. if (retlen != PAGE_SIZE) {
  576. dev_err(d->dev, "Short write to MTD device: %zd written",
  577. retlen);
  578. ret = -EIO;
  579. goto err;
  580. }
  581. return ret;
  582. err:
  583. d->curr_write_pos--;
  584. eb->active_count--;
  585. d->revmap[*bp] = PAGE_UNDEF;
  586. return ret;
  587. }
  588. static int mtdswap_move_block(struct mtdswap_dev *d, unsigned int oldblock,
  589. unsigned int *newblock)
  590. {
  591. struct mtd_info *mtd = d->mtd;
  592. struct swap_eb *eb, *oldeb;
  593. int ret;
  594. size_t retlen;
  595. unsigned int page, retries;
  596. loff_t readpos;
  597. page = d->revmap[oldblock];
  598. readpos = (loff_t) oldblock << PAGE_SHIFT;
  599. retries = 0;
  600. retry:
  601. ret = mtd_read(mtd, readpos, PAGE_SIZE, &retlen, d->page_buf);
  602. if (ret < 0 && !mtd_is_bitflip(ret)) {
  603. oldeb = d->eb_data + oldblock / d->pages_per_eblk;
  604. oldeb->flags |= EBLOCK_READERR;
  605. dev_err(d->dev, "Read Error: %d (block %u)\n", ret,
  606. oldblock);
  607. retries++;
  608. if (retries < MTDSWAP_IO_RETRIES)
  609. goto retry;
  610. goto read_error;
  611. }
  612. if (retlen != PAGE_SIZE) {
  613. dev_err(d->dev, "Short read: %zd (block %u)\n", retlen,
  614. oldblock);
  615. ret = -EIO;
  616. goto read_error;
  617. }
  618. ret = mtdswap_write_block(d, d->page_buf, page, newblock, 1);
  619. if (ret < 0) {
  620. d->page_data[page] = BLOCK_ERROR;
  621. dev_err(d->dev, "Write error: %d\n", ret);
  622. return ret;
  623. }
  624. eb = d->eb_data + *newblock / d->pages_per_eblk;
  625. d->page_data[page] = *newblock;
  626. d->revmap[oldblock] = PAGE_UNDEF;
  627. eb = d->eb_data + oldblock / d->pages_per_eblk;
  628. eb->active_count--;
  629. return 0;
  630. read_error:
  631. d->page_data[page] = BLOCK_ERROR;
  632. d->revmap[oldblock] = PAGE_UNDEF;
  633. return ret;
  634. }
  635. static int mtdswap_gc_eblock(struct mtdswap_dev *d, struct swap_eb *eb)
  636. {
  637. unsigned int i, block, eblk_base, newblock;
  638. int ret, errcode;
  639. errcode = 0;
  640. eblk_base = (eb - d->eb_data) * d->pages_per_eblk;
  641. for (i = 0; i < d->pages_per_eblk; i++) {
  642. if (d->spare_eblks < MIN_SPARE_EBLOCKS)
  643. return -ENOSPC;
  644. block = eblk_base + i;
  645. if (d->revmap[block] == PAGE_UNDEF)
  646. continue;
  647. ret = mtdswap_move_block(d, block, &newblock);
  648. if (ret < 0 && !errcode)
  649. errcode = ret;
  650. }
  651. return errcode;
  652. }
  653. static int __mtdswap_choose_gc_tree(struct mtdswap_dev *d)
  654. {
  655. int idx, stopat;
  656. if (TREE_COUNT(d, CLEAN) < LOW_FRAG_GC_TRESHOLD)
  657. stopat = MTDSWAP_LOWFRAG;
  658. else
  659. stopat = MTDSWAP_HIFRAG;
  660. for (idx = MTDSWAP_BITFLIP; idx >= stopat; idx--)
  661. if (d->trees[idx].root.rb_node != NULL)
  662. return idx;
  663. return -1;
  664. }
  665. static int mtdswap_wlfreq(unsigned int maxdiff)
  666. {
  667. unsigned int h, x, y, dist, base;
  668. /*
  669. * Calculate linear ramp down from f1 to f2 when maxdiff goes from
  670. * MAX_ERASE_DIFF to MAX_ERASE_DIFF + COLLECT_NONDIRTY_BASE. Similar
  671. * to triangle with height f1 - f1 and width COLLECT_NONDIRTY_BASE.
  672. */
  673. dist = maxdiff - MAX_ERASE_DIFF;
  674. if (dist > COLLECT_NONDIRTY_BASE)
  675. dist = COLLECT_NONDIRTY_BASE;
  676. /*
  677. * Modelling the slop as right angular triangle with base
  678. * COLLECT_NONDIRTY_BASE and height freq1 - freq2. The ratio y/x is
  679. * equal to the ratio h/base.
  680. */
  681. h = COLLECT_NONDIRTY_FREQ1 - COLLECT_NONDIRTY_FREQ2;
  682. base = COLLECT_NONDIRTY_BASE;
  683. x = dist - base;
  684. y = (x * h + base / 2) / base;
  685. return COLLECT_NONDIRTY_FREQ2 + y;
  686. }
  687. static int mtdswap_choose_wl_tree(struct mtdswap_dev *d)
  688. {
  689. static unsigned int pick_cnt;
  690. unsigned int i, idx = -1, wear, max;
  691. struct rb_root *root;
  692. max = 0;
  693. for (i = 0; i <= MTDSWAP_DIRTY; i++) {
  694. root = &d->trees[i].root;
  695. if (root->rb_node == NULL)
  696. continue;
  697. wear = d->max_erase_count - MTDSWAP_ECNT_MIN(root);
  698. if (wear > max) {
  699. max = wear;
  700. idx = i;
  701. }
  702. }
  703. if (max > MAX_ERASE_DIFF && pick_cnt >= mtdswap_wlfreq(max) - 1) {
  704. pick_cnt = 0;
  705. return idx;
  706. }
  707. pick_cnt++;
  708. return -1;
  709. }
  710. static int mtdswap_choose_gc_tree(struct mtdswap_dev *d,
  711. unsigned int background)
  712. {
  713. int idx;
  714. if (TREE_NONEMPTY(d, FAILING) &&
  715. (background || (TREE_EMPTY(d, CLEAN) && TREE_EMPTY(d, DIRTY))))
  716. return MTDSWAP_FAILING;
  717. idx = mtdswap_choose_wl_tree(d);
  718. if (idx >= MTDSWAP_CLEAN)
  719. return idx;
  720. return __mtdswap_choose_gc_tree(d);
  721. }
  722. static struct swap_eb *mtdswap_pick_gc_eblk(struct mtdswap_dev *d,
  723. unsigned int background)
  724. {
  725. struct rb_root *rp = NULL;
  726. struct swap_eb *eb = NULL;
  727. int idx;
  728. if (background && TREE_COUNT(d, CLEAN) > CLEAN_BLOCK_THRESHOLD &&
  729. TREE_EMPTY(d, DIRTY) && TREE_EMPTY(d, FAILING))
  730. return NULL;
  731. idx = mtdswap_choose_gc_tree(d, background);
  732. if (idx < 0)
  733. return NULL;
  734. rp = &d->trees[idx].root;
  735. eb = rb_entry(rb_first(rp), struct swap_eb, rb);
  736. rb_erase(&eb->rb, rp);
  737. eb->root = NULL;
  738. d->trees[idx].count--;
  739. return eb;
  740. }
  741. static unsigned int mtdswap_test_patt(unsigned int i)
  742. {
  743. return i % 2 ? 0x55555555 : 0xAAAAAAAA;
  744. }
  745. static unsigned int mtdswap_eblk_passes(struct mtdswap_dev *d,
  746. struct swap_eb *eb)
  747. {
  748. struct mtd_info *mtd = d->mtd;
  749. unsigned int test, i, j, patt, mtd_pages;
  750. loff_t base, pos;
  751. unsigned int *p1 = (unsigned int *)d->page_buf;
  752. unsigned char *p2 = (unsigned char *)d->oob_buf;
  753. struct mtd_oob_ops ops;
  754. int ret;
  755. ops.mode = MTD_OPS_AUTO_OOB;
  756. ops.len = mtd->writesize;
  757. ops.ooblen = mtd->oobavail;
  758. ops.ooboffs = 0;
  759. ops.datbuf = d->page_buf;
  760. ops.oobbuf = d->oob_buf;
  761. base = mtdswap_eb_offset(d, eb);
  762. mtd_pages = d->pages_per_eblk * PAGE_SIZE / mtd->writesize;
  763. for (test = 0; test < 2; test++) {
  764. pos = base;
  765. for (i = 0; i < mtd_pages; i++) {
  766. patt = mtdswap_test_patt(test + i);
  767. memset(d->page_buf, patt, mtd->writesize);
  768. memset(d->oob_buf, patt, mtd->oobavail);
  769. ret = mtd_write_oob(mtd, pos, &ops);
  770. if (ret)
  771. goto error;
  772. pos += mtd->writesize;
  773. }
  774. pos = base;
  775. for (i = 0; i < mtd_pages; i++) {
  776. ret = mtd_read_oob(mtd, pos, &ops);
  777. if (ret)
  778. goto error;
  779. patt = mtdswap_test_patt(test + i);
  780. for (j = 0; j < mtd->writesize/sizeof(int); j++)
  781. if (p1[j] != patt)
  782. goto error;
  783. for (j = 0; j < mtd->oobavail; j++)
  784. if (p2[j] != (unsigned char)patt)
  785. goto error;
  786. pos += mtd->writesize;
  787. }
  788. ret = mtdswap_erase_block(d, eb);
  789. if (ret)
  790. goto error;
  791. }
  792. eb->flags &= ~EBLOCK_READERR;
  793. return 1;
  794. error:
  795. mtdswap_handle_badblock(d, eb);
  796. return 0;
  797. }
  798. static int mtdswap_gc(struct mtdswap_dev *d, unsigned int background)
  799. {
  800. struct swap_eb *eb;
  801. int ret;
  802. if (d->spare_eblks < MIN_SPARE_EBLOCKS)
  803. return 1;
  804. eb = mtdswap_pick_gc_eblk(d, background);
  805. if (!eb)
  806. return 1;
  807. ret = mtdswap_gc_eblock(d, eb);
  808. if (ret == -ENOSPC)
  809. return 1;
  810. if (eb->flags & EBLOCK_FAILED) {
  811. mtdswap_handle_badblock(d, eb);
  812. return 0;
  813. }
  814. eb->flags &= ~EBLOCK_BITFLIP;
  815. ret = mtdswap_erase_block(d, eb);
  816. if ((eb->flags & EBLOCK_READERR) &&
  817. (ret || !mtdswap_eblk_passes(d, eb)))
  818. return 0;
  819. if (ret == 0)
  820. ret = mtdswap_write_marker(d, eb, MTDSWAP_TYPE_CLEAN);
  821. if (ret == 0)
  822. mtdswap_rb_add(d, eb, MTDSWAP_CLEAN);
  823. else if (ret != -EIO && !mtd_is_eccerr(ret))
  824. mtdswap_rb_add(d, eb, MTDSWAP_DIRTY);
  825. return 0;
  826. }
  827. static void mtdswap_background(struct mtd_blktrans_dev *dev)
  828. {
  829. struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
  830. int ret;
  831. while (1) {
  832. ret = mtdswap_gc(d, 1);
  833. if (ret || mtd_blktrans_cease_background(dev))
  834. return;
  835. }
  836. }
  837. static void mtdswap_cleanup(struct mtdswap_dev *d)
  838. {
  839. vfree(d->eb_data);
  840. vfree(d->revmap);
  841. vfree(d->page_data);
  842. kfree(d->oob_buf);
  843. kfree(d->page_buf);
  844. }
  845. static int mtdswap_flush(struct mtd_blktrans_dev *dev)
  846. {
  847. struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
  848. mtd_sync(d->mtd);
  849. return 0;
  850. }
  851. static unsigned int mtdswap_badblocks(struct mtd_info *mtd, uint64_t size)
  852. {
  853. loff_t offset;
  854. unsigned int badcnt;
  855. badcnt = 0;
  856. if (mtd_can_have_bb(mtd))
  857. for (offset = 0; offset < size; offset += mtd->erasesize)
  858. if (mtd_block_isbad(mtd, offset))
  859. badcnt++;
  860. return badcnt;
  861. }
  862. static int mtdswap_writesect(struct mtd_blktrans_dev *dev,
  863. unsigned long page, char *buf)
  864. {
  865. struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
  866. unsigned int newblock, mapped;
  867. struct swap_eb *eb;
  868. int ret;
  869. d->sect_write_count++;
  870. if (d->spare_eblks < MIN_SPARE_EBLOCKS)
  871. return -ENOSPC;
  872. if (header) {
  873. /* Ignore writes to the header page */
  874. if (unlikely(page == 0))
  875. return 0;
  876. page--;
  877. }
  878. mapped = d->page_data[page];
  879. if (mapped <= BLOCK_MAX) {
  880. eb = d->eb_data + (mapped / d->pages_per_eblk);
  881. eb->active_count--;
  882. mtdswap_store_eb(d, eb);
  883. d->page_data[page] = BLOCK_UNDEF;
  884. d->revmap[mapped] = PAGE_UNDEF;
  885. }
  886. ret = mtdswap_write_block(d, buf, page, &newblock, 0);
  887. d->mtd_write_count++;
  888. if (ret < 0)
  889. return ret;
  890. eb = d->eb_data + (newblock / d->pages_per_eblk);
  891. d->page_data[page] = newblock;
  892. return 0;
  893. }
  894. /* Provide a dummy swap header for the kernel */
  895. static int mtdswap_auto_header(struct mtdswap_dev *d, char *buf)
  896. {
  897. union swap_header *hd = (union swap_header *)(buf);
  898. memset(buf, 0, PAGE_SIZE - 10);
  899. hd->info.version = 1;
  900. hd->info.last_page = d->mbd_dev->size - 1;
  901. hd->info.nr_badpages = 0;
  902. memcpy(buf + PAGE_SIZE - 10, "SWAPSPACE2", 10);
  903. return 0;
  904. }
  905. static int mtdswap_readsect(struct mtd_blktrans_dev *dev,
  906. unsigned long page, char *buf)
  907. {
  908. struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
  909. struct mtd_info *mtd = d->mtd;
  910. unsigned int realblock, retries;
  911. loff_t readpos;
  912. struct swap_eb *eb;
  913. size_t retlen;
  914. int ret;
  915. d->sect_read_count++;
  916. if (header) {
  917. if (unlikely(page == 0))
  918. return mtdswap_auto_header(d, buf);
  919. page--;
  920. }
  921. realblock = d->page_data[page];
  922. if (realblock > BLOCK_MAX) {
  923. memset(buf, 0x0, PAGE_SIZE);
  924. if (realblock == BLOCK_UNDEF)
  925. return 0;
  926. else
  927. return -EIO;
  928. }
  929. eb = d->eb_data + (realblock / d->pages_per_eblk);
  930. BUG_ON(d->revmap[realblock] == PAGE_UNDEF);
  931. readpos = (loff_t)realblock << PAGE_SHIFT;
  932. retries = 0;
  933. retry:
  934. ret = mtd_read(mtd, readpos, PAGE_SIZE, &retlen, buf);
  935. d->mtd_read_count++;
  936. if (mtd_is_bitflip(ret)) {
  937. eb->flags |= EBLOCK_BITFLIP;
  938. mtdswap_rb_add(d, eb, MTDSWAP_BITFLIP);
  939. ret = 0;
  940. }
  941. if (ret < 0) {
  942. dev_err(d->dev, "Read error %d\n", ret);
  943. eb->flags |= EBLOCK_READERR;
  944. mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
  945. retries++;
  946. if (retries < MTDSWAP_IO_RETRIES)
  947. goto retry;
  948. return ret;
  949. }
  950. if (retlen != PAGE_SIZE) {
  951. dev_err(d->dev, "Short read %zd\n", retlen);
  952. return -EIO;
  953. }
  954. return 0;
  955. }
  956. static int mtdswap_discard(struct mtd_blktrans_dev *dev, unsigned long first,
  957. unsigned nr_pages)
  958. {
  959. struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
  960. unsigned long page;
  961. struct swap_eb *eb;
  962. unsigned int mapped;
  963. d->discard_count++;
  964. for (page = first; page < first + nr_pages; page++) {
  965. mapped = d->page_data[page];
  966. if (mapped <= BLOCK_MAX) {
  967. eb = d->eb_data + (mapped / d->pages_per_eblk);
  968. eb->active_count--;
  969. mtdswap_store_eb(d, eb);
  970. d->page_data[page] = BLOCK_UNDEF;
  971. d->revmap[mapped] = PAGE_UNDEF;
  972. d->discard_page_count++;
  973. } else if (mapped == BLOCK_ERROR) {
  974. d->page_data[page] = BLOCK_UNDEF;
  975. d->discard_page_count++;
  976. }
  977. }
  978. return 0;
  979. }
  980. static int mtdswap_show(struct seq_file *s, void *data)
  981. {
  982. struct mtdswap_dev *d = (struct mtdswap_dev *) s->private;
  983. unsigned long sum;
  984. unsigned int count[MTDSWAP_TREE_CNT];
  985. unsigned int min[MTDSWAP_TREE_CNT];
  986. unsigned int max[MTDSWAP_TREE_CNT];
  987. unsigned int i, cw = 0, cwp = 0, cwecount = 0, bb_cnt, mapped, pages;
  988. uint64_t use_size;
  989. char *name[] = {"clean", "used", "low", "high", "dirty", "bitflip",
  990. "failing"};
  991. mutex_lock(&d->mbd_dev->lock);
  992. for (i = 0; i < MTDSWAP_TREE_CNT; i++) {
  993. struct rb_root *root = &d->trees[i].root;
  994. if (root->rb_node) {
  995. count[i] = d->trees[i].count;
  996. min[i] = MTDSWAP_ECNT_MIN(root);
  997. max[i] = MTDSWAP_ECNT_MAX(root);
  998. } else
  999. count[i] = 0;
  1000. }
  1001. if (d->curr_write) {
  1002. cw = 1;
  1003. cwp = d->curr_write_pos;
  1004. cwecount = d->curr_write->erase_count;
  1005. }
  1006. sum = 0;
  1007. for (i = 0; i < d->eblks; i++)
  1008. sum += d->eb_data[i].erase_count;
  1009. use_size = (uint64_t)d->eblks * d->mtd->erasesize;
  1010. bb_cnt = mtdswap_badblocks(d->mtd, use_size);
  1011. mapped = 0;
  1012. pages = d->mbd_dev->size;
  1013. for (i = 0; i < pages; i++)
  1014. if (d->page_data[i] != BLOCK_UNDEF)
  1015. mapped++;
  1016. mutex_unlock(&d->mbd_dev->lock);
  1017. for (i = 0; i < MTDSWAP_TREE_CNT; i++) {
  1018. if (!count[i])
  1019. continue;
  1020. if (min[i] != max[i])
  1021. seq_printf(s, "%s:\t%5d erase blocks, erased min %d, "
  1022. "max %d times\n",
  1023. name[i], count[i], min[i], max[i]);
  1024. else
  1025. seq_printf(s, "%s:\t%5d erase blocks, all erased %d "
  1026. "times\n", name[i], count[i], min[i]);
  1027. }
  1028. if (bb_cnt)
  1029. seq_printf(s, "bad:\t%5u erase blocks\n", bb_cnt);
  1030. if (cw)
  1031. seq_printf(s, "current erase block: %u pages used, %u free, "
  1032. "erased %u times\n",
  1033. cwp, d->pages_per_eblk - cwp, cwecount);
  1034. seq_printf(s, "total erasures: %lu\n", sum);
  1035. seq_puts(s, "\n");
  1036. seq_printf(s, "mtdswap_readsect count: %llu\n", d->sect_read_count);
  1037. seq_printf(s, "mtdswap_writesect count: %llu\n", d->sect_write_count);
  1038. seq_printf(s, "mtdswap_discard count: %llu\n", d->discard_count);
  1039. seq_printf(s, "mtd read count: %llu\n", d->mtd_read_count);
  1040. seq_printf(s, "mtd write count: %llu\n", d->mtd_write_count);
  1041. seq_printf(s, "discarded pages count: %llu\n", d->discard_page_count);
  1042. seq_puts(s, "\n");
  1043. seq_printf(s, "total pages: %u\n", pages);
  1044. seq_printf(s, "pages mapped: %u\n", mapped);
  1045. return 0;
  1046. }
  1047. static int mtdswap_open(struct inode *inode, struct file *file)
  1048. {
  1049. return single_open(file, mtdswap_show, inode->i_private);
  1050. }
  1051. static const struct file_operations mtdswap_fops = {
  1052. .open = mtdswap_open,
  1053. .read = seq_read,
  1054. .llseek = seq_lseek,
  1055. .release = single_release,
  1056. };
  1057. static int mtdswap_add_debugfs(struct mtdswap_dev *d)
  1058. {
  1059. struct dentry *root = d->mtd->dbg.dfs_dir;
  1060. struct dentry *dent;
  1061. if (!IS_ENABLED(CONFIG_DEBUG_FS))
  1062. return 0;
  1063. if (IS_ERR_OR_NULL(root))
  1064. return -1;
  1065. dent = debugfs_create_file("mtdswap_stats", S_IRUSR, root, d,
  1066. &mtdswap_fops);
  1067. if (!dent) {
  1068. dev_err(d->dev, "debugfs_create_file failed\n");
  1069. return -1;
  1070. }
  1071. return 0;
  1072. }
  1073. static int mtdswap_init(struct mtdswap_dev *d, unsigned int eblocks,
  1074. unsigned int spare_cnt)
  1075. {
  1076. struct mtd_info *mtd = d->mbd_dev->mtd;
  1077. unsigned int i, eblk_bytes, pages, blocks;
  1078. int ret = -ENOMEM;
  1079. d->mtd = mtd;
  1080. d->eblks = eblocks;
  1081. d->spare_eblks = spare_cnt;
  1082. d->pages_per_eblk = mtd->erasesize >> PAGE_SHIFT;
  1083. pages = d->mbd_dev->size;
  1084. blocks = eblocks * d->pages_per_eblk;
  1085. for (i = 0; i < MTDSWAP_TREE_CNT; i++)
  1086. d->trees[i].root = RB_ROOT;
  1087. d->page_data = vmalloc(sizeof(int)*pages);
  1088. if (!d->page_data)
  1089. goto page_data_fail;
  1090. d->revmap = vmalloc(sizeof(int)*blocks);
  1091. if (!d->revmap)
  1092. goto revmap_fail;
  1093. eblk_bytes = sizeof(struct swap_eb)*d->eblks;
  1094. d->eb_data = vzalloc(eblk_bytes);
  1095. if (!d->eb_data)
  1096. goto eb_data_fail;
  1097. for (i = 0; i < pages; i++)
  1098. d->page_data[i] = BLOCK_UNDEF;
  1099. for (i = 0; i < blocks; i++)
  1100. d->revmap[i] = PAGE_UNDEF;
  1101. d->page_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
  1102. if (!d->page_buf)
  1103. goto page_buf_fail;
  1104. d->oob_buf = kmalloc(2 * mtd->oobavail, GFP_KERNEL);
  1105. if (!d->oob_buf)
  1106. goto oob_buf_fail;
  1107. mtdswap_scan_eblks(d);
  1108. return 0;
  1109. oob_buf_fail:
  1110. kfree(d->page_buf);
  1111. page_buf_fail:
  1112. vfree(d->eb_data);
  1113. eb_data_fail:
  1114. vfree(d->revmap);
  1115. revmap_fail:
  1116. vfree(d->page_data);
  1117. page_data_fail:
  1118. printk(KERN_ERR "%s: init failed (%d)\n", MTDSWAP_PREFIX, ret);
  1119. return ret;
  1120. }
  1121. static void mtdswap_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
  1122. {
  1123. struct mtdswap_dev *d;
  1124. struct mtd_blktrans_dev *mbd_dev;
  1125. char *parts;
  1126. char *this_opt;
  1127. unsigned long part;
  1128. unsigned int eblocks, eavailable, bad_blocks, spare_cnt;
  1129. uint64_t swap_size, use_size, size_limit;
  1130. int ret;
  1131. parts = &partitions[0];
  1132. if (!*parts)
  1133. return;
  1134. while ((this_opt = strsep(&parts, ",")) != NULL) {
  1135. if (kstrtoul(this_opt, 0, &part) < 0)
  1136. return;
  1137. if (mtd->index == part)
  1138. break;
  1139. }
  1140. if (mtd->index != part)
  1141. return;
  1142. if (mtd->erasesize < PAGE_SIZE || mtd->erasesize % PAGE_SIZE) {
  1143. printk(KERN_ERR "%s: Erase size %u not multiple of PAGE_SIZE "
  1144. "%lu\n", MTDSWAP_PREFIX, mtd->erasesize, PAGE_SIZE);
  1145. return;
  1146. }
  1147. if (PAGE_SIZE % mtd->writesize || mtd->writesize > PAGE_SIZE) {
  1148. printk(KERN_ERR "%s: PAGE_SIZE %lu not multiple of write size"
  1149. " %u\n", MTDSWAP_PREFIX, PAGE_SIZE, mtd->writesize);
  1150. return;
  1151. }
  1152. if (!mtd->oobsize || mtd->oobavail < MTDSWAP_OOBSIZE) {
  1153. printk(KERN_ERR "%s: Not enough free bytes in OOB, "
  1154. "%d available, %zu needed.\n",
  1155. MTDSWAP_PREFIX, mtd->oobavail, MTDSWAP_OOBSIZE);
  1156. return;
  1157. }
  1158. if (spare_eblocks > 100)
  1159. spare_eblocks = 100;
  1160. use_size = mtd->size;
  1161. size_limit = (uint64_t) BLOCK_MAX * PAGE_SIZE;
  1162. if (mtd->size > size_limit) {
  1163. printk(KERN_WARNING "%s: Device too large. Limiting size to "
  1164. "%llu bytes\n", MTDSWAP_PREFIX, size_limit);
  1165. use_size = size_limit;
  1166. }
  1167. eblocks = mtd_div_by_eb(use_size, mtd);
  1168. use_size = (uint64_t)eblocks * mtd->erasesize;
  1169. bad_blocks = mtdswap_badblocks(mtd, use_size);
  1170. eavailable = eblocks - bad_blocks;
  1171. if (eavailable < MIN_ERASE_BLOCKS) {
  1172. printk(KERN_ERR "%s: Not enough erase blocks. %u available, "
  1173. "%d needed\n", MTDSWAP_PREFIX, eavailable,
  1174. MIN_ERASE_BLOCKS);
  1175. return;
  1176. }
  1177. spare_cnt = div_u64((uint64_t)eavailable * spare_eblocks, 100);
  1178. if (spare_cnt < MIN_SPARE_EBLOCKS)
  1179. spare_cnt = MIN_SPARE_EBLOCKS;
  1180. if (spare_cnt > eavailable - 1)
  1181. spare_cnt = eavailable - 1;
  1182. swap_size = (uint64_t)(eavailable - spare_cnt) * mtd->erasesize +
  1183. (header ? PAGE_SIZE : 0);
  1184. printk(KERN_INFO "%s: Enabling MTD swap on device %lu, size %llu KB, "
  1185. "%u spare, %u bad blocks\n",
  1186. MTDSWAP_PREFIX, part, swap_size / 1024, spare_cnt, bad_blocks);
  1187. d = kzalloc(sizeof(struct mtdswap_dev), GFP_KERNEL);
  1188. if (!d)
  1189. return;
  1190. mbd_dev = kzalloc(sizeof(struct mtd_blktrans_dev), GFP_KERNEL);
  1191. if (!mbd_dev) {
  1192. kfree(d);
  1193. return;
  1194. }
  1195. d->mbd_dev = mbd_dev;
  1196. mbd_dev->priv = d;
  1197. mbd_dev->mtd = mtd;
  1198. mbd_dev->devnum = mtd->index;
  1199. mbd_dev->size = swap_size >> PAGE_SHIFT;
  1200. mbd_dev->tr = tr;
  1201. if (!(mtd->flags & MTD_WRITEABLE))
  1202. mbd_dev->readonly = 1;
  1203. if (mtdswap_init(d, eblocks, spare_cnt) < 0)
  1204. goto init_failed;
  1205. if (add_mtd_blktrans_dev(mbd_dev) < 0)
  1206. goto cleanup;
  1207. d->dev = disk_to_dev(mbd_dev->disk);
  1208. ret = mtdswap_add_debugfs(d);
  1209. if (ret < 0)
  1210. goto debugfs_failed;
  1211. return;
  1212. debugfs_failed:
  1213. del_mtd_blktrans_dev(mbd_dev);
  1214. cleanup:
  1215. mtdswap_cleanup(d);
  1216. init_failed:
  1217. kfree(mbd_dev);
  1218. kfree(d);
  1219. }
  1220. static void mtdswap_remove_dev(struct mtd_blktrans_dev *dev)
  1221. {
  1222. struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
  1223. del_mtd_blktrans_dev(dev);
  1224. mtdswap_cleanup(d);
  1225. kfree(d);
  1226. }
  1227. static struct mtd_blktrans_ops mtdswap_ops = {
  1228. .name = "mtdswap",
  1229. .major = 0,
  1230. .part_bits = 0,
  1231. .blksize = PAGE_SIZE,
  1232. .flush = mtdswap_flush,
  1233. .readsect = mtdswap_readsect,
  1234. .writesect = mtdswap_writesect,
  1235. .discard = mtdswap_discard,
  1236. .background = mtdswap_background,
  1237. .add_mtd = mtdswap_add_mtd,
  1238. .remove_dev = mtdswap_remove_dev,
  1239. .owner = THIS_MODULE,
  1240. };
  1241. static int __init mtdswap_modinit(void)
  1242. {
  1243. return register_mtd_blktrans(&mtdswap_ops);
  1244. }
  1245. static void __exit mtdswap_modexit(void)
  1246. {
  1247. deregister_mtd_blktrans(&mtdswap_ops);
  1248. }
  1249. module_init(mtdswap_modinit);
  1250. module_exit(mtdswap_modexit);
  1251. MODULE_LICENSE("GPL");
  1252. MODULE_AUTHOR("Jarkko Lavinen <jarkko.lavinen@nokia.com>");
  1253. MODULE_DESCRIPTION("Block device access to an MTD suitable for using as "
  1254. "swap space");