sqlite3-vfs-opfs.c-pp.js 55 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458
  1. //#ifnot target=node
  2. /*
  3. 2022-09-18
  4. The author disclaims copyright to this source code. In place of a
  5. legal notice, here is a blessing:
  6. * May you do good and not evil.
  7. * May you find forgiveness for yourself and forgive others.
  8. * May you share freely, never taking more than you give.
  9. ***********************************************************************
  10. This file holds the synchronous half of an sqlite3_vfs
  11. implementation which proxies, in a synchronous fashion, the
  12. asynchronous Origin-Private FileSystem (OPFS) APIs using a second
  13. Worker, implemented in sqlite3-opfs-async-proxy.js. This file is
  14. intended to be appended to the main sqlite3 JS deliverable somewhere
  15. after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js.
  16. */
  17. 'use strict';
  18. globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
  19. /**
  20. installOpfsVfs() returns a Promise which, on success, installs an
  21. sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs
  22. which accept a VFS. It is intended to be called via
  23. sqlite3ApiBootstrap.initializers or an equivalent mechanism.
  24. The installed VFS uses the Origin-Private FileSystem API for
  25. all file storage. On error it is rejected with an exception
  26. explaining the problem. Reasons for rejection include, but are
  27. not limited to:
  28. - The counterpart Worker (see below) could not be loaded.
  29. - The environment does not support OPFS. That includes when
  30. this function is called from the main window thread.
  31. Significant notes and limitations:
  32. - The OPFS features used here are only available in dedicated Worker
  33. threads. This file tries to detect that case, resulting in a
  34. rejected Promise if those features do not seem to be available.
  35. - It requires the SharedArrayBuffer and Atomics classes, and the
  36. former is only available if the HTTP server emits the so-called
  37. COOP and COEP response headers. These features are required for
  38. proxying OPFS's synchronous API via the synchronous interface
  39. required by the sqlite3_vfs API.
  40. - This function may only be called a single time. When called, this
  41. function removes itself from the sqlite3 object.
  42. All arguments to this function are for internal/development purposes
  43. only. They do not constitute a public API and may change at any
  44. time.
  45. The argument may optionally be a plain object with the following
  46. configuration options:
  47. - proxyUri: name of the async proxy JS file.
  48. - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables
  49. logging of errors. 2 enables logging of warnings and errors. 3
  50. additionally enables debugging info. Logging is performed
  51. via the sqlite3.config.{log|warn|error}() functions.
  52. - sanityChecks (=false): if true, some basic sanity tests are run on
  53. the OPFS VFS API after it's initialized, before the returned
  54. Promise resolves. This is only intended for testing and
  55. development of the VFS, not client-side use.
  56. On success, the Promise resolves to the top-most sqlite3 namespace
  57. object and that object gets a new object installed in its
  58. `opfs` property, containing several OPFS-specific utilities.
  59. */
  60. const installOpfsVfs = function callee(options){
  61. if(!globalThis.SharedArrayBuffer
  62. || !globalThis.Atomics){
  63. return Promise.reject(
  64. new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+
  65. "The server must emit the COOP/COEP response headers to enable those. "+
  66. "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep")
  67. );
  68. }else if('undefined'===typeof WorkerGlobalScope){
  69. return Promise.reject(
  70. new Error("The OPFS sqlite3_vfs cannot run in the main thread "+
  71. "because it requires Atomics.wait().")
  72. );
  73. }else if(!globalThis.FileSystemHandle ||
  74. !globalThis.FileSystemDirectoryHandle ||
  75. !globalThis.FileSystemFileHandle ||
  76. !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle ||
  77. !navigator?.storage?.getDirectory){
  78. return Promise.reject(
  79. new Error("Missing required OPFS APIs.")
  80. );
  81. }
  82. if(!options || 'object'!==typeof options){
  83. options = Object.create(null);
  84. }
  85. const urlParams = new URL(globalThis.location.href).searchParams;
  86. if(urlParams.has('opfs-disable')){
  87. //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.');
  88. return Promise.resolve(sqlite3);
  89. }
  90. if(undefined===options.verbose){
  91. options.verbose = urlParams.has('opfs-verbose')
  92. ? (+urlParams.get('opfs-verbose') || 2) : 1;
  93. }
  94. if(undefined===options.sanityChecks){
  95. options.sanityChecks = urlParams.has('opfs-sanity-check');
  96. }
  97. if(undefined===options.proxyUri){
  98. options.proxyUri = callee.defaultProxyUri;
  99. }
  100. //sqlite3.config.warn("OPFS options =",options,globalThis.location);
  101. if('function' === typeof options.proxyUri){
  102. options.proxyUri = options.proxyUri();
  103. }
  104. const thePromise = new Promise(function(promiseResolve_, promiseReject_){
  105. const loggers = [
  106. sqlite3.config.error,
  107. sqlite3.config.warn,
  108. sqlite3.config.log
  109. ];
  110. const logImpl = (level,...args)=>{
  111. if(options.verbose>level) loggers[level]("OPFS syncer:",...args);
  112. };
  113. const log = (...args)=>logImpl(2, ...args);
  114. const warn = (...args)=>logImpl(1, ...args);
  115. const error = (...args)=>logImpl(0, ...args);
  116. const toss = sqlite3.util.toss;
  117. const capi = sqlite3.capi;
  118. const util = sqlite3.util;
  119. const wasm = sqlite3.wasm;
  120. const sqlite3_vfs = capi.sqlite3_vfs;
  121. const sqlite3_file = capi.sqlite3_file;
  122. const sqlite3_io_methods = capi.sqlite3_io_methods;
  123. /**
  124. Generic utilities for working with OPFS. This will get filled out
  125. by the Promise setup and, on success, installed as sqlite3.opfs.
  126. ACHTUNG: do not rely on these APIs in client code. They are
  127. experimental and subject to change or removal as the
  128. OPFS-specific sqlite3_vfs evolves.
  129. */
  130. const opfsUtil = Object.create(null);
  131. /**
  132. Returns true if _this_ thread has access to the OPFS APIs.
  133. */
  134. const thisThreadHasOPFS = ()=>{
  135. return globalThis.FileSystemHandle &&
  136. globalThis.FileSystemDirectoryHandle &&
  137. globalThis.FileSystemFileHandle &&
  138. globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle &&
  139. navigator?.storage?.getDirectory;
  140. };
  141. /**
  142. Not part of the public API. Solely for internal/development
  143. use.
  144. */
  145. opfsUtil.metrics = {
  146. dump: function(){
  147. let k, n = 0, t = 0, w = 0;
  148. for(k in state.opIds){
  149. const m = metrics[k];
  150. n += m.count;
  151. t += m.time;
  152. w += m.wait;
  153. m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
  154. m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0;
  155. }
  156. sqlite3.config.log(globalThis.location.href,
  157. "metrics for",globalThis.location.href,":",metrics,
  158. "\nTotal of",n,"op(s) for",t,
  159. "ms (incl. "+w+" ms of waiting on the async side)");
  160. sqlite3.config.log("Serialization metrics:",metrics.s11n);
  161. W.postMessage({type:'opfs-async-metrics'});
  162. },
  163. reset: function(){
  164. let k;
  165. const r = (m)=>(m.count = m.time = m.wait = 0);
  166. for(k in state.opIds){
  167. r(metrics[k] = Object.create(null));
  168. }
  169. let s = metrics.s11n = Object.create(null);
  170. s = s.serialize = Object.create(null);
  171. s.count = s.time = 0;
  172. s = metrics.s11n.deserialize = Object.create(null);
  173. s.count = s.time = 0;
  174. }
  175. }/*metrics*/;
  176. const opfsIoMethods = new sqlite3_io_methods();
  177. const opfsVfs = new sqlite3_vfs()
  178. .addOnDispose( ()=>opfsIoMethods.dispose());
  179. let promiseWasRejected = undefined;
  180. const promiseReject = (err)=>{
  181. promiseWasRejected = true;
  182. opfsVfs.dispose();
  183. return promiseReject_(err);
  184. };
  185. const promiseResolve = ()=>{
  186. promiseWasRejected = false;
  187. return promiseResolve_(sqlite3);
  188. };
  189. const W =
  190. //#if target=es6-bundler-friendly
  191. new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url));
  192. //#elif target=es6-module
  193. new Worker(new URL(options.proxyUri, import.meta.url));
  194. //#else
  195. new Worker(options.proxyUri);
  196. //#endif
  197. setTimeout(()=>{
  198. /* At attempt to work around a browser-specific quirk in which
  199. the Worker load is failing in such a way that we neither
  200. resolve nor reject it. This workaround gives that resolve/reject
  201. a time limit and rejects if that timer expires. Discussion:
  202. https://sqlite.org/forum/forumpost/a708c98dcb3ef */
  203. if(undefined===promiseWasRejected){
  204. promiseReject(
  205. new Error("Timeout while waiting for OPFS async proxy worker.")
  206. );
  207. }
  208. }, 4000);
  209. W._originalOnError = W.onerror /* will be restored later */;
  210. W.onerror = function(err){
  211. // The error object doesn't contain any useful info when the
  212. // failure is, e.g., that the remote script is 404.
  213. error("Error initializing OPFS asyncer:",err);
  214. promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
  215. };
  216. const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
  217. const dVfs = pDVfs
  218. ? new sqlite3_vfs(pDVfs)
  219. : null /* dVfs will be null when sqlite3 is built with
  220. SQLITE_OS_OTHER. */;
  221. opfsIoMethods.$iVersion = 1;
  222. opfsVfs.$iVersion = 2/*yes, two*/;
  223. opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;
  224. opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit
  225. is undocumented/unspecified. */;
  226. opfsVfs.$zName = wasm.allocCString("opfs");
  227. // All C-side memory of opfsVfs is zeroed out, but just to be explicit:
  228. opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null;
  229. opfsVfs.addOnDispose(
  230. '$zName', opfsVfs.$zName,
  231. 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null)
  232. );
  233. /**
  234. Pedantic sidebar about opfsVfs.ondispose: the entries in that array
  235. are items to clean up when opfsVfs.dispose() is called, but in this
  236. environment it will never be called. The VFS instance simply
  237. hangs around until the WASM module instance is cleaned up. We
  238. "could" _hypothetically_ clean it up by "importing" an
  239. sqlite3_os_end() impl into the wasm build, but the shutdown order
  240. of the wasm engine and the JS one are undefined so there is no
  241. guaranty that the opfsVfs instance would be available in one
  242. environment or the other when sqlite3_os_end() is called (_if_ it
  243. gets called at all in a wasm build, which is undefined).
  244. */
  245. /**
  246. State which we send to the async-api Worker or share with it.
  247. This object must initially contain only cloneable or sharable
  248. objects. After the worker's "inited" message arrives, other types
  249. of data may be added to it.
  250. For purposes of Atomics.wait() and Atomics.notify(), we use a
  251. SharedArrayBuffer with one slot reserved for each of the API
  252. proxy's methods. The sync side of the API uses Atomics.wait()
  253. on the corresponding slot and the async side uses
  254. Atomics.notify() on that slot.
  255. The approach of using a single SAB to serialize comms for all
  256. instances might(?) lead to deadlock situations in multi-db
  257. cases. We should probably have one SAB here with a single slot
  258. for locking a per-file initialization step and then allocate a
  259. separate SAB like the above one for each file. That will
  260. require a bit of acrobatics but should be feasible. The most
  261. problematic part is that xOpen() would have to use
  262. postMessage() to communicate its SharedArrayBuffer, and mixing
  263. that approach with Atomics.wait/notify() gets a bit messy.
  264. */
  265. const state = Object.create(null);
  266. state.verbose = options.verbose;
  267. state.littleEndian = (()=>{
  268. const buffer = new ArrayBuffer(2);
  269. new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */);
  270. // Int16Array uses the platform's endianness.
  271. return new Int16Array(buffer)[0] === 256;
  272. })();
  273. /**
  274. asyncIdleWaitTime is how long (ms) to wait, in the async proxy,
  275. for each Atomics.wait() when waiting on inbound VFS API calls.
  276. We need to wake up periodically to give the thread a chance to
  277. do other things. If this is too high (e.g. 500ms) then even two
  278. workers/tabs can easily run into locking errors. Some multiple
  279. of this value is also used for determining how long to wait on
  280. lock contention to free up.
  281. */
  282. state.asyncIdleWaitTime = 150;
  283. /**
  284. Whether the async counterpart should log exceptions to
  285. the serialization channel. That produces a great deal of
  286. noise for seemingly innocuous things like xAccess() checks
  287. for missing files, so this option may have one of 3 values:
  288. 0 = no exception logging.
  289. 1 = only log exceptions for "significant" ops like xOpen(),
  290. xRead(), and xWrite().
  291. 2 = log all exceptions.
  292. */
  293. state.asyncS11nExceptions = 1;
  294. /* Size of file I/O buffer block. 64k = max sqlite3 page size, and
  295. xRead/xWrite() will never deal in blocks larger than that. */
  296. state.fileBufferSize = 1024 * 64;
  297. state.sabS11nOffset = state.fileBufferSize;
  298. /**
  299. The size of the block in our SAB for serializing arguments and
  300. result values. Needs to be large enough to hold serialized
  301. values of any of the proxied APIs. Filenames are the largest
  302. part but are limited to opfsVfs.$mxPathname bytes. We also
  303. store exceptions there, so it needs to be long enough to hold
  304. a reasonably long exception string.
  305. */
  306. state.sabS11nSize = opfsVfs.$mxPathname * 2;
  307. /**
  308. The SAB used for all data I/O between the synchronous and
  309. async halves (file i/o and arg/result s11n).
  310. */
  311. state.sabIO = new SharedArrayBuffer(
  312. state.fileBufferSize/* file i/o block */
  313. + state.sabS11nSize/* argument/result serialization block */
  314. );
  315. state.opIds = Object.create(null);
  316. const metrics = Object.create(null);
  317. {
  318. /* Indexes for use in our SharedArrayBuffer... */
  319. let i = 0;
  320. /* SAB slot used to communicate which operation is desired
  321. between both workers. This worker writes to it and the other
  322. listens for changes. */
  323. state.opIds.whichOp = i++;
  324. /* Slot for storing return values. This worker listens to that
  325. slot and the other worker writes to it. */
  326. state.opIds.rc = i++;
  327. /* Each function gets an ID which this worker writes to
  328. the whichOp slot. The async-api worker uses Atomic.wait()
  329. on the whichOp slot to figure out which operation to run
  330. next. */
  331. state.opIds.xAccess = i++;
  332. state.opIds.xClose = i++;
  333. state.opIds.xDelete = i++;
  334. state.opIds.xDeleteNoWait = i++;
  335. state.opIds.xFileSize = i++;
  336. state.opIds.xLock = i++;
  337. state.opIds.xOpen = i++;
  338. state.opIds.xRead = i++;
  339. state.opIds.xSleep = i++;
  340. state.opIds.xSync = i++;
  341. state.opIds.xTruncate = i++;
  342. state.opIds.xUnlock = i++;
  343. state.opIds.xWrite = i++;
  344. state.opIds.mkdir = i++;
  345. state.opIds['opfs-async-metrics'] = i++;
  346. state.opIds['opfs-async-shutdown'] = i++;
  347. /* The retry slot is used by the async part for wait-and-retry
  348. semantics. Though we could hypothetically use the xSleep slot
  349. for that, doing so might lead to undesired side effects. */
  350. state.opIds.retry = i++;
  351. state.sabOP = new SharedArrayBuffer(
  352. i * 4/* ==sizeof int32, noting that Atomics.wait() and friends
  353. can only function on Int32Array views of an SAB. */);
  354. opfsUtil.metrics.reset();
  355. }
  356. /**
  357. SQLITE_xxx constants to export to the async worker
  358. counterpart...
  359. */
  360. state.sq3Codes = Object.create(null);
  361. [
  362. 'SQLITE_ACCESS_EXISTS',
  363. 'SQLITE_ACCESS_READWRITE',
  364. 'SQLITE_BUSY',
  365. 'SQLITE_CANTOPEN',
  366. 'SQLITE_ERROR',
  367. 'SQLITE_IOERR',
  368. 'SQLITE_IOERR_ACCESS',
  369. 'SQLITE_IOERR_CLOSE',
  370. 'SQLITE_IOERR_DELETE',
  371. 'SQLITE_IOERR_FSYNC',
  372. 'SQLITE_IOERR_LOCK',
  373. 'SQLITE_IOERR_READ',
  374. 'SQLITE_IOERR_SHORT_READ',
  375. 'SQLITE_IOERR_TRUNCATE',
  376. 'SQLITE_IOERR_UNLOCK',
  377. 'SQLITE_IOERR_WRITE',
  378. 'SQLITE_LOCK_EXCLUSIVE',
  379. 'SQLITE_LOCK_NONE',
  380. 'SQLITE_LOCK_PENDING',
  381. 'SQLITE_LOCK_RESERVED',
  382. 'SQLITE_LOCK_SHARED',
  383. 'SQLITE_LOCKED',
  384. 'SQLITE_MISUSE',
  385. 'SQLITE_NOTFOUND',
  386. 'SQLITE_OPEN_CREATE',
  387. 'SQLITE_OPEN_DELETEONCLOSE',
  388. 'SQLITE_OPEN_MAIN_DB',
  389. 'SQLITE_OPEN_READONLY'
  390. ].forEach((k)=>{
  391. if(undefined === (state.sq3Codes[k] = capi[k])){
  392. toss("Maintenance required: not found:",k);
  393. }
  394. });
  395. state.opfsFlags = Object.assign(Object.create(null),{
  396. /**
  397. Flag for use with xOpen(). URI flag "opfs-unlock-asap=1"
  398. enables this. See defaultUnlockAsap, below.
  399. */
  400. OPFS_UNLOCK_ASAP: 0x01,
  401. /**
  402. Flag for use with xOpen(). URI flag "delete-before-open=1"
  403. tells the VFS to delete the db file before attempting to open
  404. it. This can be used, e.g., to replace a db which has been
  405. corrupted (without forcing us to expose a delete/unlink()
  406. function in the public API).
  407. Failure to unlink the file is ignored but may lead to
  408. downstream errors. An unlink can fail if, e.g., another tab
  409. has the handle open.
  410. It goes without saying that deleting a file out from under another
  411. instance results in Undefined Behavior.
  412. */
  413. OPFS_UNLINK_BEFORE_OPEN: 0x02,
  414. /**
  415. If true, any async routine which implicitly acquires a sync
  416. access handle (i.e. an OPFS lock) will release that lock at
  417. the end of the call which acquires it. If false, such
  418. "autolocks" are not released until the VFS is idle for some
  419. brief amount of time.
  420. The benefit of enabling this is much higher concurrency. The
  421. down-side is much-reduced performance (as much as a 4x decrease
  422. in speedtest1).
  423. */
  424. defaultUnlockAsap: false
  425. });
  426. /**
  427. Runs the given operation (by name) in the async worker
  428. counterpart, waits for its response, and returns the result
  429. which the async worker writes to SAB[state.opIds.rc]. The
  430. 2nd and subsequent arguments must be the aruguments for the
  431. async op.
  432. */
  433. const opRun = (op,...args)=>{
  434. const opNdx = state.opIds[op] || toss("Invalid op ID:",op);
  435. state.s11n.serialize(...args);
  436. Atomics.store(state.sabOPView, state.opIds.rc, -1);
  437. Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx);
  438. Atomics.notify(state.sabOPView, state.opIds.whichOp)
  439. /* async thread will take over here */;
  440. const t = performance.now();
  441. while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){
  442. /*
  443. The reason for this loop is buried in the details of a long
  444. discussion at:
  445. https://github.com/sqlite/sqlite-wasm/issues/12
  446. Summary: in at least one browser flavor, under high loads,
  447. the wait()/notify() pairings can get out of sync. Calling
  448. wait() here until it returns 'not-equal' gets them back in
  449. sync.
  450. */
  451. }
  452. /* When the above wait() call returns 'not-equal', the async
  453. half will have completed the operation and reported its results
  454. in the state.opIds.rc slot of the SAB. */
  455. const rc = Atomics.load(state.sabOPView, state.opIds.rc);
  456. metrics[op].wait += performance.now() - t;
  457. if(rc && state.asyncS11nExceptions){
  458. const err = state.s11n.deserialize();
  459. if(err) error(op+"() async error:",...err);
  460. }
  461. return rc;
  462. };
  463. /**
  464. Not part of the public API. Only for test/development use.
  465. */
  466. opfsUtil.debug = {
  467. asyncShutdown: ()=>{
  468. warn("Shutting down OPFS async listener. The OPFS VFS will no longer work.");
  469. opRun('opfs-async-shutdown');
  470. },
  471. asyncRestart: ()=>{
  472. warn("Attempting to restart OPFS VFS async listener. Might work, might not.");
  473. W.postMessage({type: 'opfs-async-restart'});
  474. }
  475. };
  476. const initS11n = ()=>{
  477. /**
  478. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  479. ACHTUNG: this code is 100% duplicated in the other half of
  480. this proxy! The documentation is maintained in the
  481. "synchronous half".
  482. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  483. This proxy de/serializes cross-thread function arguments and
  484. output-pointer values via the state.sabIO SharedArrayBuffer,
  485. using the region defined by (state.sabS11nOffset,
  486. state.sabS11nOffset + state.sabS11nSize]. Only one dataset is
  487. recorded at a time.
  488. This is not a general-purpose format. It only supports the
  489. range of operations, and data sizes, needed by the
  490. sqlite3_vfs and sqlite3_io_methods operations. Serialized
  491. data are transient and this serialization algorithm may
  492. change at any time.
  493. The data format can be succinctly summarized as:
  494. Nt...Td...D
  495. Where:
  496. - N = number of entries (1 byte)
  497. - t = type ID of first argument (1 byte)
  498. - ...T = type IDs of the 2nd and subsequent arguments (1 byte
  499. each).
  500. - d = raw bytes of first argument (per-type size).
  501. - ...D = raw bytes of the 2nd and subsequent arguments (per-type
  502. size).
  503. All types except strings have fixed sizes. Strings are stored
  504. using their TextEncoder/TextDecoder representations. It would
  505. arguably make more sense to store them as Int16Arrays of
  506. their JS character values, but how best/fastest to get that
  507. in and out of string form is an open point. Initial
  508. experimentation with that approach did not gain us any speed.
  509. Historical note: this impl was initially about 1% this size by
  510. using using JSON.stringify/parse(), but using fit-to-purpose
  511. serialization saves considerable runtime.
  512. */
  513. if(state.s11n) return state.s11n;
  514. const textDecoder = new TextDecoder(),
  515. textEncoder = new TextEncoder('utf-8'),
  516. viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize),
  517. viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
  518. state.s11n = Object.create(null);
  519. /* Only arguments and return values of these types may be
  520. serialized. This covers the whole range of types needed by the
  521. sqlite3_vfs API. */
  522. const TypeIds = Object.create(null);
  523. TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' };
  524. TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' };
  525. TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' };
  526. TypeIds.string = { id: 4 };
  527. const getTypeId = (v)=>(
  528. TypeIds[typeof v]
  529. || toss("Maintenance required: this value type cannot be serialized.",v)
  530. );
  531. const getTypeIdById = (tid)=>{
  532. switch(tid){
  533. case TypeIds.number.id: return TypeIds.number;
  534. case TypeIds.bigint.id: return TypeIds.bigint;
  535. case TypeIds.boolean.id: return TypeIds.boolean;
  536. case TypeIds.string.id: return TypeIds.string;
  537. default: toss("Invalid type ID:",tid);
  538. }
  539. };
  540. /**
  541. Returns an array of the deserialized state stored by the most
  542. recent serialize() operation (from from this thread or the
  543. counterpart thread), or null if the serialization buffer is
  544. empty. If passed a truthy argument, the serialization buffer
  545. is cleared after deserialization.
  546. */
  547. state.s11n.deserialize = function(clear=false){
  548. ++metrics.s11n.deserialize.count;
  549. const t = performance.now();
  550. const argc = viewU8[0];
  551. const rc = argc ? [] : null;
  552. if(argc){
  553. const typeIds = [];
  554. let offset = 1, i, n, v;
  555. for(i = 0; i < argc; ++i, ++offset){
  556. typeIds.push(getTypeIdById(viewU8[offset]));
  557. }
  558. for(i = 0; i < argc; ++i){
  559. const t = typeIds[i];
  560. if(t.getter){
  561. v = viewDV[t.getter](offset, state.littleEndian);
  562. offset += t.size;
  563. }else{/*String*/
  564. n = viewDV.getInt32(offset, state.littleEndian);
  565. offset += 4;
  566. v = textDecoder.decode(viewU8.slice(offset, offset+n));
  567. offset += n;
  568. }
  569. rc.push(v);
  570. }
  571. }
  572. if(clear) viewU8[0] = 0;
  573. //log("deserialize:",argc, rc);
  574. metrics.s11n.deserialize.time += performance.now() - t;
  575. return rc;
  576. };
  577. /**
  578. Serializes all arguments to the shared buffer for consumption
  579. by the counterpart thread.
  580. This routine is only intended for serializing OPFS VFS
  581. arguments and (in at least one special case) result values,
  582. and the buffer is sized to be able to comfortably handle
  583. those.
  584. If passed no arguments then it zeroes out the serialization
  585. state.
  586. */
  587. state.s11n.serialize = function(...args){
  588. const t = performance.now();
  589. ++metrics.s11n.serialize.count;
  590. if(args.length){
  591. //log("serialize():",args);
  592. const typeIds = [];
  593. let i = 0, offset = 1;
  594. viewU8[0] = args.length & 0xff /* header = # of args */;
  595. for(; i < args.length; ++i, ++offset){
  596. /* Write the TypeIds.id value into the next args.length
  597. bytes. */
  598. typeIds.push(getTypeId(args[i]));
  599. viewU8[offset] = typeIds[i].id;
  600. }
  601. for(i = 0; i < args.length; ++i) {
  602. /* Deserialize the following bytes based on their
  603. corresponding TypeIds.id from the header. */
  604. const t = typeIds[i];
  605. if(t.setter){
  606. viewDV[t.setter](offset, args[i], state.littleEndian);
  607. offset += t.size;
  608. }else{/*String*/
  609. const s = textEncoder.encode(args[i]);
  610. viewDV.setInt32(offset, s.byteLength, state.littleEndian);
  611. offset += 4;
  612. viewU8.set(s, offset);
  613. offset += s.byteLength;
  614. }
  615. }
  616. //log("serialize() result:",viewU8.slice(0,offset));
  617. }else{
  618. viewU8[0] = 0;
  619. }
  620. metrics.s11n.serialize.time += performance.now() - t;
  621. };
  622. return state.s11n;
  623. }/*initS11n()*/;
  624. /**
  625. Generates a random ASCII string len characters long, intended for
  626. use as a temporary file name.
  627. */
  628. const randomFilename = function f(len=16){
  629. if(!f._chars){
  630. f._chars = "abcdefghijklmnopqrstuvwxyz"+
  631. "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
  632. "012346789";
  633. f._n = f._chars.length;
  634. }
  635. const a = [];
  636. let i = 0;
  637. for( ; i < len; ++i){
  638. const ndx = Math.random() * (f._n * 64) % f._n | 0;
  639. a[i] = f._chars[ndx];
  640. }
  641. return a.join("");
  642. /*
  643. An alternative impl. with an unpredictable length
  644. but much simpler:
  645. Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36)
  646. */
  647. };
  648. /**
  649. Map of sqlite3_file pointers to objects constructed by xOpen().
  650. */
  651. const __openFiles = Object.create(null);
  652. const opTimer = Object.create(null);
  653. opTimer.op = undefined;
  654. opTimer.start = undefined;
  655. const mTimeStart = (op)=>{
  656. opTimer.start = performance.now();
  657. opTimer.op = op;
  658. ++metrics[op].count;
  659. };
  660. const mTimeEnd = ()=>(
  661. metrics[opTimer.op].time += performance.now() - opTimer.start
  662. );
  663. /**
  664. Impls for the sqlite3_io_methods methods. Maintenance reminder:
  665. members are in alphabetical order to simplify finding them.
  666. */
  667. const ioSyncWrappers = {
  668. xCheckReservedLock: function(pFile,pOut){
  669. /**
  670. As of late 2022, only a single lock can be held on an OPFS
  671. file. We have no way of checking whether any _other_ db
  672. connection has a lock except by trying to obtain and (on
  673. success) release a sync-handle for it, but doing so would
  674. involve an inherent race condition. For the time being,
  675. pending a better solution, we simply report whether the
  676. given pFile is open.
  677. Update 2024-06-12: based on forum discussions, this
  678. function now always sets pOut to 0 (false):
  679. https://sqlite.org/forum/forumpost/a2f573b00cda1372
  680. */
  681. wasm.poke(pOut, 0, 'i32');
  682. return 0;
  683. },
  684. xClose: function(pFile){
  685. mTimeStart('xClose');
  686. let rc = 0;
  687. const f = __openFiles[pFile];
  688. if(f){
  689. delete __openFiles[pFile];
  690. rc = opRun('xClose', pFile);
  691. if(f.sq3File) f.sq3File.dispose();
  692. }
  693. mTimeEnd();
  694. return rc;
  695. },
  696. xDeviceCharacteristics: function(pFile){
  697. return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
  698. },
  699. xFileControl: function(pFile, opId, pArg){
  700. /*mTimeStart('xFileControl');
  701. mTimeEnd();*/
  702. return capi.SQLITE_NOTFOUND;
  703. },
  704. xFileSize: function(pFile,pSz64){
  705. mTimeStart('xFileSize');
  706. let rc = opRun('xFileSize', pFile);
  707. if(0==rc){
  708. try {
  709. const sz = state.s11n.deserialize()[0];
  710. wasm.poke(pSz64, sz, 'i64');
  711. }catch(e){
  712. error("Unexpected error reading xFileSize() result:",e);
  713. rc = state.sq3Codes.SQLITE_IOERR;
  714. }
  715. }
  716. mTimeEnd();
  717. return rc;
  718. },
  719. xLock: function(pFile,lockType){
  720. mTimeStart('xLock');
  721. const f = __openFiles[pFile];
  722. let rc = 0;
  723. /* All OPFS locks are exclusive locks. If xLock() has
  724. previously succeeded, do nothing except record the lock
  725. type. If no lock is active, have the async counterpart
  726. lock the file. */
  727. if( !f.lockType ) {
  728. rc = opRun('xLock', pFile, lockType);
  729. if( 0===rc ) f.lockType = lockType;
  730. }else{
  731. f.lockType = lockType;
  732. }
  733. mTimeEnd();
  734. return rc;
  735. },
  736. xRead: function(pFile,pDest,n,offset64){
  737. mTimeStart('xRead');
  738. const f = __openFiles[pFile];
  739. let rc;
  740. try {
  741. rc = opRun('xRead',pFile, n, Number(offset64));
  742. if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){
  743. /**
  744. Results get written to the SharedArrayBuffer f.sabView.
  745. Because the heap is _not_ a SharedArrayBuffer, we have
  746. to copy the results. TypedArray.set() seems to be the
  747. fastest way to copy this. */
  748. wasm.heap8u().set(f.sabView.subarray(0, n), pDest);
  749. }
  750. }catch(e){
  751. error("xRead(",arguments,") failed:",e,f);
  752. rc = capi.SQLITE_IOERR_READ;
  753. }
  754. mTimeEnd();
  755. return rc;
  756. },
  757. xSync: function(pFile,flags){
  758. mTimeStart('xSync');
  759. ++metrics.xSync.count;
  760. const rc = opRun('xSync', pFile, flags);
  761. mTimeEnd();
  762. return rc;
  763. },
  764. xTruncate: function(pFile,sz64){
  765. mTimeStart('xTruncate');
  766. const rc = opRun('xTruncate', pFile, Number(sz64));
  767. mTimeEnd();
  768. return rc;
  769. },
  770. xUnlock: function(pFile,lockType){
  771. mTimeStart('xUnlock');
  772. const f = __openFiles[pFile];
  773. let rc = 0;
  774. if( capi.SQLITE_LOCK_NONE === lockType
  775. && f.lockType ){
  776. rc = opRun('xUnlock', pFile, lockType);
  777. }
  778. if( 0===rc ) f.lockType = lockType;
  779. mTimeEnd();
  780. return rc;
  781. },
  782. xWrite: function(pFile,pSrc,n,offset64){
  783. mTimeStart('xWrite');
  784. const f = __openFiles[pFile];
  785. let rc;
  786. try {
  787. f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n));
  788. rc = opRun('xWrite', pFile, n, Number(offset64));
  789. }catch(e){
  790. error("xWrite(",arguments,") failed:",e,f);
  791. rc = capi.SQLITE_IOERR_WRITE;
  792. }
  793. mTimeEnd();
  794. return rc;
  795. }
  796. }/*ioSyncWrappers*/;
  797. /**
  798. Impls for the sqlite3_vfs methods. Maintenance reminder: members
  799. are in alphabetical order to simplify finding them.
  800. */
  801. const vfsSyncWrappers = {
  802. xAccess: function(pVfs,zName,flags,pOut){
  803. mTimeStart('xAccess');
  804. const rc = opRun('xAccess', wasm.cstrToJs(zName));
  805. wasm.poke( pOut, (rc ? 0 : 1), 'i32' );
  806. mTimeEnd();
  807. return 0;
  808. },
  809. xCurrentTime: function(pVfs,pOut){
  810. /* If it turns out that we need to adjust for timezone, see:
  811. https://stackoverflow.com/a/11760121/1458521 */
  812. wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000),
  813. 'double');
  814. return 0;
  815. },
  816. xCurrentTimeInt64: function(pVfs,pOut){
  817. wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(),
  818. 'i64');
  819. return 0;
  820. },
  821. xDelete: function(pVfs, zName, doSyncDir){
  822. mTimeStart('xDelete');
  823. const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false);
  824. mTimeEnd();
  825. return rc;
  826. },
  827. xFullPathname: function(pVfs,zName,nOut,pOut){
  828. /* Until/unless we have some notion of "current dir"
  829. in OPFS, simply copy zName to pOut... */
  830. const i = wasm.cstrncpy(pOut, zName, nOut);
  831. return i<nOut ? 0 : capi.SQLITE_CANTOPEN
  832. /*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/;
  833. },
  834. xGetLastError: function(pVfs,nOut,pOut){
  835. /* TODO: store exception.message values from the async
  836. partner in a dedicated SharedArrayBuffer, noting that we'd have
  837. to encode them... TextEncoder can do that for us. */
  838. warn("OPFS xGetLastError() has nothing sensible to return.");
  839. return 0;
  840. },
  841. //xSleep is optionally defined below
  842. xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
  843. mTimeStart('xOpen');
  844. let opfsFlags = 0;
  845. if(0===zName){
  846. zName = randomFilename();
  847. }else if(wasm.isPtr(zName)){
  848. if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){
  849. /* -----------------------^^^^^ MUST pass the untranslated
  850. C-string here. */
  851. opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP;
  852. }
  853. if(capi.sqlite3_uri_boolean(zName, "delete-before-open", 0)){
  854. opfsFlags |= state.opfsFlags.OPFS_UNLINK_BEFORE_OPEN;
  855. }
  856. zName = wasm.cstrToJs(zName);
  857. //warn("xOpen zName =",zName, "opfsFlags =",opfsFlags);
  858. }
  859. const fh = Object.create(null);
  860. fh.fid = pFile;
  861. fh.filename = zName;
  862. fh.sab = new SharedArrayBuffer(state.fileBufferSize);
  863. fh.flags = flags;
  864. fh.readOnly = !(sqlite3.SQLITE_OPEN_CREATE & flags)
  865. && !!(flags & capi.SQLITE_OPEN_READONLY);
  866. const rc = opRun('xOpen', pFile, zName, flags, opfsFlags);
  867. if(!rc){
  868. /* Recall that sqlite3_vfs::xClose() will be called, even on
  869. error, unless pFile->pMethods is NULL. */
  870. if(fh.readOnly){
  871. wasm.poke(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32');
  872. }
  873. __openFiles[pFile] = fh;
  874. fh.sabView = state.sabFileBufView;
  875. fh.sq3File = new sqlite3_file(pFile);
  876. fh.sq3File.$pMethods = opfsIoMethods.pointer;
  877. fh.lockType = capi.SQLITE_LOCK_NONE;
  878. }
  879. mTimeEnd();
  880. return rc;
  881. }/*xOpen()*/
  882. }/*vfsSyncWrappers*/;
  883. if(dVfs){
  884. opfsVfs.$xRandomness = dVfs.$xRandomness;
  885. opfsVfs.$xSleep = dVfs.$xSleep;
  886. }
  887. if(!opfsVfs.$xRandomness){
  888. /* If the default VFS has no xRandomness(), add a basic JS impl... */
  889. vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){
  890. const heap = wasm.heap8u();
  891. let i = 0;
  892. for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF;
  893. return i;
  894. };
  895. }
  896. if(!opfsVfs.$xSleep){
  897. /* If we can inherit an xSleep() impl from the default VFS then
  898. assume it's sane and use it, otherwise install a JS-based
  899. one. */
  900. vfsSyncWrappers.xSleep = function(pVfs,ms){
  901. Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms);
  902. return 0;
  903. };
  904. }
  905. /**
  906. Expects an OPFS file path. It gets resolved, such that ".."
  907. components are properly expanded, and returned. If the 2nd arg
  908. is true, the result is returned as an array of path elements,
  909. else an absolute path string is returned.
  910. */
  911. opfsUtil.getResolvedPath = function(filename,splitIt){
  912. const p = new URL(filename, "file://irrelevant").pathname;
  913. return splitIt ? p.split('/').filter((v)=>!!v) : p;
  914. };
  915. /**
  916. Takes the absolute path to a filesystem element. Returns an
  917. array of [handleOfContainingDir, filename]. If the 2nd argument
  918. is truthy then each directory element leading to the file is
  919. created along the way. Throws if any creation or resolution
  920. fails.
  921. */
  922. opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){
  923. const path = opfsUtil.getResolvedPath(absFilename, true);
  924. const filename = path.pop();
  925. let dh = opfsUtil.rootDirectory;
  926. for(const dirName of path){
  927. if(dirName){
  928. dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
  929. }
  930. }
  931. return [dh, filename];
  932. };
  933. /**
  934. Creates the given directory name, recursively, in
  935. the OPFS filesystem. Returns true if it succeeds or the
  936. directory already exists, else false.
  937. */
  938. opfsUtil.mkdir = async function(absDirName){
  939. try {
  940. await opfsUtil.getDirForFilename(absDirName+"/filepart", true);
  941. return true;
  942. }catch(e){
  943. //sqlite3.config.warn("mkdir(",absDirName,") failed:",e);
  944. return false;
  945. }
  946. };
  947. /**
  948. Checks whether the given OPFS filesystem entry exists,
  949. returning true if it does, false if it doesn't or if an
  950. exception is intercepted while trying to make the
  951. determination.
  952. */
  953. opfsUtil.entryExists = async function(fsEntryName){
  954. try {
  955. const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName);
  956. await dh.getFileHandle(fn);
  957. return true;
  958. }catch(e){
  959. return false;
  960. }
  961. };
  962. /**
  963. Generates a random ASCII string, intended for use as a
  964. temporary file name. Its argument is the length of the string,
  965. defaulting to 16.
  966. */
  967. opfsUtil.randomFilename = randomFilename;
  968. /**
  969. Returns a promise which resolves to an object which represents
  970. all files and directories in the OPFS tree. The top-most object
  971. has two properties: `dirs` is an array of directory entries
  972. (described below) and `files` is a list of file names for all
  973. files in that directory.
  974. Traversal starts at sqlite3.opfs.rootDirectory.
  975. Each `dirs` entry is an object in this form:
  976. ```
  977. { name: directoryName,
  978. dirs: [...subdirs],
  979. files: [...file names]
  980. }
  981. ```
  982. The `files` and `subdirs` entries are always set but may be
  983. empty arrays.
  984. The returned object has the same structure but its `name` is
  985. an empty string. All returned objects are created with
  986. Object.create(null), so have no prototype.
  987. Design note: the entries do not contain more information,
  988. e.g. file sizes, because getting such info is not only
  989. expensive but is subject to locking-related errors.
  990. */
  991. opfsUtil.treeList = async function(){
  992. const doDir = async function callee(dirHandle,tgt){
  993. tgt.name = dirHandle.name;
  994. tgt.dirs = [];
  995. tgt.files = [];
  996. for await (const handle of dirHandle.values()){
  997. if('directory' === handle.kind){
  998. const subDir = Object.create(null);
  999. tgt.dirs.push(subDir);
  1000. await callee(handle, subDir);
  1001. }else{
  1002. tgt.files.push(handle.name);
  1003. }
  1004. }
  1005. };
  1006. const root = Object.create(null);
  1007. await doDir(opfsUtil.rootDirectory, root);
  1008. return root;
  1009. };
  1010. /**
  1011. Irrevocably deletes _all_ files in the current origin's OPFS.
  1012. Obviously, this must be used with great caution. It may throw
  1013. an exception if removal of anything fails (e.g. a file is
  1014. locked), but the precise conditions under which the underlying
  1015. APIs will throw are not documented (so we cannot tell you what
  1016. they are).
  1017. */
  1018. opfsUtil.rmfr = async function(){
  1019. const dir = opfsUtil.rootDirectory, opt = {recurse: true};
  1020. for await (const handle of dir.values()){
  1021. dir.removeEntry(handle.name, opt);
  1022. }
  1023. };
  1024. /**
  1025. Deletes the given OPFS filesystem entry. As this environment
  1026. has no notion of "current directory", the given name must be an
  1027. absolute path. If the 2nd argument is truthy, deletion is
  1028. recursive (use with caution!).
  1029. The returned Promise resolves to true if the deletion was
  1030. successful, else false (but...). The OPFS API reports the
  1031. reason for the failure only in human-readable form, not
  1032. exceptions which can be type-checked to determine the
  1033. failure. Because of that...
  1034. If the final argument is truthy then this function will
  1035. propagate any exception on error, rather than returning false.
  1036. */
  1037. opfsUtil.unlink = async function(fsEntryName, recursive = false,
  1038. throwOnError = false){
  1039. try {
  1040. const [hDir, filenamePart] =
  1041. await opfsUtil.getDirForFilename(fsEntryName, false);
  1042. await hDir.removeEntry(filenamePart, {recursive});
  1043. return true;
  1044. }catch(e){
  1045. if(throwOnError){
  1046. throw new Error("unlink(",arguments[0],") failed: "+e.message,{
  1047. cause: e
  1048. });
  1049. }
  1050. return false;
  1051. }
  1052. };
  1053. /**
  1054. Traverses the OPFS filesystem, calling a callback for each
  1055. entry. The argument may be either a callback function or an
  1056. options object with any of the following properties:
  1057. - `callback`: function which gets called for each filesystem
  1058. entry. It gets passed 3 arguments: 1) the
  1059. FileSystemFileHandle or FileSystemDirectoryHandle of each
  1060. entry (noting that both are instanceof FileSystemHandle). 2)
  1061. the FileSystemDirectoryHandle of the parent directory. 3) the
  1062. current depth level, with 0 being at the top of the tree
  1063. relative to the starting directory. If the callback returns a
  1064. literal false, as opposed to any other falsy value, traversal
  1065. stops without an error. Any exceptions it throws are
  1066. propagated. Results are undefined if the callback manipulate
  1067. the filesystem (e.g. removing or adding entries) because the
  1068. how OPFS iterators behave in the face of such changes is
  1069. undocumented.
  1070. - `recursive` [bool=true]: specifies whether to recurse into
  1071. subdirectories or not. Whether recursion is depth-first or
  1072. breadth-first is unspecified!
  1073. - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory]
  1074. specifies the starting directory.
  1075. If this function is passed a function, it is assumed to be the
  1076. callback.
  1077. Returns a promise because it has to (by virtue of being async)
  1078. but that promise has no specific meaning: the traversal it
  1079. performs is synchronous. The promise must be used to catch any
  1080. exceptions propagated by the callback, however.
  1081. */
  1082. opfsUtil.traverse = async function(opt){
  1083. const defaultOpt = {
  1084. recursive: true,
  1085. directory: opfsUtil.rootDirectory
  1086. };
  1087. if('function'===typeof opt){
  1088. opt = {callback:opt};
  1089. }
  1090. opt = Object.assign(defaultOpt, opt||{});
  1091. const doDir = async function callee(dirHandle, depth){
  1092. for await (const handle of dirHandle.values()){
  1093. if(false === opt.callback(handle, dirHandle, depth)) return false;
  1094. else if(opt.recursive && 'directory' === handle.kind){
  1095. if(false === await callee(handle, depth + 1)) break;
  1096. }
  1097. }
  1098. };
  1099. doDir(opt.directory, 0);
  1100. };
  1101. /**
  1102. impl of importDb() when it's given a function as its second
  1103. argument.
  1104. */
  1105. const importDbChunked = async function(filename, callback){
  1106. const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
  1107. const hFile = await hDir.getFileHandle(fnamePart, {create:true});
  1108. let sah = await hFile.createSyncAccessHandle();
  1109. let nWrote = 0, chunk, checkedHeader = false, err = false;
  1110. try{
  1111. sah.truncate(0);
  1112. while( undefined !== (chunk = await callback()) ){
  1113. if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
  1114. if( 0===nWrote && chunk.byteLength>=15 ){
  1115. util.affirmDbHeader(chunk);
  1116. checkedHeader = true;
  1117. }
  1118. sah.write(chunk, {at: nWrote});
  1119. nWrote += chunk.byteLength;
  1120. }
  1121. if( nWrote < 512 || 0!==nWrote % 512 ){
  1122. toss("Input size",nWrote,"is not correct for an SQLite database.");
  1123. }
  1124. if( !checkedHeader ){
  1125. const header = new Uint8Array(20);
  1126. sah.read( header, {at: 0} );
  1127. util.affirmDbHeader( header );
  1128. }
  1129. sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/);
  1130. return nWrote;
  1131. }catch(e){
  1132. await sah.close();
  1133. sah = undefined;
  1134. await hDir.removeEntry( fnamePart ).catch(()=>{});
  1135. throw e;
  1136. }finally {
  1137. if( sah ) await sah.close();
  1138. }
  1139. };
  1140. /**
  1141. Asynchronously imports the given bytes (a byte array or
  1142. ArrayBuffer) into the given database file.
  1143. Results are undefined if the given db name refers to an opened
  1144. db.
  1145. If passed a function for its second argument, its behaviour
  1146. changes: imports its data in chunks fed to it by the given
  1147. callback function. It calls the callback (which may be async)
  1148. repeatedly, expecting either a Uint8Array or ArrayBuffer (to
  1149. denote new input) or undefined (to denote EOF). For so long as
  1150. the callback continues to return non-undefined, it will append
  1151. incoming data to the given VFS-hosted database file. When
  1152. called this way, the resolved value of the returned Promise is
  1153. the number of bytes written to the target file.
  1154. It very specifically requires the input to be an SQLite3
  1155. database and throws if that's not the case. It does so in
  1156. order to prevent this function from taking on a larger scope
  1157. than it is specifically intended to. i.e. we do not want it to
  1158. become a convenience for importing arbitrary files into OPFS.
  1159. This routine rewrites the database header bytes in the output
  1160. file (not the input array) to force disabling of WAL mode.
  1161. On error this throws and the state of the input file is
  1162. undefined (it depends on where the exception was triggered).
  1163. On success, resolves to the number of bytes written.
  1164. */
  1165. opfsUtil.importDb = async function(filename, bytes){
  1166. if( bytes instanceof Function ){
  1167. return importDbChunked(filename, bytes);
  1168. }
  1169. if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
  1170. util.affirmIsDb(bytes);
  1171. const n = bytes.byteLength;
  1172. const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
  1173. let sah, err, nWrote = 0;
  1174. try {
  1175. const hFile = await hDir.getFileHandle(fnamePart, {create:true});
  1176. sah = await hFile.createSyncAccessHandle();
  1177. sah.truncate(0);
  1178. nWrote = sah.write(bytes, {at: 0});
  1179. if(nWrote != n){
  1180. toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
  1181. }
  1182. sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */;
  1183. return nWrote;
  1184. }catch(e){
  1185. if( sah ){ await sah.close(); sah = undefined; }
  1186. await hDir.removeEntry( fnamePart ).catch(()=>{});
  1187. throw e;
  1188. }finally{
  1189. if( sah ) await sah.close();
  1190. }
  1191. };
  1192. if(sqlite3.oo1){
  1193. const OpfsDb = function(...args){
  1194. const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args);
  1195. opt.vfs = opfsVfs.$zName;
  1196. sqlite3.oo1.DB.dbCtorHelper.call(this, opt);
  1197. };
  1198. OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
  1199. sqlite3.oo1.OpfsDb = OpfsDb;
  1200. OpfsDb.importDb = opfsUtil.importDb;
  1201. sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback(
  1202. opfsVfs.pointer,
  1203. function(oo1Db, sqlite3){
  1204. /* Set a relatively high default busy-timeout handler to
  1205. help OPFS dbs deal with multi-tab/multi-worker
  1206. contention. */
  1207. sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000);
  1208. }
  1209. );
  1210. }/*extend sqlite3.oo1*/
  1211. const sanityCheck = function(){
  1212. const scope = wasm.scopedAllocPush();
  1213. const sq3File = new sqlite3_file();
  1214. try{
  1215. const fid = sq3File.pointer;
  1216. const openFlags = capi.SQLITE_OPEN_CREATE
  1217. | capi.SQLITE_OPEN_READWRITE
  1218. //| capi.SQLITE_OPEN_DELETEONCLOSE
  1219. | capi.SQLITE_OPEN_MAIN_DB;
  1220. const pOut = wasm.scopedAlloc(8);
  1221. const dbFile = "/sanity/check/file"+randomFilename(8);
  1222. const zDbFile = wasm.scopedAllocCString(dbFile);
  1223. let rc;
  1224. state.s11n.serialize("This is ä string.");
  1225. rc = state.s11n.deserialize();
  1226. log("deserialize() says:",rc);
  1227. if("This is ä string."!==rc[0]) toss("String d13n error.");
  1228. vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
  1229. rc = wasm.peek(pOut,'i32');
  1230. log("xAccess(",dbFile,") exists ?=",rc);
  1231. rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile,
  1232. fid, openFlags, pOut);
  1233. log("open rc =",rc,"state.sabOPView[xOpen] =",
  1234. state.sabOPView[state.opIds.xOpen]);
  1235. if(0!==rc){
  1236. error("open failed with code",rc);
  1237. return;
  1238. }
  1239. vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
  1240. rc = wasm.peek(pOut,'i32');
  1241. if(!rc) toss("xAccess() failed to detect file.");
  1242. rc = ioSyncWrappers.xSync(sq3File.pointer, 0);
  1243. if(rc) toss('sync failed w/ rc',rc);
  1244. rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024);
  1245. if(rc) toss('truncate failed w/ rc',rc);
  1246. wasm.poke(pOut,0,'i64');
  1247. rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut);
  1248. if(rc) toss('xFileSize failed w/ rc',rc);
  1249. log("xFileSize says:",wasm.peek(pOut, 'i64'));
  1250. rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1);
  1251. if(rc) toss("xWrite() failed!");
  1252. const readBuf = wasm.scopedAlloc(16);
  1253. rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2);
  1254. wasm.poke(readBuf+6,0);
  1255. let jRead = wasm.cstrToJs(readBuf);
  1256. log("xRead() got:",jRead);
  1257. if("sanity"!==jRead) toss("Unexpected xRead() value.");
  1258. if(vfsSyncWrappers.xSleep){
  1259. log("xSleep()ing before close()ing...");
  1260. vfsSyncWrappers.xSleep(opfsVfs.pointer,2000);
  1261. log("waking up from xSleep()");
  1262. }
  1263. rc = ioSyncWrappers.xClose(fid);
  1264. log("xClose rc =",rc,"sabOPView =",state.sabOPView);
  1265. log("Deleting file:",dbFile);
  1266. vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234);
  1267. vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
  1268. rc = wasm.peek(pOut,'i32');
  1269. if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete().");
  1270. warn("End of OPFS sanity checks.");
  1271. }finally{
  1272. sq3File.dispose();
  1273. wasm.scopedAllocPop(scope);
  1274. }
  1275. }/*sanityCheck()*/;
  1276. W.onmessage = function({data}){
  1277. //log("Worker.onmessage:",data);
  1278. switch(data.type){
  1279. case 'opfs-unavailable':
  1280. /* Async proxy has determined that OPFS is unavailable. There's
  1281. nothing more for us to do here. */
  1282. promiseReject(new Error(data.payload.join(' ')));
  1283. break;
  1284. case 'opfs-async-loaded':
  1285. /* Arrives as soon as the asyc proxy finishes loading.
  1286. Pass our config and shared state on to the async
  1287. worker. */
  1288. W.postMessage({type: 'opfs-async-init',args: state});
  1289. break;
  1290. case 'opfs-async-inited': {
  1291. /* Indicates that the async partner has received the 'init'
  1292. and has finished initializing, so the real work can
  1293. begin... */
  1294. if(true===promiseWasRejected){
  1295. break /* promise was already rejected via timer */;
  1296. }
  1297. try {
  1298. sqlite3.vfs.installVfs({
  1299. io: {struct: opfsIoMethods, methods: ioSyncWrappers},
  1300. vfs: {struct: opfsVfs, methods: vfsSyncWrappers}
  1301. });
  1302. state.sabOPView = new Int32Array(state.sabOP);
  1303. state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
  1304. state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
  1305. initS11n();
  1306. if(options.sanityChecks){
  1307. warn("Running sanity checks because of opfs-sanity-check URL arg...");
  1308. sanityCheck();
  1309. }
  1310. if(thisThreadHasOPFS()){
  1311. navigator.storage.getDirectory().then((d)=>{
  1312. W.onerror = W._originalOnError;
  1313. delete W._originalOnError;
  1314. sqlite3.opfs = opfsUtil;
  1315. opfsUtil.rootDirectory = d;
  1316. log("End of OPFS sqlite3_vfs setup.", opfsVfs);
  1317. promiseResolve();
  1318. }).catch(promiseReject);
  1319. }else{
  1320. promiseResolve();
  1321. }
  1322. }catch(e){
  1323. error(e);
  1324. promiseReject(e);
  1325. }
  1326. break;
  1327. }
  1328. default: {
  1329. const errMsg = (
  1330. "Unexpected message from the OPFS async worker: " +
  1331. JSON.stringify(data)
  1332. );
  1333. error(errMsg);
  1334. promiseReject(new Error(errMsg));
  1335. break;
  1336. }
  1337. }/*switch(data.type)*/
  1338. }/*W.onmessage()*/;
  1339. })/*thePromise*/;
  1340. return thePromise;
  1341. }/*installOpfsVfs()*/;
  1342. installOpfsVfs.defaultProxyUri =
  1343. "sqlite3-opfs-async-proxy.js";
  1344. globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{
  1345. try{
  1346. let proxyJs = installOpfsVfs.defaultProxyUri;
  1347. if(sqlite3.scriptInfo.sqlite3Dir){
  1348. installOpfsVfs.defaultProxyUri =
  1349. sqlite3.scriptInfo.sqlite3Dir + proxyJs;
  1350. //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri);
  1351. }
  1352. return installOpfsVfs().catch((e)=>{
  1353. sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message);
  1354. });
  1355. }catch(e){
  1356. sqlite3.config.error("installOpfsVfs() exception:",e);
  1357. return Promise.reject(e);
  1358. }
  1359. });
  1360. }/*sqlite3ApiBootstrap.initializers.push()*/);
  1361. //#else
  1362. /* The OPFS VFS parts are elided from builds targeting node.js. */
  1363. //#endif target=node