blob: b7e6887f65913ab3dfe4344f169ce11b5852387b [file] [log] [blame]
stephan3961b262022-08-10 11:26:08 +00001/*
stephanc5313af2022-09-18 02:35:30 +00002 2022-09-18
stephan3961b262022-08-10 11:26:08 +00003
4 The author disclaims copyright to this source code. In place of a
5 legal notice, here is a blessing:
6
7 * May you do good and not evil.
8 * May you find forgiveness for yourself and forgive others.
9 * May you share freely, never taking more than you give.
10
11 ***********************************************************************
12
stephanc5313af2022-09-18 02:35:30 +000013 This file holds the synchronous half of an sqlite3_vfs
14 implementation which proxies, in a synchronous fashion, the
15 asynchronous Origin-Private FileSystem (OPFS) APIs using a second
16 Worker, implemented in sqlite3-opfs-async-proxy.js. This file is
17 intended to be appended to the main sqlite3 JS deliverable somewhere
18 after sqlite3-api-glue.js and before sqlite3-api-cleanup.js.
19
20*/
21
22'use strict';
23self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
24/**
25 sqlite3.installOpfsVfs() returns a Promise which, on success, installs
26 an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs
27 which accept a VFS. It uses the Origin-Private FileSystem API for
28 all file storage. On error it is rejected with an exception
29 explaining the problem. Reasons for rejection include, but are
30 not limited to:
31
32 - The counterpart Worker (see below) could not be loaded.
33
34 - The environment does not support OPFS. That includes when
35 this function is called from the main window thread.
36
stephan3961b262022-08-10 11:26:08 +000037
38 Significant notes and limitations:
39
40 - As of this writing, OPFS is still very much in flux and only
41 available in bleeding-edge versions of Chrome (v102+, noting that
42 that number will increase as the OPFS API matures).
43
stephanc5313af2022-09-18 02:35:30 +000044 - The OPFS features used here are only available in dedicated Worker
stephanf3860122022-09-18 17:32:35 +000045 threads. This file tries to detect that case, resulting in a
46 rejected Promise if those features do not seem to be available.
stephanc5313af2022-09-18 02:35:30 +000047
48 - It requires the SharedArrayBuffer and Atomics classes, and the
49 former is only available if the HTTP server emits the so-called
50 COOP and COEP response headers. These features are required for
51 proxying OPFS's synchronous API via the synchronous interface
52 required by the sqlite3_vfs API.
53
54 - This function may only be called a single time and it must be
55 called from the client, as opposed to the library initialization,
56 in case the client requires a custom path for this API's
57 "counterpart": this function's argument is the relative URI to
58 this module's "asynchronous half". When called, this function removes
59 itself from the sqlite3 object.
60
61 The argument may optionally be a plain object with the following
62 configuration options:
63
64 - proxyUri: as described above
65
66 - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables
67 logging of errors. 2 enables logging of warnings and errors. 3
68 additionally enables debugging info.
69
70 - sanityChecks (=false): if true, some basic sanity tests are
71 run on the OPFS VFS API after it's initialized, before the
72 returned Promise resolves.
73
74 On success, the Promise resolves to the top-most sqlite3 namespace
stephanf3860122022-09-18 17:32:35 +000075 object and that object gets a new object installed in its
76 `opfs` property, containing several OPFS-specific utilities.
stephan3961b262022-08-10 11:26:08 +000077*/
stephanc5313af2022-09-18 02:35:30 +000078sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
stephan509f4052022-09-19 09:58:01 +000079 delete sqlite3.installOpfsVfs;
80 if(self.window===self ||
81 !self.SharedArrayBuffer ||
82 !self.FileSystemHandle ||
83 !self.FileSystemDirectoryHandle ||
84 !self.FileSystemFileHandle ||
85 !self.FileSystemFileHandle.prototype.createSyncAccessHandle ||
86 !navigator.storage.getDirectory){
87 return Promise.reject(
88 new Error("This environment does not have OPFS support.")
89 );
90 }
stephanc5313af2022-09-18 02:35:30 +000091 const options = (asyncProxyUri && 'object'===asyncProxyUri) ? asyncProxyUri : {
92 proxyUri: asyncProxyUri
stephan3961b262022-08-10 11:26:08 +000093 };
stephan509f4052022-09-19 09:58:01 +000094 const urlParams = new URL(self.location.href).searchParams;
stephanc5313af2022-09-18 02:35:30 +000095 if(undefined===options.verbose){
stephan509f4052022-09-19 09:58:01 +000096 options.verbose = urlParams.has('opfs-verbose') ? 3 : 2;
stephan3961b262022-08-10 11:26:08 +000097 }
stephanc5313af2022-09-18 02:35:30 +000098 if(undefined===options.sanityChecks){
stephan509f4052022-09-19 09:58:01 +000099 options.sanityChecks = urlParams.has('opfs-sanity-check');
stephanc5313af2022-09-18 02:35:30 +0000100 }
101 if(undefined===options.proxyUri){
102 options.proxyUri = callee.defaultProxyUri;
103 }
stephanf3860122022-09-18 17:32:35 +0000104
stephanc5313af2022-09-18 02:35:30 +0000105 const thePromise = new Promise(function(promiseResolve, promiseReject){
stephan509f4052022-09-19 09:58:01 +0000106 const loggers = {
107 0:console.error.bind(console),
108 1:console.warn.bind(console),
109 2:console.log.bind(console)
stephanc5313af2022-09-18 02:35:30 +0000110 };
stephan509f4052022-09-19 09:58:01 +0000111 const logImpl = (level,...args)=>{
112 if(options.verbose>level) loggers[level]("OPFS syncer:",...args);
113 };
114 const log = (...args)=>logImpl(2, ...args);
115 const warn = (...args)=>logImpl(1, ...args);
116 const error = (...args)=>logImpl(0, ...args);
stephanc5313af2022-09-18 02:35:30 +0000117 warn("The OPFS VFS feature is very much experimental and under construction.");
118 const toss = function(...args){throw new Error(args.join(' '))};
stephanc5313af2022-09-18 02:35:30 +0000119 const capi = sqlite3.capi;
120 const wasm = capi.wasm;
121 const sqlite3_vfs = capi.sqlite3_vfs;
122 const sqlite3_file = capi.sqlite3_file;
123 const sqlite3_io_methods = capi.sqlite3_io_methods;
stephanc5313af2022-09-18 02:35:30 +0000124 const W = new Worker(options.proxyUri);
stephanf3860122022-09-18 17:32:35 +0000125 W._originalOnError = W.onerror /* will be restored later */;
stephanc5313af2022-09-18 02:35:30 +0000126 W.onerror = function(err){
stephan509f4052022-09-19 09:58:01 +0000127 // The error object doesn't contain any useful info when the
128 // failure is, e.g., that the remote script is 404.
stephanc5313af2022-09-18 02:35:30 +0000129 promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
130 };
131 const wMsg = (type,payload)=>W.postMessage({type,payload});
stephan509f4052022-09-19 09:58:01 +0000132 /**
133 Generic utilities for working with OPFS. This will get filled out
134 by the Promise setup and, on success, installed as sqlite3.opfs.
135 */
136 const opfsUtil = Object.create(null);
stephanf8150112022-09-19 17:09:09 +0000137 /**
138 Not part of the public API. Solely for internal/development
139 use.
140 */
141 opfsUtil.metrics = {
142 dump: function(){
143 let k, n = 0, t = 0;
144 for(k in metrics){
145 const m = metrics[k];
146 n += m.count;
147 t += m.time;
148 m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
149 m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0;
150 }
151 console.log("metrics for",self.location.href,":",metrics,
152 "\nTotal of",n,"op(s) for",t,"ms");
153 },
154 reset: function(){
155 let k;
156 const r = (m)=>(m.count = m.time = m.wait = 0);
157 for(k in state.opIds){
158 r(metrics[k] = Object.create(null));
159 }
160 [ // timed routines which are not in state.opIds
161 'xFileControl'
162 ].forEach((k)=>r(metrics[k] = Object.create(null)));
163 }
164 }/*metrics*/;
stephanc5313af2022-09-18 02:35:30 +0000165
166 /**
167 State which we send to the async-api Worker or share with it.
168 This object must initially contain only cloneable or sharable
169 objects. After the worker's "inited" message arrives, other types
170 of data may be added to it.
stephanf3860122022-09-18 17:32:35 +0000171
172 For purposes of Atomics.wait() and Atomics.notify(), we use a
173 SharedArrayBuffer with one slot reserved for each of the API
174 proxy's methods. The sync side of the API uses Atomics.wait()
175 on the corresponding slot and the async side uses
176 Atomics.notify() on that slot.
177
178 The approach of using a single SAB to serialize comms for all
179 instances might(?) lead to deadlock situations in multi-db
180 cases. We should probably have one SAB here with a single slot
181 for locking a per-file initialization step and then allocate a
182 separate SAB like the above one for each file. That will
183 require a bit of acrobatics but should be feasible.
stephanc5313af2022-09-18 02:35:30 +0000184 */
185 const state = Object.create(null);
186 state.verbose = options.verbose;
stephanf3860122022-09-18 17:32:35 +0000187 state.fileBufferSize =
188 1024 * 64 + 8 /* size of aFileHandle.sab. 64k = max sqlite3 page
189 size. The additional bytes are space for
190 holding BigInt results, since we cannot store
191 those via the Atomics API (which only works on
192 an Int32Array). */;
193 state.fbInt64Offset =
194 state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */;
stephanc5313af2022-09-18 02:35:30 +0000195 state.opIds = Object.create(null);
stephanf8150112022-09-19 17:09:09 +0000196 const metrics = Object.create(null);
stephanc5313af2022-09-18 02:35:30 +0000197 {
stephan3961b262022-08-10 11:26:08 +0000198 let i = 0;
stephanc5313af2022-09-18 02:35:30 +0000199 state.opIds.xAccess = i++;
200 state.opIds.xClose = i++;
201 state.opIds.xDelete = i++;
stephanf3860122022-09-18 17:32:35 +0000202 state.opIds.xDeleteNoWait = i++;
stephanc5313af2022-09-18 02:35:30 +0000203 state.opIds.xFileSize = i++;
204 state.opIds.xOpen = i++;
205 state.opIds.xRead = i++;
206 state.opIds.xSleep = i++;
207 state.opIds.xSync = i++;
208 state.opIds.xTruncate = i++;
209 state.opIds.xWrite = i++;
stephanf3860122022-09-18 17:32:35 +0000210 state.opIds.mkdir = i++;
stephanc5313af2022-09-18 02:35:30 +0000211 state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/);
stephanf8150112022-09-19 17:09:09 +0000212 opfsUtil.metrics.reset();
stephanc5313af2022-09-18 02:35:30 +0000213 }
214
215 state.sq3Codes = Object.create(null);
216 state.sq3Codes._reverse = Object.create(null);
217 [ // SQLITE_xxx constants to export to the async worker counterpart...
218 'SQLITE_ERROR', 'SQLITE_IOERR',
219 'SQLITE_NOTFOUND', 'SQLITE_MISUSE',
220 'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ',
221 'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
222 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
stephanf3860122022-09-18 17:32:35 +0000223 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
224 'SQLITE_IOERR_DELETE'
stephanc5313af2022-09-18 02:35:30 +0000225 ].forEach(function(k){
226 state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
227 state.sq3Codes._reverse[capi[k]] = k;
stephan3961b262022-08-10 11:26:08 +0000228 });
stephan3961b262022-08-10 11:26:08 +0000229
stephanc5313af2022-09-18 02:35:30 +0000230 const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n];
stephan3961b262022-08-10 11:26:08 +0000231
stephanc5313af2022-09-18 02:35:30 +0000232 /**
233 Runs the given operation in the async worker counterpart, waits
234 for its response, and returns the result which the async worker
235 writes to the given op's index in state.opSABView. The 2nd argument
236 must be a single object or primitive value, depending on the
237 given operation's signature in the async API counterpart.
238 */
239 const opRun = (op,args)=>{
stephanf8150112022-09-19 17:09:09 +0000240 const t = performance.now();
stephanf3860122022-09-18 17:32:35 +0000241 Atomics.store(state.opSABView, state.opIds[op], -1);
stephanc5313af2022-09-18 02:35:30 +0000242 wMsg(op, args);
stephanf3860122022-09-18 17:32:35 +0000243 Atomics.wait(state.opSABView, state.opIds[op], -1);
stephanf8150112022-09-19 17:09:09 +0000244 metrics[op].wait += performance.now() - t;
stephanc5313af2022-09-18 02:35:30 +0000245 return Atomics.load(state.opSABView, state.opIds[op]);
246 };
247
248 /**
249 Generates a random ASCII string len characters long, intended for
250 use as a temporary file name.
251 */
252 const randomFilename = function f(len=16){
253 if(!f._chars){
254 f._chars = "abcdefghijklmnopqrstuvwxyz"+
255 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
256 "012346789";
257 f._n = f._chars.length;
258 }
259 const a = [];
260 let i = 0;
261 for( ; i < len; ++i){
262 const ndx = Math.random() * (f._n * 64) % f._n | 0;
263 a[i] = f._chars[ndx];
264 }
265 return a.join('');
266 };
267
268 /**
269 Map of sqlite3_file pointers to objects constructed by xOpen().
270 */
271 const __openFiles = Object.create(null);
272
273 const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
274 const dVfs = pDVfs
275 ? new sqlite3_vfs(pDVfs)
276 : null /* dVfs will be null when sqlite3 is built with
277 SQLITE_OS_OTHER. Though we cannot currently handle
278 that case, the hope is to eventually be able to. */;
279 const opfsVfs = new sqlite3_vfs();
280 const opfsIoMethods = new sqlite3_io_methods();
281 opfsVfs.$iVersion = 2/*yes, two*/;
282 opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;
283 opfsVfs.$mxPathname = 1024/*sure, why not?*/;
284 opfsVfs.$zName = wasm.allocCString("opfs");
285 // All C-side memory of opfsVfs is zeroed out, but just to be explicit:
286 opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null;
287 opfsVfs.ondispose = [
288 '$zName', opfsVfs.$zName,
289 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null),
290 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose()
291 ];
stephanc5313af2022-09-18 02:35:30 +0000292 /**
293 Pedantic sidebar about opfsVfs.ondispose: the entries in that array
294 are items to clean up when opfsVfs.dispose() is called, but in this
295 environment it will never be called. The VFS instance simply
296 hangs around until the WASM module instance is cleaned up. We
297 "could" _hypothetically_ clean it up by "importing" an
298 sqlite3_os_end() impl into the wasm build, but the shutdown order
299 of the wasm engine and the JS one are undefined so there is no
300 guaranty that the opfsVfs instance would be available in one
301 environment or the other when sqlite3_os_end() is called (_if_ it
302 gets called at all in a wasm build, which is undefined).
303 */
304
305 /**
306 Installs a StructBinder-bound function pointer member of the
307 given name and function in the given StructType target object.
308 It creates a WASM proxy for the given function and arranges for
309 that proxy to be cleaned up when tgt.dispose() is called. Throws
310 on the slightest hint of error (e.g. tgt is-not-a StructType,
311 name does not map to a struct-bound member, etc.).
312
313 Returns a proxy for this function which is bound to tgt and takes
314 2 args (name,func). That function returns the same thing,
315 permitting calls to be chained.
316
317 If called with only 1 arg, it has no side effects but returns a
318 func with the same signature as described above.
319 */
320 const installMethod = function callee(tgt, name, func){
stephanf3860122022-09-18 17:32:35 +0000321 if(!(tgt instanceof sqlite3.StructBinder.StructType)){
stephanc5313af2022-09-18 02:35:30 +0000322 toss("Usage error: target object is-not-a StructType.");
323 }
324 if(1===arguments.length){
325 return (n,f)=>callee(tgt,n,f);
326 }
327 if(!callee.argcProxy){
328 callee.argcProxy = function(func,sig){
329 return function(...args){
330 if(func.length!==arguments.length){
331 toss("Argument mismatch. Native signature is:",sig);
332 }
333 return func.apply(this, args);
334 }
335 };
336 callee.removeFuncList = function(){
337 if(this.ondispose.__removeFuncList){
338 this.ondispose.__removeFuncList.forEach(
339 (v,ndx)=>{
340 if('number'===typeof v){
341 try{wasm.uninstallFunction(v)}
342 catch(e){/*ignore*/}
343 }
344 /* else it's a descriptive label for the next number in
345 the list. */
346 }
347 );
348 delete this.ondispose.__removeFuncList;
349 }
350 };
351 }/*static init*/
352 const sigN = tgt.memberSignature(name);
353 if(sigN.length<2){
354 toss("Member",name," is not a function pointer. Signature =",sigN);
355 }
356 const memKey = tgt.memberKey(name);
357 //log("installMethod",tgt, name, sigN);
358 const fProxy = 1
359 // We can remove this proxy middle-man once the VFS is working
360 ? callee.argcProxy(func, sigN)
361 : func;
362 const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true));
363 tgt[memKey] = pFunc;
364 if(!tgt.ondispose) tgt.ondispose = [];
365 if(!tgt.ondispose.__removeFuncList){
366 tgt.ondispose.push('ondispose.__removeFuncList handler',
367 callee.removeFuncList);
368 tgt.ondispose.__removeFuncList = [];
369 }
370 tgt.ondispose.__removeFuncList.push(memKey, pFunc);
371 return (n,f)=>callee(tgt, n, f);
372 }/*installMethod*/;
stephanf8150112022-09-19 17:09:09 +0000373
374 const opTimer = Object.create(null);
375 opTimer.op = undefined;
376 opTimer.start = undefined;
377 const mTimeStart = (op)=>{
378 opTimer.start = performance.now();
379 opTimer.op = op;
380 //metrics[op] || toss("Maintenance required: missing metrics for",op);
381 ++metrics[op].count;
382 };
383 const mTimeEnd = ()=>(
384 metrics[opTimer.op].time += performance.now() - opTimer.start
385 );
386
stephanc5313af2022-09-18 02:35:30 +0000387 /**
388 Impls for the sqlite3_io_methods methods. Maintenance reminder:
389 members are in alphabetical order to simplify finding them.
390 */
391 const ioSyncWrappers = {
392 xCheckReservedLock: function(pFile,pOut){
393 // Exclusive lock is automatically acquired when opened
394 //warn("xCheckReservedLock(",arguments,") is a no-op");
395 wasm.setMemValue(pOut,1,'i32');
396 return 0;
397 },
398 xClose: function(pFile){
stephanf8150112022-09-19 17:09:09 +0000399 mTimeStart('xClose');
stephanc5313af2022-09-18 02:35:30 +0000400 let rc = 0;
401 const f = __openFiles[pFile];
402 if(f){
403 delete __openFiles[pFile];
404 rc = opRun('xClose', pFile);
405 if(f.sq3File) f.sq3File.dispose();
406 }
stephanf8150112022-09-19 17:09:09 +0000407 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000408 return rc;
409 },
410 xDeviceCharacteristics: function(pFile){
411 //debug("xDeviceCharacteristics(",pFile,")");
412 return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
413 },
stephanf8150112022-09-19 17:09:09 +0000414 xFileControl: function(pFile, opId, pArg){
415 mTimeStart('xFileControl');
416 if(capi.SQLITE_FCNTL_SYNC===opId){
417 return opRun('xSync', {fid:pFile, flags:0});
418 }
419 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000420 return capi.SQLITE_NOTFOUND;
421 },
422 xFileSize: function(pFile,pSz64){
stephanf8150112022-09-19 17:09:09 +0000423 mTimeStart('xFileSize');
stephanc5313af2022-09-18 02:35:30 +0000424 const rc = opRun('xFileSize', pFile);
425 if(!isWorkerErrCode(rc)){
426 const f = __openFiles[pFile];
stephanf8150112022-09-19 17:09:09 +0000427 wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0,true) ,'i64');
stephanc5313af2022-09-18 02:35:30 +0000428 }
stephanf8150112022-09-19 17:09:09 +0000429 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000430 return rc;
431 },
432 xLock: function(pFile,lockType){
433 //2022-09: OPFS handles lock when opened
434 //warn("xLock(",arguments,") is a no-op");
435 return 0;
436 },
437 xRead: function(pFile,pDest,n,offset){
438 /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
stephanf8150112022-09-19 17:09:09 +0000439 mTimeStart('xRead');
stephanc5313af2022-09-18 02:35:30 +0000440 const f = __openFiles[pFile];
441 let rc;
442 try {
443 // FIXME(?): block until we finish copying the xRead result buffer. How?
444 rc = opRun('xRead',{fid:pFile, n, offset});
stephan862281f2022-09-19 09:25:25 +0000445 if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){
stephanf8150112022-09-19 17:09:09 +0000446 // set() seems to be the fastest way to copy this...
447 wasm.heap8u().set(f.sabView.subarray(0, n), pDest);
stephan862281f2022-09-19 09:25:25 +0000448 }
stephanc5313af2022-09-18 02:35:30 +0000449 }catch(e){
450 error("xRead(",arguments,") failed:",e,f);
451 rc = capi.SQLITE_IOERR_READ;
452 }
stephanf8150112022-09-19 17:09:09 +0000453 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000454 return rc;
455 },
456 xSync: function(pFile,flags){
stephanf8150112022-09-19 17:09:09 +0000457 return 0; // impl'd in xFileControl(). opRun('xSync', {fid:pFile, flags});
stephanc5313af2022-09-18 02:35:30 +0000458 },
459 xTruncate: function(pFile,sz64){
stephanf8150112022-09-19 17:09:09 +0000460 mTimeStart('xTruncate');
461 const rc = opRun('xTruncate', {fid:pFile, size: sz64});
462 mTimeEnd();
463 return rc;
stephanc5313af2022-09-18 02:35:30 +0000464 },
465 xUnlock: function(pFile,lockType){
466 //2022-09: OPFS handles lock when opened
467 //warn("xUnlock(",arguments,") is a no-op");
468 return 0;
469 },
470 xWrite: function(pFile,pSrc,n,offset){
471 /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
stephanf8150112022-09-19 17:09:09 +0000472 mTimeStart('xWrite');
stephanc5313af2022-09-18 02:35:30 +0000473 const f = __openFiles[pFile];
stephanf8150112022-09-19 17:09:09 +0000474 let rc;
stephanc5313af2022-09-18 02:35:30 +0000475 try {
stephanc5313af2022-09-18 02:35:30 +0000476 // FIXME(?): block from here until we finish the xWrite. How?
stephanf8150112022-09-19 17:09:09 +0000477 f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n));
478 rc = opRun('xWrite',{fid:pFile, n, offset});
stephanc5313af2022-09-18 02:35:30 +0000479 }catch(e){
480 error("xWrite(",arguments,") failed:",e,f);
stephanf8150112022-09-19 17:09:09 +0000481 rc = capi.SQLITE_IOERR_WRITE;
stephanc5313af2022-09-18 02:35:30 +0000482 }
stephanf8150112022-09-19 17:09:09 +0000483 mTimeEnd();
484 return rc;
stephanc5313af2022-09-18 02:35:30 +0000485 }
486 }/*ioSyncWrappers*/;
487
488 /**
489 Impls for the sqlite3_vfs methods. Maintenance reminder: members
490 are in alphabetical order to simplify finding them.
491 */
492 const vfsSyncWrappers = {
493 xAccess: function(pVfs,zName,flags,pOut){
stephanf8150112022-09-19 17:09:09 +0000494 mTimeStart('xAccess');
stephanc5313af2022-09-18 02:35:30 +0000495 const rc = opRun('xAccess', wasm.cstringToJs(zName));
496 wasm.setMemValue(pOut, rc ? 0 : 1, 'i32');
stephanf8150112022-09-19 17:09:09 +0000497 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000498 return 0;
499 },
500 xCurrentTime: function(pVfs,pOut){
501 /* If it turns out that we need to adjust for timezone, see:
502 https://stackoverflow.com/a/11760121/1458521 */
503 wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000),
504 'double');
505 return 0;
506 },
507 xCurrentTimeInt64: function(pVfs,pOut){
508 // TODO: confirm that this calculation is correct
509 wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(),
510 'i64');
511 return 0;
512 },
513 xDelete: function(pVfs, zName, doSyncDir){
stephanf8150112022-09-19 17:09:09 +0000514 mTimeStart('xDelete');
stephanf3860122022-09-18 17:32:35 +0000515 opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
516 /* We're ignoring errors because we cannot yet differentiate
517 between harmless and non-harmless failures. */
stephanf8150112022-09-19 17:09:09 +0000518 mTimeEnd();
stephanf3860122022-09-18 17:32:35 +0000519 return 0;
stephanc5313af2022-09-18 02:35:30 +0000520 },
521 xFullPathname: function(pVfs,zName,nOut,pOut){
522 /* Until/unless we have some notion of "current dir"
523 in OPFS, simply copy zName to pOut... */
524 const i = wasm.cstrncpy(pOut, zName, nOut);
525 return i<nOut ? 0 : capi.SQLITE_CANTOPEN
526 /*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/;
527 },
528 xGetLastError: function(pVfs,nOut,pOut){
529 /* TODO: store exception.message values from the async
530 partner in a dedicated SharedArrayBuffer, noting that we'd have
531 to encode them... TextEncoder can do that for us. */
532 warn("OPFS xGetLastError() has nothing sensible to return.");
533 return 0;
534 },
stephan8766fd22022-09-19 05:19:04 +0000535 //xSleep is optionally defined below
stephanc5313af2022-09-18 02:35:30 +0000536 xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
stephanf8150112022-09-19 17:09:09 +0000537 mTimeStart('xOpen');
stephanc5313af2022-09-18 02:35:30 +0000538 if(!f._){
539 f._ = {
540 fileTypes: {
541 SQLITE_OPEN_MAIN_DB: 'mainDb',
542 SQLITE_OPEN_MAIN_JOURNAL: 'mainJournal',
543 SQLITE_OPEN_TEMP_DB: 'tempDb',
544 SQLITE_OPEN_TEMP_JOURNAL: 'tempJournal',
545 SQLITE_OPEN_TRANSIENT_DB: 'transientDb',
546 SQLITE_OPEN_SUBJOURNAL: 'subjournal',
547 SQLITE_OPEN_SUPER_JOURNAL: 'superJournal',
548 SQLITE_OPEN_WAL: 'wal'
549 },
550 getFileType: function(filename,oflags){
551 const ft = f._.fileTypes;
552 for(let k of Object.keys(ft)){
553 if(oflags & capi[k]) return ft[k];
554 }
555 warn("Cannot determine fileType based on xOpen() flags for file",filename);
556 return '???';
557 }
558 };
559 }
560 if(0===zName){
561 zName = randomFilename();
562 }else if('number'===typeof zName){
563 zName = wasm.cstringToJs(zName);
564 }
565 const args = Object.create(null);
566 args.fid = pFile;
567 args.filename = zName;
568 args.sab = new SharedArrayBuffer(state.fileBufferSize);
569 args.fileType = f._.getFileType(args.filename, flags);
570 args.create = !!(flags & capi.SQLITE_OPEN_CREATE);
571 args.deleteOnClose = !!(flags & capi.SQLITE_OPEN_DELETEONCLOSE);
572 args.readOnly = !!(flags & capi.SQLITE_OPEN_READONLY);
573 const rc = opRun('xOpen', args);
574 if(!rc){
575 /* Recall that sqlite3_vfs::xClose() will be called, even on
576 error, unless pFile->pMethods is NULL. */
577 if(args.readOnly){
578 wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32');
579 }
580 __openFiles[pFile] = args;
581 args.sabView = new Uint8Array(args.sab);
582 args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8);
583 args.sq3File = new sqlite3_file(pFile);
584 args.sq3File.$pMethods = opfsIoMethods.pointer;
585 args.ba = new Uint8Array(args.sab);
586 }
stephanf8150112022-09-19 17:09:09 +0000587 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000588 return rc;
589 }/*xOpen()*/
590 }/*vfsSyncWrappers*/;
591
stephan8766fd22022-09-19 05:19:04 +0000592 if(dVfs){
593 opfsVfs.$xRandomness = dVfs.$xRandomness;
594 opfsVfs.$xSleep = dVfs.$xSleep;
595 }
stephanc5313af2022-09-18 02:35:30 +0000596 if(!opfsVfs.$xRandomness){
597 /* If the default VFS has no xRandomness(), add a basic JS impl... */
598 vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){
599 const heap = wasm.heap8u();
600 let i = 0;
601 for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF;
602 return i;
603 };
604 }
605 if(!opfsVfs.$xSleep){
606 /* If we can inherit an xSleep() impl from the default VFS then
stephan8766fd22022-09-19 05:19:04 +0000607 assume it's sane and use it, otherwise install a JS-based
608 one. */
609 vfsSyncWrappers.xSleep = function(pVfs,ms){
610 Atomics.wait(state.opSABView, state.opIds.xSleep, 0, ms);
611 return 0;
612 };
stephanc5313af2022-09-18 02:35:30 +0000613 }
614
615 /* Install the vfs/io_methods into their C-level shared instances... */
616 let inst = installMethod(opfsIoMethods);
617 for(let k of Object.keys(ioSyncWrappers)) inst(k, ioSyncWrappers[k]);
618 inst = installMethod(opfsVfs);
619 for(let k of Object.keys(vfsSyncWrappers)) inst(k, vfsSyncWrappers[k]);
stephanf3860122022-09-18 17:32:35 +0000620
stephanf3860122022-09-18 17:32:35 +0000621 /**
622 Syncronously deletes the given OPFS filesystem entry, ignoring
623 any errors. As this environment has no notion of "current
624 directory", the given name must be an absolute path. If the 2nd
625 argument is truthy, deletion is recursive (use with caution!).
626
627 Returns true if the deletion succeeded and fails if it fails,
628 but cannot report the nature of the failure.
629 */
stephan0e0687c2022-09-19 13:44:23 +0000630 opfsUtil.deleteEntry = function(fsEntryName,recursive=false){
stephanf3860122022-09-18 17:32:35 +0000631 return 0===opRun('xDelete', {filename:fsEntryName, recursive});
632 };
633 /**
634 Exactly like deleteEntry() but runs asynchronously.
635 */
stephan0e0687c2022-09-19 13:44:23 +0000636 opfsUtil.deleteEntryAsync = async function(fsEntryName,recursive=false){
stephanf3860122022-09-18 17:32:35 +0000637 wMsg('xDeleteNoWait', {filename: fsEntryName, recursive});
638 };
639 /**
640 Synchronously creates the given directory name, recursively, in
641 the OPFS filesystem. Returns true if it succeeds or the
642 directory already exists, else false.
643 */
644 opfsUtil.mkdir = async function(absDirName){
645 return 0===opRun('mkdir', absDirName);
646 };
647 /**
648 Synchronously checks whether the given OPFS filesystem exists,
649 returning true if it does, false if it doesn't.
650 */
651 opfsUtil.entryExists = function(fsEntryName){
652 return 0===opRun('xAccess', fsEntryName);
653 };
654
655 /**
656 Generates a random ASCII string, intended for use as a
657 temporary file name. Its argument is the length of the string,
658 defaulting to 16.
659 */
660 opfsUtil.randomFilename = randomFilename;
661
662 if(sqlite3.oo1){
663 opfsUtil.OpfsDb = function(...args){
664 const opt = sqlite3.oo1.dbCtorHelper.normalizeArgs(...args);
665 opt.vfs = opfsVfs.$zName;
666 sqlite3.oo1.dbCtorHelper.call(this, opt);
667 };
668 opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
669 }
stephanf8150112022-09-19 17:09:09 +0000670
stephanf3860122022-09-18 17:32:35 +0000671 /**
672 Potential TODOs:
673
674 - Expose one or both of the Worker objects via opfsUtil and
675 publish an interface for proxying the higher-level OPFS
676 features like getting a directory listing.
677 */
stephanc5313af2022-09-18 02:35:30 +0000678
679 const sanityCheck = async function(){
680 const scope = wasm.scopedAllocPush();
681 const sq3File = new sqlite3_file();
682 try{
683 const fid = sq3File.pointer;
684 const openFlags = capi.SQLITE_OPEN_CREATE
685 | capi.SQLITE_OPEN_READWRITE
686 //| capi.SQLITE_OPEN_DELETEONCLOSE
687 | capi.SQLITE_OPEN_MAIN_DB;
688 const pOut = wasm.scopedAlloc(8);
689 const dbFile = "/sanity/check/file";
690 const zDbFile = wasm.scopedAllocCString(dbFile);
691 let rc;
692 vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
693 rc = wasm.getMemValue(pOut,'i32');
694 log("xAccess(",dbFile,") exists ?=",rc);
695 rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile,
696 fid, openFlags, pOut);
stephan509f4052022-09-19 09:58:01 +0000697 log("open rc =",rc,"state.opSABView[xOpen] =",
698 state.opSABView[state.opIds.xOpen]);
stephanc5313af2022-09-18 02:35:30 +0000699 if(isWorkerErrCode(rc)){
700 error("open failed with code",rc);
701 return;
702 }
703 vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
704 rc = wasm.getMemValue(pOut,'i32');
705 if(!rc) toss("xAccess() failed to detect file.");
706 rc = ioSyncWrappers.xSync(sq3File.pointer, 0);
707 if(rc) toss('sync failed w/ rc',rc);
708 rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024);
709 if(rc) toss('truncate failed w/ rc',rc);
710 wasm.setMemValue(pOut,0,'i64');
711 rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut);
712 if(rc) toss('xFileSize failed w/ rc',rc);
713 log("xFileSize says:",wasm.getMemValue(pOut, 'i64'));
714 rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1);
715 if(rc) toss("xWrite() failed!");
716 const readBuf = wasm.scopedAlloc(16);
717 rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2);
718 wasm.setMemValue(readBuf+6,0);
719 let jRead = wasm.cstringToJs(readBuf);
720 log("xRead() got:",jRead);
721 if("sanity"!==jRead) toss("Unexpected xRead() value.");
stephan8766fd22022-09-19 05:19:04 +0000722 if(vfsSyncWrappers.xSleep){
723 log("xSleep()ing before close()ing...");
724 vfsSyncWrappers.xSleep(opfsVfs.pointer,2000);
725 log("waking up from xSleep()");
726 }
stephanc5313af2022-09-18 02:35:30 +0000727 rc = ioSyncWrappers.xClose(fid);
728 log("xClose rc =",rc,"opSABView =",state.opSABView);
729 log("Deleting file:",dbFile);
730 vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234);
731 vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
732 rc = wasm.getMemValue(pOut,'i32');
733 if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete().");
734 }finally{
735 sq3File.dispose();
736 wasm.scopedAllocPop(scope);
737 }
738 }/*sanityCheck()*/;
739
stephanf8150112022-09-19 17:09:09 +0000740
stephanc5313af2022-09-18 02:35:30 +0000741 W.onmessage = function({data}){
742 //log("Worker.onmessage:",data);
743 switch(data.type){
744 case 'loaded':
745 /*Pass our config and shared state on to the async worker.*/
746 wMsg('init',state);
747 break;
748 case 'inited':{
749 /*Indicates that the async partner has received the 'init',
750 so we now know that the state object is no longer subject to
751 being copied by a pending postMessage() call.*/
752 try {
stephan0e0687c2022-09-19 13:44:23 +0000753 const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, 0);
stephanc5313af2022-09-18 02:35:30 +0000754 if(rc){
755 opfsVfs.dispose();
756 toss("sqlite3_vfs_register(OPFS) failed with rc",rc);
757 }
758 if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){
759 toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS");
760 }
761 capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods);
762 state.opSABView = new Int32Array(state.opSAB);
763 if(options.sanityChecks){
764 warn("Running sanity checks because of opfs-sanity-check URL arg...");
765 sanityCheck();
766 }
stephanf3860122022-09-18 17:32:35 +0000767 W.onerror = W._originalOnError;
768 delete W._originalOnError;
769 sqlite3.opfs = opfsUtil;
stephanc5313af2022-09-18 02:35:30 +0000770 log("End of OPFS sqlite3_vfs setup.", opfsVfs);
stephan509f4052022-09-19 09:58:01 +0000771 promiseResolve(sqlite3);
stephanc5313af2022-09-18 02:35:30 +0000772 }catch(e){
773 error(e);
774 promiseReject(e);
775 }
776 break;
777 }
778 default:
779 promiseReject(e);
780 error("Unexpected message from the async worker:",data);
781 break;
782 }
783 };
784 })/*thePromise*/;
785 return thePromise;
786}/*installOpfsVfs()*/;
787sqlite3.installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js";
788}/*sqlite3ApiBootstrap.initializers.push()*/);