blob: 3fd08d9aeca4f7675eab9acad589d47f47ec8fd6 [file] [log] [blame]
stephan3961b262022-08-10 11:26:08 +00001/*
stephanc5313af2022-09-18 02:35:30 +00002 2022-09-18
stephan3961b262022-08-10 11:26:08 +00003
4 The author disclaims copyright to this source code. In place of a
5 legal notice, here is a blessing:
6
7 * May you do good and not evil.
8 * May you find forgiveness for yourself and forgive others.
9 * May you share freely, never taking more than you give.
10
11 ***********************************************************************
12
stephanc5313af2022-09-18 02:35:30 +000013 This file holds the synchronous half of an sqlite3_vfs
14 implementation which proxies, in a synchronous fashion, the
15 asynchronous Origin-Private FileSystem (OPFS) APIs using a second
16 Worker, implemented in sqlite3-opfs-async-proxy.js. This file is
17 intended to be appended to the main sqlite3 JS deliverable somewhere
18 after sqlite3-api-glue.js and before sqlite3-api-cleanup.js.
19
20*/
21
22'use strict';
23self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
24/**
25 sqlite3.installOpfsVfs() returns a Promise which, on success, installs
26 an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs
27 which accept a VFS. It uses the Origin-Private FileSystem API for
28 all file storage. On error it is rejected with an exception
29 explaining the problem. Reasons for rejection include, but are
30 not limited to:
31
32 - The counterpart Worker (see below) could not be loaded.
33
34 - The environment does not support OPFS. That includes when
35 this function is called from the main window thread.
36
stephan3961b262022-08-10 11:26:08 +000037
38 Significant notes and limitations:
39
40 - As of this writing, OPFS is still very much in flux and only
41 available in bleeding-edge versions of Chrome (v102+, noting that
42 that number will increase as the OPFS API matures).
43
stephanc5313af2022-09-18 02:35:30 +000044 - The OPFS features used here are only available in dedicated Worker
stephanf3860122022-09-18 17:32:35 +000045 threads. This file tries to detect that case, resulting in a
46 rejected Promise if those features do not seem to be available.
stephanc5313af2022-09-18 02:35:30 +000047
48 - It requires the SharedArrayBuffer and Atomics classes, and the
49 former is only available if the HTTP server emits the so-called
50 COOP and COEP response headers. These features are required for
51 proxying OPFS's synchronous API via the synchronous interface
52 required by the sqlite3_vfs API.
53
54 - This function may only be called a single time and it must be
55 called from the client, as opposed to the library initialization,
56 in case the client requires a custom path for this API's
57 "counterpart": this function's argument is the relative URI to
58 this module's "asynchronous half". When called, this function removes
59 itself from the sqlite3 object.
60
61 The argument may optionally be a plain object with the following
62 configuration options:
63
64 - proxyUri: as described above
65
66 - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables
67 logging of errors. 2 enables logging of warnings and errors. 3
68 additionally enables debugging info.
69
70 - sanityChecks (=false): if true, some basic sanity tests are
71 run on the OPFS VFS API after it's initialized, before the
72 returned Promise resolves.
73
74 On success, the Promise resolves to the top-most sqlite3 namespace
stephanf3860122022-09-18 17:32:35 +000075 object and that object gets a new object installed in its
76 `opfs` property, containing several OPFS-specific utilities.
stephan3961b262022-08-10 11:26:08 +000077*/
stephanc5313af2022-09-18 02:35:30 +000078sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
stephan509f4052022-09-19 09:58:01 +000079 delete sqlite3.installOpfsVfs;
80 if(self.window===self ||
81 !self.SharedArrayBuffer ||
82 !self.FileSystemHandle ||
83 !self.FileSystemDirectoryHandle ||
84 !self.FileSystemFileHandle ||
85 !self.FileSystemFileHandle.prototype.createSyncAccessHandle ||
86 !navigator.storage.getDirectory){
87 return Promise.reject(
88 new Error("This environment does not have OPFS support.")
89 );
90 }
stephanc5313af2022-09-18 02:35:30 +000091 const options = (asyncProxyUri && 'object'===asyncProxyUri) ? asyncProxyUri : {
92 proxyUri: asyncProxyUri
stephan3961b262022-08-10 11:26:08 +000093 };
stephan509f4052022-09-19 09:58:01 +000094 const urlParams = new URL(self.location.href).searchParams;
stephanc5313af2022-09-18 02:35:30 +000095 if(undefined===options.verbose){
stephan509f4052022-09-19 09:58:01 +000096 options.verbose = urlParams.has('opfs-verbose') ? 3 : 2;
stephan3961b262022-08-10 11:26:08 +000097 }
stephanc5313af2022-09-18 02:35:30 +000098 if(undefined===options.sanityChecks){
stephan509f4052022-09-19 09:58:01 +000099 options.sanityChecks = urlParams.has('opfs-sanity-check');
stephanc5313af2022-09-18 02:35:30 +0000100 }
101 if(undefined===options.proxyUri){
102 options.proxyUri = callee.defaultProxyUri;
103 }
stephanf3860122022-09-18 17:32:35 +0000104
stephanc5313af2022-09-18 02:35:30 +0000105 const thePromise = new Promise(function(promiseResolve, promiseReject){
stephan509f4052022-09-19 09:58:01 +0000106 const loggers = {
107 0:console.error.bind(console),
108 1:console.warn.bind(console),
109 2:console.log.bind(console)
stephanc5313af2022-09-18 02:35:30 +0000110 };
stephan509f4052022-09-19 09:58:01 +0000111 const logImpl = (level,...args)=>{
112 if(options.verbose>level) loggers[level]("OPFS syncer:",...args);
113 };
114 const log = (...args)=>logImpl(2, ...args);
115 const warn = (...args)=>logImpl(1, ...args);
116 const error = (...args)=>logImpl(0, ...args);
stephanc5313af2022-09-18 02:35:30 +0000117 warn("The OPFS VFS feature is very much experimental and under construction.");
118 const toss = function(...args){throw new Error(args.join(' '))};
stephanc5313af2022-09-18 02:35:30 +0000119 const capi = sqlite3.capi;
120 const wasm = capi.wasm;
121 const sqlite3_vfs = capi.sqlite3_vfs;
122 const sqlite3_file = capi.sqlite3_file;
123 const sqlite3_io_methods = capi.sqlite3_io_methods;
stephanc5313af2022-09-18 02:35:30 +0000124 const W = new Worker(options.proxyUri);
stephanf3860122022-09-18 17:32:35 +0000125 W._originalOnError = W.onerror /* will be restored later */;
stephanc5313af2022-09-18 02:35:30 +0000126 W.onerror = function(err){
stephan509f4052022-09-19 09:58:01 +0000127 // The error object doesn't contain any useful info when the
128 // failure is, e.g., that the remote script is 404.
stephanc5313af2022-09-18 02:35:30 +0000129 promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
130 };
131 const wMsg = (type,payload)=>W.postMessage({type,payload});
stephan509f4052022-09-19 09:58:01 +0000132 /**
133 Generic utilities for working with OPFS. This will get filled out
134 by the Promise setup and, on success, installed as sqlite3.opfs.
135 */
136 const opfsUtil = Object.create(null);
stephanf8150112022-09-19 17:09:09 +0000137 /**
138 Not part of the public API. Solely for internal/development
139 use.
140 */
141 opfsUtil.metrics = {
142 dump: function(){
stephanaec046a2022-09-19 18:22:29 +0000143 let k, n = 0, t = 0, w = 0;
144 for(k in state.opIds){
stephanf8150112022-09-19 17:09:09 +0000145 const m = metrics[k];
146 n += m.count;
147 t += m.time;
stephanaec046a2022-09-19 18:22:29 +0000148 w += m.wait;
stephanf8150112022-09-19 17:09:09 +0000149 m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
150 m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0;
151 }
stephanaec046a2022-09-19 18:22:29 +0000152 console.log(self.location.href,
153 "metrics for",self.location.href,":",metrics,
154 "\nTotal of",n,"op(s) for",t,
155 "ms (incl. "+w+" ms of waiting on the async side)");
stephanf8150112022-09-19 17:09:09 +0000156 },
157 reset: function(){
158 let k;
159 const r = (m)=>(m.count = m.time = m.wait = 0);
160 for(k in state.opIds){
161 r(metrics[k] = Object.create(null));
162 }
stephanaec046a2022-09-19 18:22:29 +0000163 //[ // timed routines which are not in state.opIds
164 // 'xFileControl'
165 //].forEach((k)=>r(metrics[k] = Object.create(null)));
stephanf8150112022-09-19 17:09:09 +0000166 }
167 }/*metrics*/;
stephanc5313af2022-09-18 02:35:30 +0000168
169 /**
170 State which we send to the async-api Worker or share with it.
171 This object must initially contain only cloneable or sharable
172 objects. After the worker's "inited" message arrives, other types
173 of data may be added to it.
stephanf3860122022-09-18 17:32:35 +0000174
175 For purposes of Atomics.wait() and Atomics.notify(), we use a
176 SharedArrayBuffer with one slot reserved for each of the API
177 proxy's methods. The sync side of the API uses Atomics.wait()
178 on the corresponding slot and the async side uses
179 Atomics.notify() on that slot.
180
181 The approach of using a single SAB to serialize comms for all
182 instances might(?) lead to deadlock situations in multi-db
183 cases. We should probably have one SAB here with a single slot
184 for locking a per-file initialization step and then allocate a
185 separate SAB like the above one for each file. That will
186 require a bit of acrobatics but should be feasible.
stephanc5313af2022-09-18 02:35:30 +0000187 */
188 const state = Object.create(null);
189 state.verbose = options.verbose;
stephanf3860122022-09-18 17:32:35 +0000190 state.fileBufferSize =
191 1024 * 64 + 8 /* size of aFileHandle.sab. 64k = max sqlite3 page
192 size. The additional bytes are space for
193 holding BigInt results, since we cannot store
194 those via the Atomics API (which only works on
195 an Int32Array). */;
196 state.fbInt64Offset =
197 state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */;
stephanc5313af2022-09-18 02:35:30 +0000198 state.opIds = Object.create(null);
stephanf8150112022-09-19 17:09:09 +0000199 const metrics = Object.create(null);
stephanc5313af2022-09-18 02:35:30 +0000200 {
stephan3961b262022-08-10 11:26:08 +0000201 let i = 0;
stephanc5313af2022-09-18 02:35:30 +0000202 state.opIds.xAccess = i++;
203 state.opIds.xClose = i++;
204 state.opIds.xDelete = i++;
stephanf3860122022-09-18 17:32:35 +0000205 state.opIds.xDeleteNoWait = i++;
stephanc5313af2022-09-18 02:35:30 +0000206 state.opIds.xFileSize = i++;
207 state.opIds.xOpen = i++;
208 state.opIds.xRead = i++;
209 state.opIds.xSleep = i++;
210 state.opIds.xSync = i++;
211 state.opIds.xTruncate = i++;
212 state.opIds.xWrite = i++;
stephanf3860122022-09-18 17:32:35 +0000213 state.opIds.mkdir = i++;
stephanc5313af2022-09-18 02:35:30 +0000214 state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/);
stephanaec046a2022-09-19 18:22:29 +0000215 state.opIds.xFileControl = state.opIds.xSync /* special case */;
stephanf8150112022-09-19 17:09:09 +0000216 opfsUtil.metrics.reset();
stephanc5313af2022-09-18 02:35:30 +0000217 }
218
219 state.sq3Codes = Object.create(null);
220 state.sq3Codes._reverse = Object.create(null);
221 [ // SQLITE_xxx constants to export to the async worker counterpart...
222 'SQLITE_ERROR', 'SQLITE_IOERR',
223 'SQLITE_NOTFOUND', 'SQLITE_MISUSE',
224 'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ',
225 'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
226 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
stephanf3860122022-09-18 17:32:35 +0000227 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
228 'SQLITE_IOERR_DELETE'
stephanc5313af2022-09-18 02:35:30 +0000229 ].forEach(function(k){
230 state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
231 state.sq3Codes._reverse[capi[k]] = k;
stephan3961b262022-08-10 11:26:08 +0000232 });
stephan3961b262022-08-10 11:26:08 +0000233
stephanc5313af2022-09-18 02:35:30 +0000234 const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n];
stephan3961b262022-08-10 11:26:08 +0000235
stephanc5313af2022-09-18 02:35:30 +0000236 /**
237 Runs the given operation in the async worker counterpart, waits
238 for its response, and returns the result which the async worker
239 writes to the given op's index in state.opSABView. The 2nd argument
240 must be a single object or primitive value, depending on the
241 given operation's signature in the async API counterpart.
242 */
243 const opRun = (op,args)=>{
stephanf8150112022-09-19 17:09:09 +0000244 const t = performance.now();
stephanf3860122022-09-18 17:32:35 +0000245 Atomics.store(state.opSABView, state.opIds[op], -1);
stephanc5313af2022-09-18 02:35:30 +0000246 wMsg(op, args);
stephanf3860122022-09-18 17:32:35 +0000247 Atomics.wait(state.opSABView, state.opIds[op], -1);
stephanf8150112022-09-19 17:09:09 +0000248 metrics[op].wait += performance.now() - t;
stephanc5313af2022-09-18 02:35:30 +0000249 return Atomics.load(state.opSABView, state.opIds[op]);
250 };
251
252 /**
253 Generates a random ASCII string len characters long, intended for
254 use as a temporary file name.
255 */
256 const randomFilename = function f(len=16){
257 if(!f._chars){
258 f._chars = "abcdefghijklmnopqrstuvwxyz"+
259 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
260 "012346789";
261 f._n = f._chars.length;
262 }
263 const a = [];
264 let i = 0;
265 for( ; i < len; ++i){
266 const ndx = Math.random() * (f._n * 64) % f._n | 0;
267 a[i] = f._chars[ndx];
268 }
269 return a.join('');
270 };
271
272 /**
273 Map of sqlite3_file pointers to objects constructed by xOpen().
274 */
275 const __openFiles = Object.create(null);
276
277 const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
278 const dVfs = pDVfs
279 ? new sqlite3_vfs(pDVfs)
280 : null /* dVfs will be null when sqlite3 is built with
281 SQLITE_OS_OTHER. Though we cannot currently handle
282 that case, the hope is to eventually be able to. */;
283 const opfsVfs = new sqlite3_vfs();
284 const opfsIoMethods = new sqlite3_io_methods();
285 opfsVfs.$iVersion = 2/*yes, two*/;
286 opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;
287 opfsVfs.$mxPathname = 1024/*sure, why not?*/;
288 opfsVfs.$zName = wasm.allocCString("opfs");
289 // All C-side memory of opfsVfs is zeroed out, but just to be explicit:
290 opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null;
291 opfsVfs.ondispose = [
292 '$zName', opfsVfs.$zName,
293 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null),
294 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose()
295 ];
stephanc5313af2022-09-18 02:35:30 +0000296 /**
297 Pedantic sidebar about opfsVfs.ondispose: the entries in that array
298 are items to clean up when opfsVfs.dispose() is called, but in this
299 environment it will never be called. The VFS instance simply
300 hangs around until the WASM module instance is cleaned up. We
301 "could" _hypothetically_ clean it up by "importing" an
302 sqlite3_os_end() impl into the wasm build, but the shutdown order
303 of the wasm engine and the JS one are undefined so there is no
304 guaranty that the opfsVfs instance would be available in one
305 environment or the other when sqlite3_os_end() is called (_if_ it
306 gets called at all in a wasm build, which is undefined).
307 */
308
309 /**
310 Installs a StructBinder-bound function pointer member of the
311 given name and function in the given StructType target object.
312 It creates a WASM proxy for the given function and arranges for
313 that proxy to be cleaned up when tgt.dispose() is called. Throws
314 on the slightest hint of error (e.g. tgt is-not-a StructType,
315 name does not map to a struct-bound member, etc.).
316
317 Returns a proxy for this function which is bound to tgt and takes
318 2 args (name,func). That function returns the same thing,
319 permitting calls to be chained.
320
321 If called with only 1 arg, it has no side effects but returns a
322 func with the same signature as described above.
323 */
324 const installMethod = function callee(tgt, name, func){
stephanf3860122022-09-18 17:32:35 +0000325 if(!(tgt instanceof sqlite3.StructBinder.StructType)){
stephanc5313af2022-09-18 02:35:30 +0000326 toss("Usage error: target object is-not-a StructType.");
327 }
328 if(1===arguments.length){
329 return (n,f)=>callee(tgt,n,f);
330 }
331 if(!callee.argcProxy){
332 callee.argcProxy = function(func,sig){
333 return function(...args){
334 if(func.length!==arguments.length){
335 toss("Argument mismatch. Native signature is:",sig);
336 }
337 return func.apply(this, args);
338 }
339 };
340 callee.removeFuncList = function(){
341 if(this.ondispose.__removeFuncList){
342 this.ondispose.__removeFuncList.forEach(
343 (v,ndx)=>{
344 if('number'===typeof v){
345 try{wasm.uninstallFunction(v)}
346 catch(e){/*ignore*/}
347 }
348 /* else it's a descriptive label for the next number in
349 the list. */
350 }
351 );
352 delete this.ondispose.__removeFuncList;
353 }
354 };
355 }/*static init*/
356 const sigN = tgt.memberSignature(name);
357 if(sigN.length<2){
358 toss("Member",name," is not a function pointer. Signature =",sigN);
359 }
360 const memKey = tgt.memberKey(name);
361 //log("installMethod",tgt, name, sigN);
362 const fProxy = 1
363 // We can remove this proxy middle-man once the VFS is working
364 ? callee.argcProxy(func, sigN)
365 : func;
366 const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true));
367 tgt[memKey] = pFunc;
368 if(!tgt.ondispose) tgt.ondispose = [];
369 if(!tgt.ondispose.__removeFuncList){
370 tgt.ondispose.push('ondispose.__removeFuncList handler',
371 callee.removeFuncList);
372 tgt.ondispose.__removeFuncList = [];
373 }
374 tgt.ondispose.__removeFuncList.push(memKey, pFunc);
375 return (n,f)=>callee(tgt, n, f);
376 }/*installMethod*/;
stephanf8150112022-09-19 17:09:09 +0000377
378 const opTimer = Object.create(null);
379 opTimer.op = undefined;
380 opTimer.start = undefined;
381 const mTimeStart = (op)=>{
382 opTimer.start = performance.now();
383 opTimer.op = op;
384 //metrics[op] || toss("Maintenance required: missing metrics for",op);
385 ++metrics[op].count;
386 };
387 const mTimeEnd = ()=>(
388 metrics[opTimer.op].time += performance.now() - opTimer.start
389 );
390
stephanc5313af2022-09-18 02:35:30 +0000391 /**
392 Impls for the sqlite3_io_methods methods. Maintenance reminder:
393 members are in alphabetical order to simplify finding them.
394 */
395 const ioSyncWrappers = {
396 xCheckReservedLock: function(pFile,pOut){
397 // Exclusive lock is automatically acquired when opened
398 //warn("xCheckReservedLock(",arguments,") is a no-op");
399 wasm.setMemValue(pOut,1,'i32');
400 return 0;
401 },
402 xClose: function(pFile){
stephanf8150112022-09-19 17:09:09 +0000403 mTimeStart('xClose');
stephanc5313af2022-09-18 02:35:30 +0000404 let rc = 0;
405 const f = __openFiles[pFile];
406 if(f){
407 delete __openFiles[pFile];
408 rc = opRun('xClose', pFile);
409 if(f.sq3File) f.sq3File.dispose();
410 }
stephanf8150112022-09-19 17:09:09 +0000411 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000412 return rc;
413 },
414 xDeviceCharacteristics: function(pFile){
415 //debug("xDeviceCharacteristics(",pFile,")");
416 return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
417 },
stephanf8150112022-09-19 17:09:09 +0000418 xFileControl: function(pFile, opId, pArg){
419 mTimeStart('xFileControl');
stephanaec046a2022-09-19 18:22:29 +0000420 const rc = (capi.SQLITE_FCNTL_SYNC===opId)
421 ? opRun('xSync', {fid:pFile, flags:0})
422 : capi.SQLITE_NOTFOUND;
stephanf8150112022-09-19 17:09:09 +0000423 mTimeEnd();
stephanaec046a2022-09-19 18:22:29 +0000424 return rc;
stephanc5313af2022-09-18 02:35:30 +0000425 },
426 xFileSize: function(pFile,pSz64){
stephanf8150112022-09-19 17:09:09 +0000427 mTimeStart('xFileSize');
stephanc5313af2022-09-18 02:35:30 +0000428 const rc = opRun('xFileSize', pFile);
429 if(!isWorkerErrCode(rc)){
stephanaec046a2022-09-19 18:22:29 +0000430 wasm.setMemValue(
431 pSz64, __openFiles[pFile].sabViewFileSize.getBigInt64(0,true),
432 'i64'
433 );
stephanc5313af2022-09-18 02:35:30 +0000434 }
stephanf8150112022-09-19 17:09:09 +0000435 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000436 return rc;
437 },
438 xLock: function(pFile,lockType){
439 //2022-09: OPFS handles lock when opened
440 //warn("xLock(",arguments,") is a no-op");
441 return 0;
442 },
443 xRead: function(pFile,pDest,n,offset){
444 /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
stephanf8150112022-09-19 17:09:09 +0000445 mTimeStart('xRead');
stephanc5313af2022-09-18 02:35:30 +0000446 const f = __openFiles[pFile];
447 let rc;
448 try {
449 // FIXME(?): block until we finish copying the xRead result buffer. How?
450 rc = opRun('xRead',{fid:pFile, n, offset});
stephan862281f2022-09-19 09:25:25 +0000451 if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){
stephanf8150112022-09-19 17:09:09 +0000452 // set() seems to be the fastest way to copy this...
453 wasm.heap8u().set(f.sabView.subarray(0, n), pDest);
stephan862281f2022-09-19 09:25:25 +0000454 }
stephanc5313af2022-09-18 02:35:30 +0000455 }catch(e){
456 error("xRead(",arguments,") failed:",e,f);
457 rc = capi.SQLITE_IOERR_READ;
458 }
stephanf8150112022-09-19 17:09:09 +0000459 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000460 return rc;
461 },
462 xSync: function(pFile,flags){
stephanaec046a2022-09-19 18:22:29 +0000463 ++metrics.xSync.count;
stephanf8150112022-09-19 17:09:09 +0000464 return 0; // impl'd in xFileControl(). opRun('xSync', {fid:pFile, flags});
stephanc5313af2022-09-18 02:35:30 +0000465 },
466 xTruncate: function(pFile,sz64){
stephanf8150112022-09-19 17:09:09 +0000467 mTimeStart('xTruncate');
468 const rc = opRun('xTruncate', {fid:pFile, size: sz64});
469 mTimeEnd();
470 return rc;
stephanc5313af2022-09-18 02:35:30 +0000471 },
472 xUnlock: function(pFile,lockType){
473 //2022-09: OPFS handles lock when opened
474 //warn("xUnlock(",arguments,") is a no-op");
475 return 0;
476 },
477 xWrite: function(pFile,pSrc,n,offset){
478 /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
stephanf8150112022-09-19 17:09:09 +0000479 mTimeStart('xWrite');
stephanc5313af2022-09-18 02:35:30 +0000480 const f = __openFiles[pFile];
stephanf8150112022-09-19 17:09:09 +0000481 let rc;
stephanc5313af2022-09-18 02:35:30 +0000482 try {
stephanc5313af2022-09-18 02:35:30 +0000483 // FIXME(?): block from here until we finish the xWrite. How?
stephanf8150112022-09-19 17:09:09 +0000484 f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n));
485 rc = opRun('xWrite',{fid:pFile, n, offset});
stephanc5313af2022-09-18 02:35:30 +0000486 }catch(e){
487 error("xWrite(",arguments,") failed:",e,f);
stephanf8150112022-09-19 17:09:09 +0000488 rc = capi.SQLITE_IOERR_WRITE;
stephanc5313af2022-09-18 02:35:30 +0000489 }
stephanf8150112022-09-19 17:09:09 +0000490 mTimeEnd();
491 return rc;
stephanc5313af2022-09-18 02:35:30 +0000492 }
493 }/*ioSyncWrappers*/;
494
495 /**
496 Impls for the sqlite3_vfs methods. Maintenance reminder: members
497 are in alphabetical order to simplify finding them.
498 */
499 const vfsSyncWrappers = {
500 xAccess: function(pVfs,zName,flags,pOut){
stephanf8150112022-09-19 17:09:09 +0000501 mTimeStart('xAccess');
stephanaec046a2022-09-19 18:22:29 +0000502 wasm.setMemValue(
503 pOut, (opRun('xAccess', wasm.cstringToJs(zName)) ? 0 : 1), 'i32'
504 );
stephanf8150112022-09-19 17:09:09 +0000505 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000506 return 0;
507 },
508 xCurrentTime: function(pVfs,pOut){
509 /* If it turns out that we need to adjust for timezone, see:
510 https://stackoverflow.com/a/11760121/1458521 */
511 wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000),
512 'double');
513 return 0;
514 },
515 xCurrentTimeInt64: function(pVfs,pOut){
516 // TODO: confirm that this calculation is correct
517 wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(),
518 'i64');
519 return 0;
520 },
521 xDelete: function(pVfs, zName, doSyncDir){
stephanf8150112022-09-19 17:09:09 +0000522 mTimeStart('xDelete');
stephanf3860122022-09-18 17:32:35 +0000523 opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
524 /* We're ignoring errors because we cannot yet differentiate
525 between harmless and non-harmless failures. */
stephanf8150112022-09-19 17:09:09 +0000526 mTimeEnd();
stephanf3860122022-09-18 17:32:35 +0000527 return 0;
stephanc5313af2022-09-18 02:35:30 +0000528 },
529 xFullPathname: function(pVfs,zName,nOut,pOut){
530 /* Until/unless we have some notion of "current dir"
531 in OPFS, simply copy zName to pOut... */
532 const i = wasm.cstrncpy(pOut, zName, nOut);
533 return i<nOut ? 0 : capi.SQLITE_CANTOPEN
534 /*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/;
535 },
536 xGetLastError: function(pVfs,nOut,pOut){
537 /* TODO: store exception.message values from the async
538 partner in a dedicated SharedArrayBuffer, noting that we'd have
539 to encode them... TextEncoder can do that for us. */
540 warn("OPFS xGetLastError() has nothing sensible to return.");
541 return 0;
542 },
stephan8766fd22022-09-19 05:19:04 +0000543 //xSleep is optionally defined below
stephanc5313af2022-09-18 02:35:30 +0000544 xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
stephanf8150112022-09-19 17:09:09 +0000545 mTimeStart('xOpen');
stephanc5313af2022-09-18 02:35:30 +0000546 if(!f._){
547 f._ = {
548 fileTypes: {
549 SQLITE_OPEN_MAIN_DB: 'mainDb',
550 SQLITE_OPEN_MAIN_JOURNAL: 'mainJournal',
551 SQLITE_OPEN_TEMP_DB: 'tempDb',
552 SQLITE_OPEN_TEMP_JOURNAL: 'tempJournal',
553 SQLITE_OPEN_TRANSIENT_DB: 'transientDb',
554 SQLITE_OPEN_SUBJOURNAL: 'subjournal',
555 SQLITE_OPEN_SUPER_JOURNAL: 'superJournal',
556 SQLITE_OPEN_WAL: 'wal'
557 },
558 getFileType: function(filename,oflags){
559 const ft = f._.fileTypes;
560 for(let k of Object.keys(ft)){
561 if(oflags & capi[k]) return ft[k];
562 }
563 warn("Cannot determine fileType based on xOpen() flags for file",filename);
564 return '???';
565 }
566 };
567 }
568 if(0===zName){
569 zName = randomFilename();
570 }else if('number'===typeof zName){
571 zName = wasm.cstringToJs(zName);
572 }
573 const args = Object.create(null);
574 args.fid = pFile;
575 args.filename = zName;
576 args.sab = new SharedArrayBuffer(state.fileBufferSize);
577 args.fileType = f._.getFileType(args.filename, flags);
578 args.create = !!(flags & capi.SQLITE_OPEN_CREATE);
579 args.deleteOnClose = !!(flags & capi.SQLITE_OPEN_DELETEONCLOSE);
580 args.readOnly = !!(flags & capi.SQLITE_OPEN_READONLY);
581 const rc = opRun('xOpen', args);
582 if(!rc){
583 /* Recall that sqlite3_vfs::xClose() will be called, even on
584 error, unless pFile->pMethods is NULL. */
585 if(args.readOnly){
586 wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32');
587 }
588 __openFiles[pFile] = args;
589 args.sabView = new Uint8Array(args.sab);
590 args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8);
591 args.sq3File = new sqlite3_file(pFile);
592 args.sq3File.$pMethods = opfsIoMethods.pointer;
593 args.ba = new Uint8Array(args.sab);
594 }
stephanf8150112022-09-19 17:09:09 +0000595 mTimeEnd();
stephanc5313af2022-09-18 02:35:30 +0000596 return rc;
597 }/*xOpen()*/
598 }/*vfsSyncWrappers*/;
599
stephan8766fd22022-09-19 05:19:04 +0000600 if(dVfs){
601 opfsVfs.$xRandomness = dVfs.$xRandomness;
602 opfsVfs.$xSleep = dVfs.$xSleep;
603 }
stephanc5313af2022-09-18 02:35:30 +0000604 if(!opfsVfs.$xRandomness){
605 /* If the default VFS has no xRandomness(), add a basic JS impl... */
606 vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){
607 const heap = wasm.heap8u();
608 let i = 0;
609 for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF;
610 return i;
611 };
612 }
613 if(!opfsVfs.$xSleep){
614 /* If we can inherit an xSleep() impl from the default VFS then
stephan8766fd22022-09-19 05:19:04 +0000615 assume it's sane and use it, otherwise install a JS-based
616 one. */
617 vfsSyncWrappers.xSleep = function(pVfs,ms){
618 Atomics.wait(state.opSABView, state.opIds.xSleep, 0, ms);
619 return 0;
620 };
stephanc5313af2022-09-18 02:35:30 +0000621 }
622
623 /* Install the vfs/io_methods into their C-level shared instances... */
624 let inst = installMethod(opfsIoMethods);
625 for(let k of Object.keys(ioSyncWrappers)) inst(k, ioSyncWrappers[k]);
626 inst = installMethod(opfsVfs);
627 for(let k of Object.keys(vfsSyncWrappers)) inst(k, vfsSyncWrappers[k]);
stephanf3860122022-09-18 17:32:35 +0000628
stephanf3860122022-09-18 17:32:35 +0000629 /**
630 Syncronously deletes the given OPFS filesystem entry, ignoring
631 any errors. As this environment has no notion of "current
632 directory", the given name must be an absolute path. If the 2nd
633 argument is truthy, deletion is recursive (use with caution!).
634
635 Returns true if the deletion succeeded and fails if it fails,
636 but cannot report the nature of the failure.
637 */
stephan0e0687c2022-09-19 13:44:23 +0000638 opfsUtil.deleteEntry = function(fsEntryName,recursive=false){
stephanf3860122022-09-18 17:32:35 +0000639 return 0===opRun('xDelete', {filename:fsEntryName, recursive});
640 };
641 /**
642 Exactly like deleteEntry() but runs asynchronously.
643 */
stephan0e0687c2022-09-19 13:44:23 +0000644 opfsUtil.deleteEntryAsync = async function(fsEntryName,recursive=false){
stephanf3860122022-09-18 17:32:35 +0000645 wMsg('xDeleteNoWait', {filename: fsEntryName, recursive});
646 };
647 /**
648 Synchronously creates the given directory name, recursively, in
649 the OPFS filesystem. Returns true if it succeeds or the
650 directory already exists, else false.
651 */
652 opfsUtil.mkdir = async function(absDirName){
653 return 0===opRun('mkdir', absDirName);
654 };
655 /**
656 Synchronously checks whether the given OPFS filesystem exists,
657 returning true if it does, false if it doesn't.
658 */
659 opfsUtil.entryExists = function(fsEntryName){
660 return 0===opRun('xAccess', fsEntryName);
661 };
662
663 /**
664 Generates a random ASCII string, intended for use as a
665 temporary file name. Its argument is the length of the string,
666 defaulting to 16.
667 */
668 opfsUtil.randomFilename = randomFilename;
669
670 if(sqlite3.oo1){
671 opfsUtil.OpfsDb = function(...args){
672 const opt = sqlite3.oo1.dbCtorHelper.normalizeArgs(...args);
673 opt.vfs = opfsVfs.$zName;
674 sqlite3.oo1.dbCtorHelper.call(this, opt);
675 };
676 opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
677 }
stephanf8150112022-09-19 17:09:09 +0000678
stephanf3860122022-09-18 17:32:35 +0000679 /**
680 Potential TODOs:
681
682 - Expose one or both of the Worker objects via opfsUtil and
683 publish an interface for proxying the higher-level OPFS
684 features like getting a directory listing.
685 */
stephanc5313af2022-09-18 02:35:30 +0000686
687 const sanityCheck = async function(){
688 const scope = wasm.scopedAllocPush();
689 const sq3File = new sqlite3_file();
690 try{
691 const fid = sq3File.pointer;
692 const openFlags = capi.SQLITE_OPEN_CREATE
693 | capi.SQLITE_OPEN_READWRITE
694 //| capi.SQLITE_OPEN_DELETEONCLOSE
695 | capi.SQLITE_OPEN_MAIN_DB;
696 const pOut = wasm.scopedAlloc(8);
697 const dbFile = "/sanity/check/file";
698 const zDbFile = wasm.scopedAllocCString(dbFile);
699 let rc;
700 vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
701 rc = wasm.getMemValue(pOut,'i32');
702 log("xAccess(",dbFile,") exists ?=",rc);
703 rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile,
704 fid, openFlags, pOut);
stephan509f4052022-09-19 09:58:01 +0000705 log("open rc =",rc,"state.opSABView[xOpen] =",
706 state.opSABView[state.opIds.xOpen]);
stephanc5313af2022-09-18 02:35:30 +0000707 if(isWorkerErrCode(rc)){
708 error("open failed with code",rc);
709 return;
710 }
711 vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
712 rc = wasm.getMemValue(pOut,'i32');
713 if(!rc) toss("xAccess() failed to detect file.");
714 rc = ioSyncWrappers.xSync(sq3File.pointer, 0);
715 if(rc) toss('sync failed w/ rc',rc);
716 rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024);
717 if(rc) toss('truncate failed w/ rc',rc);
718 wasm.setMemValue(pOut,0,'i64');
719 rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut);
720 if(rc) toss('xFileSize failed w/ rc',rc);
721 log("xFileSize says:",wasm.getMemValue(pOut, 'i64'));
722 rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1);
723 if(rc) toss("xWrite() failed!");
724 const readBuf = wasm.scopedAlloc(16);
725 rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2);
726 wasm.setMemValue(readBuf+6,0);
727 let jRead = wasm.cstringToJs(readBuf);
728 log("xRead() got:",jRead);
729 if("sanity"!==jRead) toss("Unexpected xRead() value.");
stephan8766fd22022-09-19 05:19:04 +0000730 if(vfsSyncWrappers.xSleep){
731 log("xSleep()ing before close()ing...");
732 vfsSyncWrappers.xSleep(opfsVfs.pointer,2000);
733 log("waking up from xSleep()");
734 }
stephanc5313af2022-09-18 02:35:30 +0000735 rc = ioSyncWrappers.xClose(fid);
736 log("xClose rc =",rc,"opSABView =",state.opSABView);
737 log("Deleting file:",dbFile);
738 vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234);
739 vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
740 rc = wasm.getMemValue(pOut,'i32');
741 if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete().");
742 }finally{
743 sq3File.dispose();
744 wasm.scopedAllocPop(scope);
745 }
746 }/*sanityCheck()*/;
747
stephanf8150112022-09-19 17:09:09 +0000748
stephanc5313af2022-09-18 02:35:30 +0000749 W.onmessage = function({data}){
750 //log("Worker.onmessage:",data);
751 switch(data.type){
752 case 'loaded':
753 /*Pass our config and shared state on to the async worker.*/
754 wMsg('init',state);
755 break;
756 case 'inited':{
757 /*Indicates that the async partner has received the 'init',
758 so we now know that the state object is no longer subject to
759 being copied by a pending postMessage() call.*/
760 try {
stephan0e0687c2022-09-19 13:44:23 +0000761 const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, 0);
stephanc5313af2022-09-18 02:35:30 +0000762 if(rc){
763 opfsVfs.dispose();
764 toss("sqlite3_vfs_register(OPFS) failed with rc",rc);
765 }
766 if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){
767 toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS");
768 }
769 capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods);
770 state.opSABView = new Int32Array(state.opSAB);
771 if(options.sanityChecks){
772 warn("Running sanity checks because of opfs-sanity-check URL arg...");
773 sanityCheck();
774 }
stephanf3860122022-09-18 17:32:35 +0000775 W.onerror = W._originalOnError;
776 delete W._originalOnError;
777 sqlite3.opfs = opfsUtil;
stephanc5313af2022-09-18 02:35:30 +0000778 log("End of OPFS sqlite3_vfs setup.", opfsVfs);
stephan509f4052022-09-19 09:58:01 +0000779 promiseResolve(sqlite3);
stephanc5313af2022-09-18 02:35:30 +0000780 }catch(e){
781 error(e);
782 promiseReject(e);
783 }
784 break;
785 }
786 default:
787 promiseReject(e);
788 error("Unexpected message from the async worker:",data);
789 break;
790 }
791 };
792 })/*thePromise*/;
793 return thePromise;
794}/*installOpfsVfs()*/;
795sqlite3.installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js";
796}/*sqlite3ApiBootstrap.initializers.push()*/);