blob: 5b29aa3ea50979eccd004ac7e8663d2169c23e3d [file] [log] [blame]
stephan132a87b2022-09-17 15:08:22 +00001/*
2 2022-09-16
3
4 The author disclaims copyright to this source code. In place of a
5 legal notice, here is a blessing:
6
7 * May you do good and not evil.
8 * May you find forgiveness for yourself and forgive others.
9 * May you share freely, never taking more than you give.
10
11 ***********************************************************************
12
stephane72ddfd2022-10-14 15:52:29 +000013 An Worker which manages asynchronous OPFS handles on behalf of a
14 synchronous API which controls it via a combination of Worker
15 messages, SharedArrayBuffer, and Atomics. It is the asynchronous
16 counterpart of the API defined in sqlite3-api-opfs.js.
stephan132a87b2022-09-17 15:08:22 +000017
18 Highly indebted to:
19
20 https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js
21
22 for demonstrating how to use the OPFS APIs.
stephan07315542022-09-17 20:50:12 +000023
24 This file is to be loaded as a Worker. It does not have any direct
25 access to the sqlite3 JS/WASM bits, so any bits which it needs (most
26 notably SQLITE_xxx integer codes) have to be imported into it via an
27 initialization process.
stephan132a87b2022-09-17 15:08:22 +000028*/
29'use strict';
stephan07315542022-09-17 20:50:12 +000030const toss = function(...args){throw new Error(args.join(' '))};
31if(self.window === self){
32 toss("This code cannot run from the main thread.",
33 "Load it as a Worker from a separate Worker.");
34}else if(!navigator.storage.getDirectory){
35 toss("This API requires navigator.storage.getDirectory.");
36}
stephan9a557732022-10-04 17:06:51 +000037
stephan07315542022-09-17 20:50:12 +000038/**
39 Will hold state copied to this object from the syncronous side of
40 this API.
41*/
42const state = Object.create(null);
43/**
44 verbose:
45
46 0 = no logging output
47 1 = only errors
48 2 = warnings and errors
49 3 = debug, warnings, and errors
50*/
51state.verbose = 2;
52
stephan509f4052022-09-19 09:58:01 +000053const loggers = {
54 0:console.error.bind(console),
55 1:console.warn.bind(console),
56 2:console.log.bind(console)
stephan07315542022-09-17 20:50:12 +000057};
stephan509f4052022-09-19 09:58:01 +000058const logImpl = (level,...args)=>{
59 if(state.verbose>level) loggers[level]("OPFS asyncer:",...args);
stephan07315542022-09-17 20:50:12 +000060};
stephan509f4052022-09-19 09:58:01 +000061const log = (...args)=>logImpl(2, ...args);
62const warn = (...args)=>logImpl(1, ...args);
63const error = (...args)=>logImpl(0, ...args);
stephanf8150112022-09-19 17:09:09 +000064const metrics = Object.create(null);
stephanaec046a2022-09-19 18:22:29 +000065metrics.reset = ()=>{
66 let k;
stephan5e8bb0a2022-09-20 08:27:57 +000067 const r = (m)=>(m.count = m.time = m.wait = 0);
stephanaec046a2022-09-19 18:22:29 +000068 for(k in state.opIds){
69 r(metrics[k] = Object.create(null));
70 }
stephanb8c8d4e2022-09-20 13:25:39 +000071 let s = metrics.s11n = Object.create(null);
72 s = s.serialize = Object.create(null);
73 s.count = s.time = 0;
74 s = metrics.s11n.deserialize = Object.create(null);
75 s.count = s.time = 0;
stephanaec046a2022-09-19 18:22:29 +000076};
77metrics.dump = ()=>{
78 let k, n = 0, t = 0, w = 0;
79 for(k in state.opIds){
80 const m = metrics[k];
81 n += m.count;
82 t += m.time;
stephan5e8bb0a2022-09-20 08:27:57 +000083 w += m.wait;
stephanaec046a2022-09-19 18:22:29 +000084 m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
85 }
86 console.log(self.location.href,
stephan5e8bb0a2022-09-20 08:27:57 +000087 "metrics for",self.location.href,":\n",
stephan56fae742022-09-24 10:12:19 +000088 metrics,
stephan5e8bb0a2022-09-20 08:27:57 +000089 "\nTotal of",n,"op(s) for",t,"ms",
90 "approx",w,"ms spent waiting on OPFS APIs.");
stephan56fae742022-09-24 10:12:19 +000091 console.log("Serialization metrics:",metrics.s11n);
stephanaec046a2022-09-19 18:22:29 +000092};
stephan07315542022-09-17 20:50:12 +000093
stephan07315542022-09-17 20:50:12 +000094/**
95 Map of sqlite3_file pointers (integers) to metadata related to a
96 given OPFS file handles. The pointers are, in this side of the
97 interface, opaque file handle IDs provided by the synchronous
98 part of this constellation. Each value is an object with a structure
99 demonstrated in the xOpen() impl.
100*/
101const __openFiles = Object.create(null);
102
103/**
stephan8200a6d2022-09-17 23:29:27 +0000104 Expects an OPFS file path. It gets resolved, such that ".."
stephane72ddfd2022-10-14 15:52:29 +0000105 components are properly expanded, and returned. If the 2nd arg is
106 true, the result is returned as an array of path elements, else an
107 absolute path string is returned.
stephan07315542022-09-17 20:50:12 +0000108*/
stephan8200a6d2022-09-17 23:29:27 +0000109const getResolvedPath = function(filename,splitIt){
110 const p = new URL(
111 filename, 'file://irrelevant'
112 ).pathname;
113 return splitIt ? p.split('/').filter((v)=>!!v) : p;
stephan509f4052022-09-19 09:58:01 +0000114};
stephan07315542022-09-17 20:50:12 +0000115
116/**
117 Takes the absolute path to a filesystem element. Returns an array
stephane72ddfd2022-10-14 15:52:29 +0000118 of [handleOfContainingDir, filename]. If the 2nd argument is truthy
119 then each directory element leading to the file is created along
120 the way. Throws if any creation or resolution fails.
stephan07315542022-09-17 20:50:12 +0000121*/
stephan56fae742022-09-24 10:12:19 +0000122const getDirForFilename = async function f(absFilename, createDirs = false){
stephan8200a6d2022-09-17 23:29:27 +0000123 const path = getResolvedPath(absFilename, true);
stephan07315542022-09-17 20:50:12 +0000124 const filename = path.pop();
stephan8200a6d2022-09-17 23:29:27 +0000125 let dh = state.rootDir;
126 for(const dirName of path){
127 if(dirName){
128 dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
stephan07315542022-09-17 20:50:12 +0000129 }
stephan132a87b2022-09-17 15:08:22 +0000130 }
stephan07315542022-09-17 20:50:12 +0000131 return [dh, filename];
132};
stephan132a87b2022-09-17 15:08:22 +0000133
stephan7ff8da82022-10-03 09:21:37 +0000134/**
135 Returns the sync access handle associated with the given file
136 handle object (which must be a valid handle object), lazily opening
stephan9a557732022-10-04 17:06:51 +0000137 it if needed.
stephanc7fb48d2022-10-04 09:12:05 +0000138
139 In order to help alleviate cross-tab contention for a dabase,
140 if an exception is thrown while acquiring the handle, this routine
141 will wait briefly and try again, up to 3 times. If acquisition
142 still fails at that point it will give up and propagate the
143 exception.
stephan7ff8da82022-10-03 09:21:37 +0000144*/
stephan5f0b67c2022-10-03 11:33:35 +0000145const getSyncHandle = async (fh)=>{
146 if(!fh.syncHandle){
stephanc7fb48d2022-10-04 09:12:05 +0000147 const t = performance.now();
148 log("Acquiring sync handle for",fh.filenameAbs);
149 const maxTries = 3;
150 let i = 1, ms = 300;
151 for(; true; ms *= ++i){
152 try {
153 //if(1===i) toss("Just testing.");
154 //TODO? A config option which tells it to throw here
stephan3e771c02022-10-14 13:26:18 +0000155 //randomly every now and then, for testing purposes.
stephanc7fb48d2022-10-04 09:12:05 +0000156 fh.syncHandle = await fh.fileHandle.createSyncAccessHandle();
157 break;
158 }catch(e){
159 if(i === maxTries){
160 toss("Error getting sync handle.",maxTries,
161 "attempts failed. ",fh.filenameAbs, ":", e.message);
162 throw e;
163 }
stephan3e771c02022-10-14 13:26:18 +0000164 warn("Error getting sync handle. Waiting",ms,
stephanc7fb48d2022-10-04 09:12:05 +0000165 "ms and trying again.",fh.filenameAbs,e);
166 Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms);
167 }
168 }
169 log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
stephan5f0b67c2022-10-03 11:33:35 +0000170 }
stephan5f0b67c2022-10-03 11:33:35 +0000171 return fh.syncHandle;
172};
173
stephane72ddfd2022-10-14 15:52:29 +0000174/**
175 If the given file-holding object has a sync handle attached to it,
176 that handle is remove and asynchronously closed. Though it may
177 sound sensible to continue work as soon as the close() returns
178 (noting that it's asynchronous), doing so can cause operations
179 performed soon afterwards, e.g. a call to getSyncHandle() to fail
180 because they may happen out of order from the close(). OPFS does
181 not guaranty that the actual order of operations is retained in
182 such cases. i.e. always "await" on the result of this function.
183*/
stephan5f0b67c2022-10-03 11:33:35 +0000184const closeSyncHandle = async (fh)=>{
185 if(fh.syncHandle){
stephan9a557732022-10-04 17:06:51 +0000186 log("Closing sync handle for",fh.filenameAbs);
stephan5f0b67c2022-10-03 11:33:35 +0000187 const h = fh.syncHandle;
188 delete fh.syncHandle;
189 return h.close();
190 }
191};
stephan132a87b2022-09-17 15:08:22 +0000192
stephan07315542022-09-17 20:50:12 +0000193/**
stephan72ab4002022-09-21 12:27:35 +0000194 Stores the given value at state.sabOPView[state.opIds.rc] and then
195 Atomics.notify()'s it.
stephan07315542022-09-17 20:50:12 +0000196*/
197const storeAndNotify = (opName, value)=>{
stephanc9e26022022-09-20 10:11:52 +0000198 log(opName+"() => notify(",state.opIds.rc,",",value,")");
199 Atomics.store(state.sabOPView, state.opIds.rc, value);
200 Atomics.notify(state.sabOPView, state.opIds.rc);
stephan07315542022-09-17 20:50:12 +0000201};
stephan132a87b2022-09-17 15:08:22 +0000202
stephan07315542022-09-17 20:50:12 +0000203/**
204 Throws if fh is a file-holding object which is flagged as read-only.
205*/
206const affirmNotRO = function(opName,fh){
207 if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs);
208};
209
stephane72ddfd2022-10-14 15:52:29 +0000210/**
211 We track 2 different timers: the "metrics" timer records how much
212 time we spend performing work. The "wait" timer records how much
213 time we spend waiting on the underlying OPFS timer. See the calls
214 to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd()
215 throughout this file to see how they're used.
216*/
217const __mTimer = Object.create(null);
218__mTimer.op = undefined;
219__mTimer.start = undefined;
stephanaec046a2022-09-19 18:22:29 +0000220const mTimeStart = (op)=>{
stephane72ddfd2022-10-14 15:52:29 +0000221 __mTimer.start = performance.now();
222 __mTimer.op = op;
stephanaec046a2022-09-19 18:22:29 +0000223 //metrics[op] || toss("Maintenance required: missing metrics for",op);
224 ++metrics[op].count;
225};
226const mTimeEnd = ()=>(
stephane72ddfd2022-10-14 15:52:29 +0000227 metrics[__mTimer.op].time += performance.now() - __mTimer.start
stephanaec046a2022-09-19 18:22:29 +0000228);
stephane72ddfd2022-10-14 15:52:29 +0000229const __wTimer = Object.create(null);
230__wTimer.op = undefined;
231__wTimer.start = undefined;
stephan5e8bb0a2022-09-20 08:27:57 +0000232const wTimeStart = (op)=>{
stephane72ddfd2022-10-14 15:52:29 +0000233 __wTimer.start = performance.now();
234 __wTimer.op = op;
stephan5e8bb0a2022-09-20 08:27:57 +0000235 //metrics[op] || toss("Maintenance required: missing metrics for",op);
236};
237const wTimeEnd = ()=>(
stephane72ddfd2022-10-14 15:52:29 +0000238 metrics[__wTimer.op].wait += performance.now() - __wTimer.start
stephan5e8bb0a2022-09-20 08:27:57 +0000239);
stephanaec046a2022-09-19 18:22:29 +0000240
stephan07315542022-09-17 20:50:12 +0000241/**
stephane72ddfd2022-10-14 15:52:29 +0000242 Gets set to true by the 'opfs-async-shutdown' command to quit the
243 wait loop. This is only intended for debugging purposes: we cannot
244 inspect this file's state while the tight waitLoop() is running and
245 need a way to stop that loop for introspection purposes.
stephan3c272ba2022-10-04 00:54:00 +0000246*/
247let flagAsyncShutdown = false;
248
stephan9a557732022-10-04 17:06:51 +0000249
stephan3c272ba2022-10-04 00:54:00 +0000250/**
stephan07315542022-09-17 20:50:12 +0000251 Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
stephane72ddfd2022-10-14 15:52:29 +0000252 methods, as well as helpers like mkdir(). Maintenance reminder:
253 members are in alphabetical order to simplify finding them.
stephan07315542022-09-17 20:50:12 +0000254*/
255const vfsAsyncImpls = {
stephan3c272ba2022-10-04 00:54:00 +0000256 'opfs-async-metrics': async ()=>{
257 mTimeStart('opfs-async-metrics');
stephan56fae742022-09-24 10:12:19 +0000258 metrics.dump();
stephan3c272ba2022-10-04 00:54:00 +0000259 storeAndNotify('opfs-async-metrics', 0);
stephan56fae742022-09-24 10:12:19 +0000260 mTimeEnd();
261 },
stephan3c272ba2022-10-04 00:54:00 +0000262 'opfs-async-shutdown': async ()=>{
263 flagAsyncShutdown = true;
264 storeAndNotify('opfs-async-shutdown', 0);
265 },
stephan56fae742022-09-24 10:12:19 +0000266 mkdir: async (dirname)=>{
stephan5e8bb0a2022-09-20 08:27:57 +0000267 mTimeStart('mkdir');
stephanaec046a2022-09-19 18:22:29 +0000268 let rc = 0;
stephan5e8bb0a2022-09-20 08:27:57 +0000269 wTimeStart('mkdir');
stephanaec046a2022-09-19 18:22:29 +0000270 try {
stephan56fae742022-09-24 10:12:19 +0000271 await getDirForFilename(dirname+"/filepart", true);
stephanaec046a2022-09-19 18:22:29 +0000272 }catch(e){
stephan56fae742022-09-24 10:12:19 +0000273 state.s11n.storeException(2,e);
stephanaec046a2022-09-19 18:22:29 +0000274 rc = state.sq3Codes.SQLITE_IOERR;
stephan72ab4002022-09-21 12:27:35 +0000275 }finally{
276 wTimeEnd();
stephanaec046a2022-09-19 18:22:29 +0000277 }
278 storeAndNotify('mkdir', rc);
stephan5e8bb0a2022-09-20 08:27:57 +0000279 mTimeEnd();
stephanaec046a2022-09-19 18:22:29 +0000280 },
stephan56fae742022-09-24 10:12:19 +0000281 xAccess: async (filename)=>{
stephanaec046a2022-09-19 18:22:29 +0000282 mTimeStart('xAccess');
stephan8200a6d2022-09-17 23:29:27 +0000283 /* OPFS cannot support the full range of xAccess() queries sqlite3
284 calls for. We can essentially just tell if the file is
285 accessible, but if it is it's automatically writable (unless
286 it's locked, which we cannot(?) know without trying to open
287 it). OPFS does not have the notion of read-only.
288
289 The return semantics of this function differ from sqlite3's
290 xAccess semantics because we are limited in what we can
291 communicate back to our synchronous communication partner: 0 =
292 accessible, non-0 means not accessible.
293 */
294 let rc = 0;
stephan5e8bb0a2022-09-20 08:27:57 +0000295 wTimeStart('xAccess');
stephan8200a6d2022-09-17 23:29:27 +0000296 try{
stephan56fae742022-09-24 10:12:19 +0000297 const [dh, fn] = await getDirForFilename(filename);
stephan8200a6d2022-09-17 23:29:27 +0000298 await dh.getFileHandle(fn);
299 }catch(e){
stephan56fae742022-09-24 10:12:19 +0000300 state.s11n.storeException(2,e);
stephan8200a6d2022-09-17 23:29:27 +0000301 rc = state.sq3Codes.SQLITE_IOERR;
stephan72ab4002022-09-21 12:27:35 +0000302 }finally{
303 wTimeEnd();
stephan8200a6d2022-09-17 23:29:27 +0000304 }
stephan07315542022-09-17 20:50:12 +0000305 storeAndNotify('xAccess', rc);
stephanaec046a2022-09-19 18:22:29 +0000306 mTimeEnd();
stephan07315542022-09-17 20:50:12 +0000307 },
308 xClose: async function(fid){
309 const opName = 'xClose';
stephanaec046a2022-09-19 18:22:29 +0000310 mTimeStart(opName);
stephan07315542022-09-17 20:50:12 +0000311 const fh = __openFiles[fid];
stephan5e8bb0a2022-09-20 08:27:57 +0000312 let rc = 0;
313 wTimeStart('xClose');
stephan07315542022-09-17 20:50:12 +0000314 if(fh){
315 delete __openFiles[fid];
stephan5f0b67c2022-10-03 11:33:35 +0000316 await closeSyncHandle(fh);
stephan07315542022-09-17 20:50:12 +0000317 if(fh.deleteOnClose){
318 try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
319 catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
stephan132a87b2022-09-17 15:08:22 +0000320 }
stephan07315542022-09-17 20:50:12 +0000321 }else{
stephan72ab4002022-09-21 12:27:35 +0000322 state.s11n.serialize();
stephan5e8bb0a2022-09-20 08:27:57 +0000323 rc = state.sq3Codes.SQLITE_NOTFOUND;
stephan132a87b2022-09-17 15:08:22 +0000324 }
stephan5e8bb0a2022-09-20 08:27:57 +0000325 wTimeEnd();
326 storeAndNotify(opName, rc);
stephanaec046a2022-09-19 18:22:29 +0000327 mTimeEnd();
stephan07315542022-09-17 20:50:12 +0000328 },
stephanc4b87be2022-09-20 01:28:47 +0000329 xDelete: async function(...args){
330 mTimeStart('xDelete');
331 const rc = await vfsAsyncImpls.xDeleteNoWait(...args);
332 storeAndNotify('xDelete', rc);
333 mTimeEnd();
334 },
stephan138647a2022-09-20 03:31:02 +0000335 xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){
stephan8200a6d2022-09-17 23:29:27 +0000336 /* The syncDir flag is, for purposes of the VFS API's semantics,
337 ignored here. However, if it has the value 0x1234 then: after
338 deleting the given file, recursively try to delete any empty
339 directories left behind in its wake (ignoring any errors and
340 stopping at the first failure).
341
342 That said: we don't know for sure that removeEntry() fails if
343 the dir is not empty because the API is not documented. It has,
344 however, a "recursive" flag which defaults to false, so
345 presumably it will fail if the dir is not empty and that flag
346 is false.
347 */
stephanf3860122022-09-18 17:32:35 +0000348 let rc = 0;
stephan5e8bb0a2022-09-20 08:27:57 +0000349 wTimeStart('xDelete');
stephan07315542022-09-17 20:50:12 +0000350 try {
stephan8200a6d2022-09-17 23:29:27 +0000351 while(filename){
stephan56fae742022-09-24 10:12:19 +0000352 const [hDir, filenamePart] = await getDirForFilename(filename, false);
stephan8200a6d2022-09-17 23:29:27 +0000353 if(!filenamePart) break;
stephanf3860122022-09-18 17:32:35 +0000354 await hDir.removeEntry(filenamePart, {recursive});
stephan8200a6d2022-09-17 23:29:27 +0000355 if(0x1234 !== syncDir) break;
356 filename = getResolvedPath(filename, true);
357 filename.pop();
358 filename = filename.join('/');
359 }
stephan07315542022-09-17 20:50:12 +0000360 }catch(e){
stephan56fae742022-09-24 10:12:19 +0000361 state.s11n.storeException(2,e);
stephanf3860122022-09-18 17:32:35 +0000362 rc = state.sq3Codes.SQLITE_IOERR_DELETE;
stephan132a87b2022-09-17 15:08:22 +0000363 }
stephan5e8bb0a2022-09-20 08:27:57 +0000364 wTimeEnd();
stephanf3860122022-09-18 17:32:35 +0000365 return rc;
366 },
stephan07315542022-09-17 20:50:12 +0000367 xFileSize: async function(fid){
stephanaec046a2022-09-19 18:22:29 +0000368 mTimeStart('xFileSize');
stephan07315542022-09-17 20:50:12 +0000369 const fh = __openFiles[fid];
370 let sz;
stephan5e8bb0a2022-09-20 08:27:57 +0000371 wTimeStart('xFileSize');
stephan07315542022-09-17 20:50:12 +0000372 try{
stephan7ff8da82022-10-03 09:21:37 +0000373 sz = await (await getSyncHandle(fh)).getSize();
stephan138647a2022-09-20 03:31:02 +0000374 state.s11n.serialize(Number(sz));
stephan07315542022-09-17 20:50:12 +0000375 sz = 0;
376 }catch(e){
stephan56fae742022-09-24 10:12:19 +0000377 state.s11n.storeException(2,e);
stephan07315542022-09-17 20:50:12 +0000378 sz = state.sq3Codes.SQLITE_IOERR;
stephan132a87b2022-09-17 15:08:22 +0000379 }
stephan5e8bb0a2022-09-20 08:27:57 +0000380 wTimeEnd();
stephan07315542022-09-17 20:50:12 +0000381 storeAndNotify('xFileSize', sz);
stephanaec046a2022-09-19 18:22:29 +0000382 mTimeEnd();
stephan07315542022-09-17 20:50:12 +0000383 },
stephan9a557732022-10-04 17:06:51 +0000384 xLock: async function(fid,lockType){
385 mTimeStart('xLock');
386 const fh = __openFiles[fid];
387 let rc = 0;
388 if( !fh.syncHandle ){
stephane72ddfd2022-10-14 15:52:29 +0000389 wTimeStart('xLock');
stephan9a557732022-10-04 17:06:51 +0000390 try { await getSyncHandle(fh) }
391 catch(e){
392 state.s11n.storeException(1,e);
393 rc = state.sq3Codes.SQLITE_IOERR;
394 }
stephane72ddfd2022-10-14 15:52:29 +0000395 wTimeEnd();
stephan9a557732022-10-04 17:06:51 +0000396 }
397 storeAndNotify('xLock',rc);
398 mTimeEnd();
399 },
stephan138647a2022-09-20 03:31:02 +0000400 xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags){
stephan07315542022-09-17 20:50:12 +0000401 const opName = 'xOpen';
stephanaec046a2022-09-19 18:22:29 +0000402 mTimeStart(opName);
stephanc4b87be2022-09-20 01:28:47 +0000403 const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
404 const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
stephan5e8bb0a2022-09-20 08:27:57 +0000405 wTimeStart('xOpen');
stephan07315542022-09-17 20:50:12 +0000406 try{
stephan07315542022-09-17 20:50:12 +0000407 let hDir, filenamePart;
408 try {
stephan56fae742022-09-24 10:12:19 +0000409 [hDir, filenamePart] = await getDirForFilename(filename, !!create);
stephan132a87b2022-09-17 15:08:22 +0000410 }catch(e){
stephan07315542022-09-17 20:50:12 +0000411 storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND);
stephanaec046a2022-09-19 18:22:29 +0000412 mTimeEnd();
stephan5e8bb0a2022-09-20 08:27:57 +0000413 wTimeEnd();
stephan07315542022-09-17 20:50:12 +0000414 return;
stephan132a87b2022-09-17 15:08:22 +0000415 }
stephanc4b87be2022-09-20 01:28:47 +0000416 const hFile = await hDir.getFileHandle(filenamePart, {create});
stephan07315542022-09-17 20:50:12 +0000417 /**
418 wa-sqlite, at this point, grabs a SyncAccessHandle and
stephan5f0b67c2022-10-03 11:33:35 +0000419 assigns it to the syncHandle prop of the file state
stephan07315542022-09-17 20:50:12 +0000420 object, but only for certain cases and it's unclear why it
421 places that limitation on it.
422 */
stephan5e8bb0a2022-09-20 08:27:57 +0000423 wTimeEnd();
stephan5f0b67c2022-10-03 11:33:35 +0000424 __openFiles[fid] = Object.assign(Object.create(null),{
stephan7ff8da82022-10-03 09:21:37 +0000425 filenameAbs: filename,
426 filenamePart: filenamePart,
427 dirHandle: hDir,
428 fileHandle: hFile,
429 sabView: state.sabFileBufView,
430 readOnly: create
431 ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),
432 deleteOnClose: deleteOnClose
433 });
stephan07315542022-09-17 20:50:12 +0000434 storeAndNotify(opName, 0);
435 }catch(e){
stephan5e8bb0a2022-09-20 08:27:57 +0000436 wTimeEnd();
stephan07315542022-09-17 20:50:12 +0000437 error(opName,e);
stephan56fae742022-09-24 10:12:19 +0000438 state.s11n.storeException(1,e);
stephan07315542022-09-17 20:50:12 +0000439 storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR);
440 }
stephanaec046a2022-09-19 18:22:29 +0000441 mTimeEnd();
stephan07315542022-09-17 20:50:12 +0000442 },
stephan138647a2022-09-20 03:31:02 +0000443 xRead: async function(fid,n,offset){
stephanaec046a2022-09-19 18:22:29 +0000444 mTimeStart('xRead');
stephane72ddfd2022-10-14 15:52:29 +0000445 let rc = 0, nRead;
stephan5f0b67c2022-10-03 11:33:35 +0000446 const fh = __openFiles[fid];
stephan07315542022-09-17 20:50:12 +0000447 try{
stephan5e8bb0a2022-09-20 08:27:57 +0000448 wTimeStart('xRead');
stephane72ddfd2022-10-14 15:52:29 +0000449 nRead = (await getSyncHandle(fh)).read(
stephanaec046a2022-09-19 18:22:29 +0000450 fh.sabView.subarray(0, n),
451 {at: Number(offset)}
452 );
stephan5e8bb0a2022-09-20 08:27:57 +0000453 wTimeEnd();
stephan07315542022-09-17 20:50:12 +0000454 if(nRead < n){/* Zero-fill remaining bytes */
stephanaec046a2022-09-19 18:22:29 +0000455 fh.sabView.fill(0, nRead, n);
stephan07315542022-09-17 20:50:12 +0000456 rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
457 }
458 }catch(e){
stephane72ddfd2022-10-14 15:52:29 +0000459 if(undefined===nRead) wTimeEnd();
stephan07315542022-09-17 20:50:12 +0000460 error("xRead() failed",e,fh);
stephan56fae742022-09-24 10:12:19 +0000461 state.s11n.storeException(1,e);
stephan07315542022-09-17 20:50:12 +0000462 rc = state.sq3Codes.SQLITE_IOERR_READ;
463 }
464 storeAndNotify('xRead',rc);
stephanaec046a2022-09-19 18:22:29 +0000465 mTimeEnd();
stephan07315542022-09-17 20:50:12 +0000466 },
stephan138647a2022-09-20 03:31:02 +0000467 xSync: async function(fid,flags/*ignored*/){
stephanaec046a2022-09-19 18:22:29 +0000468 mTimeStart('xSync');
stephan07315542022-09-17 20:50:12 +0000469 const fh = __openFiles[fid];
stephan72ab4002022-09-21 12:27:35 +0000470 let rc = 0;
stephan5f0b67c2022-10-03 11:33:35 +0000471 if(!fh.readOnly && fh.syncHandle){
stephan72ab4002022-09-21 12:27:35 +0000472 try {
473 wTimeStart('xSync');
stephan5f0b67c2022-10-03 11:33:35 +0000474 await fh.syncHandle.flush();
stephan72ab4002022-09-21 12:27:35 +0000475 }catch(e){
stephan56fae742022-09-24 10:12:19 +0000476 state.s11n.storeException(2,e);
stephan72ab4002022-09-21 12:27:35 +0000477 }
stephane72ddfd2022-10-14 15:52:29 +0000478 wTimeEnd();
stephan5e8bb0a2022-09-20 08:27:57 +0000479 }
stephan72ab4002022-09-21 12:27:35 +0000480 storeAndNotify('xSync',rc);
stephanaec046a2022-09-19 18:22:29 +0000481 mTimeEnd();
stephan07315542022-09-17 20:50:12 +0000482 },
stephan138647a2022-09-20 03:31:02 +0000483 xTruncate: async function(fid,size){
stephanaec046a2022-09-19 18:22:29 +0000484 mTimeStart('xTruncate');
stephan07315542022-09-17 20:50:12 +0000485 let rc = 0;
486 const fh = __openFiles[fid];
stephan5e8bb0a2022-09-20 08:27:57 +0000487 wTimeStart('xTruncate');
stephan07315542022-09-17 20:50:12 +0000488 try{
489 affirmNotRO('xTruncate', fh);
stephan7ff8da82022-10-03 09:21:37 +0000490 await (await getSyncHandle(fh)).truncate(size);
stephan07315542022-09-17 20:50:12 +0000491 }catch(e){
492 error("xTruncate():",e,fh);
stephan56fae742022-09-24 10:12:19 +0000493 state.s11n.storeException(2,e);
stephan07315542022-09-17 20:50:12 +0000494 rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
495 }
stephan5e8bb0a2022-09-20 08:27:57 +0000496 wTimeEnd();
stephan07315542022-09-17 20:50:12 +0000497 storeAndNotify('xTruncate',rc);
stephanaec046a2022-09-19 18:22:29 +0000498 mTimeEnd();
stephan07315542022-09-17 20:50:12 +0000499 },
stephan9a557732022-10-04 17:06:51 +0000500 xUnlock: async function(fid,lockType){
501 mTimeStart('xUnlock');
502 let rc = 0;
503 const fh = __openFiles[fid];
504 if( state.sq3Codes.SQLITE_LOCK_NONE===lockType
505 && fh.syncHandle ){
stephane72ddfd2022-10-14 15:52:29 +0000506 wTimeStart('xUnlock');
stephan9a557732022-10-04 17:06:51 +0000507 try { await closeSyncHandle(fh) }
508 catch(e){
509 state.s11n.storeException(1,e);
510 rc = state.sq3Codes.SQLITE_IOERR;
stephan9a557732022-10-04 17:06:51 +0000511 }
stephane72ddfd2022-10-14 15:52:29 +0000512 wTimeEnd();
stephan9a557732022-10-04 17:06:51 +0000513 }
514 storeAndNotify('xUnlock',rc);
515 mTimeEnd();
516 },
stephan138647a2022-09-20 03:31:02 +0000517 xWrite: async function(fid,n,offset){
stephanaec046a2022-09-19 18:22:29 +0000518 mTimeStart('xWrite');
stephan07315542022-09-17 20:50:12 +0000519 let rc;
stephan5e8bb0a2022-09-20 08:27:57 +0000520 wTimeStart('xWrite');
stephan07315542022-09-17 20:50:12 +0000521 try{
stephanaec046a2022-09-19 18:22:29 +0000522 const fh = __openFiles[fid];
stephan07315542022-09-17 20:50:12 +0000523 affirmNotRO('xWrite', fh);
stephanaec046a2022-09-19 18:22:29 +0000524 rc = (
stephan7ff8da82022-10-03 09:21:37 +0000525 n === (await getSyncHandle(fh))
526 .write(fh.sabView.subarray(0, n),
527 {at: Number(offset)})
stephanaec046a2022-09-19 18:22:29 +0000528 ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
stephan07315542022-09-17 20:50:12 +0000529 }catch(e){
530 error("xWrite():",e,fh);
stephan56fae742022-09-24 10:12:19 +0000531 state.s11n.storeException(1,e);
stephan07315542022-09-17 20:50:12 +0000532 rc = state.sq3Codes.SQLITE_IOERR_WRITE;
533 }
stephane72ddfd2022-10-14 15:52:29 +0000534 wTimeEnd();
stephan07315542022-09-17 20:50:12 +0000535 storeAndNotify('xWrite',rc);
stephanaec046a2022-09-19 18:22:29 +0000536 mTimeEnd();
stephan07315542022-09-17 20:50:12 +0000537 }
stephan9a557732022-10-04 17:06:51 +0000538}/*vfsAsyncImpls*/;
stephan07315542022-09-17 20:50:12 +0000539
stephan138647a2022-09-20 03:31:02 +0000540const initS11n = ()=>{
stephanb8c8d4e2022-09-20 13:25:39 +0000541 /**
stephan72ab4002022-09-21 12:27:35 +0000542 ACHTUNG: this code is 100% duplicated in the other half of this
543 proxy! The documentation is maintained in the "synchronous half".
stephanb8c8d4e2022-09-20 13:25:39 +0000544 */
stephan138647a2022-09-20 03:31:02 +0000545 if(state.s11n) return state.s11n;
stephanb8c8d4e2022-09-20 13:25:39 +0000546 const textDecoder = new TextDecoder(),
547 textEncoder = new TextEncoder('utf-8'),
548 viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize),
549 viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
stephan138647a2022-09-20 03:31:02 +0000550 state.s11n = Object.create(null);
stephanb8c8d4e2022-09-20 13:25:39 +0000551 const TypeIds = Object.create(null);
552 TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' };
553 TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' };
554 TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' };
555 TypeIds.string = { id: 4 };
stephan72ab4002022-09-21 12:27:35 +0000556 const getTypeId = (v)=>(
557 TypeIds[typeof v]
558 || toss("Maintenance required: this value type cannot be serialized.",v)
559 );
stephanb8c8d4e2022-09-20 13:25:39 +0000560 const getTypeIdById = (tid)=>{
561 switch(tid){
stephan72ab4002022-09-21 12:27:35 +0000562 case TypeIds.number.id: return TypeIds.number;
563 case TypeIds.bigint.id: return TypeIds.bigint;
564 case TypeIds.boolean.id: return TypeIds.boolean;
565 case TypeIds.string.id: return TypeIds.string;
566 default: toss("Invalid type ID:",tid);
stephanb8c8d4e2022-09-20 13:25:39 +0000567 }
568 };
stephan138647a2022-09-20 03:31:02 +0000569 state.s11n.deserialize = function(){
stephanb8c8d4e2022-09-20 13:25:39 +0000570 ++metrics.s11n.deserialize.count;
571 const t = performance.now();
stephanb8c8d4e2022-09-20 13:25:39 +0000572 const argc = viewU8[0];
stephan72ab4002022-09-21 12:27:35 +0000573 const rc = argc ? [] : null;
stephanb8c8d4e2022-09-20 13:25:39 +0000574 if(argc){
stephan72ab4002022-09-21 12:27:35 +0000575 const typeIds = [];
576 let offset = 1, i, n, v;
stephanb8c8d4e2022-09-20 13:25:39 +0000577 for(i = 0; i < argc; ++i, ++offset){
578 typeIds.push(getTypeIdById(viewU8[offset]));
579 }
580 for(i = 0; i < argc; ++i){
581 const t = typeIds[i];
582 if(t.getter){
583 v = viewDV[t.getter](offset, state.littleEndian);
584 offset += t.size;
stephan72ab4002022-09-21 12:27:35 +0000585 }else{/*String*/
stephanb8c8d4e2022-09-20 13:25:39 +0000586 n = viewDV.getInt32(offset, state.littleEndian);
587 offset += 4;
588 v = textDecoder.decode(viewU8.slice(offset, offset+n));
589 offset += n;
590 }
591 rc.push(v);
592 }
593 }
594 //log("deserialize:",argc, rc);
595 metrics.s11n.deserialize.time += performance.now() - t;
596 return rc;
597 };
stephan138647a2022-09-20 03:31:02 +0000598 state.s11n.serialize = function(...args){
stephanb8c8d4e2022-09-20 13:25:39 +0000599 const t = performance.now();
stephan72ab4002022-09-21 12:27:35 +0000600 ++metrics.s11n.serialize.count;
stephan5e8bb0a2022-09-20 08:27:57 +0000601 if(args.length){
stephanb8c8d4e2022-09-20 13:25:39 +0000602 //log("serialize():",args);
stephan72ab4002022-09-21 12:27:35 +0000603 const typeIds = [];
604 let i = 0, offset = 1;
605 viewU8[0] = args.length & 0xff /* header = # of args */;
stephanb8c8d4e2022-09-20 13:25:39 +0000606 for(; i < args.length; ++i, ++offset){
stephan72ab4002022-09-21 12:27:35 +0000607 /* Write the TypeIds.id value into the next args.length
608 bytes. */
stephanb8c8d4e2022-09-20 13:25:39 +0000609 typeIds.push(getTypeId(args[i]));
610 viewU8[offset] = typeIds[i].id;
611 }
612 for(i = 0; i < args.length; ++i) {
stephan72ab4002022-09-21 12:27:35 +0000613 /* Deserialize the following bytes based on their
614 corresponding TypeIds.id from the header. */
stephanb8c8d4e2022-09-20 13:25:39 +0000615 const t = typeIds[i];
616 if(t.setter){
617 viewDV[t.setter](offset, args[i], state.littleEndian);
618 offset += t.size;
stephan72ab4002022-09-21 12:27:35 +0000619 }else{/*String*/
stephanb8c8d4e2022-09-20 13:25:39 +0000620 const s = textEncoder.encode(args[i]);
621 viewDV.setInt32(offset, s.byteLength, state.littleEndian);
622 offset += 4;
623 viewU8.set(s, offset);
624 offset += s.byteLength;
625 }
626 }
627 //log("serialize() result:",viewU8.slice(0,offset));
stephan5e8bb0a2022-09-20 08:27:57 +0000628 }else{
stephanb8c8d4e2022-09-20 13:25:39 +0000629 viewU8[0] = 0;
stephan5e8bb0a2022-09-20 08:27:57 +0000630 }
stephanb8c8d4e2022-09-20 13:25:39 +0000631 metrics.s11n.serialize.time += performance.now() - t;
stephan138647a2022-09-20 03:31:02 +0000632 };
stephane8afca32022-09-21 14:02:47 +0000633
634 state.s11n.storeException = state.asyncS11nExceptions
stephan56fae742022-09-24 10:12:19 +0000635 ? ((priority,e)=>{
636 if(priority<=state.asyncS11nExceptions){
637 state.s11n.serialize(e.message);
638 }
639 })
stephane8afca32022-09-21 14:02:47 +0000640 : ()=>{};
641
stephan138647a2022-09-20 03:31:02 +0000642 return state.s11n;
stephanb8c8d4e2022-09-20 13:25:39 +0000643}/*initS11n()*/;
stephan138647a2022-09-20 03:31:02 +0000644
stephan5e8bb0a2022-09-20 08:27:57 +0000645const waitLoop = async function f(){
stephan138647a2022-09-20 03:31:02 +0000646 const opHandlers = Object.create(null);
stephanc9e26022022-09-20 10:11:52 +0000647 for(let k of Object.keys(state.opIds)){
648 const vi = vfsAsyncImpls[k];
649 if(!vi) continue;
stephan138647a2022-09-20 03:31:02 +0000650 const o = Object.create(null);
651 opHandlers[state.opIds[k]] = o;
652 o.key = k;
stephane72ddfd2022-10-14 15:52:29 +0000653 o.f = vi;
stephan138647a2022-09-20 03:31:02 +0000654 }
stephan5f0b67c2022-10-03 11:33:35 +0000655 /**
656 waitTime is how long (ms) to wait for each Atomics.wait().
stephanc7fb48d2022-10-04 09:12:05 +0000657 We need to wake up periodically to give the thread a chance
stephan5f0b67c2022-10-03 11:33:35 +0000658 to do other things.
659 */
stephan9a557732022-10-04 17:06:51 +0000660 const waitTime = 1000;
stephan3c272ba2022-10-04 00:54:00 +0000661 while(!flagAsyncShutdown){
stephan138647a2022-09-20 03:31:02 +0000662 try {
stephan5f0b67c2022-10-03 11:33:35 +0000663 if('timed-out'===Atomics.wait(
664 state.sabOPView, state.opIds.whichOp, 0, waitTime
665 )){
stephan5e8bb0a2022-09-20 08:27:57 +0000666 continue;
667 }
668 const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
669 Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
stephan138647a2022-09-20 03:31:02 +0000670 const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
stephan56fae742022-09-24 10:12:19 +0000671 const args = state.s11n.deserialize() || [];
stephan5f0b67c2022-10-03 11:33:35 +0000672 state.s11n.serialize(/* clear s11n to keep the caller from
673 confusing this with an exception string
674 written by the upcoming operation */);
stephan5e8bb0a2022-09-20 08:27:57 +0000675 //warn("waitLoop() whichOp =",opId, hnd, args);
676 if(hnd.f) await hnd.f(...args);
677 else error("Missing callback for opId",opId);
stephan138647a2022-09-20 03:31:02 +0000678 }catch(e){
stephan5f0b67c2022-10-03 11:33:35 +0000679 error('in waitLoop():',e);
stephan138647a2022-09-20 03:31:02 +0000680 }
stephane72ddfd2022-10-14 15:52:29 +0000681 }
stephan138647a2022-09-20 03:31:02 +0000682};
683
stephan07315542022-09-17 20:50:12 +0000684navigator.storage.getDirectory().then(function(d){
685 const wMsg = (type)=>postMessage({type});
686 state.rootDir = d;
stephan5e8bb0a2022-09-20 08:27:57 +0000687 self.onmessage = function({data}){
stephan07315542022-09-17 20:50:12 +0000688 switch(data.type){
stephan138647a2022-09-20 03:31:02 +0000689 case 'opfs-async-init':{
stephan07315542022-09-17 20:50:12 +0000690 /* Receive shared state from synchronous partner */
stephan138647a2022-09-20 03:31:02 +0000691 const opt = data.args;
692 state.littleEndian = opt.littleEndian;
stephane8afca32022-09-21 14:02:47 +0000693 state.asyncS11nExceptions = opt.asyncS11nExceptions;
stephan07315542022-09-17 20:50:12 +0000694 state.verbose = opt.verbose ?? 2;
695 state.fileBufferSize = opt.fileBufferSize;
stephan138647a2022-09-20 03:31:02 +0000696 state.sabS11nOffset = opt.sabS11nOffset;
697 state.sabS11nSize = opt.sabS11nSize;
stephanc4b87be2022-09-20 01:28:47 +0000698 state.sabOP = opt.sabOP;
699 state.sabOPView = new Int32Array(state.sabOP);
700 state.sabIO = opt.sabIO;
701 state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
stephan138647a2022-09-20 03:31:02 +0000702 state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize);
stephan07315542022-09-17 20:50:12 +0000703 state.opIds = opt.opIds;
704 state.sq3Codes = opt.sq3Codes;
705 Object.keys(vfsAsyncImpls).forEach((k)=>{
706 if(!Number.isFinite(state.opIds[k])){
707 toss("Maintenance required: missing state.opIds[",k,"]");
708 }
709 });
stephan138647a2022-09-20 03:31:02 +0000710 initS11n();
stephanaec046a2022-09-19 18:22:29 +0000711 metrics.reset();
stephan07315542022-09-17 20:50:12 +0000712 log("init state",state);
stephan138647a2022-09-20 03:31:02 +0000713 wMsg('opfs-async-inited');
stephan5e8bb0a2022-09-20 08:27:57 +0000714 waitLoop();
stephan07315542022-09-17 20:50:12 +0000715 break;
716 }
stephan3c272ba2022-10-04 00:54:00 +0000717 case 'opfs-async-restart':
718 if(flagAsyncShutdown){
719 warn("Restarting after opfs-async-shutdown. Might or might not work.");
720 flagAsyncShutdown = false;
721 waitLoop();
722 }
723 break;
724 case 'opfs-async-metrics':
725 metrics.dump();
726 break;
stephan132a87b2022-09-17 15:08:22 +0000727 }
728 };
stephan138647a2022-09-20 03:31:02 +0000729 wMsg('opfs-async-loaded');
stephan9a557732022-10-04 17:06:51 +0000730}).catch((e)=>error("error initializing OPFS asyncer:",e));