Optimize loading with multi-threaded worker loader

.trae/documents/Optimize Loading with Multi-Process Workers.md

@@ -0,0 +1,20 @@
+I will optimize the loading process by implementing a **Multi-Threaded Worker Loader**. This involves creating a pool of Web Workers to fetch assets (JavaScript, Audio, Images, Views) in parallel, offloading the network initiation and handling from the main thread.
+
+### Plan:
+1.  **Create `public/src/js/loader-worker.js`**:
+    *   This worker will handle `fetch` requests for different resource types (`text`, `blob`, `arraybuffer`).
+    *   It will transfer the data back to the main thread (using zero-copy transfer for `ArrayBuffer`).
+
+2.  **Modify `public/src/js/loader.js`**:
+    *   **Initialize Worker Pool**: Create a pool of workers (defaulting to 4) in the `Loader` class.
+    *   **Implement `workerFetch(url, type)`**: A method to distribute fetch tasks to the worker pool.
+    *   **Override `ajax(url, ...)`**: Intercept requests for static assets (`src/`, `assets/`, etc.) and route them through `workerFetch`. Keep API calls (`api/`) on the main thread to ensure session stability.
+    *   **Update `loadScript(url)`**: Change it to fetch the script content via `workerFetch` and inject it using a `<script>` tag with inline content. This ensures JS files are also loaded via the "multi-process" mechanism.
+    *   **Update `loadSound` and `RemoteFile` logic**: Since `RemoteFile` uses `loader.ajax`, routing `ajax` to workers will automatically parallelize audio loading.
+
+### Technical Details:
+*   **Concurrency**: 4 Workers will be used to maximize throughput without overloading the browser's connection limit per domain.
+*   **Resource Types**:
+    *   **JS/Views**: Fetched as `text`.
+    *   **Images**: Fetched as `blob` -> `URL.createObjectURL`.
+    *   **Audio**: Fetched as `arraybuffer` -> `AudioContext.decodeAudioData`.

public/src/js/loader-worker.js

@@ -0,0 +1,26 @@
+self.addEventListener('message', async e => {
+	const { id, url, type } = e.data
+	try{
+		const response = await fetch(url)
+		if(!response.ok){
+			throw new Error(response.status + " " + response.statusText)
+		}
+		let data
+		if(type === "arraybuffer"){
+			data = await response.arrayBuffer()
+		}else if(type === "blob"){
+			data = await response.blob()
+		}else{
+			data = await response.text()
+		}
+		self.postMessage({
+			id: id,
+			data: data
+		}, type === "arraybuffer" ? [data] : undefined)
+	}catch(e){
+		self.postMessage({
+			id: id,
+			error: e.toString()
+		})
+	}
+})

public/src/js/loader.js

@@ -11,6 +11,8 @@ class Loader{
 		this.errorMessages = []
 		this.songSearchGradient = "linear-gradient(to top, rgba(245, 246, 252, 0.08), #ff5963), "
 		
+		this.initWorkers()
+		
 		var promises = []
 		
 		promises.push(this.ajax("src/views/loader.html").then(page => {
@@ -23,6 +25,78 @@ class Loader{
 		
 		Promise.all(promises).then(this.run.bind(this))
 	}
+	initWorkers(){
+		this.workers = []
+		this.workerQueue = []
+		this.workerCallbacks = {}
+		this.workerId = 0
+		var concurrency = navigator.hardwareConcurrency || 4
+		for(var i = 0; i < concurrency; i++){
+			var worker = new Worker("src/js/loader-worker.js")
+			worker.onmessage = this.onWorkerMessage.bind(this)
+			this.workers.push({
+				worker: worker,
+				active: 0
+			})
+		}
+	}
+	onWorkerMessage(e){
+		var data = e.data
+		var callback = this.workerCallbacks[data.id]
+		if(callback){
+			delete this.workerCallbacks[data.id]
+			if(data.error){
+				callback.reject(data.error)
+			}else{
+				callback.resolve(data.data)
+			}
+			this.workerFree(callback.workerIndex)
+		}
+	}
+	workerFetch(url, type){
+		return new Promise((resolve, reject) => {
+			var id = ++this.workerId
+			this.workerQueue.push({
+				id: id,
+				url: new URL(url, location.href).href,
+				type: type,
+				resolve: resolve,
+				reject: reject
+			})
+			this.workerRun()
+		})
+	}
+	workerRun(){
+		if(this.workerQueue.length === 0){
+			return
+		}
+		var workerIndex = -1
+		var minActive = Infinity
+		for(var i = 0; i < this.workers.length; i++){
+			if(this.workers[i].active < minActive){
+				minActive = this.workers[i].active
+				workerIndex = i
+			}
+		}
+		if(workerIndex !== -1){
+			var task = this.workerQueue.shift()
+			var workerObj = this.workers[workerIndex]
+			workerObj.active++
+			this.workerCallbacks[task.id] = task
+			task.workerIndex = workerIndex
+			workerObj.worker.postMessage({
+				id: task.id,
+				url: task.url,
+				type: task.type
+			})
+		}
+	}
+	workerFree(index){
+		if(this.workers[index]){
+			this.workers[index].active--
+			this.workerRun()
+		}
+	}
 	run(){
 		this.promises = []
 		this.loaderDiv = document.querySelector("#loader")
@@ -538,6 +612,17 @@ class Loader{
 		return css.join("\n")
 	}
 	ajax(url, customRequest, customResponse){
+		if(!customResponse && (url.startsWith("src/") || url.startsWith("assets/") || url.indexOf("img/") !== -1 || url.indexOf("audio/") !== -1 || url.indexOf("fonts/") !== -1 || url.indexOf("views/") !== -1)){
+			var type = "text"
+			if(customRequest){
+				var reqStub = {}
+				customRequest(reqStub)
+				if(reqStub.responseType){
+					type = reqStub.responseType
+				}
+			}
+			return this.workerFetch(url, type)
+		}
 		var request = new XMLHttpRequest()
 		request.open("GET", url)
 		var promise = pageEvents.load(request)
@@ -557,12 +642,13 @@ class Loader{
 		return promise
 	}
 	loadScript(url){
-		var script = document.createElement("script")
 		var url = url + this.queryString
-		var promise = pageEvents.load(script)
-		script.src = url
+		return this.workerFetch(url, "text").then(code => {
+			var script = document.createElement("script")
+			code += "\n//# sourceURL=" + url
+			script.text = code
 			document.head.appendChild(script)
-		return promise
+		})
 	}
 	getCsrfToken(){
 		return this.ajax("api/csrftoken").then(response => {