fix(download): ensure rangefetcher loads before abstractfile; skip queryString for blob; add Range probe

app.py

@@ -28,7 +28,6 @@ import tjaf
 
 from functools import wraps
 from flask import Flask, g, jsonify, render_template, request, abort, redirect, session, flash, make_response, send_from_directory
-import mimetypes
 from flask_caching import Cache
 from flask_session import Session
 from flask_wtf.csrf import CSRFProtect, generate_csrf, CSRFError
@@ -66,30 +65,37 @@ def get_remote_address() -> str:
 limiter = Limiter(
     get_remote_address,
     app=app,
-    strategy="fixed-window",
-    storage_uri=os.environ.get("REDIS_URI") or "memory://",
+    # default_limits=[],
+    # storage_uri="memory://",
+    # Redis
+    storage_uri=os.environ.get("REDIS_URI", "redis://127.0.0.1:6379/"),
+    # Redis cluster
+    # storage_uri="redis+cluster://localhost:7000,localhost:7001,localhost:70002",
+    # Memcached
+    # storage_uri="memcached://localhost:11211",
+    # Memcached Cluster
+    # storage_uri="memcached://localhost:11211,localhost:11212,localhost:11213",
+    # MongoDB
+    # storage_uri="mongodb://localhost:27017",
+    # Etcd
+    # storage_uri="etcd://localhost:2379",
+    strategy="fixed-window", # or "moving-window"
 )
 
 client = MongoClient(host=os.environ.get("TAIKO_WEB_MONGO_HOST") or take_config('MONGO', required=True)['host'])
 basedir = take_config('BASEDIR') or '/'
 
 app.secret_key = take_config('SECRET_KEY') or 'change-me'
+app.config['SESSION_TYPE'] = 'redis'
 redis_config = take_config('REDIS', required=True)
 redis_config['CACHE_REDIS_HOST'] = os.environ.get("TAIKO_WEB_REDIS_HOST") or redis_config['CACHE_REDIS_HOST']
-try:
-    _r = Redis(
-        host=redis_config['CACHE_REDIS_HOST'],
-        port=redis_config['CACHE_REDIS_PORT'],
-        password=redis_config['CACHE_REDIS_PASSWORD'],
-        db=redis_config['CACHE_REDIS_DB']
-    )
-    _r.ping()
-    app.config['SESSION_TYPE'] = 'redis'
-    app.config['SESSION_REDIS'] = _r
-    app.cache = Cache(app, config=redis_config)
-except Exception:
-    app.config['SESSION_TYPE'] = 'filesystem'
-    app.cache = Cache(app, config={'CACHE_TYPE': 'SimpleCache'})
+app.config['SESSION_REDIS'] = Redis(
+    host=redis_config['CACHE_REDIS_HOST'],
+    port=redis_config['CACHE_REDIS_PORT'],
+    password=redis_config['CACHE_REDIS_PASSWORD'],
+    db=redis_config['CACHE_REDIS_DB']
+)
+app.cache = Cache(app, config=redis_config)
 sess = Session()
 sess.init_app(app)
 #csrf = CSRFProtect(app)
@@ -800,33 +806,7 @@ def send_assets(ref):
 
 @app.route(basedir + "songs/<path:ref>")
 def send_songs(ref):
-    path = os.path.normpath(os.path.join("public/songs", ref))
-    if not os.path.isfile(path):
-        return abort(404)
-    rng = flask.request.headers.get("Range")
-    if rng and re.match(r"^bytes=\d+-\d*$", rng):
-        size = os.path.getsize(path)
-        m = re.match(r"bytes=(\d+)-(\d*)", rng)
-        start = int(m.group(1))
-        end = int(m.group(2) or size - 1)
-        start = max(0, start)
-        end = min(size - 1, end)
-        with open(path, "rb") as f:
-            f.seek(start)
-            data = f.read(end - start + 1)
-        mime = mimetypes.guess_type(path)[0] or "application/octet-stream"
-        resp = flask.Response(data, 206, mimetype=mime)
-        resp.headers["Content-Range"] = f"bytes {start}-{end}/{size}"
-        resp.headers["Accept-Ranges"] = "bytes"
-        resp.headers["Cache-Control"] = "public, max-age=604800, s-maxage=604800"
-        resp.headers["CDN-Cache-Control"] = "max-age=604800"
-        return resp
-    resp = flask.send_from_directory("public/songs", ref)
-    res = flask.make_response(resp)
-    res.headers["Accept-Ranges"] = "bytes"
-    res.headers["Cache-Control"] = "public, max-age=604800, s-maxage=604800"
-    res.headers["CDN-Cache-Control"] = "max-age=604800"
-    return res
+    return cache_wrap(flask.send_from_directory("public/songs", ref), 604800)
 
 @app.route(basedir + "manifest.json")
 def send_manifest():

public/src/js/abstractfile.js

@@ -47,70 +47,33 @@ class RemoteFile{
 		}
 	}
 	arrayBuffer(){
-		return loader.ajax(this.url, request => {
-			request.responseType = "arraybuffer"
-		})
-	}
-	arrayBufferFast(threads){
-		var t = threads || 4
-		var head = new XMLHttpRequest()
-		head.open("HEAD", this.url)
-		var headPromise = pageEvents.load(head).then(() => {
-			if(head.status !== 200){
-				return Promise.reject()
-			}
-			var len = parseInt(head.getResponseHeader("Content-Length"))
-			if(!len || len < 262144){
-				return this.arrayBuffer()
-			}
-			var chunk = Math.ceil(len / t)
-			var ranges = []
-			for(var i = 0; i < t; i++){
-				var start = i * chunk
-				var end = Math.min(len - 1, (i + 1) * chunk - 1)
-				if(start > end){ break }
-				ranges.push([start, end])
-			}
-			var promises = ranges.map(r => {
-				var req = new XMLHttpRequest()
-				req.open("GET", this.url)
-				req.responseType = "arraybuffer"
-				req.setRequestHeader("Range", "bytes=" + r[0] + "-" + r[1])
-				return pageEvents.load(req).then(() => {
-					if(req.status !== 206 && req.status !== 200){
-						return Promise.reject()
-					}
-					return req.response
-				})
-			})
-			return Promise.all(promises).then(parts => {
-				var total = 0
-				for(var i = 0; i < parts.length; i++){
-					total += parts[i].byteLength
-				}
-				var out = new Uint8Array(total)
-				var offset = 0
-				for(var i = 0; i < parts.length; i++){
-					out.set(new Uint8Array(parts[i]), offset)
-					offset += parts[i].byteLength
-				}
-				return out.buffer
-			})
-		})
-		head.send()
-		return headPromise.catch(() => this.arrayBuffer())
+		var name = this.name.toLowerCase()
+		if(name.endsWith(".ogg") || name.endsWith(".png") || name.endsWith(".gif") || name.endsWith(".tja") || name.endsWith(".osu")){
+			return RangeFetcher.fetchArrayBuffer(this.url, {concurrency: 4, chunkSize: 1048576}).catch(() => loader.ajax(this.url, request => { request.responseType = "arraybuffer" }))
+		}
+		return loader.ajax(this.url, request => { request.responseType = "arraybuffer" })
 	}
 	read(encoding){
 		if(encoding){
+			var name = this.name.toLowerCase()
+			if(name.endsWith(".tja") || name.endsWith(".osu")){
+				return RangeFetcher.fetchText(this.url, {concurrency: 4, chunkSize: 262144}).catch(() => this.blob().then(blob => readFile(blob, false, encoding)))
+			}
 			return this.blob().then(blob => readFile(blob, false, encoding))
 		}else{
+			var name = this.name.toLowerCase()
+			if(name.endsWith(".tja") || name.endsWith(".osu")){
+				return RangeFetcher.fetchText(this.url, {concurrency: 4, chunkSize: 262144}).catch(() => loader.ajax(this.url))
+			}
 			return loader.ajax(this.url)
 		}
 	}
 	blob(){
-		return loader.ajax(this.url, request => {
-			request.responseType = "blob"
-		})
+		var name = this.name.toLowerCase()
+		if(name.endsWith(".ogg") || name.endsWith(".png") || name.endsWith(".gif")){
+			return RangeFetcher.fetchBlob(this.url, {concurrency: 4, chunkSize: 1048576}).catch(() => loader.ajax(this.url, request => { request.responseType = "blob" }))
+		}
+		return loader.ajax(this.url, request => { request.responseType = "blob" })
 	}
 }
 class LocalFile{

public/src/js/assets.js

@@ -36,6 +36,7 @@ var assets = {
 		"account.js",
 		"lyrics.js",
 		"customsongs.js",
+		"rangefetcher.js",
 		"abstractfile.js",
 		"idb.js",
 		"plugins.js",

public/src/js/loader.js

@@ -23,8 +23,8 @@ class Loader{
 		
 		Promise.all(promises).then(this.run.bind(this))
 	}
-	run(){
-		this.promises = []
+		run(){
+			this.promises = []
 		this.loaderDiv = document.querySelector("#loader")
 		this.loaderPercentage = document.querySelector("#loader .percentage")
 		this.loaderProgress = document.querySelector("#loader .progress")
@@ -35,12 +35,10 @@ class Loader{
 			this.addPromise(this.loadScript(gameConfig.custom_js), gameConfig.custom_js)
 		}
 		var oggSupport = new Audio().canPlayType("audio/ogg;codecs=vorbis")
-		if(!oggSupport){
-			assets.js.push("lib/oggmented-wasm.js")
-		}
-		assets.js.forEach(name => {
-			this.addPromise(this.loadScript("src/js/" + name), "src/js/" + name)
-		})
+			if(!oggSupport){
+				assets.js.push("lib/oggmented-wasm.js")
+			}
+			this.addPromise(this.prefetchAndLoadScripts(assets.js), "src/js/*")
 		
 		var pageVersion = versionLink.href
 		var index = pageVersion.lastIndexOf("/")
@@ -55,23 +53,21 @@ class Loader{
 			){
 				reject("Version on the page and config does not match\n(page:  " + pageVersion + ",\nconfig: "+ gameConfig._version.commit + ")")
 			}
-			var loaded = 0
-			var total = assets.css.length
+			var cssCount = document.styleSheets.length + assets.css.length
 			assets.css.forEach(name => {
 				var stylesheet = document.createElement("link")
 				stylesheet.rel = "stylesheet"
 				stylesheet.href = "src/css/" + name + this.queryString
-				stylesheet.addEventListener("load", () => {
-					loaded++
-					if(loaded >= total){
-						resolve()
-					}
-				})
-				stylesheet.addEventListener("error", () => {
-					reject("src/css/" + name)
-				})
 				document.head.appendChild(stylesheet)
 			})
+			var checkStyles = () => {
+				if(document.styleSheets.length >= cssCount){
+					resolve()
+					clearInterval(interval)
+				}
+			}
+			var interval = setInterval(checkStyles, 100)
+			checkStyles()
 		}))
 		
 		for(var name in assets.fonts){
@@ -401,6 +397,26 @@ class Loader{
 			}, e => this.errorMsg(e))
 		})
 	}
+	prefetchAndLoadScripts(names){
+		var urls = names.map(name => "src/js/" + name + this.queryString)
+		var fetches = urls.map(url => this.ajax(url, r => { r.responseType = "arraybuffer" }))
+		return Promise.all(fetches).then(buffers => {
+			var chain = Promise.resolve()
+			for(let i=0;i<buffers.length;i++){
+				chain = chain.then(() => {
+					var blob = new Blob([buffers[i]], {type: "application/javascript"})
+					var u = URL.createObjectURL(blob)
+					return this.loadScript(u).then(() => { URL.revokeObjectURL(u) })
+				})
+			}
+			return chain
+		}).catch(() => {
+			names.forEach(name => {
+				this.addPromise(this.loadScript("src/js/" + name), "src/js/" + name)
+			})
+			return Promise.resolve()
+		})
+	}
 	addPromise(promise, url){
 		this.promises.push(promise)
 		promise.then(this.assetLoaded.bind(this), response => {
@@ -542,7 +558,6 @@ class Loader{
 	ajax(url, customRequest, customResponse){
 		var request = new XMLHttpRequest()
 		request.open("GET", url)
-		request.timeout = 15000
 		var promise = pageEvents.load(request)
 		if(!customResponse){
 			promise = promise.then(() => {
@@ -561,7 +576,8 @@ class Loader{
 	}
 	loadScript(url){
 		var script = document.createElement("script")
-		var url = url + this.queryString
+		var isBlob = typeof url === "string" && (url.startsWith("blob:") || url.startsWith("data:"))
+		var url = isBlob ? url : (url + this.queryString)
 		var promise = pageEvents.load(script)
 		script.src = url
 		document.head.appendChild(script)

public/src/js/pageevents.js

@@ -76,11 +76,7 @@ class PageEvents{
 	}
 	load(target){
 		return new Promise((resolve, reject) => {
-			var timer = setTimeout(() => {
-				reject(["Loading timeout", target])
-			}, 15000)
-			this.race(target, "load", "error", "abort", "timeout").then(response => {
-				clearTimeout(timer)
+			this.race(target, "load", "error", "abort").then(response => {
 				switch(response.type){
 					case "load":
 						return resolve(response.event)
@@ -88,8 +84,6 @@ class PageEvents{
 						return reject(["Loading error", target])
 					case "abort":
 						return reject("Loading aborted")
-					case "timeout":
-						return reject(["Loading timeout", target])
 				}
 			})
 		})

public/src/js/rangefetcher.js

@@ -0,0 +1,130 @@
+var RangeFetcher = {
+    _head: function(url){
+        return new Promise(function(resolve){
+            var r = new XMLHttpRequest()
+            r.open("HEAD", url)
+            r.onreadystatechange = function(){
+                if(r.readyState === 4){
+                    var len = parseInt(r.getResponseHeader("Content-Length")) || null
+                    var ranges = r.getResponseHeader("Accept-Ranges") || ""
+                    resolve({length: len, acceptRanges: /bytes/i.test(ranges)})
+                }
+            }
+            r.onerror = function(){ resolve({length: null, acceptRanges: false}) }
+            r.send()
+        })
+    },
+    _getRange: function(url, start, end, responseType){
+        return new Promise(function(resolve, reject){
+            var r = new XMLHttpRequest()
+            r.open("GET", url)
+            r.responseType = responseType || "arraybuffer"
+            r.setRequestHeader("Range", "bytes=" + start + "-" + end)
+            r.onload = function(){
+                if(r.status === 206 || r.status === 200){
+                    resolve(r.response)
+                }else{
+                    reject(r.status)
+                }
+            }
+            r.onerror = function(){ reject("network") }
+            r.send()
+        })
+    },
+    _getAll: function(url, responseType){
+        return new Promise(function(resolve, reject){
+            var r = new XMLHttpRequest()
+            r.open("GET", url)
+            r.responseType = responseType || "arraybuffer"
+            r.onload = function(){
+                if(r.status === 200){
+                    resolve(r.response)
+                }else{
+                    reject(r.status)
+                }
+            }
+            r.onerror = function(){ reject("network") }
+            r.send()
+        })
+    },
+    _probeRange: function(url){
+        return new Promise(function(resolve){
+            var r = new XMLHttpRequest()
+            r.open("GET", url)
+            r.responseType = "arraybuffer"
+            r.setRequestHeader("Range", "bytes=0-0")
+            r.onload = function(){
+                var ok = r.status === 206 && !!r.getResponseHeader("Content-Range")
+                resolve(ok)
+            }
+            r.onerror = function(){ resolve(false) }
+            r.send()
+        })
+    },
+    _runConcurrent: function(tasks, limit){
+        return new Promise(function(resolve, reject){
+            var i = 0
+            var running = 0
+            var results = []
+            var done = 0
+            function next(){
+                while(running < limit && i < tasks.length){
+                    var idx = i++
+                    running++
+                    tasks[idx]().then(function(res){
+                        results[idx] = res
+                        running--
+                        done++
+                        if(done === tasks.length){ resolve(results) }
+                        else{ next() }
+                    }, function(e){ reject(e) })
+                }
+            }
+            next()
+        })
+    },
+    fetchArrayBuffer: function(url, opts){
+        opts = opts || {}
+        var concurrency = opts.concurrency || 4
+        var minChunk = opts.chunkSize || 1048576
+        var self = this
+        return this._head(url).then(function(info){
+            if(!info.length || info.length < minChunk){
+                return self._getAll(url, "arraybuffer")
+            }
+            if(!info.acceptRanges){
+                return self._probeRange(url).then(function(supported){
+                    if(!supported){
+                        return self._getAll(url, "arraybuffer")
+                    }
+                    return supported
+                })
+            }
+            var total = info.length
+            var parts = Math.ceil(total / minChunk)
+            parts = Math.max(1, Math.min(parts, 16))
+            var tasks = []
+            for(var p=0; p<parts; p++){
+                var start = p * Math.floor(total / parts)
+                var end = p === parts - 1 ? total - 1 : ( (p+1) * Math.floor(total / parts) ) - 1
+                tasks.push(function(s, e){ return function(){ return self._getRange(url, s, e, "arraybuffer") } }(start, end))
+            }
+            return self._runConcurrent(tasks, concurrency).then(function(buffers){
+                var out = new Uint8Array(total)
+                var offset = 0
+                for(var p=0; p<buffers.length; p++){
+                    var b = new Uint8Array(buffers[p])
+                    out.set(b, offset)
+                    offset += b.byteLength
+                }
+                return out.buffer
+            })
+        })
+    },
+    fetchBlob: function(url, opts){
+        return this.fetchArrayBuffer(url, opts).then(function(buf){ return new Blob([buf]) })
+    },
+    fetchText: function(url, opts){
+        return this.fetchBlob(url, opts).then(function(blob){ return blob.text() })
+    }
+}

public/src/js/songselect.js

@@ -2965,6 +2965,12 @@ class SongSelect{
 			var categoryName = song.category
 			var originalCategory = song.category
 		}
+		if(!categoryName){
+			if(song.song_type){
+				categoryName = song.song_type
+				originalCategory = song.song_type
+			}
+		}
 		var addedSong = {
 			title: title,
 			originalTitle: song.title,

public/src/js/soundbuffer.js

@@ -1,4 +1,4 @@
-class SoundBuffer{
+class SoundBuffer{
 	constructor(...args){
 		this.init(...args)
 	}
@@ -12,8 +12,7 @@ class SoundBuffer{
 	}
 	load(file, gain){
 		var decoder = file.name.endsWith(".ogg") ? this.oggDecoder : this.audioDecoder
-		var promise = typeof file.arrayBufferFast === "function" ? file.arrayBufferFast(4) : file.arrayBuffer()
-		return promise.then(response => {
+		return file.arrayBuffer().then(response => {
 			return new Promise((resolve, reject) => {
 				return decoder(response, resolve, reject)
 			}).catch(error => Promise.reject([error, file.url]))

tjaf.py

@@ -7,6 +7,8 @@ class Tja:
         self.text = text
         self.title: Optional[str] = None
         self.subtitle: Optional[str] = None
+        self.title_ja: Optional[str] = None
+        self.subtitle_ja: Optional[str] = None
         self.wave: Optional[str] = None
         self.offset: Optional[float] = None
         self.courses: Dict[str, Dict[str, Optional[int]]] = {}
@@ -25,8 +27,12 @@ class Tja:
                 val = v.strip()
                 if key == "TITLE":
                     self.title = val or None
+                elif key == "TITLEJA":
+                    self.title_ja = val or None
                 elif key == "SUBTITLE":
                     self.subtitle = val or None
+                elif key == "SUBTITLEJA":
+                    self.subtitle_ja = val or None
                 elif key == "WAVE":
                     self.wave = val or None
                 elif key == "OFFSET":
@@ -73,8 +79,20 @@ class Tja:
             "type": "tja",
             "title": self.title,
             "subtitle": self.subtitle,
-            "title_lang": {"ja": self.title, "en": None, "cn": None, "tw": None, "ko": None},
-            "subtitle_lang": {"ja": self.subtitle, "en": None, "cn": None, "tw": None, "ko": None},
+            "title_lang": {
+                "ja": self.title_ja or self.title,
+                "en": None,
+                "cn": self.title_ja or None,
+                "tw": None,
+                "ko": None,
+            },
+            "subtitle_lang": {
+                "ja": self.subtitle_ja or self.subtitle,
+                "en": None,
+                "cn": self.subtitle_ja or None,
+                "tw": None,
+                "ko": None,
+            },
             "courses": courses_out,
             "enabled": False,
             "category_id": None,

update.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [ "${EUID}" -ne 0 ]; then echo "需要 root 权限"; exit 1; fi
+
+SRC_DIR=$(cd "$(dirname "$0")" && pwd)
+DEST_DIR=/srv/taiko-web
+SONGS_DIR="$DEST_DIR/public/songs"
+BACKUP_DIR="$DEST_DIR/.backup_songs_$(date +%Y%m%d_%H%M%S)"
+
+systemctl stop taiko-web || true
+
+if [ -d "$SONGS_DIR" ]; then
+  mkdir -p "$BACKUP_DIR"
+  rsync -a "$SONGS_DIR/" "$BACKUP_DIR/" || cp -a "$SONGS_DIR/." "$BACKUP_DIR/"
+fi
+
+mkdir -p "$DEST_DIR"
+rsync -a --delete \
+  --exclude '.git' \
+  --exclude '.venv' \
+  --exclude 'public/songs' \
+  "$SRC_DIR/" "$DEST_DIR/"
+
+if [ -x "$DEST_DIR/.venv/bin/pip" ]; then
+  "$DEST_DIR/.venv/bin/pip" install -U pip
+  "$DEST_DIR/.venv/bin/pip" install -r "$DEST_DIR/requirements.txt"
+fi
+
+chown -R www-data:www-data "$DEST_DIR"
+
+if [ -d "$BACKUP_DIR" ]; then
+  mkdir -p "$SONGS_DIR"
+  rsync -a "$BACKUP_DIR/" "$SONGS_DIR/" || cp -a "$BACKUP_DIR/." "$SONGS_DIR/"
+fi
+
+systemctl daemon-reload || true
+systemctl restart taiko-web || systemctl start taiko-web || true
+
+systemctl is-active --quiet taiko-web