fix(loader): add 15s timeout and CSS load events to prevent 97% hang; show diagnostics on failure

app.py

@@ -28,6 +28,7 @@ import tjaf
 
 from functools import wraps
 from flask import Flask, g, jsonify, render_template, request, abort, redirect, session, flash, make_response, send_from_directory
+import mimetypes
 from flask_caching import Cache
 from flask_session import Session
 from flask_wtf.csrf import CSRFProtect, generate_csrf, CSRFError
@@ -65,37 +66,30 @@ def get_remote_address() -> str:
 limiter = Limiter(
     get_remote_address,
     app=app,
-    # default_limits=[],
-    # storage_uri="memory://",
-    # Redis
-    storage_uri=os.environ.get("REDIS_URI", "redis://127.0.0.1:6379/"),
-    # Redis cluster
-    # storage_uri="redis+cluster://localhost:7000,localhost:7001,localhost:70002",
-    # Memcached
-    # storage_uri="memcached://localhost:11211",
-    # Memcached Cluster
-    # storage_uri="memcached://localhost:11211,localhost:11212,localhost:11213",
-    # MongoDB
-    # storage_uri="mongodb://localhost:27017",
-    # Etcd
-    # storage_uri="etcd://localhost:2379",
-    strategy="fixed-window", # or "moving-window"
+    strategy="fixed-window",
+    storage_uri=os.environ.get("REDIS_URI") or "memory://",
 )
 
 client = MongoClient(host=os.environ.get("TAIKO_WEB_MONGO_HOST") or take_config('MONGO', required=True)['host'])
 basedir = take_config('BASEDIR') or '/'
 
 app.secret_key = take_config('SECRET_KEY') or 'change-me'
-app.config['SESSION_TYPE'] = 'redis'
 redis_config = take_config('REDIS', required=True)
 redis_config['CACHE_REDIS_HOST'] = os.environ.get("TAIKO_WEB_REDIS_HOST") or redis_config['CACHE_REDIS_HOST']
-app.config['SESSION_REDIS'] = Redis(
-    host=redis_config['CACHE_REDIS_HOST'],
-    port=redis_config['CACHE_REDIS_PORT'],
-    password=redis_config['CACHE_REDIS_PASSWORD'],
-    db=redis_config['CACHE_REDIS_DB']
-)
-app.cache = Cache(app, config=redis_config)
+try:
+    _r = Redis(
+        host=redis_config['CACHE_REDIS_HOST'],
+        port=redis_config['CACHE_REDIS_PORT'],
+        password=redis_config['CACHE_REDIS_PASSWORD'],
+        db=redis_config['CACHE_REDIS_DB']
+    )
+    _r.ping()
+    app.config['SESSION_TYPE'] = 'redis'
+    app.config['SESSION_REDIS'] = _r
+    app.cache = Cache(app, config=redis_config)
+except Exception:
+    app.config['SESSION_TYPE'] = 'filesystem'
+    app.cache = Cache(app, config={'CACHE_TYPE': 'SimpleCache'})
 sess = Session()
 sess.init_app(app)
 #csrf = CSRFProtect(app)
@@ -806,7 +800,33 @@ def send_assets(ref):
 
 @app.route(basedir + "songs/<path:ref>")
 def send_songs(ref):
-    return cache_wrap(flask.send_from_directory("public/songs", ref), 604800)
+    path = os.path.normpath(os.path.join("public/songs", ref))
+    if not os.path.isfile(path):
+        return abort(404)
+    rng = flask.request.headers.get("Range")
+    if rng and re.match(r"^bytes=\d+-\d*$", rng):
+        size = os.path.getsize(path)
+        m = re.match(r"bytes=(\d+)-(\d*)", rng)
+        start = int(m.group(1))
+        end = int(m.group(2) or size - 1)
+        start = max(0, start)
+        end = min(size - 1, end)
+        with open(path, "rb") as f:
+            f.seek(start)
+            data = f.read(end - start + 1)
+        mime = mimetypes.guess_type(path)[0] or "application/octet-stream"
+        resp = flask.Response(data, 206, mimetype=mime)
+        resp.headers["Content-Range"] = f"bytes {start}-{end}/{size}"
+        resp.headers["Accept-Ranges"] = "bytes"
+        resp.headers["Cache-Control"] = "public, max-age=604800, s-maxage=604800"
+        resp.headers["CDN-Cache-Control"] = "max-age=604800"
+        return resp
+    resp = flask.send_from_directory("public/songs", ref)
+    res = flask.make_response(resp)
+    res.headers["Accept-Ranges"] = "bytes"
+    res.headers["Cache-Control"] = "public, max-age=604800, s-maxage=604800"
+    res.headers["CDN-Cache-Control"] = "max-age=604800"
+    return res
 
 @app.route(basedir + "manifest.json")
 def send_manifest():

public/src/js/abstractfile.js

@@ -51,6 +51,55 @@ class RemoteFile{
 			request.responseType = "arraybuffer"
 		})
 	}
+	arrayBufferFast(threads){
+		var t = threads || 4
+		var head = new XMLHttpRequest()
+		head.open("HEAD", this.url)
+		var headPromise = pageEvents.load(head).then(() => {
+			if(head.status !== 200){
+				return Promise.reject()
+			}
+			var len = parseInt(head.getResponseHeader("Content-Length"))
+			if(!len || len < 262144){
+				return this.arrayBuffer()
+			}
+			var chunk = Math.ceil(len / t)
+			var ranges = []
+			for(var i = 0; i < t; i++){
+				var start = i * chunk
+				var end = Math.min(len - 1, (i + 1) * chunk - 1)
+				if(start > end){ break }
+				ranges.push([start, end])
+			}
+			var promises = ranges.map(r => {
+				var req = new XMLHttpRequest()
+				req.open("GET", this.url)
+				req.responseType = "arraybuffer"
+				req.setRequestHeader("Range", "bytes=" + r[0] + "-" + r[1])
+				return pageEvents.load(req).then(() => {
+					if(req.status !== 206 && req.status !== 200){
+						return Promise.reject()
+					}
+					return req.response
+				})
+			})
+			return Promise.all(promises).then(parts => {
+				var total = 0
+				for(var i = 0; i < parts.length; i++){
+					total += parts[i].byteLength
+				}
+				var out = new Uint8Array(total)
+				var offset = 0
+				for(var i = 0; i < parts.length; i++){
+					out.set(new Uint8Array(parts[i]), offset)
+					offset += parts[i].byteLength
+				}
+				return out.buffer
+			})
+		})
+		head.send()
+		return headPromise.catch(() => this.arrayBuffer())
+	}
 	read(encoding){
 		if(encoding){
 			return this.blob().then(blob => readFile(blob, false, encoding))

public/src/js/loader.js

@@ -55,21 +55,23 @@ class Loader{
 			){
 				reject("Version on the page and config does not match\n(page:  " + pageVersion + ",\nconfig: "+ gameConfig._version.commit + ")")
 			}
-			var cssCount = document.styleSheets.length + assets.css.length
+			var loaded = 0
+			var total = assets.css.length
 			assets.css.forEach(name => {
 				var stylesheet = document.createElement("link")
 				stylesheet.rel = "stylesheet"
 				stylesheet.href = "src/css/" + name + this.queryString
+				stylesheet.addEventListener("load", () => {
+					loaded++
+					if(loaded >= total){
+						resolve()
+					}
+				})
+				stylesheet.addEventListener("error", () => {
+					reject("src/css/" + name)
+				})
 				document.head.appendChild(stylesheet)
 			})
-			var checkStyles = () => {
-				if(document.styleSheets.length >= cssCount){
-					resolve()
-					clearInterval(interval)
-				}
-			}
-			var interval = setInterval(checkStyles, 100)
-			checkStyles()
 		}))
 		
 		for(var name in assets.fonts){
@@ -540,6 +542,7 @@ class Loader{
 	ajax(url, customRequest, customResponse){
 		var request = new XMLHttpRequest()
 		request.open("GET", url)
+		request.timeout = 15000
 		var promise = pageEvents.load(request)
 		if(!customResponse){
 			promise = promise.then(() => {

public/src/js/pageevents.js

@@ -76,7 +76,11 @@ class PageEvents{
 	}
 	load(target){
 		return new Promise((resolve, reject) => {
-			this.race(target, "load", "error", "abort").then(response => {
+			var timer = setTimeout(() => {
+				reject(["Loading timeout", target])
+			}, 15000)
+			this.race(target, "load", "error", "abort", "timeout").then(response => {
+				clearTimeout(timer)
 				switch(response.type){
 					case "load":
 						return resolve(response.event)
@@ -84,6 +88,8 @@ class PageEvents{
 						return reject(["Loading error", target])
 					case "abort":
 						return reject("Loading aborted")
+					case "timeout":
+						return reject(["Loading timeout", target])
 				}
 			})
 		})

public/src/js/songselect.js

@@ -2965,12 +2965,6 @@ class SongSelect{
 			var categoryName = song.category
 			var originalCategory = song.category
 		}
-		if(!categoryName){
-			if(song.song_type){
-				categoryName = song.song_type
-				originalCategory = song.song_type
-			}
-		}
 		var addedSong = {
 			title: title,
 			originalTitle: song.title,

public/src/js/soundbuffer.js

@@ -1,4 +1,4 @@
-class SoundBuffer{
+class SoundBuffer{
 	constructor(...args){
 		this.init(...args)
 	}
@@ -12,7 +12,8 @@
 	}
 	load(file, gain){
 		var decoder = file.name.endsWith(".ogg") ? this.oggDecoder : this.audioDecoder
-		return file.arrayBuffer().then(response => {
+		var promise = typeof file.arrayBufferFast === "function" ? file.arrayBufferFast(4) : file.arrayBuffer()
+		return promise.then(response => {
 			return new Promise((resolve, reject) => {
 				return decoder(response, resolve, reject)
 			}).catch(error => Promise.reject([error, file.url]))

tjaf.py

@@ -7,8 +7,6 @@ class Tja:
         self.text = text
         self.title: Optional[str] = None
         self.subtitle: Optional[str] = None
-        self.title_ja: Optional[str] = None
-        self.subtitle_ja: Optional[str] = None
         self.wave: Optional[str] = None
         self.offset: Optional[float] = None
         self.courses: Dict[str, Dict[str, Optional[int]]] = {}
@@ -27,12 +25,8 @@ class Tja:
                 val = v.strip()
                 if key == "TITLE":
                     self.title = val or None
-                elif key == "TITLEJA":
-                    self.title_ja = val or None
                 elif key == "SUBTITLE":
                     self.subtitle = val or None
-                elif key == "SUBTITLEJA":
-                    self.subtitle_ja = val or None
                 elif key == "WAVE":
                     self.wave = val or None
                 elif key == "OFFSET":
@@ -79,20 +73,8 @@ class Tja:
             "type": "tja",
             "title": self.title,
             "subtitle": self.subtitle,
-            "title_lang": {
-                "ja": self.title_ja or self.title,
-                "en": None,
-                "cn": self.title_ja or None,
-                "tw": None,
-                "ko": None,
-            },
-            "subtitle_lang": {
-                "ja": self.subtitle_ja or self.subtitle,
-                "en": None,
-                "cn": self.subtitle_ja or None,
-                "tw": None,
-                "ko": None,
-            },
+            "title_lang": {"ja": self.title, "en": None, "cn": None, "tw": None, "ko": None},
+            "subtitle_lang": {"ja": self.subtitle, "en": None, "cn": None, "tw": None, "ko": None},
             "courses": courses_out,
             "enabled": False,
             "category_id": None,

update.sh

@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-set -Eeuo pipefail
-
-if [ "${EUID}" -ne 0 ]; then echo "需要 root 权限"; exit 1; fi
-
-SRC_DIR=$(cd "$(dirname "$0")" && pwd)
-DEST_DIR=/srv/taiko-web
-SONGS_DIR="$DEST_DIR/public/songs"
-BACKUP_DIR="$DEST_DIR/.backup_songs_$(date +%Y%m%d_%H%M%S)"
-
-systemctl stop taiko-web || true
-
-if [ -d "$SONGS_DIR" ]; then
-  mkdir -p "$BACKUP_DIR"
-  rsync -a "$SONGS_DIR/" "$BACKUP_DIR/" || cp -a "$SONGS_DIR/." "$BACKUP_DIR/"
-fi
-
-mkdir -p "$DEST_DIR"
-rsync -a --delete \
-  --exclude '.git' \
-  --exclude '.venv' \
-  --exclude 'public/songs' \
-  "$SRC_DIR/" "$DEST_DIR/"
-
-if [ -x "$DEST_DIR/.venv/bin/pip" ]; then
-  "$DEST_DIR/.venv/bin/pip" install -U pip
-  "$DEST_DIR/.venv/bin/pip" install -r "$DEST_DIR/requirements.txt"
-fi
-
-chown -R www-data:www-data "$DEST_DIR"
-
-if [ -d "$BACKUP_DIR" ]; then
-  mkdir -p "$SONGS_DIR"
-  rsync -a "$BACKUP_DIR/" "$SONGS_DIR/" || cp -a "$BACKUP_DIR/." "$SONGS_DIR/"
-fi
-
-systemctl daemon-reload || true
-systemctl restart taiko-web || systemctl start taiko-web || true
-
-systemctl is-active --quiet taiko-web