mirror of
https://gitlab.com/vyla-entertainment/home.git
synced 2026-11-20 00:57:22 -05:00
updated debug
This commit is contained in:
181
file-updater.js
181
file-updater.js
@@ -27,7 +27,9 @@ const sha256Buf = (buf) => crypto.createHash('sha256').update(buf).digest('hex')
|
||||
const sha256File = (p) => sha256Buf(fs.readFileSync(p));
|
||||
const gitBlobSha = (buf) => crypto.createHash('sha1').update(Buffer.concat([Buffer.from(`blob ${buf.length}\0`), buf])).digest('hex');
|
||||
|
||||
function fetchBuffer(url, { headers = {}, maxBytes, hosts, redirects = 0 }) {
|
||||
const hooks = { notice: null };
|
||||
|
||||
function fetchBuffer(url, { headers = {}, maxBytes, hosts, redirects = 0, retries = 0 }) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (redirects > 5) return reject(new Error('Too many redirects'));
|
||||
let u;
|
||||
@@ -48,6 +50,15 @@ function fetchBuffer(url, { headers = {}, maxBytes, hosts, redirects = 0 }) {
|
||||
fetchBuffer(next, { headers: keep, maxBytes, hosts, redirects: redirects + 1 }).then(resolve, reject);
|
||||
return;
|
||||
}
|
||||
if (res.statusCode === 429 && retries < 4) {
|
||||
res.resume();
|
||||
const wait = Math.min(parseInt(res.headers['retry-after'], 10) || 5, 30) * 1000;
|
||||
if (hooks.notice) hooks.notice(`GitLab rate limit (HTTP 429). Retrying in ${Math.round(wait / 1000)}s (${retries + 1}/4)...`);
|
||||
setTimeout(() => {
|
||||
fetchBuffer(url, { headers, maxBytes, hosts, redirects, retries: retries + 1 }).then(resolve, reject);
|
||||
}, wait);
|
||||
return;
|
||||
}
|
||||
if (res.statusCode !== 200) {
|
||||
res.resume();
|
||||
reject(new Error(`HTTP ${res.statusCode}`));
|
||||
@@ -121,12 +132,21 @@ class FileUpdater {
|
||||
this.statePath = path.join(appDataPath, 'file-state.json');
|
||||
}
|
||||
|
||||
set onNotice(fn) {
|
||||
hooks.notice = fn;
|
||||
}
|
||||
|
||||
sdkHeaders() {
|
||||
const h = { Accept: 'application/json' };
|
||||
if (process.env.NPM_TOKEN) h.Authorization = `Bearer ${process.env.NPM_TOKEN}`;
|
||||
return h;
|
||||
}
|
||||
|
||||
liveHeaders(extra = {}) {
|
||||
const t = this.updateClient && this.updateClient.token;
|
||||
return t ? { ...extra, 'PRIVATE-TOKEN': t } : extra;
|
||||
}
|
||||
|
||||
target(rel) {
|
||||
const root = path.resolve(this.appDataPath);
|
||||
const full = path.resolve(root, ...rel.split('/'));
|
||||
@@ -139,17 +159,28 @@ class FileUpdater {
|
||||
const s = JSON.parse(fs.readFileSync(this.statePath, 'utf8'));
|
||||
return {
|
||||
seq: typeof s.seq === 'number' ? s.seq : null,
|
||||
base: s.base && typeof s.base === 'object' ? s.base : {}
|
||||
base: s.base && typeof s.base === 'object' ? s.base : {},
|
||||
remote: s.remote && typeof s.remote === 'object' ? s.remote : {},
|
||||
local: s.local && typeof s.local === 'object' ? s.local : {},
|
||||
commits: s.commits && typeof s.commits === 'object' ? s.commits : {}
|
||||
};
|
||||
} catch (e) {
|
||||
return { seq: null, base: {} };
|
||||
return { seq: null, base: {}, remote: {}, local: {}, commits: {} };
|
||||
}
|
||||
}
|
||||
|
||||
writeState(state) {
|
||||
const cur = this.readState();
|
||||
const next = {
|
||||
seq: state.seq !== undefined ? state.seq : cur.seq,
|
||||
base: state.base || cur.base,
|
||||
remote: state.remote || cur.remote,
|
||||
local: state.local || cur.local,
|
||||
commits: state.commits || cur.commits
|
||||
};
|
||||
fs.mkdirSync(this.appDataPath, { recursive: true });
|
||||
const tmp = `${this.statePath}.tmp`;
|
||||
fs.writeFileSync(tmp, JSON.stringify(state), 'utf8');
|
||||
fs.writeFileSync(tmp, JSON.stringify(next), 'utf8');
|
||||
fs.renameSync(tmp, this.statePath);
|
||||
}
|
||||
|
||||
@@ -169,7 +200,7 @@ class FileUpdater {
|
||||
const dir = path.join(this.appDataPath, root);
|
||||
if (fs.existsSync(dir)) walk(dir, root);
|
||||
}
|
||||
this.writeState({ seq: this.readState().seq, base });
|
||||
this.writeState({ seq: this.readState().seq, base, remote: {}, local: {}, commits: {} });
|
||||
}
|
||||
|
||||
zipMatches(zipPath, sha256) {
|
||||
@@ -303,7 +334,7 @@ class FileUpdater {
|
||||
const out = [];
|
||||
for (let page = 1; page <= 50; page++) {
|
||||
const url = `${REPO_API}${root}/repository/tree?recursive=true&per_page=100&ref=main&page=${page}`;
|
||||
const raw = await fetchBuffer(url, { headers: { Accept: 'application/json' }, maxBytes: 5 * 1024 * 1024, hosts: LIVE_HOSTS });
|
||||
return fetchBuffer(`${REPO_API}${root}/repository/files/${file}/raw?ref=main`, { headers: this.liveHeaders(), maxBytes: MAX_FILE, hosts: LIVE_HOSTS });
|
||||
const items = JSON.parse(raw.toString('utf8'));
|
||||
if (!Array.isArray(items) || items.length === 0) break;
|
||||
out.push(...items);
|
||||
@@ -312,34 +343,133 @@ class FileUpdater {
|
||||
return out;
|
||||
}
|
||||
|
||||
async loadArchive(root) {
|
||||
const buf = await fetchBuffer(`${REPO_API}${root}/repository/archive.zip?sha=main`, { headers: this.liveHeaders(), maxBytes: 600 * 1024 * 1024, hosts: LIVE_HOSTS });
|
||||
const zip = new AdmZip(buf);
|
||||
const map = new Map();
|
||||
for (const entry of zip.getEntries()) {
|
||||
if (entry.isDirectory) continue;
|
||||
const name = entry.entryName;
|
||||
const i = name.indexOf('/');
|
||||
if (i < 0) continue;
|
||||
map.set(`${root}/${name.slice(i + 1)}`, entry);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
fetchRepoFile(rel) {
|
||||
const hit = this.archive && this.archive.get(rel);
|
||||
if (hit) return Promise.resolve(hit.getData());
|
||||
const parts = rel.split('/');
|
||||
const root = parts.shift();
|
||||
const file = encodeURIComponent(parts.join('/'));
|
||||
return fetchBuffer(`${REPO_API}${root}/repository/files/${file}/raw?ref=main`, { maxBytes: MAX_FILE, hosts: LIVE_HOSTS });
|
||||
}
|
||||
|
||||
async headCommit(root) {
|
||||
try {
|
||||
const raw = await fetchBuffer(`${REPO_API}${root}/repository/commits/main`, {
|
||||
headers: this.liveHeaders({ Accept: 'application/json' }),
|
||||
maxBytes: 1024 * 1024,
|
||||
hosts: LIVE_HOSTS,
|
||||
retries: 3
|
||||
});
|
||||
const id = JSON.parse(raw.toString('utf8')).id;
|
||||
return /^[0-9a-f]{40}$/.test(id) ? id : null;
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
localIntact(root, state) {
|
||||
const prefix = `${root}/`;
|
||||
const keys = Object.keys(state.remote).filter((k) => k.startsWith(prefix));
|
||||
if (keys.length === 0) return false;
|
||||
for (const rel of keys) {
|
||||
try {
|
||||
if (sha256File(this.target(rel)) !== state.local[rel]) return false;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async checkRepos() {
|
||||
const state = this.readState();
|
||||
const changes = [];
|
||||
const remoteIds = {};
|
||||
const checkedRoots = [];
|
||||
const errors = [];
|
||||
const verified = { remote: {}, local: {} };
|
||||
const commits = {};
|
||||
this.archive = new Map();
|
||||
for (const root of ALLOWED_ROOTS) {
|
||||
const tree = await this.repoTree(root);
|
||||
for (const item of tree) {
|
||||
if (item.type !== 'blob') continue;
|
||||
const rel = safeRel(`${root}/${item.path}`);
|
||||
if (!rel || !/^[0-9a-f]{40}$/.test(item.id)) continue;
|
||||
const head = await this.headCommit(root);
|
||||
if (this.localIntact(root, state) && (!head || head === state.commits[root])) continue;
|
||||
let map;
|
||||
try {
|
||||
map = await this.loadArchive(root);
|
||||
} catch (e) {
|
||||
errors.push(`${root}: ${e.message}`);
|
||||
continue;
|
||||
}
|
||||
if (map.size === 0) {
|
||||
errors.push(`${root}: empty archive`);
|
||||
continue;
|
||||
}
|
||||
checkedRoots.push(root);
|
||||
if (head) commits[root] = head;
|
||||
for (const [key, entry] of map) {
|
||||
const rel = safeRel(key);
|
||||
if (!rel) continue;
|
||||
const data = entry.getData();
|
||||
const id = gitBlobSha(data);
|
||||
remoteIds[rel] = id;
|
||||
const full = this.target(rel);
|
||||
if (fs.existsSync(full) && (state.base[rel] === item.id || gitBlobSha(fs.readFileSync(full)) === item.id)) continue;
|
||||
changes.push({ path: rel, sha256: item.id, size: 0 });
|
||||
let ok = false;
|
||||
let localHash = null;
|
||||
if (fs.existsSync(full)) {
|
||||
const buf = fs.readFileSync(full);
|
||||
localHash = sha256Buf(buf);
|
||||
ok = gitBlobSha(buf) === id || (state.remote[rel] === id && state.local[rel] === localHash);
|
||||
}
|
||||
if (ok) {
|
||||
verified.remote[rel] = id;
|
||||
verified.local[rel] = localHash;
|
||||
} else {
|
||||
changes.push({ path: rel, sha256: id, size: data.length });
|
||||
this.archive.set(rel, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { seq: state.seq === null ? 0 : state.seq, changes, removed: [] };
|
||||
if (checkedRoots.length === 0) {
|
||||
if (errors.length) throw new Error(errors.join(' | '));
|
||||
return { seq: state.seq === null ? 0 : state.seq, changes: [], removed: [], verified, commits };
|
||||
}
|
||||
const tracked = new Set([...Object.keys(state.base), ...Object.keys(state.remote)]);
|
||||
const removed = [...tracked].filter((rel) =>
|
||||
checkedRoots.includes(rel.split('/')[0]) &&
|
||||
!remoteIds[rel] &&
|
||||
safeRel(rel) &&
|
||||
fs.existsSync(this.target(rel))
|
||||
);
|
||||
if (changes.length === 0 && removed.length === 0) {
|
||||
for (const k of Object.keys(verified.remote)) {
|
||||
state.remote[k] = verified.remote[k];
|
||||
state.local[k] = verified.local[k];
|
||||
}
|
||||
this.writeState({ remote: state.remote, local: state.local, commits: { ...state.commits, ...commits } });
|
||||
this.archive = null;
|
||||
}
|
||||
return { seq: state.seq === null ? 0 : state.seq, changes, removed, verified, commits };
|
||||
}
|
||||
|
||||
async applyLive(info, replacements, onFile) {
|
||||
const staged = [];
|
||||
const total = info.changes.length;
|
||||
let next = 0;
|
||||
if (!this.archive) this.archive = new Map();
|
||||
let finished = 0;
|
||||
let failure = null;
|
||||
const worker = async () => {
|
||||
@@ -356,7 +486,8 @@ class FileUpdater {
|
||||
}
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.min(8, total) }, worker));
|
||||
await Promise.all(Array.from({ length: Math.min(4, total) }, worker));
|
||||
this.archive = null;
|
||||
if (failure) throw failure;
|
||||
|
||||
const done = [];
|
||||
@@ -387,9 +518,23 @@ class FileUpdater {
|
||||
}
|
||||
|
||||
const state = this.readState();
|
||||
for (const s of staged) state.base[s.c.path] = s.c.sha256;
|
||||
for (const rel of info.removed) delete state.base[rel];
|
||||
this.writeState({ seq: info.seq, base: state.base });
|
||||
for (const s of staged) {
|
||||
state.base[s.c.path] = s.c.sha256;
|
||||
state.remote[s.c.path] = s.c.sha256;
|
||||
state.local[s.c.path] = sha256Buf(s.data);
|
||||
}
|
||||
for (const rel of info.removed) {
|
||||
delete state.base[rel];
|
||||
delete state.remote[rel];
|
||||
delete state.local[rel];
|
||||
}
|
||||
const v = info.verified || { remote: {}, local: {} };
|
||||
for (const k of Object.keys(v.remote)) {
|
||||
state.remote[k] = v.remote[k];
|
||||
state.local[k] = v.local[k];
|
||||
}
|
||||
state.commits = { ...state.commits, ...(info.commits || {}) };
|
||||
this.writeState({ seq: info.seq, base: state.base, remote: state.remote, local: state.local, commits: state.commits });
|
||||
}
|
||||
|
||||
installedSdk() {
|
||||
|
||||
Reference in New Issue
Block a user