From c3e590e782be6ac665fb5798d541e3beefb8871a Mon Sep 17 00:00:00 2001 From: Kosmos Date: Mon, 20 Apr 2026 20:07:44 +0000 Subject: [PATCH] Initial module scaffold --- README.md | 45 +++ module.json | 26 ++ scripts/adapters/foundry-runtime.js | 110 ++++++++ scripts/apps/audit-report-app.js | 421 ++++++++++++++++++++++++++++ scripts/core/analyzer.js | 60 ++++ scripts/core/finding-engine.js | 154 ++++++++++ scripts/core/path-utils.js | 120 ++++++++ scripts/core/reference-extractor.js | 102 +++++++ scripts/main.js | 22 ++ styles/audit.css | 287 +++++++++++++++++++ tools/offline-analyze.mjs | 127 +++++++++ 11 files changed, 1474 insertions(+) create mode 100644 README.md create mode 100644 module.json create mode 100644 scripts/adapters/foundry-runtime.js create mode 100644 scripts/apps/audit-report-app.js create mode 100644 scripts/core/analyzer.js create mode 100644 scripts/core/finding-engine.js create mode 100644 scripts/core/path-utils.js create mode 100644 scripts/core/reference-extractor.js create mode 100644 scripts/main.js create mode 100644 styles/audit.css create mode 100644 tools/offline-analyze.mjs diff --git a/README.md b/README.md new file mode 100644 index 0000000..5e4cb02 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# Kosmos Storage Audit + +Dieses Verzeichnis enthaelt einen ersten Modul-Prototypen fuer die Analyse von Medienreferenzen und riskanten Ablageorten in FoundryVTT. + +## Foundry Manifest + +- Manifest-URL: + - `https://gitea.kosmos.ac/kosmos/kosmos-storage-audit/raw/branch/main/module.json` +- Download-URL: + - `https://gitea.kosmos.ac/kosmos/kosmos-storage-audit/archive/main.zip` + +## Ausrichtung + +- Das Zielprodukt ist ein online nutzbares Foundry-Modul. +- Die Kernlogik unter `scripts/core/` ist bewusst runtime-neutral gehalten. +- `scripts/adapters/foundry-runtime.js` bindet die Kernlogik an die laufende Foundry-Instanz. +- `tools/offline-analyze.mjs` ist nur ein Entwicklungs-Harness, um Parser und Matching lokal gegen eine gestoppte Instanz zu pruefen. + +## Erste Kernbausteine + +- `path-utils.js` + - Normalisierung auf `data:` und `public:` +- `reference-extractor.js` + - rekursive String- und Attribut-Extraktion aus JSON/Objekten +- `finding-engine.js` + - Klassifizierung von Broken References, Orphans und Fremdverweisen +- `analyzer.js` + - Orchestrierung von Dateien, Quellen und Findings + +## Wichtige Heuristik + +- Bei mehreren Welten versucht das Tool nicht, globale "echte Orphans" im gesamten `data`-Root zu behaupten. +- Orphans werden nur in klar eingrenzbaren Bereichen gemeldet: + - weltlokale Bereiche + - explizit riskante Paket-`storage`-Ordner + - nicht offensichtliche `public`-Bereiche +- Weltverweise auf Modul- oder Systemassets gelten nicht pauschal als Problem. +- Sie werden erst dann hochgezogen, wenn das Zielasset im owning Paket selbst nicht sichtbar referenziert ist. + +## Noch offen + +- bessere Laufzeitabdeckung fuer Welteinstellungen und Compendien +- robustere Erkennung von Directory- und Wildcard-Referenzen +- UI in Foundry +- Aktionen zum sicheren Verschieben oder Migrieren diff --git a/module.json b/module.json new file mode 100644 index 0000000..0b2f3ee --- /dev/null +++ b/module.json @@ -0,0 +1,26 @@ +{ + "id": "kosmos-storage-audit", + "title": "Kosmos Storage Audit", + "description": "Analyzes media references and risky storage locations across Foundry data and public roots.", + "version": "0.0.1", + "compatibility": { + "minimum": "13", + "verified": "13" + }, + "authors": [ + { + "name": "kosmos" + } + ], + "esmodules": [ + "scripts/main.js" + ], + "styles": [ + "styles/audit.css" + ], + "languages": [], + "relationships": {}, + "url": "https://gitea.kosmos.ac/kosmos/kosmos-storage-audit", + "manifest": "https://gitea.kosmos.ac/kosmos/kosmos-storage-audit/raw/branch/main/module.json", + "download": "https://gitea.kosmos.ac/kosmos/kosmos-storage-audit/archive/main.zip" +} diff --git a/scripts/adapters/foundry-runtime.js b/scripts/adapters/foundry-runtime.js new file mode 100644 index 0000000..b9bd497 --- /dev/null +++ b/scripts/adapters/foundry-runtime.js @@ -0,0 +1,110 @@ +import { analyzeStorage } from "../core/analyzer.js"; +import { isMediaPath, normalizePath } from "../core/path-utils.js"; + +async function* walkFilePicker(storage, target = "", onProgress = null) { + onProgress?.({ phase: "files", label: `Durchsuche ${storage}:${target || "/"}` }); + const result = await FilePicker.browse(storage, target); + for (const file of result.files ?? []) { + const path = normalizePath(file); + if (!isMediaPath(path)) continue; + yield { storage, path }; + } + for (const directory of result.dirs ?? []) { + const path = normalizePath(directory); + yield* walkFilePicker(storage, path, onProgress); + } +} + +async function* listFoundryFiles(onProgress = null) { + for (const storage of ["data", "public"]) { + if (!game.data.files.storages.includes(storage)) continue; + yield* walkFilePicker(storage, "", onProgress); + } +} + +function worldCollectionEntries() { + if (!game.world) return []; + const entries = []; + for (const collection of game.collections ?? []) { + const docs = Array.from(collection.values?.() ?? []); + for (const doc of docs) { + entries.push({ + sourceType: "world-document", + sourceScope: { + ownerType: "world", + ownerId: game.world.id, + systemId: game.world.system, + subtype: doc.documentName?.toLowerCase() ?? collection.documentName?.toLowerCase() ?? "document" + }, + sourceLabel: `${doc.documentName ?? "Document"} ${doc.id}`, + value: doc.toObject ? doc.toObject() : doc + }); + } + } + return entries; +} + +function packageMetadataEntries() { + const entries = []; + for (const module of game.modules.values()) { + entries.push({ + sourceType: "module-manifest", + sourceScope: { ownerType: "module", ownerId: module.id, subtype: "manifest" }, + sourceLabel: `module.json ${module.id}`, + value: module.toObject ? module.toObject() : module + }); + } + if (game.system) { + entries.push({ + sourceType: "system-manifest", + sourceScope: { ownerType: "system", ownerId: game.system.id, subtype: "manifest" }, + sourceLabel: `system.json ${game.system.id}`, + value: game.system.toObject ? game.system.toObject() : game.system + }); + } + if (game.world) { + entries.push({ + sourceType: "world-manifest", + sourceScope: { ownerType: "world", ownerId: game.world.id, systemId: game.world.system, subtype: "manifest" }, + sourceLabel: `world.json ${game.world.id}`, + value: game.world.toObject ? game.world.toObject() : game.world + }); + } + return entries; +} + +async function* packagePackEntries(onProgress = null) { + for (const pack of game.packs.values()) { + const ownerType = pack.metadata.packageType; + const ownerId = pack.metadata.packageName; + if (!["module", "system"].includes(ownerType) || !ownerId) continue; + onProgress?.({ phase: "sources", label: `Lese Paket-Pack ${pack.collection}`, currentSource: pack.collection }); + const documents = await pack.getDocuments(); + for (const document of documents) { + yield { + sourceType: "package-pack-document", + sourceScope: { + ownerType, + ownerId, + subtype: `pack:${pack.collection}` + }, + sourceLabel: `${pack.collection} ${document.id}`, + value: document.toObject ? document.toObject() : document + }; + } + } +} + +async function* listFoundrySources(onProgress = null) { + yield* worldCollectionEntries(); + yield* packageMetadataEntries(); + yield* packagePackEntries(onProgress); +} + +export async function runRuntimeAnalysis({ onProgress }={}) { + return analyzeStorage({ + listFiles: () => listFoundryFiles(onProgress), + listSources: () => listFoundrySources(onProgress), + onProgress + }); +} diff --git a/scripts/apps/audit-report-app.js b/scripts/apps/audit-report-app.js new file mode 100644 index 0000000..4d270a3 --- /dev/null +++ b/scripts/apps/audit-report-app.js @@ -0,0 +1,421 @@ +export class StorageAuditReportApp extends foundry.applications.api.ApplicationV2 { + static DEFAULT_OPTIONS = { + id: "kosmos-storage-audit-report", + classes: ["kosmos-storage-audit"], + tag: "div", + actions: { + runAnalysis: StorageAuditReportApp.#onRunAnalysis, + toggleShowAll: StorageAuditReportApp.#onToggleShowAll + }, + window: { + title: "Kosmos Storage Audit", + icon: "fa-solid fa-broom-ball", + resizable: true + }, + position: { + width: 760, + height: 680 + } + }; + + #analysis = null; + #loading = false; + #showAll = false; + #progress = null; + + async _prepareContext() { + const findings = this.#analysis?.findings ?? []; + const visibleFindings = this.#showAll ? findings : findings.filter(f => f.severity !== "info"); + const groupedFindings = groupFindings(visibleFindings); + return { + loading: this.#loading, + hasAnalysis: !!this.#analysis, + showAll: this.#showAll, + progress: this.#progress, + summary: this.#summarize(this.#analysis), + groupedFindings, + findings: visibleFindings.slice(0, 100) + }; + } + + async _renderHTML(context) { + const container = document.createElement("div"); + container.className = "storage-audit"; + container.innerHTML = ` +
+
+

Kosmos Storage Audit

+

Prueft Medienreferenzen und markiert primaer benutzerrelevante Risiken in den Foundry-Roots data und public.

+

Orphans werden bewusst nicht global fuer den gesamten data-Root behauptet, sondern nur in klar weltlokalen oder explizit riskanten Bereichen.

+

Weltverweise auf Modul- oder Systemassets gelten nicht pauschal als Problem. Relevant sind vor allem Ziele, die im owning Paket selbst nicht sichtbar referenziert werden.

+
+
+ + +
+
+ ${renderProgress(context.progress, context.loading)} + ${renderSummary(context.summary)} + ${renderGroupedFindingList(context.groupedFindings, context.hasAnalysis, context.loading, context.showAll)} + ${renderFindingList(context.findings, context.hasAnalysis, context.loading, context.showAll)} + `; + return container; + } + + _replaceHTML(result, content) { + content.replaceChildren(result); + } + + async runAnalysis() { + this.#loading = true; + this.#progress = { + phase: "start", + label: "Initialisiere Analyse", + files: 0, + sources: 0, + references: 0, + findings: 0, + currentSource: null + }; + await this.render({ force: true }); + try { + const api = game.modules.get("kosmos-storage-audit")?.api; + this.#analysis = await api.runRuntimeAnalysis({ + onProgress: update => this.#updateProgress(update) + }); + const count = this.#analysis.findings.length; + ui.notifications.info(`Kosmos Storage Audit abgeschlossen: ${count} Findings.`); + } catch (error) { + console.error("kosmos-storage-audit | analysis failed", error); + ui.notifications.error(`Kosmos Storage Audit fehlgeschlagen: ${error.message}`); + } finally { + this.#loading = false; + await this.render({ force: true }); + } + } + + toggleShowAll() { + this.#showAll = !this.#showAll; + return this.render({ force: true }); + } + + #summarize(analysis) { + if (!analysis) return null; + const bySeverity = { high: 0, warning: 0, info: 0 }; + const byKind = {}; + const grouped = { + brokenReferences: 0, + packageReferences: 0, + orphans: 0 + }; + for (const finding of analysis.findings) { + bySeverity[finding.severity] = (bySeverity[finding.severity] ?? 0) + 1; + byKind[finding.kind] = (byKind[finding.kind] ?? 0) + 1; + } + grouped.brokenReferences = new Set( + analysis.findings.filter(f => f.kind === "broken-reference").map(f => f.target.locator ?? formatTarget(f.target)) + ).size; + grouped.packageReferences = new Set( + analysis.findings.filter(f => f.kind === "non-package-to-package-reference").map(f => f.target.locator ?? formatTarget(f.target)) + ).size; + grouped.orphans = analysis.findings.filter(f => f.kind === "orphan-file").length; + return { + files: analysis.files.length, + references: analysis.references.length, + findings: analysis.findings.length, + bySeverity, + byKind, + grouped + }; + } + + #updateProgress(update) { + this.#progress = { + ...(this.#progress ?? {}), + ...update + }; + this.render({ force: true }); + } + + static #onRunAnalysis(_event, _button) { + return this.runAnalysis(); + } + + static #onToggleShowAll(_event, _button) { + return this.toggleShowAll(); + } +} + +function renderProgress(progress, loading) { + if (!loading || !progress) return ""; + return ` +
+
+
+
+
+ ${escapeHtml(progress.label ?? "Analysiere")} + Dateien: ${progress.files ?? 0} + Quellen: ${progress.sources ?? 0} + Referenzen: ${progress.references ?? 0} + ${progress.findings != null ? `Findings: ${progress.findings}` : ""} +
+ ${progress.currentSource ? `

Aktuell: ${escapeHtml(progress.currentSource)}

` : ""} +
+ `; +} + +function renderSummary(summary) { + if (!summary) { + return ` +
+

Noch keine Analyse ausgefuehrt.

+
+ `; + } + + const kindLines = Object.entries(summary.byKind) + .sort((a, b) => b[1] - a[1]) + .map(([kind, count]) => `
  • ${kind}${count}
    • `) + .join(""); + + return ` +
    +
    +
    Dateien${summary.files}
    +
    Referenzen${summary.references}
    +
    Findings${summary.findings}
    +
    High${summary.bySeverity.high}
    +
    Warning${summary.bySeverity.warning}
    +
    +
    +

    Findings nach Typ

    +
      ${kindLines || "
    • Keine Findings0
      • "}
    +
    +
    +

    Arbeitsbloecke

    +
      +
    • Deduplizierte fehlende Ziele${summary.grouped?.brokenReferences ?? 0}
      • +
    • Unverankerte Paketziele${summary.grouped?.packageReferences ?? 0}
      • +
    • Orphan-Kandidaten${summary.grouped?.orphans ?? 0}
      • +
    +
    +
    + `; +} + +function renderGroupedFindingList(groupedFindings, hasAnalysis, loading, showAll) { + if (loading || !hasAnalysis) return ""; + + const sections = []; + if (groupedFindings.nonPackageToPackage.length) { + sections.push(renderGroupedSection( + "Unverankerte Paketziele", + "Diese Ziele liegen in Modul- oder Systemordnern, werden aus Weltdaten referenziert, sind im owning Paket selbst aber derzeit nicht als Referenz sichtbar.", + groupedFindings.nonPackageToPackage, + group => ` +
    +
    + ${group.severity} + ${group.kind} + ${group.count} Referenzen +
    +

    ${escapeHtml(group.target)}

    +

    ${escapeHtml(group.reason)}

    +
    +
    Owner-Paket
    ${escapeHtml(group.ownerLabel)}
    +
    Bewertung
    ${escapeHtml(group.explanation)}
    +
    +

    ${escapeHtml(group.recommendation)}

    + ${renderSampleSources(group.sources)} +
    + ` + )); + } + + if (groupedFindings.brokenReferences.length) { + sections.push(renderGroupedSection( + "Kaputte Ziele", + "Diese Dateien fehlen, werden aber weiterhin referenziert.", + groupedFindings.brokenReferences, + group => ` +
    +
    + ${group.severity} + ${group.kind} + ${group.count} Referenzen +
    +

    ${escapeHtml(group.target)}

    +

    ${escapeHtml(group.reason)}

    +

    ${escapeHtml(group.recommendation)}

    + ${renderSampleSources(group.sources)} +
    + ` + )); + } + + if (showAll && groupedFindings.orphans.length) { + sections.push(renderGroupedSection( + "Orphan-Kandidaten", + "Diese Dateien haben im aktuellen Analysekontext keine eingehende Referenz.", + groupedFindings.orphans, + group => ` +
    +
    + ${group.severity} + ${group.kind} +
    +

    ${escapeHtml(group.target)}

    +

    ${escapeHtml(group.reason)}

    +

    ${escapeHtml(group.recommendation)}

    +
    + ` + )); + } + + if (!sections.length) { + return ` +
    +
    +

    Arbeitsansicht

    +

    Keine gruppierten ${showAll ? "" : "warnenden "}Findings vorhanden.

    +
    +
    + `; + } + + return `
    ${sections.join("")}
    `; +} + +function renderGroupedSection(title, description, groups, renderGroup) { + const items = groups.slice(0, 12).map(renderGroup).join(""); + return ` +
    +
    +

    ${title}

    +

    ${description}

    +
    +
    ${items}
    +
    + `; +} + +function renderFindingList(findings, hasAnalysis, loading, showAll) { + if (loading) { + return `

    Analyse laeuft...

    `; + } + if (!hasAnalysis) { + return `

    Die Analyse kann direkt aus dieser Ansicht gestartet werden.

    `; + } + if (!findings.length) { + return `

    Keine ${showAll ? "" : "warnenden "}Findings gefunden.

    `; + } + + const items = findings.map(finding => ` +
    +
    + ${finding.severity} + ${finding.kind} +
    +

    ${escapeHtml(finding.reason)}

    +
    +
    Ziel
    ${escapeHtml(finding.target.locator ?? `${finding.target.storage}:${finding.target.path}`)}
    + ${finding.source ? `
    Quelle
    ${escapeHtml(finding.source.sourceLabel)}
    ` : ""} +
    +

    ${escapeHtml(finding.recommendation)}

    +
    + `).join(""); + + return ` +
    +

    Einzelfaelle

    + ${items} +
    + `; +} + +function renderSampleSources(sources) { + if (!sources.length) return ""; + const rows = sources.map(source => `
  • ${escapeHtml(source)}
    • `).join(""); + return `
    Beispiele
    `; +} + +function groupFindings(findings) { + return { + brokenReferences: groupByTarget(findings.filter(f => f.kind === "broken-reference")), + nonPackageToPackage: groupByTarget(findings.filter(f => f.kind === "non-package-to-package-reference")), + orphans: findings + .filter(f => f.kind === "orphan-file") + .sort(compareFindings) + .map(finding => ({ + kind: finding.kind, + severity: finding.severity, + target: finding.target.locator ?? formatTarget(finding.target), + reason: finding.reason, + recommendation: finding.recommendation + })) + }; +} + +function groupByTarget(findings) { + const grouped = new Map(); + for (const finding of findings) { + const target = finding.target.locator ?? formatTarget(finding.target); + const ownerLabel = deriveOwnerLabel(target); + const current = grouped.get(target) ?? { + kind: finding.kind, + severity: finding.severity, + target, + count: 0, + ownerLabel, + explanation: "Weltdaten zeigen auf ein Paketasset, fuer das in Manifesten und Paket-Packs keine sichtbare Eigenreferenz gefunden wurde.", + reason: finding.reason, + recommendation: finding.recommendation, + sources: new Set() + }; + current.count += 1; + if (finding.severity === "high") current.severity = "high"; + if (finding.source?.sourceLabel) current.sources.add(finding.source.sourceLabel); + grouped.set(target, current); + } + return [...grouped.values()] + .map(group => ({ ...group, sources: [...group.sources].slice(0, 5) })) + .sort((a, b) => compareSeverity(a.severity, b.severity) || (b.count - a.count) || a.target.localeCompare(b.target)); +} + +function deriveOwnerLabel(target) { + const [, path] = String(target).split(":", 2); + const parts = (path ?? "").split("/"); + if ((parts[0] === "modules") || (parts[0] === "systems")) { + return `${parts[0]}:${parts[1] ?? "unknown"}`; + } + return "unbekannt"; +} + +function compareFindings(a, b) { + return compareSeverity(a.severity, b.severity) + || formatTarget(a.target).localeCompare(formatTarget(b.target)); +} + +function compareSeverity(a, b) { + const order = { high: 0, warning: 1, info: 2 }; + return (order[a] ?? 9) - (order[b] ?? 9); +} + +function formatTarget(target) { + return target.locator ?? `${target.storage}:${target.path}`; +} + +function escapeHtml(value) { + return String(value ?? "") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/\"/g, """); +} diff --git a/scripts/core/analyzer.js b/scripts/core/analyzer.js new file mode 100644 index 0000000..4379ea9 --- /dev/null +++ b/scripts/core/analyzer.js @@ -0,0 +1,60 @@ +import { buildFindings, createFileRecord } from "./finding-engine.js"; +import { extractReferencesFromValue } from "./reference-extractor.js"; +import { createFileLocator } from "./path-utils.js"; + +export async function analyzeStorage({ listFiles, listSources, onProgress }={}) { + const files = []; + let fileCount = 0; + let sourceCount = 0; + let referenceCount = 0; + + onProgress?.({ phase: "files", label: "Scanne Dateien", files: 0, sources: 0, references: 0 }); + for await (const entry of listFiles()) { + files.push(createFileRecord(createFileLocator(entry.storage, entry.path), entry.size ?? null)); + fileCount += 1; + if ((fileCount % 100) === 0) { + onProgress?.({ phase: "files", label: "Scanne Dateien", files: fileCount, sources: sourceCount, references: referenceCount }); + await yieldToUI(); + } + } + + const references = []; + onProgress?.({ phase: "sources", label: "Lese Referenzen", files: fileCount, sources: 0, references: 0 }); + for await (const source of listSources()) { + sourceCount += 1; + const extracted = extractReferencesFromValue(source.value, { + sourceType: source.sourceType, + sourceScope: source.sourceScope, + sourceLabel: source.sourceLabel + }); + references.push(...extracted); + referenceCount += extracted.length; + if ((sourceCount % 50) === 0) { + onProgress?.({ + phase: "sources", + label: "Lese Referenzen", + files: fileCount, + sources: sourceCount, + references: referenceCount, + currentSource: source.sourceLabel + }); + await yieldToUI(); + } + } + + onProgress?.({ phase: "findings", label: "Klassifiziere Findings", files: fileCount, sources: sourceCount, references: referenceCount }); + const findings = buildFindings({ files, references }); + onProgress?.({ + phase: "done", + label: "Analyse abgeschlossen", + files: fileCount, + sources: sourceCount, + references: referenceCount, + findings: findings.length + }); + return { files, references, findings }; +} + +async function yieldToUI() { + await new Promise(resolve => setTimeout(resolve, 0)); +} diff --git a/scripts/core/finding-engine.js b/scripts/core/finding-engine.js new file mode 100644 index 0000000..aab9c62 --- /dev/null +++ b/scripts/core/finding-engine.js @@ -0,0 +1,154 @@ +import { classifyRisk, detectMediaKind, inferOwnerHint, isCorePublicPath, isStorageAreaPath } from "./path-utils.js"; + +function compareOwner(sourceScope, targetOwner) { + if (!sourceScope) return "unknown"; + if ((sourceScope.ownerType === "world") && (targetOwner.ownerType === "system") && (sourceScope.systemId === targetOwner.ownerId)) { + return "same-system"; + } + if ((sourceScope.ownerType === "world") && (targetOwner.ownerType === "public") && + ["cards", "icons", "nue", "sounds", "ui"].includes(targetOwner.ownerId)) { + return "core-public"; + } + if ((sourceScope.ownerType === "module") && (targetOwner.ownerType === "module")) { + return sourceScope.ownerId === targetOwner.ownerId ? "same-package" : "foreign-module"; + } + if ((sourceScope.ownerType === "system") && (targetOwner.ownerType === "system")) { + return sourceScope.ownerId === targetOwner.ownerId ? "same-package" : "cross-package"; + } + if ((sourceScope.ownerType === "world") && (targetOwner.ownerType === "world")) { + return sourceScope.ownerId === targetOwner.ownerId ? "same-world" : "cross-world"; + } + if ((sourceScope.ownerType === "world") && ((targetOwner.ownerType === "module") || (targetOwner.ownerType === "system"))) { + return "non-package-to-package"; + } + if ((sourceScope.ownerType === "world") && (targetOwner.ownerType === "public")) { + return "risky-public"; + } + if (((sourceScope.ownerType === "module") || (sourceScope.ownerType === "system")) && + ((targetOwner.ownerType === "module") || (targetOwner.ownerType === "system")) && + ((sourceScope.ownerType !== targetOwner.ownerType) || (sourceScope.ownerId !== targetOwner.ownerId))) { + return "cross-package"; + } + return "allowed"; +} + +export function buildFindings({ files, references }) { + const fileByLocator = new Map(files.map(file => [file.locator, file])); + const refsByLocator = new Map(); + const findings = []; + + for (const reference of references) { + const normalized = reference.normalized; + if (!normalized) continue; + const bucket = refsByLocator.get(normalized.locator) ?? []; + bucket.push(reference); + refsByLocator.set(normalized.locator, bucket); + } + + for (const reference of references) { + const normalized = reference.normalized; + if (!normalized) continue; + + const file = fileByLocator.get(normalized.locator); + if (!file) { + if (reference.sourceScope?.ownerType !== "world") continue; + findings.push({ + kind: "broken-reference", + severity: reference.sourceScope?.ownerType === "world" ? "high" : "warning", + target: normalized, + source: reference, + reason: `Referenced file ${normalized.locator} does not exist in the scanned roots.`, + recommendation: "Check whether the file was moved, deleted, or should be copied into a stable location.", + confidence: "high" + }); + continue; + } + + const ownerRelation = compareOwner(reference.sourceScope, file.ownerHint); + if (ownerRelation === "non-package-to-package") { + if (isAnchoredInOwningPackage(file, refsByLocator.get(normalized.locator) ?? [])) continue; + findings.push({ + kind: "non-package-to-package-reference", + severity: "high", + target: normalized, + source: reference, + reason: `${reference.sourceScope.ownerType}:${reference.sourceScope.ownerId} references package-owned storage ${normalized.locator}, but the asset is not visibly referenced by its owning package.`, + recommendation: "Review whether the file was manually placed into the package folder. If the package does not use it itself, prefer moving it into user-controlled storage.", + confidence: "high" + }); + } else if (ownerRelation === "risky-public") { + if (reference.sourceScope?.ownerType !== "world") continue; + findings.push({ + kind: "risky-public-reference", + severity: "high", + target: normalized, + source: reference, + reason: `${reference.sourceScope.ownerType}:${reference.sourceScope.ownerId} references release-public storage ${normalized.locator}.`, + recommendation: "Prefer copying the asset into world or user-controlled storage.", + confidence: "high" + }); + } + } + + for (const file of files) { + if (detectMediaKind(file.path) === "other") continue; + const refs = refsByLocator.get(file.locator) ?? []; + if (refs.length) continue; + if (!shouldReportOrphan(file, references)) continue; + + const severity = (file.riskClass === "package-module") || (file.riskClass === "package-system") || (file.riskClass === "release-public") + ? "warning" + : "info"; + + findings.push({ + kind: "orphan-file", + severity, + target: file, + source: null, + reason: `No incoming media reference was found for ${file.locator}.`, + recommendation: severity === "warning" + ? "Review whether the file is safe to remove or should be moved into a stable storage location." + : "Review whether the file is intentionally kept as reserve content.", + confidence: "medium" + }); + } + + return findings; +} + +function isAnchoredInOwningPackage(file, references) { + const ownerType = file.ownerHint.ownerType; + const ownerId = file.ownerHint.ownerId; + if (!["module", "system"].includes(ownerType) || !ownerId) return false; + return references.some(reference => + (reference.sourceScope?.ownerType === ownerType) && + (reference.sourceScope?.ownerId === ownerId) + ); +} + +function shouldReportOrphan(file, references) { + if (file.riskClass === "stable-world") { + const worldId = file.ownerHint.ownerId; + return references.some(reference => reference.sourceScope?.ownerType === "world" && reference.sourceScope.ownerId === worldId); + } + if ((file.riskClass === "package-module") || (file.riskClass === "package-system")) { + return isStorageAreaPath(file); + } + if (file.riskClass === "release-public") { + return !isCorePublicPath(file); + } + return false; +} + +export function createFileRecord(locator, size = null) { + return { + ...locator, + basename: locator.path.split("/").pop() ?? locator.path, + extension: locator.path.includes(".") ? locator.path.split(".").pop().toLowerCase() : "", + size, + mediaKind: detectMediaKind(locator.path), + ownerHint: inferOwnerHint(locator), + riskClass: classifyRisk(locator), + exists: true + }; +} diff --git a/scripts/core/path-utils.js b/scripts/core/path-utils.js new file mode 100644 index 0000000..abd401c --- /dev/null +++ b/scripts/core/path-utils.js @@ -0,0 +1,120 @@ +export const RELEASE_PREFIXES = new Set(["canvas", "cards", "docs", "fonts", "icons", "lang", "nue", "scripts", "sounds", "toolclips", "tours", "ui"]); +const DATA_PREFIXES = new Set(["assets", "modules", "systems", "worlds"]); +const MEDIA_EXTENSIONS = { + image: new Set(["avif", "gif", "jpeg", "jpg", "png", "svg", "webp"]), + video: new Set(["m4v", "mkv", "mov", "mp4", "mpeg", "mpg", "webm"]), + audio: new Set(["flac", "m4a", "mid", "midi", "mp3", "ogg", "opus", "wav", "webm"]) +}; + +export function normalizePath(path) { + const normalized = String(path ?? "") + .replace(/\\/g, "/") + .replace(/^[./]+/, "") + .replace(/^\/+/, "") + .replace(/\/{2,}/g, "/") + .trim(); + try { + return decodeURI(normalized); + } catch { + return normalized; + } +} + +export function splitResourceSuffix(path) { + const [withoutHash] = String(path ?? "").split("#", 1); + const [cleanPath] = withoutHash.split("?", 1); + return cleanPath; +} + +export function parseStoragePath(rawValue) { + if (typeof rawValue !== "string") return null; + const trimmed = rawValue.trim(); + if (!trimmed) return null; + if (/^(?:https?:|data:|blob:)/i.test(trimmed)) return null; + + const clean = normalizePath(splitResourceSuffix(trimmed)); + if (!clean) return null; + + const first = clean.split("/", 1)[0]; + let storage = null; + if (DATA_PREFIXES.has(first)) storage = "data"; + else if (RELEASE_PREFIXES.has(first)) storage = "public"; + if (!storage) return null; + + return { + storage, + path: clean, + locator: `${storage}:${clean}` + }; +} + +export function getExtension(path) { + const clean = normalizePath(path); + const index = clean.lastIndexOf("."); + return index === -1 ? "" : clean.slice(index + 1).toLowerCase(); +} + +export function detectMediaKind(path) { + const ext = getExtension(path); + for (const [kind, extensions] of Object.entries(MEDIA_EXTENSIONS)) { + if (extensions.has(ext)) return kind; + } + return "other"; +} + +export function isMediaPath(path) { + return detectMediaKind(path) !== "other"; +} + +export function classifyRisk(locator) { + const parts = locator.path.split("/"); + if (locator.storage === "public") return "release-public"; + if (parts[0] === "modules") return "package-module"; + if (parts[0] === "systems") return "package-system"; + if (parts[0] === "assets") return "stable-user"; + if (parts[0] === "worlds") return "stable-world"; + return locator.storage === "data" ? "stable-user" : "unknown"; +} + +export function inferOwnerHint(locator) { + const parts = locator.path.split("/"); + if (locator.storage === "public") { + return { ownerType: "public", ownerId: parts[0] ?? null }; + } + if (parts[0] === "assets") { + return { ownerType: "user", ownerId: null }; + } + if ((parts[0] === "worlds") && parts[1]) { + return { ownerType: "world", ownerId: parts[1] }; + } + if ((parts[0] === "modules") && parts[1]) { + return { ownerType: "module", ownerId: parts[1] }; + } + if ((parts[0] === "systems") && parts[1]) { + return { ownerType: "system", ownerId: parts[1] }; + } + if (locator.storage === "data") { + return { ownerType: "user", ownerId: parts[0] ?? null }; + } + return { ownerType: "unknown", ownerId: null }; +} + +export function isStorageAreaPath(locator) { + const parts = locator.path.split("/"); + return ((parts[0] === "modules") || (parts[0] === "systems")) && (parts[2] === "storage"); +} + +export function isCorePublicPath(locator) { + if (locator.storage !== "public") return false; + const parts = locator.path.split("/"); + return RELEASE_PREFIXES.has(parts[0] ?? ""); +} + +export function createFileLocator(storage, path) { + const cleanPath = normalizePath(path); + return { + storage, + path: cleanPath, + locator: `${storage}:${cleanPath}` + }; +} diff --git a/scripts/core/reference-extractor.js b/scripts/core/reference-extractor.js new file mode 100644 index 0000000..9a63d13 --- /dev/null +++ b/scripts/core/reference-extractor.js @@ -0,0 +1,102 @@ +import { createFileLocator, isMediaPath, normalizePath, parseStoragePath } from "./path-utils.js"; + +const ATTRIBUTE_PATTERNS = [ + /\b(?:src|href|poster)\s*=\s*["']([^"'<>]+)["']/gi, + /url\(\s*["']?([^"')]+)["']?\s*\)/gi +]; + +export function collectStringCandidates(value, visit) { + if (typeof value === "string") { + visit(value); + return; + } + if (Array.isArray(value)) { + for (const entry of value) collectStringCandidates(entry, visit); + return; + } + if ((value !== null) && (typeof value === "object")) { + for (const entry of Object.values(value)) collectStringCandidates(entry, visit); + } +} + +export function extractReferencesFromValue(value, source) { + const references = []; + collectStringCandidates(value, candidate => { + const direct = resolveReference(candidate, source); + if (direct && isMediaPath(direct.path)) { + references.push({ + sourceType: source.sourceType, + sourceScope: source.sourceScope, + sourceLabel: source.sourceLabel, + rawValue: candidate, + normalized: { + ...direct, + targetKind: "local-file" + } + }); + } + + for (const pattern of ATTRIBUTE_PATTERNS) { + pattern.lastIndex = 0; + let match; + while ((match = pattern.exec(candidate))) { + const nested = resolveReference(match[1], source); + if (!nested || !isMediaPath(nested.path)) continue; + references.push({ + sourceType: source.sourceType, + sourceScope: source.sourceScope, + sourceLabel: source.sourceLabel, + rawValue: match[1], + normalized: { + ...nested, + targetKind: "local-file" + } + }); + } + } + }); + return dedupeReferences(references); +} + +function resolveReference(rawValue, source) { + const direct = parseStoragePath(rawValue); + if (direct) return direct; + return resolvePackageRelativeReference(rawValue, source); +} + +function resolvePackageRelativeReference(rawValue, source) { + if (typeof rawValue !== "string") return null; + if (/^(?:https?:|data:|blob:)/i.test(rawValue.trim())) return null; + const path = normalizePath(rawValue); + if (!path || !path.includes("/")) return null; + + const ownerType = source.sourceScope?.ownerType; + const ownerId = source.sourceScope?.ownerId; + if (!ownerType || !ownerId) return null; + + if (ownerType === "module") { + return createFileLocator("data", `modules/${ownerId}/${path}`); + } + if (ownerType === "system") { + return createFileLocator("data", `systems/${ownerId}/${path}`); + } + if (ownerType === "world") { + return createFileLocator("data", path); + } + return null; +} + +function dedupeReferences(references) { + const seen = new Set(); + return references.filter(reference => { + const key = [ + reference.sourceType, + reference.sourceLabel, + reference.normalized?.locator ?? "", + reference.rawValue + ].join("|"); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} diff --git a/scripts/main.js b/scripts/main.js new file mode 100644 index 0000000..e5d837c --- /dev/null +++ b/scripts/main.js @@ -0,0 +1,22 @@ +import { runRuntimeAnalysis } from "./adapters/foundry-runtime.js"; +import { StorageAuditReportApp } from "./apps/audit-report-app.js"; + +Hooks.once("init", () => { + console.log("kosmos-storage-audit | init"); + game.settings.registerMenu("kosmos-storage-audit", "report", { + name: "Kosmos Storage Audit", + label: "Open Report", + hint: "Run the storage audit and inspect prioritized findings.", + icon: "fa-solid fa-broom-ball", + type: StorageAuditReportApp, + restricted: true + }); + game.modules.get("kosmos-storage-audit").api = { + runRuntimeAnalysis, + StorageAuditReportApp + }; +}); + +Hooks.once("ready", () => { + console.log("kosmos-storage-audit | ready"); +}); diff --git a/styles/audit.css b/styles/audit.css new file mode 100644 index 0000000..186ea46 --- /dev/null +++ b/styles/audit.css @@ -0,0 +1,287 @@ +.kosmos-storage-audit .window-content { + padding: 0; + overflow-y: auto; +} + +.storage-audit { + display: grid; + gap: 1rem; + padding: 1.1rem; + color: inherit; + font: inherit; + line-height: inherit; +} + +.storage-audit code { + color: inherit; +} + +.storage-audit__hero, +.storage-audit__summary, +.storage-audit__progress, +.storage-audit__group, +.storage-audit__finding, +.storage-audit__list { + border-radius: 12px; +} + +.storage-audit__hero, +.storage-audit__summary { + padding: 1.1rem 1.2rem; +} + +.storage-audit__grouped { + display: grid; + gap: 1rem; +} + +.storage-audit__progress { + padding: 1rem 1.1rem; +} + +.storage-audit__progress-bar { + height: 10px; + border-radius: 999px; + background: color-mix(in srgb, currentColor 12%, transparent); + overflow: hidden; + margin-bottom: 0.75rem; +} + +.storage-audit__progress-fill { + width: 100%; + height: 100%; + background: linear-gradient(90deg, currentColor, transparent, currentColor); + background-size: 200% 100%; + opacity: 0.35; + animation: storage-audit-progress 1.6s linear infinite; +} + +.storage-audit__progress-meta { + display: flex; + gap: 1rem; + flex-wrap: wrap; + font-size: 0.92rem; +} + +.storage-audit__progress-source { + margin-top: 0.5rem; + opacity: 0.8; + word-break: break-word; +} + +.storage-audit__group-header { + padding: 1rem 1.1rem 0; +} + +.storage-audit__group-header h3 { + margin: 0 0 0.25rem; +} + +.storage-audit__group-header p { + margin: 0; + opacity: 0.82; +} + +.storage-audit__hero { + display: flex; + justify-content: space-between; + gap: 1rem; + align-items: start; +} + +.storage-audit__hero h2, +.storage-audit__summary h3 { + margin: 0 0 0.45rem; + line-height: 1.05; +} + +.storage-audit__hero h2 { + font-weight: 800; + letter-spacing: 0.01em; +} + +.storage-audit__hero p, +.storage-audit__finding p, +.storage-audit__list p { + margin: 0; +} + +.storage-audit__hero p { + max-width: 52rem; + opacity: 0.9; +} + +.storage-audit__hero > div:first-child { + display: grid; + gap: 0.45rem; +} + +.storage-audit__actions { + display: flex; + flex-direction: column; + gap: 0.75rem; + align-items: stretch; + min-width: 12rem; +} + +.storage-audit__actions .button { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 0.5rem; + border-radius: 999px; + padding: 0.62rem 0.95rem; + font-weight: 700; + min-height: 2.55rem; +} + +.storage-audit__actions .button[disabled] { + opacity: 0.5; +} + +.storage-audit__stats { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(110px, 1fr)); + gap: 0.75rem; + margin-bottom: 1rem; +} + +.storage-audit__stats article { + background: color-mix(in srgb, currentColor 4%, transparent); + border-radius: 12px; + padding: 0.9rem; + border: 1px solid color-mix(in srgb, currentColor 10%, transparent); +} + +.storage-audit__stats span { + display: block; + opacity: 0.8; + margin-bottom: 0.18rem; +} + +.storage-audit__stats strong { line-height: 1; } + +.storage-audit__kinds ul, +.storage-audit__list { + display: grid; + gap: 0.75rem; +} + +.storage-audit__kinds ul { + list-style: none; + padding: 0; + margin: 0; +} + +.storage-audit__kinds li { + display: flex; + justify-content: space-between; + gap: 1rem; + padding: 0.35rem 0; + border-bottom: 1px solid color-mix(in srgb, currentColor 10%, transparent); +} + +.storage-audit__kinds li:last-child { + border-bottom: 0; +} + +.storage-audit__list { + padding: 1rem; +} + +.storage-audit__list--raw h3 { + margin-top: 0; +} + +.storage-audit__finding { + padding: 0.9rem 1rem; +} + +.storage-audit__finding header { + display: flex; + gap: 0.75rem; + align-items: center; + margin-bottom: 0.5rem; +} + +.storage-audit__finding header strong { opacity: 0.9; } + +.storage-audit__severity { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 4.75rem; + border-radius: 999px; + padding: 0.2rem 0.6rem; + text-transform: uppercase; + font-weight: 700; + letter-spacing: 0.04em; +} + +.severity-high .storage-audit__severity { + background: color-mix(in srgb, #b03e29 80%, transparent); + color: white; +} + +.severity-warning .storage-audit__severity { + background: color-mix(in srgb, #b8902b 80%, transparent); + color: black; +} + +.severity-info .storage-audit__severity { + background: color-mix(in srgb, #466d7a 80%, transparent); + color: white; +} + +.storage-audit__finding dl { + display: grid; + gap: 0.35rem; + margin: 0.75rem 0 0; +} + +.storage-audit__finding dl div { + display: grid; + gap: 0.2rem; +} + +.storage-audit__finding dt { + opacity: 0.75; +} + +.storage-audit__finding dd { margin: 0; } + +.storage-audit__recommendation { + margin-top: 0.75rem; + opacity: 0.9; +} + +.storage-audit__samples { + margin-top: 0.75rem; +} + +.storage-audit__samples span { + display: block; + opacity: 0.75; + margin-bottom: 0.25rem; +} + +.storage-audit__samples ul { + margin: 0; + padding-left: 1.25rem; +} + +@media (max-width: 900px) { + .storage-audit__hero { + flex-direction: column; + } + + .storage-audit__actions { + min-width: 0; + width: 100%; + } +} + +@keyframes storage-audit-progress { + 0% { background-position: 0% 0; } + 100% { background-position: 200% 0; } +} diff --git a/tools/offline-analyze.mjs b/tools/offline-analyze.mjs new file mode 100644 index 0000000..8a6ab70 --- /dev/null +++ b/tools/offline-analyze.mjs @@ -0,0 +1,127 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { ClassicLevel } from "../../barebone/fvtt/node_modules/classic-level/index.js"; +import { analyzeStorage } from "../scripts/core/analyzer.js"; +import { isMediaPath, normalizePath } from "../scripts/core/path-utils.js"; + +const [, , dataRootArg, worldIdArg, publicRootArg] = process.argv; +const dataRoot = path.resolve(dataRootArg ?? "./barebone/fvttdata/Data"); +const worldId = worldIdArg ?? "dsa5"; +const publicRoot = path.resolve(publicRootArg ?? "./barebone/fvtt/public"); +const worldJsonPath = path.join(dataRoot, "worlds", worldId, "world.json"); +const worldManifest = JSON.parse(await fs.readFile(worldJsonPath, "utf8")); +const worldSystemId = worldManifest.system ?? null; + +async function* walkDirectory(storage, baseDir, relativeDir = "") { + const absoluteDir = path.join(baseDir, relativeDir); + const entries = await fs.readdir(absoluteDir, { withFileTypes: true }); + for (const entry of entries) { + const relativePath = normalizePath(path.posix.join(relativeDir, entry.name)); + if (entry.isDirectory()) { + yield* walkDirectory(storage, baseDir, relativePath); + continue; + } + if (!isMediaPath(relativePath)) continue; + const stat = await fs.stat(path.join(baseDir, relativePath)); + yield { storage, path: relativePath, size: stat.size }; + } +} + +async function* listOfflineFiles() { + yield* walkDirectory("data", dataRoot); + yield* walkDirectory("public", publicRoot); +} + +async function* listWorldSources() { + const collectionsDir = path.join(dataRoot, "worlds", worldId, "data"); + const collectionNames = await fs.readdir(collectionsDir); + for (const collectionName of collectionNames.sort()) { + const dbPath = path.join(collectionsDir, collectionName); + const db = new ClassicLevel(dbPath, { keyEncoding: "utf8", valueEncoding: "json" }); + await db.open(); + try { + for await (const [key, value] of db.iterator()) { + yield { + sourceType: "world-document", + sourceScope: { ownerType: "world", ownerId: worldId, systemId: worldSystemId, subtype: collectionName }, + sourceLabel: `${collectionName} ${key}`, + value + }; + } + } finally { + await db.close(); + } + } + yield { + sourceType: "world-manifest", + sourceScope: { ownerType: "world", ownerId: worldId, systemId: worldSystemId, subtype: "manifest" }, + sourceLabel: `world.json ${worldId}`, + value: worldManifest + }; + + for (const packageType of ["modules", "systems"]) { + const packagesDir = path.join(dataRoot, packageType); + const packageIds = await fs.readdir(packagesDir); + for (const packageId of packageIds.sort()) { + const manifestName = packageType === "modules" ? "module.json" : "system.json"; + const manifestPath = path.join(packagesDir, packageId, manifestName); + try { + const value = JSON.parse(await fs.readFile(manifestPath, "utf8")); + yield { + sourceType: `${packageType.slice(0, -1)}-manifest`, + sourceScope: { ownerType: packageType.slice(0, -1), ownerId: packageId, subtype: "manifest" }, + sourceLabel: `${manifestName} ${packageId}`, + value + }; + for (const pack of value.packs ?? []) { + const packPath = resolvePackDirectory(path.join(packagesDir, packageId), pack.path ?? ""); + if (!packPath) continue; + const db = new ClassicLevel(packPath, { keyEncoding: "utf8", valueEncoding: "json" }); + await db.open(); + try { + for await (const [key, packValue] of db.iterator()) { + yield { + sourceType: "package-pack-document", + sourceScope: { + ownerType: packageType.slice(0, -1), + ownerId: packageId, + subtype: `pack:${pack.name ?? path.basename(packPath)}` + }, + sourceLabel: `${packageId}:${pack.name ?? path.basename(packPath)} ${key}`, + value: packValue + }; + } + } finally { + await db.close(); + } + } + } catch { + // Ignore directories without a manifest. + } + } + } +} + +function resolvePackDirectory(packageRoot, packPath) { + if (!packPath) return null; + const normalized = packPath.replace(/\\/g, "/"); + const withoutExtension = normalized.endsWith(".db") ? normalized.slice(0, -3) : normalized; + return path.join(packageRoot, withoutExtension); +} + +const result = await analyzeStorage({ + listFiles: listOfflineFiles, + listSources: listWorldSources +}); + +const summary = { + files: result.files.length, + references: result.references.length, + findings: result.findings.length, + byKind: result.findings.reduce((acc, finding) => { + acc[finding.kind] = (acc[finding.kind] ?? 0) + 1; + return acc; + }, {}) +}; + +console.log(JSON.stringify(summary, null, 2));