kosmos-storage-audit/tools/offline-analyze.mjs

import fs from "node:fs/promises";
import path from "node:path";
import { ClassicLevel } from "../../barebone/fvtt/node_modules/classic-level/index.js";
import { analyzeStorage } from "../scripts/core/analyzer.js";
import { isMediaPath, normalizePath } from "../scripts/core/path-utils.js";

const [, , dataRootArg, worldIdArg, publicRootArg] = process.argv;
const dataRoot = path.resolve(dataRootArg ?? "./barebone/fvttdata/Data");
const worldId = worldIdArg ?? "dsa5";
const publicRoot = path.resolve(publicRootArg ?? "./barebone/fvtt/public");
const worldJsonPath = path.join(dataRoot, "worlds", worldId, "world.json");
const worldManifest = JSON.parse(await fs.readFile(worldJsonPath, "utf8"));
const worldSystemId = worldManifest.system ?? null;

async function* walkDirectory(storage, baseDir, relativeDir = "") {
  const absoluteDir = path.join(baseDir, relativeDir);
  let entries;
  try {
    entries = await fs.readdir(absoluteDir, { withFileTypes: true });
  } catch (error) {
    if (error?.code === "ENOENT") return;
    throw error;
  }
  for (const entry of entries) {
    const relativePath = normalizePath(path.posix.join(relativeDir, entry.name));
    if (entry.isDirectory()) {
      yield* walkDirectory(storage, baseDir, relativePath);
      continue;
    }
    if (!isMediaPath(relativePath)) continue;
    let stat;
    try {
      stat = await fs.stat(path.join(baseDir, relativePath));
    } catch (error) {
      if (error?.code === "ENOENT") continue;
      throw error;
    }
    yield { storage, path: relativePath, size: stat.size };
  }
}

async function* listOfflineFiles() {
  yield* walkDirectory("data", dataRoot);
  yield* walkDirectory("public", publicRoot);
}

async function* listWorldSources() {
  const collectionsDir = path.join(dataRoot, "worlds", worldId, "data");
  const collectionNames = await fs.readdir(collectionsDir);
  for (const collectionName of collectionNames.sort()) {
    const dbPath = path.join(collectionsDir, collectionName);
    const db = new ClassicLevel(dbPath, { keyEncoding: "utf8", valueEncoding: "json" });
    await db.open();
    try {
      for await (const [key, value] of db.iterator()) {
        yield {
          sourceType: "world-document",
          sourceScope: { ownerType: "world", ownerId: worldId, systemId: worldSystemId, subtype: collectionName },
          sourceLabel: `${collectionName} ${key}`,
          value
        };
      }
    } finally {
      await db.close();
    }
  }
  yield {
    sourceType: "world-manifest",
    sourceScope: { ownerType: "world", ownerId: worldId, systemId: worldSystemId, subtype: "manifest" },
    sourceLabel: `world.json ${worldId}`,
    value: worldManifest
  };

  for (const packageType of ["modules", "systems"]) {
    const packagesDir = path.join(dataRoot, packageType);
    const packageIds = await fs.readdir(packagesDir);
    for (const packageId of packageIds.sort()) {
      const manifestName = packageType === "modules" ? "module.json" : "system.json";
      const manifestPath = path.join(packagesDir, packageId, manifestName);
      try {
        const value = JSON.parse(await fs.readFile(manifestPath, "utf8"));
        yield {
          sourceType: `${packageType.slice(0, -1)}-manifest`,
          sourceScope: { ownerType: packageType.slice(0, -1), ownerId: packageId, subtype: "manifest" },
          sourceLabel: `${manifestName} ${packageId}`,
          value
        };
        for (const pack of value.packs ?? []) {
          const packPath = resolvePackDirectory(path.join(packagesDir, packageId), pack.path ?? "");
          if (!packPath) continue;
          const db = new ClassicLevel(packPath, { keyEncoding: "utf8", valueEncoding: "json" });
          await db.open();
          try {
            for await (const [key, packValue] of db.iterator()) {
              yield {
                sourceType: "package-pack-document",
                sourceScope: {
                  ownerType: packageType.slice(0, -1),
                  ownerId: packageId,
                  subtype: `pack:${pack.name ?? path.basename(packPath)}`
                },
                sourceLabel: `${packageId}:${pack.name ?? path.basename(packPath)} ${key}`,
                value: packValue
              };
            }
          } finally {
            await db.close();
          }
        }
      } catch {
        // Ignore directories without a manifest.
      }
    }
  }
}

function resolvePackDirectory(packageRoot, packPath) {
  if (!packPath) return null;
  const normalized = packPath.replace(/\\/g, "/");
  const withoutExtension = normalized.endsWith(".db") ? normalized.slice(0, -3) : normalized;
  return path.join(packageRoot, withoutExtension);
}

const result = await analyzeStorage({
  listFiles: listOfflineFiles,
  listSources: listWorldSources
});

const summary = {
  files: result.files.length,
  references: result.references.length,
  findings: result.findings.length,
  byKind: result.findings.reduce((acc, finding) => {
    acc[finding.kind] = (acc[finding.kind] ?? 0) + 1;
    return acc;
  }, {})
};

console.log(JSON.stringify(summary, null, 2));