Suppress duplicate tar output in bootstrap and upgrade scripts

Verbose on the sending tar, quiet on the receiving tar, so the
file list prints once.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

backend/check-deps.ts

@@ -0,0 +1,66 @@
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+interface PackageJson {
+    dependencies?: Record<string, string>;
+    devDependencies?: Record<string, string>;
+}
+
+function readPackageJson(path: string): PackageJson {
+    const content = readFileSync(path, "utf-8");
+    return JSON.parse(content);
+}
+
+function getAllDependencyNames(pkg: PackageJson): Set<string> {
+    const names = new Set<string>();
+    for (const name of Object.keys(pkg.dependencies ?? {})) {
+        names.add(name);
+    }
+    for (const name of Object.keys(pkg.devDependencies ?? {})) {
+        names.add(name);
+    }
+    return names;
+}
+
+const diachronPkgPath = join(__dirname, "diachron", "package.json");
+const backendPkgPath = join(__dirname, "package.json");
+
+const diachronPkg = readPackageJson(diachronPkgPath);
+const backendPkg = readPackageJson(backendPkgPath);
+
+const diachronDeps = getAllDependencyNames(diachronPkg);
+const backendDeps = getAllDependencyNames(backendPkg);
+
+const duplicates: string[] = [];
+
+for (const dep of diachronDeps) {
+    if (backendDeps.has(dep)) {
+        duplicates.push(dep);
+    }
+}
+
+if (duplicates.length > 0) {
+    console.error("Error: Duplicate dependencies found.");
+    console.error("");
+    console.error(
+        "The following dependencies exist in both backend/package.json and backend/diachron/package.json:",
+    );
+    console.error("");
+    for (const dep of duplicates.sort()) {
+        console.error(`  - ${dep}`);
+    }
+    console.error("");
+    console.error(
+        "Dependencies in backend/diachron/package.json are provided by the framework",
+    );
+    console.error(
+        "and must not be duplicated in backend/package.json. Remove them from",
+    );
+    console.error("backend/package.json to fix this error.");
+    process.exit(1);
+}
+
+console.log("No duplicate dependencies found.");

backend/check.sh

@@ -11,4 +11,5 @@ out_dir="$check_dir/out"
 source "$check_dir"/../diachron/shims/common
 source "$check_dir"/../diachron/shims/node.common
 
+$ROOT/cmd tsx "$check_dir/check-deps.ts"
 $ROOT/cmd pnpm tsc --outDir "$out_dir"

backend/diachron/package.json

@@ -0,0 +1,39 @@
+{
+    "name": "express",
+    "version": "1.0.0",
+    "description": "",
+    "main": "index.js",
+    "scripts": {
+        "test": "DB_PORT=5433 DB_USER=diachron_test DB_PASSWORD=diachron_test DB_NAME=diachron_test tsx --test '**/*.{test,spec}.ts'",
+        "test:watch": "DB_PORT=5433 DB_USER=diachron_test DB_PASSWORD=diachron_test DB_NAME=diachron_test tsx --test --watch '**/*.{test,spec}.ts'",
+        "nodemon": "nodemon dist/index.js",
+        "kysely-codegen": "kysely-codegen"
+    },
+    "keywords": [],
+    "author": "",
+    "license": "ISC",
+    "packageManager": "pnpm@10.12.4",
+    "dependencies": {
+        "@types/node": "^24.10.1",
+        "@types/nunjucks": "^3.2.6",
+        "@vercel/ncc": "^0.38.4",
+        "express": "^5.1.0",
+        "kysely": "^0.28.9",
+        "nodemon": "^3.1.11",
+        "nunjucks": "^3.2.4",
+        "path-to-regexp": "^8.3.0",
+        "pg": "^8.16.3",
+        "ts-luxon": "^6.2.0",
+        "ts-node": "^10.9.2",
+        "tsx": "^4.20.6",
+        "typeid-js": "^1.2.0",
+        "typescript": "^5.9.3",
+        "zod": "^4.1.12"
+    },
+    "devDependencies": {
+        "@biomejs/biome": "2.3.10",
+        "@types/express": "^5.0.5",
+        "@types/pg": "^8.16.0",
+        "kysely-codegen": "^0.19.0"
+    }
+}

backend/package.json

@@ -1,6 +1,6 @@
 {
-    "name": "express",
-    "version": "1.0.0",
+    "name": "my app",
+    "version": "0.0.1",
     "description": "",
     "main": "index.js",
     "scripts": {
@@ -14,26 +14,7 @@
     "license": "ISC",
     "packageManager": "pnpm@10.12.4",
     "dependencies": {
-        "@types/node": "^24.10.1",
-        "@types/nunjucks": "^3.2.6",
-        "@vercel/ncc": "^0.38.4",
-        "express": "^5.1.0",
-        "kysely": "^0.28.9",
-        "nodemon": "^3.1.11",
-        "nunjucks": "^3.2.4",
-        "path-to-regexp": "^8.3.0",
-        "pg": "^8.16.3",
-        "ts-luxon": "^6.2.0",
-        "ts-node": "^10.9.2",
-        "tsx": "^4.20.6",
-        "typeid-js": "^1.2.0",
-        "typescript": "^5.9.3",
-        "zod": "^4.1.12"
     },
     "devDependencies": {
-        "@biomejs/biome": "2.3.10",
-        "@types/express": "^5.0.5",
-        "@types/pg": "^8.16.0",
-        "kysely-codegen": "^0.19.0"
     }
 }

backend/pnpm-workspace.yaml

@@ -0,0 +1,2 @@
+packages:
+  - 'diachron'

backend/tsconfig.json

@@ -10,5 +10,5 @@
         "types": ["node"],
         "outDir": "out"
     },
-    "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+    "exclude": ["**/*.spec.ts", "**/*.test.ts", "check-deps.ts"]
 }

bootstrap.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# shellcheck disable=SC2002
+
+set -eu
+set -o pipefail
+IFS=$'\n\t'
+
+# print useful message on failure
+trap 's=$?; echo >&2 "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
+
+# shellcheck disable=SC2034
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# cd "$DIR"
+here="$PWD"
+
+"$DIR/update-cached-repository.sh"
+
+# repository="${2:-https://gitea.philologue.net/philologue/diachron}"
+repository="${2:-$HOME/.cache/diachron/v1/repositories/diachron.git}"
+ref="${1:-hydrators-kysely}"
+
+echo will bootstrap ref "$ref" of repo "$repository"
+
+into=$(mktemp -d)
+cd "$into"
+echo I am in $(pwd)
+echo I will clone repository "$repository", ref "$ref"
+git clone "$repository"
+
+r=$(ls -1)
+
+cd "$r"
+
+echo I am in $(pwd)
+
+git checkout "$ref"
+
+ls
+echo working dir: $PWD
+# ls backend
+
+# exit 0
+
+tar cvf - $(cat "$PWD/file-list" | grep -v '^#') | (cd "$here" && tar xf -)
+
+echo "$ref" > .diachron-version
+
+echo "Now, run the command ./sync.sh"

diachron/common.d/check-deps

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eu
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+ROOT="$DIR/../.."
+
+cd "$ROOT/backend"
+
+"$ROOT/cmd" tsx check-deps.ts "$@"

diachron/develop.d/check-deps

@@ -0,0 +1 @@
+../common.d/check-deps

diachron/mgmt.d/check-deps

@@ -0,0 +1 @@
+../common.d/check-deps

file-list

@@ -0,0 +1,16 @@
+# please keep this file sorted alphabetically
+
+backend/diachron
+backend/package.json
+backend/pnpm-workspace.yaml
+# express/framework
+cmd
+file-list
+develop
+diachron
+logger
+master
+mgmt
+sync.sh
+templates
+upgrade.sh

sync.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-# Note: This is kind of AI slop and needs to be more carefully reviewed.
-
 set -eu
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -25,50 +23,66 @@ pnpm_checksum_var="pnpm_checksum_${platform}"
 pnpm_binary_url="${!pnpm_binary_var}"
 pnpm_checksum="${!pnpm_checksum_var}"
 
-# Set up paths for shims to use
-nodejs_dist_dir="diachron/binaries/$nodejs_dirname"
-nodejs_bin_dir="$nodejs_dist_dir/bin"
+cache_dir="$HOME/.cache/diachron/v1/binaries"
+local_dir="$DIR/diachron/binaries"
+mkdir -p "$cache_dir" "$local_dir"
 
-# Ensure correct node version is installed
-node_installed_checksum_file="$DIR/diachron/binaries/.node.checksum"
-node_installed_checksum=""
-if [ -f "$node_installed_checksum_file" ]; then
-    node_installed_checksum=$(cat "$node_installed_checksum_file")
-fi
+# read_checksum_file <path>
+# Prints the contents of a checksum file, or empty string
+# if the file does not exist.
+read_checksum_file() {
+    if [ -f "$1" ]; then
+        cat "$1"
+    fi
+}
 
-if [ "$node_installed_checksum" != "$nodejs_checksum" ]; then
+# Ensure Node.js is in the cache
+cached_node_checksum=$(read_checksum_file "$cache_dir/.node.checksum")
+if [ "$cached_node_checksum" != "$nodejs_checksum" ]; then
     echo "Downloading Node.js for $platform..."
-    node_archive="$DIR/diachron/downloads/node.tar.xz"
+    node_archive="$cache_dir/node.tar.xz"
     curl -fsSL "$nodejs_binary" -o "$node_archive"
 
     echo "Verifying checksum..."
     echo "$nodejs_checksum  $node_archive" | sha256_check
 
     echo "Extracting Node.js..."
-    tar -xf "$node_archive" -C "$DIR/diachron/binaries"
+    rm -rf "${cache_dir:?}/$nodejs_dirname"
+    tar -xf "$node_archive" -C "$cache_dir"
     rm "$node_archive"
 
-    echo "$nodejs_checksum" >"$node_installed_checksum_file"
+    echo "$nodejs_checksum" >"$cache_dir/.node.checksum"
 fi
 
-# Ensure correct pnpm version is installed
-pnpm_binary="$DIR/diachron/binaries/pnpm"
-pnpm_installed_checksum_file="$DIR/diachron/binaries/.pnpm.checksum"
-pnpm_installed_checksum=""
-if [ -f "$pnpm_installed_checksum_file" ]; then
-    pnpm_installed_checksum=$(cat "$pnpm_installed_checksum_file")
+# Copy Node.js into the working directory if needed
+local_node_checksum=$(read_checksum_file "$local_dir/.node.checksum")
+if [ "$local_node_checksum" != "$nodejs_checksum" ]; then
+    echo "Installing Node.js into project..."
+    rm -rf "${local_dir:?}/$nodejs_dirname"
+    cp -R "$cache_dir/$nodejs_dirname" "$local_dir/$nodejs_dirname"
+    echo "$nodejs_checksum" >"$local_dir/.node.checksum"
 fi
 
-if [ "$pnpm_installed_checksum" != "$pnpm_checksum" ]; then
+# Ensure pnpm is in the cache
+cached_pnpm_checksum=$(read_checksum_file "$cache_dir/.pnpm.checksum")
+if [ "$cached_pnpm_checksum" != "$pnpm_checksum" ]; then
     echo "Downloading pnpm for $platform..."
-    curl -fsSL "$pnpm_binary_url" -o "$pnpm_binary"
+    curl -fsSL "$pnpm_binary_url" -o "$cache_dir/pnpm"
 
     echo "Verifying checksum..."
-    echo "$pnpm_checksum  $pnpm_binary" | sha256_check
+    echo "$pnpm_checksum  $cache_dir/pnpm" | sha256_check
 
-    chmod +x "$pnpm_binary"
+    chmod +x "$cache_dir/pnpm"
 
-    echo "$pnpm_checksum" >"$pnpm_installed_checksum_file"
+    echo "$pnpm_checksum" >"$cache_dir/.pnpm.checksum"
+fi
+
+# Copy pnpm into the working directory if needed
+local_pnpm_checksum=$(read_checksum_file "$local_dir/.pnpm.checksum")
+if [ "$local_pnpm_checksum" != "$pnpm_checksum" ]; then
+    echo "Installing pnpm into project..."
+    cp "$cache_dir/pnpm" "$local_dir/pnpm"
+    echo "$pnpm_checksum" >"$local_dir/.pnpm.checksum"
 fi
 
 # Get golang binaries in place

update-cached-repository.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+IFS=$'\n\t'
+
+trap 's=$?; echo >&2 "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR
+
+upstream=https://gitea.philologue.net/philologue/diachron
+cache_dir="$HOME/.cache/diachron/v1/repositories"
+cached_repo="$cache_dir/diachron.git"
+
+mkdir -p "$cache_dir"
+
+if [ -d "$cached_repo" ]; then
+    echo "Updating cached repository..."
+    git -C "$cached_repo" fetch --prune origin
+else
+    echo "Creating cached repository..."
+    git clone --mirror "$upstream" "$cached_repo"
+fi
+

upgrade.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+IFS=$'\n\t'
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+new_ref="${1:?Usage: upgrade.sh <new-ref>}"
+
+cached_repo="$HOME/.cache/diachron/v1/repositories/diachron.git"
+tmpdir=""
+
+cleanup() {
+    if [ -n "$tmpdir" ]; then
+        rm -rf "$tmpdir"
+    fi
+}
+trap cleanup EXIT
+
+echo "=== Diachron Framework Upgrade ==="
+echo ""
+echo "This will replace all framework files in your project."
+echo "Make sure you have committed or backed up any local changes."
+echo ""
+read -r -p "Continue? [y/N] " answer
+if [[ ! "$answer" =~ ^[Yy]$ ]]; then
+    echo "Aborted."
+    exit 0
+fi
+
+# Update cached repository
+"$DIR/update-cached-repository.sh"
+
+# Read current version
+if [ ! -f "$DIR/.diachron-version" ]; then
+    echo "Error: .diachron-version not found." >&2
+    echo "Is this a diachron project?" >&2
+    exit 1
+fi
+old_ref=$(cat "$DIR/.diachron-version")
+
+# Verify both refs exist in cached repo
+if ! git -C "$cached_repo" rev-parse --verify "$old_ref^{commit}" >/dev/null 2>&1; then
+    echo "Error: current version '$old_ref' not found in cached repository." >&2
+    exit 1
+fi
+if ! git -C "$cached_repo" rev-parse --verify "$new_ref^{commit}" >/dev/null 2>&1; then
+    echo "Error: target version '$new_ref' not found in cached repository." >&2
+    exit 1
+fi
+
+# Require a clean working tree
+if [ -n "$(git -C "$DIR" status --porcelain)" ]; then
+    echo "Error: working tree is not clean." >&2
+    echo "Commit or stash all changes (including untracked files) before upgrading." >&2
+    exit 1
+fi
+
+echo ""
+echo "Upgrading: $old_ref -> $new_ref"
+echo ""
+
+# Read current file-list (files to remove)
+old_files=()
+while IFS= read -r line; do
+    [[ "$line" =~ ^[[:space:]]*# ]] && continue
+    [[ -z "$line" ]] && continue
+    old_files+=("$line")
+done < "$DIR/file-list"
+
+# Clone and checkout new version into a temp directory
+tmpdir=$(mktemp -d)
+git clone --quiet "$cached_repo" "$tmpdir/diachron"
+git -C "$tmpdir/diachron" checkout --quiet "$new_ref"
+
+# Read new file-list (files to add)
+new_files=()
+while IFS= read -r line; do
+    [[ "$line" =~ ^[[:space:]]*# ]] && continue
+    [[ -z "$line" ]] && continue
+    new_files+=("$line")
+done < "$tmpdir/diachron/file-list"
+
+# Remove old framework files
+for f in "${old_files[@]}"; do
+    git -C "$DIR" rm -rf --quiet --ignore-unmatch "$f"
+done
+
+# Copy in new framework files
+(cd "$tmpdir/diachron" && tar cvf - "${new_files[@]}") | (cd "$DIR" && tar xf -)
+
+# Stage them
+for f in "${new_files[@]}"; do
+    git -C "$DIR" add "$f"
+done
+
+# Update version marker
+echo "$new_ref" > "$DIR/.diachron-version"
+git -C "$DIR" add "$DIR/.diachron-version"
+
+echo "=== Upgrade staged: $old_ref -> $new_ref ==="
+echo ""
+echo "Framework files have been removed, replaced, and staged."
+echo ""
+echo "Next steps:"
+echo "  1. Review:   git diff --cached"
+echo "  2. Commit:   git commit -m 'Upgrade diachron to $new_ref'"
+echo "  3. Install:  ./sync.sh"