Run natively under systemd; Docker was unworkable in this LXC
Docker in this unprivileged Proxmox LXC was broken two independent ways: AppArmor could not load the docker-default profile, so the daemon could not start any new container, and runc could not set the net.ipv4.ip_unprivileged_port_start sysctl, which is why every service needed network_mode: host. `docker build` was impossible outright since Docker 29 removed the classic builder. The stack only survived because the containers predated the breakage - a reboot would have left the app down, and nothing could be redeployed. Postgres, MinIO and the app now run natively under systemd. Deploys build out-of-place into releases/<sha> and swap a symlink, so the build happens while the old release keeps serving and downtime is the ~3s restart rather than the ~3min build. Rollback is the same swap in reverse with no rebuild. Dockerfile and docker-compose.prod.yml are unchanged and still work on a normal host; the compose app service gains `build: .` so it can come up on a VPS that cannot reach the LAN-only Gitea registry. Documents three traps found during the migration: rewrites() in next.config.mjs is evaluated at build time so MINIO_ENDPOINT changes silently do nothing, prisma migrate deploy would fail because production has no _prisma_migrations table, and the image proxy relies on the bucket's anonymous-download policy. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
JP
164
deploy/deploy.sh
164
deploy/deploy.sh
@@ -2,28 +2,27 @@
|
||||
#
|
||||
# Deploy drinktracker to the production LXC.
|
||||
#
|
||||
# The image is built ON the LXC and tagged with the registry name that
|
||||
# docker-compose.prod.yml expects. Compose then finds it locally and never pulls, so
|
||||
# this needs no registry credentials on either machine. Pass --push to additionally
|
||||
# publish the image to the Gitea registry (requires `docker login` on the LXC).
|
||||
# The app runs natively under systemd - no Docker. Each deploy builds into
|
||||
# releases/<sha> out-of-place and swaps a symlink, so the ~3 minute build happens
|
||||
# while the old release keeps serving and downtime is only the restart.
|
||||
#
|
||||
# Usage: ./deploy/deploy.sh [--push] [--no-build] [--yes]
|
||||
# Usage: ./deploy/deploy.sh [--yes] [--no-build] [--rollback]
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${DT_HOST:-drinkadmin@192.168.2.169}"
|
||||
SSH_KEY="${DT_SSH_KEY:-$HOME/.ssh/drinktracker_ed25519}"
|
||||
REMOTE_DIR="${DT_REMOTE_DIR:-/root/drinktracker}"
|
||||
IMAGE="${DT_IMAGE:-192.168.2.140:3000/jpscott84/drinktracker:latest}"
|
||||
COMPOSE_FILE="docker-compose.prod.yml"
|
||||
APP_DIR="${DT_APP_DIR:-/opt/drinktracker}"
|
||||
SVC_USER="${DT_SVC_USER:-drinktracker}"
|
||||
KEEP_RELEASES=5
|
||||
HEALTH_URL="http://localhost:3000/"
|
||||
|
||||
PUSH=0; BUILD=1; ASSUME_YES=0
|
||||
ASSUME_YES=0; BUILD=1; ROLLBACK=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--push) PUSH=1 ;;
|
||||
--no-build) BUILD=0 ;;
|
||||
--yes|-y) ASSUME_YES=1 ;;
|
||||
--no-build) BUILD=0 ;;
|
||||
--rollback) ROLLBACK=1 ;;
|
||||
*) echo "unknown option: $arg" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
@@ -32,87 +31,118 @@ SSH=(ssh -o BatchMode=yes -o ConnectTimeout=10 -i "$SSH_KEY" "$HOST")
|
||||
say() { printf '\n\033[1;36m==> %s\033[0m\n' "$*"; }
|
||||
die() { printf '\n\033[1;31mFAILED: %s\033[0m\n' "$*" >&2; exit 1; }
|
||||
|
||||
health_poll() {
|
||||
for i in $(seq 1 30); do
|
||||
code=$("${SSH[@]}" "curl -s -o /dev/null -w '%{http_code}' -m 5 $HEALTH_URL" || echo 000)
|
||||
# 307 is the unauthenticated redirect to /login - healthy.
|
||||
case "$code" in 200|302|307) echo " healthy (HTTP $code) after $((i*2))s"; return 0 ;; esac
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ─── Rollback: repoint the symlink, no rebuild ───────────────────────
|
||||
if [ "$ROLLBACK" -eq 1 ]; then
|
||||
say "Available releases (newest last)"
|
||||
"${SSH[@]}" "ls -1tr $APP_DIR/releases"
|
||||
CURRENT=$("${SSH[@]}" "readlink $APP_DIR/current | xargs basename")
|
||||
PREV=$("${SSH[@]}" "ls -1t $APP_DIR/releases | grep -v '^$CURRENT\$' | head -1")
|
||||
[ -n "$PREV" ] || die "no other release to roll back to"
|
||||
echo " current=$CURRENT -> rolling back to $PREV"
|
||||
if [ "$ASSUME_YES" -ne 1 ]; then
|
||||
read -r -p " proceed? [y/N] " r; [[ "$r" =~ ^[Yy]$ ]] || { echo aborted; exit 1; }
|
||||
fi
|
||||
"${SSH[@]}" "sudo -n -u $SVC_USER bash -c 'ln -sfn $APP_DIR/releases/$PREV $APP_DIR/current.new && mv -Tf $APP_DIR/current.new $APP_DIR/current'"
|
||||
"${SSH[@]}" "sudo -n systemctl restart drinktracker"
|
||||
health_poll || die "app unhealthy after rollback - journalctl -u drinktracker"
|
||||
say "Rolled back to $PREV"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ─── Preflight (local) ───────────────────────────────────────────────
|
||||
say "Preflight"
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
[ -n "$(git status --porcelain)" ] && die "working tree is dirty - commit or stash first"
|
||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
[ "$BRANCH" = "main" ] || die "on branch '$BRANCH', expected main"
|
||||
|
||||
npx tsc --noEmit || die "typecheck failed"
|
||||
echo " typecheck ok, tree clean, on main"
|
||||
|
||||
# Push first so the server pulls exactly what was verified here.
|
||||
git -c credential.helper=store push origin main
|
||||
LOCAL_SHA=$(git rev-parse HEAD)
|
||||
echo " pushed $(git rev-parse --short HEAD)"
|
||||
SHA=$(git rev-parse HEAD); SHORT=${SHA:0:7}
|
||||
echo " pushed $SHORT"
|
||||
|
||||
# ─── Preflight (remote) ──────────────────────────────────────────────
|
||||
say "Checking $HOST"
|
||||
"${SSH[@]}" true || die "cannot reach $HOST with key $SSH_KEY"
|
||||
"${SSH[@]}" 'sudo -n true' 2>/dev/null \
|
||||
|| die "passwordless sudo required on the LXC (needed for $REMOTE_DIR). See deploy/README.md"
|
||||
|
||||
# A dirty server checkout means someone edited production by hand; clobbering that
|
||||
# silently would destroy the only copy of the change.
|
||||
if ! "${SSH[@]}" "sudo -n git -C $REMOTE_DIR diff --quiet && sudo -n git -C $REMOTE_DIR diff --cached --quiet"; then
|
||||
die "$REMOTE_DIR has uncommitted changes - inspect before deploying"
|
||||
fi
|
||||
for unit in postgresql@16-main minio drinktracker; do
|
||||
st=$("${SSH[@]}" "systemctl is-active $unit" || true)
|
||||
echo " $unit: $st"
|
||||
[ "$st" = "active" ] || die "$unit is not active - fix before deploying"
|
||||
done
|
||||
|
||||
if [ "$ASSUME_YES" -ne 1 ]; then
|
||||
REMOTE_SHA=$("${SSH[@]}" "sudo -n git -C $REMOTE_DIR rev-parse --short HEAD")
|
||||
echo " remote is at $REMOTE_SHA, deploying ${LOCAL_SHA:0:7}"
|
||||
read -r -p " proceed? [y/N] " reply
|
||||
[[ "$reply" =~ ^[Yy]$ ]] || { echo "aborted"; exit 1; }
|
||||
CURRENT=$("${SSH[@]}" "readlink $APP_DIR/current | xargs basename")
|
||||
echo " current release $CURRENT, deploying $SHORT"
|
||||
read -r -p " proceed? [y/N] " r; [[ "$r" =~ ^[Yy]$ ]] || { echo aborted; exit 1; }
|
||||
fi
|
||||
|
||||
# ─── Deploy ──────────────────────────────────────────────────────────
|
||||
say "Syncing $REMOTE_DIR to $LOCAL_SHA"
|
||||
"${SSH[@]}" "sudo -n git -C $REMOTE_DIR fetch origin main --quiet && sudo -n git -C $REMOTE_DIR reset --hard $LOCAL_SHA --quiet && sudo -n git -C $REMOTE_DIR log --oneline -1"
|
||||
|
||||
# SWITCHBOARD_BASE_URL arrived with the gateway migration and will be missing from
|
||||
# any .env.production written before it. Compose supplies a default, but making it
|
||||
# explicit keeps the file honest about what the app reads.
|
||||
say "Checking .env.production for new variables"
|
||||
"${SSH[@]}" "sudo -n grep -q '^SWITCHBOARD_BASE_URL=' $REMOTE_DIR/.env.production" \
|
||||
&& echo " SWITCHBOARD_BASE_URL present" \
|
||||
|| echo " NOTE: SWITCHBOARD_BASE_URL absent; compose default (http://192.168.2.11:8787/v1) applies"
|
||||
# ─── Build (out-of-place; the running release keeps serving) ─────────
|
||||
say "Syncing and building on $HOST"
|
||||
"${SSH[@]}" "sudo -n -u $SVC_USER git -C $APP_DIR/repo fetch --quiet origin main && sudo -n -u $SVC_USER git -C $APP_DIR/repo reset --hard $SHA --quiet && sudo -n -u $SVC_USER git -C $APP_DIR/repo log --oneline -1"
|
||||
|
||||
if [ "$BUILD" -eq 1 ]; then
|
||||
say "Building image on the LXC (this takes a few minutes)"
|
||||
# Pass the build context as a path rather than cd-ing in: the shell runs as the
|
||||
# deploy user, which cannot enter /root even though sudo can build there.
|
||||
"${SSH[@]}" "sudo -n docker build -t $IMAGE $REMOTE_DIR" || die "image build failed"
|
||||
# Built WITHOUT the env file, deliberately - same as the Dockerfile. next.config.mjs
|
||||
# evaluates rewrites() at build time, so sourcing the env here would bake a different
|
||||
# /minio-images destination into routes-manifest.json.
|
||||
"${SSH[@]}" "sudo -n -u $SVC_USER bash -lc '
|
||||
set -e
|
||||
cd $APP_DIR/repo
|
||||
export npm_config_cache=$APP_DIR/.npm NODE_OPTIONS=--max-old-space-size=3072
|
||||
npm ci --prefer-offline --no-audit --fund=false
|
||||
npx prisma generate
|
||||
npm run build
|
||||
'" || die "build failed"
|
||||
fi
|
||||
|
||||
if [ "$PUSH" -eq 1 ]; then
|
||||
say "Pushing image to registry"
|
||||
"${SSH[@]}" "sudo -n docker push $IMAGE" || die "registry push failed (is docker login done on the LXC?)"
|
||||
fi
|
||||
say "Assembling release $SHORT"
|
||||
# Mirrors Dockerfile:24-30 - standalone bundle plus static and public copied in.
|
||||
"${SSH[@]}" "sudo -n -u $SVC_USER bash -lc '
|
||||
set -e
|
||||
REL=$APP_DIR/releases/$SHORT
|
||||
rm -rf \$REL && mkdir -p \$REL/.next/cache
|
||||
cp -a $APP_DIR/repo/.next/standalone/. \$REL/
|
||||
cp -a $APP_DIR/repo/.next/static \$REL/.next/static
|
||||
cp -a $APP_DIR/repo/public \$REL/public
|
||||
du -sh \$REL
|
||||
'" || die "release assembly failed"
|
||||
|
||||
say "Restarting stack"
|
||||
# --no-build: compose must use the image we just built, not rebuild or pull it.
|
||||
# Compose takes the project directory from the compose file's location, which keeps
|
||||
# the project name "drinktracker" and so reuses the existing containers and volumes.
|
||||
"${SSH[@]}" "sudo -n docker compose -f $REMOTE_DIR/$COMPOSE_FILE up -d --no-build"
|
||||
# ─── Schema ──────────────────────────────────────────────────────────
|
||||
say "Backing up database before schema push"
|
||||
"${SSH[@]}" "sudo -n install -d -m 0750 /var/backups/drinktracker && sudo -n -u postgres pg_dump drinkman | gzip -9 | sudo -n tee /var/backups/drinktracker/pre-$SHORT.sql.gz >/dev/null && sudo -n ls -lh /var/backups/drinktracker/pre-$SHORT.sql.gz"
|
||||
|
||||
say "Applying schema"
|
||||
# db push, NOT migrate deploy: production has no _prisma_migrations table, so
|
||||
# migrate deploy would try to apply the init migration against populated tables.
|
||||
# --accept-data-loss means removing a field from schema.prisma DROPS the column.
|
||||
"${SSH[@]}" "sudo -n -u $SVC_USER bash -lc '
|
||||
set -a; . /etc/drinktracker/drinktracker.env; set +a
|
||||
cd $APP_DIR/repo && npx prisma db push --skip-generate --accept-data-loss
|
||||
'" || die "prisma db push failed"
|
||||
|
||||
# ─── Swap and restart ────────────────────────────────────────────────
|
||||
say "Swapping to $SHORT and restarting"
|
||||
"${SSH[@]}" "sudo -n -u $SVC_USER bash -c 'ln -sfn $APP_DIR/releases/$SHORT $APP_DIR/current.new && mv -Tf $APP_DIR/current.new $APP_DIR/current'"
|
||||
"${SSH[@]}" "sudo -n systemctl restart drinktracker"
|
||||
health_poll || die "app unhealthy - check: ${SSH[*]} 'journalctl -u drinktracker -n 50'"
|
||||
|
||||
# ─── Verify ──────────────────────────────────────────────────────────
|
||||
say "Verifying"
|
||||
"${SSH[@]}" "sudo -n docker compose -f $REMOTE_DIR/$COMPOSE_FILE ps --format 'table {{.Name}}\t{{.Status}}'" || true
|
||||
"${SSH[@]}" "curl -s -o /dev/null -w ' public=%{http_code}\n' -m 15 https://drinktracker.tenseconddelay.net/ || true"
|
||||
"${SSH[@]}" "sudo -n journalctl -u drinktracker --since '2 min ago' | grep '\[switchboard\]' | tail -3 || echo ' (no AI calls yet)'"
|
||||
|
||||
ok=0
|
||||
for i in $(seq 1 30); do
|
||||
code=$("${SSH[@]}" "curl -s -o /dev/null -w '%{http_code}' -m 5 $HEALTH_URL" || echo 000)
|
||||
# 307 is the unauthenticated redirect to /login - a healthy response here.
|
||||
case "$code" in 200|307|302) echo " app responding (HTTP $code) after ${i}0s"; ok=1; break ;; esac
|
||||
sleep 10
|
||||
done
|
||||
[ "$ok" -eq 1 ] || die "app did not become healthy - check: ${SSH[*]} 'sudo docker logs drinktracker-app-1 --tail 50'"
|
||||
say "Pruning old releases (keeping $KEEP_RELEASES)"
|
||||
"${SSH[@]}" "sudo -n -u $SVC_USER bash -c 'cd $APP_DIR/releases && ls -1t | tail -n +$((KEEP_RELEASES+1)) | xargs -r rm -rf; ls -1t'"
|
||||
|
||||
say "Recent gateway activity"
|
||||
"${SSH[@]}" "sudo -n docker logs drinktracker-app-1 --tail 200 2>&1 | grep '\[switchboard\]' | tail -5 || echo ' (no AI calls yet)'"
|
||||
|
||||
say "Deployed ${LOCAL_SHA:0:7}"
|
||||
echo "To roll back: ./deploy/deploy.sh after 'git revert', or on the LXC:"
|
||||
echo " sudo git -C $REMOTE_DIR reset --hard <previous-sha> && cd $REMOTE_DIR && sudo docker build -t $IMAGE . && sudo docker compose -f $COMPOSE_FILE up -d --no-build"
|
||||
say "Deployed $SHORT"
|
||||
echo "Roll back with: ./deploy/deploy.sh --rollback"
|
||||
|
||||
Reference in New Issue
Block a user