From dd6acfa7b8ea3fe809fa1d9f3a1d1388b26d1c03 Mon Sep 17 00:00:00 2001 From: RandyJC Date: Mon, 1 Dec 2025 14:34:24 +0100 Subject: [PATCH] fixing issues --- .env | 4 +- docker-compose.yml | 47 ++++++++--- uploader.sh | 191 +++++++++++++++++++++++++++------------------ 3 files changed, 151 insertions(+), 91 deletions(-) diff --git a/.env b/.env index 04d0a7e..8f5df1f 100644 --- a/.env +++ b/.env @@ -11,14 +11,14 @@ DROP=/mnt/user/photosync/drop IMMICH_URL=http://immich:8080 IMMICH_API_KEY=REPLACE_ME ALLOW_INSECURE_SSL=true -IMMICH_GO_VERSION=0.31.0 +IMMICH_GO_VERSION=0.27.0 # Mover tuning STABLE_FOR=5 SCAN_FALLBACK=60 # Uploader tuning -UPLOAD_CONCURRENCY=4 UPLOAD_SCAN_INTERVAL=5 +UPLOAD_CONCURRENCY=4 UPLOAD_IDLE_SLEEP=3 DELETE_ON_SUCCESS=true diff --git a/docker-compose.yml b/docker-compose.yml index 28cd4cc..07d51d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,16 +5,25 @@ services: restart: unless-stopped environment: - TZ=${TZ:-UTC} - - INBOX=${INBOX} - - DROP=${DROP} + - INBOX=/inbox + - DROP=/drop - STABLE_FOR=${STABLE_FOR:-5} - SCAN_FALLBACK=${SCAN_FALLBACK:-60} + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} volumes: - ${INBOX}:/inbox:rw - ${DROP}:/drop:rw - ./move_stable.sh:/scripts/move_stable.sh:ro - entrypoint: ["/bin/sh","-c","apk add --no-cache inotify-tools coreutils >/dev/null && chmod +x /scripts/move_stable.sh && exec /scripts/move_stable.sh"] - user: "${PUID:-1000}:${PGID:-1000}" + networks: + - npm-network + entrypoint: + - /bin/sh + - -c + - | + set -e + apk add --no-cache inotify-tools coreutils su-exec >/dev/null + exec su-exec "$${PUID}:$${PGID}" /scripts/move_stable.sh photosync-uploader: image: alpine:3.19 @@ -29,10 +38,14 @@ services: - IMMICH_GO_VERSION=${IMMICH_GO_VERSION:-0.31.0} - SCAN_DIR=/drop - SCAN_INTERVAL=${UPLOAD_SCAN_INTERVAL:-5} - - IDLE_SLEEP=${UPLOAD_IDLE_SLEEP:-3} - CONCURRENCY=${UPLOAD_CONCURRENCY:-4} + - IDLE_SLEEP=${UPLOAD_IDLE_SLEEP:-3} - DELETE_ON_SUCCESS=${DELETE_ON_SUCCESS:-true} - ALLOW_INSECURE_SSL=${ALLOW_INSECURE_SSL:-false} + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - XDG_CACHE_HOME=/tmp/immich-cache + - HOME=/tmp volumes: - ${DROP}:/drop:rw - ./uploader.sh:/scripts/uploader.sh:ro @@ -41,17 +54,25 @@ services: - -c - | set -e - apk add --no-cache curl ca-certificates >/dev/null + apk add --no-cache curl ca-certificates su-exec tar >/dev/null arch="$$(uname -m)" case "$$arch" in - x86_64) arch="amd64" ;; + x86_64) arch="x86_64" ;; aarch64) arch="arm64" ;; + arm64) arch="arm64" ;; *) echo "Unsupported architecture: $$arch"; exit 1 ;; esac - url="https://github.com/simulot/immich-go/releases/download/v$${IMMICH_GO_VERSION}/immich-go_Linux_$${arch}" + url="https://github.com/simulot/immich-go/releases/download/v$${IMMICH_GO_VERSION}/immich-go_Linux_$${arch}.tar.gz" echo "[uploader] Downloading immich-go $${IMMICH_GO_VERSION} for $$arch" - curl -fsSL -o /usr/local/bin/immich-go "$$url" - chmod +x /usr/local/bin/immich-go - chmod +x /scripts/uploader.sh - exec /scripts/uploader.sh - user: "${PUID:-1000}:${PGID:-1000}" + tmp="$$(mktemp -d)" + curl -fsSL "$$url" -o "$$tmp/immich-go.tar.gz" + tar -xzf "$$tmp/immich-go.tar.gz" -C "$$tmp" + install -m 0755 "$$tmp/immich-go" /usr/local/bin/immich-go + rm -rf "$$tmp" + exec su-exec "$${PUID}:$${PGID}" /scripts/uploader.sh + networks: + - npm-network + +networks: + npm-network: + external: true diff --git a/uploader.sh b/uploader.sh index 0fdd51f..c737a4e 100755 --- a/uploader.sh +++ b/uploader.sh @@ -1,80 +1,135 @@ #!/bin/sh set -eu -IMMICH_URL="${IMMICH_URL:?IMMICH_URL is required}" # e.g. http://immich:8080 +# -------- Config from environment -------- +IMMICH_URL="${IMMICH_URL:?IMMICH_URL is required}" IMMICH_API_KEY="${IMMICH_API_KEY:?IMMICH_API_KEY is required}" + SCAN_DIR="${SCAN_DIR:-/drop}" DELETE_ON_SUCCESS="${DELETE_ON_SUCCESS:-true}" -SCAN_INTERVAL="${SCAN_INTERVAL:-5}" -ALLOW_INSECURE_SSL="${ALLOW_INSECURE_SSL:-false}" +SCAN_INTERVAL="${SCAN_INTERVAL:-5}" # seconds between scans CONCURRENCY="${CONCURRENCY:-4}" IDLE_SLEEP="${IDLE_SLEEP:-3}" +IMMICH_CHECK_PATH="${IMMICH_CHECK_PATH:-/api/server/ping}" +CURL_OPTS="-sS -m 3 -o /dev/null -w %{http_code}" +[ "${ALLOW_INSECURE_SSL:-false}" = "true" ] && CURL_OPTS="$CURL_OPTS -k" +VERBOSE="${VERBOSE:-1}" # 0=quiet, 1=summary, 2=also list some files +LOG_FILE="${LOG_FILE:-}" + +# sanity for concurrency case "$CONCURRENCY" in ''|*[!0-9]*) CONCURRENCY=1 ;; esac [ "$CONCURRENCY" -lt 1 ] && CONCURRENCY=1 -echo "[uploader] Target: ${IMMICH_URL}" -echo "[uploader] Source (drop): ${SCAN_DIR} (scan every ${SCAN_INTERVAL}s, concurrency ${CONCURRENCY})" - -CURL_OPTS="-sS -m 5 -o /dev/null -w %{http_code}" -[ "$ALLOW_INSECURE_SSL" = "true" ] && CURL_OPTS="$CURL_OPTS -k" -PING="${IMMICH_URL%/}/api/server-info" - -wait_for_immich() { - backoff=1 - max_backoff=30 - last="" - while :; do - code="$(curl $CURL_OPTS "$PING" 2>/dev/null || true)" - case "$code" in - 2*|3*|4*) [ "$last" != "up" ] && echo "[uploader] Immich is up (HTTP $code)."; return 0 ;; - *) [ "$last" != "down" ] && echo "[uploader] Waiting for Immich at $PING ...";; - esac - last="down" - sleep "$backoff" - [ "$backoff" -lt "$max_backoff" ] && backoff=$((backoff*2)) - [ "$backoff" -gt "$max_backoff" ] && backoff="$max_backoff" - done -} - -prune_empty_parents() { - dir="$(dirname "$1")" - while [ "$dir" != "$SCAN_DIR" ]; do - rmdir "$dir" 2>/dev/null || break - dir="$(dirname "$dir")" - done -} - -upload_one() { - f="$1" - if immich-go upload from-folder \ - --server "${IMMICH_URL}" \ - --api-key "${IMMICH_API_KEY}" \ - --no-ui --on-server-errors continue \ - --pause-immich-jobs=false \ - --recursive=false \ - "$f" >/dev/null 2>&1 - then - echo "[uploader] OK: $(basename "$f")" - if [ "$DELETE_ON_SUCCESS" = "true" ]; then - rm -f -- "$f" || true - prune_empty_parents "$f" - fi - else - echo "[uploader] FAIL: $(basename "$f") — will retry next loop" +# -------- Helpers -------- +ts() { date '+%Y-%m-%d %H:%M:%S'; } +log() { + TS_NOW="$(ts)" + echo "[$TS_NOW] $*" + if [ -n "${LOG_FILE:-}" ]; then + echo "[$TS_NOW] $*" >> "$LOG_FILE" 2>/dev/null || true fi } -upload_batch() { - count="$#" - echo "[uploader] Found $count file(s) to upload" +wait_for_immich() { + backoff=1; max_backoff=15; state="init" + PING="${IMMICH_URL%/}${IMMICH_CHECK_PATH}" + while :; do + code="$(curl $CURL_OPTS "$PING" 2>/dev/null || echo 000)" + case "$code" in + 000) + [ "$state" = "down" ] || log "[uploader] Waiting for Immich at $PING (HTTP $code) ..." + sleep "$backoff"; backoff=$((backoff*2)); [ "$backoff" -gt "$max_backoff" ] && backoff="$max_backoff"; state="down" + ;; + *) + [ "$state" = "up" ] || log "[uploader] Immich reachable (HTTP $code) at $PING" + return 0 + ;; + esac + done +} + +# -------- Worker script (invoked per file) -------- +WORKER="/tmp/immich_worker.sh" +cat > "$WORKER" <<'EOF' +#!/bin/sh +set -eu +f="$1" + +ts() { date '+%Y-%m-%d %H:%M:%S'; } + +start_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")" +errf="$(mktemp -p /tmp immich_err.XXXXXX)" + +cmd="immich-go upload from-folder --server \"${IMMICH_URL}\" --api-key \"${IMMICH_API_KEY}\" --no-ui --recursive=false --pause-immich-jobs=false" +[ "${ALLOW_INSECURE_SSL:-false}" = "true" ] && cmd="$cmd --skip-verify-ssl" + +if output="$(eval "$cmd \"$f\"" 1>/dev/null 2>"$errf")" +then + end_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")" + elapsed=$((end_ms-start_ms)) + echo "[uploader] OK: $(basename "$f") (${elapsed}ms)" + if [ -n "${LOG_FILE:-}" ]; then + echo "[$(ts)] OK: $(basename "$f") (${elapsed}ms)" >> "${LOG_FILE}" 2>/dev/null || true + fi + if [ "${DELETE_ON_SUCCESS:-true}" = "true" ]; then + rm -f -- "$f" || true + d="$(dirname "$f")" + while [ "$d" != "${SCAN_DIR:-/drop}" ]; do + rmdir "$d" 2>/dev/null || break + d="$(dirname "$d")" + done + fi +else + end_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")" + elapsed=$((end_ms-start_ms)) + echo "[uploader] FAIL: $(basename "$f") (${elapsed}ms) — will retry next loop" + if [ -s "$errf" ]; then + echo " immich-go error: $(head -n 2 "$errf" | tr '\n' ' ')" >&2 + if [ -n "${LOG_FILE:-}" ]; then + printf '[%s] FAIL: %s (%sms) — ' "$(ts)" "$(basename "$f")" "$elapsed" >> "${LOG_FILE}" 2>/dev/null || true + head -n 2 "$errf" >> "${LOG_FILE}" 2>/dev/null || true + fi + fi +fi + +rm -f "$errf" 2>/dev/null || true +EOF +chmod +x "$WORKER" + +# -------- Startup info -------- +echo "[uploader] Target: ${IMMICH_URL}" +echo "[uploader] Healthcheck: ${IMMICH_CHECK_PATH}" +echo "[uploader] Scan dir: ${SCAN_DIR} (every ${SCAN_INTERVAL}s, concurrency ${CONCURRENCY})" +echo "[uploader] Delete on success: ${DELETE_ON_SUCCESS}" + +wait_for_immich + +# -------- Main loop -------- +while :; do + scan_t="$(ts)" + TMP_LIST="$(mktemp -p /tmp uploader_list.XXXXXX)" + : > "$TMP_LIST" + + find "$SCAN_DIR" -type f ! -name '.*' -size +0c -print0 2>/dev/null >> "$TMP_LIST" + + total="$(tr -cd '\0' < "$TMP_LIST" | wc -c | tr -d ' ')" + if [ "$total" -eq 0 ]; then + [ "$VERBOSE" = "1" ] && log "[scan] $scan_t — 0 files; idle ${SCAN_INTERVAL}s" + rm -f "$TMP_LIST" 2>/dev/null || true + sleep "$IDLE_SLEEP" + continue + fi + + log "[scan] $scan_t — found ${total} file(s) in $SCAN_DIR" + + # Upload files with basic concurrency running=0 pids="" - - for f in "$@"; do - upload_one "$f" & + while IFS= read -r -d '' f; do + sh "$WORKER" "$f" & pid=$! pids="$pids $pid" running=$((running+1)) @@ -86,28 +141,12 @@ upload_batch() { wait "$pid_to_wait" || true running=$((running-1)) fi - done + done < "$TMP_LIST" for pid in $pids; do wait "$pid" || true done -} -while :; do - wait_for_immich - - file_list="$(find "$SCAN_DIR" -type f -size +0c -print0 | xargs -0 -I{} printf '%s\n' "{}")" - if [ -z "$file_list" ]; then - sleep "$IDLE_SLEEP" - continue - fi - - oldifs="$IFS" - IFS=' -' - set -- $file_list - IFS="$oldifs" - - upload_batch "$@" + rm -f "$TMP_LIST" 2>/dev/null || true sleep "$SCAN_INTERVAL" done