fixing issues

2025-12-01 14:34:24 +01:00
parent 6f03b99d32
commit dd6acfa7b8
3 changed files with 151 additions and 91 deletions
--- a/.env
+++ b/.env
@@ -11,14 +11,14 @@ DROP=/mnt/user/photosync/drop
 IMMICH_URL=http://immich:8080
 IMMICH_API_KEY=REPLACE_ME
 ALLOW_INSECURE_SSL=true
-IMMICH_GO_VERSION=0.31.0
+IMMICH_GO_VERSION=0.27.0

 # Mover tuning
 STABLE_FOR=5
 SCAN_FALLBACK=60

 # Uploader tuning
-UPLOAD_CONCURRENCY=4
 UPLOAD_SCAN_INTERVAL=5
+UPLOAD_CONCURRENCY=4
 UPLOAD_IDLE_SLEEP=3
 DELETE_ON_SUCCESS=true
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,16 +5,25 @@ services:
    restart: unless-stopped
    environment:
      - TZ=${TZ:-UTC}
-      - INBOX=${INBOX}
-      - DROP=${DROP}
+      - INBOX=/inbox
+      - DROP=/drop
      - STABLE_FOR=${STABLE_FOR:-5}
      - SCAN_FALLBACK=${SCAN_FALLBACK:-60}
+      - PUID=${PUID:-1000}
+      - PGID=${PGID:-1000}
    volumes:
      - ${INBOX}:/inbox:rw
      - ${DROP}:/drop:rw
      - ./move_stable.sh:/scripts/move_stable.sh:ro
-    entrypoint: ["/bin/sh","-c","apk add --no-cache inotify-tools coreutils >/dev/null && chmod +x /scripts/move_stable.sh && exec /scripts/move_stable.sh"]
-    user: "${PUID:-1000}:${PGID:-1000}"
+    networks:
+      - npm-network
+    entrypoint:
+      - /bin/sh
+      - -c
+      - |
+        set -e
+        apk add --no-cache inotify-tools coreutils su-exec >/dev/null
+        exec su-exec "$${PUID}:$${PGID}" /scripts/move_stable.sh

  photosync-uploader:
    image: alpine:3.19
@@ -29,10 +38,14 @@ services:
      - IMMICH_GO_VERSION=${IMMICH_GO_VERSION:-0.31.0}
      - SCAN_DIR=/drop
      - SCAN_INTERVAL=${UPLOAD_SCAN_INTERVAL:-5}
-      - IDLE_SLEEP=${UPLOAD_IDLE_SLEEP:-3}
      - CONCURRENCY=${UPLOAD_CONCURRENCY:-4}
+      - IDLE_SLEEP=${UPLOAD_IDLE_SLEEP:-3}
      - DELETE_ON_SUCCESS=${DELETE_ON_SUCCESS:-true}
      - ALLOW_INSECURE_SSL=${ALLOW_INSECURE_SSL:-false}
+      - PUID=${PUID:-1000}
+      - PGID=${PGID:-1000}
+      - XDG_CACHE_HOME=/tmp/immich-cache
+      - HOME=/tmp
    volumes:
      - ${DROP}:/drop:rw
      - ./uploader.sh:/scripts/uploader.sh:ro
@@ -41,17 +54,25 @@ services:
      - -c
      - |
        set -e
-        apk add --no-cache curl ca-certificates >/dev/null
+        apk add --no-cache curl ca-certificates su-exec tar >/dev/null
        arch="$$(uname -m)"
        case "$$arch" in
-          x86_64)   arch="amd64" ;;
+          x86_64)   arch="x86_64" ;;
          aarch64)  arch="arm64" ;;
+          arm64)    arch="arm64" ;;
          *) echo "Unsupported architecture: $$arch"; exit 1 ;;
        esac
-        url="https://github.com/simulot/immich-go/releases/download/v$${IMMICH_GO_VERSION}/immich-go_Linux_$${arch}"
+        url="https://github.com/simulot/immich-go/releases/download/v$${IMMICH_GO_VERSION}/immich-go_Linux_$${arch}.tar.gz"
        echo "[uploader] Downloading immich-go $${IMMICH_GO_VERSION} for $$arch"
-        curl -fsSL -o /usr/local/bin/immich-go "$$url"
-        chmod +x /usr/local/bin/immich-go
-        chmod +x /scripts/uploader.sh
-        exec /scripts/uploader.sh
-    user: "${PUID:-1000}:${PGID:-1000}"
+        tmp="$$(mktemp -d)"
+        curl -fsSL "$$url" -o "$$tmp/immich-go.tar.gz"
+        tar -xzf "$$tmp/immich-go.tar.gz" -C "$$tmp"
+        install -m 0755 "$$tmp/immich-go" /usr/local/bin/immich-go
+        rm -rf "$$tmp"
+        exec su-exec "$${PUID}:$${PGID}" /scripts/uploader.sh
+    networks:
+      - npm-network
+
+networks:
+  npm-network:
+    external: true
--- a/uploader.sh
+++ b/uploader.sh
@@ -1,80 +1,135 @@
 #!/bin/sh
 set -eu

-IMMICH_URL="${IMMICH_URL:?IMMICH_URL is required}"     # e.g. http://immich:8080
+# -------- Config from environment --------
+IMMICH_URL="${IMMICH_URL:?IMMICH_URL is required}"
 IMMICH_API_KEY="${IMMICH_API_KEY:?IMMICH_API_KEY is required}"
+
 SCAN_DIR="${SCAN_DIR:-/drop}"
 DELETE_ON_SUCCESS="${DELETE_ON_SUCCESS:-true}"
-SCAN_INTERVAL="${SCAN_INTERVAL:-5}"
-ALLOW_INSECURE_SSL="${ALLOW_INSECURE_SSL:-false}"
+SCAN_INTERVAL="${SCAN_INTERVAL:-5}"          # seconds between scans
 CONCURRENCY="${CONCURRENCY:-4}"
 IDLE_SLEEP="${IDLE_SLEEP:-3}"
+IMMICH_CHECK_PATH="${IMMICH_CHECK_PATH:-/api/server/ping}"
+CURL_OPTS="-sS -m 3 -o /dev/null -w %{http_code}"
+[ "${ALLOW_INSECURE_SSL:-false}" = "true" ] && CURL_OPTS="$CURL_OPTS -k"

+VERBOSE="${VERBOSE:-1}"                      # 0=quiet, 1=summary, 2=also list some files
+LOG_FILE="${LOG_FILE:-}"
+
+# sanity for concurrency
 case "$CONCURRENCY" in
  ''|*[!0-9]*) CONCURRENCY=1 ;;
 esac
 [ "$CONCURRENCY" -lt 1 ] && CONCURRENCY=1

-echo "[uploader] Target: ${IMMICH_URL}"
-echo "[uploader] Source (drop): ${SCAN_DIR} (scan every ${SCAN_INTERVAL}s, concurrency ${CONCURRENCY})"
-
-CURL_OPTS="-sS -m 5 -o /dev/null -w %{http_code}"
-[ "$ALLOW_INSECURE_SSL" = "true" ] && CURL_OPTS="$CURL_OPTS -k"
-PING="${IMMICH_URL%/}/api/server-info"
-
-wait_for_immich() {
-  backoff=1
-  max_backoff=30
-  last=""
-  while :; do
-    code="$(curl $CURL_OPTS "$PING" 2>/dev/null || true)"
-    case "$code" in
-      2*|3*|4*) [ "$last" != "up" ] && echo "[uploader] Immich is up (HTTP $code)."; return 0 ;;
-      *)        [ "$last" != "down" ] && echo "[uploader] Waiting for Immich at $PING ...";;
-    esac
-    last="down"
-    sleep "$backoff"
-    [ "$backoff" -lt "$max_backoff" ] && backoff=$((backoff*2))
-    [ "$backoff" -gt "$max_backoff" ] && backoff="$max_backoff"
-  done
-}
-
-prune_empty_parents() {
-  dir="$(dirname "$1")"
-  while [ "$dir" != "$SCAN_DIR" ]; do
-    rmdir "$dir" 2>/dev/null || break
-    dir="$(dirname "$dir")"
-  done
-}
-
-upload_one() {
-  f="$1"
-  if immich-go upload from-folder \
-        --server "${IMMICH_URL}" \
-        --api-key "${IMMICH_API_KEY}" \
-        --no-ui --on-server-errors continue \
-        --pause-immich-jobs=false \
-        --recursive=false \
-        "$f" >/dev/null 2>&1
-  then
-    echo "[uploader] OK: $(basename "$f")"
-    if [ "$DELETE_ON_SUCCESS" = "true" ]; then
-      rm -f -- "$f" || true
-      prune_empty_parents "$f"
-    fi
-  else
-    echo "[uploader] FAIL: $(basename "$f") — will retry next loop"
+# -------- Helpers --------
+ts() { date '+%Y-%m-%d %H:%M:%S'; }
+log() {
+  TS_NOW="$(ts)"
+  echo "[$TS_NOW] $*"
+  if [ -n "${LOG_FILE:-}" ]; then
+    echo "[$TS_NOW] $*" >> "$LOG_FILE" 2>/dev/null || true
  fi
 }

-upload_batch() {
-  count="$#"
-  echo "[uploader] Found $count file(s) to upload"
+wait_for_immich() {
+  backoff=1; max_backoff=15; state="init"
+  PING="${IMMICH_URL%/}${IMMICH_CHECK_PATH}"
+  while :; do
+    code="$(curl $CURL_OPTS "$PING" 2>/dev/null || echo 000)"
+    case "$code" in
+      000)
+        [ "$state" = "down" ] || log "[uploader] Waiting for Immich at $PING (HTTP $code) ..."
+        sleep "$backoff"; backoff=$((backoff*2)); [ "$backoff" -gt "$max_backoff" ] && backoff="$max_backoff"; state="down"
+        ;;
+      *)
+        [ "$state" = "up" ] || log "[uploader] Immich reachable (HTTP $code) at $PING"
+        return 0
+        ;;
+    esac
+  done
+}
+
+# -------- Worker script (invoked per file) --------
+WORKER="/tmp/immich_worker.sh"
+cat > "$WORKER" <<'EOF'
+#!/bin/sh
+set -eu
+f="$1"
+
+ts() { date '+%Y-%m-%d %H:%M:%S'; }
+
+start_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")"
+errf="$(mktemp -p /tmp immich_err.XXXXXX)"
+
+cmd="immich-go upload from-folder --server \"${IMMICH_URL}\" --api-key \"${IMMICH_API_KEY}\" --no-ui --recursive=false --pause-immich-jobs=false"
+[ "${ALLOW_INSECURE_SSL:-false}" = "true" ] && cmd="$cmd --skip-verify-ssl"
+
+if output="$(eval "$cmd \"$f\"" 1>/dev/null 2>"$errf")"
+then
+  end_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")"
+  elapsed=$((end_ms-start_ms))
+  echo "[uploader] OK: $(basename "$f") (${elapsed}ms)"
+  if [ -n "${LOG_FILE:-}" ]; then
+    echo "[$(ts)] OK: $(basename "$f") (${elapsed}ms)" >> "${LOG_FILE}" 2>/dev/null || true
+  fi
+  if [ "${DELETE_ON_SUCCESS:-true}" = "true" ]; then
+    rm -f -- "$f" || true
+    d="$(dirname "$f")"
+    while [ "$d" != "${SCAN_DIR:-/drop}" ]; do
+      rmdir "$d" 2>/dev/null || break
+      d="$(dirname "$d")"
+    done
+  fi
+else
+  end_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")"
+  elapsed=$((end_ms-start_ms))
+  echo "[uploader] FAIL: $(basename "$f") (${elapsed}ms) — will retry next loop"
+  if [ -s "$errf" ]; then
+    echo "         immich-go error: $(head -n 2 "$errf" | tr '\n' ' ')" >&2
+    if [ -n "${LOG_FILE:-}" ]; then
+      printf '[%s] FAIL: %s (%sms) — ' "$(ts)" "$(basename "$f")" "$elapsed" >> "${LOG_FILE}" 2>/dev/null || true
+      head -n 2 "$errf" >> "${LOG_FILE}" 2>/dev/null || true
+    fi
+  fi
+fi
+
+rm -f "$errf" 2>/dev/null || true
+EOF
+chmod +x "$WORKER"
+
+# -------- Startup info --------
+echo "[uploader] Target: ${IMMICH_URL}"
+echo "[uploader] Healthcheck: ${IMMICH_CHECK_PATH}"
+echo "[uploader] Scan dir: ${SCAN_DIR} (every ${SCAN_INTERVAL}s, concurrency ${CONCURRENCY})"
+echo "[uploader] Delete on success: ${DELETE_ON_SUCCESS}"
+
+wait_for_immich
+
+# -------- Main loop --------
+while :; do
+  scan_t="$(ts)"
+  TMP_LIST="$(mktemp -p /tmp uploader_list.XXXXXX)"
+  : > "$TMP_LIST"
+
+  find "$SCAN_DIR" -type f ! -name '.*' -size +0c -print0 2>/dev/null >> "$TMP_LIST"
+
+  total="$(tr -cd '\0' < "$TMP_LIST" | wc -c | tr -d ' ')"
+  if [ "$total" -eq 0 ]; then
+    [ "$VERBOSE" = "1" ] && log "[scan] $scan_t — 0 files; idle ${SCAN_INTERVAL}s"
+    rm -f "$TMP_LIST" 2>/dev/null || true
+    sleep "$IDLE_SLEEP"
+    continue
+  fi
+
+  log "[scan] $scan_t — found ${total} file(s) in $SCAN_DIR"
+
+  # Upload files with basic concurrency
  running=0
  pids=""
-
-  for f in "$@"; do
-    upload_one "$f" &
+  while IFS= read -r -d '' f; do
+    sh "$WORKER" "$f" &
    pid=$!
    pids="$pids $pid"
    running=$((running+1))
@@ -86,28 +141,12 @@ upload_batch() {
      wait "$pid_to_wait" || true
      running=$((running-1))
    fi
-  done
+  done < "$TMP_LIST"

  for pid in $pids; do
    wait "$pid" || true
  done
-}

-while :; do
-  wait_for_immich
-
-  file_list="$(find "$SCAN_DIR" -type f -size +0c -print0 | xargs -0 -I{} printf '%s\n' "{}")"
-  if [ -z "$file_list" ]; then
-    sleep "$IDLE_SLEEP"
-    continue
-  fi
-
-  oldifs="$IFS"
-  IFS='
-'
-  set -- $file_list
-  IFS="$oldifs"
-
-  upload_batch "$@"
+  rm -f "$TMP_LIST" 2>/dev/null || true
  sleep "$SCAN_INTERVAL"
 done