fixing issues
This commit is contained in:
RandyJC
191
uploader.sh
191
uploader.sh
@@ -1,80 +1,135 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
IMMICH_URL="${IMMICH_URL:?IMMICH_URL is required}" # e.g. http://immich:8080
|
||||
# -------- Config from environment --------
|
||||
IMMICH_URL="${IMMICH_URL:?IMMICH_URL is required}"
|
||||
IMMICH_API_KEY="${IMMICH_API_KEY:?IMMICH_API_KEY is required}"
|
||||
|
||||
SCAN_DIR="${SCAN_DIR:-/drop}"
|
||||
DELETE_ON_SUCCESS="${DELETE_ON_SUCCESS:-true}"
|
||||
SCAN_INTERVAL="${SCAN_INTERVAL:-5}"
|
||||
ALLOW_INSECURE_SSL="${ALLOW_INSECURE_SSL:-false}"
|
||||
SCAN_INTERVAL="${SCAN_INTERVAL:-5}" # seconds between scans
|
||||
CONCURRENCY="${CONCURRENCY:-4}"
|
||||
IDLE_SLEEP="${IDLE_SLEEP:-3}"
|
||||
IMMICH_CHECK_PATH="${IMMICH_CHECK_PATH:-/api/server/ping}"
|
||||
CURL_OPTS="-sS -m 3 -o /dev/null -w %{http_code}"
|
||||
[ "${ALLOW_INSECURE_SSL:-false}" = "true" ] && CURL_OPTS="$CURL_OPTS -k"
|
||||
|
||||
VERBOSE="${VERBOSE:-1}" # 0=quiet, 1=summary, 2=also list some files
|
||||
LOG_FILE="${LOG_FILE:-}"
|
||||
|
||||
# sanity for concurrency
|
||||
case "$CONCURRENCY" in
|
||||
''|*[!0-9]*) CONCURRENCY=1 ;;
|
||||
esac
|
||||
[ "$CONCURRENCY" -lt 1 ] && CONCURRENCY=1
|
||||
|
||||
echo "[uploader] Target: ${IMMICH_URL}"
|
||||
echo "[uploader] Source (drop): ${SCAN_DIR} (scan every ${SCAN_INTERVAL}s, concurrency ${CONCURRENCY})"
|
||||
|
||||
CURL_OPTS="-sS -m 5 -o /dev/null -w %{http_code}"
|
||||
[ "$ALLOW_INSECURE_SSL" = "true" ] && CURL_OPTS="$CURL_OPTS -k"
|
||||
PING="${IMMICH_URL%/}/api/server-info"
|
||||
|
||||
wait_for_immich() {
|
||||
backoff=1
|
||||
max_backoff=30
|
||||
last=""
|
||||
while :; do
|
||||
code="$(curl $CURL_OPTS "$PING" 2>/dev/null || true)"
|
||||
case "$code" in
|
||||
2*|3*|4*) [ "$last" != "up" ] && echo "[uploader] Immich is up (HTTP $code)."; return 0 ;;
|
||||
*) [ "$last" != "down" ] && echo "[uploader] Waiting for Immich at $PING ...";;
|
||||
esac
|
||||
last="down"
|
||||
sleep "$backoff"
|
||||
[ "$backoff" -lt "$max_backoff" ] && backoff=$((backoff*2))
|
||||
[ "$backoff" -gt "$max_backoff" ] && backoff="$max_backoff"
|
||||
done
|
||||
}
|
||||
|
||||
prune_empty_parents() {
|
||||
dir="$(dirname "$1")"
|
||||
while [ "$dir" != "$SCAN_DIR" ]; do
|
||||
rmdir "$dir" 2>/dev/null || break
|
||||
dir="$(dirname "$dir")"
|
||||
done
|
||||
}
|
||||
|
||||
upload_one() {
|
||||
f="$1"
|
||||
if immich-go upload from-folder \
|
||||
--server "${IMMICH_URL}" \
|
||||
--api-key "${IMMICH_API_KEY}" \
|
||||
--no-ui --on-server-errors continue \
|
||||
--pause-immich-jobs=false \
|
||||
--recursive=false \
|
||||
"$f" >/dev/null 2>&1
|
||||
then
|
||||
echo "[uploader] OK: $(basename "$f")"
|
||||
if [ "$DELETE_ON_SUCCESS" = "true" ]; then
|
||||
rm -f -- "$f" || true
|
||||
prune_empty_parents "$f"
|
||||
fi
|
||||
else
|
||||
echo "[uploader] FAIL: $(basename "$f") — will retry next loop"
|
||||
# -------- Helpers --------
|
||||
ts() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
log() {
|
||||
TS_NOW="$(ts)"
|
||||
echo "[$TS_NOW] $*"
|
||||
if [ -n "${LOG_FILE:-}" ]; then
|
||||
echo "[$TS_NOW] $*" >> "$LOG_FILE" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
upload_batch() {
|
||||
count="$#"
|
||||
echo "[uploader] Found $count file(s) to upload"
|
||||
wait_for_immich() {
|
||||
backoff=1; max_backoff=15; state="init"
|
||||
PING="${IMMICH_URL%/}${IMMICH_CHECK_PATH}"
|
||||
while :; do
|
||||
code="$(curl $CURL_OPTS "$PING" 2>/dev/null || echo 000)"
|
||||
case "$code" in
|
||||
000)
|
||||
[ "$state" = "down" ] || log "[uploader] Waiting for Immich at $PING (HTTP $code) ..."
|
||||
sleep "$backoff"; backoff=$((backoff*2)); [ "$backoff" -gt "$max_backoff" ] && backoff="$max_backoff"; state="down"
|
||||
;;
|
||||
*)
|
||||
[ "$state" = "up" ] || log "[uploader] Immich reachable (HTTP $code) at $PING"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# -------- Worker script (invoked per file) --------
|
||||
WORKER="/tmp/immich_worker.sh"
|
||||
cat > "$WORKER" <<'EOF'
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
f="$1"
|
||||
|
||||
ts() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
|
||||
start_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")"
|
||||
errf="$(mktemp -p /tmp immich_err.XXXXXX)"
|
||||
|
||||
cmd="immich-go upload from-folder --server \"${IMMICH_URL}\" --api-key \"${IMMICH_API_KEY}\" --no-ui --recursive=false --pause-immich-jobs=false"
|
||||
[ "${ALLOW_INSECURE_SSL:-false}" = "true" ] && cmd="$cmd --skip-verify-ssl"
|
||||
|
||||
if output="$(eval "$cmd \"$f\"" 1>/dev/null 2>"$errf")"
|
||||
then
|
||||
end_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")"
|
||||
elapsed=$((end_ms-start_ms))
|
||||
echo "[uploader] OK: $(basename "$f") (${elapsed}ms)"
|
||||
if [ -n "${LOG_FILE:-}" ]; then
|
||||
echo "[$(ts)] OK: $(basename "$f") (${elapsed}ms)" >> "${LOG_FILE}" 2>/dev/null || true
|
||||
fi
|
||||
if [ "${DELETE_ON_SUCCESS:-true}" = "true" ]; then
|
||||
rm -f -- "$f" || true
|
||||
d="$(dirname "$f")"
|
||||
while [ "$d" != "${SCAN_DIR:-/drop}" ]; do
|
||||
rmdir "$d" 2>/dev/null || break
|
||||
d="$(dirname "$d")"
|
||||
done
|
||||
fi
|
||||
else
|
||||
end_ms="$(date +%s%3N 2>/dev/null || echo "$(date +%s)000")"
|
||||
elapsed=$((end_ms-start_ms))
|
||||
echo "[uploader] FAIL: $(basename "$f") (${elapsed}ms) — will retry next loop"
|
||||
if [ -s "$errf" ]; then
|
||||
echo " immich-go error: $(head -n 2 "$errf" | tr '\n' ' ')" >&2
|
||||
if [ -n "${LOG_FILE:-}" ]; then
|
||||
printf '[%s] FAIL: %s (%sms) — ' "$(ts)" "$(basename "$f")" "$elapsed" >> "${LOG_FILE}" 2>/dev/null || true
|
||||
head -n 2 "$errf" >> "${LOG_FILE}" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -f "$errf" 2>/dev/null || true
|
||||
EOF
|
||||
chmod +x "$WORKER"
|
||||
|
||||
# -------- Startup info --------
|
||||
echo "[uploader] Target: ${IMMICH_URL}"
|
||||
echo "[uploader] Healthcheck: ${IMMICH_CHECK_PATH}"
|
||||
echo "[uploader] Scan dir: ${SCAN_DIR} (every ${SCAN_INTERVAL}s, concurrency ${CONCURRENCY})"
|
||||
echo "[uploader] Delete on success: ${DELETE_ON_SUCCESS}"
|
||||
|
||||
wait_for_immich
|
||||
|
||||
# -------- Main loop --------
|
||||
while :; do
|
||||
scan_t="$(ts)"
|
||||
TMP_LIST="$(mktemp -p /tmp uploader_list.XXXXXX)"
|
||||
: > "$TMP_LIST"
|
||||
|
||||
find "$SCAN_DIR" -type f ! -name '.*' -size +0c -print0 2>/dev/null >> "$TMP_LIST"
|
||||
|
||||
total="$(tr -cd '\0' < "$TMP_LIST" | wc -c | tr -d ' ')"
|
||||
if [ "$total" -eq 0 ]; then
|
||||
[ "$VERBOSE" = "1" ] && log "[scan] $scan_t — 0 files; idle ${SCAN_INTERVAL}s"
|
||||
rm -f "$TMP_LIST" 2>/dev/null || true
|
||||
sleep "$IDLE_SLEEP"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "[scan] $scan_t — found ${total} file(s) in $SCAN_DIR"
|
||||
|
||||
# Upload files with basic concurrency
|
||||
running=0
|
||||
pids=""
|
||||
|
||||
for f in "$@"; do
|
||||
upload_one "$f" &
|
||||
while IFS= read -r -d '' f; do
|
||||
sh "$WORKER" "$f" &
|
||||
pid=$!
|
||||
pids="$pids $pid"
|
||||
running=$((running+1))
|
||||
@@ -86,28 +141,12 @@ upload_batch() {
|
||||
wait "$pid_to_wait" || true
|
||||
running=$((running-1))
|
||||
fi
|
||||
done
|
||||
done < "$TMP_LIST"
|
||||
|
||||
for pid in $pids; do
|
||||
wait "$pid" || true
|
||||
done
|
||||
}
|
||||
|
||||
while :; do
|
||||
wait_for_immich
|
||||
|
||||
file_list="$(find "$SCAN_DIR" -type f -size +0c -print0 | xargs -0 -I{} printf '%s\n' "{}")"
|
||||
if [ -z "$file_list" ]; then
|
||||
sleep "$IDLE_SLEEP"
|
||||
continue
|
||||
fi
|
||||
|
||||
oldifs="$IFS"
|
||||
IFS='
|
||||
'
|
||||
set -- $file_list
|
||||
IFS="$oldifs"
|
||||
|
||||
upload_batch "$@"
|
||||
rm -f "$TMP_LIST" 2>/dev/null || true
|
||||
sleep "$SCAN_INTERVAL"
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user