From 95b4d1fe5b1de5b67eb6be2d42c1970c66395484 Mon Sep 17 00:00:00 2001
From: ppr <ppr@noreply.example.org>
Date: Tue, 9 Apr 2024 16:22:15 -0400
Subject: [PATCH] support download quota, skip existing files, use globals and
 local vars

---
 icerbox.sh | 174 +++++++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 123 insertions(+), 51 deletions(-)

diff --git a/icerbox.sh b/icerbox.sh
index c10bc2c..9434475 100644
--- a/icerbox.sh
+++ b/icerbox.sh
@@ -2,97 +2,157 @@
 set -ueo pipefail
 
 api='https://icerbox.com/api/v1/'
-jwt='/dev/shm/icerbox.token'
+jwt=~/.icerbox.token
 cfg=~/.icerbox.conf
+ct='Content-Type: application/json'
 
 err() {
- echo "$1" >/dev/stderr
+ echo "err: ${1}" >/dev/stderr
  exit 1
 }
 
 warn() {
- echo "$1" >/dev/stderr
+ echo "warn: ${1}" >/dev/stderr
+}
+
+join_by() {
+  local d=${1-} f=${2-}
+  if shift 2; then
+    printf %s "$f" "${@/#/$d}"
+  fi
 }
 
 icerbox_handle_response() {
     test -n "$1" || err "empty http status code"
     test -s "$2" || err "empty json reponse body"
     if [ "$1" -ne 200 ]; then
-        jq -r '.status_code?' "${2}" >/dev/stderr
-        jq -r '.message?' "${2}" >/dev/stderr
+        jq '.' "${2}" >/dev/stderr || true
         err "request failed with http ${1}"
     fi
 }
 
 icerbox_auth_login() {
-    out='/dev/shm/icerbox.auth.login.json'
+    local out='/dev/shm/icerbox.auth.login.json'
     test -f "$out" && rm "$out"
-    code=$(curl -s -w '%{http_code}' -H 'Content-Type: application/json' -o "$out" -d "email=${1}" -d "password=${2}" "${api}auth/login")
-    test $? -eq 0 || err "curl request failed"
+    local code=$(curl -s -w '%{http_code}' -H "$ct" -o "$out" -d "email=${1}" -d "password=${2}" "${api}auth/login")
+    test $? -eq 0 || err "auth/login: curl request failed"
     icerbox_handle_response "$code" "$out"
     token=$(jq -r '.token?' "$out")
-    test -n "$token" || err "empty jwt token"
-    echo "$token"
+    test -n "$token" || err "auth/login: empty jwt token"
 }
 
 icerbox_auth_refresh() {
-    out='/dev/shm/icerbox.auth_refresh.json'
+    local out='/dev/shm/icerbox.auth_refresh.json'
     test -f "$out" && rm "$out"
     # force POST without --data
-    code=$(curl -s -w '%{http_code}' -XPOST -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" "${api}auth/refresh")
-    test $? -eq 0 || err "curl request failed"
+    local code=$(curl -s -w '%{http_code}' -XPOST -H "$auth" -H "$ct" -o "$out" "${api}auth/refresh")
+    test $? -eq 0 || err "auth/refresh: curl request failed"
     icerbox_handle_response "$code" "$out"
     token=$(jq -r '.token?' "$out")
-    test -n "$token" || err "empty jwt token"
-    echo "$token"
+    test -n "$token" || err "auth/refresh: empty jwt token"
 }
 
 icerbox_user_account() {
-    out='/dev/shm/icerbox.user.account.json'
+    local out='/dev/shm/icerbox.user.account.json'
     test -f "$out" && rm "$out"
-    code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" "${api}user/account")
-    test $? -eq 0 || err "curl request failed"
+    local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" "${api}user/account")
+    test $? -eq 0 || err "user/account: curl request failed"
     icerbox_handle_response "$code" "$out"
 }
 
+icerbox_download_quota() {
+	local in='/dev/shm/icerbox.user.account.json'
+	test -f "$in"
+	jq -r '.data.package.volume - .data.downloaded_today' "$in"
+}
+
 icerbox_file() {
-    out='/dev/shm/icerbox.file.json'
+    local out='/dev/shm/icerbox.file.json'
     test -f "$out" && rm "$out"
-    code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" -d id="$2" -G "${api}file")
-    test $? -eq 0 || err "curl request failed"
+    local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d id="$2" -G "${api}file")
+    test $? -eq 0 || err "file: curl request failed"
     icerbox_handle_response "$code" "$out"
-    status=$(jq -r '.data?.status?' "$out")
-    test $? -eq 0 || err "failed parsing json"
-    test -n "$status" || err "empty status"
+    local status=$(jq -r '.data?.status?' "$out")
+    test $? -eq 0 || err "file: failed parsing json"
+    test -n "$status" || err "file: empty status"
     if [ "$status" != "active" ]; then 
-        warn "file ${2} is not active"
+        warn "skip: file id ${2} is not active"
         return 1
-    else
-        return 0
     fi
+    local name=$(jq -r '.data?.name?' "$out")
+    test $? -eq 0 || err "file: failed parsing json"
+    test -n "$name" || err "file: empty name"
+    if [ -f "$name" ]; then
+        warn "skip: file exists: ${name}"
+        return 1
+    fi
+    return 0
+}
+
+icerbox_files() {
+    local out='/dev/shm/icerbox.files.json'
+    test -f "$out" && rm "$out"
+    local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d ids="$1" -G "${api}files")
+    test $? -eq 0 || err "files: curl request failed"
+    icerbox_handle_response "$code" "$out"
+}
+
+icerbox_filter_files() {
+    local in='/dev/shm/icerbox.files.json'
+    test -f "$in" || err "filter files: no input file"
+	local size_sum=0
+	local IFS=$'\t'
+	while read -a arr; do
+		if [ -f "${arr[1]}" ]; then
+            local local_size=$(stat --printf '%s' "${arr[1]}")
+            test -n "$local_size" || err "filter_files: empty file size"
+            test "$local_size" -eq 0 && err "abort: file exists and is empty: ${arr[1]}"
+            if [ $local_size -eq ${arr[2]} ]; then
+    			warn "skip: file of same size exists ${arr[1]}"
+    			continue
+            elif [ $local_size -lt ${arr[2]} ]; then
+                warn "skip: file with smaller size exists ${arr[1]}"
+    			continue
+            elif [ $local_size -gt ${arr[2]} ]; then
+                err "abort: local file is larger than remote ${arr[1]}"
+            fi
+		fi
+		size_sum=$(( $size_sum + ${arr[2]} ))
+		if [ $size_sum -gt $download_quota ]; then
+			warn "skip: above download quota"
+			break
+		fi
+        to_download+=("${arr[0]}")
+    done < <(jq -r '.data[] | select(.status="active") | [.id,.name,.size] | @tsv' $in)
+    numfmt --format='download size: %f' --to=iec $size_sum
 }
 
 icerbox_dl_ticket() {
-    out='/dev/shm/icerbox.dl.ticket.json'
+    local out='/dev/shm/icerbox.dl.ticket.json'
     test -f "$out" && rm "$out"
     # endpoint does not accept file parameter with json as content type
-    code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -o "$out" -d file="$2" "${api}dl/ticket")
+    local code=$(curl -s -w '%{http_code}' -H "$auth" -o "$out" -d file="$1" "${api}dl/ticket")
     test $? -eq 0 || err "curl request failed"
+    if [ "$code" -eq 422 ]; then
+        warn "skip: file is unavailable:  $1"
+        return
+    fi
     icerbox_handle_response "$code" "$out"
-    url=$(jq -r '.url?' "$out")
+    local url=$(jq -r '.url?' "$out")
     test $? -eq 0 || err "failed parsing json"
     test -n "$url" || err "empty ticket url"
-    echo "$url"
+    tickets+=(-OJL "$url")
 }
 
 icerbox_file_id() {
     test -n "$1" || err "empty url"
     test "${1:0:20}" = "https://icerbox.com/" || err "not a icerbox url: $1"
     test "${#1}" -ge 28 || err "url $1 does not contain valid file id"
-    echo "${1:20:8}"
+    file_ids+=("${1:20:8}")
 }
 
 test "$#" -eq 0 && err "usage $0 https://icerbox.com/abcdefgh https://icerbox.com/abcdefgh ..."
+test "$#" -gt 666 && err "too many arguments. http request will fail with 414 Request-URI Too Large"
 test -f "$cfg" && source "$cfg" || err "please set 'email' and 'password' in ${cfg}"
 test -n "$email" || err "email not set"
 test -n "$password" || err "password not set"
@@ -103,8 +163,9 @@ if [ -f $jwt ] && [ -s $jwt ]; then
     token=$(< $jwt)
     token_age=$(stat --format=%Y "$jwt")
     now=$(date +%s)
-    if [ $token_age -le $(( $now - 1800 )) ]; then
-        token=$(icerbox_auth_refresh "$token")
+    if [ $token_age -le $(( $now - 2700 )) ]; then
+        auth="Authorization: Bearer ${token}"
+        icerbox_auth_refresh
         echo "$token" > "${jwt}.tmp"
         mv "${jwt}.tmp" "$jwt"
         echo "session refreshed"
@@ -112,32 +173,43 @@ if [ -f $jwt ] && [ -s $jwt ]; then
         echo "using active session"
     fi
 else
-    token=$(icerbox_auth_login "$email" "$password")
+    icerbox_auth_login "$email" "$password"
     echo "$token" > "${jwt}.tmp"
     mv "${jwt}.tmp" "${jwt}"
     echo "new session acquired"
 fi
 
-test -n "$token" || echo "no session token"
+test -n "$token" || err "no session token"
+auth="Authorization: Bearer ${token}"
 
-#icerbox_user_account "$token"
+icerbox_user_account
+download_quota=$(icerbox_download_quota)
 
+numfmt --format='daily download limit: %f' --to=iec $download_quota
+
+# globals filled by functions
+file_ids=()
+to_download=()
 tickets=()
+
+printf "requested: %d\n" "$#"
+
 for url in $@; do
-    file_id=$(icerbox_file_id "$url")
-    icerbox_file "$token" "$file_id"
-    if [ $? -ne 0 ]; then
-        echo "Skipping $url"
-        continue
-    else
-        dl_url=$(icerbox_dl_ticket "$token" "$file_id")
-        tickets+=(-OJL "$dl_url")
-    fi
+    icerbox_file_id "$url"
 done
 
-if [ ${#tickets} -gt 0 ]; then
-    curl --progress-bar --parallel-max 10 -Z "${tickets[@]}"
-else
-    err "nothing to download"
-fi
+icerbox_files "$(join_by , "${file_ids[@]}")"
+icerbox_filter_files
 
+test ${#to_download[@]} -gt 0 || (echo "nothing to download" ; exit 1)
+
+printf "to download: %d\n" "${#to_download[@]}"
+
+for file_id in ${to_download[@]}; do
+    icerbox_dl_ticket "$file_id"
+done
+
+test ${#tickets[@]} -gt 0 || (echo "no download tickets" ; exit 1)
+
+printf "download tickets: %d\n" "$(( ${#tickets[@]} / 2 ))"
+curl --progress-bar -Z "${tickets[@]}"