From c652045e838b82d87aaf518095feec0e5d6557fb Mon Sep 17 00:00:00 2001 From: ppr Date: Sun, 14 Apr 2024 04:03:42 -0400 Subject: [PATCH] use --oauth2-bearer, avoid -XPOST, avoid -Z because of rate limiting --- icerbox.sh | 53 ++++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/icerbox.sh b/icerbox.sh index 9434475..2dc04ea 100644 --- a/icerbox.sh +++ b/icerbox.sh @@ -15,6 +15,10 @@ warn() { echo "warn: ${1}" >/dev/stderr } +nfo() { + echo "info: ${1}" >/dev/stdout +} + join_by() { local d=${1-} f=${2-} if shift 2; then @@ -44,8 +48,8 @@ icerbox_auth_login() { icerbox_auth_refresh() { local out='/dev/shm/icerbox.auth_refresh.json' test -f "$out" && rm "$out" - # force POST without --data - local code=$(curl -s -w '%{http_code}' -XPOST -H "$auth" -H "$ct" -o "$out" "${api}auth/refresh") + # force POST with empty -d + local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -H "$ct" -o "$out" -d "" "${api}auth/refresh") test $? -eq 0 || err "auth/refresh: curl request failed" icerbox_handle_response "$code" "$out" token=$(jq -r '.token?' "$out") @@ -55,7 +59,7 @@ icerbox_auth_refresh() { icerbox_user_account() { local out='/dev/shm/icerbox.user.account.json' test -f "$out" && rm "$out" - local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" "${api}user/account") + local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -H "$ct" -o "$out" "${api}user/account") test $? -eq 0 || err "user/account: curl request failed" icerbox_handle_response "$code" "$out" } @@ -69,7 +73,7 @@ icerbox_download_quota() { icerbox_file() { local out='/dev/shm/icerbox.file.json' test -f "$out" && rm "$out" - local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d id="$2" -G "${api}file") + local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -H "$ct" -o "$out" -d id="$2" -G "${api}file") test $? -eq 0 || err "file: curl request failed" icerbox_handle_response "$code" "$out" local status=$(jq -r '.data?.status?' "$out") @@ -92,7 +96,7 @@ icerbox_file() { icerbox_files() { local out='/dev/shm/icerbox.files.json' test -f "$out" && rm "$out" - local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d ids="$1" -G "${api}files") + local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -H "$ct" -o "$out" -d ids="$1" -G "${api}files") test $? -eq 0 || err "files: curl request failed" icerbox_handle_response "$code" "$out" } @@ -108,18 +112,18 @@ icerbox_filter_files() { test -n "$local_size" || err "filter_files: empty file size" test "$local_size" -eq 0 && err "abort: file exists and is empty: ${arr[1]}" if [ $local_size -eq ${arr[2]} ]; then - warn "skip: file of same size exists ${arr[1]}" + warn "skip: file of same size exists: ${arr[0]}: ${arr[1]}" continue elif [ $local_size -lt ${arr[2]} ]; then - warn "skip: file with smaller size exists ${arr[1]}" + warn "skip: file with smaller size exists: ${arr[0]}: ${arr[1]}" continue elif [ $local_size -gt ${arr[2]} ]; then - err "abort: local file is larger than remote ${arr[1]}" + err "abort: local file is larger than remote: ${arr[0]}: ${arr[1]}" fi fi size_sum=$(( $size_sum + ${arr[2]} )) if [ $size_sum -gt $download_quota ]; then - warn "skip: above download quota" + warn "break: ${arr[0]}: above download quota" break fi to_download+=("${arr[0]}") @@ -131,10 +135,10 @@ icerbox_dl_ticket() { local out='/dev/shm/icerbox.dl.ticket.json' test -f "$out" && rm "$out" # endpoint does not accept file parameter with json as content type - local code=$(curl -s -w '%{http_code}' -H "$auth" -o "$out" -d file="$1" "${api}dl/ticket") + local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -o "$out" -d file="$1" "${api}dl/ticket") test $? -eq 0 || err "curl request failed" if [ "$code" -eq 422 ]; then - warn "skip: file is unavailable: $1" + warn "skip: file is unavailable: $1" return fi icerbox_handle_response "$code" "$out" @@ -164,52 +168,47 @@ if [ -f $jwt ] && [ -s $jwt ]; then token_age=$(stat --format=%Y "$jwt") now=$(date +%s) if [ $token_age -le $(( $now - 2700 )) ]; then - auth="Authorization: Bearer ${token}" icerbox_auth_refresh echo "$token" > "${jwt}.tmp" mv "${jwt}.tmp" "$jwt" - echo "session refreshed" + nfo "session refreshed" else - echo "using active session" + nfo "using active session" fi else icerbox_auth_login "$email" "$password" echo "$token" > "${jwt}.tmp" mv "${jwt}.tmp" "${jwt}" - echo "new session acquired" + nfo "new session acquired" fi test -n "$token" || err "no session token" -auth="Authorization: Bearer ${token}" icerbox_user_account download_quota=$(icerbox_download_quota) -numfmt --format='daily download limit: %f' --to=iec $download_quota +numfmt --format='daily download limit: %f' --to=iec "$download_quota" # globals filled by functions file_ids=() to_download=() tickets=() -printf "requested: %d\n" "$#" - +nfo "requested: $#" for url in $@; do icerbox_file_id "$url" done - icerbox_files "$(join_by , "${file_ids[@]}")" icerbox_filter_files +test ${#to_download[@]} -gt 0 || (nfo "nothing to download" ; exit 1) -test ${#to_download[@]} -gt 0 || (echo "nothing to download" ; exit 1) - -printf "to download: %d\n" "${#to_download[@]}" - +nfo "to download: ${#to_download[@]}" for file_id in ${to_download[@]}; do icerbox_dl_ticket "$file_id" done +test ${#tickets[@]} -gt 0 || (nfo "no download tickets" ; exit 1) -test ${#tickets[@]} -gt 0 || (echo "no download tickets" ; exit 1) +nfo "download tickets: $(( ${#tickets[@]} / 2 ))" +curl -s -w '%{filename_effective}\t%{size_download} bytes\t%{speed_download} bytes/sec\n' "${tickets[@]}" +test $? -eq 0 || warn "curl exit error code $?" -printf "download tickets: %d\n" "$(( ${#tickets[@]} / 2 ))" -curl --progress-bar -Z "${tickets[@]}"