ppr/icerbox.sh
1
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

use --oauth2-bearer, avoid -XPOST, avoid -Z because of rate limiting

Browse Source
This commit is contained in:
ppr 2024-04-14 04:03:42 -04:00
parent 59431faca1
commit c652045e83
1 changed files with 26 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
icerbox.sh
View File

@ -15,6 +15,10 @@ warn() {
echo "warn: ${1}" >/dev/stderr echo "warn: ${1}" >/dev/stderr
} }
nfo() {
echo "info: ${1}" >/dev/stdout
}
join_by() { join_by() {
local d=${1-} f=${2-} local d=${1-} f=${2-}
if shift 2; then if shift 2; then
@ -44,8 +48,8 @@ icerbox_auth_login() {
icerbox_auth_refresh() { icerbox_auth_refresh() {
local out='/dev/shm/icerbox.auth_refresh.json' local out='/dev/shm/icerbox.auth_refresh.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
# force POST without --data # force POST with empty -d
local code=$(curl -s -w '%{http_code}' -XPOST -H "$auth" -H "$ct" -o "$out" "${api}auth/refresh") local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -H "$ct" -o "$out" -d "" "${api}auth/refresh")
test $? -eq 0 || err "auth/refresh: curl request failed" test $? -eq 0 || err "auth/refresh: curl request failed"
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
token=$(jq -r '.token?' "$out") token=$(jq -r '.token?' "$out")
@ -55,7 +59,7 @@ icerbox_auth_refresh() {
icerbox_user_account() { icerbox_user_account() {
local out='/dev/shm/icerbox.user.account.json' local out='/dev/shm/icerbox.user.account.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" "${api}user/account") local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -H "$ct" -o "$out" "${api}user/account")
test $? -eq 0 || err "user/account: curl request failed" test $? -eq 0 || err "user/account: curl request failed"
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
} }
@ -69,7 +73,7 @@ icerbox_download_quota() {
icerbox_file() { icerbox_file() {
local out='/dev/shm/icerbox.file.json' local out='/dev/shm/icerbox.file.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d id="$2" -G "${api}file") local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -H "$ct" -o "$out" -d id="$2" -G "${api}file")
test $? -eq 0 || err "file: curl request failed" test $? -eq 0 || err "file: curl request failed"
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
local status=$(jq -r '.data?.status?' "$out") local status=$(jq -r '.data?.status?' "$out")
@ -92,7 +96,7 @@ icerbox_file() {
icerbox_files() { icerbox_files() {
local out='/dev/shm/icerbox.files.json' local out='/dev/shm/icerbox.files.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d ids="$1" -G "${api}files") local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -H "$ct" -o "$out" -d ids="$1" -G "${api}files")
test $? -eq 0 || err "files: curl request failed" test $? -eq 0 || err "files: curl request failed"
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
} }
@ -108,18 +112,18 @@ icerbox_filter_files() {
test -n "$local_size" || err "filter_files: empty file size" test -n "$local_size" || err "filter_files: empty file size"
test "$local_size" -eq 0 && err "abort: file exists and is empty: ${arr[1]}" test "$local_size" -eq 0 && err "abort: file exists and is empty: ${arr[1]}"
if [ $local_size -eq ${arr[2]} ]; then if [ $local_size -eq ${arr[2]} ]; then
warn "skip: file of same size exists ${arr[1]}" warn "skip: file of same size exists: ${arr[0]}: ${arr[1]}"
continue continue
elif [ $local_size -lt ${arr[2]} ]; then elif [ $local_size -lt ${arr[2]} ]; then
warn "skip: file with smaller size exists ${arr[1]}" warn "skip: file with smaller size exists: ${arr[0]}: ${arr[1]}"
continue continue
elif [ $local_size -gt ${arr[2]} ]; then elif [ $local_size -gt ${arr[2]} ]; then
err "abort: local file is larger than remote ${arr[1]}" err "abort: local file is larger than remote: ${arr[0]}: ${arr[1]}"
fi fi
fi fi
size_sum=$(( $size_sum + ${arr[2]} )) size_sum=$(( $size_sum + ${arr[2]} ))
if [ $size_sum -gt $download_quota ]; then if [ $size_sum -gt $download_quota ]; then
warn "skip: above download quota" warn "break: ${arr[0]}: above download quota"
break break
fi fi
to_download+=("${arr[0]}") to_download+=("${arr[0]}")
@ -131,10 +135,10 @@ icerbox_dl_ticket() {
local out='/dev/shm/icerbox.dl.ticket.json' local out='/dev/shm/icerbox.dl.ticket.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
# endpoint does not accept file parameter with json as content type # endpoint does not accept file parameter with json as content type
local code=$(curl -s -w '%{http_code}' -H "$auth" -o "$out" -d file="$1" "${api}dl/ticket") local code=$(curl -s -w '%{http_code}' --oauth2-bearer "$token" -o "$out" -d file="$1" "${api}dl/ticket")
test $? -eq 0 || err "curl request failed" test $? -eq 0 || err "curl request failed"
if [ "$code" -eq 422 ]; then if [ "$code" -eq 422 ]; then
warn "skip: file is unavailable: $1" warn "skip: file is unavailable: $1"
return return
fi fi
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
@ -164,52 +168,47 @@ if [ -f $jwt ] && [ -s $jwt ]; then
token_age=$(stat --format=%Y "$jwt") token_age=$(stat --format=%Y "$jwt")
now=$(date +%s) now=$(date +%s)
if [ $token_age -le $(( $now - 2700 )) ]; then if [ $token_age -le $(( $now - 2700 )) ]; then
auth="Authorization: Bearer ${token}"
icerbox_auth_refresh icerbox_auth_refresh
echo "$token" > "${jwt}.tmp" echo "$token" > "${jwt}.tmp"
mv "${jwt}.tmp" "$jwt" mv "${jwt}.tmp" "$jwt"
echo "session refreshed" nfo "session refreshed"
else else
echo "using active session" nfo "using active session"
fi fi
else else
icerbox_auth_login "$email" "$password" icerbox_auth_login "$email" "$password"
echo "$token" > "${jwt}.tmp" echo "$token" > "${jwt}.tmp"
mv "${jwt}.tmp" "${jwt}" mv "${jwt}.tmp" "${jwt}"
echo "new session acquired" nfo "new session acquired"
fi fi
test -n "$token" || err "no session token" test -n "$token" || err "no session token"
auth="Authorization: Bearer ${token}"
icerbox_user_account icerbox_user_account
download_quota=$(icerbox_download_quota) download_quota=$(icerbox_download_quota)
numfmt --format='daily download limit: %f' --to=iec $download_quota numfmt --format='daily download limit: %f' --to=iec "$download_quota"
# globals filled by functions # globals filled by functions
file_ids=() file_ids=()
to_download=() to_download=()
tickets=() tickets=()
printf "requested: %d\n" "$#" nfo "requested: $#"
for url in $@; do for url in $@; do
icerbox_file_id "$url" icerbox_file_id "$url"
done done
icerbox_files "$(join_by , "${file_ids[@]}")" icerbox_files "$(join_by , "${file_ids[@]}")"
icerbox_filter_files icerbox_filter_files
test ${#to_download[@]} -gt 0 || (nfo "nothing to download" ; exit 1)
test ${#to_download[@]} -gt 0 || (echo "nothing to download" ; exit 1) nfo "to download: ${#to_download[@]}"
printf "to download: %d\n" "${#to_download[@]}"
for file_id in ${to_download[@]}; do for file_id in ${to_download[@]}; do
icerbox_dl_ticket "$file_id" icerbox_dl_ticket "$file_id"
done done
test ${#tickets[@]} -gt 0 || (nfo "no download tickets" ; exit 1)
test ${#tickets[@]} -gt 0 || (echo "no download tickets" ; exit 1) nfo "download tickets: $(( ${#tickets[@]} / 2 ))"
curl -s -w '%{filename_effective}\t%{size_download} bytes\t%{speed_download} bytes/sec\n' "${tickets[@]}"
test $? -eq 0 || warn "curl exit error code $?"
printf "download tickets: %d\n" "$(( ${#tickets[@]} / 2 ))"
curl --progress-bar -Z "${tickets[@]}"