ppr/icerbox.sh
1
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

support download quota, skip existing files, use globals and local vars

Browse Source
This commit is contained in:
ppr 2024-04-09 16:22:15 -04:00
parent c9abcd8922
commit 95b4d1fe5b
1 changed files with 123 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
icerbox.sh
View File

@ -2,97 +2,157 @@
set -ueo pipefail set -ueo pipefail
api='https://icerbox.com/api/v1/' api='https://icerbox.com/api/v1/'
jwt='/dev/shm/icerbox.token' jwt=~/.icerbox.token
cfg=~/.icerbox.conf cfg=~/.icerbox.conf
ct='Content-Type: application/json'
err() { err() {
echo "$1" >/dev/stderr echo "err: ${1}" >/dev/stderr
exit 1 exit 1
} }
warn() { warn() {
echo "$1" >/dev/stderr echo "warn: ${1}" >/dev/stderr
}
join_by() {
local d=${1-} f=${2-}
if shift 2; then
printf %s "$f" "${@/#/$d}"
fi
} }
icerbox_handle_response() { icerbox_handle_response() {
test -n "$1" || err "empty http status code" test -n "$1" || err "empty http status code"
test -s "$2" || err "empty json reponse body" test -s "$2" || err "empty json reponse body"
if [ "$1" -ne 200 ]; then if [ "$1" -ne 200 ]; then
jq -r '.status_code?' "${2}" >/dev/stderr jq '.' "${2}" >/dev/stderr || true
jq -r '.message?' "${2}" >/dev/stderr
err "request failed with http ${1}" err "request failed with http ${1}"
fi fi
} }
icerbox_auth_login() { icerbox_auth_login() {
out='/dev/shm/icerbox.auth.login.json' local out='/dev/shm/icerbox.auth.login.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
code=$(curl -s -w '%{http_code}' -H 'Content-Type: application/json' -o "$out" -d "email=${1}" -d "password=${2}" "${api}auth/login") local code=$(curl -s -w '%{http_code}' -H "$ct" -o "$out" -d "email=${1}" -d "password=${2}" "${api}auth/login")
test $? -eq 0 || err "curl request failed" test $? -eq 0 || err "auth/login: curl request failed"
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
token=$(jq -r '.token?' "$out") token=$(jq -r '.token?' "$out")
test -n "$token" || err "empty jwt token" test -n "$token" || err "auth/login: empty jwt token"
echo "$token"
} }
icerbox_auth_refresh() { icerbox_auth_refresh() {
out='/dev/shm/icerbox.auth_refresh.json' local out='/dev/shm/icerbox.auth_refresh.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
# force POST without --data # force POST without --data
code=$(curl -s -w '%{http_code}' -XPOST -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" "${api}auth/refresh") local code=$(curl -s -w '%{http_code}' -XPOST -H "$auth" -H "$ct" -o "$out" "${api}auth/refresh")
test $? -eq 0 || err "curl request failed" test $? -eq 0 || err "auth/refresh: curl request failed"
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
token=$(jq -r '.token?' "$out") token=$(jq -r '.token?' "$out")
test -n "$token" || err "empty jwt token" test -n "$token" || err "auth/refresh: empty jwt token"
echo "$token"
} }
icerbox_user_account() { icerbox_user_account() {
out='/dev/shm/icerbox.user.account.json' local out='/dev/shm/icerbox.user.account.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" "${api}user/account") local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" "${api}user/account")
test $? -eq 0 || err "curl request failed" test $? -eq 0 || err "user/account: curl request failed"
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
} }
icerbox_download_quota() {
local in='/dev/shm/icerbox.user.account.json'
test -f "$in"
jq -r '.data.package.volume - .data.downloaded_today' "$in"
}
icerbox_file() { icerbox_file() {
out='/dev/shm/icerbox.file.json' local out='/dev/shm/icerbox.file.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" -d id="$2" -G "${api}file") local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d id="$2" -G "${api}file")
test $? -eq 0 || err "curl request failed" test $? -eq 0 || err "file: curl request failed"
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
status=$(jq -r '.data?.status?' "$out") local status=$(jq -r '.data?.status?' "$out")
test $? -eq 0 || err "failed parsing json" test $? -eq 0 || err "file: failed parsing json"
test -n "$status" || err "empty status" test -n "$status" || err "file: empty status"
if [ "$status" != "active" ]; then if [ "$status" != "active" ]; then
warn "file ${2} is not active" warn "skip: file id ${2} is not active"
return 1 return 1
else
return 0
fi fi
local name=$(jq -r '.data?.name?' "$out")
test $? -eq 0 || err "file: failed parsing json"
test -n "$name" || err "file: empty name"
if [ -f "$name" ]; then
warn "skip: file exists: ${name}"
return 1
fi
return 0
}
icerbox_files() {
local out='/dev/shm/icerbox.files.json'
test -f "$out" && rm "$out"
local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d ids="$1" -G "${api}files")
test $? -eq 0 || err "files: curl request failed"
icerbox_handle_response "$code" "$out"
}
icerbox_filter_files() {
local in='/dev/shm/icerbox.files.json'
test -f "$in" || err "filter files: no input file"
local size_sum=0
local IFS=$'\t'
while read -a arr; do
if [ -f "${arr[1]}" ]; then
local local_size=$(stat --printf '%s' "${arr[1]}")
test -n "$local_size" || err "filter_files: empty file size"
test "$local_size" -eq 0 && err "abort: file exists and is empty: ${arr[1]}"
if [ $local_size -eq ${arr[2]} ]; then
warn "skip: file of same size exists ${arr[1]}"
continue
elif [ $local_size -lt ${arr[2]} ]; then
warn "skip: file with smaller size exists ${arr[1]}"
continue
elif [ $local_size -gt ${arr[2]} ]; then
err "abort: local file is larger than remote ${arr[1]}"
fi
fi
size_sum=$(( $size_sum + ${arr[2]} ))
if [ $size_sum -gt $download_quota ]; then
warn "skip: above download quota"
break
fi
to_download+=("${arr[0]}")
done < <(jq -r '.data[] | select(.status="active") | [.id,.name,.size] | @tsv' $in)
numfmt --format='download size: %f' --to=iec $size_sum
} }
icerbox_dl_ticket() { icerbox_dl_ticket() {
out='/dev/shm/icerbox.dl.ticket.json' local out='/dev/shm/icerbox.dl.ticket.json'
test -f "$out" && rm "$out" test -f "$out" && rm "$out"
# endpoint does not accept file parameter with json as content type # endpoint does not accept file parameter with json as content type
code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -o "$out" -d file="$2" "${api}dl/ticket") local code=$(curl -s -w '%{http_code}' -H "$auth" -o "$out" -d file="$1" "${api}dl/ticket")
test $? -eq 0 || err "curl request failed" test $? -eq 0 || err "curl request failed"
if [ "$code" -eq 422 ]; then
warn "skip: file is unavailable: $1"
return
fi
icerbox_handle_response "$code" "$out" icerbox_handle_response "$code" "$out"
url=$(jq -r '.url?' "$out") local url=$(jq -r '.url?' "$out")
test $? -eq 0 || err "failed parsing json" test $? -eq 0 || err "failed parsing json"
test -n "$url" || err "empty ticket url" test -n "$url" || err "empty ticket url"
echo "$url" tickets+=(-OJL "$url")
} }
icerbox_file_id() { icerbox_file_id() {
test -n "$1" || err "empty url" test -n "$1" || err "empty url"
test "${1:0:20}" = "https://icerbox.com/" || err "not a icerbox url: $1" test "${1:0:20}" = "https://icerbox.com/" || err "not a icerbox url: $1"
test "${#1}" -ge 28 || err "url $1 does not contain valid file id" test "${#1}" -ge 28 || err "url $1 does not contain valid file id"
echo "${1:20:8}" file_ids+=("${1:20:8}")
} }
test "$#" -eq 0 && err "usage $0 https://icerbox.com/abcdefgh https://icerbox.com/abcdefgh ..." test "$#" -eq 0 && err "usage $0 https://icerbox.com/abcdefgh https://icerbox.com/abcdefgh ..."
test "$#" -gt 666 && err "too many arguments. http request will fail with 414 Request-URI Too Large"
test -f "$cfg" && source "$cfg" || err "please set 'email' and 'password' in ${cfg}" test -f "$cfg" && source "$cfg" || err "please set 'email' and 'password' in ${cfg}"
test -n "$email" || err "email not set" test -n "$email" || err "email not set"
test -n "$password" || err "password not set" test -n "$password" || err "password not set"
@ -103,8 +163,9 @@ if [ -f $jwt ] && [ -s $jwt ]; then
token=$(< $jwt) token=$(< $jwt)
token_age=$(stat --format=%Y "$jwt") token_age=$(stat --format=%Y "$jwt")
now=$(date +%s) now=$(date +%s)
if [ $token_age -le $(( $now - 1800 )) ]; then if [ $token_age -le $(( $now - 2700 )) ]; then
token=$(icerbox_auth_refresh "$token") auth="Authorization: Bearer ${token}"
icerbox_auth_refresh
echo "$token" > "${jwt}.tmp" echo "$token" > "${jwt}.tmp"
mv "${jwt}.tmp" "$jwt" mv "${jwt}.tmp" "$jwt"
echo "session refreshed" echo "session refreshed"
@ -112,32 +173,43 @@ if [ -f $jwt ] && [ -s $jwt ]; then
echo "using active session" echo "using active session"
fi fi
else else
token=$(icerbox_auth_login "$email" "$password") icerbox_auth_login "$email" "$password"
echo "$token" > "${jwt}.tmp" echo "$token" > "${jwt}.tmp"
mv "${jwt}.tmp" "${jwt}" mv "${jwt}.tmp" "${jwt}"
echo "new session acquired" echo "new session acquired"
fi fi
test -n "$token" || echo "no session token" test -n "$token" || err "no session token"
auth="Authorization: Bearer ${token}"
#icerbox_user_account "$token" icerbox_user_account
download_quota=$(icerbox_download_quota)
numfmt --format='daily download limit: %f' --to=iec $download_quota
# globals filled by functions
file_ids=()
to_download=()
tickets=() tickets=()
printf "requested: %d\n" "$#"
for url in $@; do for url in $@; do
file_id=$(icerbox_file_id "$url") icerbox_file_id "$url"
icerbox_file "$token" "$file_id"
if [ $? -ne 0 ]; then
echo "Skipping $url"
continue
else
dl_url=$(icerbox_dl_ticket "$token" "$file_id")
tickets+=(-OJL "$dl_url")
fi
done done
if [ ${#tickets} -gt 0 ]; then icerbox_files "$(join_by , "${file_ids[@]}")"
curl --progress-bar --parallel-max 10 -Z "${tickets[@]}" icerbox_filter_files
else
err "nothing to download"
fi
test ${#to_download[@]} -gt 0 || (echo "nothing to download" ; exit 1)
printf "to download: %d\n" "${#to_download[@]}"
for file_id in ${to_download[@]}; do
icerbox_dl_ticket "$file_id"
done
test ${#tickets[@]} -gt 0 || (echo "no download tickets" ; exit 1)
printf "download tickets: %d\n" "$(( ${#tickets[@]} / 2 ))"
curl --progress-bar -Z "${tickets[@]}"