diff --git a/icerbox.sh b/icerbox.sh index c10bc2c..9434475 100644 --- a/icerbox.sh +++ b/icerbox.sh @@ -2,97 +2,157 @@ set -ueo pipefail api='https://icerbox.com/api/v1/' -jwt='/dev/shm/icerbox.token' +jwt=~/.icerbox.token cfg=~/.icerbox.conf +ct='Content-Type: application/json' err() { - echo "$1" >/dev/stderr + echo "err: ${1}" >/dev/stderr exit 1 } warn() { - echo "$1" >/dev/stderr + echo "warn: ${1}" >/dev/stderr +} + +join_by() { + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi } icerbox_handle_response() { test -n "$1" || err "empty http status code" test -s "$2" || err "empty json reponse body" if [ "$1" -ne 200 ]; then - jq -r '.status_code?' "${2}" >/dev/stderr - jq -r '.message?' "${2}" >/dev/stderr + jq '.' "${2}" >/dev/stderr || true err "request failed with http ${1}" fi } icerbox_auth_login() { - out='/dev/shm/icerbox.auth.login.json' + local out='/dev/shm/icerbox.auth.login.json' test -f "$out" && rm "$out" - code=$(curl -s -w '%{http_code}' -H 'Content-Type: application/json' -o "$out" -d "email=${1}" -d "password=${2}" "${api}auth/login") - test $? -eq 0 || err "curl request failed" + local code=$(curl -s -w '%{http_code}' -H "$ct" -o "$out" -d "email=${1}" -d "password=${2}" "${api}auth/login") + test $? -eq 0 || err "auth/login: curl request failed" icerbox_handle_response "$code" "$out" token=$(jq -r '.token?' "$out") - test -n "$token" || err "empty jwt token" - echo "$token" + test -n "$token" || err "auth/login: empty jwt token" } icerbox_auth_refresh() { - out='/dev/shm/icerbox.auth_refresh.json' + local out='/dev/shm/icerbox.auth_refresh.json' test -f "$out" && rm "$out" # force POST without --data - code=$(curl -s -w '%{http_code}' -XPOST -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" "${api}auth/refresh") - test $? -eq 0 || err "curl request failed" + local code=$(curl -s -w '%{http_code}' -XPOST -H "$auth" -H "$ct" -o "$out" "${api}auth/refresh") + test $? -eq 0 || err "auth/refresh: curl request failed" icerbox_handle_response "$code" "$out" token=$(jq -r '.token?' "$out") - test -n "$token" || err "empty jwt token" - echo "$token" + test -n "$token" || err "auth/refresh: empty jwt token" } icerbox_user_account() { - out='/dev/shm/icerbox.user.account.json' + local out='/dev/shm/icerbox.user.account.json' test -f "$out" && rm "$out" - code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" "${api}user/account") - test $? -eq 0 || err "curl request failed" + local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" "${api}user/account") + test $? -eq 0 || err "user/account: curl request failed" icerbox_handle_response "$code" "$out" } +icerbox_download_quota() { + local in='/dev/shm/icerbox.user.account.json' + test -f "$in" + jq -r '.data.package.volume - .data.downloaded_today' "$in" +} + icerbox_file() { - out='/dev/shm/icerbox.file.json' + local out='/dev/shm/icerbox.file.json' test -f "$out" && rm "$out" - code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" -d id="$2" -G "${api}file") - test $? -eq 0 || err "curl request failed" + local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d id="$2" -G "${api}file") + test $? -eq 0 || err "file: curl request failed" icerbox_handle_response "$code" "$out" - status=$(jq -r '.data?.status?' "$out") - test $? -eq 0 || err "failed parsing json" - test -n "$status" || err "empty status" + local status=$(jq -r '.data?.status?' "$out") + test $? -eq 0 || err "file: failed parsing json" + test -n "$status" || err "file: empty status" if [ "$status" != "active" ]; then - warn "file ${2} is not active" + warn "skip: file id ${2} is not active" return 1 - else - return 0 fi + local name=$(jq -r '.data?.name?' "$out") + test $? -eq 0 || err "file: failed parsing json" + test -n "$name" || err "file: empty name" + if [ -f "$name" ]; then + warn "skip: file exists: ${name}" + return 1 + fi + return 0 +} + +icerbox_files() { + local out='/dev/shm/icerbox.files.json' + test -f "$out" && rm "$out" + local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d ids="$1" -G "${api}files") + test $? -eq 0 || err "files: curl request failed" + icerbox_handle_response "$code" "$out" +} + +icerbox_filter_files() { + local in='/dev/shm/icerbox.files.json' + test -f "$in" || err "filter files: no input file" + local size_sum=0 + local IFS=$'\t' + while read -a arr; do + if [ -f "${arr[1]}" ]; then + local local_size=$(stat --printf '%s' "${arr[1]}") + test -n "$local_size" || err "filter_files: empty file size" + test "$local_size" -eq 0 && err "abort: file exists and is empty: ${arr[1]}" + if [ $local_size -eq ${arr[2]} ]; then + warn "skip: file of same size exists ${arr[1]}" + continue + elif [ $local_size -lt ${arr[2]} ]; then + warn "skip: file with smaller size exists ${arr[1]}" + continue + elif [ $local_size -gt ${arr[2]} ]; then + err "abort: local file is larger than remote ${arr[1]}" + fi + fi + size_sum=$(( $size_sum + ${arr[2]} )) + if [ $size_sum -gt $download_quota ]; then + warn "skip: above download quota" + break + fi + to_download+=("${arr[0]}") + done < <(jq -r '.data[] | select(.status="active") | [.id,.name,.size] | @tsv' $in) + numfmt --format='download size: %f' --to=iec $size_sum } icerbox_dl_ticket() { - out='/dev/shm/icerbox.dl.ticket.json' + local out='/dev/shm/icerbox.dl.ticket.json' test -f "$out" && rm "$out" # endpoint does not accept file parameter with json as content type - code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -o "$out" -d file="$2" "${api}dl/ticket") + local code=$(curl -s -w '%{http_code}' -H "$auth" -o "$out" -d file="$1" "${api}dl/ticket") test $? -eq 0 || err "curl request failed" + if [ "$code" -eq 422 ]; then + warn "skip: file is unavailable: $1" + return + fi icerbox_handle_response "$code" "$out" - url=$(jq -r '.url?' "$out") + local url=$(jq -r '.url?' "$out") test $? -eq 0 || err "failed parsing json" test -n "$url" || err "empty ticket url" - echo "$url" + tickets+=(-OJL "$url") } icerbox_file_id() { test -n "$1" || err "empty url" test "${1:0:20}" = "https://icerbox.com/" || err "not a icerbox url: $1" test "${#1}" -ge 28 || err "url $1 does not contain valid file id" - echo "${1:20:8}" + file_ids+=("${1:20:8}") } test "$#" -eq 0 && err "usage $0 https://icerbox.com/abcdefgh https://icerbox.com/abcdefgh ..." +test "$#" -gt 666 && err "too many arguments. http request will fail with 414 Request-URI Too Large" test -f "$cfg" && source "$cfg" || err "please set 'email' and 'password' in ${cfg}" test -n "$email" || err "email not set" test -n "$password" || err "password not set" @@ -103,8 +163,9 @@ if [ -f $jwt ] && [ -s $jwt ]; then token=$(< $jwt) token_age=$(stat --format=%Y "$jwt") now=$(date +%s) - if [ $token_age -le $(( $now - 1800 )) ]; then - token=$(icerbox_auth_refresh "$token") + if [ $token_age -le $(( $now - 2700 )) ]; then + auth="Authorization: Bearer ${token}" + icerbox_auth_refresh echo "$token" > "${jwt}.tmp" mv "${jwt}.tmp" "$jwt" echo "session refreshed" @@ -112,32 +173,43 @@ if [ -f $jwt ] && [ -s $jwt ]; then echo "using active session" fi else - token=$(icerbox_auth_login "$email" "$password") + icerbox_auth_login "$email" "$password" echo "$token" > "${jwt}.tmp" mv "${jwt}.tmp" "${jwt}" echo "new session acquired" fi -test -n "$token" || echo "no session token" +test -n "$token" || err "no session token" +auth="Authorization: Bearer ${token}" -#icerbox_user_account "$token" +icerbox_user_account +download_quota=$(icerbox_download_quota) +numfmt --format='daily download limit: %f' --to=iec $download_quota + +# globals filled by functions +file_ids=() +to_download=() tickets=() + +printf "requested: %d\n" "$#" + for url in $@; do - file_id=$(icerbox_file_id "$url") - icerbox_file "$token" "$file_id" - if [ $? -ne 0 ]; then - echo "Skipping $url" - continue - else - dl_url=$(icerbox_dl_ticket "$token" "$file_id") - tickets+=(-OJL "$dl_url") - fi + icerbox_file_id "$url" done -if [ ${#tickets} -gt 0 ]; then - curl --progress-bar --parallel-max 10 -Z "${tickets[@]}" -else - err "nothing to download" -fi +icerbox_files "$(join_by , "${file_ids[@]}")" +icerbox_filter_files +test ${#to_download[@]} -gt 0 || (echo "nothing to download" ; exit 1) + +printf "to download: %d\n" "${#to_download[@]}" + +for file_id in ${to_download[@]}; do + icerbox_dl_ticket "$file_id" +done + +test ${#tickets[@]} -gt 0 || (echo "no download tickets" ; exit 1) + +printf "download tickets: %d\n" "$(( ${#tickets[@]} / 2 ))" +curl --progress-bar -Z "${tickets[@]}"