support download quota, skip existing files, use globals and local vars
This commit is contained in:
parent
c9abcd8922
commit
95b4d1fe5b
174
icerbox.sh
174
icerbox.sh
@ -2,97 +2,157 @@
|
||||
set -ueo pipefail
|
||||
|
||||
api='https://icerbox.com/api/v1/'
|
||||
jwt='/dev/shm/icerbox.token'
|
||||
jwt=~/.icerbox.token
|
||||
cfg=~/.icerbox.conf
|
||||
ct='Content-Type: application/json'
|
||||
|
||||
err() {
|
||||
echo "$1" >/dev/stderr
|
||||
echo "err: ${1}" >/dev/stderr
|
||||
exit 1
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo "$1" >/dev/stderr
|
||||
echo "warn: ${1}" >/dev/stderr
|
||||
}
|
||||
|
||||
join_by() {
|
||||
local d=${1-} f=${2-}
|
||||
if shift 2; then
|
||||
printf %s "$f" "${@/#/$d}"
|
||||
fi
|
||||
}
|
||||
|
||||
icerbox_handle_response() {
|
||||
test -n "$1" || err "empty http status code"
|
||||
test -s "$2" || err "empty json reponse body"
|
||||
if [ "$1" -ne 200 ]; then
|
||||
jq -r '.status_code?' "${2}" >/dev/stderr
|
||||
jq -r '.message?' "${2}" >/dev/stderr
|
||||
jq '.' "${2}" >/dev/stderr || true
|
||||
err "request failed with http ${1}"
|
||||
fi
|
||||
}
|
||||
|
||||
icerbox_auth_login() {
|
||||
out='/dev/shm/icerbox.auth.login.json'
|
||||
local out='/dev/shm/icerbox.auth.login.json'
|
||||
test -f "$out" && rm "$out"
|
||||
code=$(curl -s -w '%{http_code}' -H 'Content-Type: application/json' -o "$out" -d "email=${1}" -d "password=${2}" "${api}auth/login")
|
||||
test $? -eq 0 || err "curl request failed"
|
||||
local code=$(curl -s -w '%{http_code}' -H "$ct" -o "$out" -d "email=${1}" -d "password=${2}" "${api}auth/login")
|
||||
test $? -eq 0 || err "auth/login: curl request failed"
|
||||
icerbox_handle_response "$code" "$out"
|
||||
token=$(jq -r '.token?' "$out")
|
||||
test -n "$token" || err "empty jwt token"
|
||||
echo "$token"
|
||||
test -n "$token" || err "auth/login: empty jwt token"
|
||||
}
|
||||
|
||||
icerbox_auth_refresh() {
|
||||
out='/dev/shm/icerbox.auth_refresh.json'
|
||||
local out='/dev/shm/icerbox.auth_refresh.json'
|
||||
test -f "$out" && rm "$out"
|
||||
# force POST without --data
|
||||
code=$(curl -s -w '%{http_code}' -XPOST -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" "${api}auth/refresh")
|
||||
test $? -eq 0 || err "curl request failed"
|
||||
local code=$(curl -s -w '%{http_code}' -XPOST -H "$auth" -H "$ct" -o "$out" "${api}auth/refresh")
|
||||
test $? -eq 0 || err "auth/refresh: curl request failed"
|
||||
icerbox_handle_response "$code" "$out"
|
||||
token=$(jq -r '.token?' "$out")
|
||||
test -n "$token" || err "empty jwt token"
|
||||
echo "$token"
|
||||
test -n "$token" || err "auth/refresh: empty jwt token"
|
||||
}
|
||||
|
||||
icerbox_user_account() {
|
||||
out='/dev/shm/icerbox.user.account.json'
|
||||
local out='/dev/shm/icerbox.user.account.json'
|
||||
test -f "$out" && rm "$out"
|
||||
code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" "${api}user/account")
|
||||
test $? -eq 0 || err "curl request failed"
|
||||
local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" "${api}user/account")
|
||||
test $? -eq 0 || err "user/account: curl request failed"
|
||||
icerbox_handle_response "$code" "$out"
|
||||
}
|
||||
|
||||
icerbox_download_quota() {
|
||||
local in='/dev/shm/icerbox.user.account.json'
|
||||
test -f "$in"
|
||||
jq -r '.data.package.volume - .data.downloaded_today' "$in"
|
||||
}
|
||||
|
||||
icerbox_file() {
|
||||
out='/dev/shm/icerbox.file.json'
|
||||
local out='/dev/shm/icerbox.file.json'
|
||||
test -f "$out" && rm "$out"
|
||||
code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -H 'Content-Type: application/json' -o "$out" -d id="$2" -G "${api}file")
|
||||
test $? -eq 0 || err "curl request failed"
|
||||
local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d id="$2" -G "${api}file")
|
||||
test $? -eq 0 || err "file: curl request failed"
|
||||
icerbox_handle_response "$code" "$out"
|
||||
status=$(jq -r '.data?.status?' "$out")
|
||||
test $? -eq 0 || err "failed parsing json"
|
||||
test -n "$status" || err "empty status"
|
||||
local status=$(jq -r '.data?.status?' "$out")
|
||||
test $? -eq 0 || err "file: failed parsing json"
|
||||
test -n "$status" || err "file: empty status"
|
||||
if [ "$status" != "active" ]; then
|
||||
warn "file ${2} is not active"
|
||||
warn "skip: file id ${2} is not active"
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
local name=$(jq -r '.data?.name?' "$out")
|
||||
test $? -eq 0 || err "file: failed parsing json"
|
||||
test -n "$name" || err "file: empty name"
|
||||
if [ -f "$name" ]; then
|
||||
warn "skip: file exists: ${name}"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
icerbox_files() {
|
||||
local out='/dev/shm/icerbox.files.json'
|
||||
test -f "$out" && rm "$out"
|
||||
local code=$(curl -s -w '%{http_code}' -H "$auth" -H "$ct" -o "$out" -d ids="$1" -G "${api}files")
|
||||
test $? -eq 0 || err "files: curl request failed"
|
||||
icerbox_handle_response "$code" "$out"
|
||||
}
|
||||
|
||||
icerbox_filter_files() {
|
||||
local in='/dev/shm/icerbox.files.json'
|
||||
test -f "$in" || err "filter files: no input file"
|
||||
local size_sum=0
|
||||
local IFS=$'\t'
|
||||
while read -a arr; do
|
||||
if [ -f "${arr[1]}" ]; then
|
||||
local local_size=$(stat --printf '%s' "${arr[1]}")
|
||||
test -n "$local_size" || err "filter_files: empty file size"
|
||||
test "$local_size" -eq 0 && err "abort: file exists and is empty: ${arr[1]}"
|
||||
if [ $local_size -eq ${arr[2]} ]; then
|
||||
warn "skip: file of same size exists ${arr[1]}"
|
||||
continue
|
||||
elif [ $local_size -lt ${arr[2]} ]; then
|
||||
warn "skip: file with smaller size exists ${arr[1]}"
|
||||
continue
|
||||
elif [ $local_size -gt ${arr[2]} ]; then
|
||||
err "abort: local file is larger than remote ${arr[1]}"
|
||||
fi
|
||||
fi
|
||||
size_sum=$(( $size_sum + ${arr[2]} ))
|
||||
if [ $size_sum -gt $download_quota ]; then
|
||||
warn "skip: above download quota"
|
||||
break
|
||||
fi
|
||||
to_download+=("${arr[0]}")
|
||||
done < <(jq -r '.data[] | select(.status="active") | [.id,.name,.size] | @tsv' $in)
|
||||
numfmt --format='download size: %f' --to=iec $size_sum
|
||||
}
|
||||
|
||||
icerbox_dl_ticket() {
|
||||
out='/dev/shm/icerbox.dl.ticket.json'
|
||||
local out='/dev/shm/icerbox.dl.ticket.json'
|
||||
test -f "$out" && rm "$out"
|
||||
# endpoint does not accept file parameter with json as content type
|
||||
code=$(curl -s -w '%{http_code}' -H "Authorization: Bearer ${1}" -o "$out" -d file="$2" "${api}dl/ticket")
|
||||
local code=$(curl -s -w '%{http_code}' -H "$auth" -o "$out" -d file="$1" "${api}dl/ticket")
|
||||
test $? -eq 0 || err "curl request failed"
|
||||
if [ "$code" -eq 422 ]; then
|
||||
warn "skip: file is unavailable: $1"
|
||||
return
|
||||
fi
|
||||
icerbox_handle_response "$code" "$out"
|
||||
url=$(jq -r '.url?' "$out")
|
||||
local url=$(jq -r '.url?' "$out")
|
||||
test $? -eq 0 || err "failed parsing json"
|
||||
test -n "$url" || err "empty ticket url"
|
||||
echo "$url"
|
||||
tickets+=(-OJL "$url")
|
||||
}
|
||||
|
||||
icerbox_file_id() {
|
||||
test -n "$1" || err "empty url"
|
||||
test "${1:0:20}" = "https://icerbox.com/" || err "not a icerbox url: $1"
|
||||
test "${#1}" -ge 28 || err "url $1 does not contain valid file id"
|
||||
echo "${1:20:8}"
|
||||
file_ids+=("${1:20:8}")
|
||||
}
|
||||
|
||||
test "$#" -eq 0 && err "usage $0 https://icerbox.com/abcdefgh https://icerbox.com/abcdefgh ..."
|
||||
test "$#" -gt 666 && err "too many arguments. http request will fail with 414 Request-URI Too Large"
|
||||
test -f "$cfg" && source "$cfg" || err "please set 'email' and 'password' in ${cfg}"
|
||||
test -n "$email" || err "email not set"
|
||||
test -n "$password" || err "password not set"
|
||||
@ -103,8 +163,9 @@ if [ -f $jwt ] && [ -s $jwt ]; then
|
||||
token=$(< $jwt)
|
||||
token_age=$(stat --format=%Y "$jwt")
|
||||
now=$(date +%s)
|
||||
if [ $token_age -le $(( $now - 1800 )) ]; then
|
||||
token=$(icerbox_auth_refresh "$token")
|
||||
if [ $token_age -le $(( $now - 2700 )) ]; then
|
||||
auth="Authorization: Bearer ${token}"
|
||||
icerbox_auth_refresh
|
||||
echo "$token" > "${jwt}.tmp"
|
||||
mv "${jwt}.tmp" "$jwt"
|
||||
echo "session refreshed"
|
||||
@ -112,32 +173,43 @@ if [ -f $jwt ] && [ -s $jwt ]; then
|
||||
echo "using active session"
|
||||
fi
|
||||
else
|
||||
token=$(icerbox_auth_login "$email" "$password")
|
||||
icerbox_auth_login "$email" "$password"
|
||||
echo "$token" > "${jwt}.tmp"
|
||||
mv "${jwt}.tmp" "${jwt}"
|
||||
echo "new session acquired"
|
||||
fi
|
||||
|
||||
test -n "$token" || echo "no session token"
|
||||
test -n "$token" || err "no session token"
|
||||
auth="Authorization: Bearer ${token}"
|
||||
|
||||
#icerbox_user_account "$token"
|
||||
icerbox_user_account
|
||||
download_quota=$(icerbox_download_quota)
|
||||
|
||||
numfmt --format='daily download limit: %f' --to=iec $download_quota
|
||||
|
||||
# globals filled by functions
|
||||
file_ids=()
|
||||
to_download=()
|
||||
tickets=()
|
||||
|
||||
printf "requested: %d\n" "$#"
|
||||
|
||||
for url in $@; do
|
||||
file_id=$(icerbox_file_id "$url")
|
||||
icerbox_file "$token" "$file_id"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Skipping $url"
|
||||
continue
|
||||
else
|
||||
dl_url=$(icerbox_dl_ticket "$token" "$file_id")
|
||||
tickets+=(-OJL "$dl_url")
|
||||
fi
|
||||
icerbox_file_id "$url"
|
||||
done
|
||||
|
||||
if [ ${#tickets} -gt 0 ]; then
|
||||
curl --progress-bar --parallel-max 10 -Z "${tickets[@]}"
|
||||
else
|
||||
err "nothing to download"
|
||||
fi
|
||||
icerbox_files "$(join_by , "${file_ids[@]}")"
|
||||
icerbox_filter_files
|
||||
|
||||
test ${#to_download[@]} -gt 0 || (echo "nothing to download" ; exit 1)
|
||||
|
||||
printf "to download: %d\n" "${#to_download[@]}"
|
||||
|
||||
for file_id in ${to_download[@]}; do
|
||||
icerbox_dl_ticket "$file_id"
|
||||
done
|
||||
|
||||
test ${#tickets[@]} -gt 0 || (echo "no download tickets" ; exit 1)
|
||||
|
||||
printf "download tickets: %d\n" "$(( ${#tickets[@]} / 2 ))"
|
||||
curl --progress-bar -Z "${tickets[@]}"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user