Revert "Merge branch 'issue/1509' into 'develop'"
This reverts merge request !2539
This commit is contained in:
parent
72189d227f
commit
124812282f
@ -391,10 +391,6 @@ config :pleroma, :rich_media,
|
|||||||
|
|
||||||
config :pleroma, :media_proxy,
|
config :pleroma, :media_proxy,
|
||||||
enabled: false,
|
enabled: false,
|
||||||
invalidation: [
|
|
||||||
enabled: false,
|
|
||||||
provider: Pleroma.Web.MediaProxy.Invalidation.Script
|
|
||||||
],
|
|
||||||
proxy_opts: [
|
proxy_opts: [
|
||||||
redirect_on_failure: false,
|
redirect_on_failure: false,
|
||||||
max_body_length: 25 * 1_048_576,
|
max_body_length: 25 * 1_048_576,
|
||||||
|
@ -254,40 +254,6 @@ This section describe PWA manifest instance-specific values. Currently this opti
|
|||||||
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
|
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
|
||||||
* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
|
* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
|
||||||
* `whitelist`: List of domains to bypass the mediaproxy
|
* `whitelist`: List of domains to bypass the mediaproxy
|
||||||
* `invalidation`: options for remove media from cache after delete object:
|
|
||||||
* `enabled`: Enables purge cache
|
|
||||||
* `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use.
|
|
||||||
|
|
||||||
### Purge cache strategy
|
|
||||||
|
|
||||||
#### Pleroma.Web.MediaProxy.Invalidation.Script
|
|
||||||
|
|
||||||
This strategy allow perform external bash script to purge cache.
|
|
||||||
Urls of attachments pass to script as arguments.
|
|
||||||
|
|
||||||
* `script_path`: path to external script.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
```elixir
|
|
||||||
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
|
|
||||||
script_path: "./installation/nginx-cache-purge.example"
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Pleroma.Web.MediaProxy.Invalidation.Http
|
|
||||||
|
|
||||||
This strategy allow perform custom http request to purge cache.
|
|
||||||
|
|
||||||
* `method`: http method. default is `purge`
|
|
||||||
* `headers`: http headers. default is empty
|
|
||||||
* `options`: request options. default is empty
|
|
||||||
|
|
||||||
Example:
|
|
||||||
```elixir
|
|
||||||
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
|
|
||||||
method: :purge,
|
|
||||||
headers: [],
|
|
||||||
options: []
|
|
||||||
```
|
|
||||||
|
|
||||||
## Link previews
|
## Link previews
|
||||||
|
|
||||||
|
@ -1,40 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
|
|
||||||
# A simple shell script to delete a media from the Nginx cache.
|
|
||||||
|
|
||||||
SCRIPTNAME=${0##*/}
|
|
||||||
|
|
||||||
# NGINX cache directory
|
|
||||||
CACHE_DIRECTORY="/tmp/pleroma-media-cache"
|
|
||||||
|
|
||||||
## Return the files where the items are cached.
|
|
||||||
## $1 - the filename, can be a pattern .
|
|
||||||
## $2 - the cache directory.
|
|
||||||
## $3 - (optional) the number of parallel processes to run for grep.
|
|
||||||
get_cache_files() {
|
|
||||||
local max_parallel=${3-16}
|
|
||||||
find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u
|
|
||||||
}
|
|
||||||
|
|
||||||
## Removes an item from the given cache zone.
|
|
||||||
## $1 - the filename, can be a pattern .
|
|
||||||
## $2 - the cache directory.
|
|
||||||
purge_item() {
|
|
||||||
for f in $(get_cache_files $1 $2); do
|
|
||||||
echo "found file: $f"
|
|
||||||
[ -f $f ] || continue
|
|
||||||
echo "Deleting $f from $2."
|
|
||||||
rm $f
|
|
||||||
done
|
|
||||||
} # purge_item
|
|
||||||
|
|
||||||
purge() {
|
|
||||||
for url in "$@"
|
|
||||||
do
|
|
||||||
echo "$SCRIPTNAME delete \`$url\` from cache ($CACHE_DIRECTORY)"
|
|
||||||
purge_item $url $CACHE_DIRECTORY
|
|
||||||
done
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
purge $1
|
|
@ -9,13 +9,11 @@ defmodule Pleroma.Object do
|
|||||||
import Ecto.Changeset
|
import Ecto.Changeset
|
||||||
|
|
||||||
alias Pleroma.Activity
|
alias Pleroma.Activity
|
||||||
alias Pleroma.Config
|
|
||||||
alias Pleroma.Object
|
alias Pleroma.Object
|
||||||
alias Pleroma.Object.Fetcher
|
alias Pleroma.Object.Fetcher
|
||||||
alias Pleroma.ObjectTombstone
|
alias Pleroma.ObjectTombstone
|
||||||
alias Pleroma.Repo
|
alias Pleroma.Repo
|
||||||
alias Pleroma.User
|
alias Pleroma.User
|
||||||
alias Pleroma.Workers.AttachmentsCleanupWorker
|
|
||||||
|
|
||||||
require Logger
|
require Logger
|
||||||
|
|
||||||
@ -190,37 +188,27 @@ defmodule Pleroma.Object do
|
|||||||
def delete(%Object{data: %{"id" => id}} = object) do
|
def delete(%Object{data: %{"id" => id}} = object) do
|
||||||
with {:ok, _obj} = swap_object_with_tombstone(object),
|
with {:ok, _obj} = swap_object_with_tombstone(object),
|
||||||
deleted_activity = Activity.delete_all_by_object_ap_id(id),
|
deleted_activity = Activity.delete_all_by_object_ap_id(id),
|
||||||
{:ok, _} <- invalid_object_cache(object) do
|
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
|
||||||
cleanup_attachments(
|
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
|
||||||
Config.get([:instance, :cleanup_attachments]),
|
with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do
|
||||||
%{"object" => object}
|
{:ok, _} =
|
||||||
)
|
Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
|
||||||
|
"object" => object
|
||||||
|
})
|
||||||
|
end
|
||||||
|
|
||||||
{:ok, object, deleted_activity}
|
{:ok, object, deleted_activity}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@spec cleanup_attachments(boolean(), %{required(:object) => map()}) ::
|
def prune(%Object{data: %{"id" => id}} = object) do
|
||||||
{:ok, Oban.Job.t() | nil}
|
|
||||||
def cleanup_attachments(true, %{"object" => _} = params) do
|
|
||||||
AttachmentsCleanupWorker.enqueue("cleanup_attachments", params)
|
|
||||||
end
|
|
||||||
|
|
||||||
def cleanup_attachments(_, _), do: {:ok, nil}
|
|
||||||
|
|
||||||
def prune(%Object{data: %{"id" => _id}} = object) do
|
|
||||||
with {:ok, object} <- Repo.delete(object),
|
with {:ok, object} <- Repo.delete(object),
|
||||||
{:ok, _} <- invalid_object_cache(object) do
|
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
|
||||||
|
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
|
||||||
{:ok, object}
|
{:ok, object}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def invalid_object_cache(%Object{data: %{"id" => id}}) do
|
|
||||||
with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
|
|
||||||
Cachex.del(:web_resp_cache, URI.parse(id).path)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def set_cache(%Object{data: %{"id" => ap_id}} = object) do
|
def set_cache(%Object{data: %{"id" => ap_id}} = object) do
|
||||||
Cachex.put(:object_cache, "object:#{ap_id}", object)
|
Cachex.put(:object_cache, "object:#{ap_id}", object)
|
||||||
{:ok, object}
|
{:ok, object}
|
||||||
|
@ -1,26 +0,0 @@
|
|||||||
# Pleroma: A lightweight social networking server
|
|
||||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
|
|
||||||
defmodule Pleroma.Web.MediaProxy.Invalidation do
|
|
||||||
@moduledoc false
|
|
||||||
|
|
||||||
@callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
|
|
||||||
|
|
||||||
alias Pleroma.Config
|
|
||||||
|
|
||||||
@spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
|
|
||||||
def purge(urls) do
|
|
||||||
[:media_proxy, :invalidation, :enabled]
|
|
||||||
|> Config.get()
|
|
||||||
|> do_purge(urls)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp do_purge(true, urls) do
|
|
||||||
provider = Config.get([:media_proxy, :invalidation, :provider])
|
|
||||||
options = Config.get(provider)
|
|
||||||
provider.purge(urls, options)
|
|
||||||
end
|
|
||||||
|
|
||||||
defp do_purge(_, _), do: :ok
|
|
||||||
end
|
|
@ -1,40 +0,0 @@
|
|||||||
# Pleroma: A lightweight social networking server
|
|
||||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
|
|
||||||
defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
|
|
||||||
@moduledoc false
|
|
||||||
@behaviour Pleroma.Web.MediaProxy.Invalidation
|
|
||||||
|
|
||||||
require Logger
|
|
||||||
|
|
||||||
@impl Pleroma.Web.MediaProxy.Invalidation
|
|
||||||
def purge(urls, opts) do
|
|
||||||
method = Map.get(opts, :method, :purge)
|
|
||||||
headers = Map.get(opts, :headers, [])
|
|
||||||
options = Map.get(opts, :options, [])
|
|
||||||
|
|
||||||
Logger.debug("Running cache purge: #{inspect(urls)}")
|
|
||||||
|
|
||||||
Enum.each(urls, fn url ->
|
|
||||||
with {:error, error} <- do_purge(method, url, headers, options) do
|
|
||||||
Logger.error("Error while cache purge: url - #{url}, error: #{inspect(error)}")
|
|
||||||
end
|
|
||||||
end)
|
|
||||||
|
|
||||||
{:ok, "success"}
|
|
||||||
end
|
|
||||||
|
|
||||||
defp do_purge(method, url, headers, options) do
|
|
||||||
case Pleroma.HTTP.request(method, url, "", headers, options) do
|
|
||||||
{:ok, %{status: status} = env} when 400 <= status and status < 500 ->
|
|
||||||
{:error, env}
|
|
||||||
|
|
||||||
{:error, error} = error ->
|
|
||||||
error
|
|
||||||
|
|
||||||
_ ->
|
|
||||||
{:ok, "success"}
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
@ -1,41 +0,0 @@
|
|||||||
# Pleroma: A lightweight social networking server
|
|
||||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
|
|
||||||
defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
|
|
||||||
@moduledoc false
|
|
||||||
|
|
||||||
@behaviour Pleroma.Web.MediaProxy.Invalidation
|
|
||||||
|
|
||||||
require Logger
|
|
||||||
|
|
||||||
@impl Pleroma.Web.MediaProxy.Invalidation
|
|
||||||
def purge(urls, %{script_path: script_path} = _options) do
|
|
||||||
args =
|
|
||||||
urls
|
|
||||||
|> List.wrap()
|
|
||||||
|> Enum.uniq()
|
|
||||||
|> Enum.join(" ")
|
|
||||||
|
|
||||||
path = Path.expand(script_path)
|
|
||||||
|
|
||||||
Logger.debug("Running cache purge: #{inspect(urls)}, #{path}")
|
|
||||||
|
|
||||||
case do_purge(path, [args]) do
|
|
||||||
{result, exit_status} when exit_status > 0 ->
|
|
||||||
Logger.error("Error while cache purge: #{inspect(result)}")
|
|
||||||
{:error, inspect(result)}
|
|
||||||
|
|
||||||
_ ->
|
|
||||||
{:ok, "success"}
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def purge(_, _), do: {:error, "not found script path"}
|
|
||||||
|
|
||||||
defp do_purge(path, args) do
|
|
||||||
System.cmd(path, args)
|
|
||||||
rescue
|
|
||||||
error -> {inspect(error), 1}
|
|
||||||
end
|
|
||||||
end
|
|
@ -27,20 +27,8 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
|||||||
|
|
||||||
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
|
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
|
||||||
|
|
||||||
prefix =
|
|
||||||
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
|
|
||||||
nil -> "media"
|
|
||||||
_ -> ""
|
|
||||||
end
|
|
||||||
|
|
||||||
base_url =
|
|
||||||
String.trim_trailing(
|
|
||||||
Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
|
|
||||||
"/"
|
|
||||||
)
|
|
||||||
|
|
||||||
# find all objects for copies of the attachments, name and actor doesn't matter here
|
# find all objects for copies of the attachments, name and actor doesn't matter here
|
||||||
object_ids_and_hrefs =
|
delete_ids =
|
||||||
from(o in Object,
|
from(o in Object,
|
||||||
where:
|
where:
|
||||||
fragment(
|
fragment(
|
||||||
@ -79,28 +67,29 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
|||||||
|> Enum.map(fn {href, %{id: id, count: count}} ->
|
|> Enum.map(fn {href, %{id: id, count: count}} ->
|
||||||
# only delete files that have single instance
|
# only delete files that have single instance
|
||||||
with 1 <- count do
|
with 1 <- count do
|
||||||
href
|
prefix =
|
||||||
|> String.trim_leading("#{base_url}/#{prefix}")
|
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
|
||||||
|> uploader.delete_file()
|
nil -> "media"
|
||||||
|
_ -> ""
|
||||||
|
end
|
||||||
|
|
||||||
{id, href}
|
base_url =
|
||||||
else
|
String.trim_trailing(
|
||||||
_ -> {id, nil}
|
Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
|
||||||
|
"/"
|
||||||
|
)
|
||||||
|
|
||||||
|
file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
|
||||||
|
|
||||||
|
uploader.delete_file(file_path)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
id
|
||||||
end)
|
end)
|
||||||
|
|
||||||
object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
|
from(o in Object, where: o.id in ^delete_ids)
|
||||||
|
|
||||||
from(o in Object, where: o.id in ^object_ids)
|
|
||||||
|> Repo.delete_all()
|
|> Repo.delete_all()
|
||||||
|
|
||||||
object_ids_and_hrefs
|
|
||||||
|> Enum.filter(fn {_, href} -> not is_nil(href) end)
|
|
||||||
|> Enum.map(&elem(&1, 1))
|
|
||||||
|> Pleroma.Web.MediaProxy.Invalidation.purge()
|
|
||||||
|
|
||||||
{:ok, :success}
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
|
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: :ok
|
||||||
end
|
end
|
||||||
|
@ -1,35 +0,0 @@
|
|||||||
defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
|
|
||||||
use ExUnit.Case
|
|
||||||
alias Pleroma.Web.MediaProxy.Invalidation
|
|
||||||
|
|
||||||
import ExUnit.CaptureLog
|
|
||||||
import Tesla.Mock
|
|
||||||
|
|
||||||
test "logs hasn't error message when request is valid" do
|
|
||||||
mock(fn
|
|
||||||
%{method: :purge, url: "http://example.com/media/example.jpg"} ->
|
|
||||||
%Tesla.Env{status: 200}
|
|
||||||
end)
|
|
||||||
|
|
||||||
refute capture_log(fn ->
|
|
||||||
assert Invalidation.Http.purge(
|
|
||||||
["http://example.com/media/example.jpg"],
|
|
||||||
%{}
|
|
||||||
) == {:ok, "success"}
|
|
||||||
end) =~ "Error while cache purge"
|
|
||||||
end
|
|
||||||
|
|
||||||
test "it write error message in logs when request invalid" do
|
|
||||||
mock(fn
|
|
||||||
%{method: :purge, url: "http://example.com/media/example1.jpg"} ->
|
|
||||||
%Tesla.Env{status: 404}
|
|
||||||
end)
|
|
||||||
|
|
||||||
assert capture_log(fn ->
|
|
||||||
assert Invalidation.Http.purge(
|
|
||||||
["http://example.com/media/example1.jpg"],
|
|
||||||
%{}
|
|
||||||
) == {:ok, "success"}
|
|
||||||
end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
|
|
||||||
end
|
|
||||||
end
|
|
@ -1,20 +0,0 @@
|
|||||||
defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do
|
|
||||||
use ExUnit.Case
|
|
||||||
alias Pleroma.Web.MediaProxy.Invalidation
|
|
||||||
|
|
||||||
import ExUnit.CaptureLog
|
|
||||||
|
|
||||||
test "it logger error when script not found" do
|
|
||||||
assert capture_log(fn ->
|
|
||||||
assert Invalidation.Script.purge(
|
|
||||||
["http://example.com/media/example.jpg"],
|
|
||||||
%{script_path: "./example"}
|
|
||||||
) == {:error, "\"%ErlangError{original: :enoent}\""}
|
|
||||||
end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\""
|
|
||||||
|
|
||||||
assert Invalidation.Script.purge(
|
|
||||||
["http://example.com/media/example.jpg"],
|
|
||||||
%{}
|
|
||||||
) == {:error, "not found script path"}
|
|
||||||
end
|
|
||||||
end
|
|
Loading…
Reference in New Issue
Block a user