From cb40602a167f4637dc6df6633ec2dfe33f774177 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 15 May 2020 21:34:46 +0300 Subject: [PATCH] added media proxy invalidation --- config/config.exs | 7 ++++ lib/pleroma/object.ex | 34 ++++++++++----- lib/pleroma/web/media_proxy/invalidation.ex | 19 +++++++++ lib/pleroma/web/media_proxy/invalidations/nginx.ex | 12 ++++++ .../web/media_proxy/invalidations/script.ex | 10 +++++ lib/pleroma/workers/attachments_cleanup_worker.ex | 49 +++++++++++++--------- 6 files changed, 101 insertions(+), 30 deletions(-) create mode 100644 lib/pleroma/web/media_proxy/invalidation.ex create mode 100644 lib/pleroma/web/media_proxy/invalidations/nginx.ex create mode 100644 lib/pleroma/web/media_proxy/invalidations/script.ex diff --git a/config/config.exs b/config/config.exs index e703c1632..5394c7c7a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -378,6 +378,13 @@ config :pleroma, :rich_media, config :pleroma, :media_proxy, enabled: false, + invalidation: [ + enabled: false, + provider: Pleroma.Web.MediaProxy.Invalidation.Script, + options: %{ + script_path: "" + } + ], proxy_opts: [ redirect_on_failure: false, max_body_length: 25 * 1_048_576, diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index e678fd415..66b233498 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -9,11 +9,13 @@ defmodule Pleroma.Object do import Ecto.Changeset alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.ObjectTombstone alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Workers.AttachmentsCleanupWorker require Logger @@ -183,27 +185,37 @@ defmodule Pleroma.Object do def delete(%Object{data: %{"id" => id}} = object) do with {:ok, _obj} = swap_object_with_tombstone(object), deleted_activity = Activity.delete_all_by_object_ap_id(id), - {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), - {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do - with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do - {:ok, _} = - Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{ - "object" => object - }) - end + {:ok, _} <- invalid_object_cache(object) do + cleanup_attachments( + Config.get([:instance, :cleanup_attachments]), + %{"object" => object} + ) {:ok, object, deleted_activity} end end - def prune(%Object{data: %{"id" => id}} = object) do + @spec cleanup_attachments(boolean(), %{required(:object) => map()}) :: + {:ok, Oban.Job.t() | nil} + def cleanup_attachments(true, %{"object" => _} = params) do + AttachmentsCleanupWorker.enqueue("cleanup_attachments", params) + end + + def cleanup_attachments(_, _), do: {:ok, nil} + + def prune(%Object{data: %{"id" => _id}} = object) do with {:ok, object} <- Repo.delete(object), - {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), - {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do + {:ok, _} <- invalid_object_cache(object) do {:ok, object} end end + def invalid_object_cache(%Object{data: %{"id" => id}}) do + with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do + Cachex.del(:web_resp_cache, URI.parse(id).path) + end + end + def set_cache(%Object{data: %{"id" => ap_id}} = object) do Cachex.put(:object_cache, "object:#{ap_id}", object) {:ok, object} diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex new file mode 100644 index 000000000..dd9a53a27 --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -0,0 +1,19 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation do + @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} + + alias Pleroma.Config + + def purge(urls) do + [:media_proxy, :invalidation, :enabled] + |> Config.get() + |> do_purge(urls) + end + + defp do_purge(true, urls) do + config = Config.get([:media_proxy, :invalidation]) + config[:provider].purge(urls, config[:options]) + :ok + end + + defp do_purge(_, _), do: :ok +end diff --git a/lib/pleroma/web/media_proxy/invalidations/nginx.ex b/lib/pleroma/web/media_proxy/invalidations/nginx.ex new file mode 100644 index 000000000..5bfdd505c --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidations/nginx.ex @@ -0,0 +1,12 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Nginx do + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, _opts) do + Enum.each(urls, fn url -> + Pleroma.HTTP.request(:purge, url, "", [], []) + end) + + {:ok, "success"} + end +end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex new file mode 100644 index 000000000..f458845a0 --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -0,0 +1,10 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Script do + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, %{script_path: script_path} = options) do + script_args = List.wrap(Map.get(options, :script_args, [])) + System.cmd(Path.expand(script_path), [urls] ++ script_args) + {:ok, "success"} + end +end diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 3c5820a86..49352db2a 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -27,8 +27,20 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + prefix = + case Pleroma.Config.get([Pleroma.Upload, :base_url]) do + nil -> "media" + _ -> "" + end + + base_url = + String.trim_trailing( + Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()), + "/" + ) + # find all objects for copies of the attachments, name and actor doesn't matter here - delete_ids = + object_ids_and_hrefs = from(o in Object, where: fragment( @@ -67,29 +79,28 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do |> Enum.map(fn {href, %{id: id, count: count}} -> # only delete files that have single instance with 1 <- count do - prefix = - case Pleroma.Config.get([Pleroma.Upload, :base_url]) do - nil -> "media" - _ -> "" - end + href + |> String.trim_leading("#{base_url}/#{prefix}") + |> uploader.delete_file() - base_url = - String.trim_trailing( - Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()), - "/" - ) - - file_path = String.trim_leading(href, "#{base_url}/#{prefix}") - - uploader.delete_file(file_path) + {id, href} + else + _ -> {id, nil} end - - id end) - from(o in Object, where: o.id in ^delete_ids) + object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end) + + from(o in Object, where: o.id in ^object_ids) |> Repo.delete_all() + + object_ids_and_hrefs + |> Enum.filter(fn {_, href} -> not is_nil(href) end) + |> Enum.map(&elem(&1, 1)) + |> Pleroma.Web.MediaProxy.Invalidation.purge() + + {:ok, :success} end - def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: :ok + def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} end