From ddb91106b6417ab58f883ad816c17a7f09fc71b2 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 May 2020 16:47:30 +0300 Subject: [PATCH] update attachments_cleanup_worker.ex --- lib/pleroma/application.ex | 2 +- lib/pleroma/web/media_proxy/invalidation.ex | 8 ++++---- lib/pleroma/web/media_proxy/media_proxy.ex | 8 ++++++-- lib/pleroma/workers/attachments_cleanup_worker.ex | 19 ++++++++++++++----- test/object_test.exs | 2 ++ test/web/media_proxy/invalidation_test.exs | 4 ++-- 6 files changed, 29 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7ddb03529..adebebc7a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -149,7 +149,7 @@ defmodule Pleroma.Application do build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), - build_cachex("deleted_urls", default_ttl: :timer.minutes(60), limit: 2500) + build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 324f8a7ee..162d20262 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -10,14 +10,14 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do alias Pleroma.Config alias Pleroma.Web.MediaProxy + def enabled, do: Config.get([:media_proxy, :invalidation, :enabled]) + @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()} def purge(urls) do prepared_urls = prepare_urls(urls) - if Config.get([:media_proxy, :invalidation, :enabled]) do - result = do_purge(prepared_urls) - MediaProxy.remove_from_deleted_urls(prepared_urls) - result + if enabled() do + do_purge(prepared_urls) else {:ok, prepared_urls} end diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 4a0fec288..b3988b946 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -13,7 +13,9 @@ defmodule Pleroma.Web.MediaProxy do def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url), 1) def remove_from_deleted_urls(urls) when is_list(urls) do - Enum.each(urls, &remove_from_deleted_urls/1) + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(urls, &Cachex.del(cache, &1)) + end) end def remove_from_deleted_urls(url) when is_binary(url) do @@ -21,7 +23,9 @@ defmodule Pleroma.Web.MediaProxy do end def put_in_deleted_urls(urls) when is_list(urls) do - Enum.each(urls, &put_in_deleted_urls/1) + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(urls, &Cachex.put(cache, &1, true)) + end) end def put_in_deleted_urls(url) when is_binary(url) do diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 5fb0b9584..816d681dd 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Web.MediaProxy use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" @@ -50,7 +51,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do end end) - Pleroma.Web.MediaProxy.put_in_deleted_urls(attachment_urls) + lock_attachments(MediaProxy.Invalidation.enabled(), attachment_urls) Enum.each(attachment_urls, fn href -> href @@ -60,20 +61,28 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do Repo.delete_all(from(o in Object, where: o.id in ^object_ids)) - cache_purge(attachment_urls) + cache_purge(MediaProxy.Invalidation.enabled(), attachment_urls) {:ok, :success} end def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} - defp cache_purge(attachment_urls) do - Pleroma.Web.MediaProxy.Invalidation.purge(attachment_urls) + defp cache_purge(true, attachment_urls) do + MediaProxy.Invalidation.purge(attachment_urls) end + defp cache_purge(_, _), do: :ok + + defp lock_attachments(true, attachment_urls) do + MediaProxy.put_in_deleted_urls(attachment_urls) + end + + defp lock_attachments(_, _), do: :ok + # we should delete 1 object for any given attachment, but don't delete # files if there are more than 1 object for it - def prepare_objects(objects, actor, names) do + defp prepare_objects(objects, actor, names) do objects |> Enum.reduce(%{}, fn %{ id: id, diff --git a/test/object_test.exs b/test/object_test.exs index 198d3b1cf..085469495 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -76,6 +76,7 @@ defmodule Pleroma.ObjectTest do describe "delete attachments" do setup do: clear_config([Pleroma.Upload]) setup do: clear_config([:instance, :cleanup_attachments]) + setup do: clear_config([:media_proxy]) test "Disabled via config" do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) @@ -109,6 +110,7 @@ defmodule Pleroma.ObjectTest do refute Object.get_by_id(attachment.id) == nil assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + refute Pleroma.Web.MediaProxy.in_deleted_urls(href) end test "in subdirectories" do diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs index edddc385b..3a9fa8c88 100644 --- a/test/web/media_proxy/invalidation_test.exs +++ b/test/web/media_proxy/invalidation_test.exs @@ -38,7 +38,7 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do assert capture_log(fn -> assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) assert Invalidation.purge([image_url]) == {:ok, [image_url]} - refute Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) end) =~ "Running cache purge: [\"#{image_url}\"]" end end @@ -57,7 +57,7 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do assert capture_log(fn -> assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) assert Invalidation.purge([image_url]) == {:ok, [image_url]} - refute Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) end) =~ "Running cache purge: [\"#{image_url}\"]" end end