From 88f0eed0f24cb05949edcea49215ee939babac58 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 12 Jan 2020 18:48:58 +0000 Subject: [PATCH] Delete attachments when status is deleted --- CHANGELOG.md | 1 + lib/pleroma/object.ex | 88 +++++++++++++++++++++++++++++++++++++++ lib/pleroma/uploaders/local.ex | 13 ++++++ lib/pleroma/uploaders/s3.ex | 14 +++++++ lib/pleroma/uploaders/uploader.ex | 3 +- test/object_test.exs | 68 ++++++++++++++++++++++++++++++ test/uploaders/local_test.exs | 21 ++++++++++ test/uploaders/s3_test.exs | 7 ++++ 8 files changed, 214 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0907fbd53..397348304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking**: MDII uploader ### Changed +- **Breaking:** attachments are removed along with statuses when there are no other references to it - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index eb37b95a6..2452a7389 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Object do require Logger + @type t() :: %__MODULE__{} + schema "objects" do field(:data, :map) @@ -79,6 +81,20 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end + @doc """ + Get a single attachment by it's name and href + """ + @spec get_attachment_by_name_and_href(String.t(), String.t()) :: Object.t() | nil + def get_attachment_by_name_and_href(name, href) do + query = + from(o in Object, + where: fragment("(?)->>'name' = ?", o.data, ^name), + where: fragment("(?)->>'href' = ?", o.data, ^href) + ) + + Repo.one(query) + end + defp warn_on_no_object_preloaded(ap_id) do "Object.normalize() called without preloaded object (#{inspect(ap_id)}). Consider preloading the object" |> Logger.debug() @@ -164,6 +180,7 @@ defmodule Pleroma.Object do def delete(%Object{data: %{"id" => id}} = object) do with {:ok, _obj} = swap_object_with_tombstone(object), + :ok <- delete_attachments(object), deleted_activity = Activity.delete_all_by_object_ap_id(id), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do @@ -171,6 +188,77 @@ defmodule Pleroma.Object do end end + defp delete_attachments(%{data: %{"attachment" => [_ | _] = attachments, "actor" => actor}}) do + hrefs = + Enum.flat_map(attachments, fn attachment -> + Enum.map(attachment["url"], & &1["href"]) + end) + + names = Enum.map(attachments, & &1["name"]) + + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + # find all objects for copies of the attachments, name and actor doesn't matter here + delete_ids = + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href'))::jsonb \\?| (?)", + o.data, + ^hrefs + ) + ) + |> Repo.all() + # we should delete 1 object for any given attachment, but don't delete files if + # there are more than 1 object for it + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) + |> Enum.map(fn {href, %{id: id, count: count}} -> + # only delete files that have single instance + with 1 <- count do + prefix = + case Pleroma.Config.get([Pleroma.Upload, :base_url]) do + nil -> "media" + _ -> "" + end + + base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url()) + + file_path = String.trim_leading(href, "#{base_url}/#{prefix}") + + uploader.delete_file(file_path) + end + + id + end) + + from(o in Object, where: o.id in ^delete_ids) + |> Repo.delete_all() + + :ok + end + + defp delete_attachments(%{data: _data}), do: :ok + def prune(%Object{data: %{"id" => id}} = object) do with {:ok, object} <- Repo.delete(object), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), diff --git a/lib/pleroma/uploaders/local.ex b/lib/pleroma/uploaders/local.ex index 36b3c35ec..2e6fe3292 100644 --- a/lib/pleroma/uploaders/local.ex +++ b/lib/pleroma/uploaders/local.ex @@ -5,10 +5,12 @@ defmodule Pleroma.Uploaders.Local do @behaviour Pleroma.Uploaders.Uploader + @impl true def get_file(_) do {:ok, {:static_dir, upload_path()}} end + @impl true def put_file(upload) do {local_path, file} = case Enum.reverse(Path.split(upload.path)) do @@ -33,4 +35,15 @@ defmodule Pleroma.Uploaders.Local do def upload_path do Pleroma.Config.get!([__MODULE__, :uploads]) end + + @impl true + def delete_file(path) do + upload_path() + |> Path.join(path) + |> File.rm() + |> case do + :ok -> :ok + {:error, posix_error} -> {:error, to_string(posix_error)} + end + end end diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index 9876b6398..feb89cea6 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Uploaders.S3 do # The file name is re-encoded with S3's constraints here to comply with previous # links with less strict filenames + @impl true def get_file(file) do config = Config.get([__MODULE__]) bucket = Keyword.fetch!(config, :bucket) @@ -35,6 +36,7 @@ defmodule Pleroma.Uploaders.S3 do ])}} end + @impl true def put_file(%Pleroma.Upload{} = upload) do config = Config.get([__MODULE__]) bucket = Keyword.get(config, :bucket) @@ -69,6 +71,18 @@ defmodule Pleroma.Uploaders.S3 do end end + @impl true + def delete_file(file) do + [__MODULE__, :bucket] + |> Config.get() + |> ExAws.S3.delete_object(file) + |> ExAws.request() + |> case do + {:ok, %{status_code: 204}} -> :ok + error -> {:error, inspect(error)} + end + end + @regex Regex.compile!("[^0-9a-zA-Z!.*/'()_-]") def strict_encode(name) do String.replace(name, @regex, "-") diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex index c0b22c28a..d71e213d2 100644 --- a/lib/pleroma/uploaders/uploader.ex +++ b/lib/pleroma/uploaders/uploader.ex @@ -36,6 +36,8 @@ defmodule Pleroma.Uploaders.Uploader do @callback put_file(Pleroma.Upload.t()) :: :ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback + @callback delete_file(file :: String.t()) :: :ok | {:error, String.t()} + @callback http_callback(Plug.Conn.t(), Map.t()) :: {:ok, Plug.Conn.t()} | {:ok, Plug.Conn.t(), file_spec()} @@ -43,7 +45,6 @@ defmodule Pleroma.Uploaders.Uploader do @optional_callbacks http_callback: 2 @spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()} - def put_file(uploader, upload) do case uploader.put_file(upload) do :ok -> {:ok, {:file, upload.path}} diff --git a/test/object_test.exs b/test/object_test.exs index 9247a6d84..b002c2bae 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -71,6 +71,74 @@ defmodule Pleroma.ObjectTest do end end + describe "delete attachments" do + clear_config([Pleroma.Upload]) + + test "in subdirectories" do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + user = insert(:user) + + {:ok, %Object{} = attachment} = + Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id) + + %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + + uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) + + path = href |> Path.dirname() |> Path.basename() + + assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + + Object.delete(note) + + assert Object.get_by_id(attachment.id) == nil + + assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") + end + + test "with dedupe enabled" do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe]) + + uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) + + File.mkdir_p!(uploads_dir) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + user = insert(:user) + + {:ok, %Object{} = attachment} = + Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id) + + %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + + filename = Path.basename(href) + + assert {:ok, files} = File.ls(uploads_dir) + assert filename in files + + Object.delete(note) + + assert Object.get_by_id(attachment.id) == nil + assert {:ok, files} = File.ls(uploads_dir) + refute filename in files + end + end + describe "normalizer" do test "fetches unknown objects by default" do %Object{} = diff --git a/test/uploaders/local_test.exs b/test/uploaders/local_test.exs index fc442d0f1..1963dac23 100644 --- a/test/uploaders/local_test.exs +++ b/test/uploaders/local_test.exs @@ -29,4 +29,25 @@ defmodule Pleroma.Uploaders.LocalTest do |> File.exists?() end end + + describe "delete_file/1" do + test "deletes local file" do + file_path = "local_upload/files/image.jpg" + + file = %Pleroma.Upload{ + name: "image.jpg", + content_type: "image/jpg", + path: file_path, + tempfile: Path.absname("test/fixtures/image_tmp.jpg") + } + + :ok = Local.put_file(file) + local_path = Path.join([Local.upload_path(), file_path]) + assert File.exists?(local_path) + + Local.delete_file(file_path) + + refute File.exists?(local_path) + end + end end diff --git a/test/uploaders/s3_test.exs b/test/uploaders/s3_test.exs index 171316340..ab7795c3b 100644 --- a/test/uploaders/s3_test.exs +++ b/test/uploaders/s3_test.exs @@ -79,4 +79,11 @@ defmodule Pleroma.Uploaders.S3Test do end end end + + describe "delete_file/1" do + test_with_mock "deletes file", ExAws, request: fn _req -> {:ok, %{status_code: 204}} end do + assert :ok = S3.delete_file("image.jpg") + assert_called(ExAws.request(:_)) + end + end end