From e688d4ee69dfbda0f8fd3a5544720a566b3946c5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 14 Apr 2020 18:59:04 +0200 Subject: [PATCH] MRF.StealEmojiPolicy: New Policy Inspired by https://git.pleroma.social/moonman/emoji-stealer-mrf/-/blob/master/steal_emoji_policy.ex --- CHANGELOG.md | 1 + docs/configuration/cheatsheet.md | 5 ++ .../web/activity_pub/mrf/steal_emoji_policy.ex | 97 ++++++++++++++++++++++ test/support/http_request_mock.ex | 4 + .../activity_pub/mrf/steal_emoji_policy_test.exs | 64 ++++++++++++++ 5 files changed, 171 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex create mode 100644 test/web/activity_pub/mrf/steal_emoji_policy_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7fb603d..acd0fe171 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix task to create trusted OAuth App. - Notifications: Added `follow_request` notification type. - Added `:reject_deletes` group to SimplePolicy +- MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances
API Changes - Mastodon API: Extended `/api/v1/instance`. diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 707d7fdbd..1b2d72087 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -149,6 +149,11 @@ config :pleroma, :mrf_user_allowlist, * `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines * `:reject` rejects the message entirely +#### mrf_steal_emoji +* `hosts`: List of hosts to steal emojis from +* `rejected_shortcodes`: Regex-list of shortcodes to reject +* `size_limit`: File size limit (in bytes), checked before an emoji is saved to the disk + ### :activitypub * `unfollow_blocked`: Whether blocks result in people getting unfollowed * `outgoing_blocks`: Whether to federate blocks to other instances diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex new file mode 100644 index 000000000..2858af9eb --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -0,0 +1,97 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do + require Logger + + alias Pleroma.Config + + @moduledoc "Detect new emojis by their shortcode and steals them" + @behaviour Pleroma.Web.ActivityPub.MRF + + defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host]) + + defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) + + defp steal_emoji({shortcode, url}) do + url = Pleroma.Web.MediaProxy.url(url) + {:ok, response} = Pleroma.HTTP.get(url) + size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) + + if byte_size(response.body) <= size_limit do + emoji_dir_path = + Config.get( + [:mrf_steal_emoji, :path], + Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") + ) + + extension = + url + |> URI.parse() + |> Map.get(:path) + |> Path.basename() + |> Path.extname() + + file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")]) + + try do + :ok = File.write(file_path, response.body) + + shortcode + rescue + e -> + Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") + nil + end + else + Logger.debug( + "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{ + size_limit + } B)" + ) + + nil + end + rescue + e -> + Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") + nil + end + + @impl true + def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do + host = URI.parse(actor).host + + if remote_host?(host) and accept_host?(host) do + installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) + + new_emojis = + foreign_emojis + |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end) + |> Enum.filter(fn {shortcode, _url} -> + reject_emoji? = + Config.get([:mrf_steal_emoji, :rejected_shortcodes], []) + |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end) + + !reject_emoji? + end) + |> Enum.map(&steal_emoji(&1)) + |> Enum.filter(& &1) + + if !Enum.empty?(new_emojis) do + Logger.info("Stole new emojis: #{inspect(new_emojis)}") + Pleroma.Emoji.reload() + end + end + + {:ok, message} + end + + def filter(message), do: {:ok, message} + + @impl true + def describe do + {:ok, %{}} + end +end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 3a95e92da..3d5128835 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1291,6 +1291,10 @@ defmodule HttpRequestMock do }} end + def get("https://example.org/emoji/firedfox.png", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}} + end + def get("https://skippers-bin.com/users/7v1w1r8ce6", _, _, _) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}} end diff --git a/test/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/web/activity_pub/mrf/steal_emoji_policy_test.exs new file mode 100644 index 000000000..8882c8c13 --- /dev/null +++ b/test/web/activity_pub/mrf/steal_emoji_policy_test.exs @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do + use Pleroma.DataCase + + alias Pleroma.Config + alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + setup do + clear_config(:mrf_steal_emoji) + + emoji_path = Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") + File.rm_rf!(emoji_path) + File.mkdir!(emoji_path) + + Pleroma.Emoji.reload() + end + + test "does nothing by default" do + installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) + refute "firedfox" in installed_emoji + + message = %{ + "type" => "Create", + "object" => %{ + "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}], + "actor" => "https://example.org/users/admin" + } + } + + assert {:ok, message} == StealEmojiPolicy.filter(message) + + installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) + refute "firedfox" in installed_emoji + end + + test "Steals emoji on unknown shortcode from allowed remote host" do + installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) + refute "firedfox" in installed_emoji + + message = %{ + "type" => "Create", + "object" => %{ + "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}], + "actor" => "https://example.org/users/admin" + } + } + + Config.put([:mrf_steal_emoji, :hosts], ["example.org"]) + Config.put([:mrf_steal_emoji, :size_limit], 284_468) + + assert {:ok, message} == StealEmojiPolicy.filter(message) + + installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) + assert "firedfox" in installed_emoji + end +end