From 3dc04b9aca99fc6888e2354fa1cd3c2838ded499 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 8 Jan 2021 13:10:49 +0000 Subject: [PATCH] Merge branch 'fix/missing_emoji_steal_dir' into 'develop' Create dir for EmojiStealPolicy automatically See merge request pleroma/pleroma!3218 --- CHANGELOG.md | 7 ++ .../web/activity_pub/mrf/steal_emoji_policy.ex | 86 ++++++++--------- .../activity_pub/mrf/steal_emoji_policy_test.exs | 105 ++++++++++++++------- 3 files changed, 122 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f32014f1c..07e3271da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## Unreleased (Patch) + +### Fixed + +- StealEmojiPolicy creates dir for emojis, if it doesn't exist. + ## [2.2.1] - 2020-12-22 ### Changed @@ -17,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Rich Media Previews sometimes showed the wrong preview due to a bug following redirects. - Fixes for the autolinker. - Forwarded reports duplication from Pleroma instances. +- Emoji Reaction activity filtering from blocked and muted accounts. -
API diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 2858af9eb..788f21261 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -10,73 +10,75 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do @moduledoc "Detect new emojis by their shortcode and steals them" @behaviour Pleroma.Web.ActivityPub.MRF - defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host]) - defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) - defp steal_emoji({shortcode, url}) do + defp steal_emoji({shortcode, url}, emoji_dir_path) do url = Pleroma.Web.MediaProxy.url(url) - {:ok, response} = Pleroma.HTTP.get(url) - size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) - if byte_size(response.body) <= size_limit do - emoji_dir_path = - Config.get( - [:mrf_steal_emoji, :path], - Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") + with {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do + size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) + + if byte_size(response.body) <= size_limit do + extension = + url + |> URI.parse() + |> Map.get(:path) + |> Path.basename() + |> Path.extname() + + file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png")) + + case File.write(file_path, response.body) do + :ok -> + shortcode + + e -> + Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") + nil + end + else + Logger.debug( + "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{ + size_limit + } B)" ) - extension = - url - |> URI.parse() - |> Map.get(:path) - |> Path.basename() - |> Path.extname() - - file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")]) - - try do - :ok = File.write(file_path, response.body) - - shortcode - rescue - e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") - nil + nil end else - Logger.debug( - "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{ - size_limit - } B)" - ) - - nil + e -> + Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") + nil end - rescue - e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") - nil end @impl true def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do host = URI.parse(actor).host - if remote_host?(host) and accept_host?(host) do + if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) + emoji_dir_path = + Config.get( + [:mrf_steal_emoji, :path], + Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") + ) + + File.mkdir_p(emoji_dir_path) + new_emojis = foreign_emojis - |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end) + |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end) |> Enum.filter(fn {shortcode, _url} -> reject_emoji? = - Config.get([:mrf_steal_emoji, :rejected_shortcodes], []) + [:mrf_steal_emoji, :rejected_shortcodes] + |> Config.get([]) |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end) !reject_emoji? end) - |> Enum.map(&steal_emoji(&1)) + |> Enum.map(&steal_emoji(&1, emoji_dir_path)) |> Enum.filter(& &1) if !Enum.empty?(new_emojis) do diff --git a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs index 3f8222736..7665d00d0 100644 --- a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do use Pleroma.DataCase alias Pleroma.Config + alias Pleroma.Emoji alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy setup_all do @@ -14,55 +15,91 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do end setup do - emoji_path = Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") - File.rm_rf!(emoji_path) - File.mkdir!(emoji_path) + emoji_path = [:instance, :static_dir] |> Config.get() |> Path.join("emoji/stolen") - Pleroma.Emoji.reload() + Emoji.reload() + + message = %{ + "type" => "Create", + "object" => %{ + "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}], + "actor" => "https://example.org/users/admin" + } + } on_exit(fn -> File.rm_rf!(emoji_path) end) - :ok + [message: message, path: emoji_path] end - test "does nothing by default" do - installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - refute "firedfox" in installed_emoji + test "does nothing by default", %{message: message} do + refute "firedfox" in installed() - message = %{ - "type" => "Create", - "object" => %{ - "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}], - "actor" => "https://example.org/users/admin" - } - } + assert {:ok, _message} = StealEmojiPolicy.filter(message) - assert {:ok, message} == StealEmojiPolicy.filter(message) - - installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - refute "firedfox" in installed_emoji + refute "firedfox" in installed() end - test "Steals emoji on unknown shortcode from allowed remote host" do - installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - refute "firedfox" in installed_emoji + test "Steals emoji on unknown shortcode from allowed remote host", %{ + message: message, + path: path + } do + refute "firedfox" in installed() + refute File.exists?(path) - message = %{ - "type" => "Create", - "object" => %{ - "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}], - "actor" => "https://example.org/users/admin" - } - } + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) - clear_config([:mrf_steal_emoji, :hosts], ["example.org"]) - clear_config([:mrf_steal_emoji, :size_limit], 284_468) + assert {:ok, _message} = StealEmojiPolicy.filter(message) - assert {:ok, message} == StealEmojiPolicy.filter(message) + assert "firedfox" in installed() + assert File.exists?(path) - installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) - assert "firedfox" in installed_emoji + assert path + |> Path.join("firedfox.png") + |> File.exists?() end + + test "reject shortcode", %{message: message} do + refute "firedfox" in installed() + + clear_config(:mrf_steal_emoji, + hosts: ["example.org"], + size_limit: 284_468, + rejected_shortcodes: [~r/firedfox/] + ) + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + refute "firedfox" in installed() + end + + test "reject if size is above the limit", %{message: message} do + refute "firedfox" in installed() + + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 50_000) + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + refute "firedfox" in installed() + end + + test "reject if host returns error", %{message: message} do + refute "firedfox" in installed() + + Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} -> + {:ok, %Tesla.Env{status: 404, body: "Not found"}} + end) + + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) + + ExUnit.CaptureLog.capture_log(fn -> + assert {:ok, _message} = StealEmojiPolicy.filter(message) + end) =~ "MRF.StealEmojiPolicy: Failed to fetch https://example.org/emoji/firedfox.png" + + refute "firedfox" in installed() + end + + defp installed, do: Emoji.get_all() |> Enum.map(fn {k, _} -> k end) end