From e9e17e5df34051bce60232890ea042582af31f8c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 00:27:51 -0500 Subject: [PATCH 001/150] Upgrade Earmark to v1.4.10 --- lib/pleroma/earmark_renderer.ex | 256 --------------------- lib/pleroma/formatter.ex | 8 + .../object_validators/audio_video_validator.ex | 3 +- lib/pleroma/web/common_api/utils.ex | 3 +- mix.exs | 2 +- mix.lock | 2 +- test/pleroma/formatter_test.exs | 7 + test/pleroma/web/common_api/utils_test.exs | 75 ++++++ 8 files changed, 95 insertions(+), 261 deletions(-) delete mode 100644 lib/pleroma/earmark_renderer.ex diff --git a/lib/pleroma/earmark_renderer.ex b/lib/pleroma/earmark_renderer.ex deleted file mode 100644 index 6211a3b4a..000000000 --- a/lib/pleroma/earmark_renderer.ex +++ /dev/null @@ -1,256 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only -# -# This file is derived from Earmark, under the following copyright: -# Copyright © 2014 Dave Thomas, The Pragmatic Programmers -# SPDX-License-Identifier: Apache-2.0 -# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex -defmodule Pleroma.EarmarkRenderer do - @moduledoc false - - alias Earmark.Block - alias Earmark.Context - alias Earmark.HtmlRenderer - alias Earmark.Options - - import Earmark.Inline, only: [convert: 3] - import Earmark.Helpers.HtmlHelpers - import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2] - import Earmark.Context, only: [append: 2, set_value: 2] - import Earmark.Options, only: [get_mapper: 1] - - @doc false - def render(blocks, %Context{options: %Options{}} = context) do - messages = get_messages(context) - - {contexts, html} = - get_mapper(context.options).( - blocks, - &render_block(&1, put_in(context.options.messages, [])) - ) - |> Enum.unzip() - - all_messages = - contexts - |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end) - - {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()} - end - - ############# - # Paragraph # - ############# - defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do - lines = convert(lines, lnb, context) - add_attrs(lines, "

#{lines.value}

", attrs, [], lnb) - end - - ######## - # Html # - ######## - defp render_block(%Block.Html{html: html}, context) do - {context, html} - end - - defp render_block(%Block.HtmlComment{lines: lines}, context) do - {context, lines} - end - - defp render_block(%Block.HtmlOneline{html: html}, context) do - {context, html} - end - - ######### - # Ruler # - ######### - defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do - add_attrs(context, "
", attrs, [], lnb) - end - - ########### - # Heading # - ########### - defp render_block( - %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs}, - context - ) do - converted = convert(content, lnb, context) - html = "#{converted.value}" - add_attrs(converted, html, attrs, [], lnb) - end - - ############## - # Blockquote # - ############## - - defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do - {context1, body} = render(blocks, context) - html = "
#{body}
" - add_attrs(context1, html, attrs, [], lnb) - end - - ######### - # Table # - ######### - - defp render_block( - %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs}, - context - ) do - {context1, html} = add_attrs(context, "", attrs, [], lnb) - context2 = set_value(context1, html) - - context3 = - if header do - append(add_trs(append(context2, ""), [header], "th", aligns, lnb), "") - else - # Maybe an error, needed append(context, html) - context2 - end - - context4 = append(add_trs(append(context3, ""), rows, "td", aligns, lnb), "") - - {context4, [context4.value, "
"]} - end - - ######## - # Code # - ######## - - defp render_block( - %Block.Code{lnb: lnb, language: language, attrs: attrs} = block, - %Context{options: options} = context - ) do - class = - if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: "" - - tag = ~s[
]
-    lines = options.render_code.(block)
-    html = ~s[#{tag}#{lines}
] - add_attrs(context, html, attrs, [], lnb) - end - - ######### - # Lists # - ######### - - defp render_block( - %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start}, - context - ) do - {context1, content} = render(items, context) - html = "<#{type}#{start}>#{content}" - add_attrs(context1, html, attrs, [], lnb) - end - - # format a single paragraph list item, and remove the para tags - defp render_block( - %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs}, - context - ) - when length(blocks) == 1 do - {context1, content} = render(blocks, context) - content = Regex.replace(~r{}, content, "") - html = "
  • #{content}
    • " - add_attrs(context1, html, attrs, [], lnb) - end - - # format a spaced list item - defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do - {context1, content} = render(blocks, context) - html = "
  • #{content}
    • " - add_attrs(context1, html, attrs, [], lnb) - end - - ################## - # Footnote Block # - ################## - - defp render_block(%Block.FnList{blocks: footnotes}, context) do - items = - Enum.map(footnotes, fn note -> - blocks = append_footnote_link(note) - %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks} - end) - - {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context) - {context1, Enum.join([~s[
    ], "
    ", html, "
    "])} - end - - ####################################### - # Isolated IALs are rendered as paras # - ####################################### - - defp render_block(%Block.Ial{verbatim: verbatim}, context) do - {context, "

    {:#{verbatim}}

    "} - end - - #################### - # IDDef is ignored # - #################### - - defp render_block(%Block.IdDef{}, context), do: {context, ""} - - ##################################### - # And here are the inline renderers # - ##################################### - - defdelegate br, to: HtmlRenderer - defdelegate codespan(text), to: HtmlRenderer - defdelegate em(text), to: HtmlRenderer - defdelegate strong(text), to: HtmlRenderer - defdelegate strikethrough(text), to: HtmlRenderer - - defdelegate link(url, text), to: HtmlRenderer - defdelegate link(url, text, title), to: HtmlRenderer - - defdelegate image(path, alt, title), to: HtmlRenderer - - defdelegate footnote_link(ref, backref, number), to: HtmlRenderer - - # Table rows - defp add_trs(context, rows, tag, aligns, lnb) do - numbered_rows = - rows - |> Enum.zip(Stream.iterate(lnb, &(&1 + 1))) - - numbered_rows - |> Enum.reduce(context, fn {row, lnb}, ctx -> - append(add_tds(append(ctx, ""), row, tag, aligns, lnb), "") - end) - end - - defp add_tds(context, row, tag, aligns, lnb) do - Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb)) - end - - defp add_td_fn(row, tag, aligns, lnb) do - fn n, ctx -> - style = - case Enum.at(aligns, n - 1, :default) do - :default -> "" - align -> " style=\"text-align: #{align}\"" - end - - col = Enum.at(row, n - 1) - converted = convert(col, lnb, set_messages(ctx, [])) - append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}") - end - end - - ############################### - # Append Footnote Return Link # - ############################### - - defdelegate append_footnote_link(note), to: HtmlRenderer - defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer - - defdelegate render_code(lines), to: HtmlRenderer - - defp code_classes(language, prefix) do - ["" | String.split(prefix || "")] - |> Enum.map(fn pfx -> "#{pfx}#{language}" end) - |> Enum.join(" ") - end -end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 0c450eae4..b0e4a84ae 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -138,6 +138,14 @@ defmodule Pleroma.Formatter do |> Enum.join("") end + def minify({text, mentions, hashtags}, type) do + {minify(text, type), mentions, hashtags} + end + + def minify(text, "text/html") do + String.replace(text, "\n", "") + end + def truncate(text, max_length \\ 200, omission \\ "...") do # Remove trailing whitespace text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}") diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 16973e5db..eaf94797a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do use Ecto.Schema - alias Pleroma.EarmarkRenderer alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes @@ -96,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do when is_binary(content) do content = content - |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer}) + |> Earmark.as_html!() |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1c74ea787..b434a069e 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -294,8 +294,9 @@ defmodule Pleroma.Web.CommonAPI.Utils do def format_input(text, "text/markdown", options) do text |> Formatter.mentions_escape(options) - |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + |> Earmark.as_html!() |> Formatter.linkify(options) + |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") end diff --git a/mix.exs b/mix.exs index 72a6346b5..feb7eefa3 100644 --- a/mix.exs +++ b/mix.exs @@ -144,7 +144,7 @@ defmodule Pleroma.Mixfile do {:ex_aws, "~> 2.1.6"}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, - {:earmark, "1.4.3"}, + {:earmark, "1.4.10"}, {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, git: "https://github.com/msantos/crypt.git", diff --git a/mix.lock b/mix.lock index 6b551a012..29439a438 100644 --- a/mix.lock +++ b/mix.lock @@ -27,7 +27,7 @@ "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, + "earmark": {:hex, :earmark, "1.4.10", "bddce5e8ea37712a5bfb01541be8ba57d3b171d3fa4f80a0be9bcf1db417bcaf", [:mix], [{:earmark_parser, ">= 1.4.10", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "12dbfa80810478e521d3ffb941ad9fbfcbbd7debe94e1341b4c4a1b2411c1c27"}, "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, diff --git a/test/pleroma/formatter_test.exs b/test/pleroma/formatter_test.exs index 5781a3f01..ceedd1b6d 100644 --- a/test/pleroma/formatter_test.exs +++ b/test/pleroma/formatter_test.exs @@ -307,4 +307,11 @@ defmodule Pleroma.FormatterTest do assert Formatter.html_escape(text, "text/plain") == expected end + + test "it minifies html" do + text = "

    \nhello

    \n

    \nworld

    \n" + expected = "

    hello

    world

    " + + assert Formatter.minify(text, "text/html") == expected + end end diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 4d6c9ea26..39ea08ca8 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -168,6 +168,81 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do end end + describe "format_input/3 with markdown" do + test "Paragraph" do + code = ~s[Hello\n\nWorld!] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    Hello

    World!

    " + end + + test "raw HTML" do + code = ~s[OwO] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    #{code}

    " + end + + test "rulers" do + code = ~s[before\n\n-----\n\nafter] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    before

    after

    " + end + + test "headings" do + code = ~s[# h1\n## h2\n### h3\n] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    h1

    h2

    h3

    ] + end + + test "blockquote" do + code = ~s[> whoms't are you quoting?] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    whoms’t are you quoting?

    " + end + + test "code" do + code = ~s[`mix`] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    mix

    ] + + code = ~s[``mix``] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    mix

    ] + + code = ~s[```\nputs "Hello World"\n```] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[
    puts "Hello World"
    ] + end + + test "lists" do + code = ~s[- one\n- two\n- three\n- four] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "" + + code = ~s[1. one\n2. two\n3. three\n4. four\n] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "
    1. one
    2. two
    3. three
    4. four
    " + end + + test "delegated renderers" do + code = ~s[a
    b] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == "

    #{code}

    " + + code = ~s[*aaaa~*] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    aaaa~

    ] + + code = ~s[**aaaa~**] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    aaaa~

    ] + + # strikethrought + code = ~s[aaaa~] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    aaaa~

    ] + end + end + describe "context_to_conversation_id" do test "creates a mapping object" do conversation_id = Utils.context_to_conversation_id("random context") From ba71bbf6101847292346ba3b1fbe78ce4c385919 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 01:53:25 -0500 Subject: [PATCH 002/150] Improve Formatter.minify/2 --- lib/pleroma/formatter.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index b0e4a84ae..61906dda6 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -143,7 +143,10 @@ defmodule Pleroma.Formatter do end def minify(text, "text/html") do - String.replace(text, "\n", "") + text + |> String.replace(">\n", ">") + |> String.replace("> ", ">") + |> String.replace(" <", "<") end def truncate(text, max_length \\ 200, omission \\ "...") do From c4f4e48e574362d1ec86eaf11a382e81ca97cb35 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 02:08:41 -0500 Subject: [PATCH 003/150] Remove some N/A tests --- test/pleroma/web/common_api/utils_test.exs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 39ea08ca8..c6abbbe84 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -187,12 +187,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do assert result == "

    before

    after

    " end - test "headings" do - code = ~s[# h1\n## h2\n### h3\n] - {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == ~s[

    h1

    h2

    h3

    ] - end - test "blockquote" do code = ~s[> whoms't are you quoting?] {result, [], []} = Utils.format_input(code, "text/markdown") @@ -224,10 +218,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do end test "delegated renderers" do - code = ~s[a
    b] - {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == "

    #{code}

    " - code = ~s[*aaaa~*] {result, [], []} = Utils.format_input(code, "text/markdown") assert result == ~s[

    aaaa~

    ] @@ -236,7 +226,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do {result, [], []} = Utils.format_input(code, "text/markdown") assert result == ~s[

    aaaa~

    ] - # strikethrought + # strikethrough code = ~s[aaaa~] {result, [], []} = Utils.format_input(code, "text/markdown") assert result == ~s[

    aaaa~

    ] From b2548cfcdabdcb90bfcc9f4022c0b1cff9157a4a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 13:54:53 -0500 Subject: [PATCH 004/150] Sanitizer: allow
    tags --- priv/scrubbers/default.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 7b06994de..0893b17e5 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -39,6 +39,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:code, []) Meta.allow_tag_with_these_attributes(:del, []) Meta.allow_tag_with_these_attributes(:em, []) + Meta.allow_tag_with_these_attributes(:hr, []) Meta.allow_tag_with_these_attributes(:i, []) Meta.allow_tag_with_these_attributes(:li, []) Meta.allow_tag_with_these_attributes(:ol, []) From f8c93246d69a193ead81248879ba260e98673b3d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 14:27:50 -0500 Subject: [PATCH 005/150] Refactor Earmark code, fix tests --- lib/pleroma/formatter.ex | 4 ++++ .../activity_pub/object_validators/audio_video_validator.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 2 +- priv/scrubbers/default.ex | 2 ++ test/pleroma/web/common_api/utils_test.exs | 10 +++++----- test/pleroma/web/common_api_test.exs | 2 +- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 61906dda6..1be12055f 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -121,6 +121,10 @@ defmodule Pleroma.Formatter do end end + def markdown_to_html(text) do + Earmark.as_html!(text) + end + def html_escape({text, mentions, hashtags}, type) do {html_escape(text, type), mentions, hashtags} end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index eaf94797a..9b38aa4c2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do when is_binary(content) do content = content - |> Earmark.as_html!() + |> Pleroma.Formatter.markdown_to_html() |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b434a069e..be86009af 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -294,7 +294,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def format_input(text, "text/markdown", options) do text |> Formatter.mentions_escape(options) - |> Earmark.as_html!() + |> Formatter.markdown_to_html() |> Formatter.linkify(options) |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 0893b17e5..4694a92a5 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -59,6 +59,8 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card"]) Meta.allow_tag_with_these_attributes(:span, []) + Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) + @allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images]) if @allow_inline_images do diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index c6abbbe84..ab6392b1f 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -178,13 +178,13 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do test "raw HTML" do code = ~s[OwO] {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == "

    #{code}

    " + assert result == ~s[OwO] end test "rulers" do code = ~s[before\n\n-----\n\nafter] {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == "

    before

    after

    " + assert result == "

    before

    after

    " end test "blockquote" do @@ -204,7 +204,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do code = ~s[```\nputs "Hello World"\n```] {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == ~s[
    puts "Hello World"
    ] + assert result == ~s[
    puts "Hello World"
    ] end test "lists" do @@ -227,9 +227,9 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do assert result == ~s[

    aaaa~

    ] # strikethrough - code = ~s[aaaa~] + code = ~s[~~aaaa~~~] {result, [], []} = Utils.format_input(code, "text/markdown") - assert result == ~s[

    aaaa~

    ] + assert result == ~s[

    aaaa~

    ] end end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 585b2c174..c1b1af073 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -558,7 +558,7 @@ defmodule Pleroma.Web.CommonAPITest do object = Object.normalize(activity) - assert object.data["content"] == "

    2hu

    alert('xss')" + assert object.data["content"] == "

    2hu

    " assert object.data["source"] == post end From f1c67115d89ddcc7b10b963579dd621fca2094db Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 18:09:49 -0500 Subject: [PATCH 006/150] Upgrade linkify, test URL issues, fixes #2026 #1942 --- test/pleroma/web/common_api/utils_test.exs | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index ab6392b1f..28b05ed91 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -175,6 +175,54 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do assert result == "

    Hello

    World!

    " end + test "links" do + code = "https://en.wikipedia.org/wiki/Animal_Crossing_(video_game)" + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    #{code}

    ] + + code = "https://github.com/pragdave/earmark/" + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    #{code}

    ] + end + + test "link with local mention" do + insert(:user, %{nickname: "lain"}) + + code = "https://example.com/@lain" + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[

    #{code}

    ] + end + + test "local mentions" do + mario = insert(:user, %{nickname: "mario"}) + luigi = insert(:user, %{nickname: "luigi"}) + + code = "@mario @luigi yo what's up?" + {result, _, []} = Utils.format_input(code, "text/markdown") + + assert result == + ~s[

    @mario @luigi yo what’s up?

    ] + end + + test "remote mentions" do + mario = insert(:user, %{nickname: "mario@mushroom.kingdom", local: false}) + luigi = insert(:user, %{nickname: "luigi@mushroom.kingdom", local: false}) + + code = "@mario@mushroom.kingdom @luigi@mushroom.kingdom yo what's up?" + {result, _, []} = Utils.format_input(code, "text/markdown") + + assert result == + ~s[

    @mario @luigi yo what’s up?

    ] + end + test "raw HTML" do code = ~s[OwO] {result, [], []} = Utils.format_input(code, "text/markdown") @@ -205,6 +253,10 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do code = ~s[```\nputs "Hello World"\n```] {result, [], []} = Utils.format_input(code, "text/markdown") assert result == ~s[
    puts "Hello World"
    ] + + code = ~s[
    \n
    ] + {result, [], []} = Utils.format_input(code, "text/markdown") + assert result == ~s[
    <div>\n</div>
    ] end test "lists" do From 642729b49fca41fb142c6121fedf35c96c03b018 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 13 Oct 2020 19:16:57 -0500 Subject: [PATCH 007/150] Fix AudioVideoValidator markdown --- lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 9b38aa4c2..fa3e2c026 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -96,6 +96,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do content = content |> Pleroma.Formatter.markdown_to_html() + |> Pleroma.Formatter.minify("text/html") |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) From 6520599b7deac56780e1496c969cc45ff2e9f5da Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Dec 2020 13:43:40 -0600 Subject: [PATCH 008/150] Update Earmark to 1.4.13, use the new compact_output mode --- lib/pleroma/formatter.ex | 2 +- mix.exs | 2 +- mix.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 1be12055f..2aa236ca9 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -122,7 +122,7 @@ defmodule Pleroma.Formatter do end def markdown_to_html(text) do - Earmark.as_html!(text) + Earmark.as_html!(text, %Earmark.Options{compact_output: true}) end def html_escape({text, mentions, hashtags}, type) do diff --git a/mix.exs b/mix.exs index feb7eefa3..06d77edb7 100644 --- a/mix.exs +++ b/mix.exs @@ -144,7 +144,7 @@ defmodule Pleroma.Mixfile do {:ex_aws, "~> 2.1.6"}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, - {:earmark, "1.4.10"}, + {:earmark, "1.4.13"}, {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, git: "https://github.com/msantos/crypt.git", diff --git a/mix.lock b/mix.lock index 29439a438..e4dd32c83 100644 --- a/mix.lock +++ b/mix.lock @@ -27,8 +27,8 @@ "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.10", "bddce5e8ea37712a5bfb01541be8ba57d3b171d3fa4f80a0be9bcf1db417bcaf", [:mix], [{:earmark_parser, ">= 1.4.10", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "12dbfa80810478e521d3ffb941ad9fbfcbbd7debe94e1341b4c4a1b2411c1c27"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, + "earmark": {:hex, :earmark, "1.4.13", "2c6ce9768fc9fdbf4046f457e207df6360ee6c91ee1ecb8e9a139f96a4289d91", [:mix], [{:earmark_parser, ">= 1.4.12", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "a0cf3ed88ef2b1964df408889b5ecb886d1a048edde53497fc935ccd15af3403"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"}, "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"}, From f318d8e56df1e30f41c7ddf2e306b3552034921f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 11 Dec 2020 17:28:00 -0600 Subject: [PATCH 009/150] Use Pleroma.Formatter.markdown_to_html/1 in the tests --- test/pleroma/earmark_renderer_test.exs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/pleroma/earmark_renderer_test.exs b/test/pleroma/earmark_renderer_test.exs index 220d97d16..3adbefc1e 100644 --- a/test/pleroma/earmark_renderer_test.exs +++ b/test/pleroma/earmark_renderer_test.exs @@ -6,74 +6,74 @@ defmodule Pleroma.EarmarkRendererTest do test "Paragraph" do code = ~s[Hello\n\nWorld!] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    Hello

    World!

    " end test "raw HTML" do code = ~s[OwO] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    #{code}

    " end test "rulers" do code = ~s[before\n\n-----\n\nafter] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    before

    after

    " end test "headings" do code = ~s[# h1\n## h2\n### h3\n] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    h1

    h2

    h3

    ] end test "blockquote" do code = ~s[> whoms't are you quoting?] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    whoms’t are you quoting?

    " end test "code" do code = ~s[`mix`] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    mix

    ] code = ~s[``mix``] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    mix

    ] code = ~s[```\nputs "Hello World"\n```] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[
    puts "Hello World"
    ] end test "lists" do code = ~s[- one\n- two\n- three\n- four] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "
    • one
    • two
    • three
    • four
    " code = ~s[1. one\n2. two\n3. three\n4. four\n] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "
    1. one
    2. two
    3. three
    4. four
    " end test "delegated renderers" do code = ~s[a
    b] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == "

    #{code}

    " code = ~s[*aaaa~*] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    aaaa~

    ] code = ~s[**aaaa~**] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    aaaa~

    ] # strikethrought code = ~s[aaaa~] - result = Earmark.as_html!(code, %Earmark.Options{renderer: Pleroma.EarmarkRenderer}) + result = Pleroma.Formatter.markdown_to_html(code) assert result == ~s[

    aaaa~

    ] end end From 7eecc3b61d6da64e0bfdc5b155cba0dae07b84d5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 16 Feb 2021 23:23:35 +0100 Subject: [PATCH 010/150] OpenAPI: MastodonAPI Timeline Controller --- .../web/api_spec/operations/timeline_operation.ex | 3 ++- .../controllers/timeline_controller_test.exs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex index cae18c758..24d792916 100644 --- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex +++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex @@ -115,7 +115,8 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do ], operationId: "TimelineController.hashtag", responses: %{ - 200 => Operation.response("Array of Status", "application/json", array_of_statuses()) + 200 => Operation.response("Array of Status", "application/json", array_of_statuses()), + 401 => Operation.response("Error", "application/json", ApiError) } } end diff --git a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs index cc409451c..ed1286675 100644 --- a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs @@ -905,10 +905,10 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do %{conn: auth_conn} = oauth_access(["read:statuses"]) res_conn = get(auth_conn, "#{base_uri}?local=true") - assert length(json_response(res_conn, 200)) == 1 + assert length(json_response_and_validate_schema(res_conn, 200)) == 1 res_conn = get(auth_conn, "#{base_uri}?local=false") - assert length(json_response(res_conn, 200)) == 2 + assert length(json_response_and_validate_schema(res_conn, 200)) == 2 end test "with default settings on private instances, returns 403 for unauthenticated users", %{ @@ -922,7 +922,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do for local <- [true, false] do res_conn = get(conn, "#{base_uri}?local=#{local}") - assert json_response(res_conn, :unauthorized) == error_response + assert json_response_and_validate_schema(res_conn, :unauthorized) == error_response end ensure_authenticated_access(base_uri) @@ -939,7 +939,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do for local <- [true, false] do res_conn = get(conn, "#{base_uri}?local=#{local}") - assert json_response(res_conn, :unauthorized) == error_response + assert json_response_and_validate_schema(res_conn, :unauthorized) == error_response end ensure_authenticated_access(base_uri) @@ -951,10 +951,10 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do clear_config([:restrict_unauthenticated, :timelines, :federated], true) res_conn = get(conn, "#{base_uri}?local=true") - assert length(json_response(res_conn, 200)) == 1 + assert length(json_response_and_validate_schema(res_conn, 200)) == 1 res_conn = get(conn, "#{base_uri}?local=false") - assert json_response(res_conn, :unauthorized) == error_response + assert json_response_and_validate_schema(res_conn, :unauthorized) == error_response ensure_authenticated_access(base_uri) end @@ -966,11 +966,11 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do clear_config([:restrict_unauthenticated, :timelines, :federated], false) res_conn = get(conn, "#{base_uri}?local=true") - assert json_response(res_conn, :unauthorized) == error_response + assert json_response_and_validate_schema(res_conn, :unauthorized) == error_response # Note: local activities get delivered as part of federated timeline res_conn = get(conn, "#{base_uri}?local=false") - assert length(json_response(res_conn, 200)) == 2 + assert length(json_response_and_validate_schema(res_conn, 200)) == 2 ensure_authenticated_access(base_uri) end From 3123ecdd6e7a189f815624ee78be4f62487aa3db Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 16 Feb 2021 23:37:16 +0100 Subject: [PATCH 011/150] OpenAPI: MastodonAPI Media Controller --- lib/pleroma/web/api_spec/operations/media_operation.ex | 1 + test/pleroma/web/mastodon_api/controllers/media_controller_test.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/media_operation.ex b/lib/pleroma/web/api_spec/operations/media_operation.ex index 85aa14869..1e245b291 100644 --- a/lib/pleroma/web/api_spec/operations/media_operation.ex +++ b/lib/pleroma/web/api_spec/operations/media_operation.ex @@ -105,6 +105,7 @@ defmodule Pleroma.Web.ApiSpec.MediaOperation do responses: %{ 200 => Operation.response("Media", "application/json", Attachment), 401 => Operation.response("Media", "application/json", ApiError), + 403 => Operation.response("Media", "application/json", ApiError), 422 => Operation.response("Media", "application/json", ApiError) } } diff --git a/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs index 6c8f984d5..39d7f99f6 100644 --- a/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/media_controller_test.exs @@ -140,7 +140,7 @@ defmodule Pleroma.Web.MastodonAPI.MediaControllerTest do conn |> get("/api/v1/media/#{object.id}") - |> json_response(403) + |> json_response_and_validate_schema(403) end end end From e47f83cfc822716c00f3fcaffe73f31208749601 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 16 Feb 2021 23:39:07 +0100 Subject: [PATCH 012/150] OpenAPI: MastodonAPI Conversation Controller --- .../web/mastodon_api/controllers/conversation_controller_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs index 3176f1296..00797a9ea 100644 --- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs @@ -214,7 +214,8 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do res_conn = get(conn, "/api/v1/statuses/#{direct.id}/context") - assert %{"ancestors" => [], "descendants" => []} == json_response(res_conn, 200) + assert %{"ancestors" => [], "descendants" => []} == + json_response_and_validate_schema(res_conn, 200) end test "Removes a conversation", %{user: user_one, conn: conn} do From 3a8404820d803ccea44071178cc90f6aafcee80b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 16 Feb 2021 23:40:50 +0100 Subject: [PATCH 013/150] Verify MastoFE Controller put_settings response --- test/pleroma/web/mastodon_api/masto_fe_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/masto_fe_controller_test.exs b/test/pleroma/web/mastodon_api/masto_fe_controller_test.exs index ea66c708f..e679d781a 100644 --- a/test/pleroma/web/mastodon_api/masto_fe_controller_test.exs +++ b/test/pleroma/web/mastodon_api/masto_fe_controller_test.exs @@ -20,7 +20,7 @@ defmodule Pleroma.Web.MastodonAPI.MastoFEControllerTest do |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:accounts"])) |> put("/api/web/settings", %{"data" => %{"programming" => "socks"}}) - assert _result = json_response(conn, 200) + assert %{} = json_response(conn, 200) user = User.get_cached_by_ap_id(user.ap_id) assert user.mastofe_settings == %{"programming" => "socks"} From 0c7c6463d13b8a4471b8721912c82fe1cbe3e91a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 Feb 2021 00:35:26 +0100 Subject: [PATCH 014/150] OpenAPI: MastodonAPI Account Controller, excluding OAuth --- .../web/mastodon_api/controllers/account_controller_test.exs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index a327c0d1d..3036e25b3 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -514,11 +514,11 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do {:ok, post_2} = CommonAPI.post(user, %{status: "second post"}) response_1 = get(conn, "/api/v1/accounts/#{user.id}/statuses?limit=1") - assert [res] = json_response(response_1, 200) + assert [res] = json_response_and_validate_schema(response_1, 200) assert res["id"] == post_2.id response_2 = get(conn, "/api/v1/accounts/#{user.id}/statuses?limit=1&max_id=#{res["id"]}") - assert [res] = json_response(response_2, 200) + assert [res] = json_response_and_validate_schema(response_2, 200) assert res["id"] == post_1.id refute response_1 == response_2 @@ -881,7 +881,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert [] == conn |> get("/api/v1/timelines/home") - |> json_response(200) + |> json_response_and_validate_schema(200) assert %{"showing_reblogs" => true} = conn @@ -892,7 +892,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert [%{"id" => ^reblog_id}] = conn |> get("/api/v1/timelines/home") - |> json_response(200) + |> json_response_and_validate_schema(200) end test "following with reblogs" do @@ -910,7 +910,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert [%{"id" => ^reblog_id}] = conn |> get("/api/v1/timelines/home") - |> json_response(200) + |> json_response_and_validate_schema(200) assert %{"showing_reblogs" => false} = conn @@ -921,7 +921,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert [] == conn |> get("/api/v1/timelines/home") - |> json_response(200) + |> json_response_and_validate_schema(200) end test "following / unfollowing errors", %{user: user, conn: conn} do From ef5de5eb398b6d4cbc1ed338f2f41d3bfa1c5fe9 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 Feb 2021 00:45:01 +0100 Subject: [PATCH 015/150] OpenAPI: MastodonAPI Status Controller --- test/pleroma/web/mastodon_api/controllers/status_controller_test.exs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f616f405e..4c0149a4c 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -81,6 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do "sensitive" => 0 }) + # Idempotency plug response means detection fail assert %{"id" => second_id} = json_response(conn_two, 200) assert id == second_id @@ -1542,7 +1543,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> assign(:token, insert(:oauth_token, user: user3, scopes: ["read:statuses"])) |> get("api/v1/timelines/home") - [reblogged_activity] = json_response(conn3, 200) + [reblogged_activity] = json_response_and_validate_schema(conn3, 200) assert reblogged_activity["reblog"]["in_reply_to_id"] == replied_to.id @@ -1896,7 +1897,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do local = Pleroma.Constants.as_local_public() assert %{"content" => "cofe", "id" => id, "visibility" => "local"} = - json_response(conn_one, 200) + json_response_and_validate_schema(conn_one, 200) assert %Activity{id: ^id, data: %{"to" => [^local]}} = Activity.get_by_id(id) end From e4743847a18cb7cbb9e607232f25eb1cf63a4551 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 Feb 2021 01:07:56 +0100 Subject: [PATCH 016/150] OpenAPI: PleromaAPI UserImport Controller --- lib/pleroma/web/api_spec/operations/user_import_operation.ex | 1 + .../pleroma/web/pleroma_api/controllers/user_import_controller_test.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/user_import_operation.ex b/lib/pleroma/web/api_spec/operations/user_import_operation.ex index 6292e2004..8df19f1fc 100644 --- a/lib/pleroma/web/api_spec/operations/user_import_operation.ex +++ b/lib/pleroma/web/api_spec/operations/user_import_operation.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Web.ApiSpec.UserImportOperation do requestBody: request_body("Parameters", import_request(), required: true), responses: %{ 200 => ok_response(), + 403 => Operation.response("Error", "application/json", ApiError), 500 => Operation.response("Error", "application/json", ApiError) }, security: [%{"oAuth" => ["write:follow"]}] diff --git a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs index 25a7f8374..d977bc3a2 100644 --- a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs @@ -83,7 +83,7 @@ defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do assert %{"error" => "Insufficient permissions: follow | write:follows."} == json_response(conn, 403) else - assert json_response(conn, 200) + assert json_response_and_validate_schema(conn, 200) end end end From a22c53810b36c5382c805e1c5ed7e1cf3d747ebc Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 Feb 2021 01:19:25 +0100 Subject: [PATCH 017/150] Remove deprecated /api/qvitter/statuses/notifications/read --- CHANGELOG.md | 3 ++ lib/pleroma/web/router.ex | 6 --- lib/pleroma/web/twitter_api/controller.ex | 33 ---------------- test/pleroma/web/twitter_api/controller_test.exs | 49 ------------------------ 4 files changed, 3 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50484aaef..ce0bb1cb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change. +### Removed +- **Breaking**: Remove deprecated `/api/qvitter/statuses/notifications/read` (replaced by `/api/v1/pleroma/notifications/read`) + ## Unreleased (Patch) ## [2.3.0] - 2020-03-01 diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index de0bd27d7..ce2d701d7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -620,12 +620,6 @@ defmodule Pleroma.Web.Router do get("/oauth_tokens", TwitterAPI.Controller, :oauth_tokens) delete("/oauth_tokens/:id", TwitterAPI.Controller, :revoke_token) - - post( - "/qvitter/statuses/notifications/read", - TwitterAPI.Controller, - :mark_notifications_as_read - ) end scope "/", Pleroma.Web do diff --git a/lib/pleroma/web/twitter_api/controller.ex b/lib/pleroma/web/twitter_api/controller.ex index 077bfa70d..e32713311 100644 --- a/lib/pleroma/web/twitter_api/controller.ex +++ b/lib/pleroma/web/twitter_api/controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do use Pleroma.Web, :controller - alias Pleroma.Notification alias Pleroma.User alias Pleroma.Web.OAuth.Token alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug @@ -15,11 +14,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do require Logger plug( - OAuthScopesPlug, - %{scopes: ["write:notifications"]} when action == :mark_notifications_as_read - ) - - plug( :skip_plug, [OAuthScopesPlug, EnsurePublicOrAuthenticatedPlug] when action == :confirm_email ) @@ -67,31 +61,4 @@ defmodule Pleroma.Web.TwitterAPI.Controller do |> put_resp_content_type("application/json") |> send_resp(status, json) end - - def mark_notifications_as_read( - %{assigns: %{user: user}} = conn, - %{"latest_id" => latest_id} = params - ) do - Notification.set_read_up_to(user, latest_id) - - notifications = Notification.for_user(user, params) - - conn - # XXX: This is a hack because pleroma-fe still uses that API. - |> put_view(Pleroma.Web.MastodonAPI.NotificationView) - |> render("index.json", %{notifications: notifications, for: user}) - end - - def mark_notifications_as_read(%{assigns: %{user: _user}} = conn, _) do - bad_request_reply(conn, "You need to specify latest_id") - end - - defp bad_request_reply(conn, error_message) do - json = error_json(conn, error_message) - json_reply(conn, 400, json) - end - - defp error_json(conn, error_message) do - %{"error" => error_message, "request" => conn.request_path} |> Jason.encode!() - end end diff --git a/test/pleroma/web/twitter_api/controller_test.exs b/test/pleroma/web/twitter_api/controller_test.exs index 583c904b2..bca9e2dad 100644 --- a/test/pleroma/web/twitter_api/controller_test.exs +++ b/test/pleroma/web/twitter_api/controller_test.exs @@ -7,59 +7,10 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth.Token import Pleroma.Factory - describe "POST /api/qvitter/statuses/notifications/read" do - test "without valid credentials", %{conn: conn} do - conn = post(conn, "/api/qvitter/statuses/notifications/read", %{"latest_id" => 1_234_567}) - assert json_response(conn, 403) == %{"error" => "Invalid credentials."} - end - - test "with credentials, without any params" do - %{conn: conn} = oauth_access(["write:notifications"]) - - conn = post(conn, "/api/qvitter/statuses/notifications/read") - - assert json_response(conn, 400) == %{ - "error" => "You need to specify latest_id", - "request" => "/api/qvitter/statuses/notifications/read" - } - end - - test "with credentials, with params" do - %{user: current_user, conn: conn} = - oauth_access(["read:notifications", "write:notifications"]) - - other_user = insert(:user) - - {:ok, _activity} = - CommonAPI.post(other_user, %{ - status: "Hey @#{current_user.nickname}" - }) - - response_conn = - conn - |> get("/api/v1/notifications") - - [notification] = json_response(response_conn, 200) - - assert notification["pleroma"]["is_seen"] == false - - response_conn = - conn - |> post("/api/qvitter/statuses/notifications/read", %{"latest_id" => notification["id"]}) - - [notification] = response = json_response(response_conn, 200) - - assert length(response) == 1 - - assert notification["pleroma"]["is_seen"] == true - end - end - describe "GET /api/account/confirm_email/:id/:token" do setup do {:ok, user} = From 65cd9cb6384676c1660aa7f4da0f98ff7f43b999 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 17 Feb 2021 09:41:40 +0100 Subject: [PATCH 018/150] TwitterAPI: Remove unused read notification function --- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 940a645bb..60266aaab 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Config alias Pleroma.Emoji alias Pleroma.Healthcheck - alias Pleroma.Notification alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug @@ -30,7 +29,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do ] ) - plug(OAuthScopesPlug, %{scopes: ["write:notifications"]} when action == :notifications_read) def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do with %User{} = user <- User.get_cached_by_nickname(nick), @@ -62,17 +60,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def notifications_read(%{assigns: %{user: user}} = conn, %{"id" => notification_id}) do - with {:ok, _} <- Notification.read_one(user, notification_id) do - json(conn, %{status: "success"}) - else - {:error, message} -> - conn - |> put_resp_content_type("application/json") - |> send_resp(403, Jason.encode!(%{"error" => message})) - end - end - def frontend_configurations(conn, _params) do render(conn, "frontend_configurations.json") end From 55bdfb075c1cc5226948e3ff9d39fdae27aa9257 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 24 Feb 2021 23:40:33 +0100 Subject: [PATCH 019/150] OpenAPI: TwitterAPI Util Controller --- .../api_spec/operations/twitter_util_operation.ex | 219 +++++++++++++++++++++ .../web/twitter_api/controllers/util_controller.ex | 24 ++- .../web/twitter_api/util_controller_test.exs | 204 +++++++++++-------- 3 files changed, 360 insertions(+), 87 deletions(-) create mode 100644 lib/pleroma/web/api_spec/operations/twitter_util_operation.ex diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex new file mode 100644 index 000000000..62c9826f6 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -0,0 +1,219 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def emoji_operation do + %Operation{ + tags: ["Emojis"], + summary: "List all custom emojis", + operationId: "UtilController.emoji", + parameters: [], + responses: %{ + 200 => + Operation.response("List", "application/json", %Schema{ + type: :object, + additionalProperties: %Schema{ + type: :object, + properties: %{ + image_url: %Schema{type: :string}, + tags: %Schema{type: :array, items: %Schema{type: :string}} + } + }, + example: %{ + "firefox" => %{ + "image_url" => "/emoji/firefox.png", + "tag" => ["Fun"] + } + } + }) + } + } + end + + def frontend_configurations_operation do + %Operation{ + tags: ["Configuration"], + summary: "Dump frontend configurations", + operationId: "UtilController.frontend_configurations", + parameters: [], + responses: %{ + 200 => + Operation.response("List", "application/json", %Schema{ + type: :object, + additionalProperties: %Schema{type: :object} + }) + } + } + end + + def change_password_operation do + %Operation{ + tags: ["Accounts"], + summary: "Change account password", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.change_password", + parameters: [ + Operation.parameter(:password, :query, :string, "Current password", required: true), + Operation.parameter(:new_password, :query, :string, "New password", required: true), + Operation.parameter( + :new_password_confirmation, + :query, + :string, + "New password, confirmation", + required: true + ) + ], + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 400 => Operation.response("Error", "application/json", ApiError), + 403 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def change_email_operation do + %Operation{ + tags: ["Accounts"], + summary: "Change account email", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.change_email", + parameters: [ + Operation.parameter(:password, :query, :string, "Current password", required: true), + Operation.parameter(:email, :query, :string, "New email", required: true) + ], + requestBody: nil, + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 400 => Operation.response("Error", "application/json", ApiError), + 403 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def update_notificaton_settings_operation do + %Operation{ + tags: ["Accounts"], + summary: "Update Notification Settings", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.update_notificaton_settings", + parameters: [ + Operation.parameter( + :block_from_strangers, + :query, + BooleanLike, + "blocks notifications from accounts you do not follow" + ), + Operation.parameter( + :hide_notification_contents, + :query, + BooleanLike, + "removes the contents of a message from the push notification" + ) + ], + requestBody: nil, + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def disable_account_operation do + %Operation{ + tags: ["Accounts"], + summary: "Disable Account", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.disable_account", + parameters: [ + Operation.parameter(:password, :query, :string, "Password") + ], + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 403 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def delete_account_operation do + %Operation{ + tags: ["Accounts"], + summary: "Delete Account", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.delete_account", + parameters: [ + Operation.parameter(:password, :query, :string, "Password") + ], + responses: %{ + 200 => + Operation.response("Success", "application/json", %Schema{ + type: :object, + properties: %{status: %Schema{type: :string, example: "success"}} + }), + 403 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def captcha_operation do + %Operation{ + summary: "Get a captcha", + operationId: "UtilController.captcha", + parameters: [], + responses: %{ + 200 => Operation.response("Success", "application/json", %Schema{type: :object}) + } + } + end + + def healthcheck_operation do + %Operation{ + tags: ["Accounts"], + summary: "Disable Account", + security: [%{"oAuth" => ["write:accounts"]}], + operationId: "UtilController.healthcheck", + parameters: [], + responses: %{ + 200 => Operation.response("Healthy", "application/json", %Schema{type: :object}), + 503 => + Operation.response("Disabled or Unhealthy", "application/json", %Schema{type: :object}) + } + } + end + + def remote_subscribe_operation do + %Operation{ + tags: ["Accounts"], + summary: "Remote Subscribe", + operationId: "UtilController.remote_subscribe", + parameters: [], + responses: %{200 => Operation.response("Web Page", "test/html", %Schema{type: :string})} + } + end +end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 60266aaab..a2e69666e 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.WebFinger + plug(Pleroma.Web.ApiSpec.CastAndValidate when action != :remote_subscribe) plug(Pleroma.Web.Plugs.FederatingPlug when action == :remote_subscribe) plug( @@ -29,6 +30,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do ] ) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.TwitterUtilOperation def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do with %User{} = user <- User.get_cached_by_nickname(nick), @@ -79,13 +81,17 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def change_password(%{assigns: %{user: user}} = conn, params) do - case CommonAPI.Utils.confirm_current_password(user, params["password"]) do + def change_password(%{assigns: %{user: user}} = conn, %{ + password: password, + new_password: new_password, + new_password_confirmation: new_password_confirmation + }) do + case CommonAPI.Utils.confirm_current_password(user, password) do {:ok, user} -> with {:ok, _user} <- User.reset_password(user, %{ - password: params["new_password"], - password_confirmation: params["new_password_confirmation"] + password: new_password, + password_confirmation: new_password_confirmation }) do json(conn, %{status: "success"}) else @@ -102,10 +108,10 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def change_email(%{assigns: %{user: user}} = conn, params) do - case CommonAPI.Utils.confirm_current_password(user, params["password"]) do + def change_email(%{assigns: %{user: user}} = conn, %{password: password, email: email}) do + case CommonAPI.Utils.confirm_current_password(user, password) do {:ok, user} -> - with {:ok, _user} <- User.change_email(user, params["email"]) do + with {:ok, _user} <- User.change_email(user, email) do json(conn, %{status: "success"}) else {:error, changeset} -> @@ -122,7 +128,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def delete_account(%{assigns: %{user: user}} = conn, params) do - password = params["password"] || "" + password = params[:password] || "" case CommonAPI.Utils.confirm_current_password(user, password) do {:ok, user} -> @@ -135,7 +141,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def disable_account(%{assigns: %{user: user}} = conn, params) do - case CommonAPI.Utils.confirm_current_password(user, params["password"]) do + case CommonAPI.Utils.confirm_current_password(user, params[:password]) do {:ok, user} -> User.set_activation_async(user, false) json(conn, %{status: "success"}) diff --git a/test/pleroma/web/twitter_api/util_controller_test.exs b/test/pleroma/web/twitter_api/util_controller_test.exs index bdbc478c3..cc17940b5 100644 --- a/test/pleroma/web/twitter_api/util_controller_test.exs +++ b/test/pleroma/web/twitter_api/util_controller_test.exs @@ -25,11 +25,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do test "it updates notification settings", %{user: user, conn: conn} do conn - |> put("/api/pleroma/notification_settings", %{ - "block_from_strangers" => true, - "bar" => 1 - }) - |> json_response(:ok) + |> put( + "/api/pleroma/notification_settings?#{ + URI.encode_query(%{ + block_from_strangers: true + }) + }" + ) + |> json_response_and_validate_schema(:ok) user = refresh_record(user) @@ -41,8 +44,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do test "it updates notification settings to enable hiding contents", %{user: user, conn: conn} do conn - |> put("/api/pleroma/notification_settings", %{"hide_notification_contents" => "1"}) - |> json_response(:ok) + |> put( + "/api/pleroma/notification_settings?#{ + URI.encode_query(%{ + hide_notification_contents: 1 + }) + }" + ) + |> json_response_and_validate_schema(:ok) user = refresh_record(user) @@ -70,7 +79,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do response = conn |> get("/api/pleroma/frontend_configurations") - |> json_response(:ok) + |> json_response_and_validate_schema(:ok) assert response == Jason.encode!(config |> Enum.into(%{})) |> Jason.decode!() end @@ -81,7 +90,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do emoji = conn |> get("/api/pleroma/emoji") - |> json_response(200) + |> json_response_and_validate_schema(200) assert Enum.all?(emoji, fn {_key, @@ -103,7 +112,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do response = conn |> get("/api/pleroma/healthcheck") - |> json_response(503) + |> json_response_and_validate_schema(503) assert response == %{} end @@ -116,7 +125,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do response = conn |> get("/api/pleroma/healthcheck") - |> json_response(200) + |> json_response_and_validate_schema(200) assert %{ "active" => _, @@ -136,7 +145,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do response = conn |> get("/api/pleroma/healthcheck") - |> json_response(503) + |> json_response_and_validate_schema(503) assert %{ "active" => _, @@ -155,8 +164,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do test "with valid permissions and password, it disables the account", %{conn: conn, user: user} do response = conn - |> post("/api/pleroma/disable_account", %{"password" => "test"}) - |> json_response(:ok) + |> post("/api/pleroma/disable_account?password=test") + |> json_response_and_validate_schema(:ok) assert response == %{"status" => "success"} ObanHelpers.perform_all() @@ -171,8 +180,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do response = conn - |> post("/api/pleroma/disable_account", %{"password" => "test1"}) - |> json_response(:ok) + |> post("/api/pleroma/disable_account?password=test1") + |> json_response_and_validate_schema(:ok) assert response == %{"error" => "Invalid password."} user = User.get_cached_by_id(user.id) @@ -252,54 +261,61 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do conn = conn |> assign(:token, nil) - |> post("/api/pleroma/change_email") + |> post( + "/api/pleroma/change_email?#{ + URI.encode_query(%{password: "hi", email: "test@test.com"}) + }" + ) - assert json_response(conn, 403) == %{"error" => "Insufficient permissions: write:accounts."} + assert json_response_and_validate_schema(conn, 403) == %{ + "error" => "Insufficient permissions: write:accounts." + } end test "with proper permissions and invalid password", %{conn: conn} do conn = - post(conn, "/api/pleroma/change_email", %{ - "password" => "hi", - "email" => "test@test.com" - }) + post( + conn, + "/api/pleroma/change_email?#{ + URI.encode_query(%{password: "hi", email: "test@test.com"}) + }" + ) - assert json_response(conn, 200) == %{"error" => "Invalid password."} + assert json_response_and_validate_schema(conn, 200) == %{"error" => "Invalid password."} end test "with proper permissions, valid password and invalid email", %{ conn: conn } do conn = - post(conn, "/api/pleroma/change_email", %{ - "password" => "test", - "email" => "foobar" - }) + post( + conn, + "/api/pleroma/change_email?#{URI.encode_query(%{password: "test", email: "foobar"})}" + ) - assert json_response(conn, 200) == %{"error" => "Email has invalid format."} + assert json_response_and_validate_schema(conn, 200) == %{ + "error" => "Email has invalid format." + } end test "with proper permissions, valid password and no email", %{ conn: conn } do - conn = - post(conn, "/api/pleroma/change_email", %{ - "password" => "test" - }) + conn = post(conn, "/api/pleroma/change_email?#{URI.encode_query(%{password: "test"})}") - assert json_response(conn, 200) == %{"error" => "Email can't be blank."} + assert %{"error" => "Missing field: email."} = json_response_and_validate_schema(conn, 400) end test "with proper permissions, valid password and blank email", %{ conn: conn } do conn = - post(conn, "/api/pleroma/change_email", %{ - "password" => "test", - "email" => "" - }) + post( + conn, + "/api/pleroma/change_email?#{URI.encode_query(%{password: "test", email: ""})}" + ) - assert json_response(conn, 200) == %{"error" => "Email can't be blank."} + assert json_response_and_validate_schema(conn, 200) == %{"error" => "Email can't be blank."} end test "with proper permissions, valid password and non unique email", %{ @@ -308,24 +324,28 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user = insert(:user) conn = - post(conn, "/api/pleroma/change_email", %{ - "password" => "test", - "email" => user.email - }) + post( + conn, + "/api/pleroma/change_email?#{URI.encode_query(%{password: "test", email: user.email})}" + ) - assert json_response(conn, 200) == %{"error" => "Email has already been taken."} + assert json_response_and_validate_schema(conn, 200) == %{ + "error" => "Email has already been taken." + } end test "with proper permissions, valid password and valid email", %{ conn: conn } do conn = - post(conn, "/api/pleroma/change_email", %{ - "password" => "test", - "email" => "cofe@foobar.com" - }) + post( + conn, + "/api/pleroma/change_email?#{ + URI.encode_query(%{password: "test", email: "cofe@foobar.com"}) + }" + ) - assert json_response(conn, 200) == %{"status" => "success"} + assert json_response_and_validate_schema(conn, 200) == %{"status" => "success"} end end @@ -336,20 +356,35 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do conn = conn |> assign(:token, nil) - |> post("/api/pleroma/change_password") + |> post( + "/api/pleroma/change_password?#{ + URI.encode_query(%{ + password: "hi", + new_password: "newpass", + new_password_confirmation: "newpass" + }) + }" + ) - assert json_response(conn, 403) == %{"error" => "Insufficient permissions: write:accounts."} + assert json_response_and_validate_schema(conn, 403) == %{ + "error" => "Insufficient permissions: write:accounts." + } end test "with proper permissions and invalid password", %{conn: conn} do conn = - post(conn, "/api/pleroma/change_password", %{ - "password" => "hi", - "new_password" => "newpass", - "new_password_confirmation" => "newpass" - }) + post( + conn, + "/api/pleroma/change_password?#{ + URI.encode_query(%{ + password: "hi", + new_password: "newpass", + new_password_confirmation: "newpass" + }) + }" + ) - assert json_response(conn, 200) == %{"error" => "Invalid password."} + assert json_response_and_validate_schema(conn, 200) == %{"error" => "Invalid password."} end test "with proper permissions, valid password and new password and confirmation not matching", @@ -357,13 +392,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do conn: conn } do conn = - post(conn, "/api/pleroma/change_password", %{ - "password" => "test", - "new_password" => "newpass", - "new_password_confirmation" => "notnewpass" - }) + post( + conn, + "/api/pleroma/change_password?#{ + URI.encode_query(%{ + password: "test", + new_password: "newpass", + new_password_confirmation: "notnewpass" + }) + }" + ) - assert json_response(conn, 200) == %{ + assert json_response_and_validate_schema(conn, 200) == %{ "error" => "New password does not match confirmation." } end @@ -372,13 +412,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do conn: conn } do conn = - post(conn, "/api/pleroma/change_password", %{ - "password" => "test", - "new_password" => "", - "new_password_confirmation" => "" - }) + post( + conn, + "/api/pleroma/change_password?#{ + URI.encode_query(%{password: "test", new_password: "", new_password_confirmation: ""}) + }" + ) - assert json_response(conn, 200) == %{ + assert json_response_and_validate_schema(conn, 200) == %{ "error" => "New password can't be blank." } end @@ -388,13 +429,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user: user } do conn = - post(conn, "/api/pleroma/change_password", %{ - "password" => "test", - "new_password" => "newpass", - "new_password_confirmation" => "newpass" - }) + post( + conn, + "/api/pleroma/change_password?#{ + URI.encode_query(%{ + password: "test", + new_password: "newpass", + new_password_confirmation: "newpass" + }) + }" + ) - assert json_response(conn, 200) == %{"status" => "success"} + assert json_response_and_validate_schema(conn, 200) == %{"status" => "success"} fetched_user = User.get_cached_by_id(user.id) assert Pleroma.Password.Pbkdf2.verify_pass("newpass", fetched_user.password_hash) == true end @@ -409,7 +455,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do |> assign(:token, nil) |> post("/api/pleroma/delete_account") - assert json_response(conn, 403) == + assert json_response_and_validate_schema(conn, 403) == %{"error" => "Insufficient permissions: write:accounts."} end @@ -417,14 +463,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do for params <- [%{"password" => "hi"}, %{}] do ret_conn = post(conn, "/api/pleroma/delete_account", params) - assert json_response(ret_conn, 200) == %{"error" => "Invalid password."} + assert json_response_and_validate_schema(ret_conn, 200) == %{ + "error" => "Invalid password." + } end end test "with proper permissions and valid password", %{conn: conn, user: user} do - conn = post(conn, "/api/pleroma/delete_account", %{"password" => "test"}) + conn = post(conn, "/api/pleroma/delete_account?password=test") ObanHelpers.perform_all() - assert json_response(conn, 200) == %{"status" => "success"} + assert json_response_and_validate_schema(conn, 200) == %{"status" => "success"} user = User.get_by_id(user.id) refute user.is_active From b6a69b5efda5f75ad716252c69ae658a4e885b0a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 24 Mar 2021 12:50:05 -0500 Subject: [PATCH 020/150] Return token's primary key with POST /oauth/token --- .../API/differences_in_mastoapi_responses.md | 24 ++++++++++++++++++++-- lib/pleroma/web/o_auth/o_auth_view.ex | 1 + test/pleroma/web/o_auth/o_auth_controller_test.exs | 6 ++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index a14fcb416..3552b12fb 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -255,9 +255,29 @@ This information is returned in the `/api/v1/accounts/verify_credentials` endpoi *Pleroma supports refreshing tokens.* -`POST /oauth/token` +### POST `/oauth/token` -Post here request with `grant_type=refresh_token` to obtain new access token. Returns an access token. +You can obtain access tokens for a user in a few additional ways. + +#### Refreshing a token + +To obtain a new access token from a refresh token, pass `grant_type=refresh_token` with the following extra parameters: + +- `refresh_token`: The refresh token. + +#### Getting a token with a password + +To obtain a token from a user's password, pass `grant_type=password` with the following extra parameters: + +- `username`: Username to authenticate. +- `password`: The user's password. + +#### Response body + +Additional fields are returned in the response: + +- `id`: The primary key of this token in Pleroma's database. +- `me` (user tokens only): The ActivityPub ID of the user who owns the token. ## Account Registration diff --git a/lib/pleroma/web/o_auth/o_auth_view.ex b/lib/pleroma/web/o_auth/o_auth_view.ex index 281bbcc3c..1419c96a2 100644 --- a/lib/pleroma/web/o_auth/o_auth_view.ex +++ b/lib/pleroma/web/o_auth/o_auth_view.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.OAuth.OAuthView do def render("token.json", %{token: token} = opts) do response = %{ + id: token.id, token_type: "Bearer", access_token: token.token, refresh_token: token.refresh_token, diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 312500feb..0fdd5b8e9 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -805,10 +805,12 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do "client_secret" => app.client_secret }) - assert %{"access_token" => token} = json_response(conn, 200) + assert %{"id" => id, "access_token" => access_token} = json_response(conn, 200) - token = Repo.get_by(Token, token: token) + token = Repo.get_by(Token, token: access_token) assert token + assert token.id == id + assert token.token == access_token assert token.scopes == app.scopes end From 3ec1dbd9223aa44205e90967175f07cc532501ab Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 3 Feb 2021 16:09:28 +0300 Subject: [PATCH 021/150] Let pins federate - save object ids on pin, instead of activity ids - pins federation - removed pinned_activities field from the users table - activityPub endpoint for user pins - pulling remote users pins --- .../API/differences_in_mastoapi_responses.md | 1 + lib/pleroma/activity.ex | 79 +++++++++------- lib/pleroma/activity/queries.ex | 5 + lib/pleroma/user.ex | 77 ++++++++------- lib/pleroma/web/activity_pub/activity_pub.ex | 60 +++++++++++- .../web/activity_pub/activity_pub_controller.ex | 8 ++ lib/pleroma/web/activity_pub/builder.ex | 32 +++++++ lib/pleroma/web/activity_pub/object_validator.ex | 11 +++ .../object_validators/pin_validator.ex | 42 +++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 56 ++++++++++- lib/pleroma/web/activity_pub/transmogrifier.ex | 9 ++ lib/pleroma/web/activity_pub/views/user_view.ex | 21 +++++ .../web/api_spec/operations/status_operation.ex | 46 ++++++++- lib/pleroma/web/api_spec/schemas/status.ex | 7 ++ lib/pleroma/web/common_api.ex | 57 +++++++---- .../controllers/fallback_controller.ex | 6 ++ .../mastodon_api/controllers/status_controller.ex | 12 +++ lib/pleroma/web/mastodon_api/views/status_view.ex | 23 +++-- lib/pleroma/web/router.ex | 1 + .../20210202110641_add_pinned_objects_to_users.exs | 9 ++ ...0210203141144_add_featured_address_to_users.exs | 23 +++++ ..._move_pinned_activities_into_pinned_objects.exs | 28 ++++++ ...6045221_remove_pinned_activities_from_users.exs | 15 +++ test/fixtures/collections/featured.json | 39 ++++++++ test/fixtures/masto_pin.json | 41 ++++++++ test/fixtures/statuses/note.json | 27 ++++++ test/fixtures/users_mock/masto_featured.json | 18 ++++ test/fixtures/users_mock/user.json | 41 ++++++++ test/pleroma/user_test.exs | 45 +++++++++ .../activity_pub/activity_pub_controller_test.exs | 105 +++++++++++++++++++++ .../pleroma/web/activity_pub/activity_pub_test.exs | 77 +++++++++++++++ .../web/activity_pub/transmogrifier_test.exs | 74 +++++++++++++++ test/pleroma/web/common_api_test.exs | 60 ++++++++++-- .../controllers/status_controller_test.exs | 32 +++++-- .../web/mastodon_api/views/status_view_test.exs | 3 +- .../twitter_api/remote_follow_controller_test.exs | 30 ++++++ test/support/factory.ex | 6 +- test/support/http_request_mock.ex | 23 +++++ 38 files changed, 1127 insertions(+), 122 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/pin_validator.ex create mode 100644 priv/repo/migrations/20210202110641_add_pinned_objects_to_users.exs create mode 100644 priv/repo/migrations/20210203141144_add_featured_address_to_users.exs create mode 100644 priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs create mode 100644 priv/repo/migrations/20210206045221_remove_pinned_activities_from_users.exs create mode 100644 test/fixtures/collections/featured.json create mode 100644 test/fixtures/masto_pin.json create mode 100644 test/fixtures/statuses/note.json create mode 100644 test/fixtures/users_mock/masto_featured.json create mode 100644 test/fixtures/users_mock/user.json diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index a14fcb416..2ff56d3ca 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -38,6 +38,7 @@ Has these additional fields under the `pleroma` object: - `thread_muted`: true if the thread the post belongs to is muted - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint. - `parent_visible`: If the parent of this post is visible to the user or not. +- `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise. ## Scheduled statuses diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index d59403884..a4cfca4c5 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -184,40 +184,48 @@ defmodule Pleroma.Activity do |> Repo.one() end - @spec get_by_id(String.t()) :: Activity.t() | nil - def get_by_id(id) do - case FlakeId.flake_id?(id) do - true -> - Activity - |> where([a], a.id == ^id) - |> restrict_deactivated_users() - |> Repo.one() + @doc """ + Gets activity by ID, doesn't load activities from deactivated actors by default. + """ + @spec get_by_id(String.t(), keyword()) :: t() | nil + def get_by_id(id, opts \\ [filter: [:restrict_deactivated]]), do: get_by_id_with_opts(id, opts) - _ -> - nil + @spec get_by_id_with_user_actor(String.t()) :: t() | nil + def get_by_id_with_user_actor(id), do: get_by_id_with_opts(id, preload: [:user_actor]) + + @spec get_by_id_with_object(String.t()) :: t() | nil + def get_by_id_with_object(id), do: get_by_id_with_opts(id, preload: [:object]) + + defp get_by_id_with_opts(id, opts) do + if FlakeId.flake_id?(id) do + query = Queries.by_id(id) + + with_filters_query = + if is_list(opts[:filter]) do + Enum.reduce(opts[:filter], query, fn + {:type, type}, acc -> Queries.by_type(acc, type) + :restrict_deactivated, acc -> restrict_deactivated_users(acc) + _, acc -> acc + end) + else + query + end + + with_preloads_query = + if is_list(opts[:preload]) do + Enum.reduce(opts[:preload], with_filters_query, fn + :user_actor, acc -> with_preloaded_user_actor(acc) + :object, acc -> with_preloaded_object(acc) + _, acc -> acc + end) + else + with_filters_query + end + + Repo.one(with_preloads_query) end end - def get_by_id_with_user_actor(id) do - case FlakeId.flake_id?(id) do - true -> - Activity - |> where([a], a.id == ^id) - |> with_preloaded_user_actor() - |> Repo.one() - - _ -> - nil - end - end - - def get_by_id_with_object(id) do - Activity - |> where(id: ^id) - |> with_preloaded_object() - |> Repo.one() - end - def all_by_ids_with_object(ids) do Activity |> where([a], a.id in ^ids) @@ -269,6 +277,11 @@ defmodule Pleroma.Activity do def get_create_by_object_ap_id_with_object(_), do: nil + @spec create_by_id_with_object(String.t()) :: t() | nil + def create_by_id_with_object(id) do + get_by_id_with_opts(id, preload: [:object], filter: [type: "Create"]) + end + defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do get_create_by_object_ap_id_with_object(ap_id) end @@ -368,12 +381,6 @@ defmodule Pleroma.Activity do end end - @spec pinned_by_actor?(Activity.t()) :: boolean() - def pinned_by_actor?(%Activity{} = activity) do - actor = user_actor(activity) - activity.id in actor.pinned_activities - end - @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do ap_id diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index a6b02a889..4632651b0 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -14,6 +14,11 @@ defmodule Pleroma.Activity.Queries do alias Pleroma.Activity alias Pleroma.User + @spec by_id(query(), String.t()) :: query() + def by_id(query \\ Activity, id) do + from(a in query, where: a.id == ^id) + end + @spec by_ap_id(query, String.t()) :: query def by_ap_id(query \\ Activity, ap_id) do from( diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c1aa0f716..b78777141 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -99,6 +99,7 @@ defmodule Pleroma.User do field(:local, :boolean, default: true) field(:follower_address, :string) field(:following_address, :string) + field(:featured_address, :string) field(:search_rank, :float, virtual: true) field(:search_type, :integer, virtual: true) field(:tags, {:array, :string}, default: []) @@ -130,7 +131,6 @@ defmodule Pleroma.User do field(:hide_followers, :boolean, default: false) field(:hide_follows, :boolean, default: false) field(:hide_favorites, :boolean, default: true) - field(:pinned_activities, {:array, :string}, default: []) field(:email_notifications, :map, default: %{"digest" => false}) field(:mascot, :map, default: nil) field(:emoji, :map, default: %{}) @@ -148,6 +148,7 @@ defmodule Pleroma.User do field(:accepts_chat_messages, :boolean, default: nil) field(:last_active_at, :naive_datetime) field(:disclose_client, :boolean, default: true) + field(:pinned_objects, :map, default: %{}) embeds_one( :notification_settings, @@ -372,8 +373,10 @@ defmodule Pleroma.User do end # Should probably be renamed or removed + @spec ap_id(User.t()) :: String.t() def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" + @spec ap_followers(User.t()) :: String.t() def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" @@ -381,6 +384,11 @@ defmodule Pleroma.User do def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" + @spec ap_featured_collection(User.t()) :: String.t() + def ap_featured_collection(%User{featured_address: fa}) when is_binary(fa), do: fa + + def ap_featured_collection(%User{} = user), do: "#{ap_id(user)}/collections/featured" + defp truncate_fields_param(params) do if Map.has_key?(params, :fields) do Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1)) @@ -443,6 +451,7 @@ defmodule Pleroma.User do :uri, :follower_address, :following_address, + :featured_address, :hide_followers, :hide_follows, :hide_followers_count, @@ -454,7 +463,8 @@ defmodule Pleroma.User do :invisible, :actor_type, :also_known_as, - :accepts_chat_messages + :accepts_chat_messages, + :pinned_objects ] ) |> cast(params, [:name], empty_values: []) @@ -686,7 +696,7 @@ defmodule Pleroma.User do |> validate_format(:nickname, local_nickname_regex()) |> put_ap_id() |> unique_constraint(:ap_id) - |> put_following_and_follower_address() + |> put_following_and_follower_and_featured_address() end def register_changeset(struct, params \\ %{}, opts \\ []) do @@ -747,7 +757,7 @@ defmodule Pleroma.User do |> put_password_hash |> put_ap_id() |> unique_constraint(:ap_id) - |> put_following_and_follower_address() + |> put_following_and_follower_and_featured_address() end def maybe_validate_required_email(changeset, true), do: changeset @@ -765,11 +775,16 @@ defmodule Pleroma.User do put_change(changeset, :ap_id, ap_id) end - defp put_following_and_follower_address(changeset) do - followers = ap_followers(%User{nickname: get_field(changeset, :nickname)}) + defp put_following_and_follower_and_featured_address(changeset) do + user = %User{nickname: get_field(changeset, :nickname)} + followers = ap_followers(user) + following = ap_following(user) + featured = ap_featured_collection(user) changeset |> put_change(:follower_address, followers) + |> put_change(:following_address, following) + |> put_change(:featured_address, featured) end defp autofollow_users(user) do @@ -2343,45 +2358,35 @@ defmodule Pleroma.User do cast(user, %{is_approved: approved?}, [:is_approved]) end - def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do - if id not in user.pinned_activities do - max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) - params = %{pinned_activities: user.pinned_activities ++ [id]} - - # if pinned activity was scheduled for deletion, we remove job - if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do - Oban.cancel_job(expiration.id) - end + @spec add_pinned_object_id(User.t(), String.t()) :: {:ok, User.t()} | {:error, term()} + def add_pinned_object_id(%User{} = user, object_id) do + if !user.pinned_objects[object_id] do + params = %{pinned_objects: Map.put(user.pinned_objects, object_id, NaiveDateTime.utc_now())} user - |> cast(params, [:pinned_activities]) - |> validate_length(:pinned_activities, - max: max_pinned_statuses, - message: "You have already pinned the maximum number of statuses" - ) + |> cast(params, [:pinned_objects]) + |> validate_change(:pinned_objects, fn :pinned_objects, pinned_objects -> + max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) + + if Enum.count(pinned_objects) <= max_pinned_statuses do + [] + else + [pinned_objects: "You have already pinned the maximum number of statuses"] + end + end) else change(user) end |> update_and_set_cache() end - def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do - params = %{pinned_activities: List.delete(user.pinned_activities, id)} - - # if pinned activity was scheduled for deletion, we reschedule it for deletion - if data["expires_at"] do - # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation - {:ok, expires_at} = - data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast() - - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: id, - expires_at: expires_at - }) - end - + @spec remove_pinned_object_id(User.t(), String.t()) :: {:ok, t()} | {:error, term()} + def remove_pinned_object_id(%User{} = user, object_id) do user - |> cast(params, [:pinned_activities]) + |> cast( + %{pinned_objects: Map.delete(user.pinned_objects, object_id)}, + [:pinned_objects] + ) |> update_and_set_cache() end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index efbf92c70..d0051d1cb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -630,7 +630,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Map.put(:type, ["Create", "Announce"]) |> Map.put(:user, reading_user) |> Map.put(:actor_id, user.ap_id) - |> Map.put(:pinned_activity_ids, user.pinned_activities) + |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects)) params = if User.blocks?(reading_user, user) do @@ -1075,8 +1075,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_unlisted(query, _), do: query - defp restrict_pinned(query, %{pinned: true, pinned_activity_ids: ids}) do - from(activity in query, where: activity.id in ^ids) + defp restrict_pinned(query, %{pinned: true, pinned_object_ids: ids}) do + from( + [activity, object: o] in query, + where: + fragment( + "(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)", + activity.data, + activity.data, + activity.data, + ^ids + ) + ) end defp restrict_pinned(query, _), do: query @@ -1419,6 +1429,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do invisible = data["invisible"] || false actor_type = data["type"] || "Person" + featured_address = data["featured"] + {:ok, pinned_objects} = fetch_and_prepare_featured_from_ap_id(featured_address) + public_key = if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do data["publicKey"]["publicKeyPem"] @@ -1447,13 +1460,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do name: data["name"], follower_address: data["followers"], following_address: data["following"], + featured_address: featured_address, bio: data["summary"] || "", actor_type: actor_type, also_known_as: Map.get(data, "alsoKnownAs", []), public_key: public_key, inbox: data["inbox"], shared_inbox: shared_inbox, - accepts_chat_messages: accepts_chat_messages + accepts_chat_messages: accepts_chat_messages, + pinned_objects: pinned_objects } # nickname can be nil because of virtual actors @@ -1591,6 +1606,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + def pin_data_from_featured_collection(%{ + "type" => type, + "orderedItems" => objects + }) + when type in ["OrderedCollection", "Collection"] do + Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end) + end + + def fetch_and_prepare_featured_from_ap_id(nil) do + {:ok, %{}} + end + + def fetch_and_prepare_featured_from_ap_id(ap_id) do + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do + {:ok, pin_data_from_featured_collection(data)} + else + e -> + Logger.error("Could not decode featured collection at fetch #{ap_id}, #{inspect(e)}") + {:ok, %{}} + end + end + + def pinned_fetch_task(nil), do: nil + + def pinned_fetch_task(%{pinned_objects: pins}) do + if Enum.all?(pins, fn {ap_id, _} -> + Object.get_cached_by_ap_id(ap_id) || + match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id)) + end) do + :ok + else + :error + end + end + def make_user_from_ap_id(ap_id) do user = User.get_cached_by_ap_id(ap_id) @@ -1598,6 +1648,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Transmogrifier.upgrade_user_from_ap_id(ap_id) else with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do + {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) + if user do user |> User.remote_user_changeset(data) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 9d3dcc7f9..5aa3b281a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -543,4 +543,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json(object.data) end end + + def pinned(conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_cached_by_nickname(nickname) do + conn + |> put_resp_header("content-type", "application/activity+json") + |> json(UserView.render("featured.json", %{user: user})) + end + end end diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index f56bfc600..91a45836f 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -273,4 +273,36 @@ defmodule Pleroma.Web.ActivityPub.Builder do "context" => object.data["context"] }, []} end + + @spec pin(User.t(), Object.t()) :: {:ok, map(), keyword()} + def pin(%User{} = user, object) do + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "target" => pinned_url(user.nickname), + "object" => object.data["id"], + "actor" => user.ap_id, + "type" => "Add", + "to" => [Pleroma.Constants.as_public()], + "cc" => [user.follower_address] + }, []} + end + + @spec unpin(User.t(), Object.t()) :: {:ok, map, keyword()} + def unpin(%User{} = user, object) do + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "target" => pinned_url(user.nickname), + "object" => object.data["id"], + "actor" => user.ap_id, + "type" => "Remove", + "to" => [Pleroma.Constants.as_public()], + "cc" => [user.follower_address] + }, []} + end + + defp pinned_url(nickname) when is_binary(nickname) do + Pleroma.Web.Router.Helpers.activity_pub_url(Pleroma.Web.Endpoint, :pinned, nickname) + end end diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 297c19cc0..11432ef38 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -30,6 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.EventValidator alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.PinValidator alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator @@ -234,6 +235,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end + def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do + with {:ok, object} <- + object + |> PinValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + def cast_and_apply(%{"type" => "ChatMessage"} = object) do ChatMessageValidator.cast_and_apply(object) end diff --git a/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex b/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex new file mode 100644 index 000000000..dca8cba6f --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.PinValidator do + use Ecto.Schema + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:target) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:type) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + defp cast_data(data) do + cast(%__MODULE__{}, data, __schema__(:fields)) + end + + defp validate_data(changeset) do + changeset + |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) + |> validate_inclusion(:type, ~w(Add Remove)) + |> validate_actor_presence() + |> validate_object_presence() + end +end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0b9a9f0c5..9d22f9d3c 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -276,10 +276,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do result = case deleted_object do %Object{} -> - with {:ok, deleted_object, activity} <- Object.delete(deleted_object), + with {:ok, deleted_object, _activity} <- Object.delete(deleted_object), {_, actor} when is_binary(actor) <- {:actor, deleted_object.data["actor"]}, %User{} = user <- User.get_cached_by_ap_id(actor) do - User.remove_pinnned_activity(user, activity) + User.remove_pinned_object_id(user, deleted_object.data["id"]) {:ok, user} = ActivityPub.decrease_note_count_if_public(user, deleted_object) @@ -312,6 +312,58 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end end + # Tasks this handles: + # - adds pin to user + # - removes expiration job for pinned activity, if was set for expiration + @impl true + def handle(%{data: %{"type" => "Add"} = data} = object, meta) do + with %User{} = user <- User.get_cached_by_ap_id(data["actor"]), + {:ok, _user} <- User.add_pinned_object_id(user, data["object"]) do + # if pinned activity was scheduled for deletion, we remove job + if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(meta[:activity_id]) do + Oban.cancel_job(expiration.id) + end + + {:ok, object, meta} + else + nil -> + {:error, :user_not_found} + + {:error, changeset} -> + if changeset.errors[:pinned_objects] do + {:error, :pinned_statuses_limit_reached} + else + changeset.errors + end + end + end + + # Tasks this handles: + # - removes pin from user + # - if activity had expiration, recreates activity expiration job + @impl true + def handle(%{data: %{"type" => "Remove"} = data} = object, meta) do + with %User{} = user <- User.get_cached_by_ap_id(data["actor"]), + {:ok, _user} <- User.remove_pinned_object_id(user, data["object"]) do + # if pinned activity was scheduled for deletion, we reschedule it for deletion + if meta[:expires_at] do + # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation + {:ok, expires_at} = + Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast(meta[:expires_at]) + + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: meta[:activity_id], + expires_at: expires_at + }) + end + + {:ok, object, meta} + else + nil -> {:error, :user_not_found} + error -> error + end + end + # Nothing to do @impl true def handle(object, meta) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8c7d6a747..270cea6dc 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -556,6 +556,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end + def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do + with :ok <- ObjectValidator.fetch_actor_and_object(data), + %Object{} <- Object.normalize(data["object"], fetch: true), + {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do + {:ok, activity} + end + end + def handle_incoming( %{"type" => "Delete"} = data, _options @@ -1000,6 +1008,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id), {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), {:ok, user} <- update_user(user, data) do + {:ok, _pid} = Task.start(fn -> ActivityPub.pinned_fetch_task(user) end) TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) {:ok, user} else diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 8adc9878a..462f3b4a7 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -6,8 +6,10 @@ defmodule Pleroma.Web.ActivityPub.UserView do use Pleroma.Web, :view alias Pleroma.Keys + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Endpoint @@ -97,6 +99,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "followers" => "#{user.ap_id}/followers", "inbox" => "#{user.ap_id}/inbox", "outbox" => "#{user.ap_id}/outbox", + "featured" => "#{user.ap_id}/collections/featured", "preferredUsername" => user.nickname, "name" => user.name, "summary" => user.bio, @@ -245,6 +248,24 @@ defmodule Pleroma.Web.ActivityPub.UserView do |> Map.merge(pagination) end + def render("featured.json", %{ + user: %{featured_address: featured_address, pinned_objects: pinned_objects} + }) do + objects = + pinned_objects + |> Enum.sort_by(fn {_, pinned_at} -> pinned_at end, &>=/2) + |> Enum.map(fn {id, _} -> + ObjectView.render("object.json", %{object: Object.get_cached_by_ap_id(id)}) + end) + + %{ + "id" => featured_address, + "type" => "OrderedCollection", + "orderedItems" => objects + } + |> Map.merge(Utils.make_json_ld_header()) + end + defp maybe_put_total_items(map, false, _total), do: map defp maybe_put_total_items(map, true, total) do diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 4bdb8e281..802fbef3e 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -182,7 +182,34 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do parameters: [id_param()], responses: %{ 200 => status_response(), - 400 => Operation.response("Error", "application/json", ApiError) + 400 => + Operation.response("Bad Request", "application/json", %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "You have already pinned the maximum number of statuses" + } + }), + 404 => + Operation.response("Not found", "application/json", %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "Record not found" + } + }), + 422 => + Operation.response( + "Unprocessable Entity", + "application/json", + %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "Someone else's status cannot be pinned" + } + } + ) } } end @@ -197,7 +224,22 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do parameters: [id_param()], responses: %{ 200 => status_response(), - 400 => Operation.response("Error", "application/json", ApiError) + 400 => + Operation.response("Bad Request", "application/json", %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "You have already pinned the maximum number of statuses" + } + }), + 404 => + Operation.response("Not found", "application/json", %Schema{ + allOf: [ApiError], + title: "Unprocessable Entity", + example: %{ + "error" => "Record not found" + } + }) } } end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 42fa98718..3d042dc19 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -194,6 +194,13 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do parent_visible: %Schema{ type: :boolean, description: "`true` if the parent post is visible to the user" + }, + pinned_at: %Schema{ + type: :string, + format: "date-time", + nullable: true, + description: + "A datetime (ISO 8601) that states when the post was pinned or `null` if the post is not pinned" } } }, diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index b003e30c7..d35a0f219 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -411,29 +411,54 @@ defmodule Pleroma.Web.CommonAPI do end end - def pin(id, %{ap_id: user_ap_id} = user) do - with %Activity{ - actor: ^user_ap_id, - data: %{"type" => "Create"}, - object: %Object{data: %{"type" => object_type}} - } = activity <- Activity.get_by_id_with_object(id), - true <- object_type in ["Note", "Article", "Question"], - true <- Visibility.is_public?(activity), - {:ok, _user} <- User.add_pinnned_activity(user, activity) do + @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} + def pin(id, %User{ap_id: actor} = user) do + with %Activity{} = activity <- create_activity_by_id(id), + true <- activity_belongs_to_actor(activity, actor), + true <- object_type_is_allowed_for_pin(activity.object), + true <- activity_is_public(activity), + {:ok, pin_data, _} <- Builder.pin(user, activity.object), + {:ok, _pin, _} <- + Pipeline.common_pipeline(pin_data, local: true, activity_id: id) do {:ok, activity} else - {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err} - _ -> {:error, dgettext("errors", "Could not pin")} + {:error, {:execute_side_effects, error}} -> error + error -> error end end + defp create_activity_by_id(id) do + with nil <- Activity.create_by_id_with_object(id) do + {:error, :not_found} + end + end + + defp activity_belongs_to_actor(%{actor: actor}, actor), do: true + defp activity_belongs_to_actor(_, _), do: {:error, :ownership_error} + + defp object_type_is_allowed_for_pin(%{data: %{"type" => type}}) do + with false <- type in ["Note", "Article", "Question"] do + {:error, :not_allowed} + end + end + + defp activity_is_public(activity) do + with false <- Visibility.is_public?(activity) do + {:error, :visibility_error} + end + end + + @spec unpin(String.t(), User.t()) :: {:ok, User.t()} | {:error, term()} def unpin(id, user) do - with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id), - {:ok, _user} <- User.remove_pinnned_activity(user, activity) do + with %Activity{} = activity <- create_activity_by_id(id), + {:ok, unpin_data, _} <- Builder.unpin(user, activity.object), + {:ok, _unpin, _} <- + Pipeline.common_pipeline(unpin_data, + local: true, + activity_id: activity.id, + expires_at: activity.data["expires_at"] + ) do {:ok, activity} - else - {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err} - _ -> {:error, dgettext("errors", "Could not unpin")} end end diff --git a/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex b/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex index d25f84837..84621500e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex @@ -30,6 +30,12 @@ defmodule Pleroma.Web.MastodonAPI.FallbackController do |> json(%{error: error_message}) end + def call(conn, {:error, status, message}) do + conn + |> put_status(status) + |> json(%{error: message}) + end + def call(conn, _) do conn |> put_status(:internal_server_error) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index b051fca74..724dc5c5d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -260,6 +260,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do def pin(%{assigns: %{user: user}} = conn, %{id: ap_id_or_id}) do with {:ok, activity} <- CommonAPI.pin(ap_id_or_id, user) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) + else + {:error, :pinned_statuses_limit_reached} -> + {:error, "You have already pinned the maximum number of statuses"} + + {:error, :ownership_error} -> + {:error, :unprocessable_entity, "Someone else's status cannot be pinned"} + + {:error, :visibility_error} -> + {:error, :unprocessable_entity, "Non-public status cannot be pinned"} + + error -> + error end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3753588f2..d0247fa4a 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -152,6 +152,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Enum.filter(& &1) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) + {pinned?, pinned_at} = pin_data(activity_object, user) + %{ id: to_string(activity.id), uri: object.data["id"], @@ -173,7 +175,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do favourited: present?(favorited), bookmarked: present?(bookmarked), muted: false, - pinned: pinned?(activity, user), + pinned: pinned?, sensitive: false, spoiler_text: "", visibility: get_visibility(activity), @@ -184,7 +186,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do language: nil, emojis: [], pleroma: %{ - local: activity.local + local: activity.local, + pinned_at: pinned_at } } end @@ -316,6 +319,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do fn for_user, user -> User.mutes?(for_user, user) end ) + {pinned?, pinned_at} = pin_data(object, user) + %{ id: to_string(activity.id), uri: object.data["id"], @@ -339,7 +344,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do favourited: present?(favorited), bookmarked: present?(bookmarked), muted: muted, - pinned: pinned?(activity, user), + pinned: pinned?, sensitive: sensitive, spoiler_text: summary, visibility: get_visibility(object), @@ -360,7 +365,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do direct_conversation_id: direct_conversation_id, thread_muted: thread_muted?, emoji_reactions: emoji_reactions, - parent_visible: visible_for_user?(reply_to, opts[:for]) + parent_visible: visible_for_user?(reply_to, opts[:for]), + pinned_at: pinned_at } } end @@ -529,8 +535,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp present?(false), do: false defp present?(_), do: true - defp pinned?(%Activity{id: id}, %User{pinned_activities: pinned_activities}), - do: id in pinned_activities + defp pin_data(%Object{data: %{"id" => object_id}}, %User{pinned_objects: pinned_objects}) do + if pinned_at = pinned_objects[object_id] do + {true, Utils.to_masto_date(pinned_at)} + else + {false, nil} + end + end defp build_emoji_map(emoji, users, current_user) do %{ diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index de0bd27d7..ccf2ef796 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -704,6 +704,7 @@ defmodule Pleroma.Web.Router do # The following two are S2S as well, see `ActivityPub.fetch_follow_information_for_user/1`: get("/users/:nickname/followers", ActivityPubController, :followers) get("/users/:nickname/following", ActivityPubController, :following) + get("/users/:nickname/collections/featured", ActivityPubController, :pinned) end scope "/", Pleroma.Web.ActivityPub do diff --git a/priv/repo/migrations/20210202110641_add_pinned_objects_to_users.exs b/priv/repo/migrations/20210202110641_add_pinned_objects_to_users.exs new file mode 100644 index 000000000..644527246 --- /dev/null +++ b/priv/repo/migrations/20210202110641_add_pinned_objects_to_users.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.AddPinnedObjectsToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add(:pinned_objects, :map) + end + end +end diff --git a/priv/repo/migrations/20210203141144_add_featured_address_to_users.exs b/priv/repo/migrations/20210203141144_add_featured_address_to_users.exs new file mode 100644 index 000000000..0f6a21611 --- /dev/null +++ b/priv/repo/migrations/20210203141144_add_featured_address_to_users.exs @@ -0,0 +1,23 @@ +defmodule Pleroma.Repo.Migrations.AddFeaturedAddressToUsers do + use Ecto.Migration + + def up do + alter table(:users) do + add(:featured_address, :string) + end + + create(index(:users, [:featured_address])) + + execute(""" + + update users set featured_address = concat(ap_id, '/collections/featured') where local = true and featured_address is null; + + """) + end + + def down do + alter table(:users) do + remove(:featured_address) + end + end +end diff --git a/priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs b/priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs new file mode 100644 index 000000000..9aee545e3 --- /dev/null +++ b/priv/repo/migrations/20210205145000_move_pinned_activities_into_pinned_objects.exs @@ -0,0 +1,28 @@ +defmodule Pleroma.Repo.Migrations.MovePinnedActivitiesIntoPinnedObjects do + use Ecto.Migration + + import Ecto.Query + + alias Pleroma.Repo + alias Pleroma.User + + def up do + from(u in User) + |> select([u], {u.id, fragment("?.pinned_activities", u)}) + |> Repo.stream() + |> Stream.each(fn {user_id, pinned_activities_ids} -> + pinned_activities = Pleroma.Activity.all_by_ids_with_object(pinned_activities_ids) + + pins = + Map.new(pinned_activities, fn %{object: %{data: %{"id" => object_id}}} -> + {object_id, NaiveDateTime.utc_now()} + end) + + from(u in User, where: u.id == ^user_id) + |> Repo.update_all(set: [pinned_objects: pins]) + end) + |> Stream.run() + end + + def down, do: :noop +end diff --git a/priv/repo/migrations/20210206045221_remove_pinned_activities_from_users.exs b/priv/repo/migrations/20210206045221_remove_pinned_activities_from_users.exs new file mode 100644 index 000000000..a3ee93f48 --- /dev/null +++ b/priv/repo/migrations/20210206045221_remove_pinned_activities_from_users.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.RemovePinnedActivitiesFromUsers do + use Ecto.Migration + + def up do + alter table(:users) do + remove(:pinned_activities) + end + end + + def down do + alter table(:users) do + add(:pinned_activities, {:array, :string}, default: []) + end + end +end diff --git a/test/fixtures/collections/featured.json b/test/fixtures/collections/featured.json new file mode 100644 index 000000000..56f8f56fa --- /dev/null +++ b/test/fixtures/collections/featured.json @@ -0,0 +1,39 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://{{domain}}/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "id": "https://{{domain}}/users/{{nickname}}/collections/featured", + "orderedItems": [ + { + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://{{domain}}/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "actor": "https://{{domain}}/users/{{nickname}}", + "attachment": [], + "attributedTo": "https://{{domain}}/users/{{nickname}}", + "cc": [ + "https://{{domain}}/users/{{nickname}}/followers" + ], + "content": "", + "id": "https://{{domain}}/objects/{{object_id}}", + "published": "2021-02-12T15:13:43.915429Z", + "sensitive": false, + "source": "", + "summary": "", + "tag": [], + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Note" + } + ], + "type": "OrderedCollection" +} diff --git a/test/fixtures/masto_pin.json b/test/fixtures/masto_pin.json new file mode 100644 index 000000000..e57a34375 --- /dev/null +++ b/test/fixtures/masto_pin.json @@ -0,0 +1,41 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "Emoji": "toot:Emoji", + "Hashtag": "as:Hashtag", + "PropertyValue": "schema:PropertyValue", + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + }, + "atomUri": "ostatus:atomUri", + "conversation": "ostatus:conversation", + "featured": { + "@id": "toot:featured", + "@type": "@id" + }, + "focalPoint": { + "@container": "@list", + "@id": "toot:focalPoint" + }, + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "movedTo": { + "@id": "as:movedTo", + "@type": "@id" + }, + "ostatus": "http://ostatus.org#", + "schema": "http://schema.org#", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "value": "schema:value" + } + ], + "id": "https://example.com/users/nickname/statuses/{{id}}", + "actor": "https://example.com/users/nickname", + "object": "https://example.com/users/nickname/statuses/101355175004496751", + "target": "https://example.com/users/nickname/collections/featured", + "type": "{{type}}" +} diff --git a/test/fixtures/statuses/note.json b/test/fixtures/statuses/note.json new file mode 100644 index 000000000..41735cbc5 --- /dev/null +++ b/test/fixtures/statuses/note.json @@ -0,0 +1,27 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://example.com/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "actor": "https://example.com/users/{{nickname}}", + "attachment": [], + "attributedTo": "https://example.com/users/{{nickname}}", + "cc": [ + "https://example.com/users/{{nickname}}/followers" + ], + "content": "Content", + "context": "https://example.com/contexts/e4b180e1-7403-477f-aeb4-de57e7a3fe7f", + "conversation": "https://example.com/contexts/e4b180e1-7403-477f-aeb4-de57e7a3fe7f", + "id": "https://example.com/objects/{{object_id}}", + "published": "2019-12-15T22:00:05.279583Z", + "sensitive": false, + "summary": "", + "tag": [], + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Note" +} diff --git a/test/fixtures/users_mock/masto_featured.json b/test/fixtures/users_mock/masto_featured.json new file mode 100644 index 000000000..646a343ad --- /dev/null +++ b/test/fixtures/users_mock/masto_featured.json @@ -0,0 +1,18 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "votersCount": "toot:votersCount" + } + ], + "id": "https://{{domain}}/users/{{nickname}}/collections/featured", + "type": "OrderedCollection", + "totalItems": 0, + "orderedItems": [] +} diff --git a/test/fixtures/users_mock/user.json b/test/fixtures/users_mock/user.json new file mode 100644 index 000000000..da2483d02 --- /dev/null +++ b/test/fixtures/users_mock/user.json @@ -0,0 +1,41 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://example.com/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "attachment": [], + "endpoints": { + "oauthAuthorizationEndpoint": "https://example.com/oauth/authorize", + "oauthRegistrationEndpoint": "https://example.com/api/v1/apps", + "oauthTokenEndpoint": "https://example.com/oauth/token", + "sharedInbox": "https://example.com/inbox" + }, + "followers": "https://example.com/users/{{nickname}}/followers", + "following": "https://example.com/users/{{nickname}}/following", + "icon": { + "type": "Image", + "url": "https://example.com/media/4e914f5b84e4a259a3f6c2d2edc9ab642f2ab05f3e3d9c52c81fc2d984b3d51e.jpg" + }, + "id": "https://example.com/users/{{nickname}}", + "image": { + "type": "Image", + "url": "https://example.com/media/f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg?name=f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg" + }, + "inbox": "https://example.com/users/{{nickname}}/inbox", + "manuallyApprovesFollowers": false, + "name": "{{nickname}}", + "outbox": "https://example.com/users/{{nickname}}/outbox", + "preferredUsername": "{{nickname}}", + "publicKey": { + "id": "https://example.com/users/{{nickname}}#main-key", + "owner": "https://example.com/users/{{nickname}}", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n" + }, + "summary": "your friendly neighborhood pleroma developer
    I like cute things and distributed systems, and really hate delete and redrafts", + "tag": [], + "type": "Person", + "url": "https://example.com/users/{{nickname}}" +} diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 6f5bcab57..d81c1b8eb 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2338,4 +2338,49 @@ defmodule Pleroma.UserTest do assert User.active_user_count(6) == 3 assert User.active_user_count(1) == 1 end + + describe "pins" do + setup do + user = insert(:user) + + [user: user, object_id: object_id_from_created_activity(user)] + end + + test "unique pins", %{user: user, object_id: object_id} do + assert {:ok, %{pinned_objects: %{^object_id => pinned_at1} = pins} = updated_user} = + User.add_pinned_object_id(user, object_id) + + assert Enum.count(pins) == 1 + + assert {:ok, %{pinned_objects: %{^object_id => pinned_at2} = pins}} = + User.add_pinned_object_id(updated_user, object_id) + + assert pinned_at1 == pinned_at2 + + assert Enum.count(pins) == 1 + end + + test "respects max_pinned_statuses limit", %{user: user, object_id: object_id} do + clear_config([:instance, :max_pinned_statuses], 1) + {:ok, updated} = User.add_pinned_object_id(user, object_id) + + object_id2 = object_id_from_created_activity(user) + + {:error, %{errors: errors}} = User.add_pinned_object_id(updated, object_id2) + assert Keyword.has_key?(errors, :pinned_objects) + end + + test "remove_pinned_object_id/2", %{user: user, object_id: object_id} do + assert {:ok, updated} = User.add_pinned_object_id(user, object_id) + + {:ok, after_remove} = User.remove_pinned_object_id(updated, object_id) + assert after_remove.pinned_objects == %{} + end + end + + defp object_id_from_created_activity(user) do + %{id: id} = insert(:note_activity, user: user) + %{object: %{data: %{"id" => object_id}}} = Activity.get_by_id_with_object(id) + object_id + end end diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 19e04d472..a9cbf90c3 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -636,6 +636,86 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/inbox", non_create_data) |> json_response(400) end + + test "accepts Add/Remove activities", %{conn: conn} do + object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" + + status = + File.read!("test/fixtures/statuses/note.json") + |> String.replace("{{nickname}}", "lain") + |> String.replace("{{object_id}}", object_id) + + object_url = "https://example.com/objects/#{object_id}" + + user = + File.read!("test/fixtures/users_mock/user.json") + |> String.replace("{{nickname}}", "lain") + + actor = "https://example.com/users/lain" + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^object_url + } -> + %Tesla.Env{ + status: 200, + body: status, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^actor + } -> + %Tesla.Env{ + status: 200, + body: user, + headers: [{"content-type", "application/activity+json"}] + } + end) + + data = %{ + "id" => "https://example.com/objects/d61d6733-e256-4fe1-ab13-1e369789423f", + "actor" => actor, + "object" => object_url, + "target" => "https://example.com/users/lain/collections/featured", + "type" => "Add", + "to" => [Pleroma.Constants.as_public()] + } + + assert "ok" == + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/inbox", data) + |> json_response(200) + + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + assert Activity.get_by_ap_id(data["id"]) + user = User.get_cached_by_ap_id(data["actor"]) + assert user.pinned_objects[data["object"]] + + data = %{ + "id" => "https://example.com/objects/d61d6733-e256-4fe1-ab13-1e369789423d", + "actor" => actor, + "object" => object_url, + "target" => "https://example.com/users/lain/collections/featured", + "type" => "Remove", + "to" => [Pleroma.Constants.as_public()] + } + + assert "ok" == + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/inbox", data) + |> json_response(200) + + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + user = refresh_record(user) + refute user.pinned_objects[data["object"]] + end end describe "/users/:nickname/inbox" do @@ -1772,4 +1852,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> json_response(403) end end + + test "pinned collection", %{conn: conn} do + clear_config([:instance, :max_pinned_statuses], 2) + user = insert(:user) + objects = insert_list(2, :note, user: user) + + Enum.reduce(objects, user, fn %{data: %{"id" => object_id}}, user -> + {:ok, updated} = User.add_pinned_object_id(user, object_id) + updated + end) + + %{nickname: nickname, featured_address: featured_address, pinned_objects: pinned_objects} = + refresh_record(user) + + %{"id" => ^featured_address, "orderedItems" => items} = + conn + |> get("/users/#{nickname}/collections/featured") + |> json_response(200) + + object_ids = Enum.map(items, & &1["id"]) + + assert Enum.all?(pinned_objects, fn {obj_id, _} -> + obj_id in object_ids + end) + end end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index c7fa452f7..081d00d45 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -235,6 +235,83 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do "url" => [%{"href" => "https://jk.nipponalba.scot/images/profile.jpg"}] } end + + test "fetches user featured collection" do + ap_id = "https://example.com/users/lain" + + featured_url = "https://example.com/users/lain/collections/featured" + + user_data = + "test/fixtures/users_mock/user.json" + |> File.read!() + |> String.replace("{{nickname}}", "lain") + |> Jason.decode!() + |> Map.put("featured", featured_url) + |> Jason.encode!() + + object_id = Ecto.UUID.generate() + + featured_data = + "test/fixtures/collections/featured.json" + |> File.read!() + |> String.replace("{{domain}}", "example.com") + |> String.replace("{{nickname}}", "lain") + |> String.replace("{{object_id}}", object_id) + + object_url = "https://example.com/objects/#{object_id}" + + object_data = + "test/fixtures/statuses/note.json" + |> File.read!() + |> String.replace("{{object_id}}", object_id) + |> String.replace("{{nickname}}", "lain") + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^ap_id + } -> + %Tesla.Env{ + status: 200, + body: user_data, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^featured_url + } -> + %Tesla.Env{ + status: 200, + body: featured_data, + headers: [{"content-type", "application/activity+json"}] + } + end) + + Tesla.Mock.mock_global(fn + %{ + method: :get, + url: ^object_url + } -> + %Tesla.Env{ + status: 200, + body: object_data, + headers: [{"content-type", "application/activity+json"}] + } + end) + + {:ok, user} = ActivityPub.make_user_from_ap_id(ap_id) + Process.sleep(50) + + assert user.featured_address == featured_url + assert Map.has_key?(user.pinned_objects, object_url) + + in_db = Pleroma.User.get_by_ap_id(ap_id) + assert in_db.featured_address == featured_url + assert Map.has_key?(user.pinned_objects, object_url) + + assert %{data: %{"id" => ^object_url}} = Object.get_by_ap_id(object_url) + end end test "it fetches the appropriate tag-restricted posts" do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 4c3fcb44a..28d7e1e3c 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase + require Pleroma.Constants + alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Tests.ObanHelpers @@ -106,6 +108,78 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert activity.data["target"] == new_user.ap_id assert activity.data["type"] == "Move" end + + test "it accepts Add/Remove activities" do + user = + "test/fixtures/users_mock/user.json" + |> File.read!() + |> String.replace("{{nickname}}", "lain") + + object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" + + object = + "test/fixtures/statuses/note.json" + |> File.read!() + |> String.replace("{{nickname}}", "lain") + |> String.replace("{{object_id}}", object_id) + + object_url = "https://example.com/objects/#{object_id}" + + actor = "https://example.com/users/lain" + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^actor + } -> + %Tesla.Env{ + status: 200, + body: user, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^object_url + } -> + %Tesla.Env{ + status: 200, + body: object, + headers: [{"content-type", "application/activity+json"}] + } + end) + + message = %{ + "id" => "https://example.com/objects/d61d6733-e256-4fe1-ab13-1e369789423f", + "actor" => actor, + "object" => object_url, + "target" => "https://example.com/users/lain/collections/featured", + "type" => "Add", + "to" => [Pleroma.Constants.as_public()], + "cc" => ["https://example.com/users/lain/followers"] + } + + assert {:ok, activity} = Transmogrifier.handle_incoming(message) + assert activity.data == message + user = User.get_cached_by_ap_id(actor) + assert user.pinned_objects[object_url] + + remove = %{ + "id" => "http://localhost:400/objects/d61d6733-e256-4fe1-ab13-1e369789423d", + "actor" => actor, + "object" => object_url, + "target" => "http://example.com/users/lain/collections/featured", + "type" => "Remove", + "to" => [Pleroma.Constants.as_public()], + "cc" => ["https://example.com/users/lain/followers"] + } + + assert {:ok, activity} = Transmogrifier.handle_incoming(remove) + assert activity.data == remove + + user = refresh_record(user) + refute user.pinned_objects[object_url] + end end describe "prepare outgoing" do diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 6619f8fc8..fa55c2832 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -827,13 +827,17 @@ defmodule Pleroma.Web.CommonAPITest do [user: user, activity: activity] end + test "activity not found error", %{user: user} do + assert {:error, :not_found} = CommonAPI.pin("id", user) + end + test "pin status", %{user: user, activity: activity} do assert {:ok, ^activity} = CommonAPI.pin(activity.id, user) - id = activity.id + %{data: %{"id" => object_id}} = Object.normalize(activity) user = refresh_record(user) - assert %User{pinned_activities: [^id]} = user + assert user.pinned_objects |> Map.keys() == [object_id] end test "pin poll", %{user: user} do @@ -845,10 +849,11 @@ defmodule Pleroma.Web.CommonAPITest do assert {:ok, ^activity} = CommonAPI.pin(activity.id, user) - id = activity.id + %{data: %{"id" => object_id}} = Object.normalize(activity) + user = refresh_record(user) - assert %User{pinned_activities: [^id]} = user + assert user.pinned_objects |> Map.keys() == [object_id] end test "unlisted statuses can be pinned", %{user: user} do @@ -859,7 +864,7 @@ defmodule Pleroma.Web.CommonAPITest do test "only self-authored can be pinned", %{activity: activity} do user = insert(:user) - assert {:error, "Could not pin"} = CommonAPI.pin(activity.id, user) + assert {:error, :ownership_error} = CommonAPI.pin(activity.id, user) end test "max pinned statuses", %{user: user, activity: activity_one} do @@ -869,8 +874,12 @@ defmodule Pleroma.Web.CommonAPITest do user = refresh_record(user) - assert {:error, "You have already pinned the maximum number of statuses"} = - CommonAPI.pin(activity_two.id, user) + assert {:error, :pinned_statuses_limit_reached} = CommonAPI.pin(activity_two.id, user) + end + + test "only public can be pinned", %{user: user} do + {:ok, activity} = CommonAPI.post(user, %{status: "private status", visibility: "private"}) + {:error, :visibility_error} = CommonAPI.pin(activity.id, user) end test "unpin status", %{user: user, activity: activity} do @@ -884,7 +893,7 @@ defmodule Pleroma.Web.CommonAPITest do user = refresh_record(user) - assert %User{pinned_activities: []} = user + assert user.pinned_objects == %{} end test "should unpin when deleting a status", %{user: user, activity: activity} do @@ -896,7 +905,40 @@ defmodule Pleroma.Web.CommonAPITest do user = refresh_record(user) - assert %User{pinned_activities: []} = user + assert user.pinned_objects == %{} + end + + test "ephemeral activity won't be deleted if was pinned", %{user: user} do + {:ok, activity} = CommonAPI.post(user, %{status: "Hello!", expires_in: 601}) + + assert Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) + + {:ok, _activity} = CommonAPI.pin(activity.id, user) + refute Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) + + user = refresh_record(user) + {:ok, _} = CommonAPI.unpin(activity.id, user) + + # recreates expiration job on unpin + assert Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) + end + + test "ephemeral activity deletion job won't be deleted on pinning error", %{ + user: user, + activity: activity + } do + clear_config([:instance, :max_pinned_statuses], 1) + + {:ok, _activity} = CommonAPI.pin(activity.id, user) + + {:ok, activity2} = CommonAPI.post(user, %{status: "another status", expires_in: 601}) + + assert Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity2.id) + + user = refresh_record(user) + {:error, :pinned_statuses_limit_reached} = CommonAPI.pin(activity2.id, user) + + assert Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity2.id) end end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f616f405e..e0d642910 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1223,6 +1223,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> json_response_and_validate_schema(200) end + test "non authenticated user", %{activity: activity} do + assert build_conn() + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses/#{activity.id}/pin") + |> json_response(403) == %{"error" => "Invalid credentials."} + end + test "/pin: returns 400 error when activity is not public", %{conn: conn, user: user} do {:ok, dm} = CommonAPI.post(user, %{status: "test", visibility: "direct"}) @@ -1231,7 +1238,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> put_req_header("content-type", "application/json") |> post("/api/v1/statuses/#{dm.id}/pin") - assert json_response_and_validate_schema(conn, 400) == %{"error" => "Could not pin"} + assert json_response_and_validate_schema(conn, 422) == %{ + "error" => "Non-public status cannot be pinned" + } + end + + test "pin by another user", %{activity: activity} do + %{conn: conn} = oauth_access(["write:accounts"]) + + assert conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses/#{activity.id}/pin") + |> json_response(422) == %{"error" => "Someone else's status cannot be pinned"} end test "unpin status", %{conn: conn, user: user, activity: activity} do @@ -1252,13 +1270,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do |> json_response_and_validate_schema(200) end - test "/unpin: returns 400 error when activity is not exist", %{conn: conn} do - conn = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/statuses/1/unpin") - - assert json_response_and_validate_schema(conn, 400) == %{"error" => "Could not unpin"} + test "/unpin: returns 404 error when activity doesn't exist", %{conn: conn} do + assert conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses/1/unpin") + |> json_response_and_validate_schema(404) == %{"error" => "Record not found"} end test "max pinned statuses", %{conn: conn, user: user, activity: activity_one} do diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 4172cc294..fbea25079 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -286,7 +286,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do direct_conversation_id: nil, thread_muted: false, emoji_reactions: [], - parent_visible: false + parent_visible: false, + pinned_at: nil } } diff --git a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs index f389c272b..fa3b29006 100644 --- a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs +++ b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs @@ -27,6 +27,16 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do body: File.read!("test/fixtures/tesla_mock/status.emelie.json") } + %{method: :get, url: "https://mastodon.social/users/emelie/collections/featured"} -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + File.read!("test/fixtures/users_mock/masto_featured.json") + |> String.replace("{{domain}}", "mastodon.social") + |> String.replace("{{nickname}}", "emelie") + } + %{method: :get, url: "https://mastodon.social/users/emelie"} -> %Tesla.Env{ status: 200, @@ -52,6 +62,16 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do headers: [{"content-type", "application/activity+json"}], body: File.read!("test/fixtures/tesla_mock/emelie.json") } + + %{method: :get, url: "https://mastodon.social/users/emelie/collections/featured"} -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + File.read!("test/fixtures/users_mock/masto_featured.json") + |> String.replace("{{domain}}", "mastodon.social") + |> String.replace("{{nickname}}", "emelie") + } end) response = @@ -70,6 +90,16 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do headers: [{"content-type", "application/activity+json"}], body: File.read!("test/fixtures/tesla_mock/emelie.json") } + + %{method: :get, url: "https://mastodon.social/users/emelie/collections/featured"} -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + File.read!("test/fixtures/users_mock/masto_featured.json") + |> String.replace("{{domain}}", "mastodon.social") + |> String.replace("{{nickname}}", "emelie") + } end) user = insert(:user) diff --git a/test/support/factory.ex b/test/support/factory.ex index af4fff45b..883cedf3c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -48,13 +48,15 @@ defmodule Pleroma.Factory do %{ ap_id: ap_id, follower_address: ap_id <> "/followers", - following_address: ap_id <> "/following" + following_address: ap_id <> "/following", + featured_address: ap_id <> "/collections/featured" } else %{ ap_id: User.ap_id(user), follower_address: User.ap_followers(user), - following_address: User.ap_following(user) + following_address: User.ap_following(user), + featured_address: User.ap_featured_collection(user) } end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index eb692fab5..9e9f1c86c 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -89,6 +89,18 @@ defmodule HttpRequestMock do }} end + def get("https://mastodon.sdf.org/users/rinpatch/collections/featured", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + File.read!("test/fixtures/users_mock/masto_featured.json") + |> String.replace("{{domain}}", "mastodon.sdf.org") + |> String.replace("{{nickname}}", "rinpatch"), + headers: [{"content-type", "application/activity+json"}] + }} + end + def get("https://patch.cx/objects/tesla_mock/poll_attachment", _, _, _) do {:ok, %Tesla.Env{ @@ -905,6 +917,17 @@ defmodule HttpRequestMock do }} end + def get("https://mastodon.social/users/lambadalambda/collections/featured", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + File.read!("test/fixtures/users_mock/masto_featured.json") + |> String.replace("{{domain}}", "mastodon.social") + |> String.replace("{{nickname}}", "lambadalambda") + }} + end + def get("https://apfed.club/channel/indio", _, _, _) do {:ok, %Tesla.Env{ From 17f28c0507e3c34ce75e63747eed9abb66713e6e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 25 Feb 2021 14:00:44 +0300 Subject: [PATCH 022/150] mastodon pins --- lib/pleroma/object/containment.ex | 8 +++ lib/pleroma/web/activity_pub/transmogrifier.ex | 17 +++-- test/fixtures/statuses/masto-note.json | 47 +++++++++++++ .../activity_pub/activity_pub_controller_test.exs | 78 ++++++++++++++++++++++ 4 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/statuses/masto-note.json diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index fb0398f92..040537acf 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -71,6 +71,14 @@ defmodule Pleroma.Object.Containment do compare_uris(id_uri, other_uri) end + # Mastodon pin activities don't have an id, so we check the object field, which will be pinned. + def contain_origin_from_id(id, %{"object" => object}) when is_binary(object) do + id_uri = URI.parse(id) + object_uri = URI.parse(object) + + compare_uris(id_uri, object_uri) + end + def contain_origin_from_id(_id, _data), do: :error def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}), diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 270cea6dc..b662f5379 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -557,10 +557,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do - with :ok <- ObjectValidator.fetch_actor_and_object(data), - %Object{} <- Object.normalize(data["object"], fetch: true), - {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do - {:ok, activity} + with {:ok, user} <- ObjectValidator.fetch_actor(data), + %Object{} <- Object.normalize(data["object"], fetch: true) do + # Mastodon sends pin/unpin objects without id, to, cc fields + data = + data + |> Map.put_new("id", Utils.generate_activity_id()) + |> Map.put_new("to", [Pleroma.Constants.as_public()]) + |> Map.put_new("cc", [user.follower_address]) + + case Pipeline.common_pipeline(data, local: false) do + {:ok, activity, _meta} -> {:ok, activity} + error -> error + end end end diff --git a/test/fixtures/statuses/masto-note.json b/test/fixtures/statuses/masto-note.json new file mode 100644 index 000000000..6b96de473 --- /dev/null +++ b/test/fixtures/statuses/masto-note.json @@ -0,0 +1,47 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "votersCount": "toot:votersCount" + } + ], + "id": "https://example.com/users/{{nickname}}/statuses/{{status_id}}", + "type": "Note", + "summary": null, + "inReplyTo": null, + "published": "2021-02-24T12:40:49Z", + "url": "https://example.com/@{{nickname}}/{{status_id}}", + "attributedTo": "https://example.com/users/{{nickname}}", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://example.com/users/{{nickname}}/followers" + ], + "sensitive": false, + "atomUri": "https://example.com/users/{{nickname}}/statuses/{{status_id}}", + "inReplyToAtomUri": null, + "conversation": "tag:example.com,2021-02-24:objectId=15:objectType=Conversation", + "content": "

    ", + "contentMap": { + "en": "

    " + }, + "attachment": [], + "tag": [], + "replies": { + "id": "https://example.com/users/{{nickname}}/statuses/{{status_id}}/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "next": "https://example.com/users/{{nickname}}/statuses/{{status_id}}/replies?only_other_accounts=true&page=true", + "partOf": "https://example.com/users/{{nickname}}/statuses/{{status_id}}/replies", + "items": [] + } + } +} diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index a9cbf90c3..d9fa25d94 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -716,6 +716,84 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do user = refresh_record(user) refute user.pinned_objects[data["object"]] end + + test "mastodon pin/unpin", %{conn: conn} do + status_id = "105786274556060421" + + status = + File.read!("test/fixtures/statuses/masto-note.json") + |> String.replace("{{nickname}}", "lain") + |> String.replace("{{status_id}}", status_id) + + status_url = "https://example.com/users/lain/statuses/#{status_id}" + + user = + File.read!("test/fixtures/users_mock/user.json") + |> String.replace("{{nickname}}", "lain") + + actor = "https://example.com/users/lain" + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^status_url + } -> + %Tesla.Env{ + status: 200, + body: status, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^actor + } -> + %Tesla.Env{ + status: 200, + body: user, + headers: [{"content-type", "application/activity+json"}] + } + end) + + data = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "actor" => actor, + "object" => status_url, + "target" => "https://example.com/users/lain/collections/featured", + "type" => "Add" + } + + assert "ok" == + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/inbox", data) + |> json_response(200) + + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + assert Activity.get_by_object_ap_id_with_object(data["object"]) + user = User.get_cached_by_ap_id(data["actor"]) + assert user.pinned_objects[data["object"]] + + data = %{ + "actor" => actor, + "object" => status_url, + "target" => "https://example.com/users/lain/collections/featured", + "type" => "Remove" + } + + assert "ok" == + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/inbox", data) + |> json_response(200) + + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + assert Activity.get_by_object_ap_id_with_object(data["object"]) + user = refresh_record(user) + refute user.pinned_objects[data["object"]] + end end describe "/users/:nickname/inbox" do From ff612750b1bae5223bca76b34a39e7d2bd05770c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 2 Mar 2021 17:24:06 +0300 Subject: [PATCH 023/150] validator renaming & add validation for target --- lib/pleroma/web/activity_pub/object_validator.ex | 4 ++-- .../{pin_validator.ex => add_remove_validator.ex} | 13 ++++++++++++- .../activity_pub/object_validators/common_validations.ex | 8 ++++++++ test/pleroma/web/activity_pub/transmogrifier_test.exs | 2 +- .../web/mastodon_api/controllers/status_controller_test.exs | 6 +++--- test/support/http_request_mock.ex | 3 ++- 6 files changed, 28 insertions(+), 8 deletions(-) rename lib/pleroma/web/activity_pub/object_validators/{pin_validator.ex => add_remove_validator.ex} (73%) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 11432ef38..14c3e8531 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Object.Containment alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator @@ -30,7 +31,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.EventValidator alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.PinValidator alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator @@ -238,7 +238,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do with {:ok, object} <- object - |> PinValidator.cast_and_validate() + |> AddRemoveValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} diff --git a/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex similarity index 73% rename from lib/pleroma/web/activity_pub/object_validators/pin_validator.ex rename to lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index dca8cba6f..73d1c03f0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/pin_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2021 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Web.ActivityPub.ObjectValidators.PinValidator do +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do use Ecto.Schema import Ecto.Changeset @@ -37,6 +37,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.PinValidator do |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) |> validate_inclusion(:type, ~w(Add Remove)) |> validate_actor_presence() + |> validate_collection_belongs_to_actor() |> validate_object_presence() end + + defp validate_collection_belongs_to_actor(changeset) do + validate_change(changeset, :target, fn :target, target -> + if String.starts_with?(target, changeset.changes[:actor]) do + [] + else + [target: "collection doesn't belong to actor"] + end + end) + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 093549a45..940430588 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do alias Pleroma.Object alias Pleroma.User + @spec validate_any_presence(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t() def validate_any_presence(cng, fields) do non_empty = fields @@ -29,6 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end end + @spec validate_actor_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t() def validate_actor_presence(cng, options \\ []) do field_name = Keyword.get(options, :field_name, :actor) @@ -47,6 +49,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end) end + @spec validate_object_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t() def validate_object_presence(cng, options \\ []) do field_name = Keyword.get(options, :field_name, :object) allowed_types = Keyword.get(options, :allowed_types, false) @@ -68,6 +71,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end) end + @spec validate_object_or_user_presence(Ecto.Changeset.t(), keyword()) :: Ecto.Changeset.t() def validate_object_or_user_presence(cng, options \\ []) do field_name = Keyword.get(options, :field_name, :object) options = Keyword.put(options, :field_name, field_name) @@ -83,6 +87,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do if actor_cng.valid?, do: actor_cng, else: object_cng end + @spec validate_host_match(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t() def validate_host_match(cng, fields \\ [:id, :actor]) do if same_domain?(cng, fields) do cng @@ -95,6 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end end + @spec validate_fields_match(Ecto.Changeset.t(), [atom()]) :: Ecto.Changeset.t() def validate_fields_match(cng, fields) do if map_unique?(cng, fields) do cng @@ -122,12 +128,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do end) end + @spec same_domain?(Ecto.Changeset.t(), [atom()]) :: boolean() def same_domain?(cng, fields \\ [:actor, :object]) do map_unique?(cng, fields, fn value -> URI.parse(value).host end) end # This figures out if a user is able to create, delete or modify something # based on the domain and superuser status + @spec validate_modification_rights(Ecto.Changeset.t()) :: Ecto.Changeset.t() def validate_modification_rights(cng) do actor = User.get_cached_by_ap_id(get_field(cng, :actor)) diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 28d7e1e3c..9bc27f89e 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -168,7 +168,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "id" => "http://localhost:400/objects/d61d6733-e256-4fe1-ab13-1e369789423d", "actor" => actor, "object" => object_url, - "target" => "http://example.com/users/lain/collections/featured", + "target" => "https://example.com/users/lain/collections/featured", "type" => "Remove", "to" => [Pleroma.Constants.as_public()], "cc" => ["https://example.com/users/lain/followers"] diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index e0d642910..99ad87d05 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1209,15 +1209,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do setup do: clear_config([:instance, :max_pinned_statuses], 1) test "pin status", %{conn: conn, user: user, activity: activity} do - id_str = to_string(activity.id) + id = activity.id - assert %{"id" => ^id_str, "pinned" => true} = + assert %{"id" => ^id, "pinned" => true} = conn |> put_req_header("content-type", "application/json") |> post("/api/v1/statuses/#{activity.id}/pin") |> json_response_and_validate_schema(200) - assert [%{"id" => ^id_str, "pinned" => true}] = + assert [%{"id" => ^id, "pinned" => true}] = conn |> get("/api/v1/accounts/#{user.id}/statuses?pinned=true") |> json_response_and_validate_schema(200) diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 9e9f1c86c..8807c2d14 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -924,7 +924,8 @@ defmodule HttpRequestMock do body: File.read!("test/fixtures/users_mock/masto_featured.json") |> String.replace("{{domain}}", "mastodon.social") - |> String.replace("{{nickname}}", "lambadalambda") + |> String.replace("{{nickname}}", "lambadalambda"), + headers: activitypub_object_headers() }} end From d1d2744ee3e6015064cf50ac5725bfe45b682466 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 3 Mar 2021 15:41:05 +0300 Subject: [PATCH 024/150] featured_address valition in AddRemoveValidator --- lib/pleroma/web/activity_pub/object_validator.ex | 2 +- .../object_validators/add_remove_validator.ex | 12 ++++++------ lib/pleroma/web/activity_pub/transmogrifier.ex | 7 +++++-- lib/pleroma/web/common_api.ex | 13 +++++++++---- test/fixtures/users_mock/user.json | 1 + .../activity_pub/activity_pub_controller_test.exs | 22 ++++++++++++++++++++++ .../web/activity_pub/transmogrifier_test.exs | 11 +++++++++++ 7 files changed, 55 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 14c3e8531..3ca9136aa 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -238,7 +238,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do with {:ok, object} <- object - |> AddRemoveValidator.cast_and_validate() + |> AddRemoveValidator.cast_and_validate(meta) |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index 73d1c03f0..885282f32 100644 --- a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -22,28 +22,28 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do field(:cc, ObjectValidators.Recipients, default: []) end - def cast_and_validate(data) do + def cast_and_validate(data, meta) do data |> cast_data() - |> validate_data() + |> validate_data(meta) end defp cast_data(data) do cast(%__MODULE__{}, data, __schema__(:fields)) end - defp validate_data(changeset) do + defp validate_data(changeset, meta) do changeset |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) |> validate_inclusion(:type, ~w(Add Remove)) |> validate_actor_presence() - |> validate_collection_belongs_to_actor() + |> validate_collection_belongs_to_actor(meta) |> validate_object_presence() end - defp validate_collection_belongs_to_actor(changeset) do + defp validate_collection_belongs_to_actor(changeset, meta) do validate_change(changeset, :target, fn :target, target -> - if String.starts_with?(target, changeset.changes[:actor]) do + if target == meta[:featured_address] do [] else [target: "collection doesn't belong to actor"] diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b662f5379..fa62e0db2 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -557,7 +557,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do - with {:ok, user} <- ObjectValidator.fetch_actor(data), + with {:ok, %User{} = user} <- ObjectValidator.fetch_actor(data), %Object{} <- Object.normalize(data["object"], fetch: true) do # Mastodon sends pin/unpin objects without id, to, cc fields data = @@ -566,7 +566,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put_new("to", [Pleroma.Constants.as_public()]) |> Map.put_new("cc", [user.follower_address]) - case Pipeline.common_pipeline(data, local: false) do + case Pipeline.common_pipeline(data, + local: false, + featured_address: user.featured_address + ) do {:ok, activity, _meta} -> {:ok, activity} error -> error end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index d35a0f219..175d690cc 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -412,14 +412,18 @@ defmodule Pleroma.Web.CommonAPI do end @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} - def pin(id, %User{ap_id: actor} = user) do + def pin(id, %User{} = user) do with %Activity{} = activity <- create_activity_by_id(id), - true <- activity_belongs_to_actor(activity, actor), + true <- activity_belongs_to_actor(activity, user.ap_id), true <- object_type_is_allowed_for_pin(activity.object), true <- activity_is_public(activity), {:ok, pin_data, _} <- Builder.pin(user, activity.object), {:ok, _pin, _} <- - Pipeline.common_pipeline(pin_data, local: true, activity_id: id) do + Pipeline.common_pipeline(pin_data, + local: true, + activity_id: id, + featured_address: user.featured_address + ) do {:ok, activity} else {:error, {:execute_side_effects, error}} -> error @@ -456,7 +460,8 @@ defmodule Pleroma.Web.CommonAPI do Pipeline.common_pipeline(unpin_data, local: true, activity_id: activity.id, - expires_at: activity.data["expires_at"] + expires_at: activity.data["expires_at"], + featured_address: user.featured_address ) do {:ok, activity} end diff --git a/test/fixtures/users_mock/user.json b/test/fixtures/users_mock/user.json index da2483d02..c722a1145 100644 --- a/test/fixtures/users_mock/user.json +++ b/test/fixtures/users_mock/user.json @@ -34,6 +34,7 @@ "owner": "https://example.com/users/{{nickname}}", "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n" }, + "featured": "https://example.com/users/{{nickname}}/collections/featured", "summary": "your friendly neighborhood pleroma developer
    I like cute things and distributed systems, and really hate delete and redrafts", "tag": [], "type": "Person", diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index d9fa25d94..cea4b3a97 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -673,6 +673,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do body: user, headers: [{"content-type", "application/activity+json"}] } + + %{method: :get, url: "https://example.com/users/lain/collections/featured"} -> + %Tesla.Env{ + status: 200, + body: + "test/fixtures/users_mock/masto_featured.json" + |> File.read!() + |> String.replace("{{domain}}", "example.com") + |> String.replace("{{nickname}}", "lain"), + headers: [{"content-type", "application/activity+json"}] + } end) data = %{ @@ -753,6 +764,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do body: user, headers: [{"content-type", "application/activity+json"}] } + + %{method: :get, url: "https://example.com/users/lain/collections/featured"} -> + %Tesla.Env{ + status: 200, + body: + "test/fixtures/users_mock/masto_featured.json" + |> File.read!() + |> String.replace("{{domain}}", "example.com") + |> String.replace("{{nickname}}", "lain"), + headers: [{"content-type", "application/activity+json"}] + } end) data = %{ diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 9bc27f89e..fb8284aaf 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -147,6 +147,17 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do body: object, headers: [{"content-type", "application/activity+json"}] } + + %{method: :get, url: "https://example.com/users/lain/collections/featured"} -> + %Tesla.Env{ + status: 200, + body: + "test/fixtures/users_mock/masto_featured.json" + |> File.read!() + |> String.replace("{{domain}}", "example.com") + |> String.replace("{{nickname}}", "lain"), + headers: [{"content-type", "application/activity+json"}] + } end) message = %{ From 3adb43cc20751540ea590645b31b985807684202 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 3 Mar 2021 18:04:06 +0300 Subject: [PATCH 025/150] refetch user on incoming add/remove activity if featured_address is nil --- lib/pleroma/web/activity_pub/transmogrifier.ex | 8 +++ lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- .../web/activity_pub/transmogrifier_test.exs | 78 ++++++++++++++++++++++ test/support/factory.ex | 4 +- 4 files changed, 90 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index fa62e0db2..c4b11a655 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -558,6 +558,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do with {:ok, %User{} = user} <- ObjectValidator.fetch_actor(data), + # maybe locally user doesn't have featured_address + {:ok, user} <- maybe_refetch_user(user), %Object{} <- Object.normalize(data["object"], fetch: true) do # Mastodon sends pin/unpin objects without id, to, cc fields data = @@ -669,6 +671,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(_, _), do: :error + defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do + {:ok, user} + end + + defp maybe_refetch_user(%User{ap_id: ap_id}), do: upgrade_user_from_ap_id(ap_id) + @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil def get_obj_helper(id, options \\ []) do options = Keyword.put(options, :fetch, true) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d0247fa4a..814b3d142 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -152,7 +152,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Enum.filter(& &1) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) - {pinned?, pinned_at} = pin_data(activity_object, user) + {pinned?, pinned_at} = pin_data(object, user) %{ id: to_string(activity.id), diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index fb8284aaf..07ed3920f 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -191,6 +191,84 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do user = refresh_record(user) refute user.pinned_objects[object_url] end + + test "Add/Remove activities for remote users without featured address" do + user = insert(:user, local: false, domain: "example.com") + + user = + user + |> Ecto.Changeset.change(featured_address: nil) + |> Repo.update!() + + %{host: host} = URI.parse(user.ap_id) + + user_data = + "test/fixtures/users_mock/user.json" + |> File.read!() + |> String.replace("{{nickname}}", user.nickname) + + object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" + + object = + "test/fixtures/statuses/note.json" + |> File.read!() + |> String.replace("{{nickname}}", user.nickname) + |> String.replace("{{object_id}}", object_id) + + object_url = "https://#{host}/objects/#{object_id}" + + actor = "https://#{host}/users/#{user.nickname}" + + featured = "https://#{host}/users/#{user.nickname}/collections/featured" + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^actor + } -> + %Tesla.Env{ + status: 200, + body: user_data, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^object_url + } -> + %Tesla.Env{ + status: 200, + body: object, + headers: [{"content-type", "application/activity+json"}] + } + + %{method: :get, url: ^featured} -> + %Tesla.Env{ + status: 200, + body: + "test/fixtures/users_mock/masto_featured.json" + |> File.read!() + |> String.replace("{{domain}}", "#{host}") + |> String.replace("{{nickname}}", user.nickname), + headers: [{"content-type", "application/activity+json"}] + } + end) + + message = %{ + "id" => "https://#{host}/objects/d61d6733-e256-4fe1-ab13-1e369789423f", + "actor" => actor, + "object" => object_url, + "target" => "https://#{host}/users/#{user.nickname}/collections/featured", + "type" => "Add", + "to" => [Pleroma.Constants.as_public()], + "cc" => ["https://#{host}/users/#{user.nickname}/followers"] + } + + assert {:ok, activity} = Transmogrifier.handle_incoming(message) + assert activity.data == message + user = User.get_cached_by_ap_id(actor) + assert user.pinned_objects[object_url] + end end describe "prepare outgoing" do diff --git a/test/support/factory.ex b/test/support/factory.ex index 883cedf3c..867076d6a 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -41,7 +41,7 @@ defmodule Pleroma.Factory do urls = if attrs[:local] == false do - base_domain = Enum.random(["domain1.com", "domain2.com", "domain3.com"]) + base_domain = attrs[:domain] || Enum.random(["domain1.com", "domain2.com", "domain3.com"]) ap_id = "https://#{base_domain}/users/#{user.nickname}" @@ -60,6 +60,8 @@ defmodule Pleroma.Factory do } end + attrs = Map.delete(attrs, :domain) + user |> Map.put(:raw_bio, user.bio) |> Map.merge(urls) From 16c96966e9f7a039a969c06bdd6c4e18ab8d432c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 9 Mar 2021 08:59:50 +0300 Subject: [PATCH 026/150] not needed --- test/fixtures/masto_pin.json | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 test/fixtures/masto_pin.json diff --git a/test/fixtures/masto_pin.json b/test/fixtures/masto_pin.json deleted file mode 100644 index e57a34375..000000000 --- a/test/fixtures/masto_pin.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - { - "Emoji": "toot:Emoji", - "Hashtag": "as:Hashtag", - "PropertyValue": "schema:PropertyValue", - "alsoKnownAs": { - "@id": "as:alsoKnownAs", - "@type": "@id" - }, - "atomUri": "ostatus:atomUri", - "conversation": "ostatus:conversation", - "featured": { - "@id": "toot:featured", - "@type": "@id" - }, - "focalPoint": { - "@container": "@list", - "@id": "toot:focalPoint" - }, - "inReplyToAtomUri": "ostatus:inReplyToAtomUri", - "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", - "movedTo": { - "@id": "as:movedTo", - "@type": "@id" - }, - "ostatus": "http://ostatus.org#", - "schema": "http://schema.org#", - "sensitive": "as:sensitive", - "toot": "http://joinmastodon.org/ns#", - "value": "schema:value" - } - ], - "id": "https://example.com/users/nickname/statuses/{{id}}", - "actor": "https://example.com/users/nickname", - "object": "https://example.com/users/nickname/statuses/101355175004496751", - "target": "https://example.com/users/nickname/collections/featured", - "type": "{{type}}" -} From 8f0778166c2e7c76975d14937ef61c05d399b560 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 9 Mar 2021 09:00:20 +0300 Subject: [PATCH 027/150] moving fixture into mastodon folder --- test/fixtures/{ => mastodon}/collections/featured.json | 0 test/pleroma/web/activity_pub/activity_pub_test.exs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename test/fixtures/{ => mastodon}/collections/featured.json (100%) diff --git a/test/fixtures/collections/featured.json b/test/fixtures/mastodon/collections/featured.json similarity index 100% rename from test/fixtures/collections/featured.json rename to test/fixtures/mastodon/collections/featured.json diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 081d00d45..64e12066e 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -252,7 +252,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do object_id = Ecto.UUID.generate() featured_data = - "test/fixtures/collections/featured.json" + "test/fixtures/mastodon/collections/featured.json" |> File.read!() |> String.replace("{{domain}}", "example.com") |> String.replace("{{nickname}}", "lain") From 5ae9b05600dd3dffc628ba25fe01b271f7bc0122 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 9 Mar 2021 09:00:44 +0300 Subject: [PATCH 028/150] separate test file for featured collection --- .../transmogrifier/add_remove_handling_test.exs | 172 +++++++++++++++++++++ .../web/activity_pub/transmogrifier_test.exs | 163 ------------------- 2 files changed, 172 insertions(+), 163 deletions(-) create mode 100644 test/pleroma/web/activity_pub/transmogrifier/add_remove_handling_test.exs diff --git a/test/pleroma/web/activity_pub/transmogrifier/add_remove_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/add_remove_handling_test.exs new file mode 100644 index 000000000..fc7757125 --- /dev/null +++ b/test/pleroma/web/activity_pub/transmogrifier/add_remove_handling_test.exs @@ -0,0 +1,172 @@ +defmodule Pleroma.Web.ActivityPub.Transmogrifier.AddRemoveHandlingTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase, async: true + + require Pleroma.Constants + + import Pleroma.Factory + + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + + test "it accepts Add/Remove activities" do + user = + "test/fixtures/users_mock/user.json" + |> File.read!() + |> String.replace("{{nickname}}", "lain") + + object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" + + object = + "test/fixtures/statuses/note.json" + |> File.read!() + |> String.replace("{{nickname}}", "lain") + |> String.replace("{{object_id}}", object_id) + + object_url = "https://example.com/objects/#{object_id}" + + actor = "https://example.com/users/lain" + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^actor + } -> + %Tesla.Env{ + status: 200, + body: user, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^object_url + } -> + %Tesla.Env{ + status: 200, + body: object, + headers: [{"content-type", "application/activity+json"}] + } + + %{method: :get, url: "https://example.com/users/lain/collections/featured"} -> + %Tesla.Env{ + status: 200, + body: + "test/fixtures/users_mock/masto_featured.json" + |> File.read!() + |> String.replace("{{domain}}", "example.com") + |> String.replace("{{nickname}}", "lain"), + headers: [{"content-type", "application/activity+json"}] + } + end) + + message = %{ + "id" => "https://example.com/objects/d61d6733-e256-4fe1-ab13-1e369789423f", + "actor" => actor, + "object" => object_url, + "target" => "https://example.com/users/lain/collections/featured", + "type" => "Add", + "to" => [Pleroma.Constants.as_public()], + "cc" => ["https://example.com/users/lain/followers"] + } + + assert {:ok, activity} = Transmogrifier.handle_incoming(message) + assert activity.data == message + user = User.get_cached_by_ap_id(actor) + assert user.pinned_objects[object_url] + + remove = %{ + "id" => "http://localhost:400/objects/d61d6733-e256-4fe1-ab13-1e369789423d", + "actor" => actor, + "object" => object_url, + "target" => "https://example.com/users/lain/collections/featured", + "type" => "Remove", + "to" => [Pleroma.Constants.as_public()], + "cc" => ["https://example.com/users/lain/followers"] + } + + assert {:ok, activity} = Transmogrifier.handle_incoming(remove) + assert activity.data == remove + + user = refresh_record(user) + refute user.pinned_objects[object_url] + end + + test "Add/Remove activities for remote users without featured address" do + user = insert(:user, local: false, domain: "example.com") + + user = + user + |> Ecto.Changeset.change(featured_address: nil) + |> Repo.update!() + + %{host: host} = URI.parse(user.ap_id) + + user_data = + "test/fixtures/users_mock/user.json" + |> File.read!() + |> String.replace("{{nickname}}", user.nickname) + + object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" + + object = + "test/fixtures/statuses/note.json" + |> File.read!() + |> String.replace("{{nickname}}", user.nickname) + |> String.replace("{{object_id}}", object_id) + + object_url = "https://#{host}/objects/#{object_id}" + + actor = "https://#{host}/users/#{user.nickname}" + + featured = "https://#{host}/users/#{user.nickname}/collections/featured" + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^actor + } -> + %Tesla.Env{ + status: 200, + body: user_data, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^object_url + } -> + %Tesla.Env{ + status: 200, + body: object, + headers: [{"content-type", "application/activity+json"}] + } + + %{method: :get, url: ^featured} -> + %Tesla.Env{ + status: 200, + body: + "test/fixtures/users_mock/masto_featured.json" + |> File.read!() + |> String.replace("{{domain}}", "#{host}") + |> String.replace("{{nickname}}", user.nickname), + headers: [{"content-type", "application/activity+json"}] + } + end) + + message = %{ + "id" => "https://#{host}/objects/d61d6733-e256-4fe1-ab13-1e369789423f", + "actor" => actor, + "object" => object_url, + "target" => "https://#{host}/users/#{user.nickname}/collections/featured", + "type" => "Add", + "to" => [Pleroma.Constants.as_public()], + "cc" => ["https://#{host}/users/#{user.nickname}/followers"] + } + + assert {:ok, activity} = Transmogrifier.handle_incoming(message) + assert activity.data == message + user = User.get_cached_by_ap_id(actor) + assert user.pinned_objects[object_url] + end +end diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 07ed3920f..4c3fcb44a 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -6,8 +6,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase - require Pleroma.Constants - alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Tests.ObanHelpers @@ -108,167 +106,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert activity.data["target"] == new_user.ap_id assert activity.data["type"] == "Move" end - - test "it accepts Add/Remove activities" do - user = - "test/fixtures/users_mock/user.json" - |> File.read!() - |> String.replace("{{nickname}}", "lain") - - object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" - - object = - "test/fixtures/statuses/note.json" - |> File.read!() - |> String.replace("{{nickname}}", "lain") - |> String.replace("{{object_id}}", object_id) - - object_url = "https://example.com/objects/#{object_id}" - - actor = "https://example.com/users/lain" - - Tesla.Mock.mock(fn - %{ - method: :get, - url: ^actor - } -> - %Tesla.Env{ - status: 200, - body: user, - headers: [{"content-type", "application/activity+json"}] - } - - %{ - method: :get, - url: ^object_url - } -> - %Tesla.Env{ - status: 200, - body: object, - headers: [{"content-type", "application/activity+json"}] - } - - %{method: :get, url: "https://example.com/users/lain/collections/featured"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/users_mock/masto_featured.json" - |> File.read!() - |> String.replace("{{domain}}", "example.com") - |> String.replace("{{nickname}}", "lain"), - headers: [{"content-type", "application/activity+json"}] - } - end) - - message = %{ - "id" => "https://example.com/objects/d61d6733-e256-4fe1-ab13-1e369789423f", - "actor" => actor, - "object" => object_url, - "target" => "https://example.com/users/lain/collections/featured", - "type" => "Add", - "to" => [Pleroma.Constants.as_public()], - "cc" => ["https://example.com/users/lain/followers"] - } - - assert {:ok, activity} = Transmogrifier.handle_incoming(message) - assert activity.data == message - user = User.get_cached_by_ap_id(actor) - assert user.pinned_objects[object_url] - - remove = %{ - "id" => "http://localhost:400/objects/d61d6733-e256-4fe1-ab13-1e369789423d", - "actor" => actor, - "object" => object_url, - "target" => "https://example.com/users/lain/collections/featured", - "type" => "Remove", - "to" => [Pleroma.Constants.as_public()], - "cc" => ["https://example.com/users/lain/followers"] - } - - assert {:ok, activity} = Transmogrifier.handle_incoming(remove) - assert activity.data == remove - - user = refresh_record(user) - refute user.pinned_objects[object_url] - end - - test "Add/Remove activities for remote users without featured address" do - user = insert(:user, local: false, domain: "example.com") - - user = - user - |> Ecto.Changeset.change(featured_address: nil) - |> Repo.update!() - - %{host: host} = URI.parse(user.ap_id) - - user_data = - "test/fixtures/users_mock/user.json" - |> File.read!() - |> String.replace("{{nickname}}", user.nickname) - - object_id = "c61d6733-e256-4fe1-ab13-1e369789423f" - - object = - "test/fixtures/statuses/note.json" - |> File.read!() - |> String.replace("{{nickname}}", user.nickname) - |> String.replace("{{object_id}}", object_id) - - object_url = "https://#{host}/objects/#{object_id}" - - actor = "https://#{host}/users/#{user.nickname}" - - featured = "https://#{host}/users/#{user.nickname}/collections/featured" - - Tesla.Mock.mock(fn - %{ - method: :get, - url: ^actor - } -> - %Tesla.Env{ - status: 200, - body: user_data, - headers: [{"content-type", "application/activity+json"}] - } - - %{ - method: :get, - url: ^object_url - } -> - %Tesla.Env{ - status: 200, - body: object, - headers: [{"content-type", "application/activity+json"}] - } - - %{method: :get, url: ^featured} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/users_mock/masto_featured.json" - |> File.read!() - |> String.replace("{{domain}}", "#{host}") - |> String.replace("{{nickname}}", user.nickname), - headers: [{"content-type", "application/activity+json"}] - } - end) - - message = %{ - "id" => "https://#{host}/objects/d61d6733-e256-4fe1-ab13-1e369789423f", - "actor" => actor, - "object" => object_url, - "target" => "https://#{host}/users/#{user.nickname}/collections/featured", - "type" => "Add", - "to" => [Pleroma.Constants.as_public()], - "cc" => ["https://#{host}/users/#{user.nickname}/followers"] - } - - assert {:ok, activity} = Transmogrifier.handle_incoming(message) - assert activity.data == message - user = User.get_cached_by_ap_id(actor) - assert user.pinned_objects[object_url] - end end describe "prepare outgoing" do From 8857242c952dcac0bc5363e1c80160efaf7a1638 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 9 Mar 2021 11:57:20 +0300 Subject: [PATCH 029/150] removeing corresponding add activity --- lib/pleroma/activity.ex | 9 ++++++ lib/pleroma/web/activity_pub/side_effects.ex | 5 ++++ test/pleroma/activity_test.exs | 22 +++++++++++++++ test/support/factory.ex | 42 ++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index a4cfca4c5..53beca5e6 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -391,4 +391,13 @@ defmodule Pleroma.Activity do end def get_by_object_ap_id_with_object(_), do: nil + + @spec add_by_params_query(String.t(), String.t(), String.t()) :: Ecto.Query.t() + def add_by_params_query(object_id, actor, target) do + object_id + |> Queries.by_object_id() + |> Queries.by_type("Add") + |> Queries.by_actor(actor) + |> where([a], fragment("?->>'target' = ?", a.data, ^target)) + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 9d22f9d3c..5fe143c2b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -340,11 +340,16 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # Tasks this handles: # - removes pin from user + # - removes corresponding Add activity # - if activity had expiration, recreates activity expiration job @impl true def handle(%{data: %{"type" => "Remove"} = data} = object, meta) do with %User{} = user <- User.get_cached_by_ap_id(data["actor"]), {:ok, _user} <- User.remove_pinned_object_id(user, data["object"]) do + data["object"] + |> Activity.add_by_params_query(user.ap_id, user.featured_address) + |> Repo.delete_all() + # if pinned activity was scheduled for deletion, we reschedule it for deletion if meta[:expires_at] do # MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation diff --git a/test/pleroma/activity_test.exs b/test/pleroma/activity_test.exs index 390a06344..962bc7e45 100644 --- a/test/pleroma/activity_test.exs +++ b/test/pleroma/activity_test.exs @@ -254,4 +254,26 @@ defmodule Pleroma.ActivityTest do assert %{id: ^id} = Activity.get_by_object_ap_id_with_object(obj_id) end + + test "add_by_params_query/3" do + user = insert(:user) + + note = insert(:note_activity, user: user) + + insert(:add_activity, user: user, note: note) + insert(:add_activity, user: user, note: note) + insert(:add_activity, user: user) + + assert Repo.aggregate(Activity, :count, :id) == 4 + + add_query = + Activity.add_by_params_query(note.data["object"], user.ap_id, user.featured_address) + + assert Repo.aggregate(add_query, :count, :id) == 2 + + Repo.delete_all(add_query) + assert Repo.aggregate(add_query, :count, :id) == 0 + + assert Repo.aggregate(Activity, :count, :id) == 2 + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index 867076d6a..5c4e65c81 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -4,6 +4,9 @@ defmodule Pleroma.Factory do use ExMachina.Ecto, repo: Pleroma.Repo + + require Pleroma.Constants + alias Pleroma.Object alias Pleroma.User @@ -225,6 +228,45 @@ defmodule Pleroma.Factory do } end + def add_activity_factory(attrs \\ %{}) do + featured_collection_activity(attrs, "Add") + end + + def remove_activity_factor(attrs \\ %{}) do + featured_collection_activity(attrs, "Remove") + end + + defp featured_collection_activity(attrs, type) do + user = attrs[:user] || insert(:user) + note = attrs[:note] || insert(:note, user: user) + + data_attrs = + attrs + |> Map.get(:data_attrs, %{}) + |> Map.put(:type, type) + + attrs = Map.drop(attrs, [:user, :note, :data_attrs]) + + data = + %{ + "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), + "target" => user.featured_address, + "object" => note.data["object"], + "actor" => note.data["actor"], + "type" => "Add", + "to" => [Pleroma.Constants.as_public()], + "cc" => [user.follower_address] + } + |> Map.merge(data_attrs) + + %Pleroma.Activity{ + data: data, + actor: data["actor"], + recipients: data["to"] + } + |> Map.merge(attrs) + end + def note_activity_factory(attrs \\ %{}) do user = attrs[:user] || insert(:user) note = attrs[:note] || insert(:note, user: user) From 2a520ba008f432e7e1fa297954966e0181245f01 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 19 Mar 2021 17:25:12 +0300 Subject: [PATCH 030/150] expanding AddRemoveValidator --- lib/pleroma/web/activity_pub/object_validator.ex | 2 +- .../object_validators/add_remove_validator.ex | 26 +++++++++++++++++----- lib/pleroma/web/activity_pub/transmogrifier.ex | 22 +++++------------- lib/pleroma/web/common_api.ex | 3 +-- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 3ca9136aa..14c3e8531 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -238,7 +238,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do with {:ok, object} <- object - |> AddRemoveValidator.cast_and_validate(meta) + |> AddRemoveValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index 885282f32..c38f86a0e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + require Pleroma.Constants + alias Pleroma.EctoType.ActivityPub.ObjectValidators @primary_key false @@ -22,28 +24,40 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do field(:cc, ObjectValidators.Recipients, default: []) end - def cast_and_validate(data, meta) do + def cast_and_validate(data) do data + |> maybe_fix_data_for_mastodon() |> cast_data() - |> validate_data(meta) + |> validate_data() + end + + defp maybe_fix_data_for_mastodon(data) do + {:ok, actor} = Pleroma.User.get_or_fetch_by_ap_id(data["actor"]) + # Mastodon sends pin/unpin objects without id, to, cc fields + data + |> Map.put_new("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id()) + |> Map.put_new("to", [Pleroma.Constants.as_public()]) + |> Map.put_new("cc", [actor.follower_address]) end defp cast_data(data) do cast(%__MODULE__{}, data, __schema__(:fields)) end - defp validate_data(changeset, meta) do + defp validate_data(changeset) do changeset |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) |> validate_inclusion(:type, ~w(Add Remove)) |> validate_actor_presence() - |> validate_collection_belongs_to_actor(meta) + |> validate_collection_belongs_to_actor() |> validate_object_presence() end - defp validate_collection_belongs_to_actor(changeset, meta) do + defp validate_collection_belongs_to_actor(changeset) do + {:ok, actor} = Pleroma.User.get_or_fetch_by_ap_id(changeset.changes[:actor]) + validate_change(changeset, :target, fn :target, target -> - if target == meta[:featured_address] do + if target == actor.featured_address do [] else [target: "collection doesn't belong to actor"] diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index c4b11a655..2172e7736 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -557,24 +557,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do - with {:ok, %User{} = user} <- ObjectValidator.fetch_actor(data), + with :ok <- ObjectValidator.fetch_actor_and_object(data), + {:ok, actor} <- Pleroma.User.get_or_fetch_by_ap_id(data["actor"]), # maybe locally user doesn't have featured_address - {:ok, user} <- maybe_refetch_user(user), - %Object{} <- Object.normalize(data["object"], fetch: true) do - # Mastodon sends pin/unpin objects without id, to, cc fields - data = - data - |> Map.put_new("id", Utils.generate_activity_id()) - |> Map.put_new("to", [Pleroma.Constants.as_public()]) - |> Map.put_new("cc", [user.follower_address]) - - case Pipeline.common_pipeline(data, - local: false, - featured_address: user.featured_address - ) do - {:ok, activity, _meta} -> {:ok, activity} - error -> error - end + {:ok, _} <- maybe_refetch_user(actor), + {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do + {:ok, activity} end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 175d690cc..b36be4d2a 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -421,8 +421,7 @@ defmodule Pleroma.Web.CommonAPI do {:ok, _pin, _} <- Pipeline.common_pipeline(pin_data, local: true, - activity_id: id, - featured_address: user.featured_address + activity_id: id ) do {:ok, activity} else From 1885268c9c242aca2a51bd15ed839bd65d6a52dc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 25 Mar 2021 13:26:54 +0300 Subject: [PATCH 031/150] expanding validator --- .../object_validators/add_remove_validator.ex | 28 +++++++++++++++------- lib/pleroma/web/activity_pub/transmogrifier.ex | 18 +------------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index c38f86a0e..f885aabe4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do require Pleroma.Constants alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.User @primary_key false @@ -25,14 +26,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do end def cast_and_validate(data) do + {:ok, actor} = User.get_or_fetch_by_ap_id(data["actor"]) + + {:ok, actor} = maybe_refetch_user(actor) + data - |> maybe_fix_data_for_mastodon() + |> maybe_fix_data_for_mastodon(actor) |> cast_data() - |> validate_data() + |> validate_data(actor) end - defp maybe_fix_data_for_mastodon(data) do - {:ok, actor} = Pleroma.User.get_or_fetch_by_ap_id(data["actor"]) + defp maybe_fix_data_for_mastodon(data, actor) do # Mastodon sends pin/unpin objects without id, to, cc fields data |> Map.put_new("id", Pleroma.Web.ActivityPub.Utils.generate_activity_id()) @@ -44,18 +48,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do cast(%__MODULE__{}, data, __schema__(:fields)) end - defp validate_data(changeset) do + defp validate_data(changeset, actor) do changeset |> validate_required([:id, :target, :object, :actor, :type, :to, :cc]) |> validate_inclusion(:type, ~w(Add Remove)) |> validate_actor_presence() - |> validate_collection_belongs_to_actor() + |> validate_collection_belongs_to_actor(actor) |> validate_object_presence() end - defp validate_collection_belongs_to_actor(changeset) do - {:ok, actor} = Pleroma.User.get_or_fetch_by_ap_id(changeset.changes[:actor]) - + defp validate_collection_belongs_to_actor(changeset, actor) do validate_change(changeset, :target, fn :target, target -> if target == actor.featured_address do [] @@ -64,4 +66,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do end end) end + + defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do + {:ok, user} + end + + defp maybe_refetch_user(%User{ap_id: ap_id}) do + Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id) + end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 2172e7736..c4caeff0a 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -534,7 +534,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) - when type in ~w{Like EmojiReact Announce} do + when type in ~w{Like EmojiReact Announce Add Remove} do with :ok <- ObjectValidator.fetch_actor_and_object(data), {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do @@ -556,16 +556,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming(%{"type" => type} = data, _options) when type in ~w(Add Remove) do - with :ok <- ObjectValidator.fetch_actor_and_object(data), - {:ok, actor} <- Pleroma.User.get_or_fetch_by_ap_id(data["actor"]), - # maybe locally user doesn't have featured_address - {:ok, _} <- maybe_refetch_user(actor), - {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do - {:ok, activity} - end - end - def handle_incoming( %{"type" => "Delete"} = data, _options @@ -659,12 +649,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(_, _), do: :error - defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(address) do - {:ok, user} - end - - defp maybe_refetch_user(%User{ap_id: ap_id}), do: upgrade_user_from_ap_id(ap_id) - @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil def get_obj_helper(id, options \\ []) do options = Keyword.put(options, :fetch, true) From 6e108b8603de45d489d4aef7e3e271bc5e8c431d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 26 Mar 2021 19:19:19 +0300 Subject: [PATCH 032/150] reading the file, instead of config keyword --- lib/pleroma/config/release_runtime_provider.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 8227195dc..70ef3bcc1 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do with_exported = if File.exists?(exported_config_path) do - exported_config = Config.Reader.read!(with_runtime_config) + exported_config = Config.Reader.read!(exported_config_path) Config.Reader.merge(with_runtime_config, exported_config) else with_runtime_config From 4d046afd2769cfdc16b2ee48e8c1d8f7f8e8ffa7 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 27 Mar 2021 09:05:33 +0300 Subject: [PATCH 033/150] tests for release config provider --- lib/pleroma/config/release_runtime_provider.ex | 17 ++++---- mix.exs | 13 +++++- .../config/temp.exported_from_db.secret.exs | 5 +++ .../config/release_runtime_provider_test.exs | 46 ++++++++++++++++++++++ 4 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 test/fixtures/config/temp.exported_from_db.secret.exs create mode 100644 test/pleroma/config/release_runtime_provider_test.exs diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 70ef3bcc1..46fa35559 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do @moduledoc """ - Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases. + Imports runtime config and `{env}.exported_from_db.secret.exs` for releases. """ @behaviour Config.Provider @@ -8,13 +8,13 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do def init(opts), do: opts @impl true - def load(config, _opts) do + def load(config, opts) do with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults()) - config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" + config_path = opts[:config_path] with_runtime_config = - if File.exists?(config_path) do + if config_path && File.exists?(config_path) do runtime_config = Config.Reader.read!(config_path) with_defaults @@ -24,7 +24,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do warning = [ IO.ANSI.red(), IO.ANSI.bright(), - "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file", + "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file", IO.ANSI.reset() ] @@ -32,13 +32,10 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do with_defaults end - exported_config_path = - config_path - |> Path.dirname() - |> Path.join("prod.exported_from_db.secret.exs") + exported_config_path = opts[:exported_config_path] with_exported = - if File.exists?(exported_config_path) do + if exported_config_path && File.exists?(exported_config_path) do exported_config = Config.Reader.read!(exported_config_path) Config.Reader.merge(with_runtime_config, exported_config) else diff --git a/mix.exs b/mix.exs index ae74f50a3..7328b533b 100644 --- a/mix.exs +++ b/mix.exs @@ -38,7 +38,7 @@ defmodule Pleroma.Mixfile do include_executables_for: [:unix], applications: [ex_syslogger: :load, syslog: :load, eldap: :transient], steps: [:assemble, &put_otp_version/1, ©_files/1, ©_nginx_config/1], - config_providers: [{Pleroma.Config.ReleaseRuntimeProvider, nil}] + config_providers: [{Pleroma.Config.ReleaseRuntimeProvider, release_config_paths()}] ] ] ] @@ -67,6 +67,17 @@ defmodule Pleroma.Mixfile do release end + defp release_config_paths do + config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" + + exported_config_path = + config_path + |> Path.dirname() + |> Path.join("#{Mix.env()}.exported_from_db.secret.exs") + + [config_path: config_path, exported_config_path: exported_config_path] + end + # Configuration for the OTP application. # # Type `mix help compile.app` for more information. diff --git a/test/fixtures/config/temp.exported_from_db.secret.exs b/test/fixtures/config/temp.exported_from_db.secret.exs new file mode 100644 index 000000000..64bee7f32 --- /dev/null +++ b/test/fixtures/config/temp.exported_from_db.secret.exs @@ -0,0 +1,5 @@ +use Mix.Config + +config :pleroma, exported_config_merged: true + +config :pleroma, :first_setting, key: "new value" diff --git a/test/pleroma/config/release_runtime_provider_test.exs b/test/pleroma/config/release_runtime_provider_test.exs new file mode 100644 index 000000000..1921698c5 --- /dev/null +++ b/test/pleroma/config/release_runtime_provider_test.exs @@ -0,0 +1,46 @@ +defmodule Pleroma.Config.ReleaseRuntimeProviderTest do + use ExUnit.Case, async: true + + alias Pleroma.Config.ReleaseRuntimeProvider + + describe "load/2" do + test "loads release defaults config and warns about non-existent runtime config" do + ExUnit.CaptureIO.capture_io(fn -> + merged = ReleaseRuntimeProvider.load([], []) + assert merged == Pleroma.Config.Holder.release_defaults() + IO.inspect(merged) + end) =~ + "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file" + end + + test "merged runtime config" do + merged = + ReleaseRuntimeProvider.load([], config_path: "test/fixtures/config/temp.secret.exs") + + assert merged[:pleroma][:first_setting] == [key: "value", key2: [Pleroma.Repo]] + assert merged[:pleroma][:second_setting] == [key: "value2", key2: ["Activity"]] + end + + test "merged exported config" do + ExUnit.CaptureIO.capture_io(fn -> + merged = + ReleaseRuntimeProvider.load([], + exported_config_path: "test/fixtures/config/temp.exported_from_db.secret.exs" + ) + + assert merged[:pleroma][:exported_config_merged] + end) =~ + "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file" + end + + test "runtime config is merged with exported config" do + merged = + ReleaseRuntimeProvider.load([], + config_path: "test/fixtures/config/temp.secret.exs", + exported_config_path: "test/fixtures/config/temp.exported_from_db.secret.exs" + ) + + assert merged[:pleroma][:first_setting] == [key2: [Pleroma.Repo], key: "new value"] + end + end +end From 9015df22291ab60c0efad328557936fd14eab2e6 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 7 Jan 2021 18:23:01 +0100 Subject: [PATCH 034/150] TagValidator: New --- .../object_validators/article_note_validator.ex | 7 +- .../object_validators/attachment_validator.ex | 1 - .../object_validators/audio_video_validator.ex | 7 +- .../object_validators/event_validator.ex | 7 +- .../object_validators/question_validator.ex | 7 +- .../object_validators/tag_validator.ex | 77 ++++++++++++++++++++++ 6 files changed, 93 insertions(+), 13 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/tag_validator.ex diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index b0388ef3b..5910f4060 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -22,8 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:bto, ObjectValidators.Recipients, default: []) field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) + embeds_many(:tag, TagValidator) field(:type, :string) field(:name, :string) @@ -90,8 +90,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast(data, __schema__(:fields) -- [:attachment, :tag]) |> cast_embed(:attachment) + |> cast_embed(:tag) end def validate_data(data_cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 3175427ad..e7b3a3922 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator import Ecto.Changeset diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 4a96fef52..a04c95f4b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:bto, ObjectValidators.Recipients, default: []) field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) + embeds_many(:tag, TagValidator) field(:type, :string) field(:name, :string) @@ -132,8 +132,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast(data, __schema__(:fields) -- [:attachment, :tag]) |> cast_embed(:attachment) + |> cast_embed(:tag) end def validate_data(data_cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index 2e26726f8..0112a074d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:bto, ObjectValidators.Recipients, default: []) field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) + embeds_many(:tag, TagValidator) field(:type, :string) field(:name, :string) @@ -81,8 +81,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast(data, __schema__(:fields) -- [:attachment, :tag]) |> cast_embed(:attachment) + |> cast_embed(:tag) end def validate_data(data_cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 6b746c997..7acb1e928 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -24,8 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:bto, ObjectValidators.Recipients, default: []) field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) + embeds_many(:tag, TagValidator) field(:type, :string) field(:content, :string) field(:context, :string) @@ -93,10 +93,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment]) + |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment, :tag]) |> cast_embed(:attachment) |> cast_embed(:anyOf) |> cast_embed(:oneOf) + |> cast_embed(:tag) end def validate_data(data_cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex new file mode 100644 index 000000000..751021585 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -0,0 +1,77 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + import Ecto.Changeset + + @primary_key false + embedded_schema do + # Common + field(:type, :string) + field(:name, :string) + + # Mention, Hashtag + field(:href, ObjectValidators.Uri) + + # Emoji + embeds_one :icon, IconObjectValidator, primary_key: false do + field(:type, :string) + field(:url, ObjectValidators.Uri) + end + + field(:updated, ObjectValidators.DateTime) + field(:id, ObjectValidators.Uri) + end + + def cast_and_validate(data) do + data + |> cast_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + def changeset(struct, %{"type" => "Mention"} = data) do + struct + |> cast(data, [:type, :name, :href]) + |> validate_required([:type, :href]) + end + + def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do + name = + cond do + "#" <> name -> name + name -> name + end + |> String.downcase() + + data = Map.put(data, "name", name) + + struct + |> cast(data, [:type, :name, :href]) + |> validate_required([:type, :name]) + end + + def changeset(struct, %{"type" => "Emoji"} = data) do + data = Map.put(data, "name", String.trim(data["name"], ":")) + + struct + |> cast(data, [:type, :name, :updated, :id]) + |> cast_embed(:icon, with: &icon_changeset/2) + |> validate_required([:type, :name, :icon]) + end + + def icon_changeset(struct, data) do + struct + |> cast(data, [:type, :url]) + |> validate_inclusion(:type, ~w[Image]) + |> validate_required([:type, :url]) + end +end From 5ae27c8451a7012b43ef9113713132158701364b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 12 Jan 2021 14:11:29 +0100 Subject: [PATCH 035/150] pipeline_test: Fix usage of %Activity{} --- lib/pleroma/web/activity_pub/object_validator.ex | 2 +- lib/pleroma/web/activity_pub/pipeline.ex | 2 ++ test/pleroma/web/activity_pub/pipeline_test.exs | 23 ++++++++++++++++------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index f75744203..15784b28c 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -169,7 +169,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def cast_and_apply(o), do: {:error, {:validator_not_set, o}} - # is_struct/1 isn't present in Elixir 1.8.x + # is_struct/1 appears in Elixir 1.11 def stringify_keys(%{__struct__: _} = object) do object |> Map.from_struct() diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 195596f94..0aa504e72 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -40,6 +40,8 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do end end + def do_common_pipeline(%{__struct__: _}, _meta), do: {:error, :is_struct} + def do_common_pipeline(object, meta) do with {_, {:ok, validated_object, meta}} <- {:validate_object, @object_validator.validate(object, meta)}, diff --git a/test/pleroma/web/activity_pub/pipeline_test.exs b/test/pleroma/web/activity_pub/pipeline_test.exs index 52fa933ee..e606fa3d1 100644 --- a/test/pleroma/web/activity_pub/pipeline_test.exs +++ b/test/pleroma/web/activity_pub/pipeline_test.exs @@ -25,9 +25,6 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do MRFMock |> expect(:pipeline_filter, fn o, m -> {:ok, o, m} end) - ActivityPubMock - |> expect(:persist, fn o, m -> {:ok, o, m} end) - SideEffectsMock |> expect(:handle, fn o, m -> {:ok, o, m} end) |> expect(:handle_after_transaction, fn m -> m end) @@ -42,6 +39,9 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do activity_with_object = %{activity | data: Map.put(activity.data, "object", object)} + ActivityPubMock + |> expect(:persist, fn _, m -> {:ok, activity, m} end) + FederatorMock |> expect(:publish, fn ^activity_with_object -> :ok end) @@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do assert {:ok, ^activity, ^meta} = Pleroma.Web.ActivityPub.Pipeline.common_pipeline( - activity, + activity.data, meta ) end @@ -59,6 +59,9 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do activity = insert(:note_activity) meta = [local: true] + ActivityPubMock + |> expect(:persist, fn _, m -> {:ok, activity, m} end) + FederatorMock |> expect(:publish, fn ^activity -> :ok end) @@ -66,29 +69,35 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do |> expect(:get, fn [:instance, :federating] -> true end) assert {:ok, ^activity, ^meta} = - Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta) + Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta) end test "it goes through validation, filtering, persisting, side effects without federation for remote activities" do activity = insert(:note_activity) meta = [local: false] + ActivityPubMock + |> expect(:persist, fn _, m -> {:ok, activity, m} end) + ConfigMock |> expect(:get, fn [:instance, :federating] -> true end) assert {:ok, ^activity, ^meta} = - Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta) + Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta) end test "it goes through validation, filtering, persisting, side effects without federation for local activities if federation is deactivated" do activity = insert(:note_activity) meta = [local: true] + ActivityPubMock + |> expect(:persist, fn _, m -> {:ok, activity, m} end) + ConfigMock |> expect(:get, fn [:instance, :federating] -> false end) assert {:ok, ^activity, ^meta} = - Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta) + Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta) end end end From 37a7f521fd4778cde48f1b003ad9695e6ea45d1f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 12 Jan 2021 09:30:22 +0100 Subject: [PATCH 036/150] Insert string-hashtags in Pipeline Cannot be done in Ecto schemas because only one type is allowed in arrays, and needs to be done before the MRFs. --- lib/pleroma/web/activity_pub/pipeline.ex | 34 +++++++++++++++++--------- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/common_api.ex | 12 +-------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 0aa504e72..e184a9376 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -42,23 +42,33 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do def do_common_pipeline(%{__struct__: _}, _meta), do: {:error, :is_struct} - def do_common_pipeline(object, meta) do - with {_, {:ok, validated_object, meta}} <- - {:validate_object, @object_validator.validate(object, meta)}, - {_, {:ok, mrfd_object, meta}} <- - {:mrf_object, @mrf.pipeline_filter(validated_object, meta)}, - {_, {:ok, activity, meta}} <- - {:persist_object, @activity_pub.persist(mrfd_object, meta)}, - {_, {:ok, activity, meta}} <- - {:execute_side_effects, @side_effects.handle(activity, meta)}, - {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do - {:ok, activity, meta} + def do_common_pipeline(message, meta) do + with {_, {:ok, message, meta}} <- {:validate, @object_validator.validate(message, meta)}, + {_, {:ok, message, meta}} <- {:fixup, validation_fixups(message, meta)}, + {_, {:ok, message, meta}} <- {:mrf, @mrf.pipeline_filter(message, meta)}, + {_, {:ok, message, meta}} <- {:persist, @activity_pub.persist(message, meta)}, + {_, {:ok, message, meta}} <- {:side_effects, @side_effects.handle(message, meta)}, + {_, {:ok, _}} <- {:federation, maybe_federate(message, meta)} do + {:ok, message, meta} else - {:mrf_object, {:reject, message, _}} -> {:reject, message} + {:mrf, {:reject, message, _}} -> {:reject, message} e -> {:error, e} end end + defp validation_fixups(message, meta) do + # Insert copy of hashtags as strings for the non-hashtag table indexing + message = + if message["tag"] do + tag = Object.hashtags(%Object{data: message}) ++ (message["tag"] || []) + Map.put(message, "tag", tag) + else + message + end + + {:ok, message, meta} + end + defp maybe_federate(%Object{}, _), do: {:ok, :not_federated} defp maybe_federate(%Activity{} = activity, meta) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8c7d6a747..4070ed14d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -564,7 +564,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Pipeline.common_pipeline(data, local: false) do {:ok, activity} else - {:error, {:validate_object, _}} = e -> + {:error, {:validate, _}} = e -> # Check if we have a create activity for this with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]), %Activity{data: %{"actor" => actor}} <- diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index b003e30c7..895baebc9 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -228,17 +228,7 @@ defmodule Pleroma.Web.CommonAPI do {:find_object, _} -> {:error, :not_found} - {:common_pipeline, - { - :error, - { - :validate_object, - { - :error, - changeset - } - } - }} = e -> + {:common_pipeline, {:error, {:validate, {:error, changeset}}}} = e -> if {:object, {"already liked by this actor", []}} in changeset.errors do {:ok, :already_liked} else From 7ebfe899007002f5bbf8744a8f0b582e0e13342e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 12 Jan 2021 11:14:09 +0100 Subject: [PATCH 037/150] object_validators: Mark validate_data as private --- .../web/activity_pub/object_validators/accept_reject_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/announce_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/answer_validator.ex | 2 +- .../web/activity_pub/object_validators/article_note_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/block_validator.ex | 2 +- .../web/activity_pub/object_validators/chat_message_validator.ex | 2 +- .../web/activity_pub/object_validators/create_chat_message_validator.ex | 2 +- .../web/activity_pub/object_validators/create_generic_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/delete_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/event_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/follow_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/like_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/question_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/undo_validator.ex | 2 +- lib/pleroma/web/activity_pub/object_validators/update_validator.ex | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex index d31e780c3..b577a1044 100644 --- a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do |> cast(data, __schema__(:fields)) end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Accept", "Reject"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex index b08a33e68..576341790 100644 --- a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do cng end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Announce"]) |> validate_required([:id, :type, :object, :actor, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index 15e4413cd..c9bd9e42d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do |> cast(data, __schema__(:fields)) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Answer"]) |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor]) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index 5910f4060..39ef6dc29 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do |> cast_embed(:tag) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Article", "Note"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index e7b3a3922..4a0d1473d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -89,7 +89,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do end end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_inclusion(:type, ~w[Document Audio Image Video]) |> validate_required([:mediaType, :url, :type]) diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index a04c95f4b..8a5a60526 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -137,7 +137,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do |> cast_embed(:tag) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Audio", "Video"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment]) diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex index c5f77bb76..88948135f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do |> cast(data, __schema__(:fields)) end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Block"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index 1189778f2..b153156b0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -67,7 +67,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do |> cast_embed(:attachment) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["ChatMessage"]) |> validate_required([:id, :actor, :to, :type, :published]) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex index 8384c16a7..7a31a99bf 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do |> validate_data(meta) end - def validate_data(cng, meta \\ []) do + defp validate_data(cng, meta) do cng |> validate_required([:id, :actor, :to, :type, :object]) |> validate_inclusion(:type, ["Create"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index bf56a918c..e06e442f4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -79,7 +79,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do |> CommonFixes.fix_actor() end - def validate_data(cng, meta \\ []) do + defp validate_data(cng, meta) do cng |> validate_required([:actor, :type, :object]) |> validate_inclusion(:type, ["Create"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex index fc1a79a72..7da67bf16 100644 --- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do Tombstone Video } - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Delete"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex index 1906e597e..ec7566515 100644 --- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do end end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["EmojiReact"]) |> validate_required([:id, :type, :object, :actor, :context, :to, :cc, :content]) diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index 0112a074d..d42458ef5 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -86,7 +86,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do |> cast_embed(:tag) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Event"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex index 6e428bacc..239cee5e7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do |> cast(data, __schema__(:fields)) end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Follow"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index 30c40b238..509da507b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do end end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Like"]) |> validate_required([:id, :type, :object, :actor, :context, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 7acb1e928..7012e2e1d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do |> cast_embed(:tag) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Question"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) diff --git a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex index 783a79ddb..e8af60ffa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do |> cast(data, __schema__(:fields)) end - def validate_data(data_cng) do + defp validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Undo"]) |> validate_required([:id, :type, :object, :actor, :to, :cc]) diff --git a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex index a66d41400..6bb1dc7fa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do |> cast(data, __schema__(:fields)) end - def validate_data(cng) do + defp validate_data(cng) do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Update"]) From ef36f7fa5cff0a0d364aff192954556b0d2b0d2a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 1 Apr 2021 13:49:04 +0200 Subject: [PATCH 038/150] Move tag fixup to object_validator --- lib/pleroma/web/activity_pub/object_validator.ex | 32 +++++++++++++++++++----- lib/pleroma/web/activity_pub/pipeline.ex | 14 ----------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 15784b28c..70d9a35a9 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -114,8 +114,33 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end def validate(%{"type" => type} = object, meta) + when type in ~w[Event Question Audio Video Article] do + validator = + case type do + "Event" -> EventValidator + "Question" -> QuestionValidator + "Audio" -> AudioVideoValidator + "Video" -> AudioVideoValidator + "Article" -> ArticleNoteValidator + end + + with {:ok, object} <- + object + |> validator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + + # Insert copy of hashtags as strings for the non-hashtag table indexing + tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) + object = Map.put(object, "tag", tag) + + {:ok, object, meta} + end + end + + def validate(%{"type" => type} = object, meta) when type in ~w[Accept Reject Follow Update Like EmojiReact Announce - Event ChatMessage Question Audio Video Article Answer] do + ChatMessage Answer] do validator = case type do "Accept" -> AcceptRejectValidator @@ -125,12 +150,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do "Like" -> LikeValidator "EmojiReact" -> EmojiReactValidator "Announce" -> AnnounceValidator - "Event" -> EventValidator "ChatMessage" -> ChatMessageValidator - "Question" -> QuestionValidator - "Audio" -> AudioVideoValidator - "Video" -> AudioVideoValidator - "Article" -> ArticleNoteValidator "Answer" -> AnswerValidator end diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index e184a9376..377eccb92 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -44,7 +44,6 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do def do_common_pipeline(message, meta) do with {_, {:ok, message, meta}} <- {:validate, @object_validator.validate(message, meta)}, - {_, {:ok, message, meta}} <- {:fixup, validation_fixups(message, meta)}, {_, {:ok, message, meta}} <- {:mrf, @mrf.pipeline_filter(message, meta)}, {_, {:ok, message, meta}} <- {:persist, @activity_pub.persist(message, meta)}, {_, {:ok, message, meta}} <- {:side_effects, @side_effects.handle(message, meta)}, @@ -56,19 +55,6 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do end end - defp validation_fixups(message, meta) do - # Insert copy of hashtags as strings for the non-hashtag table indexing - message = - if message["tag"] do - tag = Object.hashtags(%Object{data: message}) ++ (message["tag"] || []) - Map.put(message, "tag", tag) - else - message - end - - {:ok, message, meta} - end - defp maybe_federate(%Object{}, _), do: {:ok, :not_federated} defp maybe_federate(%Activity{} = activity, meta) do From e56779dd8d1668177afa199aaa836bea70e68420 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Sep 2020 11:09:11 +0200 Subject: [PATCH 039/150] Transmogrifier: Simplify fix_explicit_addressing and fix_implicit_addressing --- lib/pleroma/web/activity_pub/transmogrifier.ex | 51 +++++++--------------- .../web/activity_pub/transmogrifier_test.exs | 6 +-- 2 files changed, 19 insertions(+), 38 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 4070ed14d..047f23918 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -72,17 +72,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_explicit_addressing( - %{"to" => to, "cc" => cc} = object, - explicit_mentions, - follower_collection - ) do - explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end) + # if directMessage flag is set to true, leave the addressing alone + def fix_explicit_addressing(%{"directMessage" => true} = object, _follower_collection), + do: object + def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, follower_collection) do + explicit_mentions = + Utils.determine_explicit_mentions(object) ++ + [Pleroma.Constants.as_public(), follower_collection] + + explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end) explicit_cc = Enum.filter(to, fn x -> x not in explicit_mentions end) final_cc = (cc ++ explicit_cc) + |> Enum.filter(& &1) |> Enum.reject(fn x -> String.ends_with?(x, "/followers") and x != follower_collection end) |> Enum.uniq() @@ -91,29 +95,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("cc", final_cc) end - def fix_explicit_addressing(object, _explicit_mentions, _followers_collection), do: object - - # if directMessage flag is set to true, leave the addressing alone - def fix_explicit_addressing(%{"directMessage" => true} = object), do: object - - def fix_explicit_addressing(object) do - explicit_mentions = Utils.determine_explicit_mentions(object) - - %User{follower_address: follower_collection} = - object - |> Containment.get_actor() - |> User.get_cached_by_ap_id() - - explicit_mentions = - explicit_mentions ++ - [ - Pleroma.Constants.as_public(), - follower_collection - ] - - fix_explicit_addressing(object, explicit_mentions, follower_collection) - end - # if as:Public is addressed, then make sure the followers collection is also addressed # so that the activities will be delivered to local users. def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do @@ -137,19 +118,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_implicit_addressing(object, _), do: object - def fix_addressing(object) do - {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"]) - followers_collection = User.ap_followers(user) + {:ok, %User{follower_address: follower_collection}} = + object + |> Containment.get_actor() + |> User.get_or_fetch_by_ap_id() object |> fix_addressing_list("to") |> fix_addressing_list("cc") |> fix_addressing_list("bto") |> fix_addressing_list("bcc") - |> fix_explicit_addressing() - |> fix_implicit_addressing(followers_collection) + |> fix_explicit_addressing(follower_collection) + |> fix_implicit_addressing(follower_collection) end def fix_actor(%{"attributedTo" => actor} = object) do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 4c3fcb44a..bb0b58e4d 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -446,7 +446,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) } - fixed_object = Transmogrifier.fix_explicit_addressing(object) + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) assert Enum.all?(explicitly_mentioned_actors, &(&1 in fixed_object["to"])) refute "https://social.beepboop.ga/users/dirb" in fixed_object["to"] assert "https://social.beepboop.ga/users/dirb" in fixed_object["cc"] @@ -459,7 +459,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "cc" => [] } - fixed_object = Transmogrifier.fix_explicit_addressing(object) + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) assert user.follower_address in fixed_object["to"] refute user.follower_address in fixed_object["cc"] end @@ -473,7 +473,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "cc" => [user.follower_address, recipient.follower_address] } - fixed_object = Transmogrifier.fix_explicit_addressing(object) + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) assert user.follower_address in fixed_object["cc"] refute recipient.follower_address in fixed_object["cc"] From e2a3365b5ce86293a5fed28c06b2e7d9dd97c9d1 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Sep 2020 11:08:05 +0200 Subject: [PATCH 040/150] ObjectValidator.CommonFixes: Introduce fix_objects_defaults and fix_activity_defaults --- .../activity_pub/object_validators/recipients.ex | 22 ++++++++++------- .../object_validators/article_note_validator.ex | 3 +-- .../object_validators/audio_video_validator.ex | 3 +-- .../activity_pub/object_validators/common_fixes.ex | 28 ++++++++++++++++++---- .../object_validators/create_generic_validator.ex | 12 +--------- .../object_validators/event_validator.ex | 4 ++-- .../object_validators/question_validator.ex | 4 ++-- .../object_validators/recipients_test.exs | 2 +- .../transmogrifier/audio_handling_test.exs | 6 ++++- .../transmogrifier/event_handling_test.exs | 2 +- 10 files changed, 50 insertions(+), 36 deletions(-) diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex index af4b0e527..b76547e75 100644 --- a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -15,19 +15,23 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do def cast(data) when is_list(data) do data - |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> - case ObjectID.cast(element) do - {:ok, id} -> - {:cont, {:ok, [id | list]}} + |> Enum.reduce_while({:ok, []}, fn + nil, {:ok, list} -> + {:cont, {:ok, list}} - _ -> - {:halt, :error} - end + element, {:ok, list} -> + case ObjectID.cast(element) do + {:ok, id} -> + {:cont, {:ok, [id | list]}} + + _ -> + {:halt, {:error, element}} + end end) end - def cast(_) do - :error + def cast(data) do + {:error, data} end def dump(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index 39ef6dc29..d2026b5ea 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -79,9 +79,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> fix_url() |> Transmogrifier.fix_emoji() end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 8a5a60526..8ee432947 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -120,9 +120,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() |> fix_url() |> fix_content() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 5f2c633bc..950eb1494 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -3,26 +3,44 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object.Containment + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils - # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults - def fix_defaults(data) do + def fix_object_defaults(data) do %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"] || data["conversation"]) + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) + {:ok, to} = ObjectValidators.Recipients.cast(data["to"] || []) + {:ok, cc} = ObjectValidators.Recipients.cast(data["cc"] || []) + data |> Map.put("context", context) |> Map.put("context_id", context_id) + |> Map.put("to", to) + |> Map.put("cc", cc) + |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) end - def fix_attribution(data) do + def fix_activity_defaults(data, meta) do + object = meta[:object_data] || %{} + data - |> Map.put_new("actor", data["attributedTo"]) + |> Map.put_new("to", object["to"] || []) + |> Map.put_new("cc", object["cc"] || []) + |> Map.put_new("bto", object["bto"] || []) + |> Map.put_new("bcc", object["bcc"] || []) end def fix_actor(data) do - actor = Containment.get_actor(data) + actor = + data + |> Map.put_new("actor", data["attributedTo"]) + |> Containment.get_actor() data |> Map.put("actor", actor) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index e06e442f4..99e8dc6c7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -62,21 +62,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end end - defp fix_addressing(data, meta) do - if object = meta[:object_data] do - data - |> Map.put_new("to", object["to"] || []) - |> Map.put_new("cc", object["cc"] || []) - else - data - end - end - defp fix(data, meta) do data |> fix_context(meta) - |> fix_addressing(meta) |> CommonFixes.fix_actor() + |> CommonFixes.fix_activity_defaults(meta) end defp validate_data(cng, meta) do diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index d42458ef5..fee2e997a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -72,8 +72,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 7012e2e1d..083d08ec4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -83,8 +83,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() |> fix_closed() end diff --git a/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs b/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs index d3a2fd13f..ce8bef39f 100644 --- a/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs +++ b/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs @@ -9,7 +9,7 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.RecipientsTest do test "it asserts that all elements of the list are object ids" do list = ["https://lain.com/users/lain", "invalid"] - assert :error == Recipients.cast(list) + assert {:error, "invalid"} == Recipients.cast(list) end test "it works with a list" do diff --git a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs index e733f167d..032ad24b5 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs @@ -24,6 +24,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AudioHandlingTest do "actor" => "http://mastodon.example.org/users/admin", "object" => %{ "type" => "Audio", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], "id" => "http://mastodon.example.org/users/admin/listens/1234", "attributedTo" => "http://mastodon.example.org/users/admin", "title" => "lain radio episode 1", @@ -61,7 +63,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AudioHandlingTest do assert object.data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] - assert object.data["cc"] == [] + assert object.data["cc"] == [ + "https://channels.tests.funkwhale.audio/federation/actors/compositions/followers" + ] assert object.data["url"] == "https://channels.tests.funkwhale.audio/library/tracks/74" diff --git a/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs index c4879fda1..14f5f704a 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.EventHandlingTest do ) assert object.data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] - assert object.data["cc"] == [] + assert object.data["cc"] == ["https://mobilizon.org/@tcit/followers"] assert object.data["url"] == "https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39" From c9449326747f8d33357f5179e69d3024b39089a0 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Sep 2020 11:11:10 +0200 Subject: [PATCH 041/150] Pipeline Ingestion: Note --- .../activity_pub/object_validators/recipients.ex | 25 ++-- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/object_validator.ex | 7 +- .../object_validators/article_note_validator.ex | 29 +++- .../activity_pub/object_validators/common_fixes.ex | 18 ++- .../object_validators/common_validations.ex | 1 + .../object_validators/create_note_validator.ex | 29 ---- lib/pleroma/web/activity_pub/side_effects.ex | 15 +- lib/pleroma/web/activity_pub/transmogrifier.ex | 12 +- lib/pleroma/web/federator.ex | 5 + .../fixtures/activitypub-client-post-activity.json | 1 + test/pleroma/activity_test.exs | 4 +- .../object_validators/recipients_test.exs | 4 +- test/pleroma/notification_test.exs | 6 + .../activity_pub/activity_pub_controller_test.exs | 45 +++--- .../transmogrifier/note_handling_test.exs | 155 ++++++++++----------- .../web/activity_pub/transmogrifier_test.exs | 4 +- test/pleroma/web/federator_test.exs | 6 +- .../web/static_fe/static_fe_controller_test.exs | 13 +- 19 files changed, 202 insertions(+), 179 deletions(-) delete mode 100644 lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex index b76547e75..a03471462 100644 --- a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -13,20 +13,23 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do cast([object]) end + def cast(object) when is_map(object) do + case ObjectID.cast(object) do + {:ok, data} -> {:ok, data} + _ -> :error + end + end + def cast(data) when is_list(data) do data - |> Enum.reduce_while({:ok, []}, fn - nil, {:ok, list} -> - {:cont, {:ok, list}} + |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> + case ObjectID.cast(element) do + {:ok, id} -> + {:cont, {:ok, [id | list]}} - element, {:ok, list} -> - case ObjectID.cast(element) do - {:ok, id} -> - {:cont, {:ok, [id | list]}} - - _ -> - {:halt, {:error, element}} - end + _ -> + {:cont, {:ok, list}} + end end) end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index efbf92c70..b74af3f3b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop - @object_types ~w[ChatMessage Question Answer Audio Video Event Article] + @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note] @impl true def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 70d9a35a9..e5b35cdd4 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -101,7 +101,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta ) - when objtype in ~w[Question Answer Audio Video Event Article] do + when objtype in ~w[Question Answer Audio Video Event Article Note] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -114,7 +114,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end def validate(%{"type" => type} = object, meta) - when type in ~w[Event Question Audio Video Article] do + when type in ~w[Event Question Audio Video Article Note] do validator = case type do "Event" -> EventValidator @@ -122,6 +122,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do "Audio" -> AudioVideoValidator "Video" -> AudioVideoValidator "Article" -> ArticleNoteValidator + "Note" -> ArticleNoteValidator end with {:ok, object} <- @@ -183,7 +184,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do EventValidator.cast_and_apply(object) end - def cast_and_apply(%{"type" => "Article"} = object) do + def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do ArticleNoteValidator.cast_and_apply(object) end diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index d2026b5ea..193f85f49 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -50,6 +50,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do field(:likes, {:array, ObjectValidators.ObjectID}, default: []) field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) + + field(:replies, {:array, ObjectValidators.ObjectID}, default: []) end def cast_and_apply(data) do @@ -65,24 +67,39 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do end def cast_data(data) do - data = fix(data) - %__MODULE__{} |> changeset(data) end - defp fix_url(%{"url" => url} = data) when is_map(url) do - Map.put(data, "url", url["href"]) - end - + defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data + defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) defp fix_url(data), do: data + defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data + defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) + defp fix_tag(data), do: Map.drop(data, ["tag"]) + + defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data) + when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies), + do: Map.drop(data, ["replies"]) + + defp fix_replies(data), do: data + defp fix(data) do data |> CommonFixes.fix_actor() |> CommonFixes.fix_object_defaults() |> fix_url() + |> fix_tag() + |> fix_replies() |> Transmogrifier.fix_emoji() + |> Transmogrifier.fix_content_map() end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 950eb1494..7309f6af2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -26,14 +26,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do |> Transmogrifier.fix_implicit_addressing(follower_collection) end - def fix_activity_defaults(data, meta) do + defp fix_activity_recipients(activity, field, object) do + {:ok, data} = ObjectValidators.Recipients.cast(activity[field] || object[field]) + + Map.put(activity, field, data) + end + + def fix_activity_defaults(activity, meta) do object = meta[:object_data] || %{} - data - |> Map.put_new("to", object["to"] || []) - |> Map.put_new("cc", object["cc"] || []) - |> Map.put_new("bto", object["bto"] || []) - |> Map.put_new("bcc", object["bcc"] || []) + activity + |> fix_activity_recipients("to", object) + |> fix_activity_recipients("cc", object) + |> fix_activity_recipients("bto", object) + |> fix_activity_recipients("bcc", object) end def fix_actor(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 093549a45..85ac07044 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do fields |> Enum.map(fn field -> get_field(cng, field) end) |> Enum.any?(fn + nil -> false [] -> false _ -> true end) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex deleted file mode 100644 index a85a0298c..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex +++ /dev/null @@ -1,29 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do - use Ecto.Schema - - alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator - - import Ecto.Changeset - - @primary_key false - - embedded_schema do - field(:id, ObjectValidators.ObjectID, primary_key: true) - field(:actor, ObjectValidators.ObjectID) - field(:type, :string) - field(:to, ObjectValidators.Recipients, default: []) - field(:cc, ObjectValidators.Recipients, default: []) - field(:bto, ObjectValidators.Recipients, default: []) - field(:bcc, ObjectValidators.Recipients, default: []) - embeds_one(:object, NoteValidator) - end - - def cast_data(data) do - cast(%__MODULE__{}, data, __schema__(:fields)) - end -end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0b9a9f0c5..3234b9e43 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -203,6 +203,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.increase_replies_count(in_reply_to) end + reply_depth = (meta[:depth] || 0) + 1 + + # FIXME: Force inReplyTo to replies + if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and + object.data["replies"] != nil do + for reply_id <- object.data["replies"] do + Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ + "id" => reply_id, + "depth" => reply_depth + }) + end + end + ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn -> Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) @@ -366,7 +379,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end def handle_object_creation(%{"type" => objtype} = object, meta) - when objtype in ~w[Audio Video Question Event Article] do + when objtype in ~w[Audio Video Question Event Article Note] do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 047f23918..28bc25363 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -404,10 +404,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming( - %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, + %{"type" => "Create", "object" => %{"type" => "Page"} = object} = data, options - ) - when objtype in ~w{Note Page} do + ) do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -499,14 +498,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data, - _options + options ) - when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do + when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do data = Map.put(data, "object", strip_internal_fields(data["object"])) + options = Keyword.put(options, :local, false) with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), nil <- Activity.get_create_by_object_ap_id(obj_id), - {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do + {:ok, activity, _} <- Pipeline.common_pipeline(data, options) do {:ok, activity} else %Activity{} = activity -> {:ok, activity} diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index f5ef76d32..69cfc2d52 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -96,6 +96,11 @@ defmodule Pleroma.Web.Federator do Logger.debug("Unhandled actor #{actor}, #{inspect(e)}") {:error, e} + {:error, {:validate_object, _}} = e -> + Logger.error("Incoming AP doc validation error: #{inspect(e)}") + Logger.debug(Jason.encode!(params, pretty: true)) + e + e -> # Just drop those for now Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end) diff --git a/test/fixtures/activitypub-client-post-activity.json b/test/fixtures/activitypub-client-post-activity.json index c985e072b..e592081bc 100644 --- a/test/fixtures/activitypub-client-post-activity.json +++ b/test/fixtures/activitypub-client-post-activity.json @@ -3,6 +3,7 @@ "type": "Create", "object": { "type": "Note", + "to": ["https://www.w3.org/ns/activitystreams#Public"], "content": "It's a note" }, "to": ["https://www.w3.org/ns/activitystreams#Public"] diff --git a/test/pleroma/activity_test.exs b/test/pleroma/activity_test.exs index 390a06344..9911aa45c 100644 --- a/test/pleroma/activity_test.exs +++ b/test/pleroma/activity_test.exs @@ -123,7 +123,8 @@ defmodule Pleroma.ActivityTest do "type" => "Note", "content" => "find me!", "id" => "http://mastodon.example.org/users/admin/objects/1", - "attributedTo" => "http://mastodon.example.org/users/admin" + "attributedTo" => "http://mastodon.example.org/users/admin", + "to" => ["https://www.w3.org/ns/activitystreams#Public"] }, "to" => ["https://www.w3.org/ns/activitystreams#Public"] } @@ -132,6 +133,7 @@ defmodule Pleroma.ActivityTest do {:ok, japanese_activity} = Pleroma.Web.CommonAPI.post(user, %{status: "更新情報"}) {:ok, job} = Pleroma.Web.Federator.incoming_ap_doc(params) {:ok, remote_activity} = ObanHelpers.perform(job) + remote_activity = Activity.get_by_id_with_object(remote_activity.id) %{ japanese_activity: japanese_activity, diff --git a/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs b/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs index ce8bef39f..4cdafa898 100644 --- a/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs +++ b/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs @@ -6,10 +6,10 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.RecipientsTest do alias Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients use Pleroma.DataCase, async: true - test "it asserts that all elements of the list are object ids" do + test "it only keeps elements that are valid object ids" do list = ["https://lain.com/users/lain", "invalid"] - assert {:error, "invalid"} == Recipients.cast(list) + assert {:ok, ["https://lain.com/users/lain"]} == Recipients.cast(list) end test "it works with a list" do diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index abf1b0410..85f895f0f 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -624,6 +624,8 @@ defmodule Pleroma.NotificationTest do "actor" => user.ap_id, "object" => %{ "type" => "Note", + "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), + "to" => ["https://www.w3.org/ns/activitystreams#Public"], "content" => "message with a Mention tag, but no explicit tagging", "tag" => [ %{ @@ -655,6 +657,9 @@ defmodule Pleroma.NotificationTest do "actor" => user.ap_id, "object" => %{ "type" => "Note", + "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [other_user.ap_id], "content" => "hi everyone", "attributedTo" => user.ap_id } @@ -951,6 +956,7 @@ defmodule Pleroma.NotificationTest do "cc" => [], "object" => %{ "type" => "Note", + "id" => remote_user.ap_id <> "/objects/test", "content" => "Hello!", "tag" => [ %{ diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 19e04d472..2de52323e 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -539,7 +539,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() |> Map.put("actor", user.ap_id) - |> put_in(["object", "attridbutedTo"], user.ap_id) + |> put_in(["object", "attributedTo"], user.ap_id) conn = conn @@ -820,29 +820,34 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Instances.reachable?(sender_host) end + @tag capture_log: true test "it removes all follower collections but actor's", %{conn: conn} do [actor, recipient] = insert_pair(:user) - data = - File.read!("test/fixtures/activitypub-client-post-activity.json") - |> Jason.decode!() + to = [ + recipient.ap_id, + recipient.follower_address, + "https://www.w3.org/ns/activitystreams#Public" + ] - object = Map.put(data["object"], "attributedTo", actor.ap_id) + cc = [recipient.follower_address, actor.follower_address] - data = - data - |> Map.put("id", Utils.generate_object_id()) - |> Map.put("actor", actor.ap_id) - |> Map.put("object", object) - |> Map.put("cc", [ - recipient.follower_address, - actor.follower_address - ]) - |> Map.put("to", [ - recipient.ap_id, - recipient.follower_address, - "https://www.w3.org/ns/activitystreams#Public" - ]) + data = %{ + "@context" => ["https://www.w3.org/ns/activitystreams"], + "type" => "Create", + "id" => Utils.generate_activity_id(), + "to" => to, + "cc" => cc, + "actor" => actor.ap_id, + "object" => %{ + "type" => "Note", + "to" => to, + "cc" => cc, + "content" => "It's a note", + "attributedTo" => actor.ap_id, + "id" => Utils.generate_object_id() + } + } conn |> assign(:valid_signature, true) @@ -852,7 +857,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) - activity = Activity.get_by_ap_id(data["id"]) + assert activity = Activity.get_by_ap_id(data["id"]) assert activity.id assert actor.follower_address in activity.recipients diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index deb956410..3eeae4004 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -14,7 +14,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do import Mock import Pleroma.Factory - import ExUnit.CaptureLog setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -147,9 +146,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do data |> Map.put("object", object) - assert capture_log(fn -> - {:ok, _returned_activity} = Transmogrifier.handle_incoming(data) - end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil" + assert {:ok, _returned_activity} = Transmogrifier.handle_incoming(data) end test "it does not work for deactivated users" do @@ -221,8 +218,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) object = Object.normalize(data["object"], fetch: false) - assert Enum.at(Object.tags(object), 2) == "moo" - assert Object.hashtags(object) == ["moo"] + assert match?( + %{ + "href" => "http://localtesting.pleroma.lol/users/lain", + "name" => "@lain@localtesting.pleroma.lol", + "type" => "Mention" + }, + Enum.at(object.data["tag"], 0) + ) + + assert match?( + %{ + "href" => "http://mastodon.example.org/tags/moo", + "name" => "#moo", + "type" => "Hashtag" + }, + Enum.at(object.data["tag"], 1) + ) + + assert "moo" == Enum.at(object.data["tag"], 2) end test "it works for incoming notices with contentMap" do @@ -276,13 +290,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() |> Map.put("actor", user.ap_id) - |> Map.put("to", nil) |> Map.put("cc", nil) object = data["object"] |> Map.put("attributedTo", user.ap_id) - |> Map.put("to", nil) |> Map.put("cc", nil) |> Map.put("id", user.ap_id <> "/activities/12345678") @@ -290,8 +302,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - assert !is_nil(data["to"]) - assert !is_nil(data["cc"]) + refute is_nil(data["cc"]) end test "it strips internal likes" do @@ -330,70 +341,46 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do end test "it correctly processes messages with non-array to field" do - user = insert(:user) + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Map.put("to", "https://www.w3.org/ns/activitystreams#Public") + |> put_in(["object", "to"], "https://www.w3.org/ns/activitystreams#Public") - message = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "to" => "https://www.w3.org/ns/activitystreams#Public", - "type" => "Create", - "object" => %{ - "content" => "blah blah blah", - "type" => "Note", - "attributedTo" => user.ap_id, - "inReplyTo" => nil - }, - "actor" => user.ap_id - } + assert {:ok, activity} = Transmogrifier.handle_incoming(data) - assert {:ok, activity} = Transmogrifier.handle_incoming(message) + assert [ + "http://mastodon.example.org/users/admin/followers", + "http://localtesting.pleroma.lol/users/lain" + ] == activity.data["cc"] assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] end test "it correctly processes messages with non-array cc field" do - user = insert(:user) + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Map.put("cc", "http://mastodon.example.org/users/admin/followers") + |> put_in(["object", "cc"], "http://mastodon.example.org/users/admin/followers") - message = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "to" => user.follower_address, - "cc" => "https://www.w3.org/ns/activitystreams#Public", - "type" => "Create", - "object" => %{ - "content" => "blah blah blah", - "type" => "Note", - "attributedTo" => user.ap_id, - "inReplyTo" => nil - }, - "actor" => user.ap_id - } + assert {:ok, activity} = Transmogrifier.handle_incoming(data) - assert {:ok, activity} = Transmogrifier.handle_incoming(message) - - assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["cc"] - assert [user.follower_address] == activity.data["to"] + assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"] + assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] end test "it correctly processes messages with weirdness in address fields" do - user = insert(:user) + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Map.put("cc", ["http://mastodon.example.org/users/admin/followers", ["¿"]]) + |> put_in(["object", "cc"], ["http://mastodon.example.org/users/admin/followers", ["¿"]]) - message = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "to" => [nil, user.follower_address], - "cc" => ["https://www.w3.org/ns/activitystreams#Public", ["¿"]], - "type" => "Create", - "object" => %{ - "content" => "…", - "type" => "Note", - "attributedTo" => user.ap_id, - "inReplyTo" => nil - }, - "actor" => user.ap_id - } + assert {:ok, activity} = Transmogrifier.handle_incoming(data) - assert {:ok, activity} = Transmogrifier.handle_incoming(message) - - assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["cc"] - assert [user.follower_address] == activity.data["to"] + assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"] + assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] end end @@ -419,7 +406,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do } do clear_config([:instance, :federation_incoming_replies_max_depth], 10) - {:ok, _activity} = Transmogrifier.handle_incoming(data) + {:ok, activity} = Transmogrifier.handle_incoming(data) + + object = Object.normalize(activity.data["object"]) + + assert object.data["replies"] == items for id <- items do job_args = %{"op" => "fetch_remote", "id" => id, "depth" => 1} @@ -442,45 +433,41 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do setup do: clear_config([:instance, :federation_incoming_replies_max_depth]) setup do - user = insert(:user) + replies = %{ + "type" => "Collection", + "items" => [ + Pleroma.Web.ActivityPub.Utils.generate_object_id(), + Pleroma.Web.ActivityPub.Utils.generate_object_id() + ] + } - {:ok, activity} = CommonAPI.post(user, %{status: "post1"}) + activity = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Kernel.put_in(["object", "replies"], replies) - {:ok, reply1} = - CommonAPI.post(user, %{status: "reply1", in_reply_to_status_id: activity.id}) - - {:ok, reply2} = - CommonAPI.post(user, %{status: "reply2", in_reply_to_status_id: activity.id}) - - replies_uris = Enum.map([reply1, reply2], fn a -> a.object.data["id"] end) - - {:ok, federation_output} = Transmogrifier.prepare_outgoing(activity.data) - - Repo.delete(activity.object) - Repo.delete(activity) - - %{federation_output: federation_output, replies_uris: replies_uris} + %{activity: activity} end test "schedules background fetching of `replies` items if max thread depth limit allows", %{ - federation_output: federation_output, - replies_uris: replies_uris + activity: activity } do clear_config([:instance, :federation_incoming_replies_max_depth], 1) - {:ok, _activity} = Transmogrifier.handle_incoming(federation_output) + assert {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(activity) + object = Object.normalize(data["object"]) - for id <- replies_uris do + for id <- object.data["replies"] do job_args = %{"op" => "fetch_remote", "id" => id, "depth" => 1} assert_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker, args: job_args) end end test "does NOT schedule background fetching of `replies` beyond max thread depth limit allows", - %{federation_output: federation_output} do + %{activity: activity} do clear_config([:instance, :federation_incoming_replies_max_depth], 0) - {:ok, _activity} = Transmogrifier.handle_incoming(federation_output) + {:ok, _activity} = Transmogrifier.handle_incoming(activity) assert all_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker) == [] end @@ -498,6 +485,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do "object" => %{ "to" => ["https://www.w3.org/ns/activitystreams#Public"], "cc" => [], + "id" => Utils.generate_object_id(), "type" => "Note", "content" => "Hi", "inReplyTo" => nil, @@ -522,6 +510,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do "object" => %{ "to" => ["https://www.w3.org/ns/activitystreams#Public"], "cc" => [], + "id" => Utils.generate_object_id(), "type" => "Note", "content" => "Hi", "inReplyTo" => nil, diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index bb0b58e4d..5a3b57acb 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.CommonAPI @@ -159,8 +160,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - assert modified["@context"] == - Pleroma.Web.ActivityPub.Utils.make_json_ld_header()["@context"] + assert modified["@context"] == Utils.make_json_ld_header()["@context"] assert modified["object"]["conversation"] == modified["context"] end diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs index 532ee6d30..372b6a73a 100644 --- a/test/pleroma/web/federator_test.exs +++ b/test/pleroma/web/federator_test.exs @@ -123,7 +123,8 @@ defmodule Pleroma.Web.FederatorTest do "type" => "Note", "content" => "hi world!", "id" => "http://mastodon.example.org/users/admin/objects/1", - "attributedTo" => "http://mastodon.example.org/users/admin" + "attributedTo" => "http://mastodon.example.org/users/admin", + "to" => ["https://www.w3.org/ns/activitystreams#Public"] }, "to" => ["https://www.w3.org/ns/activitystreams#Public"] } @@ -145,7 +146,8 @@ defmodule Pleroma.Web.FederatorTest do "type" => "Note", "content" => "hi world!", "id" => "http://mastodon.example.org/users/admin/objects/1", - "attributedTo" => "http://mastodon.example.org/users/admin" + "attributedTo" => "http://mastodon.example.org/users/admin", + "to" => ["https://www.w3.org/ns/activitystreams#Public"] }, "to" => ["https://www.w3.org/ns/activitystreams#Public"] } diff --git a/test/pleroma/web/static_fe/static_fe_controller_test.exs b/test/pleroma/web/static_fe/static_fe_controller_test.exs index 2af14dfeb..5752cffda 100644 --- a/test/pleroma/web/static_fe/static_fe_controller_test.exs +++ b/test/pleroma/web/static_fe/static_fe_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do alias Pleroma.Activity alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -185,16 +186,16 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do test "302 for remote cached status", %{conn: conn, user: user} do message = %{ "@context" => "https://www.w3.org/ns/activitystreams", - "to" => user.follower_address, - "cc" => "https://www.w3.org/ns/activitystreams#Public", "type" => "Create", + "actor" => user.ap_id, "object" => %{ + "to" => user.follower_address, + "cc" => "https://www.w3.org/ns/activitystreams#Public", + "id" => Utils.generate_object_id(), "content" => "blah blah blah", "type" => "Note", - "attributedTo" => user.ap_id, - "inReplyTo" => nil - }, - "actor" => user.ap_id + "attributedTo" => user.ap_id + } } assert {:ok, activity} = Transmogrifier.handle_incoming(message) From 641184fc7aff694e4e7e802b9204a1d313c0877c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 10 Sep 2020 19:45:42 +0200 Subject: [PATCH 042/150] recipients fixes/hardening for CreateGenericValidator --- .../activity_pub/object_validators/recipients.ex | 25 +++++---- .../activity_pub/object_validators/common_fixes.ex | 34 ++++++------ .../object_validators/create_generic_validator.ex | 60 ++++++++++++++++------ .../transmogrifier/note_handling_test.exs | 12 ++--- 4 files changed, 82 insertions(+), 49 deletions(-) diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex index a03471462..06fed8fb3 100644 --- a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -15,22 +15,27 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do def cast(object) when is_map(object) do case ObjectID.cast(object) do - {:ok, data} -> {:ok, data} + {:ok, data} -> {:ok, [data]} _ -> :error end end def cast(data) when is_list(data) do - data - |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> - case ObjectID.cast(element) do - {:ok, id} -> - {:cont, {:ok, [id | list]}} + data = + data + |> Enum.reduce_while([], fn element, list -> + case ObjectID.cast(element) do + {:ok, id} -> + {:cont, [id | list]} - _ -> - {:cont, {:ok, list}} - end - end) + _ -> + {:cont, list} + end + end) + |> Enum.sort() + |> Enum.uniq() + + {:ok, data} end def cast(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 7309f6af2..009cd51b0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -9,37 +9,39 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils + def cast_recipients(message, field, field_fallback \\ []) do + {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback) + + Map.put(message, field, data) + end + def fix_object_defaults(data) do %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"] || data["conversation"]) %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) - {:ok, to} = ObjectValidators.Recipients.cast(data["to"] || []) - {:ok, cc} = ObjectValidators.Recipients.cast(data["cc"] || []) data |> Map.put("context", context) |> Map.put("context_id", context_id) - |> Map.put("to", to) - |> Map.put("cc", cc) + |> cast_recipients("to") + |> cast_recipients("cc") + |> cast_recipients("bto") + |> cast_recipients("bcc") |> Transmogrifier.fix_explicit_addressing(follower_collection) |> Transmogrifier.fix_implicit_addressing(follower_collection) end - defp fix_activity_recipients(activity, field, object) do - {:ok, data} = ObjectValidators.Recipients.cast(activity[field] || object[field]) - - Map.put(activity, field, data) - end - - def fix_activity_defaults(activity, meta) do - object = meta[:object_data] || %{} + def fix_activity_addressing(activity, _meta) do + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"]) activity - |> fix_activity_recipients("to", object) - |> fix_activity_recipients("cc", object) - |> fix_activity_recipients("bto", object) - |> fix_activity_recipients("bcc", object) + |> cast_recipients("to") + |> cast_recipients("cc") + |> cast_recipients("bto") + |> cast_recipients("bcc") + |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) end def fix_actor(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 99e8dc6c7..51d43e8d0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -10,8 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -23,6 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do field(:type, :string) field(:to, ObjectValidators.Recipients, default: []) field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) field(:object, ObjectValidators.ObjectID) field(:expires_at, ObjectValidators.DateTime) @@ -54,29 +58,38 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do |> cast(data, __schema__(:fields)) end - defp fix_context(data, meta) do - if object = meta[:object_data] do - Map.put_new(data, "context", object["context"]) - else - data - end + # CommonFixes.fix_activity_addressing adapted for Create specific behavior + defp fix_addressing(data, object) do + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["actor"]) + + data + |> CommonFixes.cast_recipients("to", object["to"]) + |> CommonFixes.cast_recipients("cc", object["cc"]) + |> CommonFixes.cast_recipients("bto", object["bto"]) + |> CommonFixes.cast_recipients("bcc", object["bcc"]) + |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) end - defp fix(data, meta) do + def fix(data, meta) do + object = meta[:object_data] + data - |> fix_context(meta) |> CommonFixes.fix_actor() - |> CommonFixes.fix_activity_defaults(meta) + |> Map.put_new("context", object["context"]) + |> fix_addressing(object) end defp validate_data(cng, meta) do + object = meta[:object_data] + cng - |> validate_required([:actor, :type, :object]) + |> validate_required([:actor, :type, :object, :to, :cc]) |> validate_inclusion(:type, ["Create"]) |> CommonValidations.validate_actor_presence() - |> CommonValidations.validate_any_presence([:to, :cc]) - |> validate_actors_match(meta) - |> validate_context_match(meta) + |> validate_actors_match(object) + |> validate_context_match(object) + |> validate_addressing_match(object) |> validate_object_nonexistence() |> validate_object_containment() end @@ -108,8 +121,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_actors_match(cng, meta) do - attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"] + def validate_actors_match(cng, object) do + attributed_to = object["attributedTo"] || object["actor"] cng |> validate_change(:actor, fn :actor, actor -> @@ -121,7 +134,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do + def validate_context_match(cng, %{"context" => object_context}) do cng |> validate_change(:context, fn :context, context -> if context == object_context do @@ -132,5 +145,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_context_match(cng, _), do: cng + def validate_addressing_match(cng, object) do + [:to, :cc, :bcc, :bto] + |> Enum.reduce(cng, fn field, cng -> + object_data = object[to_string(field)] + + validate_change(cng, field, fn field, data -> + if data == object_data do + [] + else + [{field, "field doesn't match with object (#{inspect(object_data)})"}] + end + end) + end) + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 3eeae4004..b79f2c94c 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -171,8 +171,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do assert data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] assert data["cc"] == [ - "http://mastodon.example.org/users/admin/followers", - "http://localtesting.pleroma.lol/users/lain" + "http://localtesting.pleroma.lol/users/lain", + "http://mastodon.example.org/users/admin/followers" ] assert data["actor"] == "http://mastodon.example.org/users/admin" @@ -185,8 +185,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do assert object_data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] assert object_data["cc"] == [ - "http://mastodon.example.org/users/admin/followers", - "http://localtesting.pleroma.lol/users/lain" + "http://localtesting.pleroma.lol/users/lain", + "http://mastodon.example.org/users/admin/followers" ] assert object_data["actor"] == "http://mastodon.example.org/users/admin" @@ -350,8 +350,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do assert {:ok, activity} = Transmogrifier.handle_incoming(data) assert [ - "http://mastodon.example.org/users/admin/followers", - "http://localtesting.pleroma.lol/users/lain" + "http://localtesting.pleroma.lol/users/lain", + "http://mastodon.example.org/users/admin/followers" ] == activity.data["cc"] assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] From 96212b2e32e2542964c665f091158fb1ff1d987d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 15 Sep 2020 17:22:08 +0200 Subject: [PATCH 043/150] Fix addressing --- lib/pleroma/object/fetcher.ex | 7 ++++-- .../activity_pub/object_validators/common_fixes.ex | 25 ++++++++++++---------- .../object_validators/create_generic_validator.ex | 9 ++++---- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index bcccf1c4c..82d2c8bcb 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.HTTP + alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.Repo @@ -124,12 +125,14 @@ defmodule Pleroma.Object.Fetcher do defp prepare_activity_params(data) do %{ "type" => "Create", - "to" => data["to"] || [], - "cc" => data["cc"] || [], # Should we seriously keep this attributedTo thing? "actor" => data["actor"] || data["attributedTo"], "object" => data } + |> Maps.put_if_present("to", data["to"]) + |> Maps.put_if_present("cc", data["cc"]) + |> Maps.put_if_present("bto", data["bto"]) + |> Maps.put_if_present("bcc", data["bcc"]) end def fetch_object_from_id!(id, options \\ []) do diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 009cd51b0..c958fcc5d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -9,9 +9,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils - def cast_recipients(message, field, field_fallback \\ []) do + def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback) + data = + Enum.reject(data, fn x -> + String.ends_with?(x, "/followers") and x != follower_collection + end) + Map.put(message, field, data) end @@ -24,11 +29,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do data |> Map.put("context", context) |> Map.put("context_id", context_id) - |> cast_recipients("to") - |> cast_recipients("cc") - |> cast_recipients("bto") - |> cast_recipients("bcc") - |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) |> Transmogrifier.fix_implicit_addressing(follower_collection) end @@ -36,11 +40,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"]) activity - |> cast_recipients("to") - |> cast_recipients("cc") - |> cast_recipients("bto") - |> cast_recipients("bcc") - |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) |> Transmogrifier.fix_implicit_addressing(follower_collection) end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 51d43e8d0..d2de53049 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -63,11 +63,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["actor"]) data - |> CommonFixes.cast_recipients("to", object["to"]) - |> CommonFixes.cast_recipients("cc", object["cc"]) - |> CommonFixes.cast_recipients("bto", object["bto"]) - |> CommonFixes.cast_recipients("bcc", object["bcc"]) - |> Transmogrifier.fix_explicit_addressing(follower_collection) + |> CommonFixes.cast_and_filter_recipients("to", follower_collection, object["to"]) + |> CommonFixes.cast_and_filter_recipients("cc", follower_collection, object["cc"]) + |> CommonFixes.cast_and_filter_recipients("bto", follower_collection, object["bto"]) + |> CommonFixes.cast_and_filter_recipients("bcc", follower_collection, object["bcc"]) |> Transmogrifier.fix_implicit_addressing(follower_collection) end From d1205406d9237c72d10df937dd8d2d4da2786cc5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 15 Sep 2020 18:18:57 +0200 Subject: [PATCH 044/150] ActivityPubControllerTest: Apply same addr changes to object --- lib/pleroma/web/activity_pub/utils.ex | 5 +++- .../activity_pub/activity_pub_controller_test.exs | 30 ++++++++++++++++------ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index a4dc469dc..e81623d83 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -97,7 +97,10 @@ defmodule Pleroma.Web.ActivityPub.Utils do if need_splice? do cc_list = extract_list(params["cc"]) - Map.put(params, "cc", [ap_id | cc_list]) + + params + |> Map.put("cc", [ap_id | cc_list]) + |> Kernel.put_in(["object", "cc"], [ap_id | cc_list]) else params end diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 2de52323e..f6ea9e2ca 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -649,7 +649,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do test "it inserts an incoming activity into the database", %{conn: conn, data: data} do user = insert(:user) - data = Map.put(data, "bcc", [user.ap_id]) + + data = + data + |> Map.put("bcc", [user.ap_id]) + |> Kernel.put_in(["object", "bcc"], [user.ap_id]) conn = conn @@ -666,8 +670,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do user = insert(:user) data = - Map.put(data, "to", user.ap_id) - |> Map.delete("cc") + data + |> Map.put("to", user.ap_id) + |> Map.put("cc", []) + |> Kernel.put_in(["object", "to"], user.ap_id) + |> Kernel.put_in(["object", "cc"], []) conn = conn @@ -684,8 +691,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do user = insert(:user) data = - Map.put(data, "cc", user.ap_id) - |> Map.delete("to") + data + |> Map.put("to", []) + |> Map.put("cc", user.ap_id) + |> Kernel.put_in(["object", "to"], []) + |> Kernel.put_in(["object", "cc"], user.ap_id) conn = conn @@ -703,9 +713,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do user = insert(:user) data = - Map.put(data, "bcc", user.ap_id) - |> Map.delete("to") - |> Map.delete("cc") + data + |> Map.put("to", []) + |> Map.put("cc", []) + |> Map.put("bcc", user.ap_id) + |> Kernel.put_in(["object", "to"], []) + |> Kernel.put_in(["object", "cc"], []) + |> Kernel.put_in(["object", "bcc"], user.ap_id) conn = conn From b0c778fde77f5ec2320b0bd0327e8a13b0f39a63 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 15 Sep 2020 18:19:38 +0200 Subject: [PATCH 045/150] NoteHandlingTest: remove fix_explicit_addressing-related test --- .../transmogrifier/note_handling_test.exs | 42 ++++------------------ 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index b79f2c94c..1846b2291 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI import Mock @@ -42,36 +43,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do assert Object.hashtags(object) == ["test"] end - test "it cleans up incoming notices which are not really DMs" do - user = insert(:user) - other_user = insert(:user) - - to = [user.ap_id, other_user.ap_id] - - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Jason.decode!() - |> Map.put("to", to) - |> Map.put("cc", []) - - object = - data["object"] - |> Map.put("to", to) - |> Map.put("cc", []) - - data = Map.put(data, "object", object) - - {:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data) - - assert data["to"] == [] - assert data["cc"] == to - - object_data = Object.normalize(activity, fetch: false).data - - assert object_data["to"] == [] - assert object_data["cc"] == to - end - test "it ignores an incoming notice if we already have it" do activity = insert(:note_activity) @@ -321,9 +292,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do object = Map.put(data["object"], "likes", likes) data = Map.put(data, "object", object) - {:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data) + {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(data) - refute Map.has_key?(object.data, "likes") + object = Object.normalize(activity) + + assert object.data["likes"] == [] end test "it strips internal reactions" do @@ -435,10 +408,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do setup do replies = %{ "type" => "Collection", - "items" => [ - Pleroma.Web.ActivityPub.Utils.generate_object_id(), - Pleroma.Web.ActivityPub.Utils.generate_object_id() - ] + "items" => [Utils.generate_object_id(), Utils.generate_object_id()] } activity = From 461123110b7cf47f4d2c01d1dd6992a2b63337fe Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 17 Sep 2020 16:17:16 +0200 Subject: [PATCH 046/150] Object.Fetcher: Fix getting transmogrifier reject reason --- lib/pleroma/object/fetcher.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 82d2c8bcb..4ca67f0fd 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -102,6 +102,9 @@ defmodule Pleroma.Object.Fetcher do {:transmogrifier, {:error, {:reject, e}}} -> {:reject, e} + {:transmogrifier, {:reject, e}} -> + {:reject, e} + {:transmogrifier, _} = e -> {:error, e} From 6c9f6e62c8453f023c6ec9106d1a7c3e66ab95b7 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 28 Sep 2020 19:34:27 +0200 Subject: [PATCH 047/150] transmogrifier: Fixing votes from Note to Answer --- .../object_validators/answer_validator.ex | 7 +++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 22 ++++++++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index c9bd9e42d..3451e1ff8 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations import Ecto.Changeset @@ -23,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do field(:name, :string) field(:inReplyTo, ObjectValidators.ObjectID) field(:attributedTo, ObjectValidators.ObjectID) + field(:context, :string) # TODO: Remove actor on objects field(:actor, ObjectValidators.ObjectID) @@ -46,6 +48,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do end def changeset(struct, data) do + data = + data + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() + struct |> cast(data, __schema__(:fields)) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 28bc25363..454bbce9d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -43,7 +43,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_content_map() |> fix_addressing() |> fix_summary() - |> fix_type(options) end def fix_summary(%{"summary" => nil} = object) do @@ -321,19 +320,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object - def fix_type(object, options \\ []) + defp fix_type(%{"type" => "Note", "inReplyTo" => reply_id, "name" => _} = object, options) + when is_binary(reply_id) do + options = Keyword.put(options, :fetch, true) - def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options) - when is_binary(reply_id) do - with true <- Federator.allowed_thread_distance?(options[:depth]), - {:ok, %{data: %{"type" => "Question"} = _} = _} <- get_obj_helper(reply_id, options) do + with %Object{data: %{"type" => "Question"}} <- Object.normalize(reply_id, options) do Map.put(object, "type", "Answer") else _ -> object end end - def fix_type(object, _), do: object + defp fix_type(object, _options), do: object # Reduce the object list to find the reported user. defp get_reported(objects) do @@ -501,7 +499,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do options ) when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do - data = Map.put(data, "object", strip_internal_fields(data["object"])) + fetch_options = Keyword.put(options, :depth, (options[:depth] || 0) + 1) + + object = + data["object"] + |> strip_internal_fields() + |> fix_type(fetch_options) + |> fix_in_reply_to(fetch_options) + + data = Map.put(data, "object", object) options = Keyword.put(options, :local, false) with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), From 0b88accae632e371becacb16be4e8798aa80c705 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 21 Oct 2020 01:20:06 +0200 Subject: [PATCH 048/150] fetcher_test: Fix missing mock function --- test/pleroma/object/fetcher_test.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index a7ac90348..8d9c6c3cb 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -66,6 +66,14 @@ defmodule Pleroma.Object.FetcherTest do %Tesla.Env{ status: 500 } + + %{ + method: :get, + url: "https://stereophonic.space/objects/02997b83-3ea7-4b63-94af-ef3aa2d4ed17" + } -> + %Tesla.Env{ + status: 500 + } end) :ok From 53193b84b1d07c9fd3c6b80c04e3eada4fb4cd59 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 27 Nov 2020 00:25:24 +0100 Subject: [PATCH 049/150] =?UTF-8?q?utils:=20Fix=20maybe=5Fsplice=5Frecipie?= =?UTF-8?q?nt=20when=20"object"=20isn=E2=80=99t=20a=20map?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/pleroma/maps.ex | 6 ++++++ lib/pleroma/web/activity_pub/utils.ex | 6 +++--- test/pleroma/web/activity_pub/activity_pub_controller_test.exs | 1 - 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/maps.ex b/lib/pleroma/maps.ex index 0d2e94248..b08b83305 100644 --- a/lib/pleroma/maps.ex +++ b/lib/pleroma/maps.ex @@ -12,4 +12,10 @@ defmodule Pleroma.Maps do _ -> map end end + + def safe_put_in(data, keys, value) when is_map(data) and is_list(keys) do + Kernel.put_in(data, keys, value) + rescue + _ -> data + end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index e81623d83..0d1a6d0f1 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -96,11 +96,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do !label_in_collection?(ap_id, params["cc"]) if need_splice? do - cc_list = extract_list(params["cc"]) + cc = [ap_id | extract_list(params["cc"])] params - |> Map.put("cc", [ap_id | cc_list]) - |> Kernel.put_in(["object", "cc"], [ap_id | cc_list]) + |> Map.put("cc", cc) + |> Maps.safe_put_in(["object", "cc"], cc) else params end diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index f6ea9e2ca..f3ce703e2 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -1003,7 +1003,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do "actor" => remote_actor, "content" => "test report", "id" => "https://#{remote_domain}/e3b12fd1-948c-446e-b93b-a5e67edbe1d8", - "nickname" => reported_user.nickname, "object" => [ reported_user.ap_id, note.data["object"] From 6d6bef64bf3b37457b71cf7025e84aa9017a3b86 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 25 Mar 2021 10:17:26 +0100 Subject: [PATCH 050/150] fetcher_test: Remove assert on fake Create having an ap_id --- test/pleroma/object/fetcher_test.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 8d9c6c3cb..bd0a6e497 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -132,8 +132,7 @@ defmodule Pleroma.Object.FetcherTest do {:ok, object} = Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] + assert _activity = Activity.get_create_by_object_ap_id(object.data["id"]) {:ok, object_again} = Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") From 5ef4659b373ae1106090952ff3e963b419fa1d72 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 5 Apr 2021 18:57:14 +0200 Subject: [PATCH 051/150] test/pleroma/web/common_api_test.exs: Strip : around emoji key-name --- test/pleroma/web/common_api_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 6619f8fc8..86c12f0b2 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -539,8 +539,8 @@ defmodule Pleroma.Web.CommonAPITest do spoiler_text: ":joker_smile:" }) - assert Object.normalize(reply_activity).data["emoji"][":joker_smile:"] - refute Object.normalize(reply_activity).data["emoji"][":joker_disapprove:"] + assert Object.normalize(reply_activity).data["emoji"]["joker_smile"] + refute Object.normalize(reply_activity).data["emoji"]["joker_disapprove"] end test "deactivated users can't post" do From 681a42c359b4fbae74285363c670dff18aac5918 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 8 Apr 2021 15:45:31 +0300 Subject: [PATCH 052/150] release runtime provider fix for paths --- lib/pleroma/config/release_runtime_provider.ex | 13 +++++++++---- mix.exs | 13 +------------ test/pleroma/config/release_runtime_provider_test.exs | 1 - 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 46fa35559..e5e9d3dcd 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -11,10 +11,11 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do def load(config, opts) do with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults()) - config_path = opts[:config_path] + config_path = + opts[:config_path] || System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" with_runtime_config = - if config_path && File.exists?(config_path) do + if File.exists?(config_path) do runtime_config = Config.Reader.read!(config_path) with_defaults @@ -32,10 +33,14 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do with_defaults end - exported_config_path = opts[:exported_config_path] + exported_config_path = + opts[:exported_config_path] || + config_path + |> Path.dirname() + |> Path.join("#{Pleroma.Config.get(:env)}.exported_from_db.secret.exs") with_exported = - if exported_config_path && File.exists?(exported_config_path) do + if File.exists?(exported_config_path) do exported_config = Config.Reader.read!(exported_config_path) Config.Reader.merge(with_runtime_config, exported_config) else diff --git a/mix.exs b/mix.exs index 7328b533b..fe5d9d963 100644 --- a/mix.exs +++ b/mix.exs @@ -38,7 +38,7 @@ defmodule Pleroma.Mixfile do include_executables_for: [:unix], applications: [ex_syslogger: :load, syslog: :load, eldap: :transient], steps: [:assemble, &put_otp_version/1, ©_files/1, ©_nginx_config/1], - config_providers: [{Pleroma.Config.ReleaseRuntimeProvider, release_config_paths()}] + config_providers: [{Pleroma.Config.ReleaseRuntimeProvider, []}] ] ] ] @@ -67,17 +67,6 @@ defmodule Pleroma.Mixfile do release end - defp release_config_paths do - config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" - - exported_config_path = - config_path - |> Path.dirname() - |> Path.join("#{Mix.env()}.exported_from_db.secret.exs") - - [config_path: config_path, exported_config_path: exported_config_path] - end - # Configuration for the OTP application. # # Type `mix help compile.app` for more information. diff --git a/test/pleroma/config/release_runtime_provider_test.exs b/test/pleroma/config/release_runtime_provider_test.exs index 1921698c5..6578d3268 100644 --- a/test/pleroma/config/release_runtime_provider_test.exs +++ b/test/pleroma/config/release_runtime_provider_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Config.ReleaseRuntimeProviderTest do ExUnit.CaptureIO.capture_io(fn -> merged = ReleaseRuntimeProvider.load([], []) assert merged == Pleroma.Config.Holder.release_defaults() - IO.inspect(merged) end) =~ "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file" end From c6dcd863e28531e0d21ee64a8387bd27c2c0ed31 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 16 Apr 2021 09:59:50 +0000 Subject: [PATCH 053/150] Apply rinpatch's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 62c9826f6..decb6572a 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def change_password_operation do %Operation{ - tags: ["Accounts"], + tags: ["Account credentials"], summary: "Change account password", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.change_password", From 2b4f958b2ad653ee8e294ade18aa4482e4d372e1 Mon Sep 17 00:00:00 2001 From: Sean King Date: Sun, 18 Apr 2021 14:00:18 -0600 Subject: [PATCH 054/150] Add opting out of Google FLoC to HTTPSecurityPlug headers --- lib/pleroma/web/plugs/http_security_plug.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 0025b042a..d1e6cc9d3 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -48,7 +48,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do {"x-content-type-options", "nosniff"}, {"referrer-policy", referrer_policy}, {"x-download-options", "noopen"}, - {"content-security-policy", csp_string()} + {"content-security-policy", csp_string()}, + {"permissions-policy", "interest-cohort=()"} ] headers = From efed94a23e30260bcf1b297910906b11d6e4d895 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 19 Apr 2021 16:23:57 -0500 Subject: [PATCH 055/150] Fix error response which was breaking tests related to pinned posts --- lib/pleroma/web/common_api.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 3970c19a8..1b5f8491e 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -415,7 +415,7 @@ defmodule Pleroma.Web.CommonAPI do ) do {:ok, activity} else - {:error, {:execute_side_effects, error}} -> error + {:error, {:side_effects, error}} -> error error -> error end end From 2780cdd4e7acde0f4bf4719b7c82bc7e2d1bf3b5 Mon Sep 17 00:00:00 2001 From: Sean King Date: Mon, 19 Apr 2021 16:06:19 -0600 Subject: [PATCH 056/150] Add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b0678023..bfa76a89a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change. +- HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising. ### Added From 7eded7218922b46c5cc085e715b6031ffff9b6ce Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 20 Apr 2021 12:31:14 -0500 Subject: [PATCH 057/150] Fix incorrect shell command Can't be in /opt/pleroma/bin and then call ./bin/pleroma_ctl :) --- docs/installation/otp_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 42e264e65..13f9636f3 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -290,7 +290,7 @@ nginx -t ## Create your first user and set as admin ```sh -cd /opt/pleroma/bin +cd /opt/pleroma su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin" ``` This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password. From 30b1d5093808974310a52917e6ab85d528683fae Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 20 Apr 2021 21:06:32 +0000 Subject: [PATCH 058/150] Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index decb6572a..6ddc93a92 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def change_email_operation do %Operation{ - tags: ["Accounts"], + tags: ["Account credentials"], summary: "Change account email", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.change_email", From e104829c2f5b3eae9133ea1a6a81d138c3a8e314 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 20 Apr 2021 21:06:39 +0000 Subject: [PATCH 059/150] Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 6ddc93a92..dbed1b518 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -143,7 +143,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def disable_account_operation do %Operation{ - tags: ["Accounts"], + tags: ["Account credentials"], summary: "Disable Account", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.disable_account", From 42185d87504ea595138e8e3f5bf9ce6840edd2f1 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 20 Apr 2021 21:06:45 +0000 Subject: [PATCH 060/150] Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index dbed1b518..6e6d330de 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -163,7 +163,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def delete_account_operation do %Operation{ - tags: ["Accounts"], + tags: ["Account credentials"], summary: "Delete Account", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.delete_account", From f9bedf5597dd00ce4f429a4077e7bb4473c97410 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Tue, 20 Apr 2021 21:08:31 +0000 Subject: [PATCH 061/150] Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 6e6d330de..0cafbc719 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -195,7 +195,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do def healthcheck_operation do %Operation{ tags: ["Accounts"], - summary: "Disable Account", + summary: "Quick status check on the instance", security: [%{"oAuth" => ["write:accounts"]}], operationId: "UtilController.healthcheck", parameters: [], From 0effcd2cfed36baec1d960b64c901da7e56710a8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 19 Apr 2021 15:43:17 -0500 Subject: [PATCH 062/150] Set Repo.transaction/2 timeout to infinity. Fixes pleroma.user delete_activities mix task. --- lib/pleroma/web/activity_pub/pipeline.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 377eccb92..400823094 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()} def common_pipeline(object, meta) do - case Repo.transaction(fn -> do_common_pipeline(object, meta) end) do + case Repo.transaction(fn -> do_common_pipeline(object, meta) end, timeout: :infinity) do {:ok, {:ok, activity, meta}} -> @side_effects.handle_after_transaction(meta) {:ok, activity, meta} From 9bc69196d5dfbd3fb37c0e62da19ce08fb9bf28d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 20 Apr 2021 11:10:39 -0500 Subject: [PATCH 063/150] Add utility function to return infinite timeout for SQL transactions if we detect it was called from a Mix Task --- lib/pleroma/utils.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index bc0c95332..5e2fa8bf7 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -63,4 +63,13 @@ defmodule Pleroma.Utils do end def posix_error_message(_), do: "" + + def query_timeout do + {parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2) + + cond do + parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity] + true -> [timeout: 15_000] + end + end end From 9f711ddcf84bdb5a5680e1b55afa83768014906d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 20 Apr 2021 11:16:24 -0500 Subject: [PATCH 064/150] Try to set query timeout intelligently --- lib/pleroma/web/activity_pub/pipeline.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 400823094..a0f2e0312 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do alias Pleroma.Config alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Utils alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.ObjectValidator @@ -24,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do @spec common_pipeline(map(), keyword()) :: {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()} def common_pipeline(object, meta) do - case Repo.transaction(fn -> do_common_pipeline(object, meta) end, timeout: :infinity) do + case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do {:ok, {:ok, activity, meta}} -> @side_effects.handle_after_transaction(meta) {:ok, activity, meta} From 99fd9c5e38ad08973f435f1a67d6af60d004c578 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 20 Apr 2021 12:00:02 -0500 Subject: [PATCH 065/150] OTP releases executing commands via pleroma_ctl show the parent of the process is :erl_eval --- lib/pleroma/utils.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 5e2fa8bf7..55aecc509 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -69,6 +69,7 @@ defmodule Pleroma.Utils do cond do parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity] + parent == :erl_eval -> [timeout: :infinity] true -> [timeout: 15_000] end end From 959dc6e6fc95b33700fb7e08689afb701b2877f2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 22 Apr 2021 10:11:08 -0500 Subject: [PATCH 066/150] Cleanup and ensure we obey custom Repo timeout --- lib/pleroma/utils.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 55aecc509..a446d3ae6 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Utils do eperm epipe erange erofs espipe esrch estale etxtbsy exdev )a + @repo_timeout Pleroma.Config.get([Pleroma.Repo, :timeout], 15_000) + def compile_dir(dir) when is_binary(dir) do dir |> File.ls!() @@ -64,13 +66,20 @@ defmodule Pleroma.Utils do def posix_error_message(_), do: "" + @doc """ + Returns [timeout: integer] suitable for passing as an option to Repo functions. + + This function detects if the execution was triggered from IEx shell, Mix task, or + ./bin/pleroma_ctl and sets the timeout to :infinity, else returns the default timeout value. + """ + @spec query_timeout() :: [timeout: integer] def query_timeout do {parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2) cond do parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity] parent == :erl_eval -> [timeout: :infinity] - true -> [timeout: 15_000] + true -> [timeout: @repo_timeout] end end end From d7a71a275abea6286ee116d092ddc9440a9419a5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 22 Apr 2021 10:15:05 -0500 Subject: [PATCH 067/150] Fixed pleroma.user delete_activities mix task. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfa76a89a..a1173414d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Uploading custom instance thumbnail via AdminAPI/AdminFE generated invalid URL to the image - Applying ConcurrentLimiter settings via AdminAPI - User login failures if their `notification_settings` were in a NULL state. +- Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity ## [2.3.0] - 2020-03-01 From b9a99ac0d4b245ff3df6a9aa1b4db46ee75e9d22 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 27 Apr 2021 11:54:28 -0500 Subject: [PATCH 068/150] Cache gitlab-ci based on mix.lock --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2bc571971..2651ff9e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,9 @@ variables: &global_variables MIX_ENV: test cache: &global_cache_policy - key: ${CI_COMMIT_REF_SLUG} + key: + files: + - mix.lock paths: - deps - _build @@ -171,8 +173,8 @@ spec-deploy: - apk add curl script: - curl -X POST -F"token=$API_DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" -F"variables[JOB_REF]=$CI_JOB_ID" https://git.pleroma.social/api/v4/projects/1130/trigger/pipeline - - + + stop_review_app: image: alpine:3.9 stage: deploy @@ -231,7 +233,7 @@ amd64-musl: stage: release artifacts: *release-artifacts only: *release-only - image: elixir:1.10.3-alpine + image: elixir:1.10.3-alpine cache: *release-cache variables: *release-variables before_script: &before-release-musl From 8c1d6e88395e1d7ada9d86236a7fa2339d9097e9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 29 Apr 2021 12:20:46 -0500 Subject: [PATCH 069/150] CHANGELOG: Return OAuth token `id` --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1173414d..9a0171763 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded. +- Return OAuth token `id` (primary key) in POST `/oauth/token`. ## Unreleased (Patch) From b5ae8268982524a0a4fd295ddef64e4983832489 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 29 Apr 2021 13:03:41 -0500 Subject: [PATCH 070/150] CI: Purge pleroma build directory between runs --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2651ff9e6..78e715d47 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ stages: - docker before_script: + - rm -rf _build/*/lib/pleroma - apt-get update && apt-get install -y cmake - mix local.hex --force - mix local.rebar --force @@ -31,6 +32,9 @@ before_script: - apt-get -qq update - apt-get install -y libmagic-dev +after_script: + - rm -rf _build/*/lib/pleroma + build: stage: build script: From 004bcedb074d50bc42803e4c0a884239bd504b3d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:23:11 -0500 Subject: [PATCH 071/150] Upgrade Earmark 1.4.15 --- mix.exs | 2 +- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index 06d77edb7..8ba2d8fbc 100644 --- a/mix.exs +++ b/mix.exs @@ -144,7 +144,7 @@ defmodule Pleroma.Mixfile do {:ex_aws, "~> 2.1.6"}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, - {:earmark, "1.4.13"}, + {:earmark, "1.4.15"}, {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, git: "https://github.com/msantos/crypt.git", diff --git a/mix.lock b/mix.lock index e4dd32c83..06542f18d 100644 --- a/mix.lock +++ b/mix.lock @@ -27,8 +27,8 @@ "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.13", "2c6ce9768fc9fdbf4046f457e207df6360ee6c91ee1ecb8e9a139f96a4289d91", [:mix], [{:earmark_parser, ">= 1.4.12", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "a0cf3ed88ef2b1964df408889b5ecb886d1a048edde53497fc935ccd15af3403"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"}, + "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"}, From 6727a3659f60c0e09fa6375b6c0843c01f5be3dc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:27:06 -0500 Subject: [PATCH 072/150] Remove Pleroma.Formatter.minify/2 --- lib/pleroma/formatter.ex | 11 ----------- .../activity_pub/object_validators/audio_video_validator.ex | 1 - lib/pleroma/web/common_api/utils.ex | 1 - test/pleroma/formatter_test.exs | 7 ------- 4 files changed, 20 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 2aa236ca9..baf652a5a 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -142,17 +142,6 @@ defmodule Pleroma.Formatter do |> Enum.join("") end - def minify({text, mentions, hashtags}, type) do - {minify(text, type), mentions, hashtags} - end - - def minify(text, "text/html") do - text - |> String.replace(">\n", ">") - |> String.replace("> ", ">") - |> String.replace(" <", "<") - end - def truncate(text, max_length \\ 200, omission \\ "...") do # Remove trailing whitespace text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}") diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index fa3e2c026..9b38aa4c2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -96,7 +96,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do content = content |> Pleroma.Formatter.markdown_to_html() - |> Pleroma.Formatter.minify("text/html") |> Pleroma.HTML.filter_tags() Map.put(data, "content", content) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index be86009af..4731e79be 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -296,7 +296,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do |> Formatter.mentions_escape(options) |> Formatter.markdown_to_html() |> Formatter.linkify(options) - |> Formatter.minify("text/html") |> Formatter.html_escape("text/html") end diff --git a/test/pleroma/formatter_test.exs b/test/pleroma/formatter_test.exs index ceedd1b6d..5781a3f01 100644 --- a/test/pleroma/formatter_test.exs +++ b/test/pleroma/formatter_test.exs @@ -307,11 +307,4 @@ defmodule Pleroma.FormatterTest do assert Formatter.html_escape(text, "text/plain") == expected end - - test "it minifies html" do - text = "

    \nhello

    \n

    \nworld

    \n" - expected = "

    hello

    world

    " - - assert Formatter.minify(text, "text/html") == expected - end end From 53760d2cda9b9f241355365b3fff9852bcb1a8a2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:51:18 -0500 Subject: [PATCH 073/150] Delete obsolete EarmarkRendereTests (moved to UtilsTest) --- test/pleroma/earmark_renderer_test.exs | 79 ---------------------------------- 1 file changed, 79 deletions(-) delete mode 100644 test/pleroma/earmark_renderer_test.exs diff --git a/test/pleroma/earmark_renderer_test.exs b/test/pleroma/earmark_renderer_test.exs deleted file mode 100644 index 3adbefc1e..000000000 --- a/test/pleroma/earmark_renderer_test.exs +++ /dev/null @@ -1,79 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.EarmarkRendererTest do - use ExUnit.Case - - test "Paragraph" do - code = ~s[Hello\n\nWorld!] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    Hello

    World!

    " - end - - test "raw HTML" do - code = ~s[OwO] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    #{code}

    " - end - - test "rulers" do - code = ~s[before\n\n-----\n\nafter] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    before

    after

    " - end - - test "headings" do - code = ~s[# h1\n## h2\n### h3\n] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    h1

    h2

    h3

    ] - end - - test "blockquote" do - code = ~s[> whoms't are you quoting?] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    whoms’t are you quoting?

    " - end - - test "code" do - code = ~s[`mix`] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    mix

    ] - - code = ~s[``mix``] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    mix

    ] - - code = ~s[```\nputs "Hello World"\n```] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[
    puts "Hello World"
    ] - end - - test "lists" do - code = ~s[- one\n- two\n- three\n- four] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "
    • one
    • two
    • three
    • four
    " - - code = ~s[1. one\n2. two\n3. three\n4. four\n] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "
    1. one
    2. two
    3. three
    4. four
    " - end - - test "delegated renderers" do - code = ~s[a
    b] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == "

    #{code}

    " - - code = ~s[*aaaa~*] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    aaaa~

    ] - - code = ~s[**aaaa~**] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    aaaa~

    ] - - # strikethrought - code = ~s[aaaa~] - result = Pleroma.Formatter.markdown_to_html(code) - assert result == ~s[

    aaaa~

    ] - end -end From a8fa00ef666f574aec8048626aed78a7d62e6915 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 30 Apr 2021 12:55:43 -0500 Subject: [PATCH 074/150] Fix failing remote mentions test, valid TLDs --- test/pleroma/web/common_api/utils_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs index 28b05ed91..8c79a9a83 100644 --- a/test/pleroma/web/common_api/utils_test.exs +++ b/test/pleroma/web/common_api/utils_test.exs @@ -209,10 +209,10 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do end test "remote mentions" do - mario = insert(:user, %{nickname: "mario@mushroom.kingdom", local: false}) - luigi = insert(:user, %{nickname: "luigi@mushroom.kingdom", local: false}) + mario = insert(:user, %{nickname: "mario@mushroom.world", local: false}) + luigi = insert(:user, %{nickname: "luigi@mushroom.world", local: false}) - code = "@mario@mushroom.kingdom @luigi@mushroom.kingdom yo what's up?" + code = "@mario@mushroom.world @luigi@mushroom.world yo what's up?" {result, _, []} = Utils.format_input(code, "text/markdown") assert result == From 3d742c3c1af69a9526c12a171663630b3439b5cc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 18 Mar 2021 15:31:50 -0500 Subject: [PATCH 075/150] SimplePolicy: filter nested objects --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 11 ++++++++++- test/pleroma/web/activity_pub/mrf/simple_policy_test.exs | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index bb3838d2c..b3e5d814d 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -181,6 +181,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_banner_removal(_actor_info, object), do: {:ok, object} + defp check_object(%{"object" => object} = activity) when is_map(object) do + with {:ok, _object} <- filter(object) do + {:ok, activity} + end + end + + defp check_object(object), do: {:ok, object} + @impl true def filter(%{"type" => "Delete", "actor" => actor} = object) do %{host: actor_host} = URI.parse(actor) @@ -206,7 +214,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_media_nsfw(actor_info, object), {:ok, object} <- check_ftl_removal(actor_info, object), {:ok, object} <- check_followers_only(actor_info, object), - {:ok, object} <- check_report_removal(actor_info, object) do + {:ok, object} <- check_report_removal(actor_info, object), + {:ok, object} <- check_object(object) do {:ok, object} else {:reject, nil} -> {:reject, "[SimplePolicy]"} diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index f48e5b39b..b6d9f2ded 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -260,6 +260,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert {:reject, _} = SimplePolicy.filter(remote_user) end + + test "reject Announce when object would be rejected" do + clear_config([:mrf_simple, :reject], ["blocked.tld"]) + + announce = %{ + "type" => "Announce", + "actor" => "https://okay.tld/users/alice", + "object" => %{"type" => "Note", "actor" => "https://blocked.tld/users/bob"} + } + + assert {:reject, _} = SimplePolicy.filter(announce) + end end describe "when :followers_only" do From c16c7fdb8794df8558cf8fbe4231d8f9ec01bb6d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 29 Apr 2021 11:51:49 -0500 Subject: [PATCH 076/150] SimplePolicy: filter string Objects --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 15 ++++++++++++++- test/pleroma/web/activity_pub/mrf/simple_policy_test.exs | 12 ++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index b3e5d814d..b07d70401 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -181,7 +181,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_banner_removal(_actor_info, object), do: {:ok, object} - defp check_object(%{"object" => object} = activity) when is_map(object) do + defp check_object(%{"object" => object} = activity) do with {:ok, _object} <- filter(object) do {:ok, activity} end @@ -240,6 +240,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end end + def filter(object) when is_binary(object) do + uri = URI.parse(object) + + with {:ok, object} <- check_accept(uri, object), + {:ok, object} <- check_reject(uri, object) do + {:ok, object} + else + {:reject, nil} -> {:reject, "[SimplePolicy]"} + {:reject, _} = e -> e + _ -> {:reject, "[SimplePolicy]"} + end + end + def filter(object), do: {:ok, object} @impl true diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index b6d9f2ded..8024a2459 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -272,6 +272,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert {:reject, _} = SimplePolicy.filter(announce) end + + test "reject by URI object" do + clear_config([:mrf_simple, :reject], ["blocked.tld"]) + + announce = %{ + "type" => "Announce", + "actor" => "https://okay.tld/users/alice", + "object" => "https://blocked.tld/activities/1" + } + + assert {:reject, _} = SimplePolicy.filter(announce) + end end describe "when :followers_only" do From 20878c7f9913e1501821356f24e97c2c42b00a41 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 2 Apr 2021 12:18:35 -0500 Subject: [PATCH 077/150] CHANGELOG: SimplePolicy embedded objects are now checked --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a0171763..150cd4147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Applying ConcurrentLimiter settings via AdminAPI - User login failures if their `notification_settings` were in a NULL state. - Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity +- MRF (`SimplePolicy`): Embedded objects are now checked. If any embedded object would be rejected, its parent is rejected. This fixes Announces leaking posts from blocked domains. ## [2.3.0] - 2020-03-01 From dca87c5e7b4b12918cf59a83a77be389a7e0df01 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 1 May 2021 11:28:06 -0500 Subject: [PATCH 078/150] CHANGELOG: markdown --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a0171763..ed6e548dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Applying ConcurrentLimiter settings via AdminAPI - User login failures if their `notification_settings` were in a NULL state. - Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity +- Fixed some Markdown issues, including trailing slash in links. ## [2.3.0] - 2020-03-01 From c80b1aaf514dec6b538a9833d48df027708b6b4d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 May 2021 14:27:03 -0500 Subject: [PATCH 079/150] Don't crash when email settings are invalid Fixes: https://git.pleroma.social/pleroma/pleroma/-/issues/2606 Fixes: https://gitlab.com/soapbox-pub/soapbox/-/issues/4 --- lib/pleroma/application_requirements.ex | 38 ++++++++++++++------------ test/pleroma/application_requirements_test.exs | 18 ++++++------ test/pleroma/user_test.exs | 18 ++++++++++++ 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 6ef65b263..c412dec5e 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -34,15 +34,16 @@ defmodule Pleroma.ApplicationRequirements do defp check_welcome_message_config!(:ok) do if Pleroma.Config.get([:welcome, :email, :enabled], false) and not Pleroma.Emails.Mailer.enabled?() do - Logger.error(""" - To send welcome email do you need to enable mail. - \nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true - """) + Logger.warn(""" + To send welcome emails, you need to enable the mailer. + Welcome emails will NOT be sent with the current config. - {:error, "The mail disabled."} - else - :ok + Enable the mailer: + config :pleroma, Pleroma.Emails.Mailer, enabled: true + """) end + + :ok end defp check_welcome_message_config!(result), do: result @@ -51,18 +52,21 @@ defmodule Pleroma.ApplicationRequirements do # def check_confirmation_accounts!(:ok) do if Pleroma.Config.get([:instance, :account_activation_required]) && - not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do - Logger.error( - "Account activation enabled, but no Mailer settings enabled.\n" <> - "Please set config :pleroma, :instance, account_activation_required: false\n" <> - "Otherwise setup and enable Mailer." - ) + not Pleroma.Emails.Mailer.enabled?() do + Logger.warn(""" + Account activation is required, but the mailer is disabled. + Users will NOT be able to confirm their accounts with this config. + Either disable account activation or enable the mailer. - {:error, - "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."} - else - :ok + Disable account activation: + config :pleroma, :instance, account_activation_required: false + + Enable the mailer: + config :pleroma, Pleroma.Emails.Mailer, enabled: true + """) end + + :ok end def check_confirmation_accounts!(result), do: result diff --git a/test/pleroma/application_requirements_test.exs b/test/pleroma/application_requirements_test.exs index 683ac8c96..a54c37968 100644 --- a/test/pleroma/application_requirements_test.exs +++ b/test/pleroma/application_requirements_test.exs @@ -35,13 +35,13 @@ defmodule Pleroma.ApplicationRequirementsTest do setup do: clear_config([:welcome]) setup do: clear_config([Pleroma.Emails.Mailer]) - test "raises if welcome email enabled but mail disabled" do + test "warns if welcome email enabled but mail disabled" do clear_config([:welcome, :email, :enabled], true) clear_config([Pleroma.Emails.Mailer, :enabled], false) - assert_raise Pleroma.ApplicationRequirements.VerifyError, "The mail disabled.", fn -> - capture_log(&Pleroma.ApplicationRequirements.verify!/0) - end + assert capture_log(fn -> + assert Pleroma.ApplicationRequirements.verify!() == :ok + end) =~ "Welcome emails will NOT be sent" end end @@ -57,15 +57,13 @@ defmodule Pleroma.ApplicationRequirementsTest do setup do: clear_config([:instance, :account_activation_required]) - test "raises if account confirmation is required but mailer isn't enable" do + test "warns if account confirmation is required but mailer isn't enabled" do clear_config([:instance, :account_activation_required], true) clear_config([Pleroma.Emails.Mailer, :enabled], false) - assert_raise Pleroma.ApplicationRequirements.VerifyError, - "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails.", - fn -> - capture_log(&Pleroma.ApplicationRequirements.verify!/0) - end + assert capture_log(fn -> + assert Pleroma.ApplicationRequirements.verify!() == :ok + end) =~ "Users will NOT be able to confirm their accounts" end test "doesn't do anything if account confirmation is disabled" do diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 6f5bcab57..f89ea458a 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -572,6 +572,24 @@ defmodule Pleroma.UserTest do ) end + test "it fails gracefully with invalid email config" do + cng = User.register_changeset(%User{}, @full_user_data) + + # Disable the mailer but enable all the things that want to send emails + clear_config([Pleroma.Emails.Mailer, :enabled], false) + clear_config([:instance, :account_activation_required], true) + clear_config([:instance, :account_approval_required], true) + clear_config([:welcome, :email, :enabled], true) + clear_config([:welcome, :email, :sender], "lain@lain.com") + + # The user is still created + assert {:ok, %User{nickname: "nick"}} = User.register(cng) + + # No emails are sent + ObanHelpers.perform_all() + refute_email_sent() + end + test "it requires an email, name, nickname and password, bio is optional when account_activation_required is enabled" do clear_config([:instance, :account_activation_required], true) From 90770e0841d3ffea87627b35627bfe38cad52f07 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 May 2021 14:30:21 -0500 Subject: [PATCH 080/150] CHANGELOG: don't crash so hard when email settings are invalid --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a0171763..74086a54b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded. - Return OAuth token `id` (primary key) in POST `/oauth/token`. +### Fixed +- Don't crash so hard when email settings are invalid. + ## Unreleased (Patch) ### Fixed From ab9eabdf20180f2dd8539cf5d3dc0fdc6412496b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 13:38:11 -0500 Subject: [PATCH 081/150] Add SetMeta filter to store uploaded image sizes --- lib/pleroma/upload.ex | 9 +++++- lib/pleroma/upload/filter/set_meta.ex | 36 ++++++++++++++++++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 16 ++++++++++ test/pleroma/upload/filter/set_meta_test.exs | 19 ++++++++++++ .../web/mastodon_api/views/status_view_test.exs | 5 ++- 5 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/upload/filter/set_meta.ex create mode 100644 test/pleroma/upload/filter/set_meta_test.exs diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 654711351..4d58abd48 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -23,6 +23,8 @@ defmodule Pleroma.Upload do is once created permanent and changing it (especially in uploaders) is probably a bad idea! * `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over. + * `:width` - width of the media in pixels + * `:height` - height of the media in pixels Related behaviors: @@ -32,6 +34,7 @@ defmodule Pleroma.Upload do """ alias Ecto.UUID alias Pleroma.Config + alias Pleroma.Maps require Logger @type source :: @@ -53,9 +56,11 @@ defmodule Pleroma.Upload do name: String.t(), tempfile: String.t(), content_type: String.t(), + width: integer(), + height: integer(), path: String.t() } - defstruct [:id, :name, :tempfile, :content_type, :path] + defstruct [:id, :name, :tempfile, :content_type, :width, :height, :path] defp get_description(opts, upload) do case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do @@ -89,6 +94,8 @@ defmodule Pleroma.Upload do "mediaType" => upload.content_type, "href" => url_from_spec(upload, opts.base_url, url_spec) } + |> Maps.put_if_present("width", upload.width) + |> Maps.put_if_present("height", upload.height) ], "name" => description }} diff --git a/lib/pleroma/upload/filter/set_meta.ex b/lib/pleroma/upload/filter/set_meta.ex new file mode 100644 index 000000000..cccb6c371 --- /dev/null +++ b/lib/pleroma/upload/filter/set_meta.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.SetMeta do + @moduledoc """ + Extracts metadata about the upload, such as width/height + """ + require Logger + + @behaviour Pleroma.Upload.Filter + + @spec filter(Pleroma.Upload.t()) :: + {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do + try do + image = + file + |> Mogrify.open() + |> Mogrify.verbose() + + upload = + upload + |> Map.put(:width, image.width) + |> Map.put(:height, image.height) + + {:ok, :filtered, upload} + rescue + e in ErlangError -> + Logger.warn("#{__MODULE__}: #{inspect(e)}") + {:ok, :noop} + end + end + + def filter(_), do: {:ok, :noop} +end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index bac897a57..5dbdc309e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -426,10 +426,26 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do type: type, description: attachment["name"], pleroma: %{mime_type: media_type}, + meta: render("attachment_meta.json", %{attachment: attachment}), blurhash: attachment["blurhash"] } end + def render("attachment_meta.json", %{ + attachment: %{"url" => [%{"width" => width, "height" => height} | _]} + }) + when is_integer(width) and is_integer(height) do + %{ + original: %{ + width: width, + height: height, + aspect: width / height + } + } + end + + def render("attachment_meta.json", _), do: %{} + def render("context.json", %{activity: activity, activities: activities, user: user}) do %{ancestors: ancestors, descendants: descendants} = activities diff --git a/test/pleroma/upload/filter/set_meta_test.exs b/test/pleroma/upload/filter/set_meta_test.exs new file mode 100644 index 000000000..650e527b4 --- /dev/null +++ b/test/pleroma/upload/filter/set_meta_test.exs @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.SetMetaTest do + use Pleroma.DataCase, async: true + alias Pleroma.Upload.Filter.SetMeta + + test "adds the image dimensions" do + upload = %Pleroma.Upload{ + name: "an… image.jpg", + content_type: "image/jpeg", + path: Path.absname("test/fixtures/image.jpg"), + tempfile: Path.absname("test/fixtures/image.jpg") + } + + assert {:ok, :filtered, %{width: 1024, height: 768}} = SetMeta.filter(upload) + end +end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 2de3afc4f..e6c37e782 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -458,7 +458,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do "url" => [ %{ "mediaType" => "image/png", - "href" => "someurl" + "href" => "someurl", + "width" => 200, + "height" => 100 } ], "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", @@ -474,6 +476,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do text_url: "someurl", description: nil, pleroma: %{mime_type: "image/png"}, + meta: %{original: %{width: 200, height: 100, aspect: 2}}, blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" } From 4c060ae73371a8567468186e5d1333ec00fbdf41 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 15:38:49 -0500 Subject: [PATCH 082/150] Ingest remote attachment width/height --- .../object_validators/attachment_validator.ex | 4 ++- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 ++ .../attachment_validator_test.exs | 33 ++++++++++++++++++++++ .../transmogrifier/audio_handling_test.exs | 4 ++- .../transmogrifier/video_handling_test.exs | 12 ++++++-- 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 3175427ad..a99b40adc 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -21,6 +21,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string) field(:href, ObjectValidators.Uri) field(:mediaType, :string, default: "application/octet-stream") + field(:width, :integer) + field(:height, :integer) end end @@ -52,7 +54,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do data = fix_media_type(data) struct - |> cast(data, [:type, :href, :mediaType]) + |> cast(data, [:type, :href, :mediaType, :width, :height]) |> validate_inclusion(:type, ["Link"]) |> validate_required([:type, :href, :mediaType]) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 4d9a5617e..b5767863c 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -245,6 +245,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "type" => Map.get(url || %{}, "type", "Link") } |> Maps.put_if_present("mediaType", media_type) + |> Maps.put_if_present("width", (url || %{})["width"]) + |> Maps.put_if_present("height", (url || %{})["height"]) %{ "url" => [attachment_url], diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs index b775515e0..0e49fda99 100644 --- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs @@ -72,5 +72,38 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidatorTest do assert attachment.mediaType == "image/jpeg" end + + test "it handles image dimensions" do + attachment = %{ + "url" => [ + %{ + "type" => "Link", + "mediaType" => "image/jpeg", + "href" => "https://example.com/images/1.jpg", + "width" => 200, + "height" => 100 + } + ], + "type" => "Document", + "name" => nil, + "mediaType" => "image/jpeg" + } + + {:ok, attachment} = + AttachmentValidator.cast_and_validate(attachment) + |> Ecto.Changeset.apply_action(:insert) + + assert [ + %{ + href: "https://example.com/images/1.jpg", + type: "Link", + mediaType: "image/jpeg", + width: 200, + height: 100 + } + ] = attachment.url + + assert attachment.mediaType == "image/jpeg" + end end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs index e733f167d..a21e9e3d3 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs @@ -76,7 +76,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AudioHandlingTest do "href" => "https://channels.tests.funkwhale.audio/api/v1/listen/3901e5d8-0445-49d5-9711-e096cf32e515/?upload=42342395-0208-4fee-a38d-259a6dae0871&download=false", "mediaType" => "audio/ogg", - "type" => "Link" + "type" => "Link", + "width" => nil, + "height" => nil } ] } diff --git a/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs index 6ddf7c172..62b4a2cb3 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs @@ -60,7 +60,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.VideoHandlingTest do "href" => "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4", "mediaType" => "video/mp4", - "type" => "Link" + "type" => "Link", + "width" => nil, + "height" => nil } ] } @@ -83,7 +85,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.VideoHandlingTest do "href" => "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4", "mediaType" => "video/mp4", - "type" => "Link" + "type" => "Link", + "width" => nil, + "height" => nil } ] } @@ -113,7 +117,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.VideoHandlingTest do "href" => "https://peertube.stream/static/streaming-playlists/hls/abece3c3-b9c6-47f4-8040-f3eed8c602e6/abece3c3-b9c6-47f4-8040-f3eed8c602e6-1080-fragmented.mp4", "mediaType" => "video/mp4", - "type" => "Link" + "type" => "Link", + "width" => nil, + "height" => nil } ] } From 02b9436494998e441fe2119b78c0e4f68c45a9e1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 16:16:10 -0500 Subject: [PATCH 083/150] Don't render media `meta` if nil --- lib/pleroma/web/mastodon_api/views/status_view.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 5dbdc309e..7f318e81b 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.HTML + alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -406,6 +407,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image" href = attachment_url["href"] |> MediaProxy.url() href_preview = attachment_url["href"] |> MediaProxy.preview_url() + meta = render("attachment_meta.json", %{attachment: attachment}) type = cond do @@ -426,9 +428,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do type: type, description: attachment["name"], pleroma: %{mime_type: media_type}, - meta: render("attachment_meta.json", %{attachment: attachment}), blurhash: attachment["blurhash"] } + |> Maps.put_if_present(:meta, meta) end def render("attachment_meta.json", %{ @@ -444,7 +446,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end - def render("attachment_meta.json", _), do: %{} + def render("attachment_meta.json", _), do: nil def render("context.json", %{activity: activity, activities: activities, user: user}) do %{ancestors: ancestors, descendants: descendants} = From 6f0b42656dcce9cd7e4c833be42b6544954ca93b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 19:03:10 -0500 Subject: [PATCH 084/150] Federate attachments as Links instead of Documents --- lib/pleroma/web/activity_pub/transmogrifier.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b5767863c..acb4f4b3e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -245,8 +245,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "type" => Map.get(url || %{}, "type", "Link") } |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("width", (url || %{})["width"]) - |> Maps.put_if_present("height", (url || %{})["height"]) + |> Maps.put_if_present("width", (url || %{})["width"] || data["width"]) + |> Maps.put_if_present("height", (url || %{})["height"] || data["height"]) %{ "url" => [attachment_url], @@ -963,7 +963,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object |> Map.get("attachment", []) |> Enum.map(fn data -> - [%{"mediaType" => media_type, "href" => href} | _] = data["url"] + [%{"mediaType" => media_type, "href" => href} = url | _] = data["url"] %{ "url" => href, @@ -971,6 +971,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "name" => data["name"], "type" => "Document" } + |> Maps.put_if_present("width", url["width"]) + |> Maps.put_if_present("height", url["height"]) end) Map.put(object, "attachment", attachments) From 5a57b025c7745ebdc7ecf8c7d6b75bcc6770562a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 12 May 2021 20:15:33 -0500 Subject: [PATCH 085/150] Changelog: attachment meta --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bb4b1e73..22eaa0b94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded. - Return OAuth token `id` (primary key) in POST `/oauth/token`. +- `SetMeta` upload filter for extracting attachment dimensions. +- Attachment dimensions are federated when available. ### Fixed - Don't crash so hard when email settings are invalid. From 543e9402d64bce556f85294f91dc690c9acec51f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 14 May 2021 08:38:23 -0500 Subject: [PATCH 086/150] Support blurhash --- lib/pleroma/upload.ex | 7 +++++-- lib/pleroma/upload/filter/set_meta.ex | 9 +++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 1 + mix.exs | 3 +++ mix.lock | 1 + 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 4d58abd48..5570ed104 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Upload do path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over. * `:width` - width of the media in pixels * `:height` - height of the media in pixels + * `:blurhash` - string hash of the image encoded with the blurhash algorithm (https://blurha.sh/) Related behaviors: @@ -58,9 +59,10 @@ defmodule Pleroma.Upload do content_type: String.t(), width: integer(), height: integer(), + blurhash: String.t(), path: String.t() } - defstruct [:id, :name, :tempfile, :content_type, :width, :height, :path] + defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path] defp get_description(opts, upload) do case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do @@ -98,7 +100,8 @@ defmodule Pleroma.Upload do |> Maps.put_if_present("height", upload.height) ], "name" => description - }} + } + |> Maps.put_if_present("blurhash", upload.blurhash)} else {:description_limit, _} -> {:error, :description_too_long} diff --git a/lib/pleroma/upload/filter/set_meta.ex b/lib/pleroma/upload/filter/set_meta.ex index cccb6c371..81c48228a 100644 --- a/lib/pleroma/upload/filter/set_meta.ex +++ b/lib/pleroma/upload/filter/set_meta.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Upload.Filter.SetMeta do upload |> Map.put(:width, image.width) |> Map.put(:height, image.height) + |> Map.put(:blurhash, get_blurhash(file)) {:ok, :filtered, upload} rescue @@ -33,4 +34,12 @@ defmodule Pleroma.Upload.Filter.SetMeta do end def filter(_), do: {:ok, :noop} + + defp get_blurhash(file) do + with {:ok, blurhash} <- :eblurhash.magick(file) do + blurhash + else + _ -> nil + end + end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index acb4f4b3e..f601d6111 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -973,6 +973,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do } |> Maps.put_if_present("width", url["width"]) |> Maps.put_if_present("height", url["height"]) + |> Maps.put_if_present("blurhash", data["blurhash"]) end) Map.put(object, "attachment", attachments) diff --git a/mix.exs b/mix.exs index 436381f32..08581824a 100644 --- a/mix.exs +++ b/mix.exs @@ -198,6 +198,9 @@ defmodule Pleroma.Mixfile do {:open_api_spex, git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"}, + {:eblurhash, + git: "https://github.com/zotonic/eblurhash.git", + ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"}, ## dev & test {:ex_doc, "~> 0.22", only: :dev, runtime: false}, diff --git a/mix.lock b/mix.lock index 99be81826..d24f9c699 100644 --- a/mix.lock +++ b/mix.lock @@ -29,6 +29,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, + "eblurhash": {:git, "https://github.com/zotonic/eblurhash.git", "04a0b76eadf4de1be17726f39b6313b88708fd12", [ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"]}, "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"}, From b22f54eb29237b4c34a26b497f88770dbebf5578 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 May 2021 12:26:32 -0500 Subject: [PATCH 087/150] Make prod.secret.exs optional (with warning) --- config/prod.exs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/config/prod.exs b/config/prod.exs index adbce5606..0e151000b 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -63,7 +63,12 @@ config :logger, :ex_syslogger, level: :info # Finally import the config/prod.secret.exs # which should be versioned separately. -import_config "prod.secret.exs" +if File.exists?("./config/prod.secret.exs") do + import_config "prod.secret.exs" +else + "`config/prod.secret.exs` not found. You may want to create one by running `mix pleroma.instance gen`" + |> IO.warn([]) +end if File.exists?("./config/prod.exported_from_db.secret.exs"), do: import_config("prod.exported_from_db.secret.exs") From b540fff9081765feeadcc880af43f5d5d49d1e9c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 May 2021 12:20:20 -0500 Subject: [PATCH 088/150] Docs: use `MIX_ENV=prod mix pleroma.instance gen` --- docs/installation/alpine_linux_en.md | 2 +- docs/installation/arch_linux_en.md | 2 +- docs/installation/debian_based_en.md | 2 +- docs/installation/debian_based_jp.md | 4 ++-- docs/installation/freebsd_en.md | 6 +++--- docs/installation/gentoo_en.md | 10 +++++----- docs/installation/netbsd_en.md | 4 ++-- docs/installation/openbsd_en.md | 2 +- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index 7eb1718f2..c2dbd836d 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -117,7 +117,7 @@ cd /opt/pleroma sudo -Hu pleroma mix deps.get ``` -* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen` +* Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen` * Answer with `yes` if it asks you to install `rebar3`. * This may take some time, because parts of pleroma get compiled first. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md index da78c3205..53afccc0f 100644 --- a/docs/installation/arch_linux_en.md +++ b/docs/installation/arch_linux_en.md @@ -92,7 +92,7 @@ cd /opt/pleroma sudo -Hu pleroma mix deps.get ``` -* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen` +* Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen` * Answer with `yes` if it asks you to install `rebar3`. * This may take some time, because parts of pleroma get compiled first. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index c5687a01e..a9cf86ab3 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -90,7 +90,7 @@ cd /opt/pleroma sudo -Hu pleroma mix deps.get ``` -* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen` +* Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen` * Answer with `yes` if it asks you to install `rebar3`. * This may take some time, because parts of pleroma get compiled first. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index c4bbd4780..e076e2308 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -89,7 +89,7 @@ sudo -Hu pleroma mix deps.get * コンフィギュレーションを生成します。 ``` -sudo -Hu pleroma mix pleroma.instance gen +sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen ``` * rebar3をインストールしてもよいか聞かれたら、yesを入力してください。 * このときにpleromaの一部がコンパイルされるため、この処理には時間がかかります。 @@ -103,7 +103,7 @@ sudo -Hu pleroma mv config/{generated_config.exs,prod.secret.exs} * 先程のコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。 ``` -sudo -Hu pleroma mix pleroma.instance gen +sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen ``` * そして、データベースのマイグレーションを実行します。 diff --git a/docs/installation/freebsd_en.md b/docs/installation/freebsd_en.md index 2dc466eb8..f4f4d0db9 100644 --- a/docs/installation/freebsd_en.md +++ b/docs/installation/freebsd_en.md @@ -1,8 +1,8 @@ -# Installing on FreeBSD +# Installing on FreeBSD This document was written for FreeBSD 12.1, but should be work on future releases. -## Required software +## Required software This assumes the target system has `pkg(8)`. @@ -54,7 +54,7 @@ Configure Pleroma. Note that you need a domain name at this point: ``` $ cd /home/pleroma/pleroma $ mix deps.get # Enter "y" when asked to install Hex -$ mix pleroma.instance gen # You will be asked a few questions here. +$ MIX_ENV=prod mix pleroma.instance gen # You will be asked a few questions here. $ cp config/generated_config.exs config/prod.secret.exs ``` diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md index f2380ab72..af68db70d 100644 --- a/docs/installation/gentoo_en.md +++ b/docs/installation/gentoo_en.md @@ -54,7 +54,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake sys-apps/file ``` -If you would not like to install the optional packages, remove them from this line. +If you would not like to install the optional packages, remove them from this line. If you're running this from a low-powered virtual machine, it should work though it will take some time. There were no issues on a VPS with a single core and 1GB of RAM; if you are using an even more limited device and run into issues, you can try creating a swapfile or use a more powerful machine running Gentoo to [cross build](https://wiki.gentoo.org/wiki/Cross_build_environment). If you have a wait ahead of you, now would be a good time to take a break, strech a bit, refresh your beverage of choice and/or get a snack, and reply to Arch users' posts with "I use Gentoo btw" as we do. @@ -79,12 +79,12 @@ The output from emerging postgresql should give you a command for initializing t ``` * Start postgres and enable the system service - + ```shell # /etc/init.d/postgresql-11 start # rc-update add postgresql-11 default ``` - + ### A note on licenses, the AGPL, and deployment procedures If you do not plan to make any modifications to your Pleroma instance, cloning directly from the main repo will get you what you need. However, if you plan on doing any contributions to upstream development, making changes or modifications to your instance, making custom themes, or want to play around--and let's be honest here, if you're using Gentoo that is most likely you--you will save yourself a lot of headache later if you take the time right now to fork the Pleroma repo and use that in the following section. @@ -135,7 +135,7 @@ pleroma$ mix deps.get * Generate the configuration: ```shell -pleroma$ mix pleroma.instance gen +pleroma$ MIX_ENV=prod mix pleroma.instance gen ``` * Answer with `yes` if it asks you to install `rebar3`. @@ -241,7 +241,7 @@ First, ensure that the command you will be installing into your crontab works. # /usr/bin/certbot renew --nginx ``` -Assuming not much time has passed since you got certbot working a few steps ago, you should get a message for all domains you installed certificates for saying `Cert not yet due for renewal`. +Assuming not much time has passed since you got certbot working a few steps ago, you should get a message for all domains you installed certificates for saying `Cert not yet due for renewal`. Now, run crontab as a superuser with `crontab -e` or `sudo crontab -e` as appropriate, and add the following line to your cron: diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md index 233cf28b7..22cdd5691 100644 --- a/docs/installation/netbsd_en.md +++ b/docs/installation/netbsd_en.md @@ -1,6 +1,6 @@ # Installing on NetBSD -## Required software +## Required software pkgin should have been installed by the NetBSD installer if you selected the right options. If it isn't installed, install it using pkg_add. @@ -71,7 +71,7 @@ Configure Pleroma. Note that you need a domain name at this point: ``` $ cd /home/pleroma/pleroma $ mix deps.get -$ mix pleroma.instance gen # You will be asked a few questions here. +$ MIX_ENV=prod mix pleroma.instance gen # You will be asked a few questions here. ``` Since Postgres is configured, we can now initialize the database. There should diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 0e1269ca5..017b37519 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -239,7 +239,7 @@ Enter a shell as \_pleroma (as root `su _pleroma -`) and enter pleroma's install Then follow the main installation guide: * run `mix deps.get` - * run `mix pleroma.instance gen` and enter your instance's information when asked + * run `MIX_ENV=prod mix pleroma.instance gen` and enter your instance's information when asked * copy config/generated\_config.exs to config/prod.secret.exs. The default values should be sufficient but you should edit it and check that everything seems OK. * exit your current shell back to a root one and run `psql -U postgres -f /home/_pleroma/pleroma/config/setup_db.psql` to setup the database. * return to a \_pleroma shell into pleroma's installation directory (`su _pleroma -;cd ~/pleroma`) and run `MIX_ENV=prod mix ecto.migrate` From 230ad82dadf013cb56909d1e8df2a2d652c47068 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 May 2021 13:22:07 -0500 Subject: [PATCH 089/150] gitignore `config/runtime.exs` --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f30f4cf5f..da73b6f36 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ erl_crash.dump # variables. /config/*.secret.exs /config/generated_config.exs +/config/runtime.exs /config/*.env @@ -56,4 +57,4 @@ pleroma.iml # Editor temp files /*~ -/*# \ No newline at end of file +/*# From 9b6b5ac196d9a2defb74902bffad67505b0de5c5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 May 2021 15:33:33 -0500 Subject: [PATCH 090/150] Rename upload filter to AnalyzeMetadata --- CHANGELOG.md | 2 +- lib/pleroma/upload/filter/{set_meta.ex => analyze_metadata.ex} | 2 +- .../upload/filter/{set_meta_test.exs => analyze_metadata_test.exs} | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) rename lib/pleroma/upload/filter/{set_meta.ex => analyze_metadata.ex} (95%) rename test/pleroma/upload/filter/{set_meta_test.exs => analyze_metadata_test.exs} (70%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22eaa0b94..1a69414a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded. - Return OAuth token `id` (primary key) in POST `/oauth/token`. -- `SetMeta` upload filter for extracting attachment dimensions. +- `AnalyzeMetadata` upload filter for extracting attachment dimensions. - Attachment dimensions are federated when available. ### Fixed diff --git a/lib/pleroma/upload/filter/set_meta.ex b/lib/pleroma/upload/filter/analyze_metadata.ex similarity index 95% rename from lib/pleroma/upload/filter/set_meta.ex rename to lib/pleroma/upload/filter/analyze_metadata.ex index 81c48228a..8c23076d4 100644 --- a/lib/pleroma/upload/filter/set_meta.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2021 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Upload.Filter.SetMeta do +defmodule Pleroma.Upload.Filter.AnalyzeMetadata do @moduledoc """ Extracts metadata about the upload, such as width/height """ diff --git a/test/pleroma/upload/filter/set_meta_test.exs b/test/pleroma/upload/filter/analyze_metadata_test.exs similarity index 70% rename from test/pleroma/upload/filter/set_meta_test.exs rename to test/pleroma/upload/filter/analyze_metadata_test.exs index 650e527b4..6f0e432ef 100644 --- a/test/pleroma/upload/filter/set_meta_test.exs +++ b/test/pleroma/upload/filter/analyze_metadata_test.exs @@ -2,9 +2,9 @@ # Copyright © 2017-2021 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Upload.Filter.SetMetaTest do +defmodule Pleroma.Upload.Filter.AnalyzeMetadataTest do use Pleroma.DataCase, async: true - alias Pleroma.Upload.Filter.SetMeta + alias Pleroma.Upload.Filter.AnalyzeMetadata test "adds the image dimensions" do upload = %Pleroma.Upload{ @@ -14,6 +14,6 @@ defmodule Pleroma.Upload.Filter.SetMetaTest do tempfile: Path.absname("test/fixtures/image.jpg") } - assert {:ok, :filtered, %{width: 1024, height: 768}} = SetMeta.filter(upload) + assert {:ok, :filtered, %{width: 1024, height: 768}} = AnalyzeMetadata.filter(upload) end end From 4ab3ef07d0f10815e7a91ba3143b7f97cd2a6058 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 May 2021 15:51:11 -0500 Subject: [PATCH 091/150] Check AnalyzeMetadata filter's required commands eblurhash:magick uses "convert" Fetching image metadata uses "mogrify" --- lib/pleroma/application_requirements.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index c412dec5e..294eb3b6b 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -166,7 +166,9 @@ defmodule Pleroma.ApplicationRequirements do filter_commands_statuses = [ check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"), check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"), - check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify") + check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify"), + check_filter(Pleroma.Upload.Filters.AnalyzeMetadata, "mogrify"), + check_filter(Pleroma.Upload.Filters.AnalyzeMetadata, "convert") ] preview_proxy_commands_status = From c64cbee26c7b78f9743b668724d4797faa6a942a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 May 2021 16:28:21 -0500 Subject: [PATCH 092/150] Fixed checking for Upload Filter required commands --- CHANGELOG.md | 1 + lib/pleroma/application_requirements.ex | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a69414a5..768405dd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Don't crash so hard when email settings are invalid. +- Checking activated Upload Filters for required commands. ## Unreleased (Patch) diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 294eb3b6b..ee6ee9516 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -164,11 +164,11 @@ defmodule Pleroma.ApplicationRequirements do defp check_system_commands!(:ok) do filter_commands_statuses = [ - check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"), - check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"), - check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify"), - check_filter(Pleroma.Upload.Filters.AnalyzeMetadata, "mogrify"), - check_filter(Pleroma.Upload.Filters.AnalyzeMetadata, "convert") + check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"), + check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"), + check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"), + check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"), + check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert") ] preview_proxy_commands_status = From 2d7f6ce6fb047872083c2db6ad8b75a9032211fd Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 18 May 2021 16:46:51 -0500 Subject: [PATCH 093/150] Clarify AttachmentMetadata changes --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 768405dd6..898f8adb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded. - Return OAuth token `id` (primary key) in POST `/oauth/token`. -- `AnalyzeMetadata` upload filter for extracting attachment dimensions. -- Attachment dimensions are federated when available. +- `AnalyzeMetadata` upload filter for extracting attachment dimensions and generating blurhashes. +- Attachment dimensions and blurhashes are federated when available. ### Fixed - Don't crash so hard when email settings are invalid. From 07fed0fda2473fc4e1e3b01e863217391fd2902f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 18 May 2021 17:11:25 -0500 Subject: [PATCH 094/150] Switch to aliasing `Router.Helpers` instead of importing --- lib/pleroma/web.ex | 6 ++++-- lib/pleroma/web/feed/user_controller.ex | 2 +- lib/pleroma/web/mastodon_api/controllers/auth_controller.ex | 4 ++-- lib/pleroma/web/o_auth/o_auth_controller.ex | 4 ++-- lib/pleroma/web/templates/feed/feed/tag.atom.eex | 4 ++-- lib/pleroma/web/templates/feed/feed/tag.rss.eex | 2 +- lib/pleroma/web/templates/feed/feed/user.atom.eex | 6 +++--- lib/pleroma/web/templates/feed/feed/user.rss.eex | 6 +++--- lib/pleroma/web/templates/masto_fe/index.html.eex | 2 +- lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex | 4 ++-- lib/pleroma/web/templates/o_auth/mfa/totp.html.eex | 4 ++-- lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex | 2 +- lib/pleroma/web/templates/o_auth/o_auth/register.html.eex | 2 +- lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 2 +- lib/pleroma/web/templates/twitter_api/password/reset.html.eex | 2 +- lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex | 2 +- .../web/templates/twitter_api/remote_follow/follow_login.html.eex | 2 +- .../web/templates/twitter_api/remote_follow/follow_mfa.html.eex | 2 +- lib/pleroma/web/templates/twitter_api/util/subscribe.html.eex | 2 +- lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex | 2 +- lib/pleroma/web/views/masto_fe_view.ex | 2 +- 21 files changed, 33 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index 8630f244b..24751faba 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -35,9 +35,10 @@ defmodule Pleroma.Web do import Plug.Conn import Pleroma.Web.Gettext - import Pleroma.Web.Router.Helpers import Pleroma.Web.TranslationHelpers + alias Pleroma.Web.Router.Helpers, as: Routes + plug(:set_put_layout) defp set_put_layout(conn, _) do @@ -131,7 +132,8 @@ defmodule Pleroma.Web do import Pleroma.Web.ErrorHelpers import Pleroma.Web.Gettext - import Pleroma.Web.Router.Helpers + + alias Pleroma.Web.Router.Helpers, as: Routes require Logger diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 58d35da1e..fa7879caf 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.Feed.UserController do def feed_redirect(conn, %{"nickname" => nickname}) do with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do - redirect(conn, external: "#{user_feed_url(conn, :feed, user.nickname)}.atom") + redirect(conn, external: "#{Routes.user_feed_url(conn, :feed, user.nickname)}.atom") end end diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index eb6639fc5..4920d65da 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -53,7 +53,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do defp redirect_to_oauth_form(conn, _params) do with {:ok, app} <- local_mastofe_app() do path = - o_auth_path(conn, :authorize, + Routes.o_auth_path(conn, :authorize, response_type: "code", client_id: app.client_id, redirect_uri: ".", @@ -90,7 +90,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do defp local_mastodon_post_login_path(conn) do case get_session(conn, :return_to) do nil -> - masto_fe_path(conn, :index, ["getting-started"]) + Routes.masto_fe_path(conn, :index, ["getting-started"]) return_to -> delete_session(conn, :return_to) diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 215d97b3a..42f4d768f 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -427,7 +427,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do |> Map.put("state", state) # Handing the request to Ueberauth - redirect(conn, to: o_auth_path(conn, :request, provider, params)) + redirect(conn, to: Routes.o_auth_path(conn, :request, provider, params)) end def request(%Plug.Conn{} = conn, params) do @@ -601,7 +601,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do end # Special case: Local MastodonFE - defp redirect_uri(%Plug.Conn{} = conn, "."), do: auth_url(conn, :login) + defp redirect_uri(%Plug.Conn{} = conn, "."), do: Routes.auth_url(conn, :login) defp redirect_uri(%Plug.Conn{}, redirect_uri), do: redirect_uri diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index a288539ed..de0731085 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -9,13 +9,13 @@ xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:statusnet="http://status.net/schema/api/1/"> - <%= '#{tag_feed_url(@conn, :feed, @tag)}.rss' %> + <%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %> #<%= @tag %> These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse. <%= feed_logo() %> <%= most_recent_update(@activities) %> - + <%= for activity <- @activities do %> <%= render @view_module, "_tag_activity.atom", Map.merge(assigns, prepare_activity(activity, actor: true)) %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex index eeda01a04..9c3613feb 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex @@ -5,7 +5,7 @@ #<%= @tag %> These are public toots tagged with #<%= @tag %>. You can interact with them if you have an account anywhere in the fediverse. - <%= '#{tag_feed_url(@conn, :feed, @tag)}.rss' %> + <%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %> <%= feed_logo() %> 2b90d9 <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/feed/feed/user.atom.eex b/lib/pleroma/web/templates/feed/feed/user.atom.eex index c6acd848f..5c1f0ecbc 100644 --- a/lib/pleroma/web/templates/feed/feed/user.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/user.atom.eex @@ -6,16 +6,16 @@ xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:ostatus="http://ostatus.org/schema/1.0"> - <%= user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> + <%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> <%= @user.nickname <> "'s timeline" %> <%= most_recent_update(@activities, @user) %> <%= logo(@user) %> - + <%= render @view_module, "_author.atom", assigns %> <%= if last_activity(@activities) do %> - + <% end %> <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/feed/feed/user.rss.eex b/lib/pleroma/web/templates/feed/feed/user.rss.eex index d69120480..6b842a085 100644 --- a/lib/pleroma/web/templates/feed/feed/user.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/user.rss.eex @@ -1,16 +1,16 @@ - <%= user_feed_url(@conn, :feed, @user.nickname) <> ".rss" %> + <%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".rss" %> <%= @user.nickname <> "'s timeline" %> <%= most_recent_update(@activities, @user) %> <%= logo(@user) %> - <%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss' %> + <%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss' %> <%= render @view_module, "_author.rss", assigns %> <%= if last_activity(@activities) do %> - <%= '#{user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %> + <%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %> <% end %> <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/masto_fe/index.html.eex b/lib/pleroma/web/templates/masto_fe/index.html.eex index c330960fa..6f2b98957 100644 --- a/lib/pleroma/web/templates/masto_fe/index.html.eex +++ b/lib/pleroma/web/templates/masto_fe/index.html.eex @@ -7,7 +7,7 @@ <%= Config.get([:instance, :name]) %> - + diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index 5ab59b57b..b9daa8d8b 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -7,7 +7,7 @@

    Two-factor recovery

    -<%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> +<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
    <%= label f, :code, "Recovery code" %> <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %> @@ -19,6 +19,6 @@ <%= submit "Verify" %> <% end %> -"> +"> Enter a two-factor code diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index af85777eb..29ea7c5fb 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -7,7 +7,7 @@

    Two-factor authentication

    -<%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %> +<%= form_for @conn, Routes.mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
    <%= label f, :code, "Authentication code" %> <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %> @@ -19,6 +19,6 @@ <%= submit "Verify" %> <% end %> -"> +"> Enter a two-factor recovery code diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index 4a0718851..dc4521a62 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -1,6 +1,6 @@

    Sign in with external provider

    -<%= form_for @conn, o_auth_path(@conn, :prepare_request), [as: "authorization", method: "get"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :prepare_request), [as: "authorization", method: "get"], fn f -> %>
    <%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f}) %>
    diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index facedc8db..99f900fb7 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -8,7 +8,7 @@

    Registration Details

    If you'd like to register a new account, please provide the details below.

    -<%= form_for @conn, o_auth_path(@conn, :register), [as: "authorization"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :register), [as: "authorization"], fn f -> %>
    <%= label f, :nickname, "Nickname" %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 1a85818ec..2846ec7e7 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -5,7 +5,7 @@ <% end %> -<%= form_for @conn, o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> +<%= form_for @conn, Routes.o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> <%= if @user do %>