@@ -10,7 +10,7 @@ defmodule Pleroma.PasswordResetToken do | |||
alias Pleroma.{User, PasswordResetToken, Repo} | |||
schema "password_reset_tokens" do | |||
belongs_to(:user, User) | |||
belongs_to(:user, User, type: Pleroma.FlakeId) | |||
field(:token, :string) | |||
field(:used, :boolean, default: false) | |||
@@ -8,6 +8,7 @@ defmodule Pleroma.Activity do | |||
import Ecto.Query | |||
@type t :: %__MODULE__{} | |||
@primary_key {:id, Pleroma.FlakeId, autogenerate: true} | |||
# https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19 | |||
@mastodon_notification_types %{ | |||
@@ -99,6 +99,7 @@ defmodule Pleroma.Application do | |||
], | |||
id: :cachex_idem | |||
), | |||
worker(Pleroma.FlakeId, []), | |||
worker(Pleroma.Web.Federator.RetryQueue, []), | |||
worker(Pleroma.Web.Federator, []), | |||
worker(Pleroma.Stats, []), | |||
@@ -8,7 +8,7 @@ defmodule Pleroma.Filter do | |||
alias Pleroma.{User, Repo} | |||
schema "filters" do | |||
belongs_to(:user, User) | |||
belongs_to(:user, User, type: Pleroma.FlakeId) | |||
field(:filter_id, :integer) | |||
field(:hide, :boolean, default: false) | |||
field(:whole_word, :boolean, default: true) | |||
@@ -0,0 +1,181 @@ | |||
defmodule Pleroma.FlakeId do | |||
@moduledoc """ | |||
Flake is a decentralized, k-ordered id generation service. | |||
Adapted from: | |||
* [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License, | |||
* [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0 | |||
""" | |||
@type t :: binary | |||
@behaviour Ecto.Type | |||
use GenServer | |||
require Logger | |||
alias __MODULE__ | |||
import Kernel, except: [to_string: 1] | |||
defstruct node: nil, time: 0, sq: 0 | |||
@doc "Converts a binary Flake to a String" | |||
def to_string(<<0::integer-size(64), id::integer-size(64)>>) do | |||
Kernel.to_string(id) | |||
end | |||
def to_string(flake = <<_::integer-size(64), _::integer-size(48), _::integer-size(16)>>) do | |||
encode_base62(flake) | |||
end | |||
def to_string(s), do: s | |||
def from_string(<<id::integer-size(64)>>) do | |||
<<0::integer-size(64), id::integer-size(64)>> | |||
end | |||
for i <- [-1, 0] do | |||
def from_string(unquote(i)), do: <<0::integer-size(128)>> | |||
def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>> | |||
end | |||
def from_string(string) when is_binary(string) and byte_size(string) < 18 do | |||
case Integer.parse(string) do | |||
{id, _} -> <<0::integer-size(64), id::integer-size(64)>> | |||
_ -> nil | |||
end | |||
end | |||
def from_string(string) do | |||
string |> decode_base62 |> from_integer | |||
end | |||
def to_integer(<<integer::integer-size(128)>>), do: integer | |||
def from_integer(integer) do | |||
<<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> = | |||
<<integer::integer-size(128)>> | |||
end | |||
@doc "Generates a Flake" | |||
@spec get :: binary | |||
def get, do: to_string(:gen_server.call(:flake, :get)) | |||
# -- Ecto.Type API | |||
@impl Ecto.Type | |||
def type, do: :uuid | |||
@impl Ecto.Type | |||
def cast(value) do | |||
{:ok, FlakeId.to_string(value)} | |||
end | |||
@impl Ecto.Type | |||
def load(value) do | |||
{:ok, FlakeId.to_string(value)} | |||
end | |||
@impl Ecto.Type | |||
def dump(value) do | |||
{:ok, FlakeId.from_string(value)} | |||
end | |||
def autogenerate(), do: get() | |||
# -- GenServer API | |||
def start_link do | |||
:gen_server.start_link({:local, :flake}, __MODULE__, [], []) | |||
end | |||
@impl GenServer | |||
def init([]) do | |||
{:ok, %FlakeId{node: mac(), time: time()}} | |||
end | |||
@impl GenServer | |||
def handle_call(:get, _from, state) do | |||
{flake, new_state} = get(time(), state) | |||
{:reply, flake, new_state} | |||
end | |||
# Matches when the calling time is the same as the state time. Incr. sq | |||
defp get(time, %FlakeId{time: time, node: node, sq: seq}) do | |||
new_state = %FlakeId{time: time, node: node, sq: seq + 1} | |||
{gen_flake(new_state), new_state} | |||
end | |||
# Matches when the times are different, reset sq | |||
defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do | |||
new_state = %FlakeId{time: newtime, node: node, sq: 0} | |||
{gen_flake(new_state), new_state} | |||
end | |||
# Error when clock is running backwards | |||
defp get(newtime, %FlakeId{time: time}) when newtime < time do | |||
{:error, :clock_running_backwards} | |||
end | |||
defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do | |||
<<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>> | |||
end | |||
defp nthchar_base62(n) when n <= 9, do: ?0 + n | |||
defp nthchar_base62(n) when n <= 35, do: ?A + n - 10 | |||
defp nthchar_base62(n), do: ?a + n - 36 | |||
defp encode_base62(<<integer::integer-size(128)>>) do | |||
integer | |||
|> encode_base62([]) | |||
|> List.to_string() | |||
end | |||
defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc) | |||
defp encode_base62(int, []) when int == 0, do: '0' | |||
defp encode_base62(int, acc) when int == 0, do: acc | |||
defp encode_base62(int, acc) do | |||
r = rem(int, 62) | |||
id = div(int, 62) | |||
acc = [nthchar_base62(r) | acc] | |||
encode_base62(id, acc) | |||
end | |||
defp decode_base62(s) do | |||
decode_base62(String.to_charlist(s), 0) | |||
end | |||
defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9, | |||
do: decode_base62(cs, 62 * acc + (c - ?0)) | |||
defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z, | |||
do: decode_base62(cs, 62 * acc + (c - ?A + 10)) | |||
defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z, | |||
do: decode_base62(cs, 62 * acc + (c - ?a + 36)) | |||
defp decode_base62([], acc), do: acc | |||
defp time do | |||
{mega_seconds, seconds, micro_seconds} = :erlang.timestamp() | |||
1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000) | |||
end | |||
defp mac do | |||
{:ok, addresses} = :inet.getifaddrs() | |||
ifaces_with_mac = | |||
Enum.reduce(addresses, [], fn {iface, attrs}, acc -> | |||
if attrs[:hwaddr], do: [iface | acc], else: acc | |||
end) | |||
iface = Enum.at(ifaces_with_mac, :rand.uniform(length(ifaces_with_mac)) - 1) | |||
mac(iface) | |||
end | |||
defp mac(name) do | |||
{:ok, addresses} = :inet.getifaddrs() | |||
proplist = :proplists.get_value(name, addresses) | |||
hwaddr = Enum.take(:proplists.get_value(:hwaddr, proplist), 6) | |||
<<worker::integer-size(48)>> = :binary.list_to_bin(hwaddr) | |||
worker | |||
end | |||
end |
@@ -8,7 +8,7 @@ defmodule Pleroma.List do | |||
alias Pleroma.{User, Repo, Activity} | |||
schema "lists" do | |||
belongs_to(:user, Pleroma.User) | |||
belongs_to(:user, User, type: Pleroma.FlakeId) | |||
field(:title, :string) | |||
field(:following, {:array, :string}, default: []) | |||
@@ -9,8 +9,8 @@ defmodule Pleroma.Notification do | |||
schema "notifications" do | |||
field(:seen, :boolean, default: false) | |||
belongs_to(:user, Pleroma.User) | |||
belongs_to(:activity, Pleroma.Activity) | |||
belongs_to(:user, User, type: Pleroma.FlakeId) | |||
belongs_to(:activity, Activity, type: Pleroma.FlakeId) | |||
timestamps() | |||
end | |||
@@ -96,7 +96,7 @@ defmodule Pleroma.Notification do | |||
end | |||
end | |||
def create_notifications(%Activity{id: _, data: %{"to" => _, "type" => type}} = activity) | |||
def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity) | |||
when type in ["Create", "Like", "Announce", "Follow"] do | |||
users = get_notified_from_activity(activity) | |||
@@ -17,6 +17,8 @@ defmodule Pleroma.User do | |||
@type t :: %__MODULE__{} | |||
@primary_key {:id, Pleroma.FlakeId, autogenerate: true} | |||
@email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/ | |||
@strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/ | |||
@@ -900,15 +900,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do | |||
maybe_retire_websub(user.ap_id) | |||
# Only do this for recent activties, don't go through the whole db. | |||
# Only look at the last 1000 activities. | |||
since = (Repo.aggregate(Activity, :max, :id) || 0) - 1_000 | |||
q = | |||
from( | |||
a in Activity, | |||
where: ^old_follower_address in a.recipients, | |||
where: a.id > ^since, | |||
update: [ | |||
set: [ | |||
recipients: | |||
@@ -160,7 +160,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do | |||
"partOf" => iri, | |||
"totalItems" => info.note_count, | |||
"orderedItems" => collection, | |||
"next" => "#{iri}?max_id=#{min_id - 1}" | |||
"next" => "#{iri}?max_id=#{min_id}" | |||
} | |||
if max_qid == nil do | |||
@@ -207,7 +207,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do | |||
"partOf" => iri, | |||
"totalItems" => -1, | |||
"orderedItems" => collection, | |||
"next" => "#{iri}?max_id=#{min_id - 1}" | |||
"next" => "#{iri}?max_id=#{min_id}" | |||
} | |||
if max_qid == nil do | |||
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Authorization do | |||
field(:token, :string) | |||
field(:valid_until, :naive_datetime) | |||
field(:used, :boolean, default: false) | |||
belongs_to(:user, Pleroma.User) | |||
belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) | |||
belongs_to(:app, App) | |||
timestamps() | |||
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Token do | |||
field(:token, :string) | |||
field(:refresh_token, :string) | |||
field(:valid_until, :naive_datetime) | |||
belongs_to(:user, Pleroma.User) | |||
belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) | |||
belongs_to(:app, App) | |||
timestamps() | |||
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Push.Subscription do | |||
alias Pleroma.Web.Push.Subscription | |||
schema "push_subscriptions" do | |||
belongs_to(:user, User) | |||
belongs_to(:user, User, type: Pleroma.FlakeId) | |||
belongs_to(:token, Token) | |||
field(:endpoint, :string) | |||
field(:key_p256dh, :string) | |||
@@ -265,8 +265,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do | |||
end | |||
def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do | |||
id = String.to_integer(id) | |||
with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id), | |||
activities <- | |||
ActivityPub.fetch_activities_for_context(context, %{ | |||
@@ -340,38 +338,42 @@ defmodule Pleroma.Web.TwitterAPI.Controller do | |||
end | |||
def favorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do | |||
with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, | |||
{:ok, activity} <- TwitterAPI.fav(user, id) do | |||
with {:ok, activity} <- TwitterAPI.fav(user, id) do | |||
conn | |||
|> put_view(ActivityView) | |||
|> render("activity.json", %{activity: activity, for: user}) | |||
else | |||
_ -> json_reply(conn, 400, Jason.encode!(%{})) | |||
end | |||
end | |||
def unfavorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do | |||
with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, | |||
{:ok, activity} <- TwitterAPI.unfav(user, id) do | |||
with {:ok, activity} <- TwitterAPI.unfav(user, id) do | |||
conn | |||
|> put_view(ActivityView) | |||
|> render("activity.json", %{activity: activity, for: user}) | |||
else | |||
_ -> json_reply(conn, 400, Jason.encode!(%{})) | |||
end | |||
end | |||
def retweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do | |||
with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, | |||
{:ok, activity} <- TwitterAPI.repeat(user, id) do | |||
with {:ok, activity} <- TwitterAPI.repeat(user, id) do | |||
conn | |||
|> put_view(ActivityView) | |||
|> render("activity.json", %{activity: activity, for: user}) | |||
else | |||
_ -> json_reply(conn, 400, Jason.encode!(%{})) | |||
end | |||
end | |||
def unretweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do | |||
with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)}, | |||
{:ok, activity} <- TwitterAPI.unrepeat(user, id) do | |||
with {:ok, activity} <- TwitterAPI.unrepeat(user, id) do | |||
conn | |||
|> put_view(ActivityView) | |||
|> render("activity.json", %{activity: activity, for: user}) | |||
else | |||
_ -> json_reply(conn, 400, Jason.encode!(%{})) | |||
end | |||
end | |||
@@ -556,7 +558,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do | |||
def approve_friend_request(conn, %{"user_id" => uid} = _params) do | |||
with followed <- conn.assigns[:user], | |||
uid when is_number(uid) <- String.to_integer(uid), | |||
%User{} = follower <- Repo.get(User, uid), | |||
{:ok, follower} <- User.maybe_follow(follower, followed), | |||
%Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), | |||
@@ -578,7 +579,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do | |||
def deny_friend_request(conn, %{"user_id" => uid} = _params) do | |||
with followed <- conn.assigns[:user], | |||
uid when is_number(uid) <- String.to_integer(uid), | |||
%User{} = follower <- Repo.get(User, uid), | |||
%Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), | |||
{:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"), | |||
@@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do | |||
field(:state, :string) | |||
field(:subscribers, {:array, :string}, default: []) | |||
field(:hub, :string) | |||
belongs_to(:user, User) | |||
belongs_to(:user, User, type: Pleroma.FlakeId) | |||
timestamps() | |||
end | |||
@@ -0,0 +1,52 @@ | |||
defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do | |||
use Ecto.Migration | |||
# This migrates from int serial IDs to custom Flake: | |||
# 1- create a temporary uuid column | |||
# 2- fill this column with compatibility ids (see below) | |||
# 3- remove pkeys constraints | |||
# 4- update relation pkeys with the new ids | |||
# 5- rename the temporary column to id | |||
# 6- re-create the constraints | |||
def change do | |||
# Old serial int ids are transformed to 128bits with extra padding. | |||
# The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility | |||
# with previously issued ids. | |||
#execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" | |||
#execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" | |||
execute "ALTER TABLE activities DROP CONSTRAINT activities_pkey CASCADE;" | |||
execute "ALTER TABLE users DROP CONSTRAINT users_pkey CASCADE;" | |||
execute "ALTER TABLE activities ALTER COLUMN id DROP default;" | |||
execute "ALTER TABLE users ALTER COLUMN id DROP default;" | |||
execute "ALTER TABLE activities ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" | |||
execute "ALTER TABLE users ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);" | |||
execute "ALTER TABLE activities ADD PRIMARY KEY (id);" | |||
execute "ALTER TABLE users ADD PRIMARY KEY (id);" | |||
# Fkeys: | |||
# Activities - Referenced by: | |||
# TABLE "notifications" CONSTRAINT "notifications_activity_id_fkey" FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE | |||
# Users - Referenced by: | |||
# TABLE "filters" CONSTRAINT "filters_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE | |||
# TABLE "lists" CONSTRAINT "lists_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE | |||
# TABLE "notifications" CONSTRAINT "notifications_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE | |||
# TABLE "oauth_authorizations" CONSTRAINT "oauth_authorizations_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) | |||
# TABLE "oauth_tokens" CONSTRAINT "oauth_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) | |||
# TABLE "password_reset_tokens" CONSTRAINT "password_reset_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) | |||
# TABLE "push_subscriptions" CONSTRAINT "push_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE | |||
# TABLE "websub_client_subscriptions" CONSTRAINT "websub_client_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) | |||
execute "ALTER TABLE notifications ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid);" | |||
execute "ALTER TABLE notifications ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE;" | |||
for table <- ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do | |||
execute "ALTER TABLE #{table} ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid);" | |||
execute "ALTER TABLE #{table} ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;" | |||
end | |||
end | |||
end |
@@ -797,7 +797,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do | |||
|> with_credentials(current_user.nickname, "test") | |||
|> post("/api/favorites/create/1.json") | |||
assert json_response(conn, 500) | |||
assert json_response(conn, 400) | |||
end | |||
end | |||
@@ -1621,7 +1621,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do | |||
conn = | |||
build_conn() | |||
|> assign(:user, user) | |||
|> post("/api/pleroma/friendships/approve", %{"user_id" => to_string(other_user.id)}) | |||
|> post("/api/pleroma/friendships/approve", %{"user_id" => other_user.id}) | |||
assert relationship = json_response(conn, 200) | |||
assert other_user.id == relationship["id"] | |||
@@ -1644,7 +1644,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do | |||
conn = | |||
build_conn() | |||
|> assign(:user, user) | |||
|> post("/api/pleroma/friendships/deny", %{"user_id" => to_string(other_user.id)}) | |||
|> post("/api/pleroma/friendships/deny", %{"user_id" => other_user.id}) | |||
assert relationship = json_response(conn, 200) | |||
assert other_user.id == relationship["id"] | |||