...
 
Commits (11)
......@@ -58,6 +58,7 @@ gem 'idn-ruby', require: 'idn'
gem 'kaminari', '~> 1.1'
gem 'link_header', '~> 0.0'
gem 'mime-types', '~> 3.2', require: 'mime/types/columnar'
gem 'nilsimsa', git: 'https://github.com/witgo/nilsimsa', ref: 'fd184883048b922b176939f851338d0a4971a532'
gem 'nokogiri', '~> 1.10'
gem 'nsa', '~> 0.2'
gem 'oj', '~> 3.7'
......
......@@ -12,6 +12,13 @@ GIT
specs:
http_parser.rb (0.6.1)
GIT
remote: https://github.com/witgo/nilsimsa
revision: fd184883048b922b176939f851338d0a4971a532
ref: fd184883048b922b176939f851338d0a4971a532
specs:
nilsimsa (1.1.2)
GEM
remote: https://rubygems.org/
specs:
......@@ -717,6 +724,7 @@ DEPENDENCIES
microformats (~> 4.1)
mime-types (~> 3.2)
net-ldap (~> 0.10)
nilsimsa!
nokogiri (~> 1.10)
nsa (~> 0.2)
oj (~> 3.7)
......
......@@ -4,6 +4,7 @@ class AccountsController < ApplicationController
PAGE_SIZE = 20
include AccountControllerConcern
include SignatureAuthentication
before_action :set_cache_headers
before_action :set_body_classes
......@@ -39,8 +40,8 @@ class AccountsController < ApplicationController
end
format.json do
expires_in 3.minutes, public: true
render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter
expires_in 3.minutes, public: !(authorized_fetch_mode? && signed_request_account.present?)
render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to
end
end
end
......@@ -132,4 +133,12 @@ class AccountsController < ApplicationController
filtered_statuses.paginate_by_max_id(PAGE_SIZE, params[:max_id], params[:since_id]).to_a
end
end
def restrict_fields_to
if signed_request_account.present? || public_fetch_mode?
# Return all fields
else
%i(id type preferred_username inbox public_key endpoints)
end
end
end
......@@ -4,12 +4,13 @@ class ActivityPub::CollectionsController < Api::BaseController
include SignatureVerification
include AccountOwnedConcern
before_action :require_signature!, if: :authorized_fetch_mode?
before_action :set_size
before_action :set_statuses
before_action :set_cache_headers
def show
expires_in 3.minutes, public: true
expires_in 3.minutes, public: public_fetch_mode?
render json: collection_presenter, content_type: 'application/activity+json', serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, skip_activities: true
end
......
......@@ -5,23 +5,24 @@ class ActivityPub::InboxesController < Api::BaseController
include JsonLdHelper
include AccountOwnedConcern
before_action :skip_unknown_actor_delete
before_action :require_signature!
def create
if unknown_deleted_account?
head 202
elsif signed_request_account
upgrade_account
process_payload
head 202
else
render plain: signature_verification_failure_reason, status: 401
end
upgrade_account
process_payload
head 202
end
private
def skip_unknown_actor_delete
head 202 if unknown_deleted_account?
end
def unknown_deleted_account?
json = Oj.load(body, mode: :strict)
json['type'] == 'Delete' && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists?
json.is_a?(Hash) && json['type'] == 'Delete' && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists?
rescue Oj::ParseError
false
end
......@@ -32,8 +33,12 @@ class ActivityPub::InboxesController < Api::BaseController
def body
return @body if defined?(@body)
@body = request.body.read.force_encoding('UTF-8')
@body = request.body.read
@body.force_encoding('UTF-8') if @body.present?
request.body.rewind if request.body.respond_to?(:rewind)
@body
end
......
......@@ -6,12 +6,12 @@ class ActivityPub::OutboxesController < Api::BaseController
include SignatureVerification
include AccountOwnedConcern
before_action :require_signature!, if: :authorized_fetch_mode?
before_action :set_statuses
before_action :set_cache_headers
def show
expires_in 1.minute, public: true unless page_requested?
expires_in(page_requested? ? 0 : 3.minutes, public: public_fetch_mode?)
render json: outbox_presenter, serializer: ActivityPub::OutboxSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
end
......
......@@ -7,11 +7,13 @@ class ActivityPub::RepliesController < Api::BaseController
DESCENDANTS_LIMIT = 60
before_action :require_signature!, if: :authorized_fetch_mode?
before_action :set_status
before_action :set_cache_headers
before_action :set_replies
def index
expires_in 0, public: public_fetch_mode?
render json: replies_collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json', skip_activities: true
end
......
......@@ -36,6 +36,14 @@ class ApplicationController < ActionController::Base
Rails.env.production?
end
def authorized_fetch_mode?
ENV['AUTHORIZED_FETCH'] == 'true'
end
def public_fetch_mode?
!authorized_fetch_mode?
end
def store_current_location
store_location_for(:user, request.url) unless request.format == :json
end
......@@ -152,6 +160,6 @@ class ApplicationController < ActionController::Base
end
def set_cache_headers
response.headers['Vary'] = 'Accept'
response.headers['Vary'] = 'Accept, Signature'
end
end
......@@ -11,7 +11,7 @@ module AccountControllerConcern
layout 'public'
before_action :set_instance_presenter
before_action :set_link_headers
before_action :set_link_headers, if: -> { request.format.nil? || request.format == :html }
end
private
......
......@@ -7,12 +7,20 @@ module SignatureVerification
include DomainControlHelper
def require_signature!
render plain: signature_verification_failure_reason, status: signature_verification_failure_code unless signed_request_account
end
def signed_request?
request.headers['Signature'].present?
end
def signature_verification_failure_reason
return @signature_verification_failure_reason if defined?(@signature_verification_failure_reason)
@signature_verification_failure_reason
end
def signature_verification_failure_code
@signature_verification_failure_code || 401
end
def signed_request_account
......@@ -125,11 +133,16 @@ module SignatureVerification
end
def account_from_key_id(key_id)
domain = key_id.start_with?('acct:') ? key_id.split('@').last : key_id
if domain_not_allowed?(domain)
@signature_verification_failure_code = 403
return
end
if key_id.start_with?('acct:')
stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, '')) }
elsif !ActivityPub::TagManager.instance.local_uri?(key_id)
return if domain_not_allowed?(key_id)
account = ActivityPub::TagManager.instance.uri_to_resource(key_id, Account)
account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false) }
account
......
......@@ -2,7 +2,9 @@
class FollowerAccountsController < ApplicationController
include AccountControllerConcern
include SignatureVerification
before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
before_action :set_cache_headers
def index
......@@ -17,9 +19,9 @@ class FollowerAccountsController < ApplicationController
end
format.json do
raise Mastodon::NotPermittedError if params[:page].present? && @account.user_hides_network?
raise Mastodon::NotPermittedError if page_requested? && @account.user_hides_network?
expires_in 3.minutes, public: true if params[:page].blank?
expires_in(page_requested? ? 0 : 3.minutes, public: public_fetch_mode?)
render json: collection_presenter,
serializer: ActivityPub::CollectionSerializer,
......@@ -35,12 +37,16 @@ class FollowerAccountsController < ApplicationController
@follows ||= Follow.where(target_account: @account).recent.page(params[:page]).per(FOLLOW_PER_PAGE).preload(:account)
end
def page_requested?
params[:page].present?
end
def page_url(page)
account_followers_url(@account, page: page) unless page.nil?
end
def collection_presenter
if params[:page].present?
if page_requested?
ActivityPub::CollectionPresenter.new(
id: account_followers_url(@account, page: params.fetch(:page, 1)),
type: :ordered,
......
......@@ -2,7 +2,9 @@
class FollowingAccountsController < ApplicationController
include AccountControllerConcern
include SignatureVerification
before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
before_action :set_cache_headers
def index
......@@ -17,9 +19,9 @@ class FollowingAccountsController < ApplicationController
end
format.json do
raise Mastodon::NotPermittedError if params[:page].present? && @account.user_hides_network?
raise Mastodon::NotPermittedError if page_requested? && @account.user_hides_network?
expires_in 3.minutes, public: true if params[:page].blank?
expires_in(page_requested? ? 0 : 3.minutes, public: public_fetch_mode?)
render json: collection_presenter,
serializer: ActivityPub::CollectionSerializer,
......@@ -35,12 +37,16 @@ class FollowingAccountsController < ApplicationController
@follows ||= Follow.where(account: @account).recent.page(params[:page]).per(FOLLOW_PER_PAGE).preload(:target_account)
end
def page_requested?
params[:page].present?
end
def page_url(page)
account_following_index_url(@account, page: page) unless page.nil?
end
def collection_presenter
if params[:page].present?
if page_requested?
ActivityPub::CollectionPresenter.new(
id: account_following_index_url(@account, page: params.fetch(:page, 1)),
type: :ordered,
......
......@@ -8,11 +8,12 @@ class StatusesController < ApplicationController
layout 'public'
before_action :require_signature!, only: :show, if: -> { request.format == :json && authorized_fetch_mode? }
before_action :set_status
before_action :set_instance_presenter
before_action :set_link_headers
before_action :redirect_to_original, only: [:show]
before_action :set_referrer_policy_header, only: [:show]
before_action :redirect_to_original, only: :show
before_action :set_referrer_policy_header, only: :show
before_action :set_cache_headers
before_action :set_body_classes
before_action :set_autoplay, only: :embed
......@@ -30,14 +31,14 @@ class StatusesController < ApplicationController
end
format.json do
expires_in 3.minutes, public: @status.distributable?
expires_in 3.minutes, public: @status.distributable? && public_fetch_mode?
render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter
end
end
end
def activity
expires_in 3.minutes, public: @status.distributable?
expires_in 3.minutes, public: @status.distributable? && public_fetch_mode?
render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
end
......
# frozen_string_literal: true
class TagsController < ApplicationController
include SignatureVerification
PAGE_SIZE = 20
layout 'public'
before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
before_action :set_tag
before_action :set_body_classes
before_action :set_instance_presenter
......@@ -30,7 +33,7 @@ class TagsController < ApplicationController
end
format.json do
expires_in 3.minutes, public: true
expires_in 3.minutes, public: public_fetch_mode?
@statuses = HashtagQueryService.new.call(@tag, params.slice(:any, :all, :none), current_account, params[:local]).paginate_by_max_id(PAGE_SIZE, params[:max_id])
@statuses = cache_collection(@statuses, Status)
......
......@@ -16,13 +16,15 @@ module JsonLdHelper
# The url attribute can be a string, an array of strings, or an array of objects.
# The objects could include a mimeType. Not-included mimeType means it's text/html.
def url_to_href(value, preferred_type = nil)
single_value = if value.is_a?(Array) && !value.first.is_a?(String)
value.find { |link| preferred_type.nil? || ((link['mimeType'].presence || 'text/html') == preferred_type) }
elsif value.is_a?(Array)
value.first
else
value
end
single_value = begin
if value.is_a?(Array) && !value.first.is_a?(String)
value.find { |link| preferred_type.nil? || ((link['mimeType'].presence || 'text/html') == preferred_type) }
elsif value.is_a?(Array)
value.first
else
value
end
end
if single_value.nil? || single_value.is_a?(String)
single_value
......@@ -64,7 +66,9 @@ module JsonLdHelper
def fetch_resource(uri, id, on_behalf_of = nil)
unless id
json = fetch_resource_without_id_validation(uri, on_behalf_of)
return unless json
uri = json['id']
end
......@@ -73,25 +77,20 @@ module JsonLdHelper
end
def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false)
on_behalf_of ||= Account.representative
build_request(uri, on_behalf_of).perform do |response|
unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
raise Mastodon::UnexpectedResponseError, response
end
return body_to_json(response.body_with_limit) if response.code == 200
end
# If request failed, retry without doing it on behalf of a user
return if on_behalf_of.nil?
build_request(uri).perform do |response|
unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
raise Mastodon::UnexpectedResponseError, response
end
response.code == 200 ? body_to_json(response.body_with_limit) : nil
raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
body_to_json(response.body_with_limit) if response.code == 200
end
end
def body_to_json(body, compare_id: nil)
json = body.is_a?(String) ? Oj.load(body, mode: :strict) : body
return if compare_id.present? && json['id'] != compare_id
json
rescue Oj::ParseError
nil
......@@ -105,35 +104,34 @@ module JsonLdHelper
end
end
private
def response_successful?(response)
(200...300).cover?(response.code)
end
def response_error_unsalvageable?(response)
(400...500).cover?(response.code) && response.code != 429
response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code))
end
def build_request(uri, on_behalf_of = nil)
request = Request.new(:get, uri)
request.on_behalf_of(on_behalf_of) if on_behalf_of
request.add_headers('Accept' => 'application/activity+json, application/ld+json')
request
Request.new(:get, uri).tap do |request|
request.on_behalf_of(on_behalf_of) if on_behalf_of
request.add_headers('Accept' => 'application/activity+json, application/ld+json')
end
end
def load_jsonld_context(url, _options = {}, &_block)
json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do
request = Request.new(:get, url)
request.add_headers('Accept' => 'application/ld+json')
request.perform do |res|
raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json'
res.body_with_limit
end
end
doc = JSON::LD::API::RemoteDocument.new(url, json)
block_given? ? yield(doc) : doc
end
end
......@@ -41,6 +41,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
resolve_thread(@status)
fetch_replies(@status)
check_for_spam
distribute(@status)
forward_for_reply if @status.distributable?
end
......@@ -407,6 +408,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
Account.local.where(username: local_usernames).exists?
end
def check_for_spam
spam_check = SpamCheck.new(@status)
return if spam_check.skip?
if spam_check.spam?
spam_check.flag!
else
spam_check.remember!
end
end
def forward_for_reply
return unless @json['signature'].present? && reply_to_local?
ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
......
......@@ -34,6 +34,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
def serializable_hash(options = nil)
options = serialization_options(options)
serialized_hash = serializer.serializable_hash(options)
serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields]
serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options)
{ '@context' => serialized_context }.merge(serialized_hash)
......
......@@ -40,8 +40,8 @@ class Request
set_digest! if options.key?(:body)
end
def on_behalf_of(account, key_id_format = :acct, sign_with: nil)
raise ArgumentError unless account.local?
def on_behalf_of(account, key_id_format = :uri, sign_with: nil)
raise ArgumentError, 'account must not be nil' if account.nil?
@account = account
@keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair
......
# frozen_string_literal: true
class SpamCheck
include Redisable
include ActionView::Helpers::TextHelper
NILSIMSA_COMPARE_THRESHOLD = 95
NILSIMSA_MIN_SIZE = 10
EXPIRE_SET_AFTER = 1.week.seconds
def initialize(status)
@account = status.account
@status = status
end
def skip?
already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
end
def spam?
if insufficient_data?
false
elsif nilsimsa?
any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
else
any_other_digest?('md5') { |_, other_digest| other_digest == digest }
end
end
def flag!
auto_silence_account!
auto_report_status!
end
def remember!
# The scores in sorted sets don't actually have enough bits to hold an exact
# value of our snowflake IDs, so we use it only for its ordering property. To
# get the correct status ID back, we have to save it in the string value
redis.zadd(redis_key, @status.id, digest_with_algorithm)
redis.zremrangebyrank(redis_key, '0', '-10')
redis.expire(redis_key, EXPIRE_SET_AFTER)
end
def reset!
redis.del(redis_key)
end
def hashable_text
return @hashable_text if defined?(@hashable_text)
@hashable_text = @status.text
@hashable_text = remove_mentions(@hashable_text)
@hashable_text = strip_tags(@hashable_text) unless @status.local?
@hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text)
@hashable_text = remove_whitespace(@hashable_text)
end
def insufficient_data?
hashable_text.blank?
end
def digest
@digest ||= begin
if nilsimsa?
Nilsimsa.new(hashable_text).hexdigest
else
Digest::MD5.hexdigest(hashable_text)
end
end
end
def digest_with_algorithm
if nilsimsa?
['nilsimsa', digest, @status.id].join(':')
else
['md5', digest, @status.id].join(':')
end
end
private
def remove_mentions(text)
return text.gsub(Account::MENTION_RE, '') if @status.local?
Nokogiri::HTML.fragment(text).tap do |html|
mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) }
html.traverse do |element|
element.unlink if element.name == 'a' && mentions.include?(element['href'])
end
end.to_s
end
def normalize_unicode(text)
text.unicode_normalize(:nfkc).downcase
end
def remove_whitespace(text)
text.gsub(/\s+/, ' ').strip
end
def auto_silence_account!
@account.silence!
end
def auto_report_status!
status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced'))
end
def already_flagged?
@account.silenced?
end
def trusted?
@account.trust_level > Account::TRUST_LEVELS[:untrusted]
end
def no_unsolicited_mentions?
@status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) }
end
def solicited_reply?
!@status.thread.nil? && @status.thread.mentions.where(account: @account).exists?
end
def nilsimsa_compare_value(first, second)
first = [first].pack('H*')
second = [second].pack('H*')
bits = 0
0.upto(31) do |i|
bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord
end
128 - bits # -128 <= Nilsimsa Compare Value <= 128
end
def nilsimsa?
hashable_text.size > NILSIMSA_MIN_SIZE
end
def other_digests
redis.zrange(redis_key, 0, -1)
end
def any_other_digest?(filter_algorithm)
other_digests.any? do |record|
algorithm, other_digest, status_id = record.split(':')
next unless algorithm == filter_algorithm
yield algorithm, other_digest, status_id
end
end
def matching_status_ids
if nilsimsa?
other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact
else
other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact
end
end
def redis_key
@redis_key ||= "spam_check:#{@account.id}"
end
end
......@@ -45,6 +45,7 @@
# also_known_as :string is an Array
# silenced_at :datetime
# suspended_at :datetime
# trust_level :integer
#
class Account < ApplicationRecord
......@@ -62,6 +63,11 @@ class Account < ApplicationRecord
include AccountCounters
include DomainNormalizable
TRUST_LEVELS = {
untrusted: 0,
trusted: 1,
}.freeze
enum protocol: [:ostatus, :activitypub]
validates :username, presence: true
......@@ -163,6 +169,10 @@ class Account < ApplicationRecord
last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
end
def trust_level
self[:trust_level] || 0
end
def refresh!
ResolveAccountService.new.call(acct) unless local?
end
......@@ -171,21 +181,19 @@ class Account < ApplicationRecord
silenced_at.present?
end
def silence!(date = nil)
date ||= Time.now.utc
def silence!(date = Time.now.utc)
update!(silenced_at: date)
end
def unsilence!
update!(silenced_at: nil)
update!(silenced_at: nil, trust_level: trust_level == TRUST_LEVELS[:untrusted] ? TRUST_LEVELS[:trusted] : trust_level)
end
def suspended?
suspended_at.present?
end
def suspend!(date = nil)
date ||= Time.now.utc
def suspend!(date = Time.now.utc)
transaction do
user&.disable! if local?
update!(suspended_at: date)
......
......@@ -18,7 +18,6 @@ class WebfingerSerializer < ActiveModel::Serializer
{ rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
{ rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
{ rel: 'self', type: 'application/activity+json', href: account_url(object) },
{ rel: 'magic-public-key', href: "data:application/magic-public-key,#{object.magic_key}" },
{ rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
]
end
......
......@@ -5,18 +5,18 @@ class ActivityPub::FetchRemoteStatusService < BaseService
# Should be called when uri has already been checked for locality
def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil)
@json = if prefetched_body.nil?
fetch_resource(uri, id, on_behalf_of)
else
body_to_json(prefetched_body, compare_id: id ? uri : nil)
end
@json = begin
if prefetched_body.nil?
fetch_resource(uri, id, on_behalf_of)
else
body_to_json(prefetched_body, compare_id: id ? uri : nil)
end
end
return unless supported_context? && expected_type?
return if actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id)
return if !(supported_context? && expected_type?) || actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id)
actor = ActivityPub::TagManager.instance.uri_to_resource(actor_id, Account)
actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update(actor)
actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update?(actor)
return if actor.nil? || actor.suspended?
......@@ -46,7 +46,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
end
def needs_update(actor)
def needs_update?(actor)
actor.possibly_stale?
end
end
......@@ -8,7 +8,7 @@ class BlockService < BaseService
UnfollowService.new.call(account, target_account) if account.following?(target_account)
UnfollowService.new.call(target_account, account) if target_account.following?(account)
RejectFollowService.new.call(account, target_account) if target_account.requested?(account)
RejectFollowService.new.call(target_account, account) if target_account.requested?(account)
block = account.block!(target_account)
......
......@@ -29,7 +29,7 @@ class FetchLinkCardService < BaseService
end
attach_card if @card&.persisted?
rescue HTTP::Error, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
Rails.logger.debug "Error fetching link #{@url}: #{e}"
nil
end
......
......@@ -3,7 +3,7 @@
class FetchRemoteAccountService < BaseService
def call(url, prefetched_body = nil, protocol = :ostatus)
if prefetched_body.nil?
resource_url, resource_options, protocol = FetchAtomService.new.call(url)
resource_url, resource_options, protocol = FetchResourceService.new.call(url)
else
resource_url = url
resource_options = { prefetched_body: prefetched_body }
......
......@@ -3,7 +3,7 @@
class FetchRemoteStatusService < BaseService
def call(url, prefetched_body = nil, protocol = :ostatus)
if prefetched_body.nil?
resource_url, resource_options, protocol = FetchAtomService.new.call(url)
resource_url, resource_options, protocol = FetchResourceService.new.call(url)
else
resource_url = url
resource_options = { prefetched_body: prefetched_body }
......
# frozen_string_literal: true
class FetchAtomService < BaseService
class FetchResourceService < BaseService
include JsonLdHelper
ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html'
def call(url)
return if url.blank?
result = process(url)
# retry without ActivityPub
result ||= process(url) if @unsupported_activity
result
rescue OpenSSL::SSL::SSLError => e
Rails.logger.debug "SSL error: #{e}"
nil
rescue HTTP::ConnectionError => e
Rails.logger.debug "HTTP ConnectionError: #{e}"
process(url)
rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
Rails.logger.debug "Error fetching resource #{@url}: #{e}"
nil
end
......@@ -24,32 +18,22 @@ class FetchAtomService < BaseService
def process(url, terminal = false)
@url = url
perform_request { |response| process_response(response, terminal) }
end
def perform_request(&block)
accept = 'text/html'
accept = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", application/atom+xml, ' + accept unless @unsupported_activity
Request.new(:get, @url).add_headers('Accept' => accept).perform(&block)
Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).on_behalf_of(Account.representative).perform(&block)
end
def process_response(response, terminal = false)
return nil if response.code != 200
if response.mime_type == 'application/atom+xml'
[@url, { prefetched_body: response.body_with_limit }, :ostatus]
elsif ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
body = response.body_with_limit
json = body_to_json(body)
if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present?
[json['id'], { prefetched_body: body, id: true }, :activitypub]
elsif supported_context?(json) && expected_type?(json)
[json['id'], { prefetched_body: body, id: true }, :activitypub]
else
@unsupported_activity = true
nil
end
[json['id'], { prefetched_body: body, id: true }, :activitypub] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) || expected_type?(json))
elsif !terminal
link_header = response['Link'] && parse_link_header(response)
......@@ -66,25 +50,16 @@ class FetchAtomService < BaseService
end
def process_html(response)
page = Nokogiri::HTML(response.body_with_limit)
page = Nokogiri::HTML(response.body_with_limit)
json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' }
result ||= process(json_link['href'], terminal: true) unless json_link.nil? || @unsupported_activity
result ||= process(atom_link['href'], terminal: true) unless atom_link.nil?
result
process(json_link['href'], terminal: true) unless json_link.nil?
end
def process_link_headers(link_header)
json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'])
atom_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml))
result ||= process(json_link.href, terminal: true) unless json_link.nil? || @unsupported_activity
result ||= process(atom_link.href, terminal: true) unless atom_link.nil?
result
process(json_link.href, terminal: true) unless json_link.nil?
end
def parse_link_header(response)
......
......@@ -23,6 +23,7 @@ class RemoveStatusService < BaseService
remove_from_hashtags
remove_from_public
remove_from_media if status.media_attachments.any?
remove_from_spam_check
@status.destroy!
else
......@@ -142,6 +143,10 @@ class RemoveStatusService < BaseService
redis.publish('timeline:public:local:media', @payload) if @status.local?
end
def remove_from_spam_check
redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id)
end
def lock_options
{ redis: Redis.current, key: "distribute:#{@status.id}" }
end
......
......@@ -30,7 +30,7 @@ class ResolveAccountService < BaseService
# At this point we are in need of a Webfinger query, which may
# yield us a different username/domain through a redirect
process_webfinger!
process_webfinger!(@uri)
# Because the username/domain pair may be different than what
# we already checked, we need to check if we've already got
......@@ -69,15 +69,16 @@ class ResolveAccountService < BaseService
@domain = nil if TagManager.instance.local_domain?(@domain)
end
def process_webfinger!
def process_webfinger!(uri, redirected = false)
@webfinger = Goldfinger.finger("acct:#{@uri}")
confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@')
if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero?
@username = confirmed_username
@domain = confirmed_domain
elsif @options[:redirected].nil?
@account = ResolveAccountService.new.call("#{confirmed_username}@#{confirmed_domain}", @options.merge(redirected: true))
@uri = uri
elsif !redirected
return process_webfinger!("#{confirmed_username}@#{confirmed_domain}", true)
else
raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}"
end
......
......@@ -4,64 +4,49 @@ class ResolveURLService < BaseService
include JsonLdHelper
include Authorization
attr_reader :url
def call(url, on_behalf_of: nil)
@url = url
@url = url
@on_behalf_of = on_behalf_of
return process_local_url if local_url?
process_url unless fetched_atom_feed.nil?
if local_url?
process_local_url
elsif !fetched_resource.nil?
process_url
end
end
private
def process_url
if equals_or_includes_any?(type, ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES)
FetchRemoteAccountService.new.call(atom_url, body, protocol)
FetchRemoteAccountService.new.call(resource_url, body, protocol)
elsif equals_or_includes_any?(type, ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
FetchRemoteStatusService.new.call(atom_url, body, protocol)
FetchRemoteStatusService.new.call(resource_url, body, protocol)
end
end
def fetched_atom_feed
@_fetched_atom_feed ||= FetchAtomService.new.call(url)
def fetched_resource
@fetched_resource ||= FetchResourceService.new.call(@url)
end
def atom_url
fetched_atom_feed.first
def resource_url
fetched_resource.first
end
def body
fetched_atom_feed.second[:prefetched_body]
fetched_resource.second[:prefetched_body]
end
def protocol
fetched_atom_feed.third
fetched_resource.third
end
def type
return json_data['type'] if protocol == :activitypub
case xml_root
when 'feed'
'Person'
when 'entry'
'Note'
end
end
def json_data
@_json_data ||= body_to_json(body)
end
def xml_root
xml_data.root.name
end
def xml_data
@_xml_data ||= Nokogiri::XML(body, nil, 'utf-8')
@json_data ||= body_to_json(body)
end
def local_url?
......@@ -83,10 +68,10 @@ class ResolveURLService < BaseService
def check_local_status(status)
return if status.nil?
authorize_with @on_behalf_of, status, :show?
status
rescue Mastodon::NotPermittedError
# Do not disclose the existence of status the user is not authorized to see
nil
end
end
......@@ -25,11 +25,6 @@ doc << Ox::Element.new('XRD').tap do |xrd|
link['href'] = account_url(@account)
end
xrd << Ox::Element.new('Link').tap do |link|
link['rel'] = 'magic-public-key'
link['href'] = "data:application/magic-public-key,#{@account.magic_key}"
end
xrd << Ox::Element.new('Link').tap do |link|
link['rel'] = 'http://ostatus.org/schema/1.0/subscribe'
link['template'] = "#{authorize_interaction_url}?acct={uri}"
......
......@@ -2,6 +2,7 @@
class ActivityPub::DeliveryWorker
include Sidekiq::Worker
include JsonLdHelper
STOPLIGHT_FAILURE_THRESHOLD = 10
STOPLIGHT_COOLDOWN = 60
......@@ -32,9 +33,10 @@ class ActivityPub::DeliveryWorker
private
def build_request(http_client)
request = Request.new(:post, @inbox_url, body: @json, http_client: http_client)
request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with])
request.add_headers(HEADERS)
Request.new(:post, @inbox_url, body: @json, http_client: http_client).tap do |request|
request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with])
request.add_headers(HEADERS)
end
end
def perform_request
......@@ -53,14 +55,6 @@ class ActivityPub::DeliveryWorker
.run
end
def response_successful?(response)
(200...300).cover?(response.code)
end
def response_error_unsalvageable?(response)
response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code))
end
def failure_tracker
@failure_tracker ||= DeliveryFailureTracker.new(@inbox_url)
end
......
......@@ -3,7 +3,7 @@
class Web::PushNotificationWorker
include Sidekiq::Worker
sidekiq_options backtrace: true
sidekiq_options backtrace: true, retry: 5
def perform(subscription_id, notification_id)
subscription = ::Web::PushSubscription.find(subscription_id)
......
......@@ -875,6 +875,8 @@ en:
profile: Profile
relationships: Follows and followers
two_factor_authentication: Two-factor Auth
spam_check:
spam_detected_and_silenced: This is an automated report. Spam has been detected and the sender has been silenced automatically. If this is a mistake, please unsilence the account.
statuses:
attached:
description: 'Attached: %{attached}'
......
......@@ -12,6 +12,11 @@ class FixAccountsUniqueIndex < ActiveRecord::Migration[5.2]
end
end
class StreamEntry < ApplicationRecord
# Dummy class, to make migration possible across version changes
belongs_to :account, inverse_of: :stream_entries
end
disable_ddl_transaction!
def up
......
class AddTrustLevelToAccounts < ActiveRecord::Migration[5.2]
def change
add_column :accounts, :trust_level, :integer
end
end
......@@ -148,6 +148,7 @@ ActiveRecord::Schema.define(version: 2019_07_06_233204) do
t.string "also_known_as", array: true
t.datetime "silenced_at"
t.datetime "suspended_at"
t.integer "trust_level"
t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
......
......@@ -4,7 +4,7 @@ require 'rails_helper'
RSpec.describe ActivityPub::InboxesController, type: :controller do
describe 'POST #create' do
context 'if signed_request_account' do
context 'with signed_request_account' do
it 'returns 202' do
allow(controller).to receive(:signed_request_account) do
Fabricate(:account)
......@@ -15,7 +15,7 @@ RSpec.describe ActivityPub::InboxesController, type: :controller do
end
end
context 'not signed_request_account' do
context 'without signed_request_account' do
it 'returns 401' do
allow(controller).to receive(:signed_request_account) do
false
......
......@@ -38,7 +38,7 @@ describe ApplicationController, type: :controller do
end
context 'with signature header' do
let!(:author) { Fabricate(:account) }
let!(:author) { Fabricate(:account, domain: 'example.com', uri: 'https://example.com/actor') }
context 'without body' do
before do
......
require 'rails_helper'
RSpec.describe SpamCheck do
let!(:sender) { Fabricate(:account) }
let!(:alice) { Fabricate(:account, username: 'alice') }
let!(:bob) { Fabricate(:account, username: 'bob') }
def status_with_html(text, options = {})
status = PostStatusService.new.call(sender, { text: text }.merge(options))
status.update_columns(text: Formatter.instance.format(status), local: false)
status
end
describe '#hashable_text' do
it 'removes mentions from HTML for remote statuses' do
status = status_with_html('@alice Hello')
expect(described_class.new(status).hashable_text).to eq 'hello'
end
it 'removes mentions from text for local statuses' do
status = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?")
expect(described_class.new(status).hashable_text).to eq 'hey , how are you?'
end
end
describe '#insufficient_data?' do
it 'returns true when there is no text' do
status = status_with_html('@alice')
expect(described_class.new(status).insufficient_data?).to be true
end
it 'returns false when there is text' do
status = status_with_html('@alice h')
expect(described_class.new(status).insufficient_data?).to be false
end
end
describe '#digest' do
it 'returns a string' do
status = status_with_html('@alice Hello world')
expect(described_class.new(status).digest).to be_a String
end
end
describe '#spam?' do
it 'returns false for a unique status' do
status = status_with_html('@alice Hello')
expect(described_class.new(status).spam?).to be false
end
it 'returns false for different statuses to the same recipient' do
status1 = status_with_html('@alice Hello')
described_class.new(status1).remember!
status2 = status_with_html('@alice Are you available to talk?')
expect(described_class.new(status2).spam?).to be false
end
it 'returns false for statuses with different content warnings' do
status1 = status_with_html('@alice Are you available to talk?')
described_class.new(status1).remember!
status2 = status_with_html('@alice Are you available to talk?', spoiler_text: 'This is a completely different matter than what I was talking about previously, I swear!')
expect(described_class.new(status2).spam?).to be false
end
it 'returns false for different statuses to different recipients' do
status1 = status_with_html('@alice How is it going?')
described_class.new(status1).remember!
status2 = status_with_html('@bob Are you okay?')
expect(described_class.new(status2).spam?).to be false
end
it 'returns false for very short different statuses to different recipients' do
status1 = status_with_html('@alice 🙄')
described_class.new(status1).remember!
status2 = status_with_html('@bob Huh?')
expect(described_class.new(status2).spam?).to be false
end
it 'returns false for statuses with no text' do
status1 = status_with_html('@alice')
described_class.new(status1).remember!
status2 = status_with_html('@bob')
expect(described_class.new(status2).spam?).to be false
end
it 'returns true for duplicate statuses to the same recipient' do
status1 = status_with_html('@alice Hello')
described_class.new(status1).remember!
status2 = status_with_html('@alice Hello')
expect(described_class.new(status2).spam?).to be true
end
it 'returns true for duplicate statuses to different recipients' do
status1 = status_with_html('@alice Hello')
described_class.new(status1).remember!
status2 = status_with_html('@bob Hello')
expect(described_class.new(status2).spam?).to be true
end
it 'returns true for nearly identical statuses with random numbers' do
source_text = 'Sodium, atomic number 11, was first isolated by Humphry Davy in 1807. A chemical component of salt, he named it Na in honor of the saltiest region on earth, North America.'
status1 = status_with_html('@alice ' + source_text + ' 1234')
described_class.new(status1).remember!
status2 = status_with_html('@bob ' + source_text + ' 9568')
expect(described_class.new(status2).spam?).to be true
end
end
describe '#skip?' do
it 'returns true when the sender is already silenced' do
status = status_with_html('@alice Hello')
sender.silence!
expect(described_class.new(status).skip?).to be true
end
it 'returns true when the mentioned person follows the sender' do
status = status_with_html('@alice Hello')
alice.follow!(sender)
expect(described_class.new(status).skip?).to be true
end
it 'returns false when even one mentioned person doesn\'t follow the sender' do
status = status_with_html('@alice @bob Hello')
alice.follow!(sender)
expect(described_class.new(status).skip?).to be false
end
it 'returns true when the sender is replying to a status that mentions the sender' do
parent = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?")
status = status_with_html('@alice @bob Hello', thread: parent)
expect(described_class.new(status).skip?).to be true
end
end
describe '#remember!' do
pending
end
describe '#flag!' do
let!(:status1) { status_with_html('@alice General Kenobi you are a bold one') }
let!(:status2) { status_with_html('@alice @bob General Kenobi, you are a bold one') }
before do
described_class.new(status1).remember!
described_class.new(status2).flag!
end
it 'silences the account' do
expect(sender.silenced?).to be true
end
it 'creates a report about the account' do
expect(sender.targeted_reports.unresolved.count).to eq 1
end
it 'attaches both matching statuses to the report' do
expect(sender.targeted_reports.first.status_ids).to include(status1.id, status2.id)
end
end
end
......@@ -4,6 +4,8 @@ RSpec.describe FetchRemoteAccountService, type: :service do
let(:url) { 'https://example.com/alice' }
let(:prefetched_body) { nil }
let(:protocol) { :ostatus }
let!(:representative) { Fabricate(:account) }
subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) }
let(:actor) do
......
require 'rails_helper'
RSpec.describe FetchAtomService, type: :service do
RSpec.describe FetchResourceService, type: :service do
let!(:representative) { Fabricate(:account) }
describe '#call' do
let(:url) { 'http://example.com' }
subject { FetchAtomService.new.call(url) }
context 'url is blank' do
subject { described_class.new.call(url) }
context 'with blank url' do
let(:url) { '' }
it { is_expected.to be_nil }
end
context 'request failed' do
context 'when request fails' do
before do
WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {})
stub_request(:get, url).to_return(status: 500, body: '', headers: {})
end
it { is_expected.to be_nil }
end
context 'raise OpenSSL::SSL::SSLError' do
context 'when OpenSSL::SSL::SSLError is raised' do
before do
allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError)
allow(Request).to receive_message_chain(:new, :add_headers, :on_behalf_of, :perform).and_raise(OpenSSL::SSL::SSLError)
end
it 'output log and return nil' do
expect_any_instance_of(ActiveSupport::Logger).to receive(:debug).with('SSL error: OpenSSL::SSL::SSLError')
is_expected.to be_nil
end
it { is_expected.to be_nil }