Commit c89c8613 authored by Albert Yi's avatar Albert Yi Committed by GitHub

Merge pull request #3718 from r888888888/async-uploads

Asynchronous Uploads
parents b561a6d9 06f47cc8
unicorn: bundle exec rails server
jobs: bundle exec script/delayed_job run
jobs: bundle exec script/delayed_job run
......@@ -34,7 +34,6 @@
$dest.empty();
Danbooru.RelatedTag.build_recent_and_frequent($dest);
$dest.append("<em>Loading...</em>");
$("#related-tags-container").show();
$.get("/related_tag.json", {
"query": Danbooru.RelatedTag.current_tag(),
"category": category
......@@ -93,6 +92,9 @@
}
Danbooru.RelatedTag.process_response = function(data) {
if (data.tags.length || data.wiki_page_tags.length || data.other_wikis.length) {
$("#related-tags-container").show();
}
Danbooru.RelatedTag.recent_search = data;
Danbooru.RelatedTag.build_all();
}
......
......@@ -33,7 +33,7 @@
Danbooru.Upload.initialize_submit = function() {
$("#form").submit(function(e) {
var error_messages = [];
if (($("#upload_file").val() === "") && ($("#upload_source").val() === "")) {
if (($("#upload_file").val() === "") && ($("#upload_source").val() === "") && $("#upload_md5_confirmation").val() === "") {
error_messages.push("Must choose file or specify source");
}
if (!$("#upload_rating_s").prop("checked") && !$("#upload_rating_q").prop("checked") && !$("#upload_rating_e").prop("checked") &&
......
......@@ -471,9 +471,10 @@ div#c-post-versions, div#c-artist-versions {
div#c-posts, div#c-uploads {
/* Fetch source data box */
div#source-info {
border-radius: 4px;
margin: 1em 0;
padding: 1em;
border: 1px solid gray;
border: 1px solid #666;
p {
margin: 0;
......
@import "../common/000_vars.scss";
div#related-tags-container {
display: none;
padding-right: 2em;
h1 {
......@@ -14,6 +15,7 @@ div.related-tags {
padding: 1em;
background: #EEE;
overflow: auto;
border-radius: 4px;
div.tag-column {
max-width: 15em;
......
......@@ -34,6 +34,48 @@ div#c-uploads {
div.field_with_errors {
display: inline;
}
#filedropzone {
border: 4px dashed #DDD;
padding: 0;
min-height: 100px;
display: flex;
align-items: center;
justify-content: center;
border-radius: 4px;
.placeholder {
font-style: italic;
color: #333;
height: 100%;
}
&.error {
border-color: darken(#f2dede, 30%);
background-color: #f2dede;
}
&.success {
border-color: darken(#dff0d8, 30%);
background-color: #dff0d8;
}
}
.dz-preview {
margin-bottom: 1em;
}
.dz-progress {
height: 20px;
width: 300px;
border: 1px solid #CCC;
.dz-upload {
background-color: #F5F5FF;
display: block;
height: 20px;
}
}
}
div#a-index {
......
class UploadsController < ApplicationController
before_action :member_only, except: [:index, :show]
respond_to :html, :xml, :json, :js
skip_before_action :verify_authenticity_token, only: [:preprocess]
def new
@upload = Upload.new
@upload_notice_wiki = WikiPage.titled(Danbooru.config.upload_notice_wiki_page).first
if params[:url]
download = Downloads::File.new(params[:url])
@normalized_url, _, _ = download.before_download(params[:url], {})
@post = find_post_by_url(@normalized_url)
begin
@source = Sources::Site.new(params[:url], :referer_url => params[:ref])
@remote_size = download.size
rescue Exception
end
end
@upload, @post, @source, @normalized_url, @remote_size = UploadService::ControllerHelper.prepare(
url: params[:url], ref: params[:ref]
)
respond_with(@upload)
end
def batch
@url = params.dig(:batch, :url) || params[:url]
@source = nil
if @url
@source = Sources::Site.new(@url, :referer_url => params[:ref])
@source.get
end
@source = UploadService::ControllerHelper.batch(@url, params[:ref])
respond_with(@source)
end
......@@ -56,15 +42,19 @@ class UploadsController < ApplicationController
end
end
def create
@upload = Upload.create(upload_params)
def preprocess
@upload, @post, @source, @normalized_url, @remote_size = UploadService::ControllerHelper.prepare(
url: params[:url], file: params[:file], ref: params[:ref]
)
render body: nil
end
if @upload.errors.empty?
post = @upload.process!
def create
@service = UploadService.new(upload_params)
@upload = @service.start!
if post.present? && post.valid? && post.warnings.any?
flash[:notice] = post.warnings.full_messages.join(".\n \n")
end
if @service.warnings.any?
flash[:notice] = @service.warnings.join(".\n \n")
end
save_recent_tags
......@@ -73,14 +63,6 @@ class UploadsController < ApplicationController
private
def find_post_by_url(normalized_url)
if normalized_url.nil?
Post.where("SourcePattern(lower(posts.source)) = ?", params[:url]).first
else
Post.where("SourcePattern(lower(posts.source)) IN (?)", [params[:url], @normalized_url]).first
end
end
def save_recent_tags
if @upload
tags = Tag.scan_tags(@upload.tag_string)
......@@ -94,7 +76,7 @@ class UploadsController < ApplicationController
permitted_params = %i[
file source tag_string rating status parent_id artist_commentary_title
artist_commentary_desc include_artist_commentary referer_url
md5_confirmation as_pending
md5_confirmation as_pending
]
params.require(:upload).permit(permitted_params)
......
class PixivUgoiraService
attr_reader :width, :height, :frame_data, :content_type
def save_frame_data(post)
PixivUgoiraFrameData.create(:data => @frame_data, :content_type => @content_type, :post_id => post.id)
end
def calculate_dimensions(source_path)
folder = Zip::File.new(source_path)
tempfile = Tempfile.new("ugoira-dimensions")
begin
folder.first.extract(tempfile.path) {true}
image_size = ImageSpec.new(tempfile.path)
@width = image_size.width
@height = image_size.height
ensure
tempfile.close
tempfile.unlink
end
end
def load(data)
if data[:is_ugoira]
@frame_data = data[:ugoira_frame_data]
@content_type = data[:ugoira_content_type]
end
end
def empty?
@frame_data.nil?
end
end
This diff is collapsed.
......@@ -1396,7 +1396,8 @@ class Post < ApplicationRecord
def replace!(params)
transaction do
replacement = replacements.create(params)
replacement.process!
processor = UploadService::Replacer.new(post: self, replacement: replacement)
processor.process!
replacement
end
end
......
......@@ -18,166 +18,40 @@ class PostReplacement < ApplicationRecord
self.md5_was = post.md5
end
def undo!
undo_replacement = post.replacements.create(replacement_url: original_url)
undo_replacement.process!
end
def process!
upload = nil
transaction do
upload = Upload.create!(
file: replacement_file,
source: replacement_url,
rating: post.rating,
tag_string: self.tags,
replaced_post: post,
)
upload.process_upload
upload.update(status: "completed", post_id: post.id)
md5_changed = (upload.md5 != post.md5)
if replacement_file.present?
self.replacement_url = "file://#{replacement_file.original_filename}"
else
self.replacement_url = upload.downloaded_source
concerning :Search do
class_methods do
def post_tags_match(query)
PostQueryBuilder.new(query).build(self.joins(:post))
end
# queue the deletion *before* updating the post so that we use the old
# md5/file_ext to delete the old files. if saving the post fails,
# this is rolled back so the job won't run.
if md5_changed
post.queue_delete_files(DELETION_GRACE_PERIOD)
end
self.file_ext = upload.file_ext
self.file_size = upload.file_size
self.image_height = upload.image_height
self.image_width = upload.image_width
self.md5 = upload.md5
post.md5 = upload.md5
post.file_ext = upload.file_ext
post.image_width = upload.image_width
post.image_height = upload.image_height
post.file_size = upload.file_size
post.source = final_source.presence || upload.source
post.tag_string = upload.tag_string
def search(params = {})
q = super
rescale_notes
update_ugoira_frame_data(upload)
if md5_changed
CurrentUser.as(User.system) do
post.comments.create!(body: comment_replacement_message, do_not_bump_post: true)
if params[:creator_id].present?
q = q.where(creator_id: params[:creator_id].split(",").map(&:to_i))
end
end
save!
post.save!
end
# point of no return: these things can't be rolled back, so we do them
# only after the transaction successfully commits.
upload.distribute_files(post)
post.update_iqdb_async
end
def rescale_notes
x_scale = post.image_width.to_f / post.image_width_was.to_f
y_scale = post.image_height.to_f / post.image_height_was.to_f
post.notes.each do |note|
note.rescale!(x_scale, y_scale)
end
end
def update_ugoira_frame_data(upload)
post.pixiv_ugoira_frame_data.destroy if post.pixiv_ugoira_frame_data.present?
upload.ugoira_service.save_frame_data(post) if post.is_ugoira?
end
module SearchMethods
def post_tags_match(query)
PostQueryBuilder.new(query).build(self.joins(:post))
end
def search(params = {})
q = super
if params[:creator_id].present?
q = q.where(creator_id: params[:creator_id].split(",").map(&:to_i))
end
if params[:creator_name].present?
q = q.where(creator_id: User.name_to_id(params[:creator_name]))
end
if params[:creator_name].present?
q = q.where(creator_id: User.name_to_id(params[:creator_name]))
end
if params[:post_id].present?
q = q.where(post_id: params[:post_id].split(",").map(&:to_i))
end
if params[:post_id].present?
q = q.where(post_id: params[:post_id].split(",").map(&:to_i))
end
if params[:post_tags_match].present?
q = q.post_tags_match(params[:post_tags_match])
end
if params[:post_tags_match].present?
q = q.post_tags_match(params[:post_tags_match])
q.apply_default_order(params)
end
q.apply_default_order(params)
end
end
module PresenterMethods
def comment_replacement_message
%("#{creator.name}":[/users/#{creator.id}] replaced this post with a new image:\n\n#{replacement_message})
end
def replacement_message
linked_source = linked_source(replacement_url)
linked_source_was = linked_source(post.source_was)
<<-EOS.strip_heredoc
[table]
[tbody]
[tr]
[th]Old[/th]
[td]#{linked_source_was}[/td]
[td]#{post.md5_was}[/td]
[td]#{post.file_ext_was}[/td]
[td]#{post.image_width_was} x #{post.image_height_was}[/td]
[td]#{post.file_size_was.to_s(:human_size, precision: 4)}[/td]
[/tr]
[tr]
[th]New[/th]
[td]#{linked_source}[/td]
[td]#{post.md5}[/td]
[td]#{post.file_ext}[/td]
[td]#{post.image_width} x #{post.image_height}[/td]
[td]#{post.file_size.to_s(:human_size, precision: 4)}[/td]
[/tr]
[/tbody]
[/table]
EOS
end
def linked_source(source)
# truncate long sources in the middle: "www.pixiv.net...lust_id=23264933"
truncated_source = source.gsub(%r{\Ahttps?://}, "").truncate(64, omission: "...#{source.last(32)}")
if source =~ %r{\Ahttps?://}i
%("#{truncated_source}":[#{source}])
else
truncated_source
end
end
def suggested_tags_for_removal
tags = post.tag_array.select { |tag| Danbooru.config.remove_tag_after_replacement?(tag) }
tags = tags.map { |tag| "-#{tag}" }
tags.join(" ")
end
def suggested_tags_for_removal
tags = post.tag_array.select { |tag| Danbooru.config.remove_tag_after_replacement?(tag) }
tags = tags.map { |tag| "-#{tag}" }
tags.join(" ")
end
include PresenterMethods
extend SearchMethods
end
This diff is collapsed.
......@@ -21,6 +21,7 @@
<%= hidden_field_tag :url, params[:url] %>
<%= hidden_field_tag :ref, params[:ref] %>
<%= hidden_field_tag :normalized_url, @normalized_url %>
<%= f.hidden_field :md5_confirmation %>
<%= f.hidden_field :referer_url, :value => @source.try(:referer_url) %>
<% if CurrentUser.can_upload_free? %>
......@@ -32,11 +33,15 @@
</div>
<% end %>
<div class="input">
<div class="input fallback">
<%= f.label :file %>
<%= f.file_field :file, :size => 50 %>
</div>
<div class="input" id="filedropzone">
<div class="placeholder"><span>Drag and drop a file here</span></div>
</div>
<div class="input">
<%= f.label :source %>
<% if params[:url].present? %>
......@@ -106,15 +111,15 @@
<div class="input">
<div>
<%= f.label :tag_string, "Tags" %>
<%= f.text_area :tag_string, :size => "60x5", :data => { :autocomplete => "tag-edit" } %>
<%= f.text_area :tag_string, :size => "60x5", :spellcheck => false, :data => { :autocomplete => "tag-edit" } %>
<span id="open-edit-dialog" class="ui-icon ui-icon-arrow-1-ne" title="detach" style="display: none;"/>
</div>
<%= button_tag "Related tags", :id => "related-tags-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %>
<% TagCategory.related_button_list.each do |category| %>
<%= button_tag "#{TagCategory.related_button_mapping[category]}", :id => "related-#{category}-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %>
<% end %>
<% TagCategory.related_button_list.each do |category| %>
<%= button_tag "#{TagCategory.related_button_mapping[category]}", :id => "related-#{category}-button", :type => "button", :class => "ui-button ui-widget ui-corner-all sub gradient" %>
<% end %>
</div>
<div class="input">
......@@ -143,4 +148,55 @@
Upload - <%= Danbooru.config.app_name %>
<% end %>
<% content_for(:html_header) do %>
<script async src="https://cdnjs.cloudflare.com/ajax/libs/dropzone/5.4.0/min/dropzone.min.js"></script>
<script async src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.9-1/crypto-js.min.js"></script>
<script>
$(function() {
var enabled = true;
if (!window.FileReader) {
enabled = false;
}
if (!enabled) {
$("#filedropzone").remove();
return;
}
$("#filedropzone").dropzone({
paramName: "file",
url: "/uploads/preprocess",
createImageThumbnails: false,
addRemoveLinks: false,
maxFiles: 1,
acceptedFiles: "image/jpeg,image/png,image/gif",
previewTemplate: '<div class="dz-preview dz-file-preview"><div class="dz-details"><div class="dz-filename"><span data-dz-name></span></div><div class="dz-size" data-dz-size></div></div><div class="dz-progress"><span class="dz-upload" data-dz-uploadprogress></span></div><div class="dz-error-message"><span data-dz-errormessage></span></div></div>',
init: function() {
$(".fallback").hide();
this.on("drop", function(event) {
$("#filedropzone .placeholder").hide();
});
this.on("complete", function(file) {
$("#filedropzone .dz-progress").hide();
});
this.on("addedfile", function(file) {
var reader = new FileReader()
reader.addEventListener("loadend", function() {
$("#upload_md5_confirmation").val(CryptoJS.MD5(CryptoJS.enc.Latin1.parse(this.result)).toString());
});
reader.readAsBinaryString(file);
});
this.on("success", function(file) {
$("#filedropzone").addClass("success");
});
this.on("error", function(file, msg) {
$("#filedropzone").addClass("error");
});
}
});
});
</script>
<% end %>
<%= render "uploads/secondary_links" %>
......@@ -12,7 +12,7 @@
<p>This upload has finished processing. <%= link_to "View the post", post_path(@upload.post_id) %>.</p>
<% elsif @upload.is_pending? %>
<p>This upload is waiting to be processed. Please wait a few seconds.</p>
<% elsif @upload.is_processing? %>
<% elsif @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
<p>This upload is being processed. Please wait a few seconds.</p>
<% elsif @upload.is_duplicate? %>
<p>This upload is a duplicate: <%= link_to "post ##{@upload.duplicate_post_id}", post_path(@upload.duplicate_post_id) %></p>
......@@ -42,7 +42,7 @@
Upload - <%= Danbooru.config.app_name %>
<% end %>
<% if @upload.is_pending? || @upload.is_processing? %>
<% if @upload.is_pending? || @upload.is_processing? || @upload.is_preprocessing? || @upload.is_preprocessed? %>
<% content_for(:html_header) do %>
<meta http-equiv="refresh" content="2">
<% end %>
......
unless Rails.env.development?
FFMPEG.logger.level = Logger::ERROR
end
......@@ -279,6 +279,7 @@ Rails.application.routes.draw do
resource :tag_implication_request, :only => [:new, :create]
resources :uploads do
collection do
post :preprocess
get :batch
get :image_proxy
end
......
class AddMissingFieldsToUploads < ActiveRecord::Migration[5.2]
def change
add_column :uploads, :md5, :string
add_column :uploads, :file_ext, :string
add_column :uploads, :file_size, :integer
add_column :uploads, :image_width, :integer
add_column :uploads, :image_height, :integer
add_column :uploads, :artist_commentary_desc, :text
add_column :uploads, :artist_commentary_title, :text
add_column :uploads, :include_artist_commentary, :boolean
end
end
class AddContextToUploads < ActiveRecord::Migration[5.2]
def change
add_column :uploads, :context, :text
end
end
......@@ -14,6 +14,14 @@ FactoryBot.define do
source "http://www.google.com/intl/en_ALL/images/logo.gif"
end
factory(:ugoira_upload) do
file do
f = Tempfile.new
IO.copy_stream("#{Rails.root}/test/fixtures/ugoira.zip", f.path)
ActionDispatch::Http::UploadedFile.new(tempfile: f, filename: "ugoira.zip")
end
end
factory(:jpg_upload) do
file do
f = Tempfile.new
......
......@@ -38,6 +38,15 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
assert_response :success
end
context "with a url" do
should "preprocess" do
assert_difference(-> { Upload.count }) do
get_auth new_upload_path, @user, params: {:url => "http://www.google.com/intl/en_ALL/images/logo.gif"}
assert_response :success
end
end
end
context "for a twitter post" do
should "render" do
skip "Twitter keys are not set" unless Danbooru.config.twitter_api_key
......@@ -49,13 +58,15 @@ class UploadsControllerTest < ActionDispatch::IntegrationTest
context "for a post that has already been uploaded" do
setup do
as_user do
@post = create(:post, :source => "aaa")
@post = create(:post, :source => "http://google.com/aaa")
end
end
should "initialize the post" do
get_auth new_upload_path, @user, params: {:url => "http://google.com/aaa"}
assert_response :success
assert_difference(-> { Upload.count }, 0) do
get_auth new_upload_path, @user, params: {:url => "http://google.com/aaa"}
assert_response :success
end
end
end
end
......
This diff is collapsed.
This diff is collapsed.
......@@ -28,8 +28,7 @@ class PostTest < ActiveSupport::TestCase
context "Deletion:" do
context "Expunging a post" do
setup do
@upload = FactoryBot.create(:jpg_upload)
@upload.process!
@upload = UploadService.new(FactoryBot.attributes_for(:jpg_upload)).start!
@post = @upload.post
Favorite.add(post: @post, user: @user)
end
......@@ -2677,4 +2676,19 @@ class PostTest < ActiveSupport::TestCase
end
end
end
context "#replace!" do
subject { @post.replace!(tags: "something", replacement_url: "https://danbooru.donmai.us/data/preview/download.png") }
setup do
@post = FactoryBot.create(:post)
@post.stubs(:queue_delete_files)
end
should "update the post" do
assert_changes(-> { @post.md5 }) do
subject
end
end
end
end
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment