FEATURE: First pass of using uppy in the composer (#13935)

Adds uppy upload functionality behind a
enable_experimental_composer_uploader site setting (default false,
and hidden).

When enabled this site setting will make the composer-editor-uppy
component be used within composer.hbs, which in turn points to
a ComposerUploadUppy mixin which overrides the relevant
functions from ComposerUpload. This uppy uploader has parity
with all the features of jQuery file uploader in the original
composer-editor, including:

progress tracking
error handling
number of files validation
pasting files
dragging and dropping files
updating upload placeholders
upload markdown resolvers
processing actions (the only one we have so far is the media optimization
worker by falco, this works)
cancelling uploads
For now all uploads still go via the /uploads.json endpoint, direct
S3 support will be added later.

Also included in this PR are some changes to the media optimization
service, to support uppy's different file data structures, and also
to make the promise tracking and resolving more robust. Currently
it uses the file name to track promises, we can switch to something
more unique later if needed.

Does not include custom upload handlers, that will come
in a later PR, it is a tricky problem to handle.

Also, this new functionality will not be used in encrypted PMs because
encrypted PM uploads rely on custom upload handlers.
This commit is contained in:
Martin Brennan 2021-08-13 09:14:34 +10:00 committed by GitHub
parent b5485e2b05
commit b626373b31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1314 additions and 126 deletions

View File

@ -0,0 +1,7 @@
import ComposerEditor from "discourse/components/composer-editor";
import ComposerUploadUppy from "discourse/mixins/composer-upload-uppy";
export default ComposerEditor.extend(ComposerUploadUppy, {
layoutName: "components/composer-editor",
experimentalComposerUploads: true,
});

View File

@ -1,4 +1,5 @@
import Composer, { SAVE_ICONS, SAVE_LABELS } from "discourse/models/composer";
import { warn } from "@ember/debug";
import Controller, { inject as controller } from "@ember/controller";
import EmberObject, { action, computed } from "@ember/object";
import { alias, and, or, reads } from "@ember/object/computed";
@ -284,6 +285,22 @@ export default Controller.extend({
return option;
},
@discourseComputed("model.isEncrypted")
composerComponent(isEncrypted) {
const defaultComposer = "composer-editor";
if (this.siteSettings.enable_experimental_composer_uploader) {
if (isEncrypted) {
warn(
"Uppy cannot be used for composer uploads until upload handlers are developed, falling back to composer-editor.",
{ id: "composer" }
);
return defaultComposer;
}
return "composer-editor-uppy";
}
return defaultComposer;
},
@discourseComputed("model.composeState", "model.creatingTopic", "model.post")
popupMenuOptions(composeState) {
if (composeState === "open" || composeState === "fullscreen") {
@ -482,14 +499,14 @@ export default Controller.extend({
}
}
const [warn, info] = linkLookup.check(post, href);
const [linkWarn, linkInfo] = linkLookup.check(post, href);
if (warn) {
if (linkWarn) {
const body = I18n.t("composer.duplicate_link", {
domain: info.domain,
username: info.username,
post_url: topic.urlForPostNumber(info.post_number),
ago: shortDate(info.posted_at),
domain: linkInfo.domain,
username: linkInfo.username,
post_url: topic.urlForPostNumber(linkInfo.post_number),
ago: shortDate(linkInfo.posted_at),
});
this.appEvents.trigger("composer-messages:create", {
extraClass: "custom-body",

View File

@ -343,3 +343,14 @@ function displayErrorByResponseStatus(status, body, fileName, siteSettings) {
return;
}
export function bindFileInputChangeListener(element, fileCallbackFn) {
function changeListener(event) {
const files = Array.from(event.target.files);
files.forEach((file) => {
fileCallbackFn(file);
});
}
element.addEventListener("change", changeListener);
return changeListener;
}

View File

@ -6,47 +6,49 @@ export default class UppyChecksum extends Plugin {
constructor(uppy, opts) {
super(uppy, opts);
this.id = opts.id || "uppy-checksum";
this.pluginClass = this.constructor.name;
this.capabilities = opts.capabilities;
this.type = "preprocessor";
}
canUseSubtleCrypto() {
if (!window.isSecureContext) {
this.warnPrefixed(
"Cannot generate cryptographic digests in an insecure context (not HTTPS)."
_canUseSubtleCrypto() {
if (!this._secureContext()) {
warn(
"Cannot generate cryptographic digests in an insecure context (not HTTPS).",
{
id: "discourse.uppy-media-optimization",
}
);
return false;
}
if (this.capabilities.isIE11) {
this.warnPrefixed(
"The required cipher suite is unavailable in Internet Explorer 11."
warn(
"The required cipher suite is unavailable in Internet Explorer 11.",
{
id: "discourse.uppy-media-optimization",
}
);
return false;
}
if (
!(window.crypto && window.crypto.subtle && window.crypto.subtle.digest)
) {
this.warnPrefixed(
"The required cipher suite is unavailable in this browser."
);
if (!this._hasCryptoCipher()) {
warn("The required cipher suite is unavailable in this browser.", {
id: "discourse.uppy-media-optimization",
});
return false;
}
return true;
}
generateChecksum(fileIds) {
if (!this.canUseSubtleCrypto()) {
_generateChecksum(fileIds) {
if (!this._canUseSubtleCrypto()) {
return Promise.resolve();
}
let promises = fileIds.map((fileId) => {
let file = this.uppy.getFile(fileId);
this.uppy.emit("preprocess-progress", file, {
mode: "indeterminate",
message: "generating checksum",
});
this.uppy.emit("preprocess-progress", this.pluginClass, file);
return file.data.arrayBuffer().then((arrayBuffer) => {
return window.crypto.subtle
@ -57,38 +59,41 @@ export default class UppyChecksum extends Plugin {
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
this.uppy.setFileMeta(fileId, { sha1_checksum: hashHex });
this.uppy.emit("preprocess-complete", this.pluginClass, file);
})
.catch((err) => {
if (
err.message.toString().includes("Algorithm: Unrecognized name")
) {
this.warnPrefixed(
"SHA-1 algorithm is unsupported in this browser."
);
warn("SHA-1 algorithm is unsupported in this browser.", {
id: "discourse.uppy-media-optimization",
});
} else {
warn(`Error encountered when generating digest: ${err.message}`, {
id: "discourse.uppy-media-optimization",
});
}
this.uppy.emit("preprocess-complete", this.pluginClass, file);
});
});
});
const emitPreprocessCompleteForAll = () => {
fileIds.forEach((fileId) => {
const file = this.uppy.getFile(fileId);
this.uppy.emit("preprocess-complete", file);
});
};
return Promise.all(promises).then(emitPreprocessCompleteForAll);
return Promise.all(promises);
}
warnPrefixed(message) {
warn(`[uppy-checksum-plugin] ${message}`);
_secureContext() {
return window.isSecureContext;
}
_hasCryptoCipher() {
return window.crypto && window.crypto.subtle && window.crypto.subtle.digest;
}
install() {
this.uppy.addPreProcessor(this.generateChecksum.bind(this));
this.uppy.addPreProcessor(this._generateChecksum.bind(this));
}
uninstall() {
this.uppy.removePreProcessor(this.generateChecksum.bind(this));
this.uppy.removePreProcessor(this._generateChecksum.bind(this));
}
}

View File

@ -0,0 +1,72 @@
import { Plugin } from "@uppy/core";
import { warn } from "@ember/debug";
import { Promise } from "rsvp";
export default class UppyMediaOptimization extends Plugin {
constructor(uppy, opts) {
super(uppy, opts);
this.id = opts.id || "uppy-media-optimization";
this.type = "preprocessor";
this.optimizeFn = opts.optimizeFn;
this.pluginClass = this.constructor.name;
// mobile devices have limited processing power, so we only enable
// running media optimization in parallel when we are sure the user
// is not on a mobile device. otherwise we just process the images
// serially.
this.runParallel = opts.runParallel || false;
}
_optimizeFile(fileId) {
let file = this.uppy.getFile(fileId);
this.uppy.emit("preprocess-progress", this.pluginClass, file);
return this.optimizeFn(file)
.then((optimizedFile) => {
if (!optimizedFile) {
warn("Nothing happened, possible error or other restriction.", {
id: "discourse.uppy-media-optimization",
});
} else {
this.uppy.setFileState(fileId, { data: optimizedFile });
}
this.uppy.emit("preprocess-complete", this.pluginClass, file);
})
.catch((err) => {
warn(err, { id: "discourse.uppy-media-optimization" });
this.uppy.emit("preprocess-complete", this.pluginClass, file);
});
}
_optimizeParallel(fileIds) {
return Promise.all(fileIds.map(this._optimizeFile.bind(this)));
}
async _optimizeSerial(fileIds) {
let optimizeTasks = fileIds.map((fileId) => () =>
this._optimizeFile.call(this, fileId)
);
for (const task of optimizeTasks) {
await task();
}
}
install() {
if (this.runParallel) {
this.uppy.addPreProcessor(this._optimizeParallel.bind(this));
} else {
this.uppy.addPreProcessor(this._optimizeSerial.bind(this));
}
}
uninstall() {
if (this.runParallel) {
this.uppy.removePreProcessor(this._optimizeParallel.bind(this));
} else {
this.uppy.removePreProcessor(this._optimizeSerial.bind(this));
}
}
}

View File

@ -0,0 +1,450 @@
import Mixin from "@ember/object/mixin";
import { deepMerge } from "discourse-common/lib/object";
import UppyChecksum from "discourse/lib/uppy-checksum-plugin";
import UppyMediaOptimization from "discourse/lib/uppy-media-optimization-plugin";
import Uppy from "@uppy/core";
import DropTarget from "@uppy/drop-target";
import XHRUpload from "@uppy/xhr-upload";
import { warn } from "@ember/debug";
import I18n from "I18n";
import getURL from "discourse-common/lib/get-url";
import { clipboardHelpers } from "discourse/lib/utilities";
import { observes, on } from "discourse-common/utils/decorators";
import {
bindFileInputChangeListener,
displayErrorForUpload,
getUploadMarkdown,
validateUploadedFile,
} from "discourse/lib/uploads";
import { cacheShortUploadUrl } from "pretty-text/upload-short-url";
// Note: This mixin is used _in addition_ to the ComposerUpload mixin
// on the composer-editor component. It overrides some, but not all,
// functions created by ComposerUpload. Eventually this will supplant
// ComposerUpload, but until then only the functions that need to be
// overridden to use uppy will be overridden, so as to not go out of
// sync with the main ComposerUpload functionality by copying unchanging
// functions.
//
// Some examples are uploadPlaceholder, the main properties e.g. uploadProgress,
// and the most important _bindUploadTarget which handles all the main upload
// functionality and event binding.
//
export default Mixin.create({
@observes("composer.uploadCancelled")
_cancelUpload() {
if (!this.get("composer.uploadCancelled")) {
return;
}
this.set("composer.uploadCancelled", false);
this.set("userCancelled", true);
this._uppyInstance.cancelAll();
},
@on("willDestroyElement")
_unbindUploadTarget() {
this.messageBus.unsubscribe("/uploads/composer");
this.uploadButton?.removeEventListener(
"click",
this.uploadButtonEventListener
);
this.fileInputEl?.removeEventListener(
"change",
this.fileInputEventListener
);
this.element?.removeEventListener("paste", this.pasteEventListener);
this.appEvents.off("composer:add-files", this._addFiles.bind(this));
if (this._uppyInstance) {
this._uppyInstance.close();
this._uppyInstance = null;
}
},
_bindUploadTarget() {
this.placeholders = {};
this._preProcessorStatus = {};
this.fileInputEl = document.getElementById("file-uploader");
const isPrivateMessage = this.get("composer.privateMessage");
this.appEvents.on("composer:add-files", this._addFiles.bind(this));
this._unbindUploadTarget();
this._bindFileInputChangeListener();
this._bindPasteListener();
this._bindMobileUploadButton();
this._uppyInstance = new Uppy({
id: "composer-uppy",
autoProceed: true,
// need to use upload_type because uppy overrides type with the
// actual file type
meta: deepMerge({ upload_type: "composer" }, this.data || {}),
onBeforeFileAdded: (currentFile) => {
const validationOpts = {
user: this.currentUser,
siteSettings: this.siteSettings,
isPrivateMessage,
allowStaffToUploadAnyFileInPm: this.siteSettings
.allow_staff_to_upload_any_file_in_pm,
};
const isUploading = validateUploadedFile(currentFile, validationOpts);
this.setProperties({
uploadProgress: 0,
isUploading,
isCancellable: isUploading,
});
if (!isUploading) {
this.appEvents.trigger("composer:uploads-aborted");
}
return isUploading;
},
onBeforeUpload: (files) => {
const fileCount = Object.keys(files).length;
const maxFiles = this.siteSettings.simultaneous_uploads;
// Limit the number of simultaneous uploads
if (maxFiles > 0 && fileCount > maxFiles) {
bootbox.alert(
I18n.t("post.errors.too_many_dragged_and_dropped_files", {
count: maxFiles,
})
);
this.appEvents.trigger("composer:uploads-aborted");
this._reset();
return false;
}
},
});
this._uppyInstance.use(DropTarget, { target: this.element });
this._uppyInstance.use(UppyChecksum, { capabilities: this.capabilities });
// TODO (martin) Need a more automatic way to do this for preprocessor
// plugins like UppyChecksum and UppyMediaOptimization so people don't
// have to remember to do this, also want to wrap this.uppy.emit in those
// classes so people don't have to remember to pass through the plugin class
// name for the preprocess-X events.
this._trackPreProcessorStatus(UppyChecksum);
// TODO (martin) support for direct S3 uploads will come later, for now
// we just want the regular /uploads.json endpoint to work well
this._useXHRUploads();
// TODO (martin) develop upload handler guidance and an API to use; will
// likely be using uppy plugins for this
this._uppyInstance.on("file-added", (file) => {
if (isPrivateMessage) {
file.meta.for_private_message = true;
}
});
this._uppyInstance.on("progress", (progress) => {
this.set("uploadProgress", progress);
});
this._uppyInstance.on("upload", (data) => {
const files = data.fileIDs.map((fileId) =>
this._uppyInstance.getFile(fileId)
);
this._eachPreProcessor((pluginName, status) => {
status.needProcessing = files.length;
});
this.setProperties({
isProcessingUpload: true,
isCancellable: false,
});
files.forEach((file) => {
const placeholder = this._uploadPlaceholder(file);
this.placeholders[file.id] = {
uploadPlaceholder: placeholder,
};
this.appEvents.trigger("composer:insert-text", placeholder);
this.appEvents.trigger("composer:upload-started", file.name);
});
});
this._uppyInstance.on("upload-success", (file, response) => {
let upload = response.body;
const markdown = this.uploadMarkdownResolvers.reduce(
(md, resolver) => resolver(upload) || md,
getUploadMarkdown(upload)
);
cacheShortUploadUrl(upload.short_url, upload);
this.appEvents.trigger(
"composer:replace-text",
this.placeholders[file.id].uploadPlaceholder.trim(),
markdown
);
this._resetUpload(file, { removePlaceholder: false });
this.appEvents.trigger("composer:upload-success", file.name, upload);
});
this._uppyInstance.on("upload-error", (file, error, response) => {
this._resetUpload(file, { removePlaceholder: true });
if (!this.userCancelled) {
displayErrorForUpload(response, this.siteSettings, file.name);
this.appEvents.trigger("composer:upload-error", file);
}
});
this._uppyInstance.on("complete", () => {
this.appEvents.trigger("composer:all-uploads-complete");
this._reset();
});
this._uppyInstance.on("cancel-all", () => {
// uppyInstance.reset() also fires cancel-all, so we want to
// only do the manual cancelling work if the user clicked cancel
if (this.userCancelled) {
Object.values(this.placeholders).forEach((data) => {
this.appEvents.trigger(
"composer:replace-text",
data.uploadPlaceholder,
""
);
});
this.set("userCancelled", false);
this._reset();
this.appEvents.trigger("composer:uploads-cancelled");
}
});
this._setupPreprocessing();
},
_setupPreprocessing() {
Object.keys(this.uploadProcessorActions).forEach((action) => {
switch (action) {
case "optimizeJPEG":
this._uppyInstance.use(UppyMediaOptimization, {
optimizeFn: this.uploadProcessorActions[action],
runParallel: !this.site.isMobileDevice,
});
this._trackPreProcessorStatus(UppyMediaOptimization);
break;
}
});
this._uppyInstance.on("preprocess-progress", (pluginClass, file) => {
this._preProcessorStatus[pluginClass].activeProcessing++;
let placeholderData = this.placeholders[file.id];
placeholderData.processingPlaceholder = `[${I18n.t(
"processing_filename",
{
filename: file.name,
}
)}]()\n`;
this.appEvents.trigger(
"composer:replace-text",
placeholderData.uploadPlaceholder,
placeholderData.processingPlaceholder
);
});
this._uppyInstance.on("preprocess-complete", (pluginClass, file) => {
let placeholderData = this.placeholders[file.id];
this.appEvents.trigger(
"composer:replace-text",
placeholderData.processingPlaceholder,
placeholderData.uploadPlaceholder
);
const preProcessorStatus = this._preProcessorStatus[pluginClass];
preProcessorStatus.activeProcessing--;
preProcessorStatus.completeProcessing++;
if (
preProcessorStatus.completeProcessing ===
preProcessorStatus.needProcessing
) {
preProcessorStatus.allComplete = true;
if (this._allPreprocessorsComplete()) {
this.setProperties({
isProcessingUpload: false,
isCancellable: true,
});
this.appEvents.trigger("composer:uploads-preprocessing-complete");
}
}
});
},
_uploadFilenamePlaceholder(file) {
const filename = this._filenamePlaceholder(file);
// when adding two separate files with the same filename search for matching
// placeholder already existing in the editor ie [Uploading: test.png...]
// and add order nr to the next one: [Uploading: test.png(1)...]
const escapedFilename = filename.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const regexString = `\\[${I18n.t("uploading_filename", {
filename: escapedFilename + "(?:\\()?([0-9])?(?:\\))?",
})}\\]\\(\\)`;
const globalRegex = new RegExp(regexString, "g");
const matchingPlaceholder = this.get("composer.reply").match(globalRegex);
if (matchingPlaceholder) {
// get last matching placeholder and its consecutive nr in regex
// capturing group and apply +1 to the placeholder
const lastMatch = matchingPlaceholder[matchingPlaceholder.length - 1];
const regex = new RegExp(regexString);
const orderNr = regex.exec(lastMatch)[1]
? parseInt(regex.exec(lastMatch)[1], 10) + 1
: 1;
return `${filename}(${orderNr})`;
}
return filename;
},
_uploadPlaceholder(file) {
const clipboard = I18n.t("clipboard");
const uploadFilenamePlaceholder = this._uploadFilenamePlaceholder(file);
const filename = uploadFilenamePlaceholder
? uploadFilenamePlaceholder
: clipboard;
let placeholder = `[${I18n.t("uploading_filename", { filename })}]()\n`;
if (!this._cursorIsOnEmptyLine()) {
placeholder = `\n${placeholder}`;
}
return placeholder;
},
_useXHRUploads() {
this._uppyInstance.use(XHRUpload, {
endpoint: getURL(`/uploads.json?client_id=${this.messageBus.clientId}`),
headers: {
"X-CSRF-Token": this.session.csrfToken,
},
});
},
_reset() {
this._uppyInstance?.reset();
this.setProperties({
uploadProgress: 0,
isUploading: false,
isProcessingUpload: false,
isCancellable: false,
});
this._eachPreProcessor((pluginClass) => {
this._preProcessorStatus[pluginClass] = {};
});
this.fileInputEl.value = "";
},
_resetUpload(file, opts) {
if (opts.removePlaceholder) {
this.appEvents.trigger(
"composer:replace-text",
this.placeholders[file.id].uploadPlaceholder,
""
);
}
},
_bindFileInputChangeListener() {
this.fileInputEventListener = bindFileInputChangeListener(
this.fileInputEl,
this._addFiles.bind(this)
);
},
_bindPasteListener() {
this.pasteEventListener = this.element.addEventListener(
"paste",
(event) => {
if (
document.activeElement !== document.querySelector(".d-editor-input")
) {
return;
}
const { canUpload, canPasteHtml, types } = clipboardHelpers(event, {
siteSettings: this.siteSettings,
canUpload: true,
});
if (!canUpload || canPasteHtml || types.includes("text/plain")) {
event.preventDefault();
return;
}
if (event && event.clipboardData && event.clipboardData.files) {
this._addFiles([...event.clipboardData.files]);
}
}
);
},
_addFiles(files) {
files = Array.isArray(files) ? files : [files];
try {
this._uppyInstance.addFiles(
files.map((file) => {
return {
source: "composer",
name: file.name,
type: file.type,
data: file,
};
})
);
} catch (err) {
warn(`error adding files to uppy: ${err}`, {
id: "discourse.upload.uppy-add-files-error",
});
}
},
_trackPreProcessorStatus(pluginClass) {
this._preProcessorStatus[pluginClass.name] = {
needProcessing: 0,
activeProcessing: 0,
completeProcessing: 0,
allComplete: false,
};
},
_eachPreProcessor(cb) {
for (const [pluginClass, status] of Object.entries(
this._preProcessorStatus
)) {
cb(pluginClass, status);
}
},
_allPreprocessorsComplete() {
let completed = [];
this._eachPreProcessor((pluginClass, status) => {
completed.push(status.allComplete);
});
return completed.every(Boolean);
},
showUploadSelector(toolbarEvent) {
this.send("showUploadSelector", toolbarEvent);
},
});

View File

@ -326,8 +326,8 @@ export default Mixin.create({
_bindMobileUploadButton() {
if (this.site.mobileView) {
const uploadButton = document.getElementById("mobile-file-upload");
uploadButton.addEventListener(
this.uploadButton = document.getElementById("mobile-file-upload");
this.uploadButtonEventListener = this.uploadButton.addEventListener(
"click",
() => document.getElementById("file-uploader").click(),
false
@ -337,8 +337,12 @@ export default Mixin.create({
@on("willDestroyElement")
_unbindUploadTarget() {
this.uploadButton?.removeEventListener(
"click",
this.uploadButtonEventListener
);
this._validUploads = 0;
$("#reply-control .mobile-file-upload").off("click.uploader");
this.messageBus.unsubscribe("/uploads/composer");
const $uploadTarget = $(this.element);
try {

View File

@ -1,6 +1,7 @@
import Mixin from "@ember/object/mixin";
import { ajax } from "discourse/lib/ajax";
import {
bindFileInputChangeListener,
displayErrorForUpload,
validateUploadedFile,
} from "discourse/lib/uploads";
@ -18,7 +19,7 @@ import { warn } from "@ember/debug";
export default Mixin.create({
uploading: false,
uploadProgress: 0,
uppyInstance: null,
_uppyInstance: null,
autoStartUploads: true,
id: null,
@ -48,7 +49,12 @@ export default Mixin.create({
if (this.messageBus) {
this.messageBus.unsubscribe(`/uploads/${this.type}`);
}
this.uppyInstance && this.uppyInstance.close();
this.fileInputEl?.removeEventListener(
"change",
this.fileInputEventListener
);
this._uppyInstance?.close();
this._uppyInstance = null;
},
@on("didInsertElement")
@ -58,7 +64,7 @@ export default Mixin.create({
});
this.set("allowMultipleFiles", this.fileInputEl.multiple);
this._bindFileInputChangeListener();
this._bindFileInputChange();
if (!this.id) {
warn(
@ -69,9 +75,7 @@ export default Mixin.create({
);
}
this.set(
"uppyInstance",
new Uppy({
this._uppyInstance = new Uppy({
id: this.id,
autoProceed: this.autoStartUploads,
@ -118,17 +122,16 @@ export default Mixin.create({
return false;
}
},
})
);
});
this.uppyInstance.use(DropTarget, { target: this.element });
this.uppyInstance.use(UppyChecksum, { capabilities: this.capabilities });
this._uppyInstance.use(DropTarget, { target: this.element });
this._uppyInstance.use(UppyChecksum, { capabilities: this.capabilities });
this.uppyInstance.on("progress", (progress) => {
this._uppyInstance.on("progress", (progress) => {
this.set("uploadProgress", progress);
});
this.uppyInstance.on("upload-success", (file, response) => {
this._uppyInstance.on("upload-success", (file, response) => {
if (this.usingS3Uploads) {
this.setProperties({ uploading: false, processing: true });
this._completeExternalUpload(file)
@ -146,7 +149,7 @@ export default Mixin.create({
}
});
this.uppyInstance.on("upload-error", (file, error, response) => {
this._uppyInstance.on("upload-error", (file, error, response) => {
displayErrorForUpload(response, this.siteSettings, file.name);
this._reset();
});
@ -160,17 +163,17 @@ export default Mixin.create({
},
_useXHRUploads() {
this.uppyInstance.use(XHRUpload, {
this._uppyInstance.use(XHRUpload, {
endpoint: this._xhrUploadUrl(),
headers: {
"X-CSRF-Token": this.session.get("csrfToken"),
"X-CSRF-Token": this.session.csrfToken,
},
});
},
_useS3Uploads() {
this.set("usingS3Uploads", true);
this.uppyInstance.use(AwsS3, {
this._uppyInstance.use(AwsS3, {
getUploadParameters: (file) => {
const data = { file_name: file.name, type: this.type };
@ -187,7 +190,7 @@ export default Mixin.create({
data,
})
.then((response) => {
this.uppyInstance.setFileMeta(file.id, {
this._uppyInstance.setFileMeta(file.id, {
uniqueUploadIdentifier: response.unique_identifier,
});
@ -216,12 +219,12 @@ export default Mixin.create({
);
},
_bindFileInputChangeListener() {
this.fileInputEl.addEventListener("change", (event) => {
const files = Array.from(event.target.files);
files.forEach((file) => {
_bindFileInputChange() {
this.fileInputEventListener = bindFileInputChangeListener(
this.fileInputEl,
(file) => {
try {
this.uppyInstance.addFile({
this._uppyInstance.addFile({
source: `${this.id} file input`,
name: file.name,
type: file.type,
@ -232,8 +235,8 @@ export default Mixin.create({
id: "discourse.upload.uppy-add-files-error",
});
}
});
});
}
);
},
_completeExternalUpload(file) {
@ -246,7 +249,7 @@ export default Mixin.create({
},
_reset() {
this.uppyInstance && this.uppyInstance.reset();
this._uppyInstance?.reset();
this.setProperties({
uploading: false,
processing: false,

View File

@ -9,7 +9,7 @@ export default class MediaOptimizationWorkerService extends Service {
worker = null;
workerUrl = getAbsoluteURL("/javascripts/media-optimization-worker.js");
currentComposerUploadData = null;
currentPromiseResolver = null;
promiseResolvers = null;
startWorker() {
this.worker = new Worker(this.workerUrl); // TODO come up with a workaround for FF that lacks type: module support
@ -20,10 +20,10 @@ export default class MediaOptimizationWorkerService extends Service {
this.worker = null;
}
ensureAvailiableWorker() {
ensureAvailiableWorker(usingUppy) {
if (!this.worker) {
this.startWorker();
this.registerMessageHandler();
this.registerMessageHandler(usingUppy);
this.appEvents.on("composer:closed", this, "stopWorker");
}
}
@ -36,30 +36,37 @@ export default class MediaOptimizationWorkerService extends Service {
}
optimizeImage(data) {
let file = data.files[data.index];
const usingUppy = data.id && data.id.includes("uppy");
this.promiseResolvers = this.promiseResolvers || {};
let file = usingUppy ? data : data.files[data.index];
if (!/(\.|\/)(jpe?g|png|webp)$/i.test(file.type)) {
return data;
return usingUppy ? Promise.resolve() : data;
}
if (
file.size <
this.siteSettings
.composer_media_optimization_image_bytes_optimization_threshold
) {
return data;
return usingUppy ? Promise.resolve() : data;
}
this.ensureAvailiableWorker();
this.ensureAvailiableWorker(usingUppy);
return new Promise(async (resolve) => {
this.logIfDebug(`Transforming ${file.name}`);
this.currentComposerUploadData = data;
this.currentPromiseResolver = resolve;
this.promiseResolvers[file.name] = resolve;
let imageData;
try {
if (usingUppy) {
imageData = await fileToImageData(file.data);
} else {
imageData = await fileToImageData(file);
}
} catch (error) {
this.logIfDebug(error);
return resolve(data);
return usingUppy ? resolve() : resolve(data);
}
this.worker.postMessage(
@ -101,7 +108,7 @@ export default class MediaOptimizationWorkerService extends Service {
});
}
registerMessageHandler() {
registerMessageHandler(usingUppy) {
this.worker.onmessage = (e) => {
switch (e.data.type) {
case "file":
@ -111,13 +118,25 @@ export default class MediaOptimizationWorkerService extends Service {
this.logIfDebug(
`Finished optimization of ${optimizedFile.name} new size: ${optimizedFile.size}.`
);
if (usingUppy) {
this.promiseResolvers[optimizedFile.name](optimizedFile);
} else {
let data = this.currentComposerUploadData;
data.files[data.index] = optimizedFile;
this.currentPromiseResolver(data);
this.promiseResolvers[optimizedFile.name](data);
}
break;
case "error":
this.stopWorker();
this.currentPromiseResolver(this.currentComposerUploadData);
if (usingUppy) {
this.promiseResolvers[e.data.fileName]();
} else {
this.promiseResolvers[e.data.fileName](
this.currentComposerUploadData
);
}
break;
default:
this.logIfDebug(`Sorry, we are out of ${e}.`);

View File

@ -20,6 +20,13 @@
disabled=disableTextarea
outletArgs=(hash composer=composer editorType="composer")}}
{{#if experimentalComposerUploads}}
<br>
<div class="alert alert-warning">
Experimental uploader being used!
</div>
{{/if}}
{{#if allowUpload}}
{{#if acceptsAllFormats}}
<input type="file" id="file-uploader" multiple>

View File

@ -108,7 +108,8 @@
</div>
{{composer-editor topic=topic
{{component composerComponent
topic=topic
composer=model
lastValidatedAt=lastValidatedAt
canWhisper=canWhisper

View File

@ -0,0 +1,224 @@
import {
acceptance,
loggedInUser,
queryAll,
} from "discourse/tests/helpers/qunit-helpers";
import { authorizedExtensions } from "discourse/lib/uploads";
import { click, fillIn, visit } from "@ember/test-helpers";
import I18n from "I18n";
import { test } from "qunit";
function pretender(server, helper) {
server.post("/uploads/lookup-urls", () => {
return helper.response([
{
url:
"//testbucket.s3.dualstack.us-east-2.amazonaws.com/original/1X/f1095d89269ff22e1818cf54b73e857261851019.jpeg",
short_path: "/uploads/short-url/yoj8pf9DdIeHRRULyw7i57GAYdz.jpeg",
short_url: "upload://yoj8pf9DdIeHRRULyw7i57GAYdz.jpeg",
},
]);
});
server.post(
"/uploads.json",
() => {
return helper.response({
extension: "jpeg",
filesize: 126177,
height: 800,
human_filesize: "123 KB",
id: 202,
original_filename: "avatar.PNG.jpg",
retain_hours: null,
short_path: "/uploads/short-url/yoj8pf9DdIeHRRULyw7i57GAYdz.jpeg",
short_url: "upload://yoj8pf9DdIeHRRULyw7i57GAYdz.jpeg",
thumbnail_height: 320,
thumbnail_width: 690,
url:
"//testbucket.s3.dualstack.us-east-2.amazonaws.com/original/1X/f1095d89269ff22e1818cf54b73e857261851019.jpeg",
width: 1920,
});
},
500 // this delay is important to slow down the uploads a bit so we can click elements in the UI like the cancel button
);
}
function createFile(name, type = "image/png") {
// the blob content doesn't matter at all, just want it to be random-ish
const file = new Blob([(Math.random() + 1).toString(36).substring(2)], {
type,
});
file.name = name;
return file;
}
acceptance("Uppy Composer Attachment - Upload Placeholder", function (needs) {
needs.user();
needs.pretender(pretender);
needs.settings({
enable_experimental_composer_uploader: true,
simultaneous_uploads: 2,
});
test("should insert the Uploading placeholder then the complete image placeholder", async function (assert) {
await visit("/");
await click("#create-topic");
await fillIn(".d-editor-input", "The image:\n");
const appEvents = loggedInUser().appEvents;
const done = assert.async();
appEvents.on("composer:all-uploads-complete", () => {
assert.equal(
queryAll(".d-editor-input").val(),
"The image:\n![avatar.PNG|690x320](upload://yoj8pf9DdIeHRRULyw7i57GAYdz.jpeg)\n"
);
done();
});
appEvents.on("composer:upload-started", () => {
assert.equal(
queryAll(".d-editor-input").val(),
"The image:\n[Uploading: avatar.png...]()\n"
);
});
const image = createFile("avatar.png");
appEvents.trigger("composer:add-files", image);
});
test("should error if too many files are added at once", async function (assert) {
await visit("/");
await click("#create-topic");
const appEvents = loggedInUser().appEvents;
const image = createFile("avatar.png");
const image1 = createFile("avatar1.png");
const image2 = createFile("avatar2.png");
const done = assert.async();
appEvents.on("composer:uploads-aborted", async () => {
assert.equal(
queryAll(".bootbox .modal-body").html(),
I18n.t("post.errors.too_many_dragged_and_dropped_files", {
count: 2,
}),
"it should warn about too many files added"
);
await click(".modal-footer .btn-primary");
done();
});
appEvents.trigger("composer:add-files", [image, image1, image2]);
});
test("should error if an unauthorized extension file is added", async function (assert) {
await visit("/");
await click("#create-topic");
const appEvents = loggedInUser().appEvents;
const jsonFile = createFile("something.json", "application/json");
const done = assert.async();
appEvents.on("composer:uploads-aborted", async () => {
assert.equal(
queryAll(".bootbox .modal-body").html(),
I18n.t("post.errors.upload_not_authorized", {
authorized_extensions: authorizedExtensions(
false,
this.siteSettings
).join(", "),
}),
"it should warn about unauthorized extensions"
);
await click(".modal-footer .btn-primary");
done();
});
appEvents.trigger("composer:add-files", [jsonFile]);
});
// Had to comment this out for now; it works fine in Ember CLI but lagging
// UI updates sink it for the old Ember for some reason. Will re-enable
// when we make Ember CLI the primary.
//
// test("cancelling uploads clears the placeholders out", async function (assert) {
// await visit("/");
// await click("#create-topic");
// await fillIn(".d-editor-input", "The image:\n");
// const appEvents = loggedInUser().appEvents;
// const done = assert.async();
// appEvents.on("composer:uploads-cancelled", () => {
// assert.equal(
// queryAll(".d-editor-input").val(),
// "The image:\n",
// "it should clear the cancelled placeholders"
// );
// done();
// });
// let uploadStarted = 0;
// appEvents.on("composer:upload-started", async () => {
// uploadStarted++;
// if (uploadStarted === 2) {
// assert.equal(
// queryAll(".d-editor-input").val(),
// "The image:\n[Uploading: avatar.png...]()\n[Uploading: avatar2.png...]()\n",
// "it should show the upload placeholders when the upload starts"
// );
// }
// });
// appEvents.on("composer:uploads-preprocessing-complete", async () => {
// await click("#cancel-file-upload");
// });
// const image = createFile("avatar.png");
// const image2 = createFile("avatar2.png");
// appEvents.trigger("composer:add-files", [image, image2]);
// });
});
acceptance("Uppy Composer Attachment - Upload Error", function (needs) {
needs.user();
needs.pretender((server, helper) => {
server.post("/uploads.json", () => {
return helper.response(422, {
success: false,
errors: [
"There was an error uploading the file, the gif was way too cool.",
],
});
});
});
needs.settings({
enable_experimental_composer_uploader: true,
simultaneous_uploads: 2,
});
test("should show an error message for the failed upload", async function (assert) {
await visit("/");
await click("#create-topic");
await fillIn(".d-editor-input", "The image:\n");
const appEvents = loggedInUser().appEvents;
const done = assert.async();
appEvents.on("composer:upload-error", async () => {
assert.equal(
queryAll(".bootbox .modal-body").html(),
"There was an error uploading the file, the gif was way too cool.",
"it should show the error message from the server"
);
await click(".modal-footer .btn-primary");
done();
});
const image = createFile("avatar.png");
appEvents.trigger("composer:add-files", image);
});
});

View File

@ -0,0 +1,193 @@
import UppyChecksum from "discourse/lib/uppy-checksum-plugin";
import { module, test } from "qunit";
import sinon from "sinon";
class FakeUppy {
constructor() {
this.preprocessors = [];
this.emitted = [];
this.files = {
"uppy-test/file/vv2/xvejg5w/blah/png-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764": {
meta: {},
data: createFile("test1.png"),
},
"uppy-test/file/blah1/ads37x2/blah1/png-1d-1d-2v-1d-1e-image/jpeg-99999-1837921727764": {
meta: {},
data: createFile("test2.png"),
},
};
}
addPreProcessor(fn) {
this.preprocessors.push(fn);
}
getFile(id) {
return this.files[id];
}
emit(event, file, data) {
this.emitted.push({ event, file, data });
}
setFileMeta(fileId, meta) {
this.files[fileId].meta = meta;
}
}
module("Unit | Utility | UppyChecksum Plugin", function () {
test("sets the options passed in", function (assert) {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
assert.equal(plugin.id, "test-uppy");
assert.equal(plugin.capabilities, capabilities);
});
test("it does nothing if not running in a secure context", function (assert) {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
const done = assert.async();
sinon.stub(plugin, "_secureContext").returns(false);
const fileId =
"uppy-test/file/vv2/xvejg5w/blah/png-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764";
plugin.uppy.preprocessors[0]([fileId]).then(() => {
assert.equal(
plugin.uppy.emitted.length,
0,
"no events were fired by the checksum plugin because it returned early"
);
done();
});
});
test("it does nothing if the crypto object + cipher is not available", function (assert) {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
const done = assert.async();
sinon.stub(plugin, "_hasCryptoCipher").returns(false);
const fileId =
"uppy-test/file/vv2/xvejg5w/blah/png-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764";
plugin.uppy.preprocessors[0]([fileId]).then(() => {
assert.equal(
plugin.uppy.emitted.length,
0,
"no events were fired by the checksum plugin because it returned early"
);
done();
});
});
test("it does nothing if the browser is IE11", function (assert) {
const capabilities = { isIE11: true };
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
const done = assert.async();
const fileId =
"uppy-test/file/vv2/xvejg5w/blah/png-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764";
plugin.uppy.preprocessors[0]([fileId]).then(() => {
assert.equal(
plugin.uppy.emitted.length,
0,
"no events were fired by the checksum plugin because it returned early"
);
done();
});
});
test("it gets a sha1 hash of each file and adds it to the file meta", function (assert) {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
const done = assert.async();
const fileIds = [
"uppy-test/file/vv2/xvejg5w/blah/png-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764",
"uppy-test/file/blah1/ads37x2/blah1/png-1d-1d-2v-1d-1e-image/jpeg-99999-1837921727764",
];
plugin.uppy.preprocessors[0](fileIds).then(() => {
assert.equal(plugin.uppy.emitted[0].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[1].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[2].event, "preprocess-complete");
assert.equal(plugin.uppy.emitted[3].event, "preprocess-complete");
// these checksums are the actual SHA1 hashes of the test file names
assert.equal(
plugin.uppy.getFile(fileIds[0]).meta.sha1_checksum,
"d9bafe64b034b655db018ad0226c6865300ada31"
);
assert.equal(
plugin.uppy.getFile(fileIds[1]).meta.sha1_checksum,
"cb10341e3efeab45f0bc309a1c497edca4c5a744"
);
done();
});
});
test("it does nothing if the window.crypto.subtle.digest function throws an error / rejects", function (assert) {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
const done = assert.async();
const fileIds = [
"uppy-test/file/vv2/xvejg5w/blah/png-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764",
"uppy-test/file/blah1/ads37x2/blah1/png-1d-1d-2v-1d-1e-image/jpeg-99999-1837921727764",
];
sinon
.stub(window.crypto.subtle, "digest")
.rejects({ message: "Algorithm: Unrecognized name" });
plugin.uppy.preprocessors[0](fileIds).then(() => {
assert.equal(plugin.uppy.emitted[0].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[1].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[2].event, "preprocess-complete");
assert.equal(plugin.uppy.emitted[3].event, "preprocess-complete");
assert.deepEqual(plugin.uppy.getFile(fileIds[0]).meta, {});
assert.deepEqual(plugin.uppy.getFile(fileIds[1]).meta, {});
done();
});
});
});
function createFile(name, type = "image/png") {
const file = new Blob([name], {
type,
});
file.name = name;
return file;
}

View File

@ -0,0 +1,170 @@
import UppyMediaOptimization from "discourse/lib/uppy-media-optimization-plugin";
import { module, test } from "qunit";
import { Promise } from "rsvp";
class FakeUppy {
constructor() {
this.preprocessors = [];
this.emitted = [];
this.files = {
"uppy-test/file/vv2/xvejg5w/blah/jpg-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764": {
data: "old file state",
},
"uppy-test/file/blah1/ads37x2/blah1/jpg-1d-1d-2v-1d-1e-image/jpeg-99999-1837921727764": {
data: "old file state 1",
},
};
}
addPreProcessor(fn) {
this.preprocessors.push(fn);
}
getFile(id) {
return this.files[id];
}
emit(event, file, data) {
this.emitted.push({ event, file, data });
}
setFileState(fileId, state) {
this.files[fileId] = state;
}
}
module("Unit | Utility | UppyMediaOptimization Plugin", function () {
test("sets the options passed in", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
optimizeFn: function () {
return "wow such optimized";
},
});
assert.equal(plugin.id, "test-uppy");
assert.equal(plugin.runParallel, true);
assert.equal(plugin.optimizeFn(), "wow such optimized");
});
test("installation uses the correct function", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
});
plugin._optimizeParallel = function () {
return "using parallel";
};
plugin._optimizeSerial = function () {
return "using serial";
};
plugin.install();
assert.equal(plugin.uppy.preprocessors[0](), "using parallel");
plugin.runParallel = false;
plugin.uppy.preprocessors = [];
plugin.install();
assert.equal(plugin.uppy.preprocessors[0](), "using serial");
});
test("sets the file state when successfully optimizing the file and emits events", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
optimizeFn: () => {
return Promise.resolve("new file state");
},
});
plugin.install();
const done = assert.async();
const fileId =
"uppy-test/file/vv2/xvejg5w/blah/jpg-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764";
plugin.uppy.preprocessors[0]([fileId]).then(() => {
assert.equal(plugin.uppy.emitted[0].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[1].event, "preprocess-complete");
assert.equal(plugin.uppy.getFile(fileId).data, "new file state");
done();
});
});
test("handles optimizer errors gracefully by leaving old file state and calling preprocess-complete", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
optimizeFn: () => {
return new Promise(() => {
throw new Error("bad stuff");
});
},
});
plugin.install();
const done = assert.async();
const fileId =
"uppy-test/file/vv2/xvejg5w/blah/jpg-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764";
plugin.uppy.preprocessors[0]([fileId]).then(() => {
assert.equal(plugin.uppy.emitted[0].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[1].event, "preprocess-complete");
assert.equal(plugin.uppy.getFile(fileId).data, "old file state");
done();
});
});
test("handles serial file optimization successfully", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: false,
optimizeFn: () => {
return Promise.resolve("new file state");
},
});
plugin.install();
const done = assert.async();
const fileIds = [
"uppy-test/file/vv2/xvejg5w/blah/jpg-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764",
"uppy-test/file/blah1/ads37x2/blah1/jpg-1d-1d-2v-1d-1e-image/jpeg-99999-1837921727764",
];
plugin.uppy.preprocessors[0](fileIds).then(() => {
assert.equal(plugin.uppy.emitted[0].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[1].event, "preprocess-complete");
assert.equal(plugin.uppy.emitted[2].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[3].event, "preprocess-complete");
assert.equal(plugin.uppy.getFile(fileIds[0]).data, "new file state");
assert.equal(plugin.uppy.getFile(fileIds[1]).data, "new file state");
done();
});
});
test("handles parallel file optimization successfully", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
optimizeFn: () => {
return Promise.resolve("new file state");
},
});
plugin.install();
const done = assert.async();
const fileIds = [
"uppy-test/file/vv2/xvejg5w/blah/jpg-1d-1d-2v-1d-1e-image/jpeg-9043429-1624921727764",
"uppy-test/file/blah1/ads37x2/blah1/jpg-1d-1d-2v-1d-1e-image/jpeg-99999-1837921727764",
];
plugin.uppy.preprocessors[0](fileIds).then(() => {
assert.equal(plugin.uppy.emitted[0].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[1].event, "preprocess-progress");
assert.equal(plugin.uppy.emitted[2].event, "preprocess-complete");
assert.equal(plugin.uppy.emitted[3].event, "preprocess-complete");
assert.equal(plugin.uppy.getFile(fileIds[0]).data, "new file state");
assert.equal(plugin.uppy.getFile(fileIds[1]).data, "new file state");
done();
});
});
});

View File

@ -272,6 +272,10 @@ basic:
client: true
default: false
hidden: true
enable_experimental_composer_uploader:
client: true
default: false
hidden: true
enable_direct_s3_uploads:
client: true
default: false

View File

@ -139,6 +139,7 @@ onmessage = async function (e) {
console.error(error);
postMessage({
type: "error",
file: e.data.file
});
}
break;