Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: Refactor uppy plugin interfaces (#14275) 

This abstracts interaction with uppy for uppy plugin classes
into base classes for Preprocessor  plugins, so anyone
making these uppy plugins doesn't have to think as much about uppy
underneath the hood. This also makes the logging and validation
nicer, and provides a more consistent way to emit progress and
completion events.

In a future commit, we will introduce another base class for
`UploadUploaderPlugin` which will be used to be able to hijack
the upload process to go to a different provider (e.g. for discourse-video)
This commit is contained in:
Martin Brennan 2021-09-10 09:09:47 +10:00 committed by GitHub
parent 05c356f7c6
commit 2215cc2547
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 310 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
app/assets/javascripts/discourse/app/lib/uppy-checksum-plugin.js
View File

@ -1,39 +1,31 @@
import { BasePlugin } from "@uppy/core";
import { warn } from "@ember/debug";
import { UploadPreProcessorPlugin } from "discourse/lib/uppy-plugin-base";
import { Promise } from "rsvp";
export default class UppyChecksum extends BasePlugin {
export default class UppyChecksum extends UploadPreProcessorPlugin {
static pluginId = "uppy-checksum";
constructor(uppy, opts) {
super(uppy, opts);
this.id = opts.id || "uppy-checksum";
this.pluginClass = this.constructor.name;
this.capabilities = opts.capabilities;
this.type = "preprocessor";
}
_canUseSubtleCrypto() {
if (!this._secureContext()) {
warn(
"Cannot generate cryptographic digests in an insecure context (not HTTPS).",
{
id: "discourse.uppy-media-optimization",
}
this._consoleWarn(
"Cannot generate cryptographic digests in an insecure context (not HTTPS)."
);
return false;
}
if (this.capabilities.isIE11) {
warn(
"The required cipher suite is unavailable in Internet Explorer 11.",
{
id: "discourse.uppy-media-optimization",
}
this._consoleWarn(
"The required cipher suite is unavailable in Internet Explorer 11."
);
return false;
}
if (!this._hasCryptoCipher()) {
warn("The required cipher suite is unavailable in this browser.", {
id: "discourse.uppy-media-optimization",
});
this._consoleWarn(
"The required cipher suite is unavailable in this browser."
);
return false;
}
@ -46,9 +38,9 @@ export default class UppyChecksum extends BasePlugin {
}
let promises = fileIds.map((fileId) => {
let file = this.uppy.getFile(fileId);
let file = this._getFile(fileId);
this.uppy.emit("preprocess-progress", this.pluginClass, file);
this._emitProgress(file);
return file.data.arrayBuffer().then((arrayBuffer) => {
return window.crypto.subtle
@ -58,22 +50,22 @@ export default class UppyChecksum extends BasePlugin {
const hashHex = hashArray
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
this.uppy.setFileMeta(fileId, { sha1_checksum: hashHex });
this.uppy.emit("preprocess-complete", this.pluginClass, file);
this._setFileMeta(fileId, { sha1_checksum: hashHex });
this._emitComplete(file);
})
.catch((err) => {
if (
err.message.toString().includes("Algorithm: Unrecognized name")
) {
warn("SHA-1 algorithm is unsupported in this browser.", {
id: "discourse.uppy-media-optimization",
});
this._consoleWarn(
"SHA-1 algorithm is unsupported in this browser."
);
} else {
warn(`Error encountered when generating digest: ${err.message}`, {
id: "discourse.uppy-media-optimization",
});
this._consoleWarn(
`Error encountered when generating digest: ${err.message}`
);
}
this.uppy.emit("preprocess-complete", this.pluginClass, file);
this._emitComplete(file);
});
});
});
@ -90,10 +82,10 @@ export default class UppyChecksum extends BasePlugin {
}
install() {
this.uppy.addPreProcessor(this._generateChecksum.bind(this));
this._install(this._generateChecksum.bind(this));
}
uninstall() {
this.uppy.removePreProcessor(this._generateChecksum.bind(this));
this._uninstall(this._generateChecksum.bind(this));
}
}

37
app/assets/javascripts/discourse/app/lib/uppy-media-optimization-plugin.js
View File

@ -1,15 +1,12 @@
import { BasePlugin } from "@uppy/core";
import { warn } from "@ember/debug";
import { UploadPreProcessorPlugin } from "discourse/lib/uppy-plugin-base";
import { Promise } from "rsvp";
export default class UppyMediaOptimization extends BasePlugin {
export default class UppyMediaOptimization extends UploadPreProcessorPlugin {
static pluginId = "uppy-media-optimization";
constructor(uppy, opts) {
super(uppy, opts);
this.id = opts.id || "uppy-media-optimization";
this.type = "preprocessor";
this.optimizeFn = opts.optimizeFn;
this.pluginClass = this.constructor.name;
// mobile devices have limited processing power, so we only enable
// running media optimization in parallel when we are sure the user
@ -19,27 +16,27 @@ export default class UppyMediaOptimization extends BasePlugin {
}
_optimizeFile(fileId) {
let file = this.uppy.getFile(fileId);
let file = this._getFile(fileId);
this.uppy.emit("preprocess-progress", this.pluginClass, file);
this._emitProgress(file);
return this.optimizeFn(file, { stopWorkerOnError: !this.runParallel })
.then((optimizedFile) => {
if (!optimizedFile) {
warn("Nothing happened, possible error or other restriction.", {
id: "discourse.uppy-media-optimization",
});
this._consoleWarn(
"Nothing happened, possible error or other restriction."
);
} else {
this.uppy.setFileState(fileId, {
this._setFileState(fileId, {
data: optimizedFile,
size: optimizedFile.size,
});
}
this.uppy.emit("preprocess-complete", this.pluginClass, file);
this._emitComplete(file);
})
.catch((err) => {
warn(err, { id: "discourse.uppy-media-optimization" });
this.uppy.emit("preprocess-complete", this.pluginClass, file);
this._consoleWarn(err);
this._emitComplete(file);
});
}
@ -59,17 +56,17 @@ export default class UppyMediaOptimization extends BasePlugin {
install() {
if (this.runParallel) {
this.uppy.addPreProcessor(this._optimizeParallel.bind(this));
this._install(this._optimizeParallel.bind(this));
} else {
this.uppy.addPreProcessor(this._optimizeSerial.bind(this));
this._install(this._optimizeSerial.bind(this));
}
}
uninstall() {
if (this.runParallel) {
this.uppy.removePreProcessor(this._optimizeParallel.bind(this));
this._uninstall(this._optimizeParallel.bind(this));
} else {
this.uppy.removePreProcessor(this._optimizeSerial.bind(this));
this._uninstall(this._optimizeSerial.bind(this));
}
}
}

50
app/assets/javascripts/discourse/app/lib/uppy-plugin-base.js Normal file
View File

@ -0,0 +1,50 @@
import { BasePlugin } from "@uppy/core";
import { warn } from "@ember/debug";
export class UppyPluginBase extends BasePlugin {
constructor(uppy, opts) {
super(uppy, opts);
this.id = this.constructor.pluginId;
}
_consoleWarn(msg) {
warn(msg, { id: `discourse.${this.id}` });
}
_getFile(fileId) {
return this.uppy.getFile(fileId);
}
_setFileMeta(fileId, meta) {
this.uppy.setFileMeta(fileId, meta);
}
_setFileState(fileId, state) {
this.uppy.setFileState(fileId, state);
}
}
export class UploadPreProcessorPlugin extends UppyPluginBase {
static pluginType = "preprocessor";
constructor(uppy, opts) {
super(uppy, opts);
this.type = this.constructor.pluginType;
}
_install(fn) {
this.uppy.addPreProcessor(fn);
}
_uninstall(fn) {
this.uppy.removePreProcessor(fn);
}
_emitProgress(file) {
this.uppy.emit("preprocess-progress", this.id, file);
}
_emitComplete(file) {
this.uppy.emit("preprocess-complete", this.id, file);
}
}

142
app/assets/javascripts/discourse/app/mixins/composer-upload-uppy.js
View File

@ -1,4 +1,5 @@
import Mixin from "@ember/object/mixin";
import ExtendableUploader from "discourse/mixins/extendable-uploader";
import { ajax } from "discourse/lib/ajax";
import { deepMerge } from "discourse-common/lib/object";
import UppyChecksum from "discourse/lib/uppy-checksum-plugin";
@ -31,7 +32,7 @@ import { cacheShortUploadUrl } from "pretty-text/upload-short-url";
// and the most important _bindUploadTarget which handles all the main upload
// functionality and event binding.
//
export default Mixin.create({
export default Mixin.create(ExtendableUploader, {
@observes("composerModel.uploadCancelled")
_cancelUpload() {
if (!this.get("composerModel.uploadCancelled")) {
@ -154,14 +155,12 @@ export default Mixin.create({
});
this._uppyInstance.on("upload", (data) => {
this._addNeedProcessing(data.fileIDs.length);
const files = data.fileIDs.map((fileId) =>
this._uppyInstance.getFile(fileId)
);
this._eachPreProcessor((pluginName, status) => {
status.needProcessing = files.length;
});
this.setProperties({
isProcessingUpload: true,
isCancellable: false,
@ -179,6 +178,7 @@ export default Mixin.create({
});
this._uppyInstance.on("upload-success", (file, response) => {
this._inProgressUploads--;
let upload = response.body;
const markdown = this.uploadMarkdownResolvers.reduce(
(md, resolver) => resolver(upload) || md,
@ -229,13 +229,6 @@ export default Mixin.create({
this._setupPreProcessors();
this._setupUIPlugins();
// TODO (martin) Need a more automatic way to do this for preprocessor
// plugins like UppyChecksum and UppyMediaOptimization so people don't
// have to remember to do this, also want to wrap this.uppy.emit in those
// classes so people don't have to remember to pass through the plugin class
// name for the preprocess-X events.
this._trackPreProcessorStatus(UppyChecksum);
},
_handleUploadError(file, error, response) {
@ -255,31 +248,34 @@ export default Mixin.create({
},
_setupPreProcessors() {
this.uploadPreProcessors.forEach(({ pluginClass, optionsResolverFn }) => {
this._uppyInstance.use(
pluginClass,
optionsResolverFn({
composerModel: this.composerModel,
composerElement: this.composerElement,
capabilities: this.capabilities,
isMobileDevice: this.site.isMobileDevice,
})
);
this._trackPreProcessorStatus(pluginClass);
});
const checksumPreProcessor = {
pluginClass: UppyChecksum,
optionsResolverFn: ({ capabilities }) => {
return {
capabilities,
};
},
};
// It is important that the UppyChecksum preprocessor is the last one to
// be added; the preprocessors are run in order and since other preprocessors
// may modify the file (e.g. the UppyMediaOptimization one), we need to
// checksum once we are sure the file data has "settled".
this._uppyInstance.use(UppyChecksum, { capabilities: this.capabilities });
[this.uploadPreProcessors, checksumPreProcessor]
.flat()
.forEach(({ pluginClass, optionsResolverFn }) => {
this._useUploadPlugin(
pluginClass,
optionsResolverFn({
composerModel: this.composerModel,
composerElement: this.composerElement,
capabilities: this.capabilities,
isMobileDevice: this.site.isMobileDevice,
})
);
});
this._uppyInstance.on("preprocess-progress", (pluginClass, file) => {
this._debugLog(
`[${pluginClass}] processing file ${file.name} (${file.id})`
);
this._preProcessorStatus[pluginClass].activeProcessing++;
this._onPreProcessProgress((file) => {
let placeholderData = this.placeholders[file.id];
placeholderData.processingPlaceholder = `[${I18n.t(
"processing_filename",
@ -295,39 +291,25 @@ export default Mixin.create({
);
});
this._uppyInstance.on("preprocess-complete", (pluginClass, file) => {
this._debugLog(
`[${pluginClass}] completed processing file ${file.name} (${file.id})`
);
let placeholderData = this.placeholders[file.id];
this.appEvents.trigger(
`${this.eventPrefix}:replace-text`,
placeholderData.processingPlaceholder,
placeholderData.uploadPlaceholder
);
const preProcessorStatus = this._preProcessorStatus[pluginClass];
preProcessorStatus.activeProcessing--;
preProcessorStatus.completeProcessing++;
if (
preProcessorStatus.completeProcessing ===
preProcessorStatus.needProcessing
) {
preProcessorStatus.allComplete = true;
if (this._allPreprocessorsComplete()) {
this.setProperties({
isProcessingUpload: false,
isCancellable: true,
});
this._debugLog("All upload preprocessors complete.");
this.appEvents.trigger(
`${this.eventPrefix}:uploads-preprocessing-complete`
);
}
this._onPreProcessComplete(
(file) => {
let placeholderData = this.placeholders[file.id];
this.appEvents.trigger(
`${this.eventPrefix}:replace-text`,
placeholderData.processingPlaceholder,
placeholderData.uploadPlaceholder
);
},
() => {
this.setProperties({
isProcessingUpload: false,
isCancellable: true,
});
this.appEvents.trigger(
`${this.eventPrefix}:uploads-preprocessing-complete`
);
}
});
);
},
_setupUIPlugins() {
@ -505,14 +487,7 @@ export default Mixin.create({
isProcessingUpload: false,
isCancellable: false,
});
this._eachPreProcessor((pluginClass) => {
this._preProcessorStatus[pluginClass] = {
needProcessing: 0,
activeProcessing: 0,
completeProcessing: 0,
allComplete: false,
};
});
this._resetPreProcessors();
this.fileInputEl.value = "";
},
@ -578,31 +553,6 @@ export default Mixin.create({
}
},
_trackPreProcessorStatus(pluginClass) {
this._preProcessorStatus[pluginClass.name] = {
needProcessing: 0,
activeProcessing: 0,
completeProcessing: 0,
allComplete: false,
};
},
_eachPreProcessor(cb) {
for (const [pluginClass, status] of Object.entries(
this._preProcessorStatus
)) {
cb(pluginClass, status);
}
},
_allPreprocessorsComplete() {
let completed = [];
this._eachPreProcessor((pluginClass, status) => {
completed.push(status.allComplete);
});
return completed.every(Boolean);
},
showUploadSelector(toolbarEvent) {
this.send("showUploadSelector", toolbarEvent);
},

171
app/assets/javascripts/discourse/app/mixins/extendable-uploader.js Normal file
View File

@ -0,0 +1,171 @@
import Mixin from "@ember/object/mixin";
/**
* Use this mixin with any component that needs to upload files or images
* with Uppy. This mixin makes it easier to tell Uppy to use certain uppy plugins
* as well as tracking all of the state of preprocessor plugins. For example,
* you may have multiple preprocessors:
*
* - UppyMediaOptimization
* - UppyChecksum
*
* Once installed with _useUploadPlugin(PluginClass, opts), we track the following
* status for every preprocessor plugin:
*
* - needProcessing - The total number of files that have been added to uppy that
* will need to be run through the preprocessor plugins.
* - activeProcessing - The number of files that are currently being processed,
* which is determined by the preprocess-progress event.
* - completeProcessing - The number of files that have completed being processed,
* which is determined by the preprocess-complete event.
* - allComplete - Whether all files have completed the preprocessing for the plugin.
*
* There is a caveat - you must call _addNeedProcessing(data.fileIDs.length) when
* handling the "upload" event with uppy, otherwise this mixin does not know how
* many files need to be processed.
*
* If you need to do something else on progress or completion of preprocessors,
* hook into the _onPreProcessProgress(callback) or _onPreProcessComplete(callback, allCompleteCallback)
* functions. Note the _onPreProcessComplete function takes a second callback
* that is fired only when _all_ of the files have been preprocessed for all
* preprocessor plugins.
*
* A preprocessor is considered complete if the completeProcessing count is
* equal to needProcessing, at which point the allComplete prop is set to true.
* If all preprocessor plugins have allComplete set to true, then the allCompleteCallback
* is called for _onPreProcessComplete.
*
* To completely reset the preprocessor state for all plugins, call _resetPreProcessors.
*
* See ComposerUploadUppy for an example of a component using this mixin.
*/
export default Mixin.create({
_useUploadPlugin(pluginClass, opts = {}) {
if (!this._uppyInstance) {
return;
}
if (!pluginClass.pluginId) {
throw new Error(
"The uppy plugin should have a static pluginId that is used to uniquely identify it."
);
}
if (
!pluginClass.pluginType ||
!["preprocessor", "uploader"].includes(pluginClass.pluginType)
) {
throw new Error(
`The uppy plugin ${pluginClass.pluginId} should have a static pluginType that should be preprocessor or uploader`
);
}
this._uppyInstance.use(
pluginClass,
Object.assign(opts, {
id: pluginClass.pluginId,
type: pluginClass.pluginType,
})
);
if (pluginClass.pluginType === "preprocessor") {
this._trackPreProcessorStatus(pluginClass.pluginId);
}
},
// TODO (martin) This and _onPreProcessComplete will need to be tweaked
// if we ever add support for "determinate" preprocessors for uppy, which
// means the progress will have a value rather than a started/complete
// state ("indeterminate").
//
// See: https://uppy.io/docs/writing-plugins/#Progress-events
_onPreProcessProgress(callback) {
this._uppyInstance.on("preprocess-progress", (pluginId, file) => {
this._debugLog(`[${pluginId}] processing file ${file.name} (${file.id})`);
this._preProcessorStatus[pluginId].activeProcessing++;
callback(file);
});
},
_onPreProcessComplete(callback, allCompleteCallback) {
this._uppyInstance.on("preprocess-complete", (pluginId, file) => {
this._debugLog(
`[${pluginId}] completed processing file ${file.name} (${file.id})`
);
callback(file);
this._completePreProcessing(pluginId, (allComplete) => {
if (allComplete) {
this._debugLog("All upload preprocessors complete.");
allCompleteCallback();
}
});
});
},
_resetPreProcessors() {
this._eachPreProcessor((pluginId) => {
this._preProcessorStatus[pluginId] = {
needProcessing: 0,
activeProcessing: 0,
completeProcessing: 0,
allComplete: false,
};
});
},
_trackPreProcessorStatus(pluginId) {
if (!this._preProcessorStatus) {
this._preProcessorStatus = {};
}
this._preProcessorStatus[pluginId] = {
needProcessing: 0,
activeProcessing: 0,
completeProcessing: 0,
allComplete: false,
};
},
_addNeedProcessing(fileCount) {
this._eachPreProcessor((pluginName, status) => {
status.needProcessing += fileCount;
});
},
_eachPreProcessor(cb) {
for (const [pluginId, status] of Object.entries(this._preProcessorStatus)) {
cb(pluginId, status);
}
},
_allPreprocessorsComplete() {
let completed = [];
this._eachPreProcessor((pluginId, status) => {
completed.push(status.allComplete);
});
return completed.every(Boolean);
},
_completePreProcessing(pluginId, callback) {
const preProcessorStatus = this._preProcessorStatus[pluginId];
preProcessorStatus.activeProcessing--;
preProcessorStatus.completeProcessing++;
if (
preProcessorStatus.completeProcessing ===
preProcessorStatus.needProcessing
) {
preProcessorStatus.allComplete = true;
preProcessorStatus.needProcessing = 0;
if (this._allPreprocessorsComplete()) {
callback(true);
} else {
callback(false);
}
}
},
});

8
app/assets/javascripts/discourse/tests/unit/lib/uppy-checksum-plugin-test.js
View File

@ -40,10 +40,9 @@ module("Unit | Utility | UppyChecksum Plugin", function () {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
assert.equal(plugin.id, "test-uppy");
assert.equal(plugin.id, "uppy-checksum");
assert.equal(plugin.capabilities, capabilities);
});
@ -51,7 +50,6 @@ module("Unit | Utility | UppyChecksum Plugin", function () {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
@ -75,7 +73,6 @@ module("Unit | Utility | UppyChecksum Plugin", function () {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
@ -99,7 +96,6 @@ module("Unit | Utility | UppyChecksum Plugin", function () {
const capabilities = { isIE11: true };
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
@ -121,7 +117,6 @@ module("Unit | Utility | UppyChecksum Plugin", function () {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();
@ -155,7 +150,6 @@ module("Unit | Utility | UppyChecksum Plugin", function () {
const capabilities = {};
const fakeUppy = new FakeUppy();
const plugin = new UppyChecksum(fakeUppy, {
id: "test-uppy",
capabilities,
});
plugin.install();

8
app/assets/javascripts/discourse/tests/unit/lib/uppy-media-optimization-plugin-test.js
View File

@ -37,13 +37,12 @@ module("Unit | Utility | UppyMediaOptimization Plugin", function () {
test("sets the options passed in", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
optimizeFn: function () {
return "wow such optimized";
},
});
assert.equal(plugin.id, "test-uppy");
assert.equal(plugin.id, "uppy-media-optimization");
assert.equal(plugin.runParallel, true);
assert.equal(plugin.optimizeFn(), "wow such optimized");
});
@ -51,7 +50,6 @@ module("Unit | Utility | UppyMediaOptimization Plugin", function () {
test("installation uses the correct function", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
});
plugin._optimizeParallel = function () {
@ -71,7 +69,6 @@ module("Unit | Utility | UppyMediaOptimization Plugin", function () {
test("sets the file state when successfully optimizing the file and emits events", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
optimizeFn: () => {
return Promise.resolve("new file state");
@ -93,7 +90,6 @@ module("Unit | Utility | UppyMediaOptimization Plugin", function () {
test("handles optimizer errors gracefully by leaving old file state and calling preprocess-complete", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
optimizeFn: () => {
return new Promise(() => {
@ -117,7 +113,6 @@ module("Unit | Utility | UppyMediaOptimization Plugin", function () {
test("handles serial file optimization successfully", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: false,
optimizeFn: () => {
return Promise.resolve("new file state");
@ -144,7 +139,6 @@ module("Unit | Utility | UppyMediaOptimization Plugin", function () {
test("handles parallel file optimization successfully", function (assert) {
const fakeUppy = new FakeUppy();
const plugin = new UppyMediaOptimization(fakeUppy, {
id: "test-uppy",
runParallel: true,
optimizeFn: () => {
return Promise.resolve("new file state");