Skip to content

Removed mobiledoc rendering and HTML-to-mobiledoc conversion #28906

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
kevinansfield wants to merge 1 commit into
base: main
Choose a base branch
from
+919 −1,221
Draft

Removed mobiledoc rendering and HTML-to-mobiledoc conversion #28906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 0 additions & 21 deletions ghost/core/core/server/api/endpoints/utils/serializers/input/pages.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,12 @@ const tpl = require('@tryghost/tpl');
const url = require('./utils/url');
const slugFilterOrder = require('./utils/slug-filter-order');
const localUtils = require('../../index');
const mobiledoc = require('../../../../../lib/mobiledoc');
const postsMetaSchema = require('../../../../../data/schema').tables.posts_meta;
const postsSchema = require('../../../../../data/schema').tables.posts;
const clean = require('./utils/clean');
const lexical = require('../../../../../lib/lexical');

const messages = {
failedHtmlToMobiledoc: 'Failed to convert HTML to Mobiledoc',
failedHtmlToLexical: 'Failed to convert HTML to Lexical'
};

Expand Down Expand Up @@ -178,25 +176,6 @@ module.exports = {
const html = frame.data.pages[0].html;

if (frame.options.source === 'html' && !_.isEmpty(html)) {
if (process.env.CI) {
console.time('htmlToMobiledocConverter (page)'); // eslint-disable-line no-console
}

try {
frame.data.pages[0].mobiledoc = JSON.stringify(mobiledoc.htmlToMobiledocConverter(html));
} catch (err) {
throw new ValidationError({
message: tpl(messages.failedHtmlToMobiledoc),
err
});
}

if (process.env.CI) {
console.timeEnd('htmlToMobiledocConverter (page)'); // eslint-disable-line no-console
}

// normally we don't allow both mobiledoc+lexical but the model layer will remove lexical
// if mobiledoc is already present to avoid migrating formats outside of an explicit conversion
if (process.env.CI) {
console.time('htmlToLexicalConverter (page)'); // eslint-disable-line no-console
}
Expand Down
21 changes: 0 additions & 21 deletions ghost/core/core/server/api/endpoints/utils/serializers/input/posts.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,12 @@ const tpl = require('@tryghost/tpl');
const url = require('./utils/url');
const slugFilterOrder = require('./utils/slug-filter-order');
const localUtils = require('../../index');
const mobiledoc = require('../../../../../lib/mobiledoc');
const postsMetaSchema = require('../../../../../data/schema').tables.posts_meta;
const postsSchema = require('../../../../../data/schema').tables.posts;
const clean = require('./utils/clean');
const lexical = require('../../../../../lib/lexical');

const messages = {
failedHtmlToMobiledoc: 'Failed to convert HTML to Mobiledoc',
failedHtmlToLexical: 'Failed to convert HTML to Lexical'
};

Expand Down Expand Up @@ -222,25 +220,6 @@ module.exports = {
const html = frame.data.posts[0].html;

if (frame.options.source === 'html' && !_.isEmpty(html)) {
if (process.env.CI) {
console.time('htmlToMobiledocConverter (post)'); // eslint-disable-line no-console
}

try {
frame.data.posts[0].mobiledoc = JSON.stringify(mobiledoc.htmlToMobiledocConverter(html));
} catch (err) {
throw new ValidationError({
message: tpl(messages.failedHtmlToMobiledoc),
err
});
}

if (process.env.CI) {
console.timeEnd('htmlToMobiledocConverter (post)'); // eslint-disable-line no-console
}

// normally we don't allow both mobiledoc+lexical but the model layer will remove lexical
// if mobiledoc is already present to avoid migrating formats outside of an explicit conversion
if (process.env.CI) {
console.time('htmlToLexicalConverter (post)'); // eslint-disable-line no-console
}
Expand Down
15 changes: 10 additions & 5 deletions ghost/core/core/server/data/importer/importers/data/posts-importer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const _ = require('lodash');
const crypto = require('crypto');
const BaseImporter = require('./base');
const mobiledocLib = require('../../../../lib/mobiledoc');
const lexicalLib = require('../../../../lib/lexical');
const validator = require('@tryghost/validator');
const postsMetaSchema = require('../../../schema').tables.posts_meta;
const metaAttrs = _.keys(_.omit(postsMetaSchema, ['id']));
Expand Down Expand Up @@ -244,8 +245,7 @@ class PostsImporter extends BaseImporter {
mobiledoc = JSON.parse(model.mobiledoc);

if (!mobiledoc.cards || !_.isArray(mobiledoc.cards)) {
model.mobiledoc = mobiledocLib.blankDocument;
mobiledoc = model.mobiledoc;
mobiledoc = mobiledocLib.blankDocument;
}
} catch (err) {
mobiledoc = mobiledocLib.blankDocument;
Expand All @@ -270,11 +270,16 @@ class PostsImporter extends BaseImporter {
}
});

// keep the cleaned-up mobiledoc and let the model convert it to lexical and
// render the html on save - this exercises the same mobiledoc -> lexical
// import path that real legacy exports go through
model.mobiledoc = JSON.stringify(mobiledoc);
model.html = mobiledocLib.render(JSON.parse(model.mobiledoc));
delete model.html;
} else if (model.html && !model.lexical) {
model.mobiledoc = JSON.stringify(mobiledocLib.htmlToMobiledocConverter(model.html));
model.html = mobiledocLib.render(JSON.parse(model.mobiledoc));
// html-only imports have no mobiledoc to convert, so turn the html into lexical
model.lexical = JSON.stringify(lexicalLib.htmlToLexicalConverter(model.html));
delete model.mobiledoc;
delete model.html;
}

this.sanitizePostsMeta(model);
Expand Down
5 changes: 3 additions & 2 deletions ...t/core/core/server/data/migrations/versions/5.0/2022-05-21-00-00-regenerate-posts-html.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const logging = require('@tryghost/logging');
const {createIrreversibleMigration} = require('../../utils');
const mobiledocLib = require('../../../../lib/mobiledoc');
const lexicalLib = require('../../../../lib/lexical');
const {mobiledocToLexical} = require('@tryghost/kg-converters');
const htmlToPlaintext = require('@tryghost/html-to-plaintext');

module.exports = createIrreversibleMigration(async (knex) => {
Expand Down Expand Up @@ -43,7 +44,7 @@ module.exports = createIrreversibleMigration(async (knex) => {
continue;
}

const html = mobiledocLib.render(mobiledoc);
const html = await lexicalLib.render(mobiledocToLexical(post.mobiledoc));

const updatedAttrs = {
html
Expand Down
111 changes: 0 additions & 111 deletions ghost/core/core/server/lib/mobiledoc.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
const path = require('path');
const errors = require('@tryghost/errors');
const logging = require('@tryghost/logging');
const config = require('../../shared/config');
const storage = require('../adapters/storage');
const storageUtils = require('../adapters/storage/utils');

let cardFactory;
let cards;
let mobiledocHtmlRenderer;

module.exports = {
get blankDocument() {
Expand Down Expand Up @@ -53,116 +49,9 @@ module.exports = {
return cards;
},

get atoms() {
return require('@tryghost/kg-default-atoms').atoms;
},

get mobiledocHtmlRenderer() {
if (!mobiledocHtmlRenderer) {
const {MobiledocHtmlRenderer} = require('@tryghost/kg-mobiledoc-html-renderer');

mobiledocHtmlRenderer = new MobiledocHtmlRenderer({
cards: this.cards,
atoms: this.atoms,
unknownCardHandler(args) {
logging.error(new errors.InternalServerError({
message: 'Mobiledoc card \'' + args.env.name + '\' not found.'
}));
}
});
}

return mobiledocHtmlRenderer;
},

render(mobiledoc, options) {
return this.mobiledocHtmlRenderer.render(mobiledoc, options);
},

get htmlToMobiledocConverter() {
try {
if (process.env.CI) {
console.time('require @tryghost/html-to-mobiledoc'); // eslint-disable-line no-console
}

const {htmlToMobiledoc} = require('@tryghost/html-to-mobiledoc');

if (process.env.CI) {
console.timeEnd('require @tryghost/html-to-mobiledoc'); // eslint-disable-line no-console
}

return htmlToMobiledoc;
} catch (err) {
return () => {
throw new errors.InternalServerError({
message: 'Unable to convert from source HTML to Mobiledoc',
context: 'The html-to-mobiledoc package was not installed',
help: 'Please review any errors from the install process by checking the Ghost logs',
code: 'HTML_TO_MOBILEDOC_INSTALLATION',
err: err
});
};
}
},

// used when force-rerendering post content to ensure that old image card
// payloads contain width/height values to be used when generating srcsets
populateImageSizes: async function (mobiledocJson) {
// do not require image-size until it's requested to avoid circular dependencies
// shared/url-utils > server/lib/mobiledoc > server/lib/image/image-size > server/adapters/storage/utils
const {imageSize} = require('./image');

async function getUnsplashSize(url) {
const parsedUrl = new URL(url);
parsedUrl.searchParams.delete('w');
parsedUrl.searchParams.delete('fit');
parsedUrl.searchParams.delete('crop');
parsedUrl.searchParams.delete('dpr');

return await imageSize.getImageSizeFromUrl(parsedUrl.href);
}

const mobiledoc = JSON.parse(mobiledocJson);

const sizePromises = mobiledoc.cards.map(async (card) => {
const [cardName, payload] = card;

const needsFilling = cardName === 'image' && payload && payload.src && (!payload.width || !payload.height);
if (!needsFilling) {
return;
}

const isUnsplash = payload.src.match(/images\.unsplash\.com/);
const isRelativeImagePath = /^\/content\/images\//.test(payload.src);
try {
let size;
if (isUnsplash) {
size = await getUnsplashSize(payload.src);
} else if (isRelativeImagePath || storageUtils.isLocalImage(payload.src)) {
size = await imageSize.getOriginalImageSizeFromStorageUrl(payload.src);
} else if (storageUtils.isInternalImage(payload.src)) {
size = await imageSize.getImageSizeFromUrl(payload.src);
}

if (size && size.width && size.height) {
payload.width = size.width;
payload.height = size.height;
}
} catch (e) {
// TODO: use debug instead?
logging.error(e);
}
});

await Promise.all(sizePromises);

return JSON.stringify(mobiledoc);
},

// allow config changes to be picked up - useful in tests
reload() {
cardFactory = null;
cards = null;
mobiledocHtmlRenderer = null;
}
};
39 changes: 22 additions & 17 deletions ghost/core/core/server/models/post.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,15 +556,22 @@ Post = ghostBookshelf.Model.extend({
let tagsToSave;
const ops = [];

// normally we don't allow both mobiledoc & lexical through at the API level but there's
// an exception for ?source=html which always sets both when the lexical editor is enabled.
// That's necessary because at the input serializer layer we don't have access to the
// actual model to check if this would result in a change of format

// We don't allow both mobiledoc & lexical to be stored at once. A single save can
// still momentarily carry both: the stored format plus an incoming one (e.g. editing
// a lexical post by sending mobiledoc). Resolve to a single format here; any remaining
// mobiledoc is converted to lexical further down.
if (this.previous('mobiledoc') && this.get('lexical')) {
// post is stored as mobiledoc - keep it (explicit conversion happens via convert_to_lexical)
this.set('lexical', null);
} else if (this.get('mobiledoc') && this.get('lexical')) {
this.set('mobiledoc', null);
// both formats are set on this save - prefer lexical unless mobiledoc is the only
// format that was supplied (e.g. editing a lexical post by sending mobiledoc), in
// which case the incoming mobiledoc wins and is converted to lexical below
if (this.hasChanged('mobiledoc') && !this.hasChanged('lexical')) {
this.set('lexical', null);
} else {
this.set('mobiledoc', null);
}
}

// CASE: disallow published -> scheduled
Expand Down Expand Up @@ -688,29 +695,27 @@ Post = ghostBookshelf.Model.extend({
this.set('lexical', JSON.stringify(lexicalLib.blankDocument));
}

// If we're force re-rendering we want to make sure that all image cards
// have original dimensions stored in the payload for use by card renderers
if (options.force_rerender && this.get('mobiledoc')) {
this.set('mobiledoc', await mobiledocLib.populateImageSizes(this.get('mobiledoc')));
}

// CASE: mobiledoc has changed, generate html
// CASE: content is still stored as mobiledoc, convert it to lexical so it can be
// rendered. We no longer render mobiledoc directly, so this permanently migrates
// the post to lexical and the lexical block below generates the html.
// CASE: mobiledoc has changed
// CASE: ?force_rerender=true passed via Admin API
// CASE: html is null, but mobiledoc exists (only important for migrations & importing)
if (
!this.get('lexical') &&
this.get('mobiledoc') &&
(
this.hasChanged('mobiledoc')
|| options.force_rerender
|| (!this.get('html') && (options.migrating || options.importing))
)
) {
try {
this.set('html', mobiledocLib.render(JSON.parse(this.get('mobiledoc'))));
this.set('lexical', mobiledocToLexical(this.get('mobiledoc')));
this.set('mobiledoc', null);
} catch (err) {
throw new errors.ValidationError({
message: tpl(messages.invalidMobiledocStructure),
help: 'https://ghost.org/docs/publishing/'
help: messages.invalidMobiledocStructureHelp
});
}
}
Expand Down Expand Up @@ -927,7 +932,7 @@ Post = ghostBookshelf.Model.extend({
let authorId = await this.contextUser(options);
const authorExists = await ghostBookshelf.model('User').findOne({id: authorId}, {transacting: options.transacting});
if (!authorExists) {
authorId = (await ghostBookshelf.model('User').getOwnerUser()).get('id');
authorId = (await ghostBookshelf.model('User').getOwnerUser({transacting: options.transacting})).get('id');
}
ops.push(async function updateRevisions() {
const revisionModels = await ghostBookshelf.model('PostRevision')
Expand Down
29 changes: 13 additions & 16 deletions ghost/core/core/server/services/email-service/email-renderer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ const {getEmailDesign} = require('../email-rendering/email-design');
const {registerHelpers} = require('./helpers/register-helpers');
const crypto = require('crypto');
const {getPostAccessFilter} = require('../members/content-gating');
const {mobiledocToLexical} = require('@tryghost/kg-converters');
/** @import {TemplateDelegate} from 'handlebars' */

const DEFAULT_LOCALE = 'en-gb';
Expand Down Expand Up @@ -463,22 +464,18 @@ class EmailRenderer {
async renderPostBaseHtml(post, newsletter) {
const postUrl = this.#getPostUrl(post);

let html;
if (post.get('lexical')) {
// only lexical's renderer is async
html = await this.#renderers.lexical.render(
post.get('lexical'),
{
target: 'email',
postUrl,
design: this.#getEmailDesign(newsletter)
}
);
} else {
html = this.#renderers.mobiledoc.render(
JSON.parse(post.get('mobiledoc')), {target: 'email', postUrl}
);
}
// posts are migrated to lexical on save, but legacy content may still be stored as
// mobiledoc - convert it to lexical so it can be rendered.
const lexical = post.get('lexical') || mobiledocToLexical(post.get('mobiledoc'));

const html = await this.#renderers.lexical.render(
lexical,
{
target: 'email',
postUrl,
design: this.#getEmailDesign(newsletter)
}
);
return html;
}

Expand Down
Loading
Loading