diff --git a/package-lock.json b/package-lock.json index f19ad567ebe2..1f69c25f9b3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3271,6 +3271,25 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/@isomorphic-git/idb-keyval": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/@isomorphic-git/idb-keyval/-/idb-keyval-3.3.2.tgz", + "integrity": "sha512-r8/AdpiS0/WJCNR/t/gsgL+M8NMVj/ek7s60uz3LmpCaTF2mEVlZJlB01ZzalgYzRLXwSPC92o+pdzjM7PN/pA==" + }, + "node_modules/@isomorphic-git/lightning-fs": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@isomorphic-git/lightning-fs/-/lightning-fs-4.6.0.tgz", + "integrity": "sha512-tfon8f1h6LawjFI/d8lZPWRPTxmdvyTMbkT/j5yo6dB0hALhKw5D9JsdCcUu/D1pAcMMiU7GZFDsDGqylerr7g==", + "dependencies": { + "@isomorphic-git/idb-keyval": "3.3.2", + "isomorphic-textencoder": "1.0.1", + "just-debounce-it": "1.1.0", + "just-once": "1.1.0" + }, + "bin": { + "superblocktxt": "src/superblocktxt.js" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "dev": true, @@ -8815,6 +8834,11 @@ "version": "3.2.4", "license": "MIT" }, + "node_modules/async-lock": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.1.tgz", + "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==" + }, "node_modules/async-mutex": { "version": "0.3.2", "license": "MIT", @@ -10458,6 +10482,11 @@ "node": ">=0.10.0" } }, + "node_modules/clean-git-ref": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/clean-git-ref/-/clean-git-ref-2.0.1.tgz", + "integrity": "sha512-bLSptAy2P0s6hU4PzuIMKmMJJSE6gLXGH1cntDu7bWJUksvuM+7ReOK61mozULErYvP6a15rnYl0zFDef+pyPw==" + }, "node_modules/clean-regexp": { "version": "1.0.0", "dev": true, @@ -11475,6 +11504,17 @@ "typescript": ">=3" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/create-ecdh": { "version": "4.0.4", "license": "MIT", @@ -12464,6 +12504,10 @@ "resolved": "packages/decap-cms-backend-bitbucket", "link": true }, + "node_modules/decap-cms-backend-git": { + "resolved": "packages/decap-cms-backend-git", + "link": true + }, "node_modules/decap-cms-backend-git-gateway": { "resolved": "packages/decap-cms-backend-git-gateway", "link": true @@ -12625,6 +12669,20 @@ "node": ">=0.10" } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dedent": { "version": "0.7.0", "license": "MIT" @@ -12944,6 +13002,11 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, + "node_modules/diff3": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/diff3/-/diff3-0.0.3.tgz", + "integrity": "sha512-iSq8ngPOt0K53A6eVr4d5Kn6GNrM2nQZtC740pzIriHtn4pOQ2lyzEXQMBeVcWERN0ye7fhBsk9PbLLQOnUx/g==" + }, "node_modules/diffie-hellman": { "version": "5.0.3", "license": "MIT", @@ -14472,6 +14535,11 @@ "version": "2.1.1", "license": "MIT" }, + "node_modules/fast-text-encoding": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", + "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==" + }, "node_modules/fastest-levenshtein": { "version": "1.0.16", "dev": true, @@ -17118,6 +17186,59 @@ "version": "1.0.2", "license": "MIT" }, + "node_modules/isomorphic-git": { + "version": "1.25.7", + "resolved": "https://registry.npmjs.org/isomorphic-git/-/isomorphic-git-1.25.7.tgz", + "integrity": "sha512-KE10ejaIsEpQ+I/apS33qqTjyzCXgOniEaL32DwNbXtboKG8H3cu+RiBcdp3G9w4MpOOTQfGPsWp4i8UxRfDLg==", + "dependencies": { + "async-lock": "^1.1.0", + "clean-git-ref": "^2.0.1", + "crc-32": "^1.2.0", + "diff3": "0.0.3", + "ignore": "^5.1.4", + "minimisted": "^2.0.0", + "pako": "^1.0.10", + "pify": "^4.0.1", + "readable-stream": "^3.4.0", + "sha.js": "^2.4.9", + "simple-get": "^4.0.1" + }, + "bin": { + "isogit": "cli.cjs" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/isomorphic-git/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "engines": { + "node": ">=6" + } + }, + "node_modules/isomorphic-git/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/isomorphic-textencoder": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/isomorphic-textencoder/-/isomorphic-textencoder-1.0.1.tgz", + "integrity": "sha512-676hESgHullDdHDsj469hr+7t3i/neBKU9J7q1T4RHaWwLAsaQnywC0D1dIUId0YZ+JtVrShzuBk1soo0+GVcQ==", + "dependencies": { + "fast-text-encoding": "^1.0.0" + } + }, "node_modules/isstream": { "version": "0.1.2", "dev": true, @@ -18158,11 +18279,21 @@ "node": ">=4.0" } }, + "node_modules/just-debounce-it": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/just-debounce-it/-/just-debounce-it-1.1.0.tgz", + "integrity": "sha512-87Nnc0qZKgBZuhFZjYVjSraic0x7zwjhaTMrCKlj0QYKH6lh0KbFzVnfu6LHan03NO7J8ygjeBeD0epejn5Zcg==" + }, "node_modules/just-extend": { "version": "6.2.0", "dev": true, "license": "MIT" }, + "node_modules/just-once": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/just-once/-/just-once-1.1.0.tgz", + "integrity": "sha512-+rZVpl+6VyTilK7vB/svlMPil4pxqIJZkbnN7DKZTOzyXfun6ZiFeq2Pk4EtCEHZ0VU4EkdFzG8ZK5F3PErcDw==" + }, "node_modules/jwt-decode": { "version": "3.1.2", "license": "MIT" @@ -20129,6 +20260,17 @@ "node": ">=6" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/min-indent": { "version": "1.0.1", "license": "MIT", @@ -20181,6 +20323,14 @@ "node": ">= 6" } }, + "node_modules/minimisted": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/minimisted/-/minimisted-2.0.1.tgz", + "integrity": "sha512-1oPjfuLQa2caorJUM8HV8lGgWCc0qqAO1MNv/k05G4qslmsndV/5WdNZrqCiyqiz3wohia2Ij2B7w2Dr7/IyrA==", + "dependencies": { + "minimist": "^1.2.5" + } + }, "node_modules/minipass": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", @@ -28978,6 +29128,30 @@ ], "license": "MIT" }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/simple-git": { "version": "3.20.0", "license": "MIT", @@ -33231,6 +33405,7 @@ "decap-cms-backend-aws-cognito-github-proxy": "^3.1.2", "decap-cms-backend-azure": "^3.1.1", "decap-cms-backend-bitbucket": "^3.1.2", + "decap-cms-backend-git": "^3.1.1", "decap-cms-backend-git-gateway": "^3.1.1", "decap-cms-backend-github": "^3.1.2", "decap-cms-backend-gitlab": "^3.1.3", @@ -33335,6 +33510,31 @@ "react": "^18.2.0" } }, + "packages/decap-cms-backend-git": { + "version": "3.1.3", + "license": "MIT", + "dependencies": { + "@isomorphic-git/lightning-fs": "^4.6.0", + "apollo-cache-inmemory": "^1.6.2", + "apollo-client": "^2.6.3", + "apollo-link-context": "^1.0.18", + "apollo-link-http": "^1.5.15", + "isomorphic-git": "^1.25.7", + "js-base64": "^3.0.0", + "semaphore": "^1.1.0" + }, + "peerDependencies": { + "@emotion/react": "^11.11.1", + "@emotion/styled": "^11.11.0", + "decap-cms-lib-auth": "^3.0.0", + "decap-cms-lib-util": "^3.0.0", + "decap-cms-ui-default": "^3.0.0", + "immutable": "^3.7.6", + "lodash": "^4.17.11", + "prop-types": "^15.7.2", + "react": "^18.2.0" + } + }, "packages/decap-cms-backend-git-gateway": { "version": "3.1.1", "license": "MIT", diff --git a/packages/decap-cms-app/package.json b/packages/decap-cms-app/package.json index 2160fda489d8..ff8f61eabaef 100644 --- a/packages/decap-cms-app/package.json +++ b/packages/decap-cms-app/package.json @@ -33,6 +33,7 @@ "decap-cms-backend-aws-cognito-github-proxy": "^3.1.2", "decap-cms-backend-azure": "^3.1.1", "decap-cms-backend-bitbucket": "^3.1.2", + "decap-cms-backend-git": "^3.1.1", "decap-cms-backend-git-gateway": "^3.1.1", "decap-cms-backend-github": "^3.1.2", "decap-cms-backend-gitlab": "^3.1.3", diff --git a/packages/decap-cms-app/src/extensions.js b/packages/decap-cms-app/src/extensions.js index eef9ebd77063..429819159b42 100644 --- a/packages/decap-cms-app/src/extensions.js +++ b/packages/decap-cms-app/src/extensions.js @@ -3,6 +3,7 @@ import { DecapCmsCore as CMS } from 'decap-cms-core'; // Backends import { AzureBackend } from 'decap-cms-backend-azure'; import { AwsCognitoGitHubProxyBackend } from 'decap-cms-backend-aws-cognito-github-proxy'; +import { GitProxyBackendGenerator } from 'decap-cms-backend-git'; import { GitHubBackend } from 'decap-cms-backend-github'; import { GitLabBackend } from 'decap-cms-backend-gitlab'; import { GiteaBackend } from 'decap-cms-backend-gitea'; @@ -33,8 +34,10 @@ import * as locales from 'decap-cms-locales'; // Register all the things CMS.registerBackend('git-gateway', GitGatewayBackend); +CMS.registerBackend('git-gateway-large', GitProxyBackendGenerator(GitGatewayBackend)); CMS.registerBackend('azure', AzureBackend); CMS.registerBackend('aws-cognito-github-proxy', AwsCognitoGitHubProxyBackend); +CMS.registerBackend('github-large', GitProxyBackendGenerator(GitHubBackend)); CMS.registerBackend('github', GitHubBackend); CMS.registerBackend('gitlab', GitLabBackend); CMS.registerBackend('gitea', GiteaBackend); diff --git a/packages/decap-cms-backend-git/README.md b/packages/decap-cms-backend-git/README.md new file mode 100644 index 000000000000..6989d5d2093e --- /dev/null +++ b/packages/decap-cms-backend-git/README.md @@ -0,0 +1,7 @@ +# Git backend with fast read access + +## Code structure + +`Implementation` for [File Management System API](https://github.com/decaporg/decap-cms/tree/main/packages/decap-cms-lib-util/README.md). + +Look at tests or types for more info. diff --git a/packages/decap-cms-backend-git/package.json b/packages/decap-cms-backend-git/package.json new file mode 100644 index 000000000000..fda5acfd3acb --- /dev/null +++ b/packages/decap-cms-backend-git/package.json @@ -0,0 +1,42 @@ +{ + "name": "decap-cms-backend-git", + "description": "Git backend for Decap CMS with fast read access", + "version": "3.1.3", + "repository": "https://github.com/decaporg/decap-cms/tree/main/packages/decap-cms-backend-git", + "bugs": "https://github.com/decaporg/decap-cms/issues", + "license": "MIT", + "module": "dist/esm/index.js", + "main": "dist/decap-cms-backend-git.js", + "keywords": [ + "decap-cms", + "backend", + "gitlab" + ], + "sideEffects": false, + "scripts": { + "develop": "npm run build:esm -- --watch", + "build": "cross-env NODE_ENV=production webpack", + "build:esm": "cross-env NODE_ENV=esm babel src --out-dir dist/esm --ignore \"**/__tests__\" --root-mode upward --extensions \".js,.jsx,.ts,.tsx\"" + }, + "dependencies": { + "@isomorphic-git/lightning-fs": "^4.6.0", + "apollo-cache-inmemory": "^1.6.2", + "apollo-client": "^2.6.3", + "apollo-link-context": "^1.0.18", + "apollo-link-http": "^1.5.15", + "isomorphic-git": "^1.25.7", + "js-base64": "^3.0.0", + "semaphore": "^1.1.0" + }, + "peerDependencies": { + "@emotion/react": "^11.11.1", + "@emotion/styled": "^11.11.0", + "decap-cms-lib-auth": "^3.0.0", + "decap-cms-lib-util": "^3.0.0", + "decap-cms-ui-default": "^3.0.0", + "immutable": "^3.7.6", + "lodash": "^4.17.11", + "prop-types": "^15.7.2", + "react": "^18.2.0" + } +} diff --git a/packages/decap-cms-backend-git/src/implementation.ts b/packages/decap-cms-backend-git/src/implementation.ts new file mode 100644 index 000000000000..cf7a1af56025 --- /dev/null +++ b/packages/decap-cms-backend-git/src/implementation.ts @@ -0,0 +1,195 @@ +import git from 'isomorphic-git'; +import http from 'isomorphic-git/http/web'; +import FS from '@isomorphic-git/lightning-fs'; + +import type { + AssetProxy, + Config, + Credentials, + Cursor, + Entry, + Implementation, + ImplementationFile, + PersistOptions, + User, +} from 'decap-cms-lib-util'; + +const corsProxy = 'https://cors.isomorphic-git.org'; +const dir = '/repo'; +let singleton: Promise; + +function determineRepositoryURL(backend: any): string { + const name = backend.name as string; // TOOD consolidate CmsConfig and Config + if (name.startsWith('github')) { + return `https://github.com/${backend.repo}.git`; + } else if (name.startsWith('gitlab')) { + return `https://gitlab.com/${backend.repo}.git`; + } else if (name.startsWith('git-gateway')) { + if (!backend.repo) { + throw new Error( + "Repository URL mandatory for 'git-gateway' configuration (https://[...]/repoName.git)", + ); + } + return backend.repo!; + } + throw new Error("Can't determine repository URL"); +} + +export default function GitProxyBackEndGenerator(T: any) { + class GitProxyBackend implements Implementation { + backend: Implementation; + config: Config; + fs: any; + pfs: any; + repositoryUrl: string; + repository: Promise; + constructor(config: Config, options = {}) { + this.backend = new T(config, options); + this.config = config; + this.fs = new FS('decapfs'); + this.pfs = this.fs.promises; + this.repositoryUrl = determineRepositoryURL(config.backend); + if (!singleton) { + singleton = this.getRepository(); + } + this.repository = singleton; + } + + async getRepository() { + const branch = this.config.backend.branch || 'main'; + try { + await this.pfs.stat(dir); + } catch (e) { + await this.pfs.mkdir(dir); + await git.init({ + fs: this.fs, + dir, + defaultBranch: branch, + }); + } + await git.addRemote({ + fs: this.fs, + dir, + url: this.repositoryUrl, + remote: 'origin', + force: true, + }); + await git.fetch({ + fs: this.fs, + http, + dir, + corsProxy, + remote: 'origin', + ref: branch, + singleBranch: true, + depth: 1, + }); + await git.checkout({ + fs: this.fs, + dir, + ref: branch, + force: true, + track: false, + }); + return this.pfs.readdir(dir); + } + + isGitBackend() { + return true; + } + + async entriesByFolder(folder: string, extension: string) { + try { + await this.repository; + const files = await this.pfs.readdir(`${dir}/${folder}`); + const relevantFiles = files.filter((name: string) => name.endsWith(extension)); + return Promise.all( + relevantFiles.map(async (filename: string) => { + const path = `${folder}/${filename}`; + const fullPath = `${dir}/${path}`; + const data = await this.pfs.readFile(fullPath, 'utf8'); + return { + data, + file: { path, id: path }, + }; + }), + ); + } catch { + return []; + } + } + async getEntry(path: string) { + await this.repository; + const data = await this.pfs.readFile(`${dir}/${path}`, 'utf8'); + return { + file: { path, id: null }, + data, + }; + } + + status() { + return this.backend.status(); + } + authComponent() { + return this.backend.authComponent(); + } + restoreUser(user: User) { + return this.backend.restoreUser(user); + } + authenticate(credentials: Credentials) { + return this.backend.authenticate(credentials); + } + logout() { + return this.backend.logout(); + } + getToken() { + return this.backend.getToken(); + } + traverseCursor(cursor: Cursor, action: string) { + return this.backend.traverseCursor!(cursor, action); + } + entriesByFiles(files: ImplementationFile[]) { + return this.backend.entriesByFiles(files); + } + unpublishedEntries() { + return this.backend.unpublishedEntries(); + } + unpublishedEntry(args: { id?: string; collection?: string; slug?: string }) { + return this.backend.unpublishedEntry(args); + } + unpublishedEntryDataFile(collection: string, slug: string, path: string, id: string) { + return this.backend.unpublishedEntryDataFile(collection, slug, path, id); + } + unpublishedEntryMediaFile(collection: string, slug: string, path: string, id: string) { + return this.backend.unpublishedEntryMediaFile(collection, slug, path, id); + } + deleteUnpublishedEntry(collection: string, slug: string) { + return this.backend.deleteUnpublishedEntry(collection, slug); + } + persistEntry(entry: Entry, opts: PersistOptions) { + return this.backend.persistEntry(entry, opts); + } + updateUnpublishedEntryStatus(collection: string, slug: string, newStatus: string) { + return this.backend.updateUnpublishedEntryStatus(collection, slug, newStatus); + } + publishUnpublishedEntry(collection: string, slug: string) { + return this.backend.publishUnpublishedEntry(collection, slug); + } + getMedia(folder?: string) { + return this.backend.getMedia(folder); + } + getMediaFile(path: string) { + return this.backend.getMediaFile(path); + } + persistMedia(file: AssetProxy, opts: PersistOptions) { + return this.backend.persistMedia(file, opts); + } + deleteFiles(paths: string[], commitMessage: string) { + return this.backend.deleteFiles(paths, commitMessage); + } + getDeployPreview(collectionName: string, slug: string) { + return this.backend.getDeployPreview(collectionName, slug); + } + } + return GitProxyBackend; +} diff --git a/packages/decap-cms-backend-git/src/index.ts b/packages/decap-cms-backend-git/src/index.ts new file mode 100644 index 000000000000..d7af953d4c25 --- /dev/null +++ b/packages/decap-cms-backend-git/src/index.ts @@ -0,0 +1,6 @@ +import GitProxyBackendGenerator from './implementation'; + +export const DecapCmsBackendGitlab = { + GitProxyBackendGenerator, +}; +export { GitProxyBackendGenerator }; diff --git a/packages/decap-cms-backend-git/webpack.config.js b/packages/decap-cms-backend-git/webpack.config.js new file mode 100644 index 000000000000..42edd361d4a7 --- /dev/null +++ b/packages/decap-cms-backend-git/webpack.config.js @@ -0,0 +1,3 @@ +const { getConfig } = require('../../scripts/webpack.js'); + +module.exports = getConfig();