From bc31d8a42ca333eff57297294ffccf3ebbd84238 Mon Sep 17 00:00:00 2001 From: cxres Date: Sun, 23 Feb 2020 00:02:03 +0530 Subject: [PATCH 1/3] Fix cjs to esm in rdf-query.js Changed the cjs require statements to esm imports at the top of the file for consistency. Moved solid-namespace from depDependencies to dependencies in Package.json --- package.json | 4 ++-- src/utils/rdf-query.js | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 6403e5f..17bfbb4 100644 --- a/package.json +++ b/package.json @@ -57,13 +57,13 @@ "jsdoc-to-markdown": "^5.0.3", "regenerator-runtime": "^0.13.3", "solid-auth-cli": "^1.0.10", - "solid-namespace": "^0.2.0", "solid-rest": "^1.1.2", "standard": "^14.3.1", "webpack": "^4.41.5", "webpack-cli": "^3.3.10" }, "dependencies": { - "n3": "^1.3.5" + "n3": "^1.3.5", + "solid-namespace": "^0.2.0" } } diff --git a/src/utils/rdf-query.js b/src/utils/rdf-query.js index a337c74..c1581ee 100644 --- a/src/utils/rdf-query.js +++ b/src/utils/rdf-query.js @@ -4,8 +4,10 @@ * by Jeff Zucker with contributions from Otto_A_A and Alain Bourgeois * © 2019, Jeff Zucker, may be freely distributed using an MIT license */ -const N3 = require('n3') -const ns = require('solid-namespace')() +import * as N3 from 'n3'; +import solidNS from 'solid-namespace'; + +const ns = solidNS(); const { DataFactory } = N3 const { namedNode, literal } = DataFactory From 12939c241dbc63ad05a4ec0141881f55d0f39b2d Mon Sep 17 00:00:00 2001 From: cxres Date: Tue, 17 Mar 2020 10:42:12 +0530 Subject: [PATCH 2/3] Unified Copy Function The new `copy` function unifies both `copyFile` and `copyFolder`. It determines whether the source is a file or folder from the network response (without the need for an extra network request) instead of the trailing slash on provided source url. --- src/SolidApi.js | 103 ++++++++++++++++++++++++--- tests/SolidApi.composed.test.js | 121 +++++++++++++++----------------- 2 files changed, 148 insertions(+), 76 deletions(-) diff --git a/src/SolidApi.js b/src/SolidApi.js index ec95a9b..3411c20 100644 --- a/src/SolidApi.js +++ b/src/SolidApi.js @@ -497,7 +497,8 @@ class SolidAPI { if (!(await this._linkUrlsDefined(metaFrom, metaTo))) { return undefined } - return this.copyFile(metaFrom, metaTo, { withAcl: options.withAcl, withMeta: false }) + const metaResponse = await this.get(metaFrom); + return this.pasteFile(metaResponse, metaTo, { withAcl: options.withAcl, withMeta: false }) } /** @@ -541,7 +542,7 @@ class SolidAPI { if (options.agent === AGENT.TO_TARGET) { content = content.replace(new RegExp('<' + getRootUrl(oldTargetFile) + 'profile/card#', 'g'), '', 'g'), '') - } + } if (options.agent === AGENT.TO_SOURCE) { content = content.replace(new RegExp('', 'g'), '<' + getRootUrl(oldTargetFile) + 'profile/card#me>') @@ -609,7 +610,65 @@ class SolidAPI { } /** - * Copy a file (url ending with file name) or folder (url ending with "/"). + * Paste file contents. + * @param {Promise} getResoponse + * @param {string} to + * @param {WriteOptions} [options] + * @returns {Promise} + */ + async pasteFile(getResponse, to, options) { + const from = getResponse.url; + const content = await getResponse.blob() + const contentType = getResponse.headers.get('content-type') + const putResponse = await this.putFile(to, content, contentType, options) + + // Optionally copy ACL and Meta Files + // TODO: What do we want to do when the source has no acl, but the target has one? + // Currently it keeps the old acl. + await this.copyLinksForItem(from, to, options, getResponse, putResponse) + + return putResponse + } + + /** + * Copies Folder contents. + * @param {Promise} getResoponse + * @param {string} to + * @param {WriteOptions} [options] + * @returns {Promise} Resolves to an array of responses to copy further or paste. + */ + async copyFolderContents(getResponse, to, options) { + const from = getResponse.url; + const { folders, files } = await parseFolderResponse(getResponse) + const folderResponse = await this.createFolder(to, options) + + await this.copyLinksForItem(from, to, options, undefined, folderResponse) + + const foldersCreation = folders.map(async ({ name }) => { + const folderResp = await this.get(`${from}${name}/`, { headers: { Accept: 'text/turtle' } }) + return this.copyFolderContents(folderResp, `${to}${name}/`, options) + }) + + const filesCreation = files.map(async ({ name }) => { + try { + const fileResp = await this.get(`${from}${name}`) + return this.pasteFile(fileResp, `${to}${name}`, options) + } + catch (error) { + return err.message.includes('already existed') // Don't throw when merge=KEEP_TARGET and it tried to overwrite a file + } + }) + + const creationResults = await composedFetch([ + ...foldersCreation, + ...filesCreation, + ]).then(responses => responses.filter(item => !(item instanceof FetchError))) + + return [folderResponse].concat(...creationResults) // Alternative to Array.prototype.flat + } + + /** + * Copy a file or folder. * Per default existing folders will be deleted before copying and links will be copied. * @param {string} from * @param {string} to @@ -618,16 +677,40 @@ class SolidAPI { * The first one will be the folder specified by "to". * If it is a folder, the others will be creation responses from the contents in arbitrary order. */ - copy (from, to, options) { - // TBD: Rewrite to detect folders not by url (ie remove areFolders) - if (areFolders(from, to)) { - return this.copyFolder(from, to, options) + async copy (from, to, options) { + options = { + ...defaultWriteOptions, + ...options } - if (areFiles(from, to)) { - return this.copyFile(from, to, options) + if (typeof from !== 'string' || typeof to !== 'string') { + throw toFetchError(new Error(`The from and to parameters of copyFolder must be strings. Found: ${from} and ${to}`)) } - toFetchError(new Error('Cannot copy from a folder url to a file url or vice versa')) + let fromItem = await this.get(from); + // TBD: Check if response is text && RDF and parse RDF for this check + // Optimization: Reuse the RDF hence created for parseFolderResponse + const fromItemType = (fromItem.url.endsWith('/') && fromItem.headers.get('content-type') === 'text/turtle') ? + 'Container' : 'Resource'; + + if (fromItemType === 'Resource') { + if (to.endsWith('/')) { + throw toFetchError(new Error('May not copy file to a folder')) + } + return this.pasteFile(fromItem, to, options); + } + else if (fromItemType === 'Container') { + // TBD: Add additional check to see if response can be converted to turtle + // and avoid this additional fetch. For now, this test is redundant because + // of the test above and default response being 'text/turtle'. + if (fromItem.headers.get('content-type') !== 'text/turtle') { + fromItem = await this.get(url, { headers: { Accept: 'text/turtle' } }) + } + to = to.endsWith('/') ? to : `${to}/` + return this.copyFolderContents(fromItem, to, options); + } + else { + throw toFetchError(new Error(`Unrecognized item type ${fromItemType}`)) + } } /** diff --git a/tests/SolidApi.composed.test.js b/tests/SolidApi.composed.test.js index dae1ebc..ba9a640 100644 --- a/tests/SolidApi.composed.test.js +++ b/tests/SolidApi.composed.test.js @@ -198,75 +198,64 @@ describe('composed methods', () => { }) describe('copy', () => { - describe('copyFile', () => { - test('rejects with 404 on inexistent file', () => rejectsWithStatus(api.copyFile(inexistentFile.url, filePlaceholder.url), 404)) - test('rejects if no second url is specified', () => expect(api.copyFile(childFile.url)).rejects.toBeDefined()) - test('resolves with 201', () => resolvesWithStatus(api.copyFile(childFile.url, filePlaceholder.url), 201)) - test('resolves and has same content and contentType afterwards', async () => { - await resolvesWithStatus(api.copyFile(childFile.url, filePlaceholder.url), 201) - await expect(api.itemExists(filePlaceholder.url)).resolves.toBe(true) - const fromResponse = await api.get(childFile.url) - const toResponse = await api.get(filePlaceholder.url) - expect(fromResponse.headers.get('Content-Type')).toBe(toResponse.headers.get('Content-Type')) - expect(await fromResponse.text()).toBe(await toResponse.text()) - }) - test('rejects when copying to existent file with merge=KEEP_TARGET', () => { - return expect(api.copyFile(childFile.url, childFileTwo.url, { merge: MERGE.KEEP_TARGET })).rejects.toThrowError('already existed') - }) - test('rejects when copying from folder', () => { - return expect(api.copyFile(childOne.url, childFileTwo.url)).rejects.toBeDefined() - }) + test('rejects with 404 on inexistent source', () => rejectsWithStatus(api.copy(inexistentFile.url, filePlaceholder.url), 404)) + test('rejects if no destination is specified', () => expect(api.copy(childFile.url)).rejects.toBeDefined()) + test('resolves with 201 when copying a file', () => resolvesWithStatus(api.copy(childFile.url, filePlaceholder.url), 201)) + test('resolves when copying a file and has same content and contentType afterwards', async () => { + await resolvesWithStatus(api.copy(childFile.url, filePlaceholder.url), 201) + await expect(api.itemExists(filePlaceholder.url)).resolves.toBe(true) + const fromResponse = await api.get(childFile.url) + const toResponse = await api.get(filePlaceholder.url) + expect(fromResponse.headers.get('Content-Type')).toBe(toResponse.headers.get('Content-Type')) + expect(await fromResponse.text()).toBe(await toResponse.text()) + }) + test('rejects when copying to existent file with merge=KEEP_TARGET', () => { + return expect(api.copy(childFile.url, childFileTwo.url, { merge: MERGE.KEEP_TARGET })).rejects.toThrowError('already existed') }) - describe('copyFolder', () => { - test('rejects with 404 on inexistent folder', async () => rejectsWithStatus(api.copyFolder(inexistentFolder.url, inexistentFolder.url), 404)) - test('rejects if no second url is specified', () => expect(api.copyFolder(emptyFolder.url)).rejects.toBeDefined()) - test('resolves and copies empty folder', async () => { - await expect(api.copyFolder(emptyFolder.url, folderPlaceholder.url)).resolves.toBeDefined() - await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true) - }) - test('resolves copying folder with depth 1', () => { - return expect(api.copyFolder(childOne.url, folderPlaceholder.url)).resolves.toBeDefined() - }) - test('resolves with 201 and copies folder with depth 1 including its contents', async () => { - const responses = await api.copyFolder(childOne.url, folderPlaceholder.url) - expect(responses).toHaveLength(childOne.contents.length + 1) - expect(responses[0]).toHaveProperty('url', apiUtils.getParentUrl(folderPlaceholder.url)) - expect(responses[0]).toHaveProperty('status', 201) - - await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true) - await expect(api.itemExists(`${folderPlaceholder.url}${emptyFolder.name}/`)).resolves.toBe(true) - await expect(api.itemExists(`${folderPlaceholder.url}${childFile.name}`)).resolves.toBe(true) - }) - test('resolves copying folder with depth 2', async () => { - const responses = await api.copyFolder(parentFolder.url, folderPlaceholder.url) - expect(responses).toHaveLength(parentFolder.contents.length + 1) - expect(responses[0]).toHaveProperty('url', apiUtils.getParentUrl(folderPlaceholder.url)) - expect(responses[0]).toHaveProperty('status', 201) - - await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true) - // Note: Could test for others to exist too - }) - test('replaces existing folders per default', async () => { - await expect(api.copyFolder(childTwo.url, childOne.url)).resolves.toBeDefined() - await expect(api.itemExists(emptyFolder.url)).resolves.toBe(false) // empty folder was only in childOne - await expect(api.itemExists(childFile.url)).resolves.toBe(true) // childFile is in both - }) - test('overwrites files from target folder with merge=KEEP_SOURCE', async () => { - await expect(api.copyFolder(childTwo.url, childOne.url, { merge: MERGE.KEEP_SOURCE })).resolves.toBeDefined() - await expect(api.itemExists(emptyFolder.url)).resolves.toBe(true) - await expect(api.get(childFile.url).then(res => res.text())).resolves.toBe(childFileTwo.content) - }) - test('keeps files from target folder with merge=KEEP_TARGET', async () => { - await expect(api.copyFolder(childTwo.url, childOne.url, { merge: MERGE.KEEP_TARGET })).resolves.toBeDefined() - await expect(api.itemExists(emptyFolder.url)).resolves.toBe(true) - await expect(api.get(childFile.url).then(res => res.text())).resolves.toBe(childFile.content) - }) - test('throws some kind of error when called on file', async () => { - await expect(api.copyFolder(childFile.url, childTwo.url)).rejects.toBeDefined() - }) - test.todo('throws flattened errors when it fails in multiple levels') + test('resolves and copies empty folder', async () => { + await expect(api.copyFolder(emptyFolder.url, folderPlaceholder.url)).resolves.toBeDefined() + await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true) + }) + test('resolves copying folder with depth 1', () => { + return expect(api.copyFolder(childOne.url, folderPlaceholder.url)).resolves.toBeDefined() + }) + test('resolves with 201 and copies folder with depth 1 including its contents', async () => { + const responses = await api.copyFolder(childOne.url, folderPlaceholder.url) + expect(responses).toHaveLength(childOne.contents.length + 1) + expect(responses[0]).toHaveProperty('url', apiUtils.getParentUrl(folderPlaceholder.url)) + expect(responses[0]).toHaveProperty('status', 201) + + await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true) + await expect(api.itemExists(`${folderPlaceholder.url}${emptyFolder.name}/`)).resolves.toBe(true) + await expect(api.itemExists(`${folderPlaceholder.url}${childFile.name}`)).resolves.toBe(true) + }) + test('resolves copying folder with depth 2', async () => { + const responses = await api.copyFolder(parentFolder.url, folderPlaceholder.url) + expect(responses).toHaveLength(parentFolder.contents.length + 1) + expect(responses[0]).toHaveProperty('url', apiUtils.getParentUrl(folderPlaceholder.url)) + expect(responses[0]).toHaveProperty('status', 201) + + await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true) + // Note: Could test for others to exist too + }) + test('replaces existing folders per default', async () => { + await expect(api.copyFolder(childTwo.url, childOne.url)).resolves.toBeDefined() + await expect(api.itemExists(emptyFolder.url)).resolves.toBe(false) // empty folder was only in childOne + await expect(api.itemExists(childFile.url)).resolves.toBe(true) // childFile is in both + }) + test('overwrites files from target folder with merge=KEEP_SOURCE', async () => { + await expect(api.copyFolder(childTwo.url, childOne.url, { merge: MERGE.KEEP_SOURCE })).resolves.toBeDefined() + await expect(api.itemExists(emptyFolder.url)).resolves.toBe(true) + await expect(api.get(childFile.url).then(res => res.text())).resolves.toBe(childFileTwo.content) + }) + test('keeps files from target folder with merge=KEEP_TARGET', async () => { + await expect(api.copyFolder(childTwo.url, childOne.url, { merge: MERGE.KEEP_TARGET })).resolves.toBeDefined() + await expect(api.itemExists(emptyFolder.url)).resolves.toBe(true) + await expect(api.get(childFile.url).then(res => res.text())).resolves.toBe(childFile.content) }) + test.todo('rejects when trying to copy a file to a folder') + test.todo('throws flattened errors when it fails in multiple levels') }) describe('move', () => { From 83615741e4eeff39d07a623dd30c26ae7b8a88e8 Mon Sep 17 00:00:00 2001 From: cxres Date: Tue, 17 Mar 2020 11:57:26 +0530 Subject: [PATCH 3/3] Optimize copyLinksForItems Reuses prior get and put responses to get links for copying. This saves unnecessary multiple head calls being made to fetch links. --- src/SolidApi.js | 72 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/src/SolidApi.js b/src/SolidApi.js index 3411c20..d6d32b3 100644 --- a/src/SolidApi.js +++ b/src/SolidApi.js @@ -387,6 +387,23 @@ class SolidAPI { return parsedFolder } + /** + * Extract acl and meta links from a response + * @param {Promise} response + * @param {object} [options] specify if links should be checked for existence or not + * @returns {Promise} + */ + async extractItemLinks (response, options = { links: LINKS.INCLUDE_POSSIBLE }) { + if (options.links === LINKS.EXCLUDE) { + toFetchError(new Error('Invalid option LINKS.EXCLUDE for getItemLinks')) + } + const links = await getLinksFromResponse(response); + if (options.links === LINKS.INCLUDE) { + await this._removeInexistingLinks(links) + } + return links + } + /** * Get acl and meta links of an item * @param {string} url @@ -485,15 +502,15 @@ class SolidAPI { /** * Copy a meta file - * @param {string} oldTargetFile - * @param {string} newTargetFile + * @param {Object} oldTargetLinks + * @param {Object} newTargetLinks * @param {WriteOptions} [options] * @returns {Promise} creation response */ - async copyMetaFileForItem (oldTargetFile, newTargetFile, options = {}) { + async copyMetaFileForItem (oldTargetLinks, newTargetLinks, options = {}) { // TODO: Default options? - const { meta: metaFrom } = await this.getItemLinks(oldTargetFile) - const { meta: metaTo } = await this.getItemLinks(newTargetFile) + const { meta: metaFrom } = oldTargetLinks + const { meta: metaTo } = newTargetLinks if (!(await this._linkUrlsDefined(metaFrom, metaTo))) { return undefined } @@ -505,18 +522,20 @@ class SolidAPI { * Copy an ACL file * @param {string} oldTargetFile Url of the file the acl file targets (e.g. file.ttl for file.ttl.acl) * @param {string} newTargetFile Url of the new file targeted (e.g. new-file.ttl for new-file.ttl.acl) + * @param {Object} oldTargetLinks + * @param {Object} newTargetLinks * @param {WriteOptions} [options] * @returns {Promise} creation response */ - async copyAclFileForItem (oldTargetFile, newTargetFile, options) { + async copyAclFileForItem (oldTargetFile, newTargetFile, oldTargetLinks, newTargetLinks, options) { options = { ...defaultWriteOptions, ...({ agent: AGENT.NO_MODIFY }), ...options } - const { acl: aclFrom } = await this.getItemLinks(oldTargetFile) - const { acl: aclTo } = await this.getItemLinks(newTargetFile) + const { acl: aclFrom } = oldTargetLinks + const { acl: aclTo } = newTargetLinks if (!(await this._linkUrlsDefined(aclFrom, aclTo))) { return undefined @@ -573,6 +592,39 @@ class SolidAPI { return responses.filter(res => res && !(res instanceof Error)) } + /** + * Optimized Copy links for an item (for unified copy used by pasteFile and copyFolderContents). + * Use withAcl and withMeta options to specify which links to copy + * Does not throw if the links don't exist. + * @param {string} oldTargetFile Url of the file the acl file targets (e.g. file.ttl for file.ttl.acl) + * @param {string} newTargetFile Url of the new file targeted (e.g. new-file.ttl for new-file.ttl.acl) + * @param {Promise} getResponse Response from the get call + * @param {Promise} putResponse Response from the put call + * @param {WriteOptions} [options] + * @returns {Promise} creation responses + */ + async copyLinksForItem_ (oldTargetFile, newTargetFile, getResponse, putResponse, options) { + // TODO: Default options? + const responses = [] + let oldTargetLinks, newTargetsLinks; + + // Don't extract links if nothing is to run ahead + if (options.withMeta || options.withAcl) { + oldTargetLinks = await this.extractItemLinks(getResponse) + newTargetLinks = await this.extractItemLinks(putResource) + } + + if (options.withMeta) { + responses.push(await this.copyMetaFileForItem(oldTargetLinks, newTargetLinks, options) + .catch(assertResponseStatus(404))) + } + if (options.withAcl) { + responses.push(await this.copyAclFileForItem(oldTargetFile, newTargetFile, oldTargetLinks, newTargetLinks, options) + .catch(assertResponseStatus(404))) + } + return responses.filter(res => res && !(res instanceof Error)) + } + /** * Copy a folder and all contents. * Per default existing folders will be deleted before copying and links will be copied. @@ -625,7 +677,7 @@ class SolidAPI { // Optionally copy ACL and Meta Files // TODO: What do we want to do when the source has no acl, but the target has one? // Currently it keeps the old acl. - await this.copyLinksForItem(from, to, options, getResponse, putResponse) + await this.copyLinksForItem_(from, to, getResponse, putResponse, options) return putResponse } @@ -642,7 +694,7 @@ class SolidAPI { const { folders, files } = await parseFolderResponse(getResponse) const folderResponse = await this.createFolder(to, options) - await this.copyLinksForItem(from, to, options, undefined, folderResponse) + await this.copyLinksForItem_(from, to, getResponse, folderResponse, options) const foldersCreation = folders.map(async ({ name }) => { const folderResp = await this.get(`${from}${name}/`, { headers: { Accept: 'text/turtle' } })