Skip to content

Unified copy #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
CxRes wants to merge 4 commits into from
Closed

Unified copy #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bc31d8a
Fix cjs to esm in rdf-query.js
CxRes Feb 22, 2020
da5fd33
Merge branch 'master' of https://github.com/jeff-zucker/solid-file-cl…
CxRes Mar 15, 2020
12939c2
Unified Copy Function
CxRes Mar 17, 2020
8361574
Optimize copyLinksForItems
CxRes Mar 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 153 additions & 18 deletions src/SolidApi.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,23 @@ class SolidAPI {
return parsedFolder
}

/**
* Extract acl and meta links from a response
* @param {Promise<Response>} response
* @param {object} [options] specify if links should be checked for existence or not
* @returns {Promise<Links>}
*/
async extractItemLinks (response, options = { links: LINKS.INCLUDE_POSSIBLE }) {
if (options.links === LINKS.EXCLUDE) {
toFetchError(new Error('Invalid option LINKS.EXCLUDE for getItemLinks'))
}
const links = await getLinksFromResponse(response);
if (options.links === LINKS.INCLUDE) {
await this._removeInexistingLinks(links)
}
return links
}

/**
* Get acl and meta links of an item
* @param {string} url
Expand Down Expand Up @@ -485,37 +502,40 @@ class SolidAPI {

/**
* Copy a meta file
* @param {string} oldTargetFile
* @param {string} newTargetFile
* @param {Object} oldTargetLinks
* @param {Object} newTargetLinks
* @param {WriteOptions} [options]
* @returns {Promise<Response|undefined>} creation response
*/
async copyMetaFileForItem (oldTargetFile, newTargetFile, options = {}) {
async copyMetaFileForItem (oldTargetLinks, newTargetLinks, options = {}) {
// TODO: Default options?
const { meta: metaFrom } = await this.getItemLinks(oldTargetFile)
const { meta: metaTo } = await this.getItemLinks(newTargetFile)
const { meta: metaFrom } = oldTargetLinks
const { meta: metaTo } = newTargetLinks
if (!(await this._linkUrlsDefined(metaFrom, metaTo))) {
return undefined
}
return this.copyFile(metaFrom, metaTo, { withAcl: options.withAcl, withMeta: false })
const metaResponse = await this.get(metaFrom);
return this.pasteFile(metaResponse, metaTo, { withAcl: options.withAcl, withMeta: false })
}

/**
* Copy an ACL file
* @param {string} oldTargetFile Url of the file the acl file targets (e.g. file.ttl for file.ttl.acl)
* @param {string} newTargetFile Url of the new file targeted (e.g. new-file.ttl for new-file.ttl.acl)
* @param {Object} oldTargetLinks
* @param {Object} newTargetLinks
* @param {WriteOptions} [options]
* @returns {Promise<Response>} creation response
*/
async copyAclFileForItem (oldTargetFile, newTargetFile, options) {
async copyAclFileForItem (oldTargetFile, newTargetFile, oldTargetLinks, newTargetLinks, options) {
options = {
...defaultWriteOptions,
...({ agent: AGENT.NO_MODIFY }),
...options
}

const { acl: aclFrom } = await this.getItemLinks(oldTargetFile)
const { acl: aclTo } = await this.getItemLinks(newTargetFile)
const { acl: aclFrom } = oldTargetLinks
const { acl: aclTo } = newTargetLinks

if (!(await this._linkUrlsDefined(aclFrom, aclTo))) {
return undefined
Expand All @@ -541,7 +561,7 @@ class SolidAPI {
if (options.agent === AGENT.TO_TARGET) {
content = content.replace(new RegExp('<' + getRootUrl(oldTargetFile) + 'profile/card#', 'g'), '</profile/card#')
content = content.replace(new RegExp('<' + getRootUrl(oldTargetFile) + 'profile/card#me>', 'g'), '</profile/card#me>')
}
}
if (options.agent === AGENT.TO_SOURCE) {
content = content.replace(new RegExp('</profile/card#', 'g'), '<' + getRootUrl(oldTargetFile) + 'profile/card#')
content = content.replace(new RegExp('</profile/card#me>', 'g'), '<' + getRootUrl(oldTargetFile) + 'profile/card#me>')
Expand Down Expand Up @@ -572,6 +592,39 @@ class SolidAPI {
return responses.filter(res => res && !(res instanceof Error))
}

/**
* Optimized Copy links for an item (for unified copy used by pasteFile and copyFolderContents).
* Use withAcl and withMeta options to specify which links to copy
* Does not throw if the links don't exist.
* @param {string} oldTargetFile Url of the file the acl file targets (e.g. file.ttl for file.ttl.acl)
* @param {string} newTargetFile Url of the new file targeted (e.g. new-file.ttl for new-file.ttl.acl)
* @param {Promise<Response>} getResponse Response from the get call
* @param {Promise<Response>} putResponse Response from the put call
* @param {WriteOptions} [options]
* @returns {Promise<Response[]>} creation responses
*/
async copyLinksForItem_ (oldTargetFile, newTargetFile, getResponse, putResponse, options) {
// TODO: Default options?
const responses = []
let oldTargetLinks, newTargetsLinks;

// Don't extract links if nothing is to run ahead
if (options.withMeta || options.withAcl) {
oldTargetLinks = await this.extractItemLinks(getResponse)
newTargetLinks = await this.extractItemLinks(putResource)
}

if (options.withMeta) {
responses.push(await this.copyMetaFileForItem(oldTargetLinks, newTargetLinks, options)
.catch(assertResponseStatus(404)))
}
if (options.withAcl) {
responses.push(await this.copyAclFileForItem(oldTargetFile, newTargetFile, oldTargetLinks, newTargetLinks, options)
.catch(assertResponseStatus(404)))
}
return responses.filter(res => res && !(res instanceof Error))
}

/**
* Copy a folder and all contents.
* Per default existing folders will be deleted before copying and links will be copied.
Expand Down Expand Up @@ -609,7 +662,65 @@ class SolidAPI {
}

/**
* Copy a file (url ending with file name) or folder (url ending with "/").
* Paste file contents.
* @param {Promise<Response>} getResoponse
* @param {string} to
* @param {WriteOptions} [options]
* @returns {Promise<Response>}
*/
async pasteFile(getResponse, to, options) {
const from = getResponse.url;
const content = await getResponse.blob()
const contentType = getResponse.headers.get('content-type')
const putResponse = await this.putFile(to, content, contentType, options)

// Optionally copy ACL and Meta Files
// TODO: What do we want to do when the source has no acl, but the target has one?
// Currently it keeps the old acl.
await this.copyLinksForItem_(from, to, getResponse, putResponse, options)

return putResponse
}

/**
* Copies Folder contents.
* @param {Promise<Response>} getResoponse
* @param {string} to
* @param {WriteOptions} [options]
* @returns {Promise<Response[]>} Resolves to an array of responses to copy further or paste.
*/
async copyFolderContents(getResponse, to, options) {
const from = getResponse.url;
const { folders, files } = await parseFolderResponse(getResponse)
const folderResponse = await this.createFolder(to, options)

await this.copyLinksForItem_(from, to, getResponse, folderResponse, options)

const foldersCreation = folders.map(async ({ name }) => {
const folderResp = await this.get(`${from}${name}/`, { headers: { Accept: 'text/turtle' } })
return this.copyFolderContents(folderResp, `${to}${name}/`, options)
})

const filesCreation = files.map(async ({ name }) => {
try {
const fileResp = await this.get(`${from}${name}`)
return this.pasteFile(fileResp, `${to}${name}`, options)
}
catch (error) {
return err.message.includes('already existed') // Don't throw when merge=KEEP_TARGET and it tried to overwrite a file
}
})

const creationResults = await composedFetch([
...foldersCreation,
...filesCreation,
]).then(responses => responses.filter(item => !(item instanceof FetchError)))

return [folderResponse].concat(...creationResults) // Alternative to Array.prototype.flat
}

/**
* Copy a file or folder.
* Per default existing folders will be deleted before copying and links will be copied.
* @param {string} from
* @param {string} to
Expand All @@ -618,16 +729,40 @@ class SolidAPI {
* The first one will be the folder specified by "to".
* If it is a folder, the others will be creation responses from the contents in arbitrary order.
*/
copy (from, to, options) {
// TBD: Rewrite to detect folders not by url (ie remove areFolders)
if (areFolders(from, to)) {
return this.copyFolder(from, to, options)
async copy (from, to, options) {
options = {
...defaultWriteOptions,
...options
}
if (areFiles(from, to)) {
return this.copyFile(from, to, options)
if (typeof from !== 'string' || typeof to !== 'string') {
throw toFetchError(new Error(`The from and to parameters of copyFolder must be strings. Found: ${from} and ${to}`))
}

toFetchError(new Error('Cannot copy from a folder url to a file url or vice versa'))
let fromItem = await this.get(from);
// TBD: Check if response is text && RDF and parse RDF for this check
// Optimization: Reuse the RDF hence created for parseFolderResponse
const fromItemType = (fromItem.url.endsWith('/') && fromItem.headers.get('content-type') === 'text/turtle') ?
'Container' : 'Resource';

if (fromItemType === 'Resource') {
if (to.endsWith('/')) {
throw toFetchError(new Error('May not copy file to a folder'))
}
return this.pasteFile(fromItem, to, options);
}
else if (fromItemType === 'Container') {
// TBD: Add additional check to see if response can be converted to turtle
// and avoid this additional fetch. For now, this test is redundant because
// of the test above and default response being 'text/turtle'.
if (fromItem.headers.get('content-type') !== 'text/turtle') {
fromItem = await this.get(url, { headers: { Accept: 'text/turtle' } })
}
to = to.endsWith('/') ? to : `${to}/`
return this.copyFolderContents(fromItem, to, options);
}
else {
throw toFetchError(new Error(`Unrecognized item type ${fromItemType}`))
}
}

/**
Expand Down
121 changes: 55 additions & 66 deletions tests/SolidApi.composed.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,75 +198,64 @@ describe('composed methods', () => {
})

describe('copy', () => {
describe('copyFile', () => {
test('rejects with 404 on inexistent file', () => rejectsWithStatus(api.copyFile(inexistentFile.url, filePlaceholder.url), 404))
test('rejects if no second url is specified', () => expect(api.copyFile(childFile.url)).rejects.toBeDefined())
test('resolves with 201', () => resolvesWithStatus(api.copyFile(childFile.url, filePlaceholder.url), 201))
test('resolves and has same content and contentType afterwards', async () => {
await resolvesWithStatus(api.copyFile(childFile.url, filePlaceholder.url), 201)
await expect(api.itemExists(filePlaceholder.url)).resolves.toBe(true)
const fromResponse = await api.get(childFile.url)
const toResponse = await api.get(filePlaceholder.url)
expect(fromResponse.headers.get('Content-Type')).toBe(toResponse.headers.get('Content-Type'))
expect(await fromResponse.text()).toBe(await toResponse.text())
})
test('rejects when copying to existent file with merge=KEEP_TARGET', () => {
return expect(api.copyFile(childFile.url, childFileTwo.url, { merge: MERGE.KEEP_TARGET })).rejects.toThrowError('already existed')
})
test('rejects when copying from folder', () => {
return expect(api.copyFile(childOne.url, childFileTwo.url)).rejects.toBeDefined()
})
test('rejects with 404 on inexistent source', () => rejectsWithStatus(api.copy(inexistentFile.url, filePlaceholder.url), 404))
test('rejects if no destination is specified', () => expect(api.copy(childFile.url)).rejects.toBeDefined())
test('resolves with 201 when copying a file', () => resolvesWithStatus(api.copy(childFile.url, filePlaceholder.url), 201))
test('resolves when copying a file and has same content and contentType afterwards', async () => {
await resolvesWithStatus(api.copy(childFile.url, filePlaceholder.url), 201)
await expect(api.itemExists(filePlaceholder.url)).resolves.toBe(true)
const fromResponse = await api.get(childFile.url)
const toResponse = await api.get(filePlaceholder.url)
expect(fromResponse.headers.get('Content-Type')).toBe(toResponse.headers.get('Content-Type'))
expect(await fromResponse.text()).toBe(await toResponse.text())
})
test('rejects when copying to existent file with merge=KEEP_TARGET', () => {
return expect(api.copy(childFile.url, childFileTwo.url, { merge: MERGE.KEEP_TARGET })).rejects.toThrowError('already existed')
})

describe('copyFolder', () => {
test('rejects with 404 on inexistent folder', async () => rejectsWithStatus(api.copyFolder(inexistentFolder.url, inexistentFolder.url), 404))
test('rejects if no second url is specified', () => expect(api.copyFolder(emptyFolder.url)).rejects.toBeDefined())
test('resolves and copies empty folder', async () => {
await expect(api.copyFolder(emptyFolder.url, folderPlaceholder.url)).resolves.toBeDefined()
await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true)
})
test('resolves copying folder with depth 1', () => {
return expect(api.copyFolder(childOne.url, folderPlaceholder.url)).resolves.toBeDefined()
})
test('resolves with 201 and copies folder with depth 1 including its contents', async () => {
const responses = await api.copyFolder(childOne.url, folderPlaceholder.url)
expect(responses).toHaveLength(childOne.contents.length + 1)
expect(responses[0]).toHaveProperty('url', apiUtils.getParentUrl(folderPlaceholder.url))
expect(responses[0]).toHaveProperty('status', 201)

await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true)
await expect(api.itemExists(`${folderPlaceholder.url}${emptyFolder.name}/`)).resolves.toBe(true)
await expect(api.itemExists(`${folderPlaceholder.url}${childFile.name}`)).resolves.toBe(true)
})
test('resolves copying folder with depth 2', async () => {
const responses = await api.copyFolder(parentFolder.url, folderPlaceholder.url)
expect(responses).toHaveLength(parentFolder.contents.length + 1)
expect(responses[0]).toHaveProperty('url', apiUtils.getParentUrl(folderPlaceholder.url))
expect(responses[0]).toHaveProperty('status', 201)

await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true)
// Note: Could test for others to exist too
})
test('replaces existing folders per default', async () => {
await expect(api.copyFolder(childTwo.url, childOne.url)).resolves.toBeDefined()
await expect(api.itemExists(emptyFolder.url)).resolves.toBe(false) // empty folder was only in childOne
await expect(api.itemExists(childFile.url)).resolves.toBe(true) // childFile is in both
})
test('overwrites files from target folder with merge=KEEP_SOURCE', async () => {
await expect(api.copyFolder(childTwo.url, childOne.url, { merge: MERGE.KEEP_SOURCE })).resolves.toBeDefined()
await expect(api.itemExists(emptyFolder.url)).resolves.toBe(true)
await expect(api.get(childFile.url).then(res => res.text())).resolves.toBe(childFileTwo.content)
})
test('keeps files from target folder with merge=KEEP_TARGET', async () => {
await expect(api.copyFolder(childTwo.url, childOne.url, { merge: MERGE.KEEP_TARGET })).resolves.toBeDefined()
await expect(api.itemExists(emptyFolder.url)).resolves.toBe(true)
await expect(api.get(childFile.url).then(res => res.text())).resolves.toBe(childFile.content)
})
test('throws some kind of error when called on file', async () => {
await expect(api.copyFolder(childFile.url, childTwo.url)).rejects.toBeDefined()
})
test.todo('throws flattened errors when it fails in multiple levels')
test('resolves and copies empty folder', async () => {
await expect(api.copyFolder(emptyFolder.url, folderPlaceholder.url)).resolves.toBeDefined()
await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true)
})
test('resolves copying folder with depth 1', () => {
return expect(api.copyFolder(childOne.url, folderPlaceholder.url)).resolves.toBeDefined()
})
test('resolves with 201 and copies folder with depth 1 including its contents', async () => {
const responses = await api.copyFolder(childOne.url, folderPlaceholder.url)
expect(responses).toHaveLength(childOne.contents.length + 1)
expect(responses[0]).toHaveProperty('url', apiUtils.getParentUrl(folderPlaceholder.url))
expect(responses[0]).toHaveProperty('status', 201)

await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true)
await expect(api.itemExists(`${folderPlaceholder.url}${emptyFolder.name}/`)).resolves.toBe(true)
await expect(api.itemExists(`${folderPlaceholder.url}${childFile.name}`)).resolves.toBe(true)
})
test('resolves copying folder with depth 2', async () => {
const responses = await api.copyFolder(parentFolder.url, folderPlaceholder.url)
expect(responses).toHaveLength(parentFolder.contents.length + 1)
expect(responses[0]).toHaveProperty('url', apiUtils.getParentUrl(folderPlaceholder.url))
expect(responses[0]).toHaveProperty('status', 201)

await expect(api.itemExists(folderPlaceholder.url)).resolves.toBe(true)
// Note: Could test for others to exist too
})
test('replaces existing folders per default', async () => {
await expect(api.copyFolder(childTwo.url, childOne.url)).resolves.toBeDefined()
await expect(api.itemExists(emptyFolder.url)).resolves.toBe(false) // empty folder was only in childOne
await expect(api.itemExists(childFile.url)).resolves.toBe(true) // childFile is in both
})
test('overwrites files from target folder with merge=KEEP_SOURCE', async () => {
await expect(api.copyFolder(childTwo.url, childOne.url, { merge: MERGE.KEEP_SOURCE })).resolves.toBeDefined()
await expect(api.itemExists(emptyFolder.url)).resolves.toBe(true)
await expect(api.get(childFile.url).then(res => res.text())).resolves.toBe(childFileTwo.content)
})
test('keeps files from target folder with merge=KEEP_TARGET', async () => {
await expect(api.copyFolder(childTwo.url, childOne.url, { merge: MERGE.KEEP_TARGET })).resolves.toBeDefined()
await expect(api.itemExists(emptyFolder.url)).resolves.toBe(true)
await expect(api.get(childFile.url).then(res => res.text())).resolves.toBe(childFile.content)
})
test.todo('rejects when trying to copy a file to a folder')
test.todo('throws flattened errors when it fails in multiple levels')
})

describe('move', () => {
Expand Down