From 496027ed4c6d97991807b92c661bebb1bb07545c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 6 Sep 2021 15:00:30 +1000 Subject: [PATCH 1/4] feat: make testmark output for ipld/ipld repo --- README.md | 6 +++ js/make-testmark.js | 128 ++++++++++++++++++++++++++++++++++++++++++++ js/package.json | 3 +- js/test.js | 26 ++------- js/util.js | 26 +++++++++ 5 files changed, 166 insertions(+), 23 deletions(-) create mode 100644 js/make-testmark.js create mode 100644 js/util.js diff --git a/README.md b/README.md index 0201776..a2bfdf0 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,12 @@ Or, in the [go](./go/) directory, run: go test ``` +## Generating testmark output for ipld.io + +Each codec tested here has a corresponding file in the https://github.com/ipld/ipld repository which generates the https://ipld.io website containing the fixture data in [testmark](https://github.com/warpfork/go-testmark) format. The filename per codec is is `specs/codecs//fixtures/cross-codec/index.md`. + +The [js/make-testmark.js](js/make-testmark.js) program can be used to update those files when the data is updated here. Run it with `node js/make-testmark.js `. + ## License Licensed under either of diff --git a/js/make-testmark.js b/js/make-testmark.js new file mode 100644 index 0000000..b1375b7 --- /dev/null +++ b/js/make-testmark.js @@ -0,0 +1,128 @@ +import fs from 'fs' +import { parse, patch, toString } from 'testmark.js' +import { codecs } from './codecs.js' +import { fixtureDirectories, loadFixture } from './util.js' + +async function makeTestmark () { + if (process.argv.length === 2) { + throw new Error('Usage `make-testmark.js ') + } + const repoRoot = new URL(process.argv[2], import.meta.url) + + const fixtures = [] + + for (const { name, url } of fixtureDirectories()) { + fixtures.push([name, await loadFixture(url)]) + } + + for (const codec of Object.keys(codecs)) { + const dir = new URL(`specs/codecs/${codec}/fixtures/cross-codec/`, repoRoot) + let stat + try { + stat = await fs.promises.stat(dir) + } catch (err) { + console.error(`Directory does not exist: ${dir.pathname}`) + process.exit(1) + } + if (!stat.isDirectory()) { + console.error(`Is not a directory: ${dir.pathname}`) + process.exit(1) + } + const fixtureFile = new URL('index.md', dir) + let tmDoc = parse(`# Cross-codec fixtures for ${codec}\n`) + try { + stat = await fs.promises.stat(fixtureFile) + if (!stat.isFile()) { + console.error(`Is not a file: ${fixtureFile.pathname}`) + process.exit(1) + } + const contents = await fs.promises.readFile(fixtureFile, 'utf8') + tmDoc = parse(contents) + } catch (err) { + } + + const patchHunks = [] + for (let [name, fixture] of fixtures) { + if (fixture[codec]) { + name = name.replace(/\s/g, '__') + if (tmDoc.hunksByName.has(name)) { // patch it + patchHunks.push({ + name, + blockTag: '', + body: fixture[codec].bytes.toString('hex') + }) + patchHunks.push({ + name: `${name}-cid-${codec}`, + blockTag: '', + body: fixture[codec].cid + }) + if (codec === 'dag-json') { + // special case for dag-json since a human should be able to read it + patchHunks.push({ + name: `${name}-string-form`, + blockTag: 'json', + body: new TextDecoder().decode(fixture[codec].bytes) + }) + } + for (const altCodec of Object.keys(fixture)) { + if (altCodec === codec) { + continue + } + patchHunks.push({ + name: `${name}-cid-${altCodec}`, + blockTag: '', + body: fixture[altCodec].cid + }) + } + } else { // create it + let section = `## ${name} + +### Bytes +[testmark]:# (${name}) +\`\`\` +${fixture[codec].bytes.toString('hex')} +\`\`\` +` + if (codec === 'dag-json') { + // special case for dag-json since a human should be able to read it + section += ` +### String form +[testmark]:# (${name}-string-form) +\`\`\` +${new TextDecoder().decode(fixture[codec].bytes)} +\`\`\` +` + } + + section += ` +### ${codec} CID +[testmark]:# (${name}-cid-${codec}) +\`\`\` +${fixture[codec].cid} +\`\`\` +` + for (const altCodec of Object.keys(fixture)) { + if (altCodec === codec) { + continue + } + section += ` +### ${altCodec} CID +[testmark]:# (${name}-cid-${altCodec}) +\`\`\` +${fixture[altCodec].cid} +\`\`\` +` + } + tmDoc.lines = tmDoc.lines.concat(section.split('\n')) + } + } + } + tmDoc = patch(tmDoc, patchHunks) + await fs.promises.writeFile(fixtureFile, toString(tmDoc), 'utf8') + } +} + +makeTestmark().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/js/package.json b/js/package.json index c7d0834..bcc8777 100644 --- a/js/package.json +++ b/js/package.json @@ -26,6 +26,7 @@ "chai": "^4.3.4", "ipld-garbage": "^4.0.3", "mocha": "^9.0.3", - "multiformats": "^9.4.3" + "multiformats": "^9.4.3", + "testmark.js": "^1.0.0" } } diff --git a/js/test.js b/js/test.js index a170b36..4f6f78e 100644 --- a/js/test.js +++ b/js/test.js @@ -1,24 +1,17 @@ /* eslint-env mocha */ -import fs from 'fs' -import path from 'path' import chai from 'chai' import { sha256 } from 'multiformats/hashes/sha2' import * as Block from 'multiformats/block' import { codecs } from './codecs.js' +import { fixtureDirectories, loadFixture } from './util.js' const { assert } = chai -const fixturesDir = new URL('../fixtures/', import.meta.url) describe('Codec fixtures', () => { - for (const dir of fs.readdirSync(fixturesDir)) { - const dirUrl = new URL(`./${dir}/`, fixturesDir) - const stat = fs.statSync(dirUrl) - if (!stat.isDirectory()) { - continue - } - it(dir, async () => { - const data = await loadFixture(dirUrl) + for (const { name, url } of fixtureDirectories()) { + it(name, async () => { + const data = await loadFixture(url) for (const [fromCodec, { bytes }] of Object.entries(data)) { const value = codecs[fromCodec].codec.decode(bytes) for (const [toCodec, { cid }] of Object.entries(data)) { @@ -29,14 +22,3 @@ describe('Codec fixtures', () => { }) } }) - -async function loadFixture (dir) { - const data = {} - for (const file of await fs.promises.readdir(dir)) { - const ext = path.extname(file).slice(1) - const cid = file.substring(0, file.length - ext.length - 1) - const bytes = await fs.promises.readFile(new URL(file, dir)) - data[ext] = { cid, bytes } - } - return data -} diff --git a/js/util.js b/js/util.js new file mode 100644 index 0000000..9970fe0 --- /dev/null +++ b/js/util.js @@ -0,0 +1,26 @@ +import fs from 'fs' +import path from 'path' + +export const fixturesDir = new URL('../fixtures/', import.meta.url) + +export async function loadFixture (dir) { + const data = {} + for (const file of await fs.promises.readdir(dir)) { + const ext = path.extname(file).slice(1) + const cid = file.substring(0, file.length - ext.length - 1) + const bytes = await fs.promises.readFile(new URL(file, dir)) + data[ext] = { cid, bytes } + } + return data +} + +export function * fixtureDirectories () { + for (const name of fs.readdirSync(fixturesDir)) { + const url = new URL(`./${name}/`, fixturesDir) + const stat = fs.statSync(url) + if (!stat.isDirectory()) { + continue + } + yield { name, url } + } +} From 1c08c7ded08243f6d9180f57f50e45fa23fe5b49 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 9 Sep 2021 14:19:33 +1000 Subject: [PATCH 2/4] fix: cleanup output, use path naming, linebreak hex @ 120chars --- js/make-testmark.js | 53 +++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/js/make-testmark.js b/js/make-testmark.js index b1375b7..eb162df 100644 --- a/js/make-testmark.js +++ b/js/make-testmark.js @@ -29,7 +29,18 @@ async function makeTestmark () { process.exit(1) } const fixtureFile = new URL('index.md', dir) - let tmDoc = parse(`# Cross-codec fixtures for ${codec}\n`) + let tmDoc = parse(`--- +templateEngineOverride: md +--- + +# Cross-codec fixtures for ${codec} + +## Introduction + +_TODO_ + +## Fixtures +`) try { stat = await fs.promises.stat(fixtureFile) if (!stat.isFile()) { @@ -44,22 +55,22 @@ async function makeTestmark () { const patchHunks = [] for (let [name, fixture] of fixtures) { if (fixture[codec]) { - name = name.replace(/\s/g, '__') - if (tmDoc.hunksByName.has(name)) { // patch it + name = name.replace(/[\s/]/g, '__') + if (tmDoc.hunksByName.has(`${name}/${codec}/bytes`)) { // patch it patchHunks.push({ - name, + name: `${name}/${codec}/bytes`, blockTag: '', - body: fixture[codec].bytes.toString('hex') + body: fixture[codec].bytes.toString('hex').replace(/(.{120})/g, '$1\n') }) patchHunks.push({ - name: `${name}-cid-${codec}`, + name: `${name}/${codec}/cid`, blockTag: '', body: fixture[codec].cid }) if (codec === 'dag-json') { // special case for dag-json since a human should be able to read it patchHunks.push({ - name: `${name}-string-form`, + name: `${name}/${codec}/string`, blockTag: 'json', body: new TextDecoder().decode(fixture[codec].bytes) }) @@ -69,34 +80,37 @@ async function makeTestmark () { continue } patchHunks.push({ - name: `${name}-cid-${altCodec}`, + name: `${name}/${altCodec}/cid`, blockTag: '', body: fixture[altCodec].cid }) } } else { // create it - let section = `## ${name} + let section = `### ${name} + +**Bytes** -### Bytes -[testmark]:# (${name}) +[testmark]:# (${name}/${codec}/bytes) \`\`\` -${fixture[codec].bytes.toString('hex')} +${fixture[codec].bytes.toString('hex').replace(/(.{120})/g, '$1\n')} \`\`\` ` if (codec === 'dag-json') { // special case for dag-json since a human should be able to read it section += ` -### String form -[testmark]:# (${name}-string-form) -\`\`\` +**String form** + +[testmark]:# (${name}/${codec}/string) +\`\`\`json ${new TextDecoder().decode(fixture[codec].bytes)} \`\`\` ` } section += ` -### ${codec} CID -[testmark]:# (${name}-cid-${codec}) +**${codec} CID** + +[testmark]:# (${name}/${codec}/cid) \`\`\` ${fixture[codec].cid} \`\`\` @@ -106,8 +120,9 @@ ${fixture[codec].cid} continue } section += ` -### ${altCodec} CID -[testmark]:# (${name}-cid-${altCodec}) +**${altCodec} CID** + +[testmark]:# (${name}/${altCodec}/cid) \`\`\` ${fixture[altCodec].cid} \`\`\` From 7988570e1f6a7a71c1bc9d3bb606c5feed680723 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 9 Sep 2021 16:50:14 +1000 Subject: [PATCH 3/4] fix: switch from crlf handling to fixture files as binary --- .gitattributes | 2 -- fixtures/.gitattributes | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 .gitattributes create mode 100644 fixtures/.gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index fb9a97d..0000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -* text=auto -*.* text eol=lf diff --git a/fixtures/.gitattributes b/fixtures/.gitattributes new file mode 100644 index 0000000..f0471e1 --- /dev/null +++ b/fixtures/.gitattributes @@ -0,0 +1 @@ +*.* binary From 008c4f7bd42c0a3e9f62b3e943d5485015e5f514 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 9 Sep 2021 17:16:09 +1000 Subject: [PATCH 4/4] fix: ignore .gitattributes as a test fixture --- go/codecs_test.go | 2 +- js/util.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/codecs_test.go b/go/codecs_test.go index 730899c..2f2ece2 100644 --- a/go/codecs_test.go +++ b/go/codecs_test.go @@ -18,7 +18,7 @@ func TestCodecs(t *testing.T) { for _, dir := range dirs { fixtureName := dir.Name() if !dir.IsDir() { - t.Fatalf("%v is not a directory", fixtureName) + continue } if reason, blacklisted := FixtureBlacklist[fixtureName]; blacklisted { fmt.Printf("Skipping fixture '%v': %v\n", fixtureName, reason) diff --git a/js/util.js b/js/util.js index 9970fe0..588d439 100644 --- a/js/util.js +++ b/js/util.js @@ -16,11 +16,11 @@ export async function loadFixture (dir) { export function * fixtureDirectories () { for (const name of fs.readdirSync(fixturesDir)) { - const url = new URL(`./${name}/`, fixturesDir) - const stat = fs.statSync(url) + const stat = fs.statSync(new URL(`./${name}`, fixturesDir)) if (!stat.isDirectory()) { continue } + const url = new URL(`./${name}/`, fixturesDir) yield { name, url } } }