Generate testmark output for ipld/ipld (ipld.io)

README.md

@@ -65,6 +65,12 @@ Or, in the [go](./go/) directory, run:
 go test
 ```
 
+## Generating testmark output for ipld.io
+
+Each codec tested here has a corresponding file in the https://github.com/ipld/ipld repository which generates the https://ipld.io website containing the fixture data in [testmark](https://github.com/warpfork/go-testmark) format. The filename per codec is is `specs/codecs/<CODEC>/fixtures/cross-codec/index.md`.
+
+The [js/make-testmark.js](js/make-testmark.js) program can be used to update those files when the data is updated here. Run it with `node js/make-testmark.js <path/to/ipld/ipld/repository>`.
+
 ## License
 
 Licensed under either of

fixtures/.gitattributes

@@ -0,0 +1 @@
+*.* binary

go/codecs_test.go

@@ -18,7 +18,7 @@ func TestCodecs(t *testing.T) {
 	for _, dir := range dirs {
 		fixtureName := dir.Name()
 		if !dir.IsDir() {
-			t.Fatalf("%v is not a directory", fixtureName)
+			continue
 		}
 		if reason, blacklisted := FixtureBlacklist[fixtureName]; blacklisted {
 			fmt.Printf("Skipping fixture '%v': %v\n", fixtureName, reason)

js/make-testmark.js

@@ -0,0 +1,143 @@
+import fs from 'fs'
+import { parse, patch, toString } from 'testmark.js'
+import { codecs } from './codecs.js'
+import { fixtureDirectories, loadFixture } from './util.js'
+
+async function makeTestmark () {
+  if (process.argv.length === 2) {
+    throw new Error('Usage `make-testmark.js <path/to/ipld/metarepo>')
+  }
+  const repoRoot = new URL(process.argv[2], import.meta.url)
+
+  const fixtures = []
+
+  for (const { name, url } of fixtureDirectories()) {
+    fixtures.push([name, await loadFixture(url)])
+  }
+
+  for (const codec of Object.keys(codecs)) {
+    const dir = new URL(`specs/codecs/${codec}/fixtures/cross-codec/`, repoRoot)
+    let stat
+    try {
+      stat = await fs.promises.stat(dir)
+    } catch (err) {
+      console.error(`Directory does not exist: ${dir.pathname}`)
+      process.exit(1)
+    }
+    if (!stat.isDirectory()) {
+      console.error(`Is not a directory: ${dir.pathname}`)
+      process.exit(1)
+    }
+    const fixtureFile = new URL('index.md', dir)
+    let tmDoc = parse(`---
+templateEngineOverride: md
+---
+
+# Cross-codec fixtures for ${codec}
+
+## Introduction
+
+_TODO_
+
+## Fixtures
+`)
+    try {
+      stat = await fs.promises.stat(fixtureFile)
+      if (!stat.isFile()) {
+        console.error(`Is not a file: ${fixtureFile.pathname}`)
+        process.exit(1)
+      }
+      const contents = await fs.promises.readFile(fixtureFile, 'utf8')
+      tmDoc = parse(contents)
+    } catch (err) {
+    }
+
+    const patchHunks = []
+    for (let [name, fixture] of fixtures) {
+      if (fixture[codec]) {
+        name = name.replace(/[\s/]/g, '__')
+        if (tmDoc.hunksByName.has(`${name}/${codec}/bytes`)) { // patch it
+          patchHunks.push({
+            name: `${name}/${codec}/bytes`,
+            blockTag: '',
+            body: fixture[codec].bytes.toString('hex').replace(/(.{120})/g, '$1\n')
+          })
+          patchHunks.push({
+            name: `${name}/${codec}/cid`,
+            blockTag: '',
+            body: fixture[codec].cid
+          })
+          if (codec === 'dag-json') {
+            // special case for dag-json since a human should be able to read it
+            patchHunks.push({
+              name: `${name}/${codec}/string`,
+              blockTag: 'json',
+              body: new TextDecoder().decode(fixture[codec].bytes)
+            })
+          }
+          for (const altCodec of Object.keys(fixture)) {
+            if (altCodec === codec) {
+              continue
+            }
+            patchHunks.push({
+              name: `${name}/${altCodec}/cid`,
+              blockTag: '',
+              body: fixture[altCodec].cid
+            })
+          }
+        } else { // create it
+          let section = `### ${name}
+
+**Bytes**
+
+[testmark]:# (${name}/${codec}/bytes)
+\`\`\`
+${fixture[codec].bytes.toString('hex').replace(/(.{120})/g, '$1\n')}
+\`\`\`
+`
+          if (codec === 'dag-json') {
+            // special case for dag-json since a human should be able to read it
+            section += `
+**String form**
+
+[testmark]:# (${name}/${codec}/string)
+\`\`\`json
+${new TextDecoder().decode(fixture[codec].bytes)}
+\`\`\`
+`
+          }
+
+          section += `
+**${codec} CID**
+
+[testmark]:# (${name}/${codec}/cid)
+\`\`\`
+${fixture[codec].cid}
+\`\`\`
+`
+          for (const altCodec of Object.keys(fixture)) {
+            if (altCodec === codec) {
+              continue
+            }
+            section += `
+**${altCodec} CID**
+
+[testmark]:# (${name}/${altCodec}/cid)
+\`\`\`
+${fixture[altCodec].cid}
+\`\`\`
+`
+          }
+          tmDoc.lines = tmDoc.lines.concat(section.split('\n'))
+        }
+      }
+    }
+    tmDoc = patch(tmDoc, patchHunks)
+    await fs.promises.writeFile(fixtureFile, toString(tmDoc), 'utf8')
+  }
+}
+
+makeTestmark().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})

js/package.json

@@ -26,6 +26,7 @@
     "chai": "^4.3.4",
     "ipld-garbage": "^4.0.3",
     "mocha": "^9.0.3",
-    "multiformats": "^9.4.3"
+    "multiformats": "^9.4.3",
+    "testmark.js": "^1.0.0"
   }
 }

js/test.js

@@ -1,24 +1,17 @@
 /* eslint-env mocha */
 
-import fs from 'fs'
-import path from 'path'
 import chai from 'chai'
 import { sha256 } from 'multiformats/hashes/sha2'
 import * as Block from 'multiformats/block'
 import { codecs } from './codecs.js'
+import { fixtureDirectories, loadFixture } from './util.js'
 
 const { assert } = chai
-const fixturesDir = new URL('../fixtures/', import.meta.url)
 
 describe('Codec fixtures', () => {
-  for (const dir of fs.readdirSync(fixturesDir)) {
-    const dirUrl = new URL(`./${dir}/`, fixturesDir)
-    const stat = fs.statSync(dirUrl)
-    if (!stat.isDirectory()) {
-      continue
-    }
-    it(dir, async () => {
-      const data = await loadFixture(dirUrl)
+  for (const { name, url } of fixtureDirectories()) {
+    it(name, async () => {
+      const data = await loadFixture(url)
       for (const [fromCodec, { bytes }] of Object.entries(data)) {
         const value = codecs[fromCodec].codec.decode(bytes)
         for (const [toCodec, { cid }] of Object.entries(data)) {
@@ -29,14 +22,3 @@ describe('Codec fixtures', () => {
     })
   }
 })
-
-async function loadFixture (dir) {
-  const data = {}
-  for (const file of await fs.promises.readdir(dir)) {
-    const ext = path.extname(file).slice(1)
-    const cid = file.substring(0, file.length - ext.length - 1)
-    const bytes = await fs.promises.readFile(new URL(file, dir))
-    data[ext] = { cid, bytes }
-  }
-  return data
-}

js/util.js

@@ -0,0 +1,26 @@
+import fs from 'fs'
+import path from 'path'
+
+export const fixturesDir = new URL('../fixtures/', import.meta.url)
+
+export async function loadFixture (dir) {
+  const data = {}
+  for (const file of await fs.promises.readdir(dir)) {
+    const ext = path.extname(file).slice(1)
+    const cid = file.substring(0, file.length - ext.length - 1)
+    const bytes = await fs.promises.readFile(new URL(file, dir))
+    data[ext] = { cid, bytes }
+  }
+  return data
+}
+
+export function * fixtureDirectories () {
+  for (const name of fs.readdirSync(fixturesDir)) {
+    const stat = fs.statSync(new URL(`./${name}`, fixturesDir))
+    if (!stat.isDirectory()) {
+      continue
+    }
+    const url = new URL(`./${name}/`, fixturesDir)
+    yield { name, url }
+  }
+}