diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e8354a1c..ae7d1b3f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,8 +4,10 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive - name: Log in to the Container registry uses: docker/login-action@v1 with: @@ -16,16 +18,18 @@ jobs: run: docker pull ghcr.io/tmck-code/pokesay:latest - name: Make dist directory run: mkdir -p dist/bin/ dist/packages/ dist/tarballs/ + - name: Build cows + run: docker run -v ${PWD}:/usr/local/src -v ${PWD}/build/cows:/tmp/cows -v ${PWD}/build/pokesprite:/tmp/original/pokesprite -u root --platform linux/amd64 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_cowfiles.sh - name: Build go binary assets - run: docker run -v ${PWD}:/usr/local/src -u root --platform linux/amd64 -e VERSION=0.1 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_assets.sh + run: docker run -v ${PWD}:/usr/local/src -v ${PWD}/build/cows:/tmp/cows -u root --platform linux/amd64 -e VERSION=0.1 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_assets.sh + - name: Run unit tests + run: docker run -v ${PWD}:/usr/local/src -u root ghcr.io/tmck-code/pokesay:latest gotestsum --debug - name: Build go binaries run: docker run -v ${PWD}:/usr/local/src -u root --platform linux/amd64 -e VERSION=0.1 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_bin.sh - name: Check build run: ls -alh dist/bin - name: Test build run: echo w | ./dist/bin/pokesay-0.1-linux-amd64 - - name: Run unit tests - run: docker run -v ${PWD}:/usr/local/src -u root ghcr.io/tmck-code/pokesay:latest gotestsum --debug - name: Build deb package run: docker run -v ${PWD}:/usr/local/src -u root -e VERSION=0.1 -e DEBUG=1 ghcr.io/tmck-code/pokesay:latest bash -c "./build/scripts/build_packages.sh deb" - name: Build arch package diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..fd254cc7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "build/pokesprite"] + path = build/pokesprite + url = git@github.com:msikma/pokesprite diff --git a/build/Dockerfile b/build/Dockerfile index 0638caf6..24d8da27 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -15,30 +15,15 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /tmp/* /var/tmp/* + +RUN useradd u -m + WORKDIR /usr/local/src/ -ADD src/ /usr/local/src/src/ ADD go.* /usr/local/src/ +ADD src/ /usr/local/src/src/ RUN go mod tidy \ && go get -v github.com/mitchellh/go-wordwrap \ - && go install gotest.tools/gotestsum@latest - -RUN useradd u -m -USER u - -RUN git clone -q --depth 1 https://github.com/msikma/pokesprite /tmp/original/pokesprite - -ARG DEBUG -ENV DEBUG=$DEBUG -RUN echo "Debug mode is set to: '$DEBUG'" -# Convert all of the pokesprite .pngs -> cowfiles for the terminal -RUN go run /usr/local/src/src/bin/convert/png_convert.go \ - -from /tmp/original/pokesprite/ \ - -to /tmp/cows/ \ - -padding 4 \ - -skip '["resources/", "misc/", "icons/", "items/", "items-outline/"]' \ - && mv -v /tmp/cows/pokemon-gen8 /tmp/cows/gen8 \ - && mv -v /tmp/cows/pokemon-gen7x /tmp/cows/gen7x \ - && cat /tmp/original/pokesprite/data/pokemon.json | jq -c .[] > /tmp/cows/pokemon.json \ - && rm -rf /tmp/original/pokesprite + && go install gotest.tools/gotestsum@latest \ + && chown -R u:u /go ADD . . diff --git a/build/Makefile b/build/Makefile index df9482ca..bf59dade 100644 --- a/build/Makefile +++ b/build/Makefile @@ -29,23 +29,29 @@ build/docker: build/cows: @$(echo) -e "\e[48;5;30m> Building cows\e[0m" @rm -rf cows.tar.gz cows/ - @docker rm -f pokebuilder > /dev/null 2>&1 - @docker create \ + @mkdir -p cows/ + @docker run --rm \ --platform linux/amd64 \ - --name pokebuilder $(DOCKER_IMAGE) - docker cp pokebuilder:$(DOCKER_OUTPUT_DIR)/ cows/ - @tar czf cows.tar.gz cows/ - @rm -rf cows/ - @docker rm -f pokebuilder > /dev/null 2>&1 + -v $(PWD)/../:/usr/local/src \ + -v $(PWD)/pokesprite:/tmp/original/pokesprite \ + -v $(PWD)/cows:/tmp/cows \ + -u u \ + $(DOCKER_IMAGE) \ + build/scripts/build_cowfiles.sh + @tar -czf cows.tar.gz ./cows @du -sh cows.tar.gz # generate embedded bin files for category/metadata/the actual pokemon build/assets: @$(echo) -e "\e[48;5;30m> Building assets\e[0m" @mkdir -p $(PWD)/assets + @rm -rf $(PWD)/cows/ + @tar -xzf cows.tar.gz @docker run --rm \ -v $(PWD)/../:/usr/local/src \ + -v $(PWD)/cows:/tmp/cows \ --platform linux/amd64 \ + -u u \ --name pokesay \ $(DOCKER_IMAGE) \ build/scripts/build_assets.sh @@ -59,7 +65,7 @@ build/bin: -e VERSION=$(VERSION) \ -e DEBUG=$(DEBUG) \ --platform linux/amd64 \ - --user root \ + --user u \ --name pokesay \ $(DOCKER_IMAGE) \ /usr/local/src/build/scripts/build_bin.sh diff --git a/build/pokesprite b/build/pokesprite new file mode 160000 index 00000000..c5aaa610 --- /dev/null +++ b/build/pokesprite @@ -0,0 +1 @@ +Subproject commit c5aaa610ff2acdf7fd8e2dccd181bca8be9fcb3e diff --git a/build/scripts/build_cowfiles.sh b/build/scripts/build_cowfiles.sh new file mode 100755 index 00000000..ed47c46b --- /dev/null +++ b/build/scripts/build_cowfiles.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +mkdir -p /tmp/convert/ + +# Convert all of the pokesprite .pngs -> cowfiles for the terminal +go run /usr/local/src/src/bin/convert/png_convert.go \ + -from /tmp/original/pokesprite/ \ + -tmpDir /tmp/convert/ \ + -to /tmp/cows/ \ + -padding 4 \ + -skip '["resources/", "misc/", "icons/", "items/", "items-outline/"]' \ + && mv -v /tmp/cows/pokemon-gen8 /tmp/cows/gen8 > /dev/null \ + && mv -v /tmp/cows/pokemon-gen7x /tmp/cows/gen7x > /dev/null \ + && cat /tmp/original/pokesprite/data/pokemon.json | jq -c .[] > /tmp/cows/pokemon.json diff --git a/src/bin/convert/png_convert.go b/src/bin/convert/png_convert.go index ae4bc846..4158cb1a 100644 --- a/src/bin/convert/png_convert.go +++ b/src/bin/convert/png_convert.go @@ -6,7 +6,12 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" + "sync" + "time" + + "github.com/schollz/progressbar/v3" "github.com/tmck-code/pokesay/src/bin" "github.com/tmck-code/pokesay/src/pokedex" @@ -18,6 +23,7 @@ var ( type CowBuildArgs struct { FromDir string + TmpDir string ToDir string SkipDirs []string Padding int @@ -26,13 +32,14 @@ type CowBuildArgs struct { func parseArgs() CowBuildArgs { fromDir := flag.String("from", ".", "from dir") + tmpDir := flag.String("tmpDir", "/tmp/convert/", "temporary directory for intermediate files") toDir := flag.String("to", ".", "to dir") skipDirs := flag.String("skip", "'[\"resources\"]'", "JSON array of dir patterns to skip converting") padding := flag.Int("padding", 2, "the number of spaces to pad from the left") flag.Parse() - args := CowBuildArgs{FromDir: *fromDir, ToDir: *toDir, Padding: *padding, Debug: DEBUG} + args := CowBuildArgs{FromDir: *fromDir, TmpDir: *tmpDir, ToDir: *toDir, Padding: *padding, Debug: DEBUG} json.Unmarshal([]byte(*skipDirs), &args.SkipDirs) if args.Debug { @@ -41,44 +48,32 @@ func parseArgs() CowBuildArgs { return args } -func main() { - args := parseArgs() - - fpaths := pokedex.FindFiles(args.FromDir, ".png", args.SkipDirs) - - // Ensure that the destination dir exists - os.MkdirAll(args.ToDir, 0755) - - fmt.Println("Converting PNGs -> cowfiles") - pbar := bin.NewProgressBar(len(fpaths)) +func worker(args CowBuildArgs, jobs <-chan string, pbar *progressbar.ProgressBar, dataSet map[string]struct{}, nDuplicates *int, nFailures *int, mu *sync.Mutex, wg *sync.WaitGroup) { + defer wg.Done() - allData := make([]string, 0, len(fpaths)) - nDuplicates, nFailures := 0, 0 + for f := range jobs { + data, err := pokedex.ConvertPngToCow(args.FromDir, f, args.TmpDir, args.ToDir, args.Padding) + pbar.Add(1) - for _, f := range fpaths { - data, err := pokedex.ConvertPngToCow(args.FromDir, f, args.ToDir, args.Padding) if err != nil { - nFailures++ + mu.Lock() + *nFailures++ + mu.Unlock() continue } // check if this cawfile is a duplicate of one that has already been written - found := false - for _, existingData := range allData { - if existingData == data { - found = true - break - } - } - if found { + mu.Lock() + if _, found := dataSet[data]; found { if args.Debug { fmt.Println("Skipping duplicate data for", f) } - nDuplicates++ - pbar.Add(1) + *nDuplicates++ + mu.Unlock() continue } - allData = append(allData, data) + dataSet[data] = struct{}{} + mu.Unlock() destDirpath := filepath.Join( args.ToDir, @@ -89,8 +84,54 @@ func main() { destFpath := filepath.Join(destDirpath, strings.ReplaceAll(filepath.Base(f), ".png", ".cow")) pokedex.WriteToCowfile(data, destDirpath, destFpath) - pbar.Add(1) } +} + +func main() { + args := parseArgs() + + fpaths := pokedex.FindFiles(args.FromDir, ".png", args.SkipDirs) + + // Ensure that the destination dir exists + os.MkdirAll(args.ToDir, 0755) + + fmt.Println("Converting PNGs -> cowfiles") + pbar := bin.NewProgressBar(len(fpaths)) + + dataSet := make(map[string]struct{}) + nDuplicates, nFailures := 0, 0 + + nWorkers := runtime.NumCPU() + var wg sync.WaitGroup + var mu sync.Mutex + + // Create a channel to distribute work + jobs := make(chan string, len(fpaths)) + + // Send all file paths to the jobs channel + for _, f := range fpaths { + jobs <- f + } + close(jobs) + + // Start worker goroutines + for w := 0; w < nWorkers; w++ { + wg.Add(1) + go worker(args, jobs, &pbar, dataSet, &nDuplicates, &nFailures, &mu, &wg) + } + + // Wait for all workers to finish + wg.Wait() fmt.Println("\nFinished converting", len(fpaths), "pokesprite PNGs -> cowfiles") fmt.Println("(skipped", nDuplicates, "duplicates and", nFailures, "failures)") + + // wait for progress bar to finish + time.Sleep(100 * time.Millisecond) + + if args.Debug && len(pokedex.Failures) > 0 { + fmt.Println("failures:") + for _, f := range pokedex.Failures { + fmt.Println(" -", f) + } + } } diff --git a/src/pokedex/convert.go b/src/pokedex/convert.go index 6ce84729..9b8a0e68 100644 --- a/src/pokedex/convert.go +++ b/src/pokedex/convert.go @@ -2,6 +2,9 @@ package pokedex import ( "bufio" + "crypto/rand" + "encoding/hex" + "errors" "fmt" "os" "os/exec" @@ -10,7 +13,7 @@ import ( ) var ( - failures []string + Failures []string COLOUR_RESET string = fmt.Sprintf("%s[%dm\n", "\x1b", 39) ) @@ -33,18 +36,26 @@ func FindFiles(dirpath string, ext string, skip []string) []string { // img2xterm converts an image to a cowfile, returning the result as a byte slice func img2xterm(sourceFpath string) ([]byte, error) { - return exec.Command("bash", "-c", fmt.Sprintf("/usr/local/bin/img2xterm %s", sourceFpath)).Output() + return exec.Command("bash", "-c", fmt.Sprintf("/usr/local/bin/img2xterm %s 2>&1", sourceFpath)).Output() } // autoCrop trims the whitespace from the edges of an image, in place -func autoCrop(sourceFpath string) { - destFpath := fmt.Sprintf("/tmp/%s", filepath.Base(sourceFpath)) - _, err := exec.Command( - "bash", "-c", fmt.Sprintf("/usr/bin/convert %s -trim +repage %s", sourceFpath, destFpath), +func autoCrop(sourceFpath string, tmpDirpath string) (string, error) { + // Generate a random suffix to avoid conflicts when running in parallel + randomBytes := make([]byte, 8) + rand.Read(randomBytes) + randomSuffix := hex.EncodeToString(randomBytes) + + destFpath := fmt.Sprintf("%s/%s-%s", tmpDirpath, randomSuffix, filepath.Base(sourceFpath)) + // fmt.Println("Auto-cropping", sourceFpath, "->", destFpath) + output, err := exec.Command( + "bash", "-c", fmt.Sprintf("/usr/bin/convert %s -trim +repage %s 2>&1", sourceFpath, destFpath), ).Output() - Check(err) + if err != nil { + return "", fmt.Errorf("auto-crop failed for %s: (%v) - %s", sourceFpath, err, strings.Trim(string(output), "\n")) + } - os.Rename(destFpath, sourceFpath) + return destFpath, nil } func countLineLeftPadding(line string) int { @@ -107,28 +118,46 @@ func padLeft(cowfile []byte, n int) []string { return converted } -func ConvertPngToCow(sourceDirpath string, sourceFpath string, destDirpath string, extraPadding int) (string, error) { +func ConvertPngToCow(sourceDirpath string, sourceFpath string, tmpDirpath string, destDirpath string, extraPadding int) (string, error) { // Trim the whitespace from the edges of the images. This helps with the conversion - autoCrop(sourceFpath) + tmpFpath, err := autoCrop(sourceFpath, tmpDirpath) + if err != nil { + // If autoCrop fails, still try to convert the original image + Failures = append(Failures, string(err.Error())) + return "", err + } + // Some conversions are failing with something about colour channels - // Can't be bothered resolving atm, so just skip past any failed conversions - converted, _ := img2xterm(sourceFpath) + output, err := img2xterm(tmpFpath) + if err != nil { + failureMsg := fmt.Sprintf("failed to convert %s: (%v) - %s", tmpFpath, err, strings.Trim(string(output), "\n")) + Failures = append(Failures, failureMsg) + return "", errors.New(failureMsg) + } - if len(converted) == 0 { - failures = append(failures, sourceFpath) - return "", fmt.Errorf("failed to convert %s", sourceFpath) + if len(output) == 0 { + failureMsg := fmt.Sprintf("failed to convert %s: no output", tmpFpath) + Failures = append(Failures, failureMsg) + return "", errors.New(failureMsg) } - final := stripEmptyLines(padLeft(converted, extraPadding)) + final := stripEmptyLines(padLeft(output, extraPadding)) return strings.Join(final, "\n") + COLOUR_RESET, nil } func WriteToCowfile(data string, destDirpath string, destFpath string) { // Ensure that the destination dir exists - os.MkdirAll(destDirpath, 0755) + err := os.MkdirAll(destDirpath, 0755) + if err != nil { + fmt.Printf("Failed to create directory %s: %v\n", destDirpath, err) + Check(err) + } ostream, err := os.Create(destFpath) - Check(err) + if err != nil { + fmt.Printf("Failed to create file %s: %v\n", destFpath, err) + Check(err) + } defer ostream.Close() writer := bufio.NewWriter(ostream)