diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ae7d1b3f..ad24ce3c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: - name: Make dist directory run: mkdir -p dist/bin/ dist/packages/ dist/tarballs/ - name: Build cows - run: docker run -v ${PWD}:/usr/local/src -v ${PWD}/build/cows:/tmp/cows -v ${PWD}/build/pokesprite:/tmp/original/pokesprite -u root --platform linux/amd64 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_cowfiles.sh + run: docker run -v ${PWD}:/usr/local/src -v ${PWD}/build/cows:/tmp/cows -v ${PWD}/build/pokesprite-ansi:/tmp/original/cows -u root --platform linux/amd64 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_cowfiles.sh - name: Build go binary assets run: docker run -v ${PWD}:/usr/local/src -v ${PWD}/build/cows:/tmp/cows -u root --platform linux/amd64 -e VERSION=0.1 ghcr.io/tmck-code/pokesay:latest ./build/scripts/build_assets.sh - name: Run unit tests diff --git a/.gitmodules b/.gitmodules index fd254cc7..eeee9973 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ -[submodule "build/pokesprite"] - path = build/pokesprite - url = git@github.com:msikma/pokesprite +[submodule "build/pokesprite-ansi"] + path = build/pokesprite-ansi + url = git@github.com:tmck-code/pokesprite-ansi + shallow = true diff --git a/build/Dockerfile b/build/Dockerfile index 24d8da27..a838d9c2 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,24 +3,15 @@ FROM golang:1.24 WORKDIR /usr/local/src ENV DEBIAN_FRONTEND=noninteractive -ENV BUILD_DEPS="make gcc" -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - $BUILD_DEPS libmagickwand-dev ncurses-dev jq imagemagick tree \ - && git clone -q --depth 1 https://github.com/denilsonsa/img2xterm \ - && (cd img2xterm && make && make install) \ - && rm -rf img2xterm \ - && apt-get purge -y $BUILD_DEPS \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf /tmp/* /var/tmp/* - +RUN apt update \ + && apt upgrade -y \ + && apt install -y --no-install-recommends jq tree RUN useradd u -m -WORKDIR /usr/local/src/ ADD go.* /usr/local/src/ ADD src/ /usr/local/src/src/ + RUN go mod tidy \ && go get -v github.com/mitchellh/go-wordwrap \ && go install gotest.tools/gotestsum@latest \ diff --git a/build/Dockerfile.convert b/build/Dockerfile.convert new file mode 100644 index 00000000..00779908 --- /dev/null +++ b/build/Dockerfile.convert @@ -0,0 +1,26 @@ +FROM golang:1.24 + +WORKDIR /usr/local/src + +ENV DEBIAN_FRONTEND=noninteractive +ENV BUILD_DEPS="make gcc" +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + $BUILD_DEPS libmagickwand-dev ncurses-dev imagemagick tree \ + && git clone -q --depth 1 https://github.com/denilsonsa/img2xterm \ + && (cd img2xterm && make && make install) \ + && rm -rf img2xterm \ + && apt-get purge -y $BUILD_DEPS \ + && apt-get autoremove --purge -y \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/* /var/tmp/* + +RUN useradd u -m + +ADD go.* /usr/local/src/ +ADD src /usr/local/src/src/ + +RUN go mod tidy \ + && chown -R u:u /go + +ADD . . diff --git a/build/Makefile b/build/Makefile index ee3a94b7..56172812 100644 --- a/build/Makefile +++ b/build/Makefile @@ -26,27 +26,32 @@ build/docker: --build-arg DEBUG=$(DEBUG) \ .. +build/docker-convert: + @$(echo) -e "\e[48;5;30m> Building docker image $(DOCKER_IMAGE)-convert\e[0m" + @docker build \ + --platform linux/amd64 \ + -f Dockerfile.convert \ + -t $(DOCKER_REPO)/pokesay-convert:$(DOCKER_TAG) \ + --build-arg DEBUG=$(DEBUG) \ + .. + build/cows: @$(echo) -e "\e[48;5;30m> Building cows\e[0m" - @rm -rf cows.tar.gz cows/ - @mkdir -p cows/ + @rm -rf $(PWD)/cows && mkdir -p $(PWD)/cows @docker run --rm \ --platform linux/amd64 \ -v $(PWD)/../:/usr/local/src \ - -v $(PWD)/pokesprite:/tmp/original/pokesprite \ + -v $(PWD)/pokesprite-ansi:/tmp/original/cows \ -v $(PWD)/cows:/tmp/cows \ + -e DEBUG=$(DEBUG) \ -u u \ $(DOCKER_IMAGE) \ build/scripts/build_cowfiles.sh - @tar -czf cows.tar.gz ./cows - @du -sh cows.tar.gz # generate embedded bin files for category/metadata/the actual pokemon build/assets: - @$(echo) -e "\e[48;5;30m> Building assets\e[0m" + @$(echo) -e "\n\e[48;5;30m> Building assets\e[0m" @mkdir -p $(PWD)/assets - @rm -rf $(PWD)/cows/ - @tar -xzf cows.tar.gz @docker run --rm \ -v $(PWD)/../:/usr/local/src \ -v $(PWD)/cows:/tmp/cows \ @@ -58,7 +63,7 @@ build/assets: @tree -L 1 $(PWD)/assets/ build/bin: - @$(echo) -e "\e[48;5;30m> Building binaries\e[0m" + @$(echo) -e "\n\e[48;5;30m> Building binaries\e[0m" @mkdir -p $(PWD)/../dist/bin/ $(PWD)/../dist/packages/ $(PWD)/../dist/tarballs/ @docker run --rm \ -v $(PWD)/../:/usr/local/src \ @@ -72,7 +77,7 @@ build/bin: @tree $(PWD)/../dist/ build/deb: - @$(echo) -e "\e[48;5;30m> Building DEB packages\e[0m" + @$(echo) -e "\n\e[48;5;30m> Building DEB packages\e[0m" @docker run --rm \ -v $(PWD)/../:/usr/local/src \ -e VERSION=$(VERSION) \ @@ -82,7 +87,7 @@ build/deb: bash -c "/usr/local/src/build/scripts/build_packages.sh deb" build/arch: - @$(echo) -e "\e[48;5;30m> Building ARCH packages\e[0m" + @$(echo) -e "\n\e[48;5;30m> Building ARCH packages\e[0m" @docker run --rm\ -v $(PWD)/../:/usr/local/src \ -e VERSION=$(VERSION) \ @@ -92,11 +97,11 @@ build/arch: bash -c "useradd u -m && VERSION=$(VERSION) /usr/local/src/build/scripts/build_packages.sh arch" build/packages: build/deb build/arch - @$(echo) -e "\e[48;5;30m> Built packages:\e[0m" + @$(echo) -e "\n\e[48;5;30m> Built packages:\e[0m" @tree $(PWD)/../dist/ test: - @$(echo) -e "\e[48;5;30m> Running tests\e[0m" + @$(echo) -e "\n\e[48;5;30m> Running tests\e[0m" @docker run --rm \ -v $(PWD)/../:/usr/local/src \ --platform linux/amd64 \ @@ -105,5 +110,5 @@ test: $(DOCKER_IMAGE) \ gotestsum --format dots -.PHONY: all clean build/docker build/cows build/assets build/bin test +.PHONY: all clean build/docker build/docker-convert build/cows build/assets build/bin test .PHONY: build/deb build/arch build/packages diff --git a/build/cows.tar.gz b/build/cows.tar.gz deleted file mode 100644 index 4a4cad25..00000000 Binary files a/build/cows.tar.gz and /dev/null differ diff --git a/build/pokesprite b/build/pokesprite deleted file mode 160000 index c5aaa610..00000000 --- a/build/pokesprite +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c5aaa610ff2acdf7fd8e2dccd181bca8be9fcb3e diff --git a/build/pokesprite-ansi b/build/pokesprite-ansi new file mode 160000 index 00000000..a17818e6 --- /dev/null +++ b/build/pokesprite-ansi @@ -0,0 +1 @@ +Subproject commit a17818e6f14d6a28181b642fb36a118c34f7483a diff --git a/build/scripts/build_cowfiles.sh b/build/scripts/build_cowfiles.sh index ed47c46b..d37d5ac8 100755 --- a/build/scripts/build_cowfiles.sh +++ b/build/scripts/build_cowfiles.sh @@ -2,15 +2,39 @@ set -euo pipefail -mkdir -p /tmp/convert/ +skip=(resources/ misc/ icons/ items/ items-outline) +declare -A h -# Convert all of the pokesprite .pngs -> cowfiles for the terminal -go run /usr/local/src/src/bin/convert/png_convert.go \ - -from /tmp/original/pokesprite/ \ - -tmpDir /tmp/convert/ \ - -to /tmp/cows/ \ - -padding 4 \ - -skip '["resources/", "misc/", "icons/", "items/", "items-outline/"]' \ - && mv -v /tmp/cows/pokemon-gen8 /tmp/cows/gen8 > /dev/null \ - && mv -v /tmp/cows/pokemon-gen7x /tmp/cows/gen7x > /dev/null \ - && cat /tmp/original/pokesprite/data/pokemon.json | jq -c .[] > /tmp/cows/pokemon.json +cd /tmp/original/cows/ +fpaths=$(find . -iname '*.cow' -type f | sort -h) + +nOldFpaths=$(echo "$fpaths" | wc -l | tr -d '\n') +i=1 +echo "$fpaths" | while read f; do + progress=$(( ((i * 100)) / nOldFpaths)) + for s in "${skip[@]}"; do + if [[ "$f" == *"$s"* ]]; then + echo -e "[$i/$nOldFpaths $progress%] \e[0;33m skipping: $f\e[0m" + ((i++)) + continue 2 + fi + done + + s=$(sha256sum "$f" | cut -d' ' -f1) + if [[ ${h[$s]:-} ]]; then + echo -e "[$i/$nOldFpaths $progress%] \e[0;31m duplicate: $f\e[0m" + else + echo -e "[$i/$nOldFpaths $progress%] \e[0;32m copying: $f > /tmp/cows/$f\e[0m" + cp --parents "$f" /tmp/cows/ + h[$s]=1 + fi + ((i++)) +done + +# rename dirs & generate pokemon.json metadata file +mv -v /tmp/cows/pokemon-gen8 /tmp/cows/gen8 +mv -v /tmp/cows/pokemon-gen7x /tmp/cows/gen7x +cat /tmp/original/cows/data/pokemon.json | jq -c .[] > /tmp/cows/pokemon.json + +nNewFpaths=$(find /tmp/cows/ -iname '*.cow' | wc -l | tr -d '\n') +echo -e "\n\e[1;32m✔ all done, total files: $nNewFpaths / $nOldFpaths\e[0m" diff --git a/src/bin/convert/png_convert.go b/src/bin/convert/png_convert.go index 4158cb1a..ed447b0d 100644 --- a/src/bin/convert/png_convert.go +++ b/src/bin/convert/png_convert.go @@ -22,12 +22,13 @@ var ( ) type CowBuildArgs struct { - FromDir string - TmpDir string - ToDir string - SkipDirs []string - Padding int - Debug bool + FromDir string + TmpDir string + ToDir string + SkipDirs []string + SkipDuplicates bool + Padding int + Debug bool } func parseArgs() CowBuildArgs { @@ -35,11 +36,12 @@ func parseArgs() CowBuildArgs { tmpDir := flag.String("tmpDir", "/tmp/convert/", "temporary directory for intermediate files") toDir := flag.String("to", ".", "to dir") skipDirs := flag.String("skip", "'[\"resources\"]'", "JSON array of dir patterns to skip converting") + skipDuplicates := flag.Bool("skipDuplicates", false, "whether to skip duplicate images") padding := flag.Int("padding", 2, "the number of spaces to pad from the left") flag.Parse() - args := CowBuildArgs{FromDir: *fromDir, TmpDir: *tmpDir, ToDir: *toDir, Padding: *padding, Debug: DEBUG} + args := CowBuildArgs{FromDir: *fromDir, TmpDir: *tmpDir, ToDir: *toDir, SkipDuplicates: *skipDuplicates, Padding: *padding, Debug: DEBUG} json.Unmarshal([]byte(*skipDirs), &args.SkipDirs) if args.Debug { @@ -53,12 +55,12 @@ func worker(args CowBuildArgs, jobs <-chan string, pbar *progressbar.ProgressBar for f := range jobs { data, err := pokedex.ConvertPngToCow(args.FromDir, f, args.TmpDir, args.ToDir, args.Padding) - pbar.Add(1) if err != nil { mu.Lock() *nFailures++ mu.Unlock() + pbar.Add(1) continue } @@ -66,11 +68,15 @@ func worker(args CowBuildArgs, jobs <-chan string, pbar *progressbar.ProgressBar mu.Lock() if _, found := dataSet[data]; found { if args.Debug { - fmt.Println("Skipping duplicate data for", f) + fmt.Print("\r\x1b[J") // clear the progress bar before printing debug log + fmt.Println("Detected duplicate:", f) } *nDuplicates++ - mu.Unlock() - continue + if args.SkipDuplicates { + mu.Unlock() + pbar.Add(1) + continue + } } dataSet[data] = struct{}{} mu.Unlock() @@ -84,6 +90,7 @@ func worker(args CowBuildArgs, jobs <-chan string, pbar *progressbar.ProgressBar destFpath := filepath.Join(destDirpath, strings.ReplaceAll(filepath.Base(f), ".png", ".cow")) pokedex.WriteToCowfile(data, destDirpath, destFpath) + pbar.Add(1) } } @@ -119,14 +126,10 @@ func main() { wg.Add(1) go worker(args, jobs, &pbar, dataSet, &nDuplicates, &nFailures, &mu, &wg) } - - // Wait for all workers to finish wg.Wait() - fmt.Println("\nFinished converting", len(fpaths), "pokesprite PNGs -> cowfiles") - fmt.Println("(skipped", nDuplicates, "duplicates and", nFailures, "failures)") - // wait for progress bar to finish - time.Sleep(100 * time.Millisecond) + pbar.Finish() + time.Sleep(100 * time.Millisecond) // wait a moment to let the progress bar finish cleanly if args.Debug && len(pokedex.Failures) > 0 { fmt.Println("failures:") @@ -134,4 +137,15 @@ func main() { fmt.Println(" -", f) } } + + totalSucceeded := len(fpaths) - nFailures + if args.SkipDuplicates { + totalSucceeded -= nDuplicates + } + fmt.Printf("\n- converted %d/%d PNGs -> ANSI\n", totalSucceeded, len(fpaths)) + if args.SkipDuplicates { + fmt.Printf("- skipped %d duplicates\n- noticed %d failures\n\n", nDuplicates, nFailures) + } else { + fmt.Printf("- ignored %d duplicates\n- noticed %d failures\n\n", nDuplicates, nFailures) + } } diff --git a/src/pokedex/convert.go b/src/pokedex/convert.go index 9b8a0e68..6065a929 100644 --- a/src/pokedex/convert.go +++ b/src/pokedex/convert.go @@ -46,8 +46,7 @@ func autoCrop(sourceFpath string, tmpDirpath string) (string, error) { rand.Read(randomBytes) randomSuffix := hex.EncodeToString(randomBytes) - destFpath := fmt.Sprintf("%s/%s-%s", tmpDirpath, randomSuffix, filepath.Base(sourceFpath)) - // fmt.Println("Auto-cropping", sourceFpath, "->", destFpath) + destFpath := filepath.Join(tmpDirpath, fmt.Sprintf("%s-%s", randomSuffix, filepath.Base(sourceFpath))) output, err := exec.Command( "bash", "-c", fmt.Sprintf("/usr/bin/convert %s -trim +repage %s 2>&1", sourceFpath, destFpath), ).Output()