citadel

My dotfiles, scripts and nix configs
git clone git://jb55.com/citadel
Log | Files | Refs | README | LICENSE

parallel-chunked (432B)

      1 #!/usr/bin/env bash
      2 
      3 usage () {
      4   printf "%s <file> <process-script>\n\n" "$0" >&2
      5   printf "splits a file into chunks and process it line by line in parallel\n" >&2
      6   exit 1
      7 }
      8 
      9 if [ -z "$1" ]; then
     10   usage
     11 fi
     12 
     13 outfile="$1"
     14 shift
     15 
     16 blocksize=$(bc <<<"$(stat -c %s "$outfile") / $(nproc)")
     17 printf 'parallel with %s chunks...\n' "$(numfmt --to=iec "$blocksize")" >&2
     18 exec parallel -k --pipepart --block "$blocksize" -a "$outfile" "$@"