As an extension of my previous post on parallel execution I present turboprop. The initial version of this script will perform an optimization of a mysql database with multiple tables running at the same time. In the future it may be extended to allow for more operations from the command line such as mysql dumps.

<code>#!/bin/bash
# turboprop

# how many jobs to run at one time
JOBS_AT_ONCE=20
# Command to run in parallel in this case mysqlcheck -o
COMMAND="mysqlcheck -o"

# Print usage information and exit
print_usage(){
    echo -e "\n" \
    "usage: ./turboprop -d databasename \n" \
    "Optimizes mysql tables in parallel\n" \
    "-d <databasename>      Database to optimize\n" \
    "-h                     this help\n" \
    "\n" && exit 1
}

###
# Options
###

# Loop through $@ to find flags
while getopts ":d:" FLAG; do
    case "${FLAG}" in
        d) # Database name
            DB=${OPTARG} ;;
        h) # Print usage
            print_usage;;
        [:?]) print_usage;;
    esac
done

[ ! ${DB} ] && print_usage

###
# Functions
###

# The bgxupdate and bgxlimit functions below allow for
# running X jobs in parallel in bash.  They are taken from:
# http://stackoverflow.com/questions/1537956/bash-limit-the-number-of-concurrent-jobs/1685440#1685440

# bgxupdate - update active processes in a group.
#   Works by transferring each process to new group
#   if it is still active.
# in:  bgxgrp - current group of processes.
# out: bgxgrp - new group of processes.
# out: bgxcount - number of processes in new group.

bgxupdate() {
    bgxoldgrp=${bgxgrp}
    bgxgrp=""
    ((bgxcount = 0))
    bgxjobs=" $(jobs -pr | tr '\n' ' ')"
    for bgxpid in ${bgxoldgrp} ; do
        echo "${bgxjobs}" | grep " ${bgxpid} " >/dev/null 2>&1
        if [[ $? -eq 0 ]] ; then
            bgxgrp="${bgxgrp} ${bgxpid}"
            ((bgxcount = bgxcount + 1))
        fi
    done
}

# bgxlimit - start a sub-process with a limit.

#   Loops, calling bgxupdate until there is a free
#   slot to run another sub-process. Then runs it
#   an updates the process group.
# in:  $1     - the limit on processes.
# in:  $2+    - the command to run for new process.
# in:  bgxgrp - the current group of processes.
# out: bgxgrp - new group of processes

bgxlimit() {
    bgxmax=$1 ; shift
    bgxupdate
    while [[ ${bgxcount} -ge ${bgxmax} ]] ; do
        sleep 1
        bgxupdate
    done
    if [[ "$1" != "-" ]] ; then
        $* &
        bgxgrp="${bgxgrp} $!"
    fi
}

###
# Program Execution
###

bgxgrp="process_group_1"
for TABLE in `mysql ${DB} -e 'show tables'`
do
    bgxlimit ${JOBS_AT_ONCE} ${COMMAND} ${TABLE}
done

# Wait until all queued processes are done.

bgxupdate
while [[ ${bgxcount} -ne 0 ]] ; do
    oldcount=${bgxcount}
    while [[ ${oldcount} -eq ${bgxcount} ]] ; do
        sleep 1
        bgxupdate
    done
done
</code>