Run ALL the things – everywhere!

Yes, that meme is a bit overused and trite. That’s okay, it’s still fun. At least, I think it is, and since I’m the author, my opinion is the one that counts.

So why am I using it? Well, I came across some information I needed to collect from all of our Linux systems the other day. We have an in-house routine called ‘rrun’ that will let us launch commands on a specified set of systems, as root, on demand. Simple solution, right? Well, not really – unfortunately, the thing I needed to run wouldn’t run properly inside of the ‘rrun’ tool. What’s a poor deprived sysadmin soul to do in this situation?

Hopefully not what I did. I basically reinvented the wheel – though I think I made it better.

I remembered using an expect-based script many years ago that would ssh out to various systems and run commands for you, and thinking it was a wonderful thing. Well, I didn’t have that script any longer, and since I didn’t really want to re-learn Tcl, I looked for alternatives. I found Python’s pexpect module, which is basically a reimplementation of expect in Python.

After a bit of thinking and a lot of coding, I came up with the code you see below. If you like it, feel free to use it, though do be warned that the version I’m posting has not been extensively tested or Fred-proofed. I’ve also got some work left on refining the debugging levels and such, but that’s for later.

And yes, I did have firearms on the brain when I was writing it.  🙂

#!/usr/bin/python
#
# Clone of 'rrun', an internal program that runs a command as root on
# multiple target systems.
#
# It requires that you have sudo available on the target(s) and that you
# can run the given command under sudo. It does not require SSH keys be set
# up, since it handles the password dialogs for both SSH login and sudo
# access.
#
# Any vulgarities in this code are the result of being lazy about
# case sensitivity checks, and are not deliberate. If you decide to be
# offended, you need to get over it.

import pexpect
from optparse import OptionParser
import os
import getpass
import signal
import sys
from datetime import datetime

DEBUG = 0
jams = []
misses = []
hits = []

def getTargets(hostspec):
  global DEBUG
  if os.path.isfile(hostspec):
    if DEBUG:
      print "Reading hosts from file "+hostspec
    fh = open(hostspec, 'r')
    hosts=fh.read()
  else:
    if DEBUG:
      print "Using hosts from command line."
    hosts=hostspec
  return hosts.split()

def loadAmmunition(cmdspec):
  global DEBUG
  if os.path.isfile(cmdspec):
    if DEBUG:
      print "Reading commands from file "+cmdspec
    fh = open(cmdspec, 'r')
    commands = fh.read()
    fh.close()
  else:
    if DEBUG:
      print "Using commands from command line."
    commands = cmdspec
  return commands

def readPassword():
  return getpass.getpass("Use what password? ")

def pullTrigger(target, cmds, passwd):
  global DEBUG, jams, misses, hits
  rangeHot = "\$ "
  # First, we launch the ssh process and get logged in to the target
  # Set a 5 minute timeout on commands, not 30 seconds
  proc = pexpect.spawn("ssh "+target, timeout=300)
  while True:
    index = proc.expect(["The authenticity of host", "assword:", "Permission denied", rangeHot, pexpect.EOF, pexpect.TIMEOUT])
    if index == 0:
      proc.sendline("yes")
    elif index == 1:
      proc.sendline(passwd)
    elif index == 2:
      jams.append(target)
      if DEBUG:
        print "Dud cartridge. Clearing chamber, proceeding with firing plan..."
      proc.kill(signal.SIGKILL)
      return
    elif index == 3:
      break
    elif index == 4:
      jams.append(target)
      if DEBUG:
        print "Cartridge jammed, clearing chamber, proceeding with firing plan."
      proc.kill(signal.SIGKILL)
      return
    elif index == 5:
      jams.append(target)
      if DEBUG:
        print "Squib load. clearing chamber, proceeding with firing plan."
      proc.kill(signal.SIGKILL)
      return

  # We're logged in. Create the shell file with the commands.
  proc.sendline("echo "+cmds+" > /tmp/expectcmd.sh")
  index = proc.expect([rangeHot, pexpect.EOF, pexpect.TIMEOUT])
  if index != 0:
    misses.append(target)
    if DEBUG:
      print "Stop firing into the ceiling! Proceeding with firing plan."
    proc.kill(aignal.SIGKILL)
    return

  # Go root (if indicated by sys.argv[0])
  if (sys.argv[0].endswith("rlaunch") ):
    if DEBUG:
      print "Becoming root inside expect spawn."
    rangeHot = becomeRoot(proc, passwd)
    if rangeHot == "EOF":
      misses.append(target)
      if DEBUG:
        print "Missed target low. Proceeding with firing plan."
      proc.kill(signal.SIGKILL)
      return
    if rangeHot == "TIMEOUT":
      misses.append(target)
      if DEBUG:
        print "Missed target high. Proceeding with firing plan."
      proc.kill(signal.SIGKILL)
      return

  # Execute the command, redirecting stdout/stderr
  proc.sendline("/bin/sh /tmp/expectcmd.sh > /tmp/expectcmd.out 2>/tmp/expectcmd.err")
  index = proc.expect([rangeHot, pexpect.EOF, pexpect.TIMEOUT])
  if index != 0:
    misses.append(target)
    if DEBUG:
      print "Missed wide left. Proceeding with firing plan."
    proc.kill(signal.SIGKILL)
    return

  # A hit! A veritable hit! O frabjious day!
  hits.append(target)
  if ( sys.argv[0].endswith("rlaunch") ):
    rangeHot = exitRoot(proc)
  proc.sendline("exit")

def exitRoot(proc):
  global DEBUG
  # Quick and dirty. This should really be nicer, but I'm lazy and it's
  # almost guaranteed to work if you actually got this far.
  if DEBUG:
    print "Leaving root shell."
  proc.sendline("exit")
  proc.expect("\$ ")
  return "\$ " 


def becomeRoot(proc, passwd):
  proc.sendline("sudo su -")
  while True:
    index = proc.expect(["assword", "\# ", pexpect.EOF, pexpect.TIMEOUT])
    if index == 0:
      proc.sendline(passwd)
    elif index == 1:
      return "\# "
    elif index == 2:
      return "EOF"
    elif index == 3:
      return "TIMEOUT"

def cleanBrass(target, passwd):
  global DEBUG
  rangeHot = "\$ "
  if DEBUG:
    print "Cleaning up spent brass for target "+target
  # First, we launch the ssh process and get logged in to the target
  # Set a 5 minute timeout on commands, not 30 seconds
  proc = pexpect.spawn("ssh "+target, timeout=300)
  # We don't handle certain types of things we do in pulling the trigger since
  # we already know we succeeded once so we will succeed again.
  while True:
    index = proc.expect(["The authenticity of host", "assword:", rangeHot])
    if index == 0:
      proc.sendline("yes")
    elif index == 1:
      proc.sendline(passwd)
    elif index == 2:
      break

  # Go root (if indicated by sys.argv[0])
  if (sys.argv[0].endswith("rlaunch") ):
    if DEBUG:
      print "Becoming root inside expect spawn."
    rangeHot = becomeRoot(proc, passwd)
    if rangeHot == "EOF":
      if DEBUG:
        print "Spent brass behind you, not on range."
      proc.kill(signal.SIGKILL)
      return
    if rangeHot == "TIMEOUT":
      if DEBUG:
        print "Can't find any spent brass.."
      proc.kill(signal.SIGKILL)
      return

  # Execute the command, redirecting stdout/stderr
  proc.sendline("/bin/rm -rf /tmp/expectcmd.sh /tmp/expectcmd.out /tmp/expectcmd.err")
  proc.expect(rangeHot)
  if (sys.argv[0].endswith("rlaunch") ):
    rangeHot = exitRoot(proc)
  proc.sendline("exit")


def collectTarget(target, passwd):
  global DEBUG
  if DEBUG:
    print "Collecting results from target "+target
  proc = pexpect.spawn("scp "+target+":/tmp/expectcmd.out "+target+".out")
  while True:
    index = proc.expect(["assword:", "\$ ", pexpect.EOF, pexpect.TIMEOUT])
    if index == 0:
      proc.sendline(passwd)
    elif index == 1:
      break
    elif index == 2:
      break
    elif index == 3:
      if DEBUG:
        print "Can't find target. Proceeding to next collection."
      break
  proc = pexpect.spawn("scp "+target+":/tmp/expectcmd.err "+target+".err")
  while True:
    index = proc.expect(["assword:", "\$ ", pexpect.EOF, pexpect.TIMEOUT])
    if index == 0:
      proc.sendline(passwd)
    elif index == 1:
      break
    elif index == 2:
      break
    elif index == 3:
      if DEBUG:
        print "Can't find target. Proceeding to next collection."
      break

def setupTargetFile(dirname):
  if os.path.isdir(dirname):
    d = datetime.now()
    os.rename(dirname, dirname+d.isoformat('@'))
  os.mkdir(dirname)

def main():
  global DEBUG, jams, misses, hits

  # Set up command line options / arguments
  parser = OptionParser()
  parser.disable_interspersed_args()
  parser.set_defaults(saveResults=True)
  parser.add_option("-c", "--commands", dest="cmdspec", help="one-line command or file with commands to run", metavar="CMDSPEC", default="pyexpcmds")
  parser.add_option("-H", "--hosts", dest="hostspec", help="hosts to run the command(s) on", metavar="HOSTSPEC", default="pyexphosts")
  parser.add_option("-r", "--results", dest="resdir", help="store results files in DIR", metavar="DIR", default="pyexpresults")
  parser.add_option("-R", "--no-results", dest="nolog", action="store_true")
  parser.add_option("-p", "--password", dest="passwd", help="optional password to use (if not specified, you will be prompted)", metavar="PASSWORD")
  parser.add_option ("-d", "--debug", action="store_true", dest="debug", help="print debugging messages")
  parser.add_option ("-n", "--no-clean", action="store_true", dest="nocleanup", help="Do not clean up the results files on the target systems")
 
  (options, args) = parser.parse_args()

  if options.debug:
    DEBUG=1
  
  targets = getTargets(options.hostspec)

  cmds = loadAmmunition(options.cmdspec)

  if not options.passwd:
    password = readPassword()
  else:
    password = options.passwd

  for target in targets:
    if DEBUG:
      print "Launching commands at target "+target
    pullTrigger(target, cmds, password)

  if options.nolog:
    if DEBUG:
      print "Discarding targets."
  else:
    if DEBUG:
      print "Collecting targets..."
    setupTargetFile(options.resdir)
    os.chdir(options.resdir)
    for target in hits:
      collectTarget(target, password)
    os.chdir('..')

  if not options.nocleanup:
    if DEBUG:
      print "Cleaning up spent brass from misses."
    for target in misses:
      cleanBrass(target, password)

    if DEBUG:
      print "Cleaning up spent brass from hits."
    for target in hits:
      cleanBrass(target, password)

  if DEBUG:
    if (len(jams)):
      print "Jams noticed:"
      for target in jams:
        print "Target "+target
    if (len(misses)):
      print "Misses noticed:"
      for target in misses:
        print "Target: "+target
    print "All ammuntion spent. Hope you had fun at the range!"

if __name__ == "__main__":
  main()

12 Responses to “Run ALL the things – everywhere!”