python fabric erroneous duplicate executions of task

Question

I'm starting out with fabric and running in to an issue where tasks are being executed multiple times. I only intend for the task to be run.

Here's my fabfile (both nodes and hubs are lists. nodes has multiple hosts. hubs has only one host) import requests import json from fabric.api import cd, env, execute, roles, run, settings, task, runs_once

buildconfig_location = 'http://10.102.0.69'

def get_environment_configuration(environment_name):
    request = requests.get( "{}/api/v1/environments/{}".format(buildconfig_location, environment_name) )
    environment_configuration = json.loads(request.text)

    return environment_configuration

def get_hub_servers(server_list):
    hub = [
        server['ip_address']
        for server in server_list
        if server['active'] and server['unit_tests']]

    return hub

def get_node_servers(server_list):
    nodes = [
        server['ip_address']
        for server in server_list
        if server['active'] and server['unit_tests'] is False]

    return nodes

def set_hosts(environment_configuration):
    environment_configuration['servers']

    env.roledefs = {
        'hub': [
            server['ip_address']
            for server in environment_configuration['servers']
            if server['active'] and server['unit_tests']],
        'node': [
            server['ip_address']
            for server in environment_configuration['servers']
            if server['active'] and server['unit_tests'] is False],
    }

def start_node():
    with settings(user="automation1"):
        with cd('/home/automation1/inventory.robot/grid'):
            output = run('nohup ./start_node.sh > node_out 2>&1 &')

def start_hub():
    with settings(user="automation1"):
        with cd('/home/automation1/inventory.robot/grid'):
            run('nohup ./start_hub.sh > hub_out 2>&1 &')

def robot_test():
    with settings(user="automation1"):
        with cd('/home/automation1/inventory.robot/grid'):
            run('python grid.py')

def kill_node():
    with settings(user="automation1"):
        with cd('/home/automation1/inventory.robot/grid'):
            run('./kill_node.pl')

@task
@runs_once
def robot_test():
    environment_configuration = get_environment_configuration('Selenium')
    #set_hosts(environment_configuration)
    nodes = get_node_servers(environment_configuration['servers'])
    hubs = get_hub_servers(environment_configuration['servers'])

    execute(start_hub, hosts=hubs)
    execute(start_node, hosts=nodes)
    execute(robot_test, hosts=hubs)

    all = hubs + nodes
    execute(kill_node, hosts=(hubs + nodes))

Here's the output:

[10.102.0.101] Executing task 'start_hub'
[10.102.0.101] run: nohup ./start_hub.sh > hub_out 2>&1 &
[10.102.0.102] Executing task 'start_node'
[10.102.0.102] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.103] Executing task 'start_node'
[10.102.0.103] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.104] Executing task 'start_node'
[10.102.0.104] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.105] Executing task 'start_node'
[10.102.0.105] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.106] Executing task 'start_node'
[10.102.0.106] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.107] Executing task 'start_node'
[10.102.0.107] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.108] Executing task 'start_node'
[10.102.0.108] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.109] Executing task 'start_node'
[10.102.0.109] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.110] Executing task 'start_node'
[10.102.0.110] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.101] Executing task 'robot_test'
[10.102.0.101] Executing task 'start_hub'
[10.102.0.101] run: nohup ./start_hub.sh > hub_out 2>&1 &
[10.102.0.102] Executing task 'start_node'
[10.102.0.102] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.103] Executing task 'start_node'
[10.102.0.103] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.104] Executing task 'start_node'
[10.102.0.104] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.105] Executing task 'start_node'
[10.102.0.105] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.106] Executing task 'start_node'
[10.102.0.106] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.107] Executing task 'start_node'
[10.102.0.107] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.108] Executing task 'start_node'
[10.102.0.108] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.109] Executing task 'start_node'
[10.102.0.109] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.110] Executing task 'start_node'
[10.102.0.110] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.101] Executing task 'robot_test'
[10.102.0.101] Executing task 'start_hub'
[10.102.0.101] run: nohup ./start_hub.sh > hub_out 2>&1 &
[10.102.0.102] Executing task 'start_node'
[10.102.0.102] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.103] Executing task 'start_node'
[10.102.0.103] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.104] Executing task 'start_node'
[10.102.0.104] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.105] Executing task 'start_node'
[10.102.0.105] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.106] Executing task 'start_node'
[10.102.0.106] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.107] Executing task 'start_node'
[10.102.0.107] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.108] Executing task 'start_node'
[10.102.0.108] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.109] Executing task 'start_node'
[10.102.0.109] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.110] Executing task 'start_node'
[10.102.0.110] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.101] Executing task 'robot_test'
[10.102.0.101] Executing task 'start_hub'
[10.102.0.101] run: nohup ./start_hub.sh > hub_out 2>&1 &
[10.102.0.102] Executing task 'start_node'
[10.102.0.102] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.103] Executing task 'start_node'
[10.102.0.103] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.104] Executing task 'start_node'
[10.102.0.104] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.105] Executing task 'start_node'
[10.102.0.105] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.106] Executing task 'start_node'
[10.102.0.106] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.107] Executing task 'start_node'
[10.102.0.107] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.108] Executing task 'start_node'
[10.102.0.108] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.109] Executing task 'start_node'
[10.102.0.109] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.110] Executing task 'start_node'
[10.102.0.110] run: nohup ./start_node.sh > node_out 2>&1 &
[10.102.0.101] Executing task 'robot_test'
^C
Stopped.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.
Disconnecting from [email protected]... done.

If I comment out 'execute(robot_test, hosts=hubs)' I get output closer to what I expect. I'm aware that this has something to do with execute and the warning on that page but I don't understand what the issue is here.

Answer 1

robot_test() is recursive, it's calling itself in an endless loop, that's why it's executing until you cancel.

Remove the call to robot_test() inside robot_test(). What are you trying to get with that call? Because I guess it has a purpose, it's not a typo.

@task
@runs_once
def robot_test():
  environment_configuration = get_environment_configuration('Selenium')
  #set_hosts(environment_configuration)
  nodes = get_node_servers(environment_configuration['servers'])
  hubs = get_hub_servers(environment_configuration['servers'])

  execute(start_hub, hosts=hubs)
  execute(start_node, hosts=nodes)

  all = hubs + nodes
  execute(kill_node, hosts=(hubs + nodes))

If you really want to execute it recursively, you need a parameter to know when the recursion should stop.

Answer 2

You should just add the runs_once decorator to each of the functions that should be run only once. e.g.:

@runs_once
def start_node():
    with settings(user="automation1"):
        with cd('/home/automation1/inventory.robot/grid'):
            output = run('nohup ./start_node.sh > node_out 2>&1 &')

@runs_once
def start_hub():
    with settings(user="automation1"):
        with cd('/home/automation1/inventory.robot/grid'):
            run('nohup ./start_hub.sh > hub_out 2>&1 &')

@runs_once
def robot_test():
    with settings(user="automation1"):
        with cd('/home/automation1/inventory.robot/grid'):
            run('python grid.py')

Without the runs_once decorator, each function will be run once per host instead of once per host set.

---

For once per host, you can do something like this:

from fabric.context_managers import env

env.hosts = hubs
execute(start_hub)
env.hosts = nodes
execute(start_node)
env.hosts = hubs
execute(robot_test)