#!/usr/bin/env python
#------------------------------------------------------------------------------
# Copyright 2008-2011 Istituto Nazionale di Fisica Nucleare (INFN)
#
# Licensed under the EUPL, Version 1.1 only (the "Licence").
# You may not use this work except in compliance with the Licence.
# You may obtain a copy of the Licence at:
#
# http://www.osor.eu/eupl/european-union-public-licence-eupl-v.1.1
#
# Unless required by applicable law or agreed to in
# writing, software distributed under the Licence is
# distributed on an "AS IS" basis,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied.
# See the Licence for the specific language governing
# permissions and limitations under the Licence.
#------------------------------------------------------------------------------
"""
WNoDeS process running on the bait.
"""
import ConfigParser
import SocketServer
import commands
import fnmatch
import logging
import optparse
import os
import pickle
import socket
import sys
import threading
import time
#import MySQLdb
#import re
#import random
#import subprocess
#import zipfile
#import logging.handlers


__short_name__ = os.path.basename(os.path.splitext(sys.argv[0])[0])
__dir_name__ = os.path.dirname(sys.argv[0])

try:
    from wnodes.utils import utils
    from wnodes.utils.utils import synchronized
    try:
        from wnodes.utils import wsocket
    except ImportError:
        sys.exit("%s: python module 'wsocket' not found." % __short_name__)
except ImportError:
    sys.exit("%s: python module 'utils' not found." % __short_name__)


class WnodBait(wsocket.ClientRequestHandler):
    """ Define the behavior of the WNoDeS bait. """

    # lock used to rotate log-files
    LOCK_LOG = threading.Lock()

    # Lock used in order to access to the shared data structure batchBaitStatus
    LOCK_BBS = threading.Lock()

    # Lock used in order to serialize job operation
    LOCK_RESOURCES = threading.Lock()

    # Lock used in order to access to the shared data structure virtualMachines
    LOCK_VM = threading.Lock()

    # Lock used in order to access to the shared data structure vmImageAvailabe
    LOCK_IMAGE = threading.Lock()

    # Lock used in order to update status file wnodes_current_vm.log
    LOCK_CURRENT_STATUS = threading.Lock()

    # lock used to rotate log-files
    LOCK_LOGFILE = threading.Lock()

    def __init__(self, NS_HOST, NS_PORT):

        wsocket.ClientRequestHandler.__init__(self, NS_HOST, NS_PORT)

        self.NS_HOST = NS_HOST
        self.NS_PORT = NS_PORT

        # Gets configuration from WNoDeS Name Server
        # and puts inside self.config
        ret_code = self._retrieveConfig(self.NS_HOST, self.NS_PORT)
        if ret_code[0] == 1:
            sys.exit(ret_code[1])
        try:
            # import batch commands
            if self.config['BATCH_SYSTEM_TYPE'].lower() == 'lsf':
                try:
                    #from wnodes.utils.batch import lsfCmds
                    from wnodes.utils.batch_lsf import LsfCommands
                except ImportError:
                    sys.exit("%s: python module 'batch' not found."
                             % __short_name__)
                #self.batchCmd =  lsfCmds('/etc/profile.d/lsf.sh')
                self.batchCmd = LsfCommands(self.config['LSF_PROFILE'])

            elif self.config['BATCH_SYSTEM_TYPE'].lower() == 'pbs' or 'torque':
                try:
                    #from wnodes.utils.batch import pbsCmds
                    from wnodes.utils.batch_pbs import PbsCommands
                except:
                    sys.exit("%s: python module 'batch' not found."
                             % __short_name__)
                #self.batchCmd =  pbsCmds()
                self.batchCmd = PbsCommands()

            else:
                sys.exit('Batch system not supported')
        except KeyError:
            sys.exit('batchSystemType variable is not defined ' +
                     'in the configuration array')

        # Define host-name
        self.localhost = socket.gethostname().split('.')[0]
        self.localdomain = '.'.join(socket.gethostname().split('.')[1:])

        self.DEFAULT_LOGGING_INFO = logging.INFO

        # set up logging
        self.wnodes_logger = logging.getLogger('WnodLogger')
        self.wnodes_logger.setLevel(self.DEFAULT_LOGGING_INFO)
        formatter = logging.Formatter("%(asctime)s - " +
                                      "%(levelname)s - " +
                                      "%(message)s")
        handler = utils.CompressedRotatingFileHandler(
                      os.path.join(
                                   __dir_name__,
                                   "/var/log/wnodes/bait",
                                   self.config['LOG_FILE_NAME']),
                      maxBytes=self.config['MAX_LOG_FILE_SIZE'],
                      backupCount=self.config['MAX_COUNT_LOG_FILE'])
        handler.setFormatter(formatter)
        self.wnodes_logger.addHandler(handler)

        # verify config
        self._verifyMadatoryConfigParams()

        # Define HV host-name
        if self.config['ENABLE_MIXED_MODE'].upper() == 'NO':
            msg = {'WhoIsTheHV': [self.localhost]}
            OUTPUT = self.sendRequest(self.NS_HOST, self.NS_PORT, msg)

            if OUTPUT[0] == 0:
                OUTPUT = OUTPUT[1]
                if OUTPUT[0] == 0:
                    self.HV_HOST = OUTPUT[1]
                else:
                    sys.exit('I cannot retrieve HV HOSTNAME')
            else:
                sys.exit('I cannot retrieve HV HOSTNAME')
        else:
            self.HV_HOST = self.localhost

        # Define HV port
        self.HV_PORT = self.config['HV_PORT']

        # Define vmMandatoryParameters
        # MEM, CPU, BANDWIDTH
        # STORAGE parameter mandatory only with LVM enabled
        if self.config['USE_LVM'].upper() == 'YES':
            self.vmMandatoryParameters = ['MEM', 'CPU', 'BANDWIDTH', 'STORAGE']
        else:
            self.vmMandatoryParameters = ['MEM', 'CPU', 'BANDWIDTH']

        # Initialize local data structures
        self.batchJobs = {}
        self.virtualMachines = {}
        self.vmImagesAvailable = {}
        self.batchBaitStatus = ['START', '', 0, 0, 0, {}]
        self.localDNS = {}

        # Evaluate the wnodes_preexec return status
        try:
            self.PX_FAILED_RETURN_STATUS = int(self.config[
                                                  'PX_FAILED_RETURN_STATUS'])
        except KeyError:
            sys.exit('Cannot find PX_FAILED_RETURN_STATUS option, ' +
                     'in the bait configuration file. ' +
                     'Please add it in the name-server folder')
        except:
            sys.exit('Cannot change type to PX_FAILED_RETURN_STATUS variable' +
                     '(%s)' % str(self.config['PX_FAILED_RETURN_STATUS']) +
                     '. Please use a number')

    def _changeBaitStatus(self, *arguments):
        self.updateLog(' Function: %s . Variables: %s' % (utils.whoami(),
                                                          str(arguments)))
        self.batchBaitStatus[0] = arguments[0]
        self.batchBaitStatus[1] = arguments[1]
        self.batchBaitStatus[2] = time.time()

    def _checkResourcesAvailability(self, *arguments):
        # Add all mandatory parameters, if the list provided is not complete
        # Evaluate the availability of each parameter and cast it as an integer
        # Initialize self.batchJobs in order to reserve some resources (?)
        self.updateLog(' Function: %s . Variables: %s' % (utils.whoami(),
                                                          str(arguments)))
        vmParameters = arguments[0]
        batchJobId = vmParameters['BATCH_JOBID']
        EXCEED = False

        for param in self.vmMandatoryParameters:
            # Check if every mandatory parameter has been specified
            if not param in vmParameters:
                self.updateLog('The following mandatory parameter: %s ' % param
                               + 'has not been specified. Use default value',
                               'info', batchJobId)
                vmParameters[param] = int(self.config['DEFAULT_VM_%s' % param])
            # Force a cast to integer for each parameter value
            vmParameters[param] = int(vmParameters[param])
            # Check if every parameter does not exceed
            if vmParameters[param] > int(self.config['MAX_VM_%s' % param]):
                EXCEED = True

        self.updateLog('%s' % str(vmParameters), 'info')
        if not EXCEED:
            self.updateLog('VM requested parameters do not exceed MAX values.',
                           'info')
            self.updateLog('Check whether there are still available resources',
                           'info')

            RESOURCE_AVAILABLE = self.accessResourceAvailable('GET',
                                                              vmParameters)

            if RESOURCE_AVAILABLE[0] == 0:
                self.updateLog('There are still resource available. ' +
                               'I can RESERVE them to the VM %s' % batchJobId,
                               'info', batchJobId)

                self.batchJobs[batchJobId] = ['RESERVED', '', '', '',
                                              time.time(), '', '', 0, 0, 2, '',
                                              {'RESERVED': 1}, '']
                return [0, vmParameters]

            else:
                # One or more parameter exceeded the available resources
                return [1, 'There are not enough resources ' +
                        'to execute the virtual resources']

        else:
            # One or more parameter exceeded the max values
            self.updateLog('VM request parameters exceed one (or more) ' +
                           'of MAX value, See below the details ', 'warning')

            for k in self.vmMandatoryParameters:
                if vmParameters[k] > int(self.config['MAX_VM_%s' % k]):
                    self.updateLog('Parameters WNoDeS_VM_%s:%s > MAX_VM_%s:%s'
                                   % (k, str(vmParameters[k]),
                                      k, str(self.config['MAX_VM_%s' % k])),
                                   'error', batchJobId)

            return [1, 'requested vm parameters exceeded the max values']

    def _get_id_from_hostname(self, hostname):
        """
        returns the virtual machine id from its hostname
        """
        self.updateLog(' Function: %s .' % utils.whoami() +
                       ' Variables: %s' % hostname, 'debug')
        for key in self.virtualMachines:
            if self.virtualMachines[key][1].split('.')[0] \
                 == hostname.split('.')[0]:
                return key

    def _get_jobId_from_hostname(self, hostname):
        """
        returns the job_id related to the hostname
        of a running VM.
        """
        self.updateLog(' Function: %s .' % utils.whoami() +
                       ' Variables: %s' % hostname, 'debug')
        for key in self.virtualMachines:
            if self.virtualMachines[key][1].split('.')[0] \
                 == hostname.split('.')[0]:
                return self.virtualMachines[key][0]

    @synchronized(LOCK_VM)
    def _lookingForVirtualMachine(self, *arguments):
        """
         return:
         [self.vm_action_regenerate: string
         FIRST_VM_ID: string(?)
         REQUEST_NEW_ADDRESS: bool)
         RESERVED_VM: list]
        """
        self.updateLog(' Function: %s .' % utils.whoami() +
                       ' Variables: %s' % str(arguments))

        vmParameters = arguments[0]
        batchJobId = vmParameters['BATCH_JOBID']

        REGENERATE = 'NO'
        mandatoryParam = self.vmMandatoryParameters[:]
        if not vmParameters['TYPE'].upper() == 'BATCH_REAL':
            mandatoryParam.append('IMG')
            mandatoryParam.append('NETWORK_TYPE')

        vm = [ID for ID in self.virtualMachines.keys()
              if self.virtualMachines[ID][2][0] == 'OFF'
              and self.virtualMachines[ID][2][1] == 'AVAILABLE']

        self.updateLog('VM = %s' % str(vm))
        if len(vm) > 0:
            for ID in vm:
                for param in mandatoryParam:
                    if (self.virtualMachines[ID][3][param] ==
                        vmParameters[param]):

                        REGENERATE = 'NO'
                    else:
                        REGENERATE = 'YES'
                        break
                if REGENERATE == 'NO':
                    self.updateLog('Found a VirtaulMachine:%s '
                                   % self.virtualMachines[ID][0] +
                                   'with the same parameter. I Can RECYCLE it',
                                   'info', batchJobId)
                    self.virtualMachines[ID][2] = ['OFF', 'RESERVED']
                    self.batchJobs[batchJobId][5] = 'RECYCLE'
                    return ['RECYCLE', ID, False, []]

                else:
                    pass
            if REGENERATE == 'YES':
                self.updateLog('I did not found a VirtaulMachine:%s ' +
                               'with the same parameter. ' +
                               'I have to REGENERATE one or more available VM',
                               'info',
                               batchJobId)

                self.batchJobs[batchJobId][5] = 'REGENERATE'
                if not vmParameters['TYPE'].upper() == 'BATCH_REAL':
                    mandatoryParam.remove('IMG')
                    mandatoryParam.remove('NETWORK_TYPE')


                if len(vm) == 1:
                    VM_ID_TO_REGENERATE = vm[0]
                    FOUND_ONE_VM = True

                elif len(vm) > 1:
                    FOUND = True
                    for ID in vm:
                        VM_ID_TO_REGENERATE = ID
                        for param in mandatoryParam:
                            if (self.virtualMachines[ID][3][param] >=
                                vmParameters[param]):

                                if not FOUND:
                                    FOUND = True
                            else:
                                FOUND = False
                                break
                        if FOUND:
                            FOUND_ONE_VM = True
                            break
                        else:
                            FOUND_ONE_VM = False

                if FOUND_ONE_VM:
                    self.updateLog('Found a VirtualMachine:%s '
                                   % self.virtualMachines[
                                       VM_ID_TO_REGENERATE][1] +
                                   'which matches requested parameters.' +
                                   'I Can REGENERATE it',
                                   'info', batchJobId)
                    self.virtualMachines[VM_ID_TO_REGENERATE][2] = ['OFF',
                                                                    'RESERVED']
                    if not vmParameters['TYPE'].upper() == 'BATCH_REAL':
                        if (self.virtualMachines[VM_ID_TO_REGENERATE][3]
                                                        ['NETWORK_TYPE'] ==
                            vmParameters['NETWORK_TYPE']):
    
                            return ['REGENERATE', VM_ID_TO_REGENERATE, False, []]
                        else:
                            return ['REGENERATE', VM_ID_TO_REGENERATE, True, []]
                    else:
                        return ['REGENERATE', VM_ID_TO_REGENERATE, False, []]
                else:
                    self.updateLog('In order to satisfy this request I have ' +
                                   'to destroy more than one VirtualMachine',
                                   'info', batchJobId)
                    RESERVED_RESOURCES = {}
                    RESERVED_VM = []
                    REQUEST_NEW_ADDRESS = False

                    for param in mandatoryParam:
                        RESERVED_RESOURCES[param] = 0

                    for ID in vm:
                        for param in mandatoryParam:
                            RESERVED_RESOURCES[param] = \
                            (RESERVED_RESOURCES[param] + 
                             self.virtualMachines[ID][3][param])
                            
                        RESERVED_VM.append(ID)
                        self.updateLog('This VM %s ' % str(ID) +
                                       'will be destroy ' +
                                       'in order to free resources',
                                       'info', batchJobId)
                        for param in mandatoryParam:
                            if (RESERVED_RESOURCES[param] >=
                                    vmParameters[param]):
                                ENOUGH_RESOURCES = True
                            else:
                                ENOUGH_RESOURCES = False
                                break

                        if ENOUGH_RESOURCES:
                            break
                        else:
                            pass

                    for ID in RESERVED_VM:
                        if not vmParameters['TYPE'].upper() == 'BATCH_REAL':
                            if (self.virtualMachines[ID][3]['NETWORK_TYPE'] ==
                                    vmParameters['NETWORK_TYPE']):
    
                                FIRST_VM_ID = ID
                                RESERVED_VM.remove(ID)
                                REQUEST_NEW_ADDRESS = False
                                break
                            else:
                                REQUEST_NEW_ADDRESS = True
                        else:
                            REQUEST_NEW_ADDRESS = True

                    if REQUEST_NEW_ADDRESS:
                        FIRST_VM_ID = RESERVED_VM.pop(0)
                    if not vmParameters['TYPE'].upper() == 'BATCH_REAL':
                        self.updateLog('This VMs %s will be destroy '
                                       % str(RESERVED_VM), 'info', batchJobId)
                        self.updateLog('This VM %s will be regenerate'
                                       % str(FIRST_VM_ID), 'info', batchJobId)
                    self.virtualMachines[FIRST_VM_ID][2] = ['OFF', 'RESERVED']
                    for ID in RESERVED_VM:
                        self.virtualMachines[ID][2] = ['OFF', 'DESTROYING']

                    return ['REGENERATE', FIRST_VM_ID,
                            REQUEST_NEW_ADDRESS, RESERVED_VM]
            else:
                pass

        else:
            self.updateLog('There are no VirtualMachines available, ' +
                           ' so I create a new one', 'info', batchJobId)
            self.batchJobs[batchJobId][5] = 'NEW'
            return ['CREATE', '', 'YES', []]

    def _retrieveConfig(self, ns_host, ns_port):
        """
        Retrieve an updated BAIT configuration
        sending a request to the NameServer
        """
        msg = {'get_config_from_file': ['BAIT']}
        msg = str(pickle.dumps(msg).replace('\n', '1%2'))
        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((ns_host, int(ns_port)))
            sock.sendall(msg + '\n')
            socketfile = sock.makefile('r', 0)
            response = socketfile.readline().strip()
            sock.shutdown(2)
            sock.close()
            try:
                self.config = pickle.loads(response.replace('1%2', '\n'))[1]
                return [0, 'Configuration updated']
#                return [0, 'Configuration of host: % updated'
#                        % self.localhost]
            except pickle.UnpicklingError:
                msg = 'Data received cannot be loaded'

                return [1, msg]
            except Exception:
                msg = ('Data received from cannot be loaded for this reason:' +
                       ' %s, %s, %s' % ((sys.exc_info()[:])))

                return [1, msg]
        except socket.error:
            msg = ('Server HOST %s:%s is unreachable. '
                   % (ns_host, str(ns_port)) +
                   'I cannot retrieve BAIT configuration')
            return [1, msg]

    def _verifyMadatoryConfigParams(self):

        if type(self.config) == type({}):
            self.mP = []
            self.mandatoryParams = [
                              'BATCH_SYSTEM_TYPE',
                              'HV_PORT',
                              'BAIT_PORT',
                              'LOG_FILE_NAME',
                              'MAX_LOG_FILE_SIZE',
                              'MAX_COUNT_LOG_FILE',
                              'VM_UNREACH_TIMEOUT',
                              'STATUS_RETRY_COUNT',
                              'SCHEDULING_INTERVAL',
                              'RESERVATION_LENGTH',
                              'USE_LVM',
                              'TYPE',
                              'PX_FAILED_RETURN_STATUS',
                              'ENABLE_MIXED_MODE',
                              'MAX_VM_MEM',
                              'MAX_VM_BANDWIDTH',
                              'MAX_VM_CPU',
                              'MAX_VM_STORAGE',
                              'MIN_VM_MEM',
                              'MIN_VM_BANDWIDTH',
                              'MIN_VM_CPU',
                              'MIN_VM_STORAGE',
                              ]
            try:
                if self.config['BATCH_SYSTEM_TYPE'].upper() == 'lsf':
                    self.mandatoryParams.append('LSF_PROFILE')
                else:
                    pass
            except KeyError:
                pass
            
            [ self.mP.append(p) for p in self.mandatoryParams if not self.config.has_key(p)]

            if len(self.mP) > 0:
                sys.exit('found missing mandatory config parameters %s' % str(self.mP))
            else:
                self.updateLog('configuration is reliable')
        else:
            sys.exit('config is not formatted as expected %s' % str(self.config))



    @synchronized(LOCK_RESOURCES)
    def accessResourceAvailable(self, *arguments):

        ACTION = arguments[0]
        vmParameters = arguments[1]

        self.updateLog(' Function: %s . Variables: %s' % (utils.whoami(),
                                                          str(arguments)))

        # Initialized local data structures
        if self.config['USE_LVM'].upper() == 'YES':
            self.vmMandatoryParameters = ['MEM', 'CPU', 'BANDWIDTH', 'STORAGE']
        else:
            self.vmMandatoryParameters = ['MEM', 'CPU', 'BANDWIDTH']

        if ACTION == 'PUT':
            # Released the resources defined in vmParameters
            batchJobId = vmParameters['BATCH_JOBID']
            self.updateLog('Releasing resources')

            OPEN = 'YES'

            self.updateLog('Put1:Resources Available: %s'
                           % str(self.batchBaitStatus[5]),
                           'info', batchJobId)
            for i in self.vmMandatoryParameters:
                self.batchBaitStatus[5][i] = (self.batchBaitStatus[5][i] +
                                              vmParameters[i])
                self.updateLog('The following parameter: %s=%s '
                               % (i, vmParameters[i]) +
                               'is being released', 'info', batchJobId)
                if (self.batchBaitStatus[5][i] <
                        int(self.config['MIN_VM_%s' % i])):
                    OPEN = 'NO'

            if OPEN == 'YES':
                self.updateBaitBatchStatus(
                    ['OPEN', 'Resources are greater than the MIN values'])

            self.updateLog('Put2:Resources Available: %s'
                           % str(self.batchBaitStatus[5]),
                           'info', batchJobId)
            return [0, 'Resources released']

        elif ACTION == 'GET':
            # Remove the resources defined in vmParameters
            batchJobId = vmParameters['BATCH_JOBID']
            self.updateLog('Acquiring resources')

            RESOURCE_AVAILABLE = True

            self.updateLog('Get: Resources Available: %s'
                           % str(self.batchBaitStatus[5]),
                           'info', batchJobId)
            
            #Update available memory
            msg = {'getResourceAvailable': [None]}
            RESOURCES = self.sendRequest(self.HV_HOST, self.HV_PORT, msg)


            if RESOURCES[0] == 0:
                RESOURCES = RESOURCES[1]
                self.batchBaitStatus[5]['MEM'] = RESOURCES[1]['MEM']
                
            else:
                self.batchBaitStatus[5]['MEM'] = 0
                RESOURCE_AVAILABLE = False
                
            for i in self.vmMandatoryParameters:

                if self.batchBaitStatus[5][i] >= vmParameters[i]:
                    self.updateLog('The following resource parameter: %s=%s'
                                    % (i, vmParameters[i]) +
                                   ' is available', 'info', batchJobId)
                    if not RESOURCE_AVAILABLE == False:
                        RESOURCE_AVAILABLE = True
                else:
                    if i == 'MEM':
                        vm = [ID for ID in self.virtualMachines.keys()
                              if self.virtualMachines[ID][2][0] == 'OFF'
                              and self.virtualMachines[ID][2][1] == 'AVAILABLE']
                        if len(vm) > 0:
                            TOT_MEM_ALLOCATED_TO_VM = 0
                            for k in vm:
                                TOT_MEM_ALLOCATED_TO_VM = \
                                TOT_MEM_ALLOCATED_TO_VM + \
                                self.virtualMachines[k][3]['MEM']
                            
                            if self.batchBaitStatus[5]['MEM'] + TOT_MEM_ALLOCATED_TO_VM >= vmParameters['MEM']:
                                self.updateLog('The following resource parameter: %s=%s'
                                    % (i, vmParameters[i]) +
                                   ' is available', 'info', batchJobId)
                                RESOURCE_AVAILABLE = True
                            else:
                                self.updateLog('The following resource parameter: %s=%s'
                                       % (i, vmParameters[i]) +
                                       ' is not available', 'info', batchJobId)
                                RESOURCE_AVAILABLE = False
                        else:
                            RESOURCE_AVAILABLE = False
                            
                    else:
                        self.updateLog('The following resource parameter: %s=%s'
                                       % (i, vmParameters[i]) +
                                       ' is not available', 'info', batchJobId)
                        RESOURCE_AVAILABLE = False

            if RESOURCE_AVAILABLE:
                # DA rivedere meglio se aprire sempre

                CLOSE = False
                RESOURCES_UNDER_MIN_VALUE = []
                for i in self.vmMandatoryParameters:
                    self.batchBaitStatus[5][i] = \
                        int(self.batchBaitStatus[5][i]) - vmParameters[i]

                    if (self.batchBaitStatus[5][i] <
                            int(self.config['MIN_VM_%s' % i])):

                        RESOURCES_UNDER_MIN_VALUE.append(i)
                        CLOSE = True

                self.updateLog('Get2:Resources Available: %s'
                               % str(self.batchBaitStatus[5]),
                               'info', batchJobId)

                if CLOSE:
                    self.updateBaitBatchStatus(
                        ['CLOSED_FULL',
                         'Resource %s is lower than the MIN value'
                         % str(RESOURCES_UNDER_MIN_VALUE)])

                self.updateLog('Resources acquired')
                for i in self.vmMandatoryParameters:
                    self.updateLog('%s %s is available'
                                   % (i, self.batchBaitStatus[5][i]),
                                   'info', batchJobId)

                return [0, 'Resources acquired']

            else:

                self.updateLog('VM requested parameters exceeded one ' +
                               'of the available value. ' +
                               'See below the details ',
                               'warning', batchJobId)
                for i in self.vmMandatoryParameters:
                    self.updateLog('Parameters WNoDeS_VM_%s:%s '
                                   % (i, vmParameters[i]) +
                                   '-- %s_AVAILABLE_VALUE:%s'
                                   % (i, self.batchBaitStatus[5][i]))
                return [1, 'Not enough resources to satisfy the request']

        elif ACTION == 'SET':
            # Set the resources defined in vmParameters

            for i in self.vmMandatoryParameters:
                self.batchBaitStatus[5][i] = vmParameters[i]

    def checkSSH(self, host):
        serverSocket = socket.socket()
        serverSocket.settimeout(0.25)
        try:
            serverSocket.connect((host, 22))
            return 0
        except socket.error:
            return 1

    def destroyVMInstance(self, *arguments):
        """
        This method will destroy selected VM instances
        Input:
        - VM list ID
        Output:
        - [0,1]
        """
        try:
            self.updateLog('destroyVMInstance arguments %s'
                           % arguments[0], "debug")
            hostname = arguments[0][0]
            vm_id = self._get_id_from_hostname(hostname)
            job_id = self._get_jobId_from_hostname(hostname)
            self.updateLog('Destroying VM' +
                           'named: %s, ' % hostname +
                           'ID: %s, ' % vm_id +
                           'jobId %s' % job_id,
                           "debug")
            if self.batchCmd.bkill(' %s' % job_id):
                msg = ('Virtual Machine ' +
                       'named: %s, ' % hostname +
                       'ID: %s, ' % vm_id +
                       'jobId %s' % job_id, +
                       'successfully destroyed')
                return [0, msg]
            else:
                msg = ('Cannot find any VM ' +
                       'named: %s, ' % hostname +
                       'id: %s, ' % vm_id +
                       'jobId %s' % job_id, +
                       'which can be destroyed')
                return [1, msg]

        except:
            msg = ('Error in function destroyVMInstance: %s, %s, %s'
                   % sys.exc_info()[:])
            return [1, msg]

    def getStatus(self, *arguments):

        #Update available memory
        msg = {'getResourceAvailable': [None]}
        RESOURCES = self.sendRequest(self.HV_HOST, self.HV_PORT, msg)


        if RESOURCES[0] == 0:
            RESOURCES = RESOURCES[1]
            self.batchBaitStatus[5]['MEM'] = RESOURCES[1]['MEM']
            
        else:
            self.batchBaitStatus[5]['MEM'] = 0
        
        return [0, [self.batchBaitStatus,
                    self.batchJobs,
                    self.virtualMachines]]

    def getConfig(self, *arguments):
        """
        Return Current BAIT configuration
        """
        try:
            return [0, [self.config]]
        except:
            msg = "configuration doesn't exist"
            self.updateLog(msg, 'error')
            return [1, msg]

    def reloadConfig(self, *arguments):
        """
        Alert the bait in order to reload its configuration
        """
        # Get configuration from WNoDeS Name Server
        return self._retrieveConfig(self.NS_HOST, self.NS_PORT)

    def ping(self, *arguments):
        """
        Return bait version. Used to check bait process availability
        """
        self.updateLog(' Function: %s . Variables: %s'
                       % (utils.whoami(), str(arguments)))
        try:
            from wnodes import bait
            a = sys.version_info
            return [0, {'Python': '%s.%s.%s' % (a[0], a[1], a[2]),
                        'bait': bait.get_version()}]
        except:
            self.updateLog('Ping: failed to get the version', 'info')
            return [1, False]

    def reportPreExecutionScript(self, *arguments):

        self.updateLog(' Function: %s . Variables: %s'
                       % (utils.whoami(), str(arguments)))

        PRE_EXEC_REPORT = arguments[0]
        sourceAddress = arguments[1]
        batchJobId = PRE_EXEC_REPORT[0]
        PRE_EXEC_STATUS = PRE_EXEC_REPORT[1]
        PRE_EXEC_DETAILS = PRE_EXEC_REPORT[2]

        if (batchJobId in self.batchJobs and
            sourceAddress in self.batchJobs[batchJobId][2]):

            self.updateLog('Report from %s for job %s can be accepted'
                           % (sourceAddress, batchJobId))
            self.batchJobs[batchJobId][9] = PRE_EXEC_STATUS
            self.batchJobs[batchJobId][10] = PRE_EXEC_DETAILS

            if PRE_EXEC_STATUS == 0:
                return [0, '']
            else:
                self.updateStatusAndTimeStamp(batchJobId,
                                              'READYtoRUN',
                                              self.batchJobs[batchJobId][4])
                return [self.PX_FAILED_RETURN_STATUS, '']
        else:
            self.updateLog('Report from %s for job %s cannot be accepted'
                           % (sourceAddress, batchJobId), 'warning')
            if self.config['BATCH_SYSTEM_TYPE'].upper() == 'LSF':
                return [self.PX_FAILED_RETURN_STATUS, '']
            elif self.config['BATCH_SYSTEM_TYPE'].upper() == 'PBS':
                self.updateLog('BATCH_SYSTEM_TYPE is %s'
                               % (self.config['BATCH_SYSTEM_TYPE']))
                return [self.PX_FAILED_RETURN_STATUS, '']

    def reportPostExecutionScript(self, *arguments):

        self.updateLog(' Function: %s . Variables: %s'
                       % (utils.whoami(), str(arguments)))

        batchJobId = arguments[0][0]

        if batchJobId in self.batchJobs:
            self.updateStatusAndTimeStamp(batchJobId,
                                          'FINISHED',
                                          self.batchJobs[batchJobId][4])

        return [0, 'Job finished']

    def requestVMInstance(self, *arguments):

        """
        This method handles all the requests to create a VM.
        Only the pre exec on the BAIT send this request.
        Input:
        - VM parameters is a dictionary with a parameter:value pairs
        Output:
        - [0,1]
        """
        self.updateLog(' Function: %s . Variables: %s'
                       % (utils.whoami(), str(arguments)))

        parameters = arguments[0]
        sourceAddr = arguments[1]
        batchJobId = parameters[0]
        vmParameters = parameters[1]

        # Check whether the job is already handled
        if batchJobId in self.batchJobs:
            return [self.PX_FAILED_RETURN_STATUS, '']

        # Check if the type of the request is supported
        SUPPORTED_JOB_TYPE = []
        try:
            for TYPE in self.config['TYPE'].split(';'):
                SUPPORTED_JOB_TYPE.append(TYPE.upper())
            if not vmParameters['TYPE'].upper() in SUPPORTED_JOB_TYPE:
                self.updateLog('Job type is not supported by this BAIT %s - %s'
                               % (str(vmParameters['TYPE'].upper()),
                                  str(SUPPORTED_JOB_TYPE)))
                return [self.PX_FAILED_RETURN_STATUS, '']

        except:
            return [self.PX_FAILED_RETURN_STATUS, '']

        # check if the job is still active in the cluster
        bjobsOutput = self.batchCmd.bjobs(jobid=batchJobId)
        if not bjobsOutput[0] == 0:

            self.updateLog('%s: %s: batch job %s '
                           % (batchJobId, utils.whoami(), batchJobId) +
                           'does not belong to batch cluster. ' +
                           'Request cannot be accepted ',
                           "error", batchJobId)

            return [self.PX_FAILED_RETURN_STATUS,
                    'Batch job does not belong to batch cluster']

        else:

            #check source address
            if ((not (sourceAddr in self.localhost))
                and (not (self.localhost in sourceAddr))):

                self.updateLog('%s: %s: request rejected '
                               % (batchJobId, utils.whoami()) +
                               'because received from a host: %s '
                               % sourceAddr +
                               'different from the BAIT',
                               "error", batchJobId)
                return [self.PX_FAILED_RETURN_STATUS,
                        'Request received from a non-enable host']

            else:

                # support for mixed mode. Run a batch job on the HV
                if vmParameters['TYPE'].upper() == 'BATCH_REAL' and \
                    self.config['ENABLE_MIXED_MODE'].upper() == 'NO':
                    self.updateLog('Mixed mode is not enabled', 'info',
                                   batchJobId)
                    return [self.PX_FAILED_RETURN_STATUS,
                            'Mixed mode is not enabled']

                elif vmParameters['TYPE'].upper() == 'BATCH_REAL' and \
                    self.config['ENABLE_MIXED_MODE'].upper() == 'YES':

                    self.updateLog('New request to run a batch job on the HV',
                                   'info', batchJobId)
                    
                    #check resources
                    ACK = self._checkResourcesAvailability(vmParameters)

                    if ACK[0] == 0:
                        self.updateLog('There are resources available ' +
                                       'to execute the job.',
                                       'info', batchJobId)
                        vmParameters = ACK[1]
                    else:
                        self.updateLog('There are not resources available ' +
                                       'to execute the job.',
                                       'info', batchJobId)
                        return [self.PX_FAILED_RETURN_STATUS, ACK[1]]

                    # try to remove virtual resources
                    requestToHyperVisor = \
                        self._lookingForVirtualMachine(vmParameters)

                    
                    ACTION = requestToHyperVisor[0]
                    VM_ID = requestToHyperVisor[1]
                    REQUEST_NEW_ADDRESS = requestToHyperVisor[2]
                    VM_TO_BE_DESTROYED = requestToHyperVisor[3]


                    if ACTION == 'REGENERATE' or ACTION == 'RECYCLE':
                        VM_TO_BE_DESTROYED.append(VM_ID)
                        MSG = {'do_destroy': [VM_TO_BE_DESTROYED, True]}

                        output = self.sendRequest(self.HV_HOST, self.HV_PORT, MSG)
                    

                    # UPDATE self.batchJobs
                    self.batchJobs[batchJobId][1] = 'NoId'
                    self.batchJobs[batchJobId][2] = self.localhost
                    self.batchJobs[batchJobId][3] = vmParameters

                    jCK = GuardianAngel(self, batchJobId)
                    jCK.start()

                    if self.HV_HOST == self.localhost:

                        # the bait is a service running on the HV

                        self.updateStatusAndTimeStamp(
                            batchJobId, 'RUN', self.batchJobs[batchJobId][4])

                        return [0, 'VM successfully created']
                    else:

                        # The bait is running on a dedicated VM.
                        # So I need to migrate the job execution on the HV

                        vmParameters['RESERVATION_ID'] = \
                            ('%s-%s' % (vmParameters['USER_DETAILS'][0],
                                        batchJobId))

                        if self.config['BATCH_SYSTEM_TYPE'].upper() == 'LSF':

                            RES_START = time.strftime(
                                            "%m:%d:%H:%M",
                                            time.localtime(time.time() + 70))
                            RES_END = time.strftime(
                                "%m:%d:%H:%M",
                                time.localtime(
                                    time.time() +
                                    int(self.config['RESERVATION_LENGTH'])))
#                            breserveOption = (
#                                '-m "%s" '
#                                % vmParameters['HOSTNAME']
#                                + '-n 1 -o -b %s'
#                                % RES_START
#                                + ' -e %s'
#                                % RES_END
#                                + ' -N %s'
#                                % vmParameters['RESERVATION_ID']
#                                + ' -u %s'
#                                % vmParameters['USER_DETAILS'][0])
                            breserveOption = ('-n 1 -o -b %s -e %s'
                                              % (RES_START, RES_END))

                            brsvaddoutput = self.batchCmd.breserve('add',
                                vmParameters['RESERVATION_ID'],
                                vmParameters['HOSTNAME'],
                                breserveOption,
                                vmParameters['USER_DETAILS'][0])

                            self.updateLog('%s' % str(brsvaddoutput),
                                           'info', batchJobId)

                        elif self.config['BATCH_SYSTEM_TYPE'].upper() == 'PBS':
                            brsvaddoutput = [0, '']

                        if brsvaddoutput[0] == 0:
                            self.batchJobs[batchJobId][12] = \
                                vmParameters['RESERVATION_ID']
                            self.updateStatusAndTimeStamp(
                                batchJobId,
                                'READYtoMODIFY',
                                self.batchJobs[batchJobId][4])
                            self.batchJobs[batchJobId][3] = vmParameters
                            return [self.PX_FAILED_RETURN_STATUS,
                                    'VM successfully created']

                        else:
                            self.updateStatusAndTimeStamp(
                                batchJobId,
                                'FINISHED',
                                self.batchJobs[batchJobId][4])

                            return [self.PX_FAILED_RETURN_STATUS,
                                    'Failed to create a reservation']

                    self.updateLog('New request to instantiate a VM',
                                   'info',
                                   batchJobId)

                #Create a Virtual Machine
                else:

                    # set,  if not, which vm image must be used
                    if not 'IMG' in vmParameters:
                        msg = {'selectVmType': [vmParameters['USER_DETAILS'],
                                                self.localhost]}
                        output = self.sendRequest(self.NS_HOST,
                                                  self.NS_PORT,
                                                  msg)
                        if output[0] == 0:
                            output = output[1]
                            if output[0] == 0:
                                for key in output[1].keys():
                                    vmParameters[key] = output[1][key]
                            else:
                                self.updateLog('Method error: %s' % output[1],
                                               'error', batchJobId)
                                return  [self.PX_FAILED_RETURN_STATUS,
                                         'Method error: %s' % output[1]]
                        else:
                            self.updateLog('Communication error: %s'
                                           % output[1], 'error', batchJobId)
                            return  [self.PX_FAILED_RETURN_STATUS,
                                     'Communication error: %s' % output[1]]

                    #check if the TAG has a vm image file associated
                    msg = {'get_image_info': [vmParameters['IMG']]}
                    BAIT_VM_FULL_PATH = self.sendRequest(self.NS_HOST,
                                                         self.NS_PORT,
                                                         msg)

                    if BAIT_VM_FULL_PATH[0] == 0:
                        BAIT_VM_FULL_PATH = BAIT_VM_FULL_PATH[1]
                        if BAIT_VM_FULL_PATH[0] == 0:
                            self.updateLog('%s: ' % batchJobId +
                                           'Successfully got VM_TAG file name'
                                           + ' %s' % str(BAIT_VM_FULL_PATH),
                                           'info', batchJobId)
                        else:
                            self.updateLog('%s' % BAIT_VM_FULL_PATH[1],
                                           'error', batchJobId)
                            return  [self.PX_FAILED_RETURN_STATUS,
                                     'Method error: %s' % BAIT_VM_FULL_PATH[1]]
                    else:
                        self.updateLog('Communication error: %s'
                                       % BAIT_VM_FULL_PATH[1],
                                       'error', batchJobId)
                        return  [self.PX_FAILED_RETURN_STATUS,
                                 'Communication error: %s'
                                 % BAIT_VM_FULL_PATH[1]]

                    #check resources
                    ACK = self._checkResourcesAvailability(vmParameters)

                    if ACK[0] == 0:
                        self.updateLog('%s: There are resources available '
                                       % batchJobId +
                                       'to execute the virtual resource. ' +
                                       'Try to create it',
                                       'info', batchJobId)
                        vmParameters = ACK[1]
                    else:
                        self.updateLog('%s: There are not resources available '
                                       % batchJobId +
                                       'to execute the virtual resource.',
                                       'info', batchJobId)
                        return [self.PX_FAILED_RETURN_STATUS, ACK[1]]

                    # try to create the virtual resource
                    requestToHyperVisor = \
                        self._lookingForVirtualMachine(vmParameters)

                    ACTION = requestToHyperVisor[0]
                    VM_ID = requestToHyperVisor[1]
                    REQUEST_NEW_ADDRESS = requestToHyperVisor[2]
                    VM_TO_BE_DESTROYED = requestToHyperVisor[3]

                    if REQUEST_NEW_ADDRESS:

                        # Check hostname availability

                        if 'ENABLED_VLAN' in self.config:
                            self.updateLog('This HyperVisor support only ' +
                                           'the following VLANs: %s'
                                           % str(self.config['ENABLED_VLAN']),
                                           'info', batchJobId)
                        # Find the ENABLED_VLAN for this bait
                        PIECE_MATCHED = False
                        HOST_GROUPS = []
                        for KEY in self.config.keys():
                            if KEY.startswith('HOST_GROUP'):
                                HOST_GROUPS.append(KEY)

                        for HOST_GROUP in HOST_GROUPS:
                            for HOST_PARSING_RULE in (self.config
                                                      [HOST_GROUP].split(';')):
                                HOSTNAME = self.localhost
                                if fnmatch.fnmatch(HOSTNAME,
                                                   HOST_PARSING_RULE):
                                    PIECE_MATCHED = True
                                    break
                            if PIECE_MATCHED:
                                HOST_GROUP = HOST_GROUP.split('_')
                                ENABLED_VLAN = ('ENABLED_VLAN_%s_%s'
                                                % (HOST_GROUP[1],
                                                   HOST_GROUP[2]))
                                break
                        if PIECE_MATCHED == False:
                            return [self.PX_FAILED_RETURN_STATUS,
                                    'I could not found a rule for my hostname '
                                    + '%s' % self.localhost]
                        elif not ENABLED_VLAN in self.config:
                            return [self.PX_FAILED_RETURN_STATUS,
                                    'There is no an ENABLED_VLAN ' +
                                    'for host group %s' % '_'.join(HOST_GROUP)]

                        msg = {'acquire_Node':
                               ['VM_HOST',
                               self.HV_HOST,
                               vmParameters['NETWORK_TYPE'].upper(),
                               self.config[ENABLED_VLAN]]}
                        output = self.sendRequest(self.NS_HOST,
                                                  self.NS_PORT,
                                                  msg)

                        self.updateLog('Allocated host: %s'
                                       % str(output), 'info', batchJobId)

                        if output[0] == 0:
                            vmNetParameters = output[1]

                            if vmNetParameters[0] == 1:
                                self.accessResourceAvailable('PUT',
                                                             vmParameters)
                                self.batchJobs.pop(batchJobId)
                                if ACTION == 'REGENERATE':
                                    self.virtualMachines[VM_ID][2] = \
                                        ['OFF', 'AVAILABLE']

                                    for ID in VM_TO_BE_DESTROYED:
                                        self.virtualMachines[ID][2] = \
                                            ['OFF', 'AVAILABLE']

                                return [self.PX_FAILED_RETURN_STATUS,
                                        vmNetParameters[1]]

                            else:
                                vmNetParameters = vmNetParameters[1]
                                vmParameters['VLAN'] = vmNetParameters[1]
                                vmParameters['MAC'] = vmNetParameters[2]
                                vmParameters['HOSTNAME'] = vmNetParameters[0]
                                vmParameters['VM_TYPE'] = 'VM_HOST'
                        else:
                            self.accessResourceAvailable('PUT', vmParameters)
                            self.batchJobs.pop(batchJobId)
                            if ACTION == 'REGENERATE':
                                self.virtualMachines[VM_ID][2] = \
                                    ['OFF', 'AVAILABLE']

                                for ID in VM_TO_BE_DESTROYED:
                                    self.virtualMachines[ID][2] = \
                                        ['OFF', 'AVAILABLE']

                            return [self.PX_FAILED_RETURN_STATUS, output[1]]

                        if ACTION == 'CREATE':
                            MSG = {'do_%s' % ACTION.lower(): [vmParameters]}
                        else:
                            for PARAM in vmParameters.keys():
                                self.virtualMachines[VM_ID][3][PARAM] = \
                                    vmParameters[PARAM]

                            MSG = {'do_%s' % ACTION.lower():
                                   [VM_ID,
                                    self.virtualMachines[VM_ID][3],
                                    VM_TO_BE_DESTROYED,
                                    REQUEST_NEW_ADDRESS]}
                    else:
                        for PARAM in vmParameters.keys():
                            self.virtualMachines[VM_ID][3][PARAM] = \
                                vmParameters[PARAM]

                        if ACTION == 'REGENERATE':
                            MSG = {'do_%s' % ACTION.lower():
                                   [VM_ID,
                                    self.virtualMachines[VM_ID][3],
                                    VM_TO_BE_DESTROYED,
                                    REQUEST_NEW_ADDRESS]}
                        else:
                            MSG = {'do_%s' % ACTION.lower():
                                   [VM_ID,
                                    self.virtualMachines[VM_ID][3],
                                    VM_TO_BE_DESTROYED]}
                    # Send the request to the HV to instantiates the VM
                    output = self.sendRequest(self.HV_HOST, self.HV_PORT, MSG)

                    if output[0] == 0:
                        output = output[1]
                        if output[0] == 0:

                            self.updateLog(
                                'Virtual resource successfully created. %s'
                                % str(output[1]),
                                'info',
                                batchJobId)
                            vmId = output[1][0]
                            self.updateLog('5 %s'
                                           % str(self.virtualMachines.keys()))
                            self.virtualMachines[vmId] = output[1][1]
                            self.updateLog('5 %s'
                                           % str(self.virtualMachines.keys()))
                            if ACTION == 'REGENERATE':
                                for ID in output[2]:
                                    self.updateLog(
                                        '4 %s'
                                        % str(self.virtualMachines.keys()))
                                    self.virtualMachines.pop(ID)
                                    self.updateLog(
                                        '4 %s'
                                        % str(self.virtualMachines.keys()))

                            vmParameters = self.virtualMachines[vmId][3]

                            # UPDATE self.batchJobs
                            self.batchJobs[batchJobId][1] = vmParameters['ID']
                            self.batchJobs[batchJobId][2] = \
                                vmParameters['HOSTNAME']

                            jCK = GuardianAngel(self, batchJobId)
                            jCK.start()

                            if vmParameters['TYPE'] == 'BATCH':
#                                vmParameters['RESERVATION_ID'] = \
#                                    ('%s'
#                                     % (str(random.randint(1, 10000000))))
                                vmParameters['RESERVATION_ID'] = \
                                    ('%s-%s'
                                     % (vmParameters['USER_DETAILS'][0],
                                        batchJobId))
                                if (self.config['BATCH_SYSTEM_TYPE'].upper() ==
                                    'LSF'):

#                                    vmParameters['RESERVATION_ID'] = \
#                                        ('%s-%s'
#                                         % (vmParameters['USER_DETAILS'][0],
#                                            batchJobId))

                                    RES_START = time.strftime(
                                                    "%m:%d:%H:%M",
                                                    time.localtime(time.time()
                                                                   + 70))
                                    RES_END = time.strftime(
                                        "%m:%d:%H:%M",
                                        time.localtime(time.time() + int(
                                        self.config['RESERVATION_LENGTH'])))

#                                    breserveOption = (
#                                        '-m "%s" -n 1 -o -b %s'
#                                        % (vmParameters['HOSTNAME'],
#                                           RES_START) +
#                                        ' -e %s -N %s -u %s'
#                                        % (RES_END,
#                                           vmParameters['RESERVATION_ID'],
#                                           vmParameters['USER_DETAILS'][0]))
                                    breserveOption = ('-n 1 -o -b %s -e %s'
                                                      % (RES_START, RES_END))
#                                    brsvaddoutput = commands.getstatusoutput(
#                                        'source /etc/profile.d/lsf.sh; '
#                                        + 'brsvadd -m "%s" '
#                                        % vmParameters['HOSTNAME'] +
#                                        '-n 1 -o -b %s -e %s -N %s -u %s'
#                                        % (RES_START, RES_END,
#                                           vmParameters['RESERVATION_ID'],
#                                           vmParameters['USER_DETAILS'][0]))
                                    brsvaddoutput = self.batchCmd.breserve(
                                        'add',
                                        vmParameters['RESERVATION_ID'],
                                        vmParameters['HOSTNAME'],
                                        breserveOption,
                                        vmParameters['USER_DETAILS'][0])
                                    self.updateLog('%s'
                                                   % str(brsvaddoutput),
                                                   'info',
                                                   batchJobId)

                                elif (self.config['BATCH_SYSTEM_TYPE'].upper()
                                     == 'PBS'):

                                    brsvaddoutput = [0, '']

                                if brsvaddoutput[0] == 0:
                                    self.batchJobs[batchJobId][12] = \
                                        vmParameters['RESERVATION_ID']
                                    self.updateStatusAndTimeStamp(
                                        batchJobId,
                                        'READYtoMODIFY',
                                        self.batchJobs[batchJobId][4])
                                    self.virtualMachines[\
                                        vmParameters['ID']][3] = vmParameters

                                    return [self.PX_FAILED_RETURN_STATUS,
                                            'VM successfully created']

                                else:
                                        self.updateStatusAndTimeStamp(
                                            batchJobId,
                                            'FINISHED',
                                            self.batchJobs[batchJobId][4])
                                        return [
                                            self.PX_FAILED_RETURN_STATUS,
                                            'Failed creating a reservation']

                            elif vmParameters['TYPE'] == 'CLOUD':
                                self.updateStatusAndTimeStamp(
                                    batchJobId,
                                    'READYforSETUP',
                                    self.batchJobs[batchJobId][4])
                                self.virtualMachines[vmParameters['ID']][3] = \
                                    vmParameters

                                return [0, 'VM successfully created']

                        else:
                            self.updateLog('Method error: %s'
                                           % output[1],
                                           'error', batchJobId)
                            if REQUEST_NEW_ADDRESS:
                                    msg = {'release_Node':
                                           [vmParameters['HOSTNAME'],
                                            self.HV_HOST,
                                            vmParameters['VM_TYPE']]}
                                    output = self.sendRequest(self.NS_HOST,
                                                              self.NS_PORT,
                                                              msg)
                                    self.updateLog('Release output %s'
                                                   % str(output))
                            self.accessResourceAvailable('PUT', vmParameters)
                            self.batchJobs.pop(batchJobId)
                            return  [self.PX_FAILED_RETURN_STATUS,
                                     'Method error: %s' % output[1]]
                    else:
                        self.updateLog('Communication error: %s'
                                       % output[1],
                                       'error', batchJobId)
                        if REQUEST_NEW_ADDRESS:
                                msg = {'release_Node':
                                       [vmParameters['HOSTNAME'],
                                        self.HV_HOST,
                                        vmParameters['VM_TYPE']]}
                                output = self.sendRequest(self.NS_HOST,
                                                          self.NS_PORT, msg)
                                self.updateLog('Release output %s'
                                               % str(output))
                        self.accessResourceAvailable('PUT', vmParameters)
                        self.accessResourceAvailable('PUT', vmParameters)
                        self.batchJobs.pop(batchJobId)
                        return  [self.PX_FAILED_RETURN_STATUS,
                                 'Communication error: %s' % output[1]]

    def setConfig(self, ns_host, ns_port):
        """
        Return Current BAIT configuration
        """

        config = {}

        msg = {'get_config_from_file': ['BAIT']}
        msg = str(pickle.dumps(msg).replace('\n', '1%2'))
        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect((ns_host, int(ns_port)))
            sock.sendall(msg + '\n')
            socketfile = sock.makefile('r', 0)
            response = socketfile.readline().strip()
            sock.shutdown(2)
            sock.close()
            try:

                config = pickle.loads(response.replace('1%2', '\n'))

            except pickle.UnpicklingError:
                sys.exit('Data received cannot be loaded')
            except Exception:
                sys.exit('Data received from cannot be loaded for this reason:'
                         + ' %s, %s, %s' % (sys.exc_info()[:]))
        except socket.error:
            sys.exit('Server HOST %s:%s is unreachable. '
                     % (ns_host, ns_port) +
                     'Cannot retrieve BAIT configuration')
        return config

    @synchronized(LOCK_LOG)
    def updateLog(self, msg, level=None, jobid=None):
        """
        Emit a string to the WNoDeS log file.
        Thread safe.
        """
        if level is None:
            level = self.DEFAULT_LOGGING_INFO
        else:
            level = utils.LOG_LEVELS.get(level, logging.NOTSET)

        msg = '%s - %s' % (jobid, msg)
        self.wnodes_logger.log(level, msg)

    def updateStatusAndTimeStamp(self, *arguments):
        """ Update time stamp in batchJobs dictionary """

        self.updateLog(' Function: %s . Variables: %s'
                       % (utils.whoami(), str(arguments)))

        batchJobId = arguments[0]
        newStatus = arguments[1]
        lastTS = arguments[2]

        self.batchJobs[batchJobId][0] = newStatus
        self.batchJobs[batchJobId][4] = time.time()
        timeSpentLastStatus = self.batchJobs[batchJobId][4] - lastTS
        self.batchJobs[batchJobId][7] = (self.batchJobs[batchJobId][7] +
                                         timeSpentLastStatus)
        if newStatus in self.batchJobs[batchJobId][11]:
            self.batchJobs[batchJobId][11][newStatus] = \
                self.batchJobs[batchJobId][11][newStatus] + 1
        else:
            self.batchJobs[batchJobId][11][newStatus] = 1

    @synchronized(LOCK_BBS)
    def updateBaitBatchStatus(self, *arguments):
        self.updateLog(' Function: %s . Variables: %s'
                       % (utils.whoami(), str(arguments)))
        NEW_STATUS = arguments[0][0]
        COMMENT = arguments[0][1]
        # NEW_STATUS can be: CLOSED_FULL; CLOSED_ADMIN, START, OPEN, OPEN_ADMIN

        # ACTION can be: OPEN, CLOSE

        PREVIOUS_STATUS = self.batchBaitStatus[0]

        if NEW_STATUS == 'CLOSED_FULL':
            if PREVIOUS_STATUS == 'OPEN_ADMIN' or PREVIOUS_STATUS == 'OPEN':
                self._changeBaitStatus(NEW_STATUS, COMMENT)
#                badmin = self.batchCmd.badmin('hclose',
#                                              '%s.%s'
#                                              % (self.localhost,
#                                                 self.localdomain))
                valid_host = self.localhost
                if self.localdomain not in valid_host:
                    valid_host += '.' + self.localdomain
                badmin = self.batchCmd.badmin('hclose', '%s' % valid_host)
#                commands.getstatusoutput(\
#                    'source /etc/profile.d/lsf.sh; badmin.org hclose %s'
#                    % self.localhost)
                return [0, 'Status changed in %s' % NEW_STATUS]

            else:
                self.updateLog('The current status: %s '
                               % PREVIOUS_STATUS +
                               'cannot be changed to the new status: %s'
                               % NEW_STATUS)
                return [0, 'host already in this status']

        elif NEW_STATUS == 'CLOSED_ADMIN':
            if (PREVIOUS_STATUS == 'OPEN_ADMIN'
                or PREVIOUS_STATUS == 'OPEN'
                or PREVIOUS_STATUS == 'START'):

                self._changeBaitStatus(NEW_STATUS, COMMENT)
#                badmin = self.batchCmd.badmin('hclose',
#                                              '%s.%s'
#                                              % (self.localhost,
#                                                 self.localdomain))
                valid_host = self.localhost
                if self.localdomain not in valid_host:
                    valid_host += '.' + self.localdomain
                badmin = self.batchCmd.badmin('hclose',
                                              '%s'
                                              % valid_host)
#                commands.getstatusoutput(\
#                    'source /etc/profile.d/lsf.sh; badmin.org hclose %s'
#                    % self.localhost)
                return [0, 'Status changed in %s' % NEW_STATUS]

            elif PREVIOUS_STATUS == 'CLOSED_FULL':
                self._changeBaitStatus(NEW_STATUS, COMMENT)
                return [0, 'Status changed in %s' % NEW_STATUS]
            else:
                self.updateLog('The current status: %s '
                               % PREVIOUS_STATUS +
                               'cannot be changed to the new status: %s'
                               % NEW_STATUS)
                return [0, 'host already in this status']

        elif NEW_STATUS == 'OPEN':
            if PREVIOUS_STATUS == 'START' or  PREVIOUS_STATUS == 'CLOSED_FULL':
                self._changeBaitStatus(NEW_STATUS, COMMENT)
#                badmin = self.batchCmd.badmin(\
#                            'hopen', '%s.%s'
#                            % (self.localhost, self.localdomain))
                valid_host = self.localhost
                if self.localdomain not in valid_host:
                    valid_host += '.' + self.localdomain
                badmin = self.batchCmd.badmin('hopen', '%s' % valid_host)
#                commands.getstatusoutput(\
#                    'source /etc/profile.d/lsf.sh; badmin.org hopen %s'
#                    % self.localhost)
                return [0, 'Status changed in %s' % NEW_STATUS]

            elif PREVIOUS_STATUS == 'OPEN_ADMIN':
                self._changeBaitStatus(NEW_STATUS, COMMENT)
                return [0, 'Status changed in %s' % NEW_STATUS]
            else:
                self.updateLog('The current status: %s '
                               % PREVIOUS_STATUS +
                               'cannot be changed to the new status: %s'
                               % NEW_STATUS)
                return [0, 'host already in this status']

        elif NEW_STATUS == 'OPEN_ADMIN':
            if PREVIOUS_STATUS == 'START' or PREVIOUS_STATUS == 'CLOSED_ADMIN':
                self._changeBaitStatus(NEW_STATUS, COMMENT)
#                badmin = self.batchCmd.badmin(
#                            'hopen',
#                            '%s.%s'
#                            % (self.localhost, self.localdomain))
                valid_host = self.localhost
                if self.localdomain not in valid_host:
                    valid_host += '.' + self.localdomain
                badmin = self.batchCmd.badmin('hopen', '%s' % valid_host)
#                commands.getstatusoutput(
#                    'source /etc/profile.d/lsf.sh; ' +
#                    'badmin.org hopen %s'
#                    % self.localhost)
                return [0, 'Status changed in %s' % NEW_STATUS]
            else:
                self.updateLog('The current status: %s '
                               % PREVIOUS_STATUS +
                               'cannot be changed to the new status: %s'
                               % NEW_STATUS)
                return [0, 'host already in this status']
        else:
            return [0, 'Nothing to do']

    @synchronized(LOCK_VM)
    def updateVMStatus(self, *arguments):
        """
        Update in a synchronized way self.virtualMachines local data structure
        """

        self.updateLog(' Function: %s . Variables: %s' % (utils.whoami(),
                                                          str(arguments)))
        UPDATED_VirtualMachines = arguments[0]

        self.updateLog('3 %s' % str(self.virtualMachines.keys()))
        if type(UPDATED_VirtualMachines) == type({}):
            self.virtualMachines = UPDATED_VirtualMachines

        self.updateLog('3 %s' % str(self.virtualMachines.keys()))

    def updateConfig(self, *arguments):
        """
        Update one or more key of the config dictionary.
        params[1] MUST be a dictionary
        """

        configParameters = arguments[0]

        try:
            if len(configParameters.keys()) > 0:

                self.updateLog('New configuration parameters available')
                for param in configParameters.keys():

                    try:
                        self.updateLog('NEW Param: %s; OLD Param: %s'
                                       % (configParameters[param],
                                          self.config[param]))
                        self.config[param] = configParameters[param]
                    except:
                        self.updateLog('NEW Param: %s is not supported'
                                       % (configParameters[param]))

                return 0

            else:
                self.updateLog('Received dictionary is EMPTY')
                return 1
        except AttributeError:
            self.updateLog('Received string is non formatted as expected. ' +
                           'It is not a Dictionary. "%s"'
                           % str(configParameters))
            return 1


class GuardianAngel(threading.Thread):
    """ A thread will be instantiated for each VM.
    This tread handles all those status
    which are not managed from a WNoDeS component.
    """

    def __init__(self, bait_instance, batchJobId):

        threading.Thread.__init__(self)
        self.batchJobId = batchJobId
        self.wb = bait_instance

    def run(self):

        self.wb.updateLog('Thread started to handle status change for job %s'
                          % self.batchJobId)

        LAST_JOB_STATUS = ''

        while True:
            if self.batchJobId in self.wb.batchJobs:
                VM_ID = self.wb.batchJobs[self.batchJobId][1]

                if VM_ID in self.wb.virtualMachines:
                    vmParameters = self.wb.virtualMachines[VM_ID][3]
                else:
                    vmParameters = self.wb.batchJobs[self.batchJobId][3]

                bjobsOutput = self.wb.batchCmd.bjobs(jobid=self.batchJobId)
                batchJobStatus = bjobsOutput[1].split()[2]

                if batchJobStatus == LAST_JOB_STATUS:
                    time.sleep(15)
                else:
                    LAST_JOB_STATUS = batchJobStatus

                self.wb.updateLog('Job status: %s  ' % str(batchJobStatus),
                                  'info',
                                  self.batchJobId)

                if batchJobStatus in ['DONE', 'EXIT', 'UNKWN', 'ZOMBIE']:
                    self.wb.updateLog('Job already finished  %s'
                                      % str(self.batchJobId),
                                      'info',
                                      self.batchJobId)
                    self.wb.updateStatusAndTimeStamp(
                        self.batchJobId,
                        'FINISHED',
                        self.wb.batchJobs[self.batchJobId][4])
                else:
                    pass

                if self.wb.batchJobs[self.batchJobId][0] == 'READYtoMODIFY':

                    time.sleep(int(
                        self.wb.config['SCHEDULING_INTERVAL']) / 5)

                    if self.wb.config['BATCH_SYSTEM_TYPE'].upper() == 'LSF':
#                        bmodoutput = self.wb.batchCmd.bmod(
#                                        '-U %s %s'
#                                        % (vmParameters['RESERVATION_ID'],
#                                           self.batchJobId))
                        bmodoutput = self.wb.batchCmd.bmod(
                                        '-U %s'
                                        % vmParameters['RESERVATION_ID'],
                            self.batchJobId)

                    elif self.wb.config['BATCH_SYSTEM_TYPE'].upper() == 'PBS':
                        while True:
                            bjobsOutput = self.wb.batchCmd.bjobs(
                                              jobid=self.batchJobId)
                            batchJobStatus = bjobsOutput[1].split()[2]
                            if batchJobStatus == 'PEND':
#                                bmodoutput = self.wb.batchCmd.breserve('add',
#                                    self.batchJobId, vmParameters['HOSTNAME'],
#                                    vmParameters['RESERVATION_ID'])
                                bmodoutput = self.wb.batchCmd.breserve('add',
                                    self.batchJobId, vmParameters['HOSTNAME'])
                                break
                            else:
                                time.sleep(5)

                    self.wb.updateLog('%s' % str(bmodoutput),
                                      'info', self.batchJobId)
                    self.wb.updateStatusAndTimeStamp(
                        self.batchJobId,
                        'READYforSETUP',
                        self.wb.batchJobs[self.batchJobId][4])

                elif self.wb.batchJobs[self.batchJobId][0] == 'READYforSETUP':
                    self.wb.updateLog('Host %s is ready for setup'
                                      % vmParameters['HOSTNAME'],
                                      'info',
                                      self.batchJobId)

                    """
                    CHANGE STATUS FROM READYforSETUP
                    to SETUPDONE or HOSTunREACH or JOBREQUEUE
                    """
                    while True:

                        PING_HOST = commands.getstatusoutput(
                            'ping -c2 %s' % vmParameters['HOSTNAME'])

                        SSH_HOST = self.wb.checkSSH(vmParameters['HOSTNAME'])

                        if PING_HOST[0] == 0 and SSH_HOST == 0:
                            self.wb.updateLog('Host %s is UP'
                                              % vmParameters['HOSTNAME'],
                                              'info',
                                              self.batchJobId)
                            # Does the VM require a specific setup ?

                            VM_SETUP = False
                            for VM_PARAM_NAME in vmParameters.keys():
                                if 'VM_CONFIG_' in VM_PARAM_NAME:
                                    VM_SETUP = True
                                    break
                                else:
                                    VM_SETUP = False

                            if VM_SETUP:
                                self.wb.updateLog(
                                'There is at least one VM setup action. ' +
                                'Send a request to HV ' +
                                'in order to setup the VM proprerly',
                                'info',
                                self.batchJobId)
                                msg = {'vmSetup':
                                       [self.wb.batchJobs
                                            [self.batchJobId][1], '']}
                                output = self.wb.sendRequest(self.wb.HV_HOST,
                                                             self.wb.HV_PORT,
                                                             msg)
                                if output[0] == 0:
                                    output = output[1]
                                    if output[0] == 0:

                                        self.wb.updateLog('Virtual resource ' +
                                            'setup successfully completed. %s'
                                            % str(output[1]),
                                            'info',
                                            self.batchJobId)
                                        self.wb.updateStatusAndTimeStamp(
                                            self.batchJobId,
                                            'SETUPDONE',
                                            self.wb.batchJobs
                                                [self.batchJobId][4])
                                    else:

                                        self.wb.updateLog('Remote method ' +
                                            'error: %s' % output[1],
                                            'error',
                                            self.batchJobId)
                                        self.wb.updateStatusAndTimeStamp(
                                            self.batchJobId,
                                            'JOBREQUEUE',
                                            self.wb.batchJobs
                                                [self.batchJobId][4])
                                else:

                                    self.wb.updateLog('Communication error: %s'
                                                      % output[1],
                                                      'error',
                                                      self.batchJobId)
                                    self.wb.updateStatusAndTimeStamp(
                                        self.batchJobId,
                                        'JOBREQUEUE',
                                        self.wb.batchJobs[self.batchJobId][4])
                                break

                            else:
                                self.wb.updateLog('There is no setup action' +
                                    ' for this VM. Proceed with the next step',
                                    'info',
                                    self.batchJobId)
                                self.wb.updateStatusAndTimeStamp(
                                    self.batchJobId,
                                    'SETUPDONE',
                                    self.wb.batchJobs[self.batchJobId][4])
                                break
                        else:
                            self.wb.updateLog('Host %s is not still UP'
                                              % vmParameters['HOSTNAME'],
                                              'info',
                                              self.batchJobId)
                            try:
                                TIMEOUT = int(vmParameters['UNREACH_TIMEOUT'])
                            except KeyError:
                                TIMEOUT = int(
                                    self.wb.config['VM_UNREACH_TIMEOUT'])

                            if ((time.time() -
                                    self.wb.batchJobs[self.batchJobId][4])
                                    > TIMEOUT):

                                self.wb.updateLog('Host %s is not still UP. '
                                                  % vmParameters['HOSTNAME']
                                                  + 'TIMEOUT',
                                                  'info',
                                                  self.batchJobId)
                                self.wb.updateStatusAndTimeStamp(
                                    self.batchJobId,
                                    'HOSTunREACH',
                                    self.wb.batchJobs[self.batchJobId][4])
                                break
                            else:
                                time.sleep(1)

                elif self.wb.batchJobs[self.batchJobId][0] == 'SETUPDONE':
                    self.wb.updateLog('Virtual resource is now available',
                                      'info',
                                      self.batchJobId)
                    if vmParameters['TYPE'] == 'CLOUD':
                        self.wb.updateStatusAndTimeStamp(
                            self.batchJobId,
                            'HOST_AVAILABLE',
                            self.wb.batchJobs[self.batchJobId][4])
                    if vmParameters['TYPE'] == 'BATCH':
                        self.wb.updateStatusAndTimeStamp(
                            self.batchJobId,
                            'READYtoRUN',
                            self.wb.batchJobs[self.batchJobId][4])
#                        self.wb.batchCmd.badmin('hopen', '%s.%s' % (
#                            self.wb.batchJobs[self.batchJobId][2],
#                            self.wb.localdomain))
                        valid_host = self.wb.batchJobs[self.batchJobId][2]
                        if self.wb.localdomain not in valid_host:
                            valid_host += '.' + self.wb.localdomain
                        self.wb.batchCmd.badmin('hopen', '%s' % valid_host)

                elif self.wb.batchJobs[self.batchJobId][0] == 'HOSTunREACH':
                    self.wb.updateLog('Virtual resource is unreachable',
                                      'info', self.batchJobId)
                    self.wb.updateLog('%s' % str(vmParameters),
                                      'info', self.batchJobId)
                    if not (self.wb.batchJobs
                                [self.batchJobId][11]['HOSTunREACH']
                            > int(self.wb.config['STATUS_RETRY_COUNT'])):
                        msg = {'do_regenerate': [vmParameters['ID'],
                                                 vmParameters,
                                                 [],
                                                 False]}
                        output = self.wb.sendRequest(self.wb.HV_HOST,
                                                     self.wb.HV_PORT, msg)
                        if output[0] == 0:
                            output = output[1]
                            if output[0] == 0:

                                self.wb.updateLog('Virtual Resource ' +
                                    'successfully regenerated. %s'
                                    % str(output[1]),
                                    'info',
                                    self.batchJobId)
                                vmId = output[1][0]

                                self.wb.updateLog(
                                    '2 %s' %
                                    str(self.wb.virtualMachines.keys()))
                                self.wb.virtualMachines[vmId] = output[1][1]
                                self.wb.updateLog(
                                    '2 %s'
                                    % str(self.wb.virtualMachines.keys()))
                                vmParameters = self.wb.virtualMachines[vmId][3]
                                self.wb.batchJobs[self.batchJobId][1] = \
                                    vmParameters['ID']
                                self.wb.batchJobs[self.batchJobId][2] = \
                                    vmParameters['HOSTNAME']
                                self.wb.updateLog('Proceed with the next step',
                                                  'info', self.batchJobId)
                                self.wb.updateStatusAndTimeStamp(
                                    self.batchJobId,
                                    'READYforSETUP',
                                    self.wb.batchJobs[self.batchJobId][4])
                            else:
                                self.wb.updateLog('Remote method error: %s'
                                                  % output[1], 'error',
                                                  self.batchJobId)
                                self.wb.updateStatusAndTimeStamp(
                                    self.batchJobId,
                                    'JOBREQUEUE',
                                    self.wb.batchJobs[self.batchJobId][4])
                        else:

                            self.wb.updateLog('Communication error: %s'
                                              % output[1],
                                              'error',
                                              self.batchJobId)
                            self.wb.updateStatusAndTimeStamp(
                                self.batchJobId,
                                'JOBREQUEUE',
                                self.wb.batchJobs[self.batchJobId][4])
                    else:
                        self.wb.updateLog('The job %s has hit the RETRY LIMIT '
                            % self.wb.batchJobs[self.batchJobId][2]
                            + 'COUNT in the the following state: HOSTunREACH. '
                            + 'I will requeue it',
                            'info', self.batchJobId)
                        self.wb.updateStatusAndTimeStamp(
                            self.batchJobId,
                            'JOBREQUEUE',
                            self.wb.batchJobs[self.batchJobId][4])

                elif self.wb.batchJobs[self.batchJobId][0] == 'JOBREQUEUE':
                    """ At this point we have to:
                        1. Release RESOURCE
                        2. Destroy the VM and Release the hostname
                        3. Remove the job
                    """

                    self.wb.accessResourceAvailable('PUT', vmParameters)

                    msg = {'do_destroy': [[vmParameters['ID']], True]}
                    output = self.wb.sendRequest(self.wb.HV_HOST,
                                                 self.wb.HV_PORT,
                                                 msg)
                    self.wb.updateLog('1 %s'
                                      % str(self.wb.virtualMachines.keys()))
                    if output[1][0] == 0:
                        self.wb.virtualMachines.pop(vmParameters['ID'])
                    self.wb.updateLog('1 %s'
                                      % str(self.wb.virtualMachines.keys()))
                    if vmParameters['TYPE'] == 'BATCH':
#                        bmod = self.wb.batchCmd.bmod('-Un %s'
#                                                     % self.batchJobId)
                        bmod = self.wb.batchCmd.bmod('-Un', self.batchJobId)
                        self.wb.updateLog('%s'
                                           % str(bmod),
                                          'info', self.batchJobId)
                    elif vmParameters['TYPE'] == 'CLOUD':
                        bkill = self.wb.batchCmd.bkill(' %s' % self.batchJobId)
#                    bmod = commands.getstatusoutput\
#                        ('source /etc/profile.d/lsf.sh; bmod -Un %s'
#                         % self.batchJobId)

                    if self.wb.config['BATCH_SYSTEM_TYPE'].upper() == 'LSF':
                        rm_reservation = self.wb.batchCmd.breserve(
                                            'del', '%s'
                                            % vmParameters['RESERVATION_ID'])
                    elif self.wb.config['BATCH_SYSTEM_TYPE'].upper() == 'PBS':
                        rm_reservation = self.wb.batchCmd.breserve(
                                            'del',
                                            self.batchJobId)
#                    rm_reservation = commands.getstatusoutput(
#                                        'source /etc/profile.d/lsf.sh; \
#                                        brsvdel %s'
#                                        % vmParameters['RESERVATION_ID'])
                    self.wb.batchJobs.pop(self.batchJobId)
#                    commands.getstatusoutput('source /etc/profile.d/lsf.sh; '
#                                             + 'brequeue %s'
#                                             % self.batchJobId)
#                    commands.getstatusoutput('source /etc/profile.d/lsf.sh; '
#                                             + 'btop %s'
#                                             % self.batchJobId)
                    self.wb.updateLog('%s' % str(rm_reservation),
                                      'info', self.batchJobId)

                elif self.wb.batchJobs[self.batchJobId][0] == 'READYtoRUN':

                    if self.wb.batchJobs[self.batchJobId][9] == 2:

                        if (time.time() - self.wb.batchJobs[self.batchJobId][4]
                            > int(self.wb.config['VM_UNREACH_TIMEOUT'])):

                            if not int(
                                self.wb.batchJobs
                                [self.batchJobId][11]['READYtoRUN']) >\
                                int(self.wb.config['STATUS_RETRY_COUNT']):

                                self.wb.updateLog('Preexec script for job %s '
                                    % self.wb.batchJobs[self.batchJobId][2]
                                    + 'has not been yet executed. '
                                    + 'Force job execution',
                                    'info',
                                    self.batchJobId)
#                                brunOutput = self.wb.batchCmd.brun(
#                                        self.wb.batchJobs[self.batchJobId][2],
#                                        self.wb.localdomain, self.batchJobId)
                                valid_host = \
                                    self.wb.batchJobs[self.batchJobId][2]

                                if self.wb.localdomain not in valid_host:
                                    valid_host += '.' + self.wb.localdomain
                                self.wb.updateLog('valid host %s'
                                                  % valid_host,
                                                  'info',
                                                  self.batchJobId)
                                brunOutput = self.wb.batchCmd.brun(
                                                valid_host,
                                                self.batchJobId)
                                self.wb.updateStatusAndTimeStamp(
                                    self.batchJobId,
                                    'READYtoRUN',
                                    self.wb.batchJobs[self.batchJobId][4])
                            else:
                                self.wb.updateLog(
                                    'The job %s has hit '
                                    % self.wb.batchJobs[self.batchJobId][2]
                                    + 'the RETRY LIMIT COUNT in the the '
                                    + 'following state:'
                                    + 'PENDING FORCE JOB EXECUTION',
                                    'info',
                                    self.batchJobId)
                                self.wb.updateStatusAndTimeStamp(
                                    self.batchJobId,
                                    'JOBREQUEUE',
                                    self.wb.batchJobs[self.batchJobId][4])
                        else:
                            time.sleep(1)

                    elif self.wb.batchJobs[self.batchJobId][9] == 1:
                        if int(self.wb.batchJobs
                               [self.batchJobId][11]['READYtoRUN']) >\
                                int(self.wb.config['STATUS_RETRY_COUNT']):
                            self.wb.updateLog(
                                'The job %s has hit the RETRY LIMIT COUNT '
                                % self.wb.batchJobs[self.batchJobId][2]
                                + 'in the the following state:'
                                + 'PENDING PREEXEC ECEUTION FAILED',
                                'info',
                                self.batchJobId)
                            self.wb.updateStatusAndTimeStamp(
                                self.batchJobId,
                                'JOBREQUEUE',
                                self.wb.batchJobs[self.batchJobId][4])
                        else:
                            pass
                    elif self.wb.batchJobs[self.batchJobId][9] == 0:
                        self.wb.updateStatusAndTimeStamp(
                            self.batchJobId,
                            'RUN',
                            self.wb.batchJobs[self.batchJobId][4])

                elif self.wb.batchJobs[self.batchJobId][0] == 'RUN':

                    self.wb.updateLog('Let me try whether the host %s '
                                       % self.wb.batchJobs[self.batchJobId][2]
                                       + 'is still reacheable or not',
                                      'info', self.batchJobId)

                    PING_HOST = commands.getstatusoutput('ping -c2 %s'
                                    % self.wb.batchJobs[self.batchJobId][2])
                    SSH_HOST = self.wb.checkSSH(
                                    self.wb.batchJobs[self.batchJobId][2])

                    if PING_HOST[0] == 0 and SSH_HOST == 0:
                        self.wb.updateLog('Host %s is really reachable'
                            % self.wb.batchJobs[self.batchJobId][2],
                            'info', self.batchJobId)

                        bjobsOutput = self.wb.batchCmd.bjobs(
                                          jobid=self.batchJobId)
#                        bjobsOutput = commands.getstatusoutput(
#                                          'source /etc/profile.d/lsf.sh; ' +
#                                          'bjobs -w %s | grep %s'
#                                          % (self.batchJobId,
#                                             self.batchJobId))
                        batchJobStatus = bjobsOutput[1].split()[2]

                        if batchJobStatus == 'RUN':
                            self.wb.updateLog('Batch job %s is really RUNNING'
                                              % self.batchJobId, 'info',
                                              self.batchJobId)
                            time.sleep(60)

                        elif batchJobStatus == 'PEND':
                            self.wb.updateLog('Batch job %s is still PENDING'
                                              % self.batchJobId, 'info',
                                              self.batchJobId)
                            if (self.wb.config['BATCH_SYSTEM_TYPE'].upper()
                                    == 'PBS'):
                                time.sleep(60)
                                self.wb.updateLog('Using PBS as batch system '
                                    + 'we are experiecing a strange behaviour.'
                                    + ' Pre exec report has been sent '
                                    + 'but the job is not really running, '
                                    + 'thus I have to force it again ',
                                    'info', self.batchJobId)
#                                brunOutput = self.wb.batchCmd.brun(
#                                        self.wb.batchJobs[self.batchJobId][2],
#                                        self.wb.localdomain,
#                                        self.batchJobId)
                                valid_host = (self.wb.batchJobs
                                                [self.batchJobId][2])
                                if self.wb.localdomain not in valid_host:
                                    valid_host += '.' + self.wb.localdomain
                                self.wb.updateLog('valid host %s'
                                                  % valid_host,
                                                  'info',
                                                  self.batchJobId)
                                brunOutput = self.wb.batchCmd.brun(
                                                valid_host, self.batchJobId)

                        elif (batchJobStatus == 'DONE'
                              or batchJobStatus == 'EXIT'):
                            self.wb.updateLog('Batch job %s is TERMINATED'
                                              % self.batchJobId,
                                              'info',
                                              self.batchJobId)
                            self.wb.updateStatusAndTimeStamp(
                                self.batchJobId,
                                'FINISHED',
                                self.wb.batchJobs[self.batchJobId][4])

                        elif (batchJobStatus == 'UNKWN'
                              or batchJobStatus == 'ZOMBIE'):
                            self.wb.updateLog('Batch job %s '
                                              % self.batchJobId
                                              + 'is in a bad STATE %s'
                                              % batchJobStatus,
                                              'info',
                                              self.batchJobId)

                        else:
                            self.wb.updateLog('Unknown STATUS %s:'
                                              % batchJobStatus,
                                              'info', self.batchJobId)
                            pass

                    else:
                        self.wb.updateLog('While the job %s '
                                          % self.batchJobId
                                          + 'is running the VM %s '
                                          % vmParameters['HOSTNAME']
                                          + 'become UNREACHABLE',
                                          'error',
                                          self.batchJobId)

                elif self.wb.batchJobs[self.batchJobId][0] == 'FINISHED':

                    self.wb.accessResourceAvailable('PUT', vmParameters)

                    if vmParameters['TYPE'] == 'BATCH':

                        if (int(self.wb.config['RESERVATION_LENGTH']) \
                            > self.wb.batchJobs[self.batchJobId][7]):

                            if (self.wb.config['BATCH_SYSTEM_TYPE'].upper() \
                                == 'LSF'):

                                rm_reservation = self.wb.batchCmd.breserve(
                                    'del', '%s'
                                    % vmParameters['RESERVATION_ID'])

                            elif (self.wb.config['BATCH_SYSTEM_TYPE'].upper() \
                                  == 'PBS'):

                                rm_reservation = self.wb.batchCmd.breserve(
                                                    'del', self.batchJobId)

                            self.wb.updateLog('%s'
                                              % str(rm_reservation),
                                              'info',
                                              self.batchJobId)

                        try:
                            self.wb.config['ALWAYS_DESTROY']
                        except:
                            self.wb.config['ALWAYS_DESTROY'] = 'NO'
                            
                        if self.wb.config['ALWAYS_DESTROY'].upper() == 'NO':
                            valid_host = self.wb.batchJobs[self.batchJobId][2]
                            if self.wb.localdomain not in valid_host:
                                valid_host += '.' + self.wb.localdomain
                            self.wb.batchCmd.badmin('hclose', '%s' % valid_host)
    
                            self.wb.virtualMachines[vmParameters['ID']][2] = \
                                ['OFF', 'AVAILABLE']
    
                            self.wb.batchJobs.pop(self.batchJobId)
    
                            msg = {'vmUpdateStatus':
                                   [vmParameters['ID'],
                                    self.wb.virtualMachines[vmParameters['ID']]]}
                            output = self.wb.sendRequest(
                                                         self.wb.HV_HOST,
                                                         self.wb.HV_PORT,
                                                         msg
                                                         )
                        elif self.wb.config['ALWAYS_DESTROY'].upper() == 'YES':
                            msg = {'do_destroy': [[vmParameters['ID']], True]}
                            output = self.wb.sendRequest(
                                                         self.wb.HV_HOST,
                                                         self.wb.HV_PORT,
                                                         msg
                                                         )
                            self.wb.batchJobs.pop(self.batchJobId)
                        else:
                            pass

                    elif vmParameters['TYPE'] == 'CLOUD':

                        msg = {'do_destroy':
                               [[vmParameters['ID']], True]}
                        output = self.wb.sendRequest(self.wb.HV_HOST,
                                                     self.wb.HV_PORT, msg)
                        self.wb.batchJobs.pop(self.batchJobId)

                    elif vmParameters['TYPE'] == 'BATCH_REAL':
                        self.wb.batchJobs.pop(self.batchJobId)

                else:
                    time.sleep(2)
            else:
                self.wb.updateLog('Batch job %s is not managed any more. '
                                  % self.batchJobId
                                  + 'Thread can be killed',
                                  'info',
                                  self.batchJobId)
                break


def main():
    """
    Entry point for the WNoDeS process running on the bait.
    """

    # parse options from command line
    p = optparse.OptionParser()
    p.add_option('-d', '--daemonize', action='store_true', dest='daemon',
                 help='daemonize process')
    p.add_option('-c', '--configFile',
                 dest='configFile',
                 default=os.path.join(__dir_name__,
                                      '/etc/wnodes/bait',
                                      'wnodes.ini'),
                 help='set configuration file [default: %default]',
                 metavar='FILE')
    (options, args) = p.parse_args()  # @UnusedVariable

    # check the presence of a previous wnodes_bait process pid file
    pid = "/var/run/%s.pid" % __short_name__
    if os.path.isfile(pid):
        try:
            stale_pid = int(open(pid).readline().strip("\n"))
        except ValueError:
            sys.exit("%s: cannot read pid file."
                     % __short_name__)
        try:
            os.kill(stale_pid, 0)
            sys.exit("%s: wnodes_bait already running: Killed"
                     % __short_name__)
        except OSError:
            sys.stderr.write("%s: stale pid found. Cannot be killed\n"
                             % __short_name__)
#             se non si puo' killare il precedenre wnod_bait
#             non si dovrebbe uscire?
#            sys.exit("%s: stale pid found. Cannot be killed\n"
#                     % __short_name__)

    # load Nameserver host & port
    # reading the configuration file
    try:
        if os.path.isfile(options.configFile):
            conf = ConfigParser.RawConfigParser()
            try:
                conf.read(options.configFile)
            except ConfigParser.MissingSectionHeaderError:
                sys.stdout.write('There is no SECTION header [NameServer] ' +
                                 'in configuration file)\n')
                sys.exit(1)
            except:
                sys.stdout.write('Error reading configuration: %s\n'
                                 % str(sys.exc_info()[0]))
                sys.exit(1)
            try:
                NS_HOST = conf.get('NAMESERVER', 'NS_HOST')
                NS_PORT = conf.get('NAMESERVER', 'NS_PORT')
            except ConfigParser.NoSectionError:
                sys.stdout.write('There is no SECTION ' +
                                 'for WNoDeS Name Server host\n')
                sys.exit(1)
            except:
                sys.stdout.write('Error reading configuration: %sn\n'
                                 % str(sys.exc_info()[0]))
                sys.exit(1)
        else:
            sys.stdout.write('Configuration file location is not present: %s\n'
                             % options.configFile)
            sys.exit(1)
    except IOError:
        sys.stdout.write('IOEroor')
        sys.exit(1)

    # initialize wb object from class WnodeBait
    # (inherited from wsocket.ClientRequestHandler)
    wb = WnodBait(NS_HOST, NS_PORT)

    if options.daemon is True:

        # daemonize the current process
        utils.daemonize(stdout="/tmp/%s.stdout"
                        % __short_name__,
                        stderr="/tmp/%s.stderr"
                        % __short_name__)

        # write the process pid inside the pid.file
        of = open(pid, "w")
        of.write("%i\n" % os.getpid())
        of.close()

        # ask to the the HV the available resouces
        msg = {'getResourceAvailable': [None]}
        RESOURCES = wb.sendRequest(wb.HV_HOST, wb.HV_PORT, msg)

        while True:
            if RESOURCES[0] == 0:
                RESOURCES = RESOURCES[1]

                # get slot number available on this host
                if wb.config['BATCH_SYSTEM_TYPE'].lower() == 'lsf':

                    try:
                        bhost = wb.batchCmd.bhost(wb.localhost)[1].split()
                        slots = bhost[3]
                        slots = int(slots)
                    except:
                        slots = 0

                    RESOURCES[1]['CPU'] = slots

                elif (wb.config['BATCH_SYSTEM_TYPE'].lower() == 'pbs' or
                      wb.config['BATCH_SYSTEM_TYPE'].lower() == 'torque'):

                    try:
                        from xml.etree import ElementTree
                    except ImportError:
                        try:
                            from elementtree import ElementTree
                        except ImportError:
                            slots = 0
                    try:

                        bhost = wb.batchCmd.bhost('%s.%s' %
                                                  (wb.localhost,
                                                   wb.localdomain))[1]
                        bhost_info = ElementTree.fromstring(bhost)
                        slots = 0
                        for i in bhost_info.getiterator():
                            if i.tag == 'np':
                                slots = int(i.text)
                                break

                    except:
                        slots = 0

                RESOURCES[1]['CPU'] = slots

                wb.updateLog(str(RESOURCES))
                break
            else:
                wb.updateLog('Connection error requesting available resources',
                             'error')
                time.sleep(5)
                msg = {'getResourceAvailable': [None]}
                RESOURCES = wb.sendRequest(wb.HV_HOST, wb.HV_PORT, msg)

        msg = {'getVMAvailable': [None]}
        VMS = wb.sendRequest(wb.HV_HOST, wb.HV_PORT, msg)

        msg = {'getImageAvailable': [None]}
        IMAGES = wb.sendRequest(wb.HV_HOST, wb.HV_PORT, msg)

    if VMS[0] == 0:
        VMS = VMS[1]
    else:
        wb.updateLog('Connection error requesting available VM', 'error')
        VMS = VMS[1]

    if IMAGES[0] == 0:
        IMAGES = IMAGES[1]
    else:
        wb.updateLog('Connection error requesting available IMAGES', 'error')
        IMAGES = IMAGES[1]

    if RESOURCES[0] == 0 or VMS[0] == 0 or IMAGES[0] == 0:

        wb.updateLog('Successfully retrived the RESOURCES, VMS and IMAGES' +
                     ' available from HV:%s' % wb.HV_HOST)

        wb.accessResourceAvailable('SET', RESOURCES[1])
        # Is this the right way ???
        wb.virtualMachines = VMS[1]

        CURRENT_STATUS = 'DATA AVAILABLE UPDATED'

        wb.updateBaitBatchStatus(['OPEN',
                                  'Everything is OK, ' +
                                  'the BAIT process can start'])
    else:

        wb.updateLog('There was a problem retriving Information from the HV:%s'
                     % wb.HV_HOST)
        CURRENT_STATUS = 'DATA NOT AVAILABLE'
        wb.updateBaitBatchStatus(['CLOSE_ADMIN',
                                  'There is a problem with DATA. ' +
                                  'BAIT process cannot start'])

    wb.updateLog('###### BAIT Server Starting ... ######')
    wb.updateLog('######')
    wb.updateLog('###### CURRENT HV: %s' % wb.HV_HOST)
    wb.updateLog('###### CURRENT DATA STATUS: %s' % CURRENT_STATUS)
    wb.updateLog('###### CURRENT BAIT STATUS: %s : %s'
                 % (wb.batchBaitStatus[0], wb.batchBaitStatus[1]))
    wb.updateLog('######')
    wb.updateLog('###### CURRENT VM AVAILABLE OM THE HV')
    for ID in wb.virtualMachines.keys():
        wb.updateLog(('###### -- %s : %s')
                     % (str(ID), str(wb.virtualMachines[ID])))
    wb.updateLog('######')
    wb.updateLog('###### CURRENT RESOURCES ON THE HV')
    for res in wb.batchBaitStatus[5].keys():
        wb.updateLog(('###### -- %s : %s')
                     % (str(res), str(wb.batchBaitStatus[5][res])))
    wb.updateLog('######')
    wb.updateLog('###### %s' % str(wb.batchBaitStatus))
    wb.updateLog('######################################')

    serviceDispatchTable = {
                'destroyVMInstance': wb.destroyVMInstance,
                'getConfig': wb.getConfig,
                'getStatus': wb.getStatus,
                'ping': wb.ping,
                'reloadConfig': wb.reloadConfig,
                'requestVMInstance': wb.requestVMInstance,
                'reportPostExecutionScript': wb.reportPostExecutionScript,
                'reportPreExecutionScript': wb.reportPreExecutionScript,
                'updateBaitBatchStatus': wb.updateBaitBatchStatus,
                'updateConfig': wb.updateConfig,
                            }
    wsocket.initServerRequestHandler(wb, serviceDispatchTable)
    BaitServerHandler = wsocket.ServerRequestHandler

    srv = SocketServer.ThreadingTCPServer(('',
                                           int(wb.config['BAIT_PORT'])),
                                          BaitServerHandler)
    srv.daemon_threads = True
    srv.allow_reuse_address = True
    srv.serve_forever()

if __name__ == "__main__":
    main()
