#
# ghettovcb.py
#
# mkbackup ghettovcb frontent
#

import os, string, subprocess, fnmatch, platform
import posixpath

from datetime import datetime, timedelta

import paramiko
import scp


from archiver import *
import cron

GHETTO_VERSION='04/24/2010'

# ---------------------------------------------------------------------------
class MySCPClient(scp.SCPClient):
    """MySCPClient.put will convert all '\r\n' into '\n'
       MySCPClient.get will will download the remote dir at same level as local dir
    """ 
    
    # -----------------------------------------------------------------------
    def _send_files(self, files):
        for name in files:
            basename = os.path.basename(name)
            (mode, size, mtime, atime) = self._read_stats(name)
            if self.preserve_times:
                self._send_time(mtime, atime)
            file_hdl = file(name, 'rb')
            buffer=file_hdl.read()
            file_hdl.close()
            buffer=buffer.replace('\r', '')
            size=len(buffer)
            self.channel.sendall('C%s %d %s\n' % (mode, size, basename))
            self._recv_confirm()
            file_pos = 0
            buff_size = self.buff_size
            chan = self.channel
            while file_pos < size:
                buf=buffer[file_pos:file_pos+buff_size]
                chan.sendall(buf)
                file_pos += len(buf) 
                if self.callback:
                    self.callback(file_pos, size)
            chan.sendall('\x00')

    # -----------------------------------------------------------------------
    def get(self, remote_path, local_path = '',
            recursive = False, preserve_times = False):
        self._recv_topdir = local_path or os.getcwd()
        return scp.SCPClient.get(self, remote_path, local_path, recursive, preserve_times)
    
    # -----------------------------------------------------------------------
    def _recv_pushd(self, cmd):
        parts = cmd.split()
        try:
            mode = int(parts[0], 8)
            path = os.path.join(self._recv_dir, parts[2])
        except:
            self.channel.send('\x01')
            raise SCPException('Bad directory format')
        else:
            if self._recv_dir==self._recv_topdir and (not os.path.exists(self._recv_dir) or os.path.isdir(self._recv_dir)):
                # use self._recv_dir instead of os.path.join(self._recv_dir, parts[2])
                path=self._recv_dir
                self._recv_topdir=''

        try:
            if not os.path.exists(path):
                os.mkdir(path, mode)
            elif os.path.isdir(path):
                os.chmod(path, mode)
            else:
                raise SCPException('%s: Not a directory' % path)
            self._dirtimes[path] = (self._utime)
            self._utime = None
            self._recv_dir = path
        except (OSError, SCPException), e:
            self.channel.send('\x01'+e.message)
            raise
        

# ---------------------------------------------------------------------------
def RunCommand(t, command, log=None):
    chan=t.open_session()
    exit_code=None
    chan.set_combine_stderr(True)
    chan.exec_command(command)
    output=''
    while 1:
        try:
            x=chan.recv(1024)
            if len(x)==0:
                break
            output+=x
        except socket.timeout:
            status=-1
            break

    if exit_code==None:
        exit_code=chan.recv_exit_status()
        chan.close()
        
    if log:
        l=log.debug
        if exit_code!=0:
            l=log.warning
            
        if output or exit_code!=0:
            l('code=%d command=%s', exit_code, command)
            for line in output.split('\n'):
                l('> %s', line)
        
    return exit_code, output
  

class Ghettovcb(Archiver):
    
    name='ghettovcb'
    exe='ghettovcb'

    types=dict( backup='backup', copy='copy', move='move',)
    variables=dict(vm='vm_name')

    # -----------------------------------------------------------------------
    def __init__(self):
        Archiver.__init__(self)
        
        self.host=None
        self.login=None
        self.password=None
        self.local=None
        self.remote_temp=None
        self.vm_list=[]
        self.vm_exclude=[]
        
        self.ghettovcb=None
        self.ghettovcb_bin=None
        self.global_conf=None
        self.temp_target=None
        self.current_vm=None
        self.vm_var=None
        self.default_var=None
        
        self.destination=None
        self.type='backup'
        self.target=''
        
        self.scp_bin=None
        self.scp_args=None

        
    # -----------------------------------------------------------------------
    def read_vm_conf(self, filename, default={}):
        """load important variables from VM configuration backup script""" 
        values=default.copy()
        for line in open(filename, 'r'):
            for var in ('VM_BACKUP_VOLUME', 'ENABLE_COMPRESSION', 'LAST_MODIFIED_DATE', 'ENABLE_NON_PERSISTENT_NFS',):
                if line.startswith('printUsage() {'):
                    break
                if line.startswith('%s=' % var):
                    _k, v=line.split('=', 1)
                    values[var]=v.strip()
                    
        return values
    
    # -----------------------------------------------------------------------
    def load(self, job, manager):
        
        log=manager.log
        now=manager.now
        
        errors, warnings, extra={}, {}, ''

        try:
            self.host=job['host'].encode('ascii')
        except KeyError:
            errors['host']='option mandatory'
        except UnicodeEncodeError:
            errors['host']='invalid hostname or ip address'
            
        else:
            if not valid_ipRE.match(self.host):
                if not valid_hostnameRE.match(self.host):
                    errors['host']='invalid hostname or ip address'
                else:
                    try:
                        ip=socket.gethostbyname(self.host)
                    except socket.gaierror:
                        errors['host']='cannot resolve hostname'

        try:
            self.port=int(job.get('port', 22))
        except ValueError:
            errors['port']='must be an integer'
        else:
            if not (0<self.port and self.port<65535):
                errors['port']='must be an integer between 1 and 65535'

        for name in ('login', 'password', 'local', 'remote_temp',):
            value=job.get(name, None)
            if not value:
                errors[name]='option mandatory'
            else:
                setattr(self, name, value)

        try:
            self.vm_list=quoted_string_list(job.get('vm_list', ''))
        except ValueError:
            errors['vm_list']='not a valid quoted string list'

        try:
            self.vm_exclude=quoted_string_list(job.get('vm_exclude', ''))
        except ValueError:
            errors['vm_exclude']='not a valid quoted string list'

        if job.get('target', None):
            self.target=job['target']


        self.global_conf=job.get('global_conf', None)

        if not os.path.isdir(self.local):
            errors['local']='not a directory'
        else:
            # search ghettoVCB.sh whatever the case
            self.ghettovcb=None
            list_dir=os.listdir(self.local)
            if 'ghettoVCB.sh' in list_dir:
                self.ghettovcb='ghettoVCB.sh'
            else:
                for filename in os.listdir(self.local):
                    if filename.lower()=='ghettovcb.sh':
                        self.ghettovcb=filename
                        break

            # search the global_conf whatever the case
            if self.global_conf:
                if not self.global_conf in list_dir:
                    global_conf=None
                    for filename in list_dir:
                        if filename.lower()==self.global_conf.lower():
                            global_conf=filename
                            break
                    if not global_conf:
                        errors['global_conf']='global_conf file not found'
                    else:
                        self.global_conf=global_conf

            if not self.ghettovcb:
                errors['local']='ghettoVCB.sh not found'
            
            # read ghettoVCB.sh  and extract default variables
            self.default_var={}
            if self.ghettovcb:
                self.default_var=self.read_vm_conf(os.path.join(self.local, self.ghettovcb))
                
                
            if self.default_var.get('LAST_MODIFIED_DATE','')!=GHETTO_VERSION:
                warning['local']='ghettoVCB.sh version is different, use the one provided with MKBackup or be warned'
            
            
            # read and extract gloabl_conf
            if self.global_conf:
                self.default_var=self.read_vm_conf(os.path.join(self.local, self.global_conf), self.default_var)
            
            # read vm config file if it exist
            self.vm_var=dict()
            for current_vm in self.vm_list:
                filename=os.path.join(self.local, current_vm)
                if os.path.isfile(filename):
                    self.vm_var[current_vm]=self.read_vm_conf(filename, self.default_var)
                else:
                    self.vm_var[current_vm]=self.default_var
            
        self.destination=job.get('destination', None)
        if self.destination:
            if not self.vm_list:
                 warnings['destination']='vm_list must be set to use "destination"'
                 
            self.destination=self.destination.replace('\n','')
            try:
                self.destination=Destinations(self.destination, self)
            except (DestinationSyntaxError, cron.CronException), e:
                errors['destination']='syntax error: %s' % (str(e), )

        if job.get('scp_bin', None):
            try:
                self.scp_args=quoted_string_list(job.get('scp_bin'))
            except ValueError:
                errors['scp_bin']='not a valid quoted string list'
            else:
                self.scp_bin=self.scp_args[0]
                self.scp_args[0]=os.path.basename(self.scp_args[0])
                if not os.path.isfile(self.scp_bin):
                    errors['scp_bin']='file not found'


        if not errors:

            # upload and setup files on the VMWARE server

            t=paramiko.Transport((self.host, self.port))
            t.connect(username=self.login, password=self.password)
            
            scpclient=MySCPClient(t)
            scpclient.put(self.local, self.remote_temp, recursive=True)

            self.temp_target=posixpath.join(self.remote_temp, os.path.basename(self.local))
            self.ghettovcb_bin=posixpath.join(self.temp_target, self.ghettovcb)
            self.current_vm_file=posixpath.join(self.temp_target, 'current_vm')
            self.exclude_vm_file=posixpath.join(self.temp_target, 'exclude_vm')
            
            _exit_code, _output=RunCommand(t, 'chmod +x "%s"' % self.ghettovcb_bin, log)

            self.cmd_line='cd "%s" ; ./"%s" -d debug -c "%s"' % (self.temp_target, self.ghettovcb, self.temp_target)
            
            if self.vm_list:
                self.cmd_line+=' -f "%s"' % (self.current_vm_file,)       
            else:
                self.cmd_line+=' -a'       
                if self.vm_exclude:
                    self.cmd_line+=' -e "%s"' % (self.exclude_vm_file,)       
                    for i, vm in enumerate(self.vm_exclude):
                        if i==0:
                            _exit_code, _output=RunCommand(t, 'echo "%s" > "%s"' % (vm, self.exclude_vm_file), log)
                        else:
                            _exit_code, _output=RunCommand(t, 'echo "%s" >> "%s"' % (vm, self.exclude_vm_file), log)
            if self.global_conf:
                self.cmd_line+=' -g "%s"' % (self.global_conf,)       

            extra+='cmd_line=%s\n' % self.cmd_line
            t.close()
            
            if self.scp_bin:
                self.scp_args+=[ '-P', str(self.port), '-q' ]
                if sys.platform in ('win32', ):
                    # putty allow cmd line password
                    self.scp_args+=[ '-batch', '-pw', '<password>' ]
                extra+='scp_cmd=%s %s@%s:%%source%% %%destination%%\n' % (' '.join(self.scp_args), self.login, self.host, ) 
                
        return errors, warnings, extra

    # -----------------------------------------------------------------------
    def run(self, command, job, manager):

        status=0
        name=job['name']
        
        if self.vm_list:
            vm_list=self.vm_list
        else:
            vm_list=[ None, ]
                
        for self.current_vm in vm_list:
            if self.current_vm:
                job['name']='%s.%s' % (name, self.current_vm)
                
            result=self.sub_run(command, job, manager)

            code=send_mail_report(job, result, manager)
            status=max(status, code)

        job['name']=name

        return status
        
    # -----------------------------------------------------------------------
    def sub_run(self, command, job, manager):

        log=manager.log
        now=manager.now

        start=int(time.time())

        t=paramiko.Transport((self.host, self.port))
        t.get_security_options()._set_ciphers(['blowfish-cbc', ])
        t.connect(username=self.login, password=self.password)
        t.set_keepalive(300)
        if self.current_vm:
            # create the current_vm_file
            _exit_code, _output=RunCommand(t, 'echo "%s" > "%s"' % (self.current_vm, self.current_vm_file), log)
        chan=t.open_session()
        chan.set_combine_stderr(True)
        log.info('start: %s', self.cmd_line)
        chan.exec_command(self.cmd_line)
        output, line_buf='', ''
        while True:
            x=chan.recv(1024)
            if len(x)==0:
                break
            output+=x
            line_buf=line_buf+x
            lines=line_buf.split('\n')
            line_buf=lines[-1]
            for line in lines[:-1]:
                log.debug(line)

        if line_buf:
            log.debug(line_buf)
            
        ghettovcb_exit_code=chan.recv_exit_status()
        chan.close()
        end=int(time.time())

        # search the target and make a directory listing 
        dir_lst, up_lst, df, scp_process, target=None, None, None, None, None
        if self.current_vm and self.vm_var[self.current_vm]['ENABLE_NON_PERSISTENT_NFS']!=1:
            target=posixpath.join(self.vm_var[self.current_vm]['VM_BACKUP_VOLUME'], self.current_vm)
            compress=self.vm_var[self.current_vm]['ENABLE_COMPRESSION']=='1'
            if compress:
                _exit_code, dir_lst=RunCommand(t, 'ls -l "%s"' % target, log)
                dir_lst='== directory '+target+'\n'+dir_lst
            else:
                _exit_code, up_lst=RunCommand(t, 'ls "%s"' % target, log)
                dir_lst=None
                # search the target directory, take the last one
                target_name=''
                for line in up_lst.split('\n'):
                    if line.startswith(self.current_vm):
                        if line>target_name:
                           target_name=line
                up_lst='== directory '+target+'\n'+up_lst
                if target_name: 
                    target=posixpath.join(target, target_name)
                    _exit_code, dir_lst=RunCommand(t, 'ls -l "%s"' % target, log)
                    dir_lst='== directory '+target+'\n'+dir_lst
            # if destination, move or copy backup
            if self.destination:
                self.type, self.target=self.destination.match(now, self.night_shift, variables=dict(vm=self.current_vm))
                if self.type in ('move', 'copy'):
                    
                    if not compress:
                        # create the target dir, if dont exist
                        try:
                            os.mkdir(self.target)
                        except OSError, e:
                            log.debug('creating directory %s: %s', self.target, e)
                        except:
                            log.error('creating directory %s: %s', self.target, e)
    
                    if not self.scp_bin:
                        # use paramiko
                        scpclient=MySCPClient(t, buff_size=65536)
                        log.info('scp -r %s %s', target, self.target)
                        scpclient.get(target, self.target, recursive=True)
                    else:
                        # use local SCP
                        scp_args=self.scp_args[:]
                        if not compress:
                            if sys.platform in ('win32', ):
                                scp_args.append('-unsafe')
                            #scp_args.append('-r')
                            wildcard='/*'
                        else:
                            wildcard=''
                            
                        scp_args+=[ '%s@%s:%s%s' % (self.login, self.host, target, wildcard), self.target]
                        scp_cmd=' '.join(scp_args)
                        log.info('scp_cmd=%s', scp_cmd)
                        mypass={ '<password>' : self.password}
                        scp_args=map(lambda x: mypass.get(x, x), scp_args)
                        scp_process=subprocess.Popen(scp_args, executable=self.scp_bin, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                        scp_out, scp_err=scp_process.communicate()
    
                    log.info('scp end')
                        
                        
                if self.type=='move':
                    # delete the source
                    log.info('delete %s', target)
                    _exit_code, _rm_out=RunCommand(t, 'rm -r "%s"' % target, log)
    
            # make the disk space usage
            _exit_code, df=RunCommand(t, 'df', log)

        t.close()

        if ghettovcb_exit_code==0 and (not scp_process or scp_process.returncode==0):
            backup_status='OK'
        else:
            backup_status='ERR'

        status=u''
        status+='name=%s\r\n' % job['name']
        status+='program=%s\r\n' % self.name
        status+='version=%s\r\n' % self.__version__
        status+='status=%s\r\n' % backup_status
        status+='hostname=%s\r\n' % manager.hostname
        
        status+='exit_code=%d\r\n' % ghettovcb_exit_code
        status+='start=%s\r\n' % time.ctime(start)
        status+='end=%s\r\n' % time.ctime(end)
        if target:
            status+='remote_target=%s\r\n' % target

        status+='type=%s\r\n' % self.type
        status+='target=%s\r\n' % self.target
        if self.target and scp_process:
            status+='scp_exit_code=%d\n' % scp_process.returncode
            
        status+='start_epoch=%d\r\n' % start
        status+='end_epoch=%d\r\n' % end

        attachements=[ # the type, subtype and coding is not used
                       ('output.txt', None, output, 'text', 'plain', 'utf-8'),
                    ]
        if dir_lst:
            attachements.append(('dir.txt', None, dir_lst, 'text', 'plain', 'utf-8'))

        if up_lst:
            attachements.append(('dir_up.txt', None, up_lst, 'text', 'plain', 'utf-8'))

        if df:
            attachements.append(('df.txt', None, df, 'text', 'plain', 'utf-8'),)

        if scp_process and scp_out:
            attachements.append(('scp_out.txt', None, scp_out, 'text', 'plain', 'cp850'))
        if scp_process and scp_err:
            attachements.append(('scp_err.txt', None, scp_err, 'text', 'plain', 'cp850'))

        
        return backup_status, status, attachements

