#!/bin/env python
#
# mkbackup.py
#
# (c) Alain Spineux alain.spineux@gmail.com
#
# mkbackup is a front-end for popular and free linux and windows archiver tools.
#
# Supported archiver:
# - ntbackup
# - wbadmin (next to come)
# - tar (next to come)

# Features:
# - job definitions are stored in a configuration file
# - the destination of the archive can be set to different location depending
#   on the day, the week or the month to allow very complex schema of backup
# - send and email at the end of job
# - the email contains all information to evalute the reliability of the backup jobs
#
#
# mkbackup is released under the GNU GPL license
#

import sys, os, subprocess, time, smtplib, calendar, re, socket, urllib2, logging, logging.handlers, codecs

from optparse import OptionParser
import ConfigParser

import email
from email.Utils import parseaddr, formataddr, formatdate
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
from email.MIMEBase import MIMEBase

from datetime import datetime, timedelta

import cron

__version__='0.1.6'
last_version_url='http://www.magikmon.com/download/mkbackup_last.txt'

windows_codepage='cp1252'

atom=r"[a-zA-Z0-9_#\$&'*+/=?\^`{}~|\-]+"
dot_atom=atom  +  r"(?:\."  +  atom  +  ")*"
quoted=r'"(?:\\[^\r\n]|[^\\"])*"'
local="(?:"  +  dot_atom  +  "|"  +  quoted  +  ")"
domain_lit=r"\[(?:\\\S|[\x21-\x5a\x5e-\x7e])*\]"
domain="(?:"  +  dot_atom  +  "|"  +  domain_lit  +  ")"
addr_spec=local  +  "\@"  +  domain
postfix_restricted_rfc2822_address_name=local
postfix_restricted_rfc2822_email_address=addr_spec
cyrus_mailbox_name=r"[a-zA-Z0-9_#$'=`{}~|-]+(?:\.[a-zA-Z0-9_#$'=`{}~|-]+)*"

domain_nameRE=re.compile('^'+dot_atom+'$')
email_addressRE=re.compile('^'+postfix_restricted_rfc2822_email_address+'$')
valid_ipRE=re.compile('^([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$')
valid_hostnameRE=re.compile('^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\\-]*[A-Za-z0-9])$')

boolean={ 'on':True, 'yes':True, 'true':True, '1':True, 1:True, 'off':False, 'no':False, 'false':False, '0':False, 0:False}

# ---------------------------------------------------------------------------
def update_check():
    up2date, msg='yes', ''
    try:
        data=urllib2.urlopen(last_version_url).read()
        lines=data.split('\n')
        current=__version__.split('.')
        last=lines[0].split('.')
        for i in range(min(len(current), len(last))):
            if int(current[i])<int(last[i]):
                up2date='no'
                msg='\n'.join(lines[1:])
                break
            
    except urllib2.URLError:
        up2date='unk'
    except Exception, e:
        up2date='unk'

    return up2date, msg

# ---------------------------------------------------------------------------
def check_boolean(job, errors, options):
    for option, default in options:
        value=boolean.get(job.get(option, default).lower(), None)
        if value==None:
            errors[option]='boolean must have value in (on, yes, true, 1, off, no, false and 0)'
            value=default
        job[option]=value

# ---------------------------------------------------------------------------
def check_mail_config(job, errors):
    
    new_errors={}
#    smtp_host=job.setdefault('smtp_host', '127.0.0.1')
#    smtp_port=job.setdefault('smtp_port', '25')

    try:
        job['smtp_host']=smtp_host=job['smtp_host'].encode(windows_codepage)
    except UnicodeEncodeError:
        new_errors['smtp_host']='invalid hostname or ip address'
    else:
        if not valid_ipRE.match(smtp_host):
            if not valid_hostnameRE.match(smtp_host):
                new_errors['smtp_host']='invalid hostname or ip address'
            else:
                try:
                    ip=socket.gethostbyname(smtp_host)
                except socket.gaierror:
                    new_errors['smtp_host']='cannot resolve address'

    smtp_port=job['smtp_port']
    try:
        port=int(smtp_port)
    except Exception:
        new_errors['smtp_port']='must be an integer'
    else:
        if not (0<port and port<65535):
            new_errors['smtp_port']='must be an integer between 1 and 65535'
        else:
            job['smtp_port']=port
        
    sender=job.get('sender', None)
    if not sender:
        new_errors['sender']='option mandatory'
    elif not email_addressRE.match(sender):
        new_errors['sender']='invalid email address'

    recipients=job.get('recipients')
    if not recipients:
        new_errors['recipients']='option mandatory'
    else:
        recipients=recipients.split()
        bad=[]
        for recipient in recipients:
            if not email_addressRE.match(recipient):
                bad.append(recipient)
        if bad:
            new_errors['recipients']='invalid email addresses: %s' % ' '.join(bad)
        else:
            job['recipients']=recipients

    errors.update(new_errors)
    return new_errors

# ---------------------------------------------------------------------------
def sendmail(sender, recipients, subject, text, attachements, smtp_host, smtp_port):

    msg=MIMEMultipart()
    msg.preamble='' # This line is not visible on mime enable MUA
    msg.epilogue=''
    
    msg['From'] = formataddr((sender, sender)) # Display name, email address
    msg['To'] =  ', '.join([ formataddr((recipient, recipient)) for recipient in recipients ])
    msg['Date'] = formatdate(localtime=True)
    msg['Subject'] = subject

    core=MIMEText(text.encode('utf-8'), 'plain', 'utf-8')
    msg.attach(core)
    
    for filename, target, data, maintype, subtype, charset in attachements:
        
        if not data and target and os.path.exists(target):
            data=open(target, 'rb').read()

        if data:
            maintype, subtype = 'application', 'octet-stream'
            if maintype=='text':
                attachement=MIMEText(data, subtype, charset)
            else:
                attachement=MIMEBase(maintype, subtype)
                attachement.set_payload(data) #, charset) #dont use the charset, here !
            email.Encoders.encode_base64(attachement)
    
        attachement.add_header('Content-Disposition', 'attachment', filename=filename)
        msg.attach(attachement)
    
    body=msg.as_string()
    
    smtp=smtplib.SMTP(smtp_host, smtp_port)
    ret=smtp.sendmail(sender, recipients, body)

def convert_args_into_windows_codepage(args):
    return map(lambda x: x.encode(windows_codepage), args)


# ===========================================================================
class NtBackup:
    # http://support.microsoft.com/kb/814583
    # http://support.microsoft.com/default.aspx?scid=kb;en-us;233427
    #
    # Normal - selected files, marking their archive attributes
    # Copy - selected files without marking their archive attributes. This is good for making tape copies that do not interfere with archive backups, since it does not set the archive attribute.
    # Incremental - selected files, marking their archive attributes, but only backs up the ones that have changed since the last backup.
    # Differential - selected files, NOT marking their archive attributes, but only backs up the ones that have changed since the last backup.
    # Daily - only backs up files that have changed that day, does not mark their archive attributes.
    #
    # ntbackup stuff
    #   http://www.fishbrains.com/2007/11/12/utilizing-the-built-in-windows-backup-ntbackupexe-for-windows/
    #   http://episteme.arstechnica.com/eve/forums/a/tpc/f/12009443/m/165002540831
    
    name='ntbackup'
    exe='ntbackup.exe'
    ev_logtype='Application'
    ev_source='NTBackup'
    
    boolean={ True: 'yes', False:'no'}
    
    types=dict(normal='normal', full='normal', incremental='incremental', inc='incremental', differential='differential', diff='differential', copy='copy', daily='daily')

    # -----------------------------------------------------------------------
    def run(self, command, job, log):

        errors, warning={}, {}

        up2date, up2date_msg=job['up2date'], job['up2date_msg']
        
        #
        # check job config
        #
        
        selection=job.get('selection', None)
        if not selection:
            errors['selection']='option mandatory'
        elif selection.startswith('@'):
            if not os.path.isfile(selection[1:]):
                errors['selection']='file not found: %s' % (selection[1:],)
        elif os.path.isfile(selection) and selection[-4].lower()=='.bks':
            error['selection']='looks like a ".bks" file and need to be prefixed by a @'
        elif not os.path.isdir(selection) and not os.path.isfile(selection):
            errors['selection']='file or directory not found'

        check_boolean(job, errors, [('verify', 'no'), ('restricted', 'no'), ])

        logdir_default=os.path.join(os.environ.get('USERPROFILE'), 'Local Settings\Application Data\Microsoft\Windows NT\NTBackup\data')
        log.info('logdir_default=%s', logdir_default)
        logdir=job.setdefault('logdir', logdir_default)
        if not os.path.isdir(logdir):
            errors['logdir']='directory not found'
    
        program_exe=job.setdefault(self.name, self.exe)
        if os.path.basename(program_exe)!=program_exe and not os.path.isfile(program_exe):
            errors[self.name]='file not found'
        else:
            # TODO: search in %PATH%
            pass

        destination=job.get('destination', None)
        if not destination:
            errors['destination']='option mandatory'
        else:
            destination=destination.replace('\n','')
            try:
                destination=Destinations(destination, self)
            except (DestinationSyntaxError, cron.CronException), e:
                errors['destination']='syntax error: %s' % (str(e), )

        mail_config=not check_mail_config(job, errors)

        
        msg_body=u''
        if up2date=='no':
            msg_body+='\n%s\n' %up2date_msg
            
        if errors:
            log.error('Error in section: %s', job['name'])
            for k, v in errors.iteritems():
                log.error('%s=%r %s', k, job.get(k,''), v)
                msg_body+='%s=%r\n    %s\n\n' % (k, job.get(k,''), v)
            if mail_config and not command=='check':
                subject='MKBACKUP: CONFIG ERR %s ' % (job['name'], )
                sendmail(job['sender'], job['recipients'], subject, msg_body, [], job['smtp_host'], job['smtp_port'])

            # --------->  RETURN <-----------
            return

        log.info('No error in section: %s', job['name'])

        #
        # 
        #
        
        # C:\WINDOWS\system32\ntbackup.exe backup "@m:\asx\src\magik\job1.bks" /a /d "Set created 14/11/2009 at 2:29" /v:no /r:no /rs:no /hc:off /m normal /j "backup" /l:s /f "s:\Backup.bkf"
        
        now=datetime.now()
        if command in ('check', 'checkmail'):
            msg_body+='Destinations day by day:\n\n'
            log.info('Destinations day by day:')
            for i in range(destination.max_weekdivisor*7):
                today=now+timedelta(days=i)
                typ, target=destination.match(today)
                if typ=='none':
                    ty='none'
                    target='-----------------'
                elif typ==None:
                    ty=''
                    target='??? no target ???'
                else:
                    ty=typ
                msg_body+='    %s %-12s %s\n' % (today.strftime('%a %d %b %Y'), ty, target)
                log.info('    %s %-12s %s', today.strftime('%a %d %b %Y'), ty, target)
                if typ!='none' and typ!=None and not os.path.isdir(os.path.dirname(target)):
                    msg_body+='        directory not found !!!\n'
                    log.warning('        directory not found: %s', os.path.dirname(target))

        typ, target=destination.match(now)
        target_dir=os.path.dirname(target)
        
        if typ=='none':
            cmdline='no backup today'
            log.info('no backup today')
        elif typ==None:
            cmdline='no target for today !'
            log.error('no target for today')
        else:
            args=[ program_exe, 'backup', job.get('selection'), '/J', job_name, '/M', typ, '/F', target, '/rs:no', ]
            if job.get('description', None): args.extend(['/d', job.get('description')])
            args.append('/v:'+self.boolean[job.get('verify')])
            args.append('/r:'+self.boolean[job.get('restricted')])
            args.append('/l:s') # logging [s]ummary
            # '/hc:off'
        
            cmdline=' '.join(args)
            log.info('cmdline=%s', cmdline)

        if command=='checkmail':
            msg_body+='\nCommand line:\n\n%s\n' % cmdline
            subject='MKBACKUP: CONFIG OK %s ' % (job['name'], )
            sendmail(job['sender'], job['recipients'], subject, msg_body, [], job['smtp_host'], job['smtp_port'])
            
        if command in ('check', 'checkmail'):
            return

        if typ=='none' or typ==None:
            log.info('bye')
            return
        
        pargs=convert_args_into_windows_codepage(args)
        start=int(time.time())
        process=subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        std_out, std_err=process.communicate()
        end=int(time.time())
        
        logfile, last=None, 0
        for filename in os.listdir(logdir):
            full_filename=os.path.join(logdir, filename)
            current=os.stat(full_filename).st_mtime
            if last<current:
                logfile, last=full_filename, current
                
        log.info('logfile=%s', logfile)
        
        ev_status, ev_out=ReadEvLog(NtBackup.ev_logtype, NtBackup.ev_source, start)
        
        status=u''
        status+='status=%s\r\n' % ev_status
        status+='version=%s\r\n' % __version__
        status+='returncode=%r\r\n' % process.returncode
        status+='start=%s\r\n' % time.ctime(start)
        status+='end=%s\r\n' % time.ctime(end)
        
        status+='selection=%s\r\n' % job.get('selection')
        status+='target=%s\r\n' % target
        stat=os.stat(target)
        status+='target_size=%d\r\n' % stat.st_size
        status+='target_mtime_epoch=%d\r\n' % stat.st_mtime
        status+='target_mtime=%s\r\n' % time.ctime(stat.st_mtime)
        
        #import ctypes
        #free_bytes = ctypes.c_ulonglong(0)
        #ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(target_dir), None, None, ctypes.pointer(free_bytes))
        #print free_bytes.value

        import win32file
        free_bytes, total_bytes, total_free_bytes=win32file.GetDiskFreeSpaceEx(target_dir)
        status+='target_free_space=%d\r\n' % total_free_bytes

        status+='start_epoch=%d\r\n' % start
        status+='end_epoch=%d\r\n' % end
        status+='cmdline=%s\r\n' % cmdline


        for line in status.split('\n'):
            if line:
                log.info('    %s', line)
        
        #
        # dir of target directory
        #
        if True:
            dir_out=u'  epoch   |          time          |      size     |    filename\r\n'
            for filename in os.listdir(target_dir):
                full_filename=os.path.join(target_dir, filename)
                stat=os.stat(full_filename)
                if os.path.isdir(full_filename):
                    size='<DIR>'
                else:
                    size='%15d' % stat.st_size 
                dir_out+='%d %s %15s %s\r\n' % (stat.st_mtime, time.ctime(stat.st_mtime), size, filename)
                if last<current:
                    logfile, last=full_filename, current
        else: # OLD code, but use CP850 in my XP FR_en
            args=[ os.environ['comspec'], '/u', '/c', 'dir', target_dir ]
            pargs=convert_args_into_windows_codepage(args)
            process=subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            dir_out, err=process.communicate()

        attachements=[ # the type, subtype and coding is not used
                       ('log.txt', logfile, None, 'text', 'plain', 'utf-16'),
                       ('dir.txt', None, dir_out.encode('utf-8'), 'text', 'plain', 'utf-8'),
                       ('evlog.txt', None, ev_out.encode('utf-8'), 'text', 'plain', 'utf-8'), 
                       ('config.ini', job['config'], None, 'text', 'plain', windows_codepage),
                       ('status.txt', None, status.encode('utf-8'), 'text', 'plain', 'utf-8'),
                    ]
        if job['selection'].startswith('@'):
            attachements.append(('selection.bks', job['selection'][1:], None, 'text', 'plain', 'utf-16'))
        if std_out:
            attachements.append(('stdout.txt', None, std_out, 'text', 'plain', 'cp850'))
        if std_err:
            attachements.append(('stderr.txt', None, std_err, 'text', 'plain', 'cp850'))
        
        msg_body+='\n'+status
        subject='MKBACKUP BACKUP %s %s' % (ev_status, job['name'])
        sendmail(job['sender'], job['recipients'], subject, msg_body, attachements, job['smtp_host'], job['smtp_port'])
        
archivers=dict((x.name, x) for x in (NtBackup, ))


#======================================================================
#
# Event Viewer logging
#
#======================================================================
import win32evtlog
import win32evtlogutil
import winerror
import win32con

evt_dict={
        win32con.EVENTLOG_AUDIT_FAILURE:'AUDIT_FAILURE',
        win32con.EVENTLOG_AUDIT_SUCCESS:'AUDIT_SUCCESS',
        win32con.EVENTLOG_INFORMATION_TYPE:'INF',
        win32con.EVENTLOG_WARNING_TYPE:'WAR',
        win32con.EVENTLOG_ERROR_TYPE:'ERR'
        }

def FormatEv(ev_obj, logtype):
    computer=str(ev_obj.ComputerName)
    # cat=str(ev_obj.EventCategory)
    level=str(ev_obj.EventType )
    record=str(ev_obj.RecordNumber)
    evt_id=str(winerror.HRESULT_CODE(ev_obj.EventID))
    evt_type=evt_dict.get(ev_obj.EventType, 'UNK')
    msg=win32evtlogutil.SafeFormatMessage(ev_obj, logtype)
    epoch=int(ev_obj.TimeGenerated)
    msg=u'======== eventid=%d eventtype=%s epoch=%d time="%s" =====\r\n%s' % ( ev_obj.EventID, evt_type, epoch, time.ctime(epoch), msg)
    #print ev_obj.EventID, evt_type, int(ev_obj.TimeGenerated), level, msg.encode('UTF-8')
    return msg 

def ReadEvLog(logtype, source, start, end=None):
    # If any event is not 'INFO' => return 'ERR'
    # If the last event is not an 'INFO' => return 'WAR' (in fact when not event are present)
    # Else return 'OK'
    flags = win32evtlog.EVENTLOG_BACKWARDS_READ|win32evtlog.EVENTLOG_SEQUENTIAL_READ
    hand=win32evtlog.OpenEventLog(None, logtype) # None for localhost
    cont, output, status='first', u'', 'WAR'
    while cont:
        events=win32evtlog.ReadEventLog(hand,flags,0)
        for ev_obj in events:
            src=str(ev_obj.SourceName)
            if src!=source:
                break
            if int(ev_obj.TimeGenerated)<start:
                cont=False
                break
            if ev_obj.EventType!=win32con.EVENTLOG_INFORMATION_TYPE:
                status='ERR'

            if cont=='first':
                #print 'first %r %r %r %r' % (ev_obj.EventID=='8001', ev_obj.EventType==win32con.EVENTLOG_INFORMATION_TYPE, ev_obj.EventID, ev_obj.EventType)
                cont=True
                if ev_obj.EventID==8019 and ev_obj.EventType==win32con.EVENTLOG_INFORMATION_TYPE:
                    status='OK'
                
            if output:
                output=FormatEv(ev_obj, logtype)+u'\r\n'+output
            else:
            	output=FormatEv(ev_obj, logtype)

        cont=cont and events
    win32evtlog.CloseEventLog(hand)
    return status, output

#======================================================================
#
# Destinations
#
#======================================================================
class DestinationSyntaxError(Exception):
    pass

class Destinations:
    def __init__(self, raw, archiver):
        
        self.destinations=[]
        st=raw
        self.max_weekdivisor=1
        
        while st:
            if st[0]=='<':
                try:
                    pos=st.index('>')
                except ValueError:
                    raise DestinationSyntaxError, 'a ">" is missing'
                
                selector=st[1:pos]
                st=st[pos+1:]
            
                pos=st.find('<')
                if pos>0:
                    target=st[:pos]
                    st=st[pos:]
                else:
                    target=st
                    st=''
                
                typ, period=selector.split('=', 1)
                
                try:
                    if typ.lower()=='none':
                        typ='none'
                    else:
                        typ=archiver.types[typ.lower()]
                except KeyError:
                    raise DestinationSyntaxError, 'type "%s" unknow' % (typ, )

                #print "SELECTOR<%s=%s>%s" % (typ,period,target )
                #print "CARRY=%s" % st
                period=cron.Cron(period)
                if period.weekdivisor!=None:
                    self.max_weekdivisor=max(self.max_weekdivisor, period.weekdivisor)
            else:
                typ='full'
                period=None
                target=st
                st=''
                
            self.destinations.append((typ, period, target))


    def match(self, day):
        #print '---------------------------------'
        globals=dict(day=day.day, month=day.month, year=day.year, weekday=day.weekday, yearday=day.timetuple().tm_yday)
        for typ, period, target in self.destinations:
            #print typ, period, target
            if period:
                #print 'Month', day.month-1, period.months
                if period.months and not day.month-1 in period.months:
                    continue
                #print 'Day of week', day.isoweekday()-1, period.daysofweek
                if period.daysofweek and not day.isoweekday()-1 in period.daysofweek:
                    continue
                #print 'Day of month', day.day-1, period.daysofmonth
                if period.daysofmonth and not day.day-1 in period.daysofmonth:
                    continue
                if period.firstdayofweek!=None:
                    epoch=calendar.timegm(day.timetuple())
                    nday=int(epoch/86400)-period.firstdayofweek+3
                    week=int(nday/7)%period.weekdivisor
                    if period.weekselector!=None and week!=period.weekselector:
                        continue
                    globals['epoch']=epoch
                    globals['week']=week

            #print 'MATCH', typ, target
            
            for exp in re.findall('\{([^}]*)\}',target):
                target=target.replace('${%s}' % (exp,), str(eval(exp, globals, {})))
            target=target.encode('utf-8')
            target=day.strftime(target)
            target=target.decode('utf-8')

            return typ, target
            
        return None, None

#======================================================================
#
# Main
#
#======================================================================

#
# Parse the command line
#
parser=parser=OptionParser(version='%%prog %s' % __version__ )
parser.set_usage=('%prog [options] command [job..]\n'+
                  '\tcommand in "backup", "check", "checkmail"')

parser.add_option("-c", "--config", dest="config", default='mkbackup.ini', help="use another ini file", metavar="config.ini")
parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False, help="switch to debug level")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="write log to the terminal")
parser.add_option("-l", "--logfile", dest="logfile", default='mkbackup.log', help="log to this file", metavar="mkbackup.log")

cmd_options, cmd_args=parser.parse_args(sys.argv)

print 'enable logging in %s' % cmd_options.logfile
root_handler=logging.handlers.RotatingFileHandler(cmd_options.logfile, 'a', 1024**2, 2)
root_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)-3.3s %(message)s', '%Y-%m-%d %H:%M:%S'))
log=logging.getLogger()
log.setLevel(logging.INFO)
log.addHandler(root_handler)

if cmd_options.debug:
    log.setLevel(logging.DEBUG)
else:
    # this stop early the logging process of loglevel<=DEBUG and should 
    # improve performance a little bit
    logging.disable(logging.DEBUG)

if cmd_options.verbose:
   console=logging.StreamHandler()
   console.setFormatter(logging.Formatter('%(asctime)s,%(msecs)03d %(levelname)-3.3s %(message)s', '%H:%M:%S'))
   log.addHandler(console)

print 'logging enabled'
log.info('start %r', sys.argv)

if len(cmd_args)<2:
    parser.error('no "command" set')
    
command=cmd_args[1]
job_list=cmd_args[2:]

if not job_list:
    parser.error('no "job"')

if command not in ('backup', 'check', 'checkmail'):
    parser.error('invalid command "%s"' % command)
#
# load the configuration file
#
config_default=dict(verify='no',
                    update_check='yes',
                    smtp_port='25',
                    smtp_host='127.0.0.1',
                    )

config=ConfigParser.RawConfigParser(config_default)
try:
    config.readfp(codecs.open(cmd_options.config, 'r', windows_codepage))
except IOError:
    log.exception('error reading configuration file %s', cmd_options.config)
    sys.exit(1)

up2date, up2date_msg='unk', ''
check_for_update=config.get('DEFAULT', 'update_check')
if boolean.get(check_for_update.lower(), None)!=False:
    up2date, up2date_msg=update_check()
    if up2date=='no':
        log.warning('!'*60)
        for line in up2date_msg.split('\n'):
            if line:
                log.warning(line)
        log.warning('!'*60)

err=0
for job_name in job_list:
    try:
        program=config.get(job_name, 'program')
    except ConfigParser.NoOptionError, e:
        log.error('no "program" set for job "%s"',job_name)
        err+=1
    except ConfigParser.NoSectionError, e:
        log.error('job not found: %s',job_name)
        err+=1
    else:
        if not program in archivers:
            log.error('program "%s" unknow in job "%s"', program, job_name)
            err+=1

if err>0:
    sys.exit(3)

for job_name in job_list:

    job=dict(config.items(job_name))
    program=job.get('program')
    archiver=archivers[program]()
    job=dict(config.items(job_name))
    job['name']=job_name
    job['config']=cmd_options.config
    job['up2date']=up2date
    job['up2date_msg']=up2date_msg

    log.info('start command=%s job=%s archiver=%s', command, job_name, program)
    try:
        archiver.run(command, job, log)
    except:
        log.exception('running command=%s job=%s archiver=%s', command, job_name, program)
    else:
        log.info('end command=%s job=%s archiver=%s', command, job_name, program)


#<full,first su>S:\monthly\data-full-month-${month%2}.bkf
#<full,su>S:\week${week%4}\data-full-%a.bkf
#<inc,mo-sa>S:\week${week%4}\data-inc-%a.bkf



#<full,week%4=0>S:\data-full.bkf
#<inc,week%4=1>S:\data-inc.bkf

#
#w1/4
#jan/2,su
#feb-dec/2


# when start a week
# firstday=sat
#last day OF THE MONTH
#1st mon,2nd tue,3th wed,4th thu,last sat  OF THE MONTH
#1st week,2nd week,3th week,4th week,last week OF THE MONTH
#<> => empty selector to allow "no backup"
# empty target => no backup



# http://www.scalabium.com/faq/dct0082.htm
# To detect the code page of Windows operation system you must call the GetACP function from Windows API.
# If you needs to read the code page of "DOS" sessions, you must call the GetOEMCP function from Windows API. This function will return the value:
# http://msdn.microsoft.com/en-us/library/dd318070%28VS.85%29.aspx

#


# windows unicode file http://www.helpware.net/FAR/help/Unicode2.htm
# Universal Encoding Detector http://chardet.feedparser.org/docs/supported-encodings.html


# wbadmin
#
# How to use wbadmin to backup in Windows Vista 2008
# http://www.howtonetworking.com/Windows/wbadmin1.htm
#
# Backup Version and Space Management in Windows Server Backup
# https://blogs.technet.com/filecab/archive/2009/06/22/backup-version-and-space-management-in-windows-server-backup.aspx
#
# Customizing Windows Server Backup Schedule
# http://blogs.technet.com/filecab/archive/2009/04/13/customizing-windows-server-backup-schedule.aspx
#
# %windir%\logs\windowsserverbackup
#
# Output of command "Vshadow -wm2" ran on an elevated command prompt
#
# Deleting System State Backup:
# wbadmin delete systemstatebackup -deleteOldest
# wbadmin delete systemstatebackup -keepversions:10   
# The above command will keep the latest 10 versions and delete the rest all the system state backups.
#
# Deleting other backups:  Backup application stores multiple backup versions in the VSS shadow copies. Hence, older backup version can be deleted by deleting older shadow copy. Commands to list and delete VSS shadow copies are below. They need to be run in an elevated command window.
# vssadmin list shadows /for=x: ? for listing the snapshots on x: where x: is the backup location
# vssadmin delete shadows /for=x:  /oldest ? for delete the oldest shadow copy. It can be called multiple times in case there is need to delete multiple older backups.
# Note: wbadmin get versions or backup UI would still report deleted backups until next backup runs. At end of each backup ? non-existent backups are removed from the backup catalog.
#
# Win2k8: the backups are stored inside the <drive_letter>:\WindowsImageBackup\<machineName>\SystemStateBackup
# Win2k8 R2: the backups are stored inside the <drive_letter>:\WindowsImageBackup\<machineName>\BackupSet<..>\ path

