#!/usr/bin/env python2.7
import sys, os, re, argparse, subprocess, math, datetime
from ucscGb.encode import track, mkChangeNotes, encodeUtils
from ucscGb.qa.encode import tableCheck as qa
from ucscGb.gbData import ra, ucscUtils
#from ucsc.gbData import ucscUtils

def getMethods(qaDir, args, user, d):
    cmd = "curl -X GET http://genomewiki.soe.ucsc.edu/genecats/index.php/ENCODE_QA 2>/dev/null"
    p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
    output = p.stdout.read().split("\n")
    r1 = re.compile('<li class="toclevel.*="tocnumber">(\S+)<.*="toctext">(.*)<\/span>')
    r2 = re.compile('&amp;')
    if os.path.exists(qaDir + '/methods'):
        return

    f = open(qaDir + '/methods', 'w')
    f.write(args.composite + "\n")
    f.write(args.database + "\n")
    f.write("Release " + args.release + "\n")
    f.write("Redmine #" + args.redmine + "\n")
    f.write(str(d) + "\n")
    f.write(user + "\n")
    f.write("\n")
    for line in output:

        m1 = r1.match(line)

        if m1:
            tabs = ""
            for char in m1.group(1):
                if char == ".":
                    tabs = tabs + "    "
            if tabs == "":
                f.write("\n")
            m2 = r2.search(line)
            second = r2.sub('&', m1.group(2))
            f.write("%s%s %s\n" % (tabs, m1.group(1), second))
    return

def writeClaimMail(args, user, qaDir):
    f = open(qaDir + '/claimMail', 'w')
    mail = """Hi Cricket,

I am claiming the %s (Release %s) %s track.
Please let me know if there is another track you'd like me to work on instead.
http://redmine.soe.ucsc.edu/issues/%s

%s
""" % (args.composite, args.release, args.database, args.redmine, user)

    f.write(mail)
    f.close()
    return

def makeLinks(c, args, qaDir):

    notesfile = c._notesDirectory + "%s.release%s.notes" % (args.composite, args.release)
    downloads = c.downloadsDirectory + "release%s" % args.release

    if not os.path.exists(notesfile):
        return ("""
FATAL ERROR: Notes file for %s release %s doesn't exist in %s
You may need to 'git pull' or tell the wrangler to check in notes file
""" % (args.composite, args.release, c._notesDirectory))
    if os.path.exists(qaDir + "/notes.file"):
        os.remove(qaDir + "/notes.file")
    os.symlink(notesfile, qaDir + "/notes.file")

    if os.path.exists(qaDir + "/downloads"):
        os.remove(qaDir + "/downloads")
    os.symlink(downloads, qaDir + "/downloads")

    return notesfile

def parseNotes(lines):
    tables = set()
    gbdbs = set()
    files = set()
    supplemental = set()
    others = set()
    size = ""
    switch = 0
    p1 = re.compile('New Tables')
    p2 = re.compile('New Download Files')
    p3 = re.compile('New Gbdb Files')
    p4 = re.compile('New Supplemental Files')
    p5 = re.compile('New Other Files')
    s = re.compile('Total: (\d+) MB')
    e = re.compile('^$')
    for i in lines:
        if p1.search(i):
            switch = 1
            continue
        if p2.search(i):
            switch = 2
            continue
        if p3.search(i):
            switch = 3
            continue
        if p4.search(i):
            switch = 4
            continue
        if p5.search(i):
            switch = 5
            continue

        sm = s.match(i)
        if sm:
            size = sm.group(1)

        if e.match(i):
            switch = 0
            #print "empty line"

        if switch == 1:
            tables.add(i)
        if switch == 2:
            files.add(i)
        if switch == 3:
            gbdbs.add(i)
        if switch == 4:
            supplemental.add(i)
        if switch == 5:
            others.add(i)

    return tables, gbdbs, files, supplemental, others, str(size)

def writeTableMail(tables, args, user, qaDir):
    tablestr = ""
    if tables:
        sep = ""
        tablestr = sep.join(list(sorted(tables)))
        mail = """Hi Pushers,

For the *%s* database (2 part request):

1) Please push trackDb and friends

2) Please push these %s tables:

%s

from mysqlbeta -> mysqlrr

Reason: releasing ENCODE %s track on %s to the RR http://redmine.soe.ucsc.edu/issues/%s

Thank you!

%s
""" % (args.database, len(tables), tablestr, args.composite, args.database, args.redmine, user)
    else:
        mail = """Hi Pushers,

For the *%s* database:

1) Please push trackDb and friends

from mysqlbeta -> mysqlrr

Reason: releasing _____(fill in metadata or trackDb changes)____ for ENCODE %s track on %s to the RR

Thank you!

%s
""" % (args.database, args.composite, args.database, user)

    f = open(qaDir + "/pushTableMail", "w")
    f.write(mail)
    f.close

    if not tablestr:
        return ""

    f = open(qaDir + "/newTables", "w")
    f.write(tablestr)
    f.close

    return tablestr

def writeGbdbMail(gbdbs, args, user, qaDir):
    sep = ""
    gbdbstr = sep.join(list(sorted(gbdbs)))
    mail = """
Hi Pushers,

Please push these %s gbdb files:

%s

\tfrom hgwdev --> hgnfs1

Reason: staging %s %s track on beta, http://redmine.soe.ucsc.edu/issues/%s

Thank You!

%s
""" % (len(gbdbs), gbdbstr, args.composite, args.database, args.redmine, user)
    f = open(qaDir + "/pushGbdbsMail", "w")
    f.write(mail)
    f.close()
    return gbdbstr

def writeFileMail(files, others, args, user, qaDir, c, path, notesObj):
    sep = ""
    filestr = sep.join(list(sorted(files | others)))
    filesize = ucscUtils.makeFileSizes(notesObj.pushFiles, notesObj.releasePath)
    mail = """Hi Pushers,

Please push:

%s/*

from hgwdev to hgdownload:

%s

excluding md5sum.history

The total size of the new files should be: %s MB

Please note the destination on hgdownload is *one directory above the location on dev*

Reason: releasing ENCODE %s on %s to the RR http://redmine.soe.ucsc.edu/issues/%s

Thanks!

%s
""" % (path, c._rrHttpDir, filesize, args.composite, args.database, args.redmine, user)

    f = open(qaDir + "/pushFilesMail", "w")
    f.write(mail)
    f.close()
    subIds = dict()
    mdb = c.alphaMetaDb
    p = re.compile('.*(wgE.*)')
    for i in files:
        m = p.match(i)
        file = m.group(1)
        filestanza = mdb.filter(lambda s: re.match(".*%s.*" % file, s['fileName']), lambda s: s)
        if filestanza:
            for j in filestanza:
                if 'subId' in j:
                    if not j['subId'] in subIds:
                        subIds[j['subId']] = 1

    f = open(qaDir + "/subIds", "w")
    for i in subIds.keys():
        f.write(i + "\n")
    f.close()

    return filestr, set(sorted(subIds.keys()))

def writeHtml(args, c, qaDir):
    f = open(c._trackDbPath, "r")
    lines = f.readlines()
    f.close
    short = ""
    long = ""
    for i in lines:
        m = re.match('((long|short)Label)\s+(.*)', i)

        if m:
            if m.group(1) == 'longLabel':
                long = m.group(3)
            if m.group(1) == 'shortLabel':
                short = m.group(3)

    html = """<TR>
    <TD><A HREF="http://hgdownload.soe.ucsc.edu/goldenPath/%s/encodeDCC/%s/"
    TARGET=_BLANK>%s</A></TD>
    <TD>%s</TD>
</TR>""" % (args.database, args.composite, short, long)

    f = open(qaDir + "/htmlDownloadSnippet", "w")
    f.write(html)
    f.close()
    return short, long

def writeScriptOut(qaDir, output):
    f = open(qaDir + "/script.output", "w")
    sep = "\n"
    outstr = sep.join(output)
    f.write(outstr)
    f.close()
    return

def lastInitDate(sqlfile):
    if not os.path.exists(sqlfile):
        return 0
    else:
        f = open(sqlfile, "r")
        lines = f.readlines()
        f.close()
        grab = 0
        for i in lines:
            if grab == 1:
                return i.rstrip()
            if re.match('initdate={', i):
                grab = 1

def writeBetaMdb(mdb, qaDir):

    f = open(qaDir + "/beta.mdb.ra", "w")
    for i in sorted(mdb):
        for j in mdb[i]:
            f.write("%s %s\n" % (j, mdb[i][j]))
        f.write("\n")

    f.close()

def writeSql(tablestr, filestr, gbdbstr, d, short, long, args, notes, size, user, qaDir, initDate):
    newYN = "Y"
    track = short
    releaseLog = long
    if int(args.release) > 1:
        newYN = "N"
        releaseLog = "%s (Release %s)" % (long, args.release)
        track = "%s (Release %s)" % (short, args.release)

    sql = """tbls={
%s
}
files={
%s
}
sizeMB={
%s
}
currLoc={
hgwbeta
}
reviewer={
%s
}
initdate={
%s
}
lastdate={
%s
}
track={
%s
}
releaseLog={
%s
}
priority={
L
}
dbs={
%s
}
newYN={
%s
}
rank={
0
}
ndxYN={
Y
}
joinerYN={
X
}
makeDocYN={
X
}
openIssues={
http://redmine.soe.ucsc.edu/issues/%s
}
bounces={
0
}
notes={
%s
}
releaseLogUrl={
../../cgi-bin/hgTrackUi?db=%s&g=%s
}
pushState={
D
}""" % (tablestr, filestr + "\n" + gbdbstr, size, user, str(initDate), str(d), track, releaseLog, args.database, newYN, args.redmine, notes, args.database, args.composite)
    f = open(qaDir + "/release.sql", "w")
    f.write(sql)
    f.close()
    return

def writePushHtmlMail(args, user, qaDir):

    downloads = 'downloads.html'
    if re.match('mm\S+', args.database):
        downloads = 'downloadsMouse.html'

    mail = """Hi Pushers,

Please push the following file:

/usr/local/apache/htdocs/ENCODE/%s

from hgwbeta --> RR.
Reason: added the newly released ENCODE %s %s

Thanks!

%s
""" % (downloads, args.composite, args.database, user)
    f = open(qaDir + "/pushHtmlMail", "w")
    f.write(mail)
    f.close()
    return

def writeCheckList(files, filename, qaDir):
    delim = "\n"
    filelist = delim.join(list(files))
    f = open(qaDir + "/%s" % filename, "w")
    f.write(filelist)
    f.close
    return

def changeStatus(subIds):
    errors = list()
    for i in subIds:
        cmd = "encodeStatus.pl %s reviewing 2>&1" % (i)
        p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
        output = p.stdout.read()
        lines = output.split("\n")[0]
        if lines:
            if re.search("ERROR.*reviewing.*displayed", lines) or re.search("ERROR.*reviewing.*loaded", lines):
                errors.append(i)
    if errors:
        print "The following subIds had errors while setting to 'reviewing':"
        for i in errors:
            print i
        print "\n\n\nFATAL ERROR: SubIds are not set correctly to proceed with QA, please ask your wrangler to properly set the subIds to approved\n\n\n"
        sys.exit()
    return

def checkWithMkNotes(ourset, mkNotesSet, status):
    compset = set()
    errors = []
    for i in ourset:
        compset.add(i.rstrip())
    if mkNotesSet != compset:
        errors.append("""
FATAL ERROR: The list of %s in the notes file is out of date.
You may need to do a 'git pull'.
""" % status)
    return errors


def main():
    parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description='Initializes QA directory for claiming a release',
    epilog=
"""Example:
encodeQaInit hg19 wgEncodeSydhTfbs 1 69
encodeQaInit hg18 wgEncodeHudsonalphaChipSeq 3 504
"""
    )
    parser.add_argument('database', help='The database, typically hg19 or mm9')
    parser.add_argument('composite', help='The composite name, wgEncodeCshlLongRnaSeq for instance')
    parser.add_argument('release', help='The new release to be released')
    parser.add_argument('redmine', help='The Redmine issue number')
    parser.add_argument('-t', '--test', action="store_true", default=0, help="Test mode:doesn't change status to reviewing, outputs to test qa Directory")
    parser.add_argument('-m', '--mdb', default=0, help='use a different mdb composite name')



    #if len(sys.argv) == 1:
    #    parser.print_help()
    #    return

    args = parser.parse_args()

    releaseOld = 'solo'
    if not args.release.isdigit():
        parser.print_help()
    if int(args.release) > 1:
        releaseOld = int(args.release) - 1
    specialMdb = None
    if args.mdb:
        specialMdb = args.mdb

    mkNotesArgs = { 'database': args.database,
                    'composite': args.composite,
                    'releaseNew': args.release,
                    'releaseOld': str(releaseOld),
                    'ignore' : 1,
                    'loose': 1,
                    'verbose': 1,
                    'summary': 0,
                    'specialMdb': specialMdb}
    errors = []
    c = track.CompositeTrack(args.database, args.composite, None, specialMdb)
    qaDir = ""
    if args.test:
        qaDir = c.qaInitDirTest + "release%s" % args.release
    else:
        qaDir = c.qaInitDir + "release%s" % args.release
    d = datetime.date.today()
    user = os.environ['USER']

    if not os.path.exists(qaDir):
        os.makedirs(qaDir)

    print "Making QA directory: %s" % qaDir
    notesFile = makeLinks(c, args, qaDir)


    print "Getting methods"
    getMethods(qaDir, args, user, d)

    print "Checking notes file"
    if re.search('FATAL ERROR', notesFile):
        print notesFile
        return
    m = re.match('.*(kent/.*)', notesFile)
    notes = m.group(1)
    f = open(notesFile, "r")
    lines = f.readlines()
    if not re.match('mkChangeNotes v2', lines[0]):
        print "notes files is not the correct version"
        return

    writeClaimMail(args, user, qaDir)
    genNotes = mkChangeNotes.makeNotes(mkNotesArgs)
    if genNotes.errors:
        print "NON-FATAL ERROR: Local mkChangeNotes found errors in the release"
    #   return

    betaMdb = genNotes.newMdb
    if args.composite not in c.alphaMetaDb:
        print "FATAL ERROR: There is no composite level stanza in the alpha metaDb."
        return
    betaMdb[args.composite] = c.alphaMetaDb[args.composite]
    writeBetaMdb(betaMdb, qaDir)


    print "Parsing notes file"
    (tables, gbdbs, files, supplemental, others, size) = parseNotes(lines)
    print "Comparing notes file to local notes"
    errors.extend(checkWithMkNotes(tables, genNotes.newTables, "new tables"))
    errors.extend(checkWithMkNotes(gbdbs, genNotes.newGbdbs, "new gbdbs"))
    errors.extend(checkWithMkNotes(files, genNotes.newFiles, "new download files"))
    errors.extend(checkWithMkNotes(supplemental, genNotes.newSupplemental, "new supplemental files"))
    errors.extend(checkWithMkNotes(others, genNotes.newOthers, "new other files"))

    if errors:
        for i in errors:
            print i
        return

    print "Writing mails and friends"
    tablestr = writeTableMail(tables, args, user, qaDir)
    gbdbstr = writeGbdbMail(gbdbs, args, user, qaDir)
    (filestr, subIds) = writeFileMail(files, supplemental | others, args, user, qaDir, c, genNotes.releasePath, genNotes)
    (short, long) = writeHtml(args, c, qaDir)

    writePushHtmlMail(args, user, qaDir)
    writeCheckList(genNotes.pushFiles, "checkPushFilesList", qaDir)
    writeCheckList(genNotes.fullFiles, "fullFilesListNoRevoked", qaDir)
    writeCheckList(genNotes.fullTables, "allTables", qaDir)

    initDate = lastInitDate(qaDir + "/release.sql")
    if not initDate:
        initDate = d
    writeSql(tablestr, filestr, gbdbstr, d, short, long, args, notes, size, user, qaDir, initDate)

    if not args.test:
        print "Changing subIds' statuses to reviewing"
        changeStatus(subIds)

    output = []

    tables = genNotes.newTables

    if tables:

        print "Running qa tests"
        output.append("Checking for missing table descriptions:")
        (tableDescOutput, noDescription) = qa.checkTableDescriptions(args.database, tables)
        output.extend(tableDescOutput)

        output.append("Checking for missing table indexes:")
        (tableIndexOut, missingIndex) = qa.checkTableIndex(args.database, tables)
        output.extend(tableIndexOut)

        output.append("Checking table names for errors:")
        (tableNameOut, badTableNames) = qa.checkTableName(tables)
        output.extend(tableNameOut)

        trackDbPath = c.currentTrackDb
        if trackDbPath:
            output.append("Checking short and long labels in trackDb:")
            (labelOut, badLabels) = qa.checkLabels(trackDbPath)
            output.extend(labelOut)
        else:
            print "Could not determine track Db to use, does the composite have the alpha tag in trackDb.wgEncode.ra?"

        output.append("Checking tables for coordinate errors:")
        (coordsOut, badCoords) = qa.checkTableCoords(args.database, tables)
        output.extend(coordsOut)

        output.append("Checking tables for positional errors:")
        (posOut, badPos) = qa.positionalTblCheck(args.database, tables)
        output.extend(posOut)

        output.append("Counts per chromosome:")
        output.append("")
        (countChromOut, tableCounts) = qa.countPerChrom(args.database, tables)
        output.extend(countChromOut)

        writeScriptOut(qaDir, output)
    else:
        print "No new tables, skipping qaTests."

    print "Done!"


if __name__ == "__main__":
    main()
