#!/usr/bin/env python
"""
@file    buildHTMLDocs.py
@author  Daniel Krajzewicz
@author  Michael Behrisch
@date    2011-10-20
@version $Id: buildHTMLDocs.py 18096 2015-03-17 09:50:59Z behrisch $

Converts wiki-documentation into HTML pages.

Determines what to convert, first: if a command line argument is given,
it is interpreted as the page to convert. Otherwise, "Special:AllPages" is
downloaded and parsed for obtaining the list of all pages which will be
converted in subsequent steps.

For each of the pages to convert, the HTML-representation of the
page is downloaded and stripped from wiki-header/footer, first.
Then, the image-links are extracted from the HTML page and stored 
temporarily, the links themselves are patched to point to local pages/images
(if the page behind the link exists).
The page is saved into options.mirror/<PAGE_PATH>. 

After parsing all pages, the images are downloaded and stored into 
options.mirror/images.

After downloading all data, the title page is extracted and the content
included in this page is extracted. This content is embedded into "index.html"
between the <!-- nav begins --> / <!-- nav ends --> markers. 
All pages downloaded earlier are loaded, and embedded into the index.html
between the <!-- content begins --> / <!-- content ends --> markers. Then,
the page is saved into options.output/<PAGE_PATH>. All images are copied
from options.mirror/images to options.output/images.

SUMO, Simulation of Urban MObility; see http://sumo.dlr.de/
Copyright (C) 2011-2015 DLR (http://www.dlr.de/) and contributors

This file is part of SUMO.
SUMO is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.
"""
import urllib
import os
import sys
import shutil
import datetime
from optparse import OptionParser

from mirrorWiki import readParsePage, readParseEditPage


def patchLinks(page, name):
    images = set()
    level = len(name.split("/")) - 1
    level = "../" * level
    b = page.find("<a href")
    while b >= 0:
        # images/files
        if page[b + 9:].startswith("/wiki/File:") or page[b + 9:].startswith("/wiki/Image:"):
            b2 = b
            b = page.find(":", b) + 1
            images.add(page[b2 + 9:page.find("\"", b)])
            page = page[:b2 + 9] + level + "images/" + page[b:]
        # pages (HTML)
        elif page[b + 9:].startswith("/wiki/"):
            e = b + 9 + 6
            e2 = page.find("\"", b + 9)
            link = page[e:e2]
            if link.find("action=edit") < 0:
                if link.find("#") > 0:
                    link = level + link.replace("#", ".html#")
                elif link.find("#") < 0 and not (link.endswith(".png") or link.endswith(".jpg") or link.endswith(".svg")):
                    link = level + link + ".html"
                page = page[:b + 9] + link + page[e2:]
            else:
                page = page[:b + 9] + "http://sourceforge.net/" + page[b + 10:]
        b = page.find("<a href", b + 1)
    return page, images, level


def patchImages(page, name):
    images = set()
    level = len(name.split("/")) - 1
    level = "../" * level
    b = page.find("<img ")
    b = page.find("src", b)
    while b >= 0:
        b += 5
        e = page.find("\"", b + 2)
        add = page[b:e]
        l = add[add.rfind("/"):]
        if add.find("thumb") >= 0:
            l = l[l.find("-") + 1:]
        images.add(add)
        page = page[:b] + level + "images/" + l + page[e:]
        b = page.find("<img", b + 1)
        b = page.find("src", b)
    page = page.replace(".svg.png", ".svg")
    return page, images


def parseWikiLink(l):
    if l.find("[[") >= 0:
        # internal link
        b = l.find("[") + 2
        e = l.find("]", b)
        t = l[b:e]
        if t.find("|") < 0:
            link = t
            text = t
        else:
            link, text = t.split("|")
        link = link.replace(" ", "_")
        if link.find("#") >= 0:
            link = link.replace("#", ".html#")
        else:
            link = link + ".html"
        # external link
    elif l.find("[") >= 0:
        b = l.find("[") + 1
        e = l.find("]", b)
        t = l[b:e]
        link = t[:t.find(" ")]
        text = t[t.find(" ") + 1:]
    else:
        # text
        text = l[l.find(" ") + 1:]
        link = ""
    return text, link

optParser = OptionParser()
optParser.add_option("-m", "--mirror", default="mirror", help="mirror folder")
optParser.add_option("-o", "--output", default="docs", help="output folder")
optParser.add_option("-i", "--index", default=os.path.join(os.path.dirname(
    __file__), "..", "..", "docs", "wiki", "index.html"), help="index template file")
optParser.add_option("-r", "--version", help="add version info")
(options, args) = optParser.parse_args()

try:
    os.mkdir(options.mirror)
except:
    pass
try:
    os.mkdir(options.mirror + "/images")
except:
    pass
images = set()
if len(args) == 0:
    p = readParsePage("Special:AllPages")
    p = p[p.find("<input type=\"submit\" value=\"Go\" />"):]
    p = p[p.find("<table "):]
    pages = p.split("<a ")
else:
    pages = ['href="/wiki/%s"' % a for a in args]
for p in pages:
    if not p.startswith("href"):
        continue
    b = p.find("/wiki/")
    e = p.find("\"", b)
    name = p[b + 6:e]
    if name.endswith(".css"):
        print "Skipping css-file %s" % name
        continue
    print "Fetching %s" % name
    c = readParsePage(name)
    if name.find("/") > 0:
        try:
            os.makedirs(os.path.join(options.mirror, name[:name.rfind("/")]))
        except:
            pass
    if True:  # name.find(".")<0:
        c, pi, level = patchLinks(c, name)
        for i in pi:
            images.add(i)
        c, pi = patchImages(c, name)
        for i in pi:
            images.add(i)
    name = name + ".html"
    fd = open(os.path.join(options.mirror, name), "w")
    fd.write(c)
    fd.close()

imageFiles = []
for i in images:
    print "Fetching image %s" % i
    if i.find(":") >= 0:
        f = urllib.urlopen("http://sumo.dlr.de%s" % i)
        c = f.read()
        b = c.find("<div class=\"fullImageLink\" id=\"file\">")
        b = c.find("href=", b) + 6
        e = c.find("\"", b + 1)
        f = urllib.urlopen("http://sumo.dlr.de/%s" % c[b:e])
        i = i[i.find(":") + 1:]
    else:
        f = urllib.urlopen("http://sumo.dlr.de/%s" % i)
        i = i[i.rfind("/") + 1:]
    if i.find("px-") >= 0:
        i = i[:i.find('-') + 1]
    fd = open(os.path.join(options.mirror, "images", i), "wb")
    fd.write(f.read())
    fd.close()
    imageFiles.append(os.path.join("images", i))

# build navigation
nav = readParseEditPage("SUMO_User_Documentation")
lines = nav[nav.find("="):].split("\n")
level = 0
c = ""
hadHeader = False
for l in lines:
    if len(l) == 0:
        continue
    if l[0] == '=':
        text, link = parseWikiLink(" " + l.replace("=", ""))
        if hadHeader:
            if level > 0:
                c += "</ul></li>\n" * level
        spc = ' ' * (level + 1)
        c = c + spc + "<li>"
        if link != "":
            c = c + "<a href=\"" + link + "\">"
        c = c + text
        if link != "":
            c = c + "</a>"
        c = c + "</li>\n"
        hadHeader = True
        level = 0
        continue
    if l[0].find('*') < 0:
        continue
    text, link = parseWikiLink(l)
    nLevel = l.count('*')
    if level > nLevel:
        c = c + ("</ul></li>\n" * (level - nLevel))
    if level < nLevel:
        c = c + \
            ('<li style="list-style: none; display: inline"><ul>\n' *
             (nLevel - level))
    spc = ' ' * (nLevel + 1)
    #+ str(level) + "-" + str(nLevel)
    c = c + spc + "<li>"
    if link != "":
        c = c + '<a href="' + link + '">' + text + '</a>'
    else:
        c = c + text
    c = c + "</li>\n"
    level = nLevel
if level > 0:
    c += "</ul></li>\n" * level

# get template and embed navigation
fd = open(options.index)
tpl = fd.read()
fd.close()
b = tpl.find("<!-- nav begins -->")
b = tpl.find(">", b) + 1
e = tpl.find("<!-- nav ends -->")
tpl = tpl[:b] + c + tpl[e:]

# build HTML pages
try:
    os.mkdir(options.output)
except:
    pass
try:
    os.mkdir(options.output + "/images")
except:
    pass
for p in pages:
    if not p.startswith("href"):
        continue
    b = p.find("/wiki/")
    e = p.find("\"", b)
    name = p[b + 6:e]
    if name.endswith(".css"):
        print "Skipping css-file %s" % name
        continue
    fromStr = 'generated on %s from <a href="http://sumo.dlr.de/wiki/%s">the wiki page for %s</a>' % (
        datetime.datetime.now(), name, name)
    name = name + ".html"
    t = os.path.join(options.output, name)
    fd = open(os.path.join(options.mirror, name))
    c = fd.read()
    if options.version:
        fromStr += " for SUMO %s" % options.version
    c = c.replace(
        '<div id="siteSub">From Sumo</div>', '<div id="siteSub">%s</div>' % fromStr)
    fd.close()
    #
    if name.find('/') >= 0:
        level = name.count("/")
    else:
        level = name.count("\\")
    level = "../" * level
    # title
    cc = tpl.replace("<title>", "<title>" + name[:name.rfind(".html")] + " - ")
    # css inclusion
    cc = cc.replace("sumo.css", level + "sumo.css")
    cc = cc.replace("logo.png", level + "logo.png")
    # links
    b = cc.find("<a href=")
    while b >= 0:
        b = cc.find('"', b)
        if not cc[b + 1:].startswith("http"):
            cc = cc[:b + 1] + level + cc[b + 1:]
        b = cc.find("<a href=", b)
    # content
    b = cc.find("<!-- content begins -->")
    e = cc.find("<!-- content ends -->", b)
    e = cc.find("<", e + 1) - 1
    cc = cc[:b] + c + cc[e:]

    try:
        os.makedirs(os.path.split(t)[0])
    except:
        pass
    fd = open(t, "w")
    fd.write(cc)
    fd.close()
for i in imageFiles:
    shutil.copy(
        os.path.join(options.mirror, i), os.path.join(options.output, i))
if os.path.exists(os.path.join(options.output, 'SUMO_User_Documentation.html')):
    shutil.copy(os.path.join(options.output, 'SUMO_User_Documentation.html'),
                os.path.join(options.output, 'index.html'))
