genDocOMC.mos

/*
 * genDocOMC.mos  - HTML documentation generation script for use with Open Modelica
 *
 * Based on https://github.com/OpenModelica/OMCompiler/blob/master/Examples/GenerateDoc.mos
 *
 *
 * Copyright (C) 1998-CurrentYear, Linköping University
 * Copyright (C) 2012-CurrentYear, Dietmar Winkler <dietmar.winkler@dwe.no>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http:www.gnu.org/licenses/>.
 *
 */

/*
 * Generates Modelica documentation automatically in about 8 minutes on Linux, 30 on Windows
 *   Includes Modelica standard libraries
 *   Most Modelica builtin operators
 *   Some non-Modelica-but-used-in-MSL operators
 *   OpenModelica scripting
 *
 * $Id$
 */
shouldEcho := true;

echo(shouldEcho);

setDebugFlags("nogen"); /* Don't generate dll/so-files when instantiating classes (performance) */
setCommandLineOptions("--noSimplify");

/* Some variables need to be provided as env variabels,e.g.,:
 *  $ export MODELICAPATH="/path/to/library/root/dir"
 *  $ export releaseVersion="3.2"
 */
rootDir := getEnvironmentVar("MODELICAPATH");
relVer  := getEnvironmentVar("releaseVersion");
outDir  := getEnvironmentVar("outDir");
htmlDir := getEnvironmentVar("htmlDir");
genIcons:= getEnvironmentVar("genIcons")=="true";

setModelicaPath(rootDir);
"Load MSL";
loadModel(Modelica,{relVer});
loadModel(ModelicaServices,{relVer});
loadModel(ModelicaReference,{relVer});
loadModel(ObsoleteModelica3);
// loadModel(ModelicaTest);
"Load extra lib(s)";
// loadModel(BioChem);
/* Load extra libraries directly from a custom file location (even outside the path) using: */
// loadFile("/path/to/library/package.mo");

getErrorString();
getClassNames();
getErrorString();

// rmSymlinks
// (false) should we keep readable names (nice online, if the server handles symbolic links), or
// (true) use the hashed names in the links (nice if you want a zipped version to work on Windows). This removes the readable names, keeping only the hashed and de-duped images.
rmSymlinks := true;
// should we generate documentation for OM built-in functions?
genBuiltin := false;
"Defining and creating the output directory for html files:";
MSLdir := dirname(OpenModelica.Scripting.getSourceFile(Modelica));
"Define where the HTML files should go:";
targetDir := MSLdir+htmlDir;
// Create the target directory:
OpenModelica.Scripting.system("mkdir -p \""+targetDir+"\"");
// Define the relative path to the root directory depending on MSL release or web version
MSLrelease := true;
if MSLrelease then
  relativeRootDir := "../../../";
  else
  relativeRootDir := "../";
end if;

echo(false);
//self:=getInstallationDirectoryPath() + "/share/doc/omc/testmodels/GenerateDoc.mos";getErrorString();
if genIcons then
  OpenModelica.Scripting.system("rm -rf \""+outDir+"/Icons\"");getErrorString();
  profile:=" -m cProfile -o generate_icons_profile.out ";
  py:=getInstallationDirectoryPath() + "/share/doc/omc/testmodels/generate_icons.py";getErrorString();
  commands:={"python "+profile+py+" --quiet --output-dir \""+outDir+"/Icons\" "+typeNameString(cl) for cl in getClassNames(builtin=false)};getErrorString();
  res:=OpenModelica.Scripting.system_parallel(commands);getErrorString();
  OpenModelica.Scripting.system("rm -f *.json");getErrorString();
end if;
OpenModelica.Scripting.system("cp -r '"+outDir+"/Icons' '"+targetDir+"'");getErrorString();

allClassNames:=getClassNames(builtin=genBuiltin,recursive=true,sort=true);
OpenModelica.Scripting.system("date -u +%FT%TZ > tmp");
genTimeStamp := readFile("tmp");
version := "<hr />Generated at " + genTimeStamp +  " by <a href=\"http://openmodelica.org\"> "+getVersion()+"</a>";

echo(shouldEcho);
"classNames";
classNames := getClassNames(builtin=genBuiltin,sort=true);

"Defining functions used in this script...";
loadString("
function filename \"Replace characters that mess with filesystems and shell expansions\"
  input String str;
  output String ostr =
        OpenModelica.Scripting.stringReplace(
        OpenModelica.Scripting.stringReplace(
        OpenModelica.Scripting.stringReplace(
        OpenModelica.Scripting.stringReplace(str,\"/\",\"Division\"),
        \"*\",\"Multiplication\"),
        \"<\",\"x3C\"),
        \">\",\"x3E\")
;
end filename;

function uriEncode
  input String str;
  output String uri;
algorithm
  uri := OpenModelica.Scripting.stringReplace(str,\"'\",\"%27\");
  uri := OpenModelica.Scripting.stringReplace(uri,\" \",\"%20\");
  uri := OpenModelica.Scripting.stringReplace(uri,\"<\",\"x3C\");
  uri := OpenModelica.Scripting.stringReplace(uri,\">\",\"x3E\");
end uriEncode;

function last
  input String str[:];
  output String ostr = str[end];
end last;

function preSuffixIfNotEmpty
  input String prefix;
  input String str;
  input String suffix;
  output String out = if str <> \"\" then prefix + str + suffix else \"\";
end preSuffixIfNotEmpty;

function svgIcon
  input String file;
  input String link;
  input String cssClass;
  input Boolean rmSymlinks;
  output String tag;
protected
  String hashedName;
algorithm
  if OpenModelica.Scripting.regularFileExists(file) then
    hashedName := if rmSymlinks then OpenModelica.Scripting.dirname(file) + \"/\" + OpenModelica.Scripting.basename(OpenModelica.Scripting.realpath(file)) else file;
    tag := \"<a href=\\\"\" + uriEncode(if link == file then hashedName else link) + \"\\\"><img class=\\\"\"+cssClass+\" \\\" src=\\\"\"+ hashedName +\"\\\" /></a>&nbsp;\";
  else
    tag := \"\";
  end if;
end svgIcon;

function head
  input String strs[:];
  input String svgFile;
  input Boolean rmSymlinks;
  output String head;
protected
  String compound = \"\", file;
algorithm
  head :=
\"<head>
  <meta http-equiv=\\\"Content-Type\\\" content=\\\"text/html; charset=utf-8\\\" />
  <link href=\\\"style.css\\\" rel=\\\"stylesheet\\\" type=\\\"text/css\\\" />
  <title>\"+sum(s + \".\" for s in strs[1:end-1])+strs[end]+\"</title>
  </head>
  <body>
  <h1>\" + svgIcon(\"Icons/\" + svgFile,\"Icons/\" + svgFile,\"svgiconhead\",rmSymlinks=rmSymlinks) + \"<a class=\\\"omc-h1-a\\\" href=\\\"index.html\\\">.</a>\";
  for ident in strs[1:end-1] loop
    compound := if compound == \"\" then ident else compound+\".\"+ident;
    file := filename(compound+\".html\");
    head := head + \"<a class=\\\"omc-h1-a\\\" href = \\\"\"+file+\"\\\">\"+ident+\"</a>.\";
  end for;
  head := head + \"<span class=\\\"omc-h1\\\">\" + strs[end] + \"</span></h1>\";
end head;

function itemString
  input Item item;
  input String version;
  output String res;
protected
  String docInfo, revision;
algorithm
  docInfo := if item.docInfo[1] == \"\"
    then preSuffixIfNotEmpty(\"<h2><a name=\\\"info\\\">Information</a></h2><pre>\", item.comment, \"</pre>\")
    else if OpenModelica.Scripting.regexBool(item.docInfo[1],\"<html>\", caseInsensitive=true)
    then \"<h2><a name=\\\"info\\\">Information</a></h2>\" + item.docInfo[1]
    else preSuffixIfNotEmpty(\"<h2><a name=\\\"info\\\">Information</a></h2><pre>\", toHtml(item.docInfo[1]), \"</pre>\");
  revision := if item.docInfo[2] <> \"\" then \"<h2><a name=\\\"revisions\\\">Revisions</a></h2>\" + item.docInfo[2] else \"\";
  res := item.head + docInfo + item.interface + item.short + item.contents + revision + version + \"</body>\";
end itemString;

function itemFile
  input Item item;
  output String file = item.file;
end itemFile;

function toHtml
  input String str;
  output String ostr;
protected
  Integer n;
  String[2] matches;
algorithm
  (n,matches) := OpenModelica.Scripting.regex(str,\"^[:space:]*<html>(.*)</html>[:space:]*\",caseInsensitive=true,maxMatches=2);
  ostr := if n==2 then matches[2] else OpenModelica.Scripting.stringReplace(OpenModelica.Scripting.stringReplace(OpenModelica.Scripting.stringReplace(str,\"&\",\"&amp;\"),\"<\",\"&lt;\"),\">\",\"&gt;\");
end toHtml;

record Item
  String file;
  String head;
  String docInfo[3];
  String contents;
  String comment;
  String interface;
  String short;
end Item;

function stringAdd
  input String a,b;
  output String c = a+b;
end stringAdd;
");

"Start calculate items";
echo(false);
items:={Item(
      filename(OpenModelica.Scripting.typeNameString(c))+".html",
      head(OpenModelica.Scripting.typeNameStrings(c),filename(OpenModelica.Scripting.typeNameString(c))+".svg",rmSymlinks=rmSymlinks),
      OpenModelica.Scripting.getDocumentationAnnotation(c),
      preSuffixIfNotEmpty(
        "<h2>Contents</h2>\n<table><tr><th>Name</th><th>Description</th></tr>",
        sum("<tr><td>"+ svgIcon("Icons/" + filename(OpenModelica.Scripting.typeNameString(cl)) + ".svg",filename(OpenModelica.Scripting.typeNameString(cl)) + ".html","svgiconsmall",rmSymlinks=rmSymlinks) +"<a href=\"" + uriEncode(filename(OpenModelica.Scripting.typeNameString(cl))) + ".html\">" +
              last(OpenModelica.Scripting.typeNameStrings(cl)) + "</a></td>" + "<td>" +
              toHtml(OpenModelica.Scripting.getClassComment(cl)) +
             "</td></tr>\n"
             for cl in OpenModelica.Scripting.getClassNames(c,qualified=true,sort=false)),
        "</table>"
        ),
      toHtml(OpenModelica.Scripting.getClassComment(c)),
      preSuffixIfNotEmpty("\n<h4><a name=\"interface\">Interface</a></h4>\n<blockquote><pre>",toHtml(OpenModelica.Scripting.list(c,interfaceOnly=true)),"</pre></blockquote>"),
      preSuffixIfNotEmpty("\n<h4><a name=\"definition\">Definition</a></h4>\n<blockquote><pre>",toHtml(OpenModelica.Scripting.list(c,shortOnly=true)),"</pre></blockquote>")
    ) for c in allClassNames};
echo(shouldEcho);
"Start writing items to file";
echo(false);
writeFile(stringAdd(targetDir,itemFile(items)) /* Vector of filenames */,itemString(items,version) /* Vector of file contents */);
stringAdd(targetDir,itemFile(items));
getErrorString();

echo(shouldEcho);
"Finished writing " + String(size(items,1)) + " items";
echo(false);

dirs := "";
replaceCommands := "";
filetmp := stringAdd(targetDir,"index.html");
writeFile(filetmp, "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" /><title>Modelica Documentation</title><link href=\"style.css\" rel=\"stylesheet\" type=\"text/css\" /></head><body><h1>Modelica Documentation</h1>\n");
writeFile(filetmp, "<p>This is a listing of the available documentation of the Modelica Standard Library and its related libraries.</p>\n", append = true);
writeFile(filetmp, "<h2>Libraries</h2>\n", append = true);
writeFile(filetmp, "<table summary=\"Libraries\"><tr><th>Name</th><th>Description</th><th>Version</th></tr>", append = true);
for cl in classNames loop
  file := getSourceFile(cl);
  base := basename(file);
  if base <> "ModelicaBuiltin.mo" then
    base := basename(file);
    modelVersion := getVersion(cl);
    contentStr := typeNameString(cl);
    if base == "package.mo" then
      /* The replaceCommands are used in the Python-script generated later. These are sed-style replacements with & as delimiter to avoid escaping slashes. */
      replaceCommands := replaceCommands + "\n (re.compile(r'^[Mm][Oo][Dd][Ee][Ll][Ii][Cc][Aa]://"+contentStr+"/'),'"+relativeRootDir+contentStr+"/'),";
      dirpaths := strtok(OpenModelica.Scripting.dirname(file),"/");
      dirs := dirs + " \"" + dirpaths[end] + "\" ";
    end if;
    comment := toHtml(getClassComment(cl)); // let's escape "&" on the fly
    fileName := filename(contentStr);
    writeFile(filetmp, "<tr><td>" + svgIcon("Icons/" + fileName + ".svg",fileName + ".html","svgiconsmall",rmSymlinks=rmSymlinks) + "<a href=\"" +  uriEncode(fileName) + ".html\">" + contentStr + "</a></td><td>" + comment + "</td><td>" + modelVersion + "</td></tr>", append = true);

  end if;
end for;
writeFile(filetmp, "</table>\n", append = true);

if genBuiltin then
  writeFile(filetmp, "<h2>Builtin Environment</h2>\n", append = true);
  writeFile(filetmp, "<p>This is a listing of builtin Modelica functions of OpenModelica and the documentation for OpenModelica-specific scripting.</p>\n", append = true);

  writeFile(filetmp, "<table summary=\"Builtin Environment\"><tr><th>Name</th><th>Description</th><th>Version</th></tr>\n", append = true);
  for cl in classNames loop
    file := getSourceFile(cl);
    base := basename(file);
    if base == "ModelicaBuiltin.mo" then
      contentStr := typeNameString(cl);
      modelVersion := getVersion(cl);
      comment := toHtml(getClassComment(cl)); // lets escape "&" on the fly
      fileName := filename(contentStr);
      writeFile(filetmp, "<tr><td><a href=\"" + uriEncode(fileName) + ".html\">" + contentStr + "</a></td><td>" + comment + "</td><td>" + modelVersion + "</td></tr>", append = true);

    end if;
  end for;
  writeFile(filetmp, "</table>\n", append = true);
end if;
//writeFile(filetmp, version + " (<a href=\"ModelicaDocumentation.tar.xz\">Offline version</a>)", append = true);
writeFile(filetmp, version, append = true);
writeFile(filetmp, "</body>\n</html>", append = true);

writeFile("Tidy.py","#!/usr/bin/env python
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import subprocess as sub
import re
import glob
import sys
import os

os.chdir('"+targetDir+"') # this is needed for check_local_files()

repls = ["+replaceCommands+"
 (re.compile(r'/Modelica/'), r'/Modelica%20"+relVer+"/'),
 (re.compile(r'org/Modelica%20"+relVer+"/'), r'org/Modelica/'),
 (re.compile(r'/ModelicaReference/'), r'/ModelicaReference%20"+relVer+"/'),
 (re.compile(r'/ModelicaServices/'), r'/ModelicaServices%20"+relVer+"/'),
 (re.compile(r' '), r'%20'),
 (re.compile(r'\\''), r'%27'),
 (re.compile(r'[Mm][Oo][Dd][Ee][Ll][Ii][Cc][Aa]://([A-Za-z0-9.\\'()_ %]*)#'), r'\\1.html#'),
 (re.compile(r'[Mm][Oo][Dd][Ee][Ll][Ii][Cc][Aa]://([A-Za-z0-9.\\'()_ %]*)'), r'\\1.html'),
 (re.compile(r'<[/]?body[^>]*>'), r'')
]

def linkreplace(link):
  for (regex,repl) in repls:
    try:
      link = regex.sub(repl,link)
    except:
      pass
  return link

def check_local_files(uri, tag):
  uri = uri.replace('%20', ' ')
  uri = uri.replace('%27', '\\\'')
  with open('MissingFiles.log', 'a') as log_file:
    if not uri.startswith(('http://','https://','mailto:','ftp://','#')):
      filename = uri.split('#', 1)[0]
      if not os.path.exists(filename):
        log_msg = tag+'Not found: %s\\n' % uri
        sys.stdout.write(log_msg)
        log_file.write(log_msg) # lets also record that in the log file

for filepath in sorted(glob.glob('"+targetDir+"*.html')):
  tag = '[Checking file %s]:\\n' % filepath
#  sys.stdout.write(tag)  # not much point in writing the tags to stdout
  sys.stderr.write(tag)
  pid = sub.call(['tidy', '-utf8', '-modify', '-asxhtml', '--add-xml-decl', 'yes', '-quiet', filepath])
  if pid != 2:
    with open(filepath,'r') as html_file:
      soup = BeautifulSoup(html_file, 'html.parser', from_encoding='utf-8')
      for a in soup.findAll('a'):
        try:
          a['href'] = linkreplace(a['href'])
          check_local_files(a['href'], tag)
        except:
          pass
      for a in soup.findAll('link'):
        try:
          a['href'] = linkreplace(a['href'],filepath)
        except:
          pass
      for img in soup.findAll('img'):
        try:
          img['src'] = linkreplace(img['src'])
          check_local_files(img['src'], tag)
        except:
          pass
    with open(filepath,'w') as html_file:
      html_file.write(soup.__str__())
  else:
    print('Tidy failed with %s, skipping link-replacement for %s!'  % (pid,filepath))
");

writeFile(stringAdd(targetDir,"style.css"),".omc-h1
{
    font-family: mono;
    font-size: x-large;
    color:rgb(153,0,0);
    vertical-align: middle;
}

.omc-h1-a
{
    font-family: mono;
    font-size: large;
    vertical-align: middle;
}

table
{
    border-color: black;
    border-width: 1px 1px 1px 1px;
    border-style: solid;
    border-spacing: 0;
    border-collapse: collapse;
}

table td,th
{
    border-color: black;
    border-width: 1px 1px 1px 1px;
    border-style: solid;
    margin: 0;
    padding: 4px;
}

pre
{
    white-space: pre-wrap; /* CSS 3 */
}

.svgiconhead {
    height: 32px;
    width: 32px;
    vertical-align: top;
    -o-object-fit: contain;
}

.svgiconsmall {
    height: 20px;
    width: 20px;
    vertical-align: top;
    -o-object-fit: contain;
}


");

writeFile("FilterTidy.sh","#!/bin/bash
grep -v DOCTYPE tidy.err | grep -v \"<body>\" |grep -v \"<html>\\|</html>\" |grep -v apos |grep -v lacks |grep -v HTML5 |grep -B1 Warning > tidy.filtered
");

writeFile("Documentation.html","<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\">
<html>
<head>
<title>Redirect to Modelica Documentation</title>
<meta http-equiv=\"REFRESH\" content=\"0;url=Modelica%20"+relVer+"/Resources/helpOM/index.html\"></head>
<body>
You are getting redirected to the HTML documentation of the Modelica Standard Library.
</body>
</html>
");

writeFile("fix-case-sensitive.py","#!/usr/bin/env python
# Finds files *.html that have the same case-insensitive names and renames one or more of them
# Example: Ab.html AB.html ab.html becomes: Ab.1.html ab.2.html AB.html

import glob
import os
import re

def getFiles():
  files = sorted([f for f in glob.glob('"+targetDir+"*.html') if os.path.isfile(f)], key=str.lower)
  return files

def update(subst):
  for file in glob.glob('*.html'):
    if os.path.isfile(file):
      with open (file, 'r' ) as f: orig = f.read()
      patched = orig
      for s in subst.items():
        patched = re.sub('\\\\b'+s[0],s[1],patched)
      if patched is not orig:
        with open (file, 'w' ) as f: f.write(patched)
  return

def makeCaseSensitive():
  files = getFiles()
  last = ''
  idx = 0
  repls = {}
  for file in files:
    upper = file.upper()
    if upper == last:
      while True:
        idx += 1
        nfile = file.split('.')
        nfile.insert(len(nfile)-1,str(idx))
        nfile = '.'.join(nfile)
        if not os.path.isfile(nfile):
          print('Renaming file %s to %s' % (file,nfile))
          repls[file] = nfile
          os.rename(file, nfile)
          break
    else:
      idx = 0
    last = upper
  return repls

print('Running Python script: makeCaseSensitive')
repls = makeCaseSensitive()
update(repls)
");

echo(shouldEcho);

"Removing previous generation output files...";
OpenModelica.Scripting.system("rm -f tidy.out tidy.err tidy.filtered");

"Copying original HTML files to 'old-html-tmp'";
OpenModelica.Scripting.system("mkdir -p old-html-tmp");
OpenModelica.Scripting.system("cp \""+targetDir+"\"*.html old-html-tmp/");

"Copying HTML start page to root directory";
OpenModelica.Scripting.system("cp Documentation.html \""+rootDir+"\"/");

"Tidy.py";
OpenModelica.Scripting.system("python Tidy.py 2> tidy.err");

// Make zip (case-insensitive)
OpenModelica.Scripting.system("python ./fix-case-sensitive.py");

"FilterTidy.sh";
OpenModelica.Scripting.system("bash FilterTidy.sh");

"Remove symlinks if rmSymlinks=true";
if rmSymlinks then
    OpenModelica.Scripting.system("find \""+targetDir+"\" -type l -delete");
end if;

//OpenModelica.Scripting.system("rm -f ModelicaDocumentation.tar.xz");
//OpenModelica.Scripting.system("mkdir -p " + dirs);
//"tar";
//dirs;
//cmd := "tar cJf ModelicaDocumentation.tar.xz --dereference style.css *.html " + dirs + " genDocOMC.mos FindFiles.log tidy.log";
//OpenModelica.Scripting.system(cmd);
getErrorString();