Merge branch 'develop'

bootstrap.sh

@@ -1,4 +1,4 @@
-
+#!/bin/bash
 #======================================
 # LaMachine v2
 #  by Maarten van Gompel
@@ -20,7 +20,7 @@ boldblue=${bold}$(tput setaf 4) #  blue
 normal=$(tput sgr0)
 
 echo "${bold}=====================================================================${normal}"
-echo "           ,              ${bold}LaMachine v2.2.9${normal} - NLP Software distribution" #NOTE FOR DEVELOPER: also change version number in codemeta.json *AND* roles/lamachine-core/defaults/main.yml -> lamachine_version!
+echo "           ,              ${bold}LaMachine v2.2.10${normal} - NLP Software distribution" #NOTE FOR DEVELOPER: also change version number in codemeta.json *AND* roles/lamachine-core/defaults/main.yml -> lamachine_version!
 echo "          ~)                     (http://proycon.github.io/LaMachine)"
 echo "           (----í         Language Machines research group"
 echo "            /| |\         Centre of Language and Speech Technology"

builds/builds.py

@@ -64,6 +64,12 @@
         "version": 'stable',
         "context": "ubuntu1604",
     },
+    {
+        "name": 'stable-venv-ubuntu1804',
+        "flavour": 'local',
+        "version": 'stable',
+        "context": "ubuntu1804",
+    },
     {
         "name": 'stable-venv-arch',
         "flavour": 'local',

builds/context/ubuntu1604/Dockerfile

@@ -17,5 +17,5 @@ USER lamachine
 WORKDIR /home/lamachine
 RUN wget https://raw.githubusercontent.com/proycon/LaMachine/$BRANCH/bootstrap.sh
 RUN chmod a+x bootstrap.sh
-RUN ./bootstrap.sh --flavour local --branch $BRANCH --version $VERSION --env virtualenv --noninteractive --private --verbose --name $NAME --hostname lamachine-$NAME
+RUN /bin/bash bootstrap.sh --flavour local --branch $BRANCH --version $VERSION --env virtualenv --noninteractive --private --verbose --name $NAME --hostname lamachine-$NAME
 CMD /bin/bash -l /home/lamachine/lamachine-$NAME-activate

builds/context/ubuntu1804/Dockerfile

@@ -0,0 +1,21 @@
+FROM ubuntu:bionic
+ARG NAME=stable-venv-ubuntu1804
+ARG VERSION=stable
+ARG BRANCH=master
+EXPOSE 80
+USER root
+MAINTAINER Maarten van Gompel <proycon@anaproy.nl>
+LABEL Description="LaMachine Local Test"
+RUN apt-get update
+RUN apt-get install -m -y python sudo apt-utils wget locales
+RUN sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen
+RUN locale-gen
+RUN useradd -ms /bin/bash lamachine
+RUN echo "lamachine:lamachine" | chpasswd
+RUN echo "lamachine ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER lamachine
+WORKDIR /home/lamachine
+RUN wget https://raw.githubusercontent.com/proycon/LaMachine/$BRANCH/bootstrap.sh
+RUN chmod a+x bootstrap.sh
+RUN /bin/bash bootstrap.sh --flavour local --branch $BRANCH --version $VERSION --env virtualenv --noninteractive --private --verbose --name $NAME --hostname lamachine-$NAME
+CMD /bin/bash -l /home/lamachine/lamachine-$NAME-activate

codemeta.json

@@ -10,7 +10,7 @@
     "@type": "SoftwareSourceCode",
     "identifier": "lamachine",
     "name": "LaMachine",
-    "version": "2.2.8",
+    "version": "2.2.10",
     "description": "LaMachine is a unified software distribution for Natural Language Processing. We integrate numerous open-source NLP tools, programming libraries, web-services, and web-applications in a single Virtual Research Environment that can be installed on a wide variety of machines.",
     "license": "https://spdx.org/licenses/GPL-3.0",
 	"url": "https://proycon.github.io/LaMachine",

roles/alex-asr/meta/main.yml

@@ -1,3 +1,4 @@
 ---
 dependencies:
   - { role: python-core }
+  - { role: kaldi }

roles/alex-asr/tasks/main.yml

@@ -1,9 +1,47 @@
 ---
+#this script does most of what https://github.com/UFAL-DSG/alex-asr/blob/master/prepare_env.sh does, but with the latest kaldi that's already in LaMachine
+
+  - name: Obtaining PyFST
+    include_role:
+      name: lamachine-git
+    vars:
+      package:
+         user: "UFAL-DSG"
+         repo: "pyfst"
+
+  - name: Obtaining Alex-ASR #we need to do this in a seperate step as we are preparing the environment before actually installing
+    include_role:
+      name: lamachine-git
+    vars:
+      package:
+         user: "UFAL-DSG"
+         repo: "alex-asr"
+
+  - name: Creating lib directory
+    file:
+        path: "{{ source_path }}/alex-asr/libs"
+        state: directory
+        owner: "{{unix_user}}"
+        group: "{{unix_group}}"
+        mode: u+rwx,g+rwsx,a+rx
+
+  - name: Linking library path for kaldi
+    file:
+      src: "{{lm_prefix}}/opt/kaldi"
+      dest: "{{lm_prefix}}/src/alex-asr/libs/kaldi"
+      state: link
+
+  - name: Linking library path for pyfst
+    file:
+      src: "{{source_path}}/pyfst"
+      dest: "{{lm_prefix}}/src/alex-asr/libs/pyfst"
+      state: link
+
   - name: Installing Alex-ASR
     include_role:
       name: lamachine-python-install
     vars:
       package:
          github_user: "UFAL-DSG"
          github_repo: "alex-asr"
-         #pip: "alex_asr"
+

roles/gecco/tasks/main.yml

@@ -10,6 +10,13 @@
         - myspell-nl
         - libhunspell-dev
         - libaspell-dev
+        - hunspell
+        - hunspell-nl
+        - hunspell-en-gb
+        - hunspell-en-us
+        - hunspell-es
+        - hunspell-fr
+        - hunspell-de
       when: root and (ansible_distribution|lower == "debian" or ansible_distribution|lower == "ubuntu" or  ansible_distribution|lower == "linux mint")
 
     - name: Installing global dependencies for Gecco (RedHat)
@@ -21,7 +28,14 @@
       with_items:
         - aspell
         - aspell-devel
+        - hunspell
         - hunspell-devel
+        - hunspell-nl
+        - hunspell-en-US
+        - hunspell-en-GB
+        - hunspell-es
+        - hunspell-fr
+        - hunspell-de
       when: root and (ansible_distribution|lower == "centos" or ansible_distribution|lower == "rhel" or ansible_distribution|lower == "redhat" or ansible_distribution|lower == "fedora")
 
     - name: Installing global dependencies for Gecco (Arch)
@@ -33,6 +47,11 @@
       with_items:
         - aspell
         - hunspell
+        - hunspell-en
+        - hunspell-nl
+        - hunspell-es
+        - hunspell-fr
+        - hunspell-de
       when: root and (ansible_distribution|lower == "arch")
 
     - name: Installing global dependencies for Gecco (Mac OS X)

roles/kaldi/defaults/main.yml

@@ -2,5 +2,5 @@
 kaldi_source: "https://github.com/kaldi-asr/kaldi.git"
 kaldi_branch: master
 kaldi_test_ignore_errors: no
-kaldi_configure_options: ""
+kaldi_configure_options: "--shared"
 

roles/kaldi/tasks/main.yml

@@ -22,6 +22,11 @@
       version: "{{kaldi_branch}}"
       force: yes
 
+  - name: Patch OpenFST makefile so that we can link with it statically (needed for Alex-ASR)
+    shell: sed -i "s/--enable-far --enable-ngram-fsts/--enable-far --with-pic --enable-ngram-fsts/" Makefile
+    args:
+       chdir: "{{lm_prefix}}/opt/kaldi/tools"
+
   - name: Compiling kaldi/tools
     shell: make -j $(nproc)
     args :
@@ -59,14 +64,27 @@
         export KALDI_ROOT={{lm_prefix}}/opt/kaldi
       mode: u+rwx,a+rx
 
+  - name: "Kaldi - Get latest commit hash"
+    shell: "git describe --tags || git rev-parse HEAD"
+    args:
+      chdir: "{{ lm_prefix }}/opt/kaldi"
+    register: latest_commit
+    changed_when: False
+
   - name: Register Kaldi
     include_role:
        name: lamachine-register
     vars:
        metadata:
+         identifier: kaldi
          name: Kaldi
+         version: "{{ latest_commit.stdout }}"
          author: (multiple)
-         homepage: "http://kaldi-asr.org/"
+         url: "http://kaldi-asr.org/"
+         codeRepository: "https://github.com/kaldi-asr/kaldi"
          license: Apache
-         summary: "Automatic Speech Recognition Toolkit"
+         description: "Automatic Speech Recognition Toolkit"
+         programmingLanguage: "C++"
+         interfaceType: CLI
+         keywords: [ "speech recognition", "asr", "cuda" ]
 

roles/lamachine-core/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-lamachine_version: "2.2.9" #also change in bootstrap.sh and codemeta.json
+lamachine_version: "2.2.10" #also change in bootstrap.sh and codemeta.json
 conf_name: "default"
 version: "stable" #stable, development or custom
 controller: internal

roles/lamachine-core/tasks/arch.yml

@@ -12,7 +12,7 @@
       pacman:
         name: "{{item}}"
         state: installed
-      with_items: [pkg-config, git, mercurial, autoconf-archive, icu, libxml2, zlib, libtar, boost, boost-libs, nano, make, wget curl, libexttextcat, perl, python, python-virtualenv, python-pip]
+      with_items: [pkg-config, git, mercurial, autoconf-archive, icu, libxml2, zlib, libtar, boost, boost-libs, nano, make, wget curl, libexttextcat, perl, python, python-virtualenv, python-pip, zip, unzip, pcre]
 
       #############################################################
       # ANSIBLE installation so we can bootstrap ourselves

roles/lamachine-core/tasks/debian.yml

@@ -33,7 +33,7 @@
         name: '*'
         state: latest
         update-cache: yes
-        allow-unauthenticated: yes #there may be unauthenticated packages from other sources already present
+        #allow-unauthenticated: yes #there may be unauthenticated packages from other sources already present
       ignore_errors: "{% if ansible_user_id == 'travis' %}yes{% else %}no{% endif %}" #bit unelegant but sometimes the system is too messy and fails on something (like travis)
       when: nosysupdate is not defined or nosysupdate == false
 
@@ -45,7 +45,7 @@
       apt:
         name: "{{item}}"
         state: present
-      with_items: [coreutils, procps, lsof, dirmngr, pkg-config, git-core, mercurial, nano, make, gcc, g++, autoconf, automake, autoconf-archive, libtool, autotools-dev, libicu-dev, libxml2-dev, libxslt1-dev, libbz2-dev, zlib1g-dev, libtar-dev, libaspell-dev, libhunspell-dev,  python3, python3-pip, python3-wheel, python3-dev, libboost-all-dev, libboost-python-dev, wget, curl, libexttextcat-dev, libcurl4-gnutls-dev, poppler-utils, perl]
+      with_items: [coreutils, procps, lsof, dirmngr, pkg-config, git-core, mercurial, nano, make, gcc, g++, autoconf, automake, autoconf-archive, libtool, autotools-dev, libicu-dev, libxml2-dev, libxslt1-dev, libbz2-dev, zlib1g-dev, libtar-dev, libaspell-dev, libhunspell-dev,  python3, python3-pip, python3-wheel, python3-dev, libboost-all-dev, libboost-python-dev, wget, curl, libexttextcat-dev, libcurl4-gnutls-dev, poppler-utils, perl, zip, unzip, libpcre3-dev, libpcre3]
       #dirmngr is needed for gpg apt-get key
 
     - name: Installing python-virtualenv

roles/lamachine-core/tasks/redhat.yml

@@ -31,7 +31,7 @@
       yum:
         name: "{{item}}"
         state: installed
-      with_items: [psmisc, pkgconfig, git, mercurial, icu, libtool, autoconf, automake, autoconf-archive, nano, make, gcc, gcc-c++, libxml2, libxml2-devel, libxslt, libxslt-devel, libtar, libtar-devel, zlib, zlib-devel, bzip2, bzip2-devel, libcurl, gnutls-devel, libcurl-devel, wget, curl, libexttextcat, libexttextcat-devel]
+      with_items: [psmisc, pkgconfig, git, mercurial, icu, libtool, autoconf, automake, autoconf-archive, nano, make, gcc, gcc-c++, libxml2, libxml2-devel, libxslt, libxslt-devel, libtar, libtar-devel, zlib, zlib-devel, bzip2, bzip2-devel, libcurl, gnutls-devel, libcurl-devel, wget, curl, libexttextcat, libexttextcat-devel, zip, unzip, pcre, pcre-devel]
 
     - name: Installing Python base packages for CentOS/RHEL (yum)
       become: true