vmc

commit 309ea700936c07c0491f4ae1f7b2d677e9d0a815
parent daf4d6c289b6fbf81373d1c1066cd1183c29e76f
Author: umhau <umhau@alum.gcc.edu>
Date:   Wed,  5 Jul 2017 17:13:53 -0400

this should keep conflicting files out of the way. (I hope there is not a problem with the python packages themselves)

Diffstat:
M
README.md
 | 
37
+++++++++++++++++++++++++++----------
M
install.sh
 | 
128
+++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------
A
lib/move_ps_files.sh
 | 
82
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
M
lib/voicemodel.sh
 | 
20
++++++++++----------
M
vmc
 | 
22
++++++++++++++++------

5 files changed, 224 insertions(+), 65 deletions(-)
diff --git a/README.md b/README.md
@@ -11,14 +11,15 @@ Note this tool has only been tested with Linux Mint 17.3 & 18 and Ubuntu GNOME
 
 **Please see the LICENSE file for terms of use.**
 
-Linux/Unix installation
+installation
 -------------------------------------------------------------------------------
 
+This is tested on Ubuntu GNOME 17.04.  Further testing has not been performed.
+
 You should install dependencies first; this ensures that python-dev, 
 PocketSphinx, etc. are available.  Second, install vmc.  Some of the packages 
 need to be installed within the user's home directory; ~/tools is recommended.  
-This should be specified when installing the dependencies. Full installation on 
-an AMD64 computer running Mint 18 would look like this:
+This should be specified when installing the dependencies. 
 
 Commands:
 
@@ -45,15 +46,21 @@ Usage Examples
 
 Add to a preexisting set of recordings, and adapt an existing acoustic model.
 
-    vmc en-us -adapt /extant/model/location -addrecordings /audio/files/location /dictation/file/location.txt 5
+    vmc en-us \
+    -adapt /extant/model/location \
+    -addrecordings /audio/files/location /dictation/file/location.txt 5
 
 Create a new model, and create a new set of audio recordings.
 
-    vmc en-us -create /place/to/put/model -newrecordings /place/to/put/audio/files /dictation/file/location.txt 5 
+    vmc en-us \
+    -create /place/to/put/model \
+    -newrecordings /place/to/put/audio/files /dictation/file/location.txt 5 
 
 Import a previously created set of recordings, and adapt a preexisting model.
 
-    vmc en-us -adapt /extant/model/location -importrecordings /audio/files/location
+    vmc en-us \
+    -adapt /extant/model/location \
+    -importrecordings /audio/files/location
 
 File Structure
 -------------------------------------------------------------------------------
@@ -69,20 +76,20 @@ the en-us files have very default names.
 Most files have default names, or are named according to the model name. File 
 structure is as follows (incomplete, only showing commonly-used files):
 
-    audio-recordings
+    [audio-recordings]
     - [model name].fileids
     - [model name].transcription
     - mdef
     - mdef.txt
     
-    acoustic-model
+    [acoustic-model]
     - feat.params
 
 Background
 -------------------------------------------------------------------------------
 
-This tools brings together a number of disparate data files that are needed for 
-creating a voice model.  This graph illustrates the data process involved:
+This tool brings together a number of disparate data files that are needed for 
+creating a voice model.  This graph illustrates the algorithm:
 
                    word domain
                         +
@@ -106,3 +113,13 @@ creating a voice model.  This graph illustrates the data process involved:
 Each of these steps, starting with the sentence list (given) and ending with 
 the voice model are contained within this tool.
 
+The 'word domain' is the set of sentences, words and phrases used in the 
+training and in the use case scenario.  They must be as identical as possible
+to enable accurate recognition.
+
+Todo
+-------------------------------------------------------------------------------
+
+- clean up VMC script (add functions, make options tidier, etc.)
+
+- make sure that the process of removing conflicting libs actually works.
diff --git a/install.sh b/install.sh
@@ -2,7 +2,9 @@
 # 
 # USAGE =======================================================================
 # 
-#       bash install.sh -no-deps
+#       bash install.sh 
+#       bash install.sh -inc-deps (just the program files, used in development)
+#       bash install.sh -refresh (removes program files & CMU Sphinx)
 # 
 # NOTES =======================================================================
 # 
@@ -21,28 +23,34 @@
 # 
 # SET VARIABLES ===============================================================
 
+option="$1"
+
 # Absolute path to this script & containing folder.  stackoverflow.com/q/242538
-script=$(readlink -f "$0"); scriptpath=$(dirname "$script") 
+script=$(readlink -f "$0")
+scriptpath=$(dirname "$script") 
+
+# CMU Sphinx install location - my static repo or the official source?
+CMUsrc="cmusphinx" # or "umhau"
 
-libdir=/opt/vmc/lib
 install_dir="/home/$USER/CMU_Sphinx"
 
-# check number of cores (speeds compilation)
-CORES=$(nproc --all 2>&1)
+# FUNCTIONS ===================================================================
 
-# CMU Sphinx install location - my static repo or the official source?
-CMUsrc="cmusphinx" # or "umhau"
+look_for_and_remove_old_installation() {
 
-# CHECK FOR PREVIOUS INSTALLATION =============================================
+    if [ -d /opt/vmc ]; then
 
-if [ -d /opt/vmc/ ]; then
-    bash "vmc -remove 1>/dev/null"; echo "Removed vmc"
-fi
+        sudo rm -rf /opt/vmc
+        sudo rm -f /usr/local/bin/vmc
+        sudo rm -f /usr/local/bin/lmc
+
+    fi
 
-# INSTALL VMC DEPENDENCIES ====================================================
-if [ ! "$1" == '-no-deps' ]; then
+}
 
-    echo "Installing dependencies. To continue, press [enter]."; read
+install_dependencies() {
+
+    local CORES=$(nproc --all 2>&1)
 
     if [ ! -d $install_dir ]; then mkdir $install_dir; fi
 
@@ -67,7 +75,7 @@ if [ ! "$1" == '-no-deps' ]; then
     fi
 
     # check for and install sphinxtrain
-    echo "Checking for sphinxtrain...."
+    echo -n "Checking for sphinxtrain...."
     if [ ! -d $install_dir/sphinxtrain/ ]; then
         echo "installing..."; cd $install_dir
         git clone "https://github.com/$CMUsrc/sphinxtrain.git"
@@ -97,38 +105,80 @@ if [ ! "$1" == '-no-deps' ]; then
         echo "PocketSphinx already installed."
     fi
 
-else echo -n "NOT installing dependencies. To continue, press [enter]."; read
-fi
+}
+
+create_vmc_directories() {
+
+    sudo mkdir -p /opt/vmc
+    sudo mkdir -p /opt/vmc/lib
+
+
+}
+
+install_program_files() {
+
+    # move library
+    sudo cp -r $scriptpath/lib/* /opt/vmc/lib/
+
+    sudo tar -xf $scriptpath/lib/cmusphinx-en-us-ptm-5.2.tar.gz -C /opt/vmc/lib/
+    sudo mv /opt/vmc/lib/cmusphinx-en-us-ptm-5.2 /opt/vmc/lib/en-us
+
+    # move vmc into user's path & set as executable
+    sudo cp $scriptpath/vmc /usr/local/bin/vmc
+    sudo chmod +x /usr/local/bin/vmc
+
+    # move lmc into user's path  & set as executable
+    sudo cp $scriptpath/lmc /usr/local/bin/lmc
+    sudo chmod +x /usr/local/bin/lmc
+
+}
+
+copy_CMU_Sphinx_binaries() {
+
+    sudo cp /usr/local/libexec/sphinxtrain/bw /opt/vmc/lib
+    sudo cp /usr/local/libexec/sphinxtrain/map_adapt /opt/vmc/lib
+    sudo cp /usr/local/libexec/sphinxtrain/mk_s2sendump /opt/vmc/lib
+    sudo cp /usr/local/libexec/sphinxtrain/mllr_solve /opt/vmc/lib
+
+}
+
+remove_CMU_Sphinx() {
+
+    # this is all the stuff I'm aware of. CMU Sphinx docs are opaque on this.
+
+    sudo rm -f /usr/local/lib/libpocketsphinx*
+    sudo rm -f /usr/local/lib/libsphinx*
+    sudo rm -fr /usr/local/lib/sphinxtrain
+
+    sudo rm -fr /usr/local/libexec/sphinxtrain
+
+    sudo rm -fr $install_dir
+
+}
+
+
+# MAIN ========================================================================
 
-# MOVE VMC FILES ==============================================================
+main() {
 
-# get sudo 
-sudo ls 1>/dev/null; echo -en "\nInstalling vmc..."
+    sudo ls 1>/dev/null; echo "installing voice model creator"
 
-# create vmc directories
-sudo mkdir -p $libdir; sudo mkdir -p $libdir
+    look_for_and_remove_old_installation
 
-# move library
-sudo cp -r $scriptpath/lib/* $libdir/
+    if [ "$option" == "-inc-deps" ]; then install_dependencies;
+    elif [ "$option" == "-refresh" ]; then remove_CMU_Sphinx; install_dependencies;
+    else echo "not installing dependencies"; fi
 
-sudo tar -xf $scriptpath/lib/cmusphinx-en-us-ptm-5.2.tar.gz -C $libdir
-sudo mv  $libdir/cmusphinx-en-us-ptm-5.2 $libdir/en-us
+    create_vmc_directories
 
-# move vmc into user's path & set as executable
-sudo cp $scriptpath/vmc /usr/local/bin/vmc
-sudo chmod +x /usr/local/bin/vmc
+    install_program_files
 
-# move lmc into user's path  & set as executable
-sudo cp $scriptpath/lmc /usr/local/bin/lmc
-sudo chmod +x /usr/local/bin/lmc
+    copy_CMU_Sphinx_binaries
 
-# GET SPHINXTRAIN BINARIES ========================================================================
+    bash $scriptpath/lib/move_ps_files.sh -out_of_lib
 
-# copy binary tools into model folder
-sudo cp /usr/local/libexec/sphinxtrain/bw $libdir
-sudo cp /usr/local/libexec/sphinxtrain/map_adapt $libdir
-sudo cp /usr/local/libexec/sphinxtrain/mk_s2sendump $libdir
-sudo cp /usr/local/libexec/sphinxtrain/mllr_solve $libdir
+    echo -e "\nVMC installation complete"
 
-echo "done."
+}
 
+main
diff --git a/lib/move_ps_files.sh b/lib/move_ps_files.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# USAGE =======================================================================
+#
+#   a number of shared libraries are kept in /usr/local/lib/.  They are 
+#   incompatible with the python pocketsphinx package.  This script moves them 
+#   between their installed location and the installation directory for the
+#   rest of the CMU Sphinx dependencies. 
+#
+#   bash move_ps_files.sh -into_lib
+#   bash move_ps_files.sh -out_of_lib
+#
+#   This script is run at the beginning and at the end of the VMC program.
+#
+# VARIABLES ===================================================================
+
+option="$1"
+lib_backup_dir="/home/$USER/CMU_Sphinx/shared_library_backup"
+
+# FUNCTIONS ===================================================================
+
+move_files_out_of_lib() {
+
+
+    sudo mv /usr/local/lib/libpocketsphinx* $lib_backup_dir
+    sudo mv /usr/local/lib/libsphinx* $lib_backup_dir
+    sudo mv /usr/local/lib/sphinxtrain $lib_backup_dir
+
+}
+
+copy_files_into_lib() {
+
+    sudo cp $lib_backup_dir/libpocketsphinx* /usr/local/lib
+    sudo cp $lib_backup_dir/libsphinx* /usr/local/lib
+    sudo cp -r "$lib_backup_dir/sphinxtrain" /usr/local/lib
+
+}
+
+delete_files_from_lib() {
+
+    sudo rm -f /usr/local/lib/libpocketsphinx*
+    sudo rm -f /usr/local/lib/libsphinx*
+    sudo rm -fr /usr/local/lib/sphinxtrain
+
+}
+
+clean_lib_directory() {
+
+    if [ -d "$lib_backup_dir" ]
+    then
+        delete_files_from_lib
+    else
+        mkdir -p "$lib_backup_dir"
+        move_files_out_of_lib
+    fi
+
+
+}
+
+ensure_files_are_present() {
+
+    if [ -d "$lib_backup_dir" ] # if no backup made, assume files are present
+    then 
+        :
+    else
+        copy_files_into_lib
+    fi
+
+}
+
+main() {
+
+    if   [ $option == "-into_lib" ]; then ensure_files_are_present;
+    elif [ $option == "-out_of_lib" ]; then clean_lib_directory; 
+    else echo "bad options given"
+    fi
+
+}
+
+# COMMANDS ====================================================================
+
+main
diff --git a/lib/voicemodel.sh b/lib/voicemodel.sh
@@ -33,7 +33,7 @@ dict="/usr/local/lib/python2.7/dist-packages/pocketsphinx/model/cmudict-en-us.di
 # convert binary mdef file to .txt --------------------------------------------
 cd $acoustic_model_dir
 sudo pocketsphinx_mdef_convert \
-    -text $acoustic_model_dir/mdef $acoustic_model_dir/mdef.txt &> /dev/null
+    -text $acoustic_model_dir/mdef $acoustic_model_dir/mdef.txt #&> /dev/null
 
 # run tools to create voice model ---------------------------------------------
 cd $audio_file_dir
@@ -42,8 +42,8 @@ sudo sphinx_fe \
     -argfile $acoustic_model_dir/feat.params \
     -samprate 16000 \
     c $audio_file_dir/$model_name.fileids \
-    -di . -do . -ei wav -eo mfc -mswav yes \
-    &> /dev/null
+    -di . -do . -ei wav -eo mfc -mswav yes #\
+    # &> /dev/null
 
 sudo $libdir/bw \
     -hmmdir $acoustic_model_dir \
@@ -52,13 +52,13 @@ sudo $libdir/bw \
     -cmn current -agc none -dictfn $dict  \
     -ctlfn $audio_file_dir/$model_name.fileids \
     -lsnfn $audio_file_dir/$model_name.transcription \
-    -accumdir . \
-    &> /dev/null
+    -accumdir . #\
+    # &> /dev/null
 
 sudo $libdir/mllr_solve \
     -meanfn $acoustic_model_dir/means  \
     -varfn $acoustic_model_dir/variances \
-    -outmllrfn mllr_matrix -accumdir . &> /dev/null
+    -outmllrfn mllr_matrix -accumdir . #&> /dev/null
 
 sudo $libdir/map_adapt \
     -moddeffn $acoustic_model_dir/mdef.txt \
@@ -71,12 +71,12 @@ sudo $libdir/map_adapt \
     -mapmeanfn $acoustic_model_dir/means \
     -mapvarfn $acoustic_model_dir/variances \
     -mapmixwfn $acoustic_model_dir/mixture_weights \
-    -maptmatfn $acoustic_model_dir/transition_matrices\
-    &> /dev/null
+    -maptmatfn $acoustic_model_dir/transition_matrices #\
+    # &> /dev/null
 
 sudo $libdir/mk_s2sendump \
     -pocketsphinx yes \
     -moddeffn $acoustic_model_dir/mdef.txt \
     -mixwfn $acoustic_model_dir/mixture_weights \
-    -sendumpfn $acoustic_model_dir/sendump \
-    &> /dev/null
+    -sendumpfn $acoustic_model_dir/sendump #\
+    # &> /dev/null
diff --git a/vmc b/vmc
@@ -62,7 +62,6 @@ else
 fi
 
 current_number_of_recordings="0" # changed below, if there are any.
-libdir="/opt/vmc/lib"
 
 # COMMANDS ====================================================================
 
@@ -74,7 +73,7 @@ if [ $1 == '-remove' ]; then
     sudo rm -f /usr/local/bin/vmc
     sudo rm -f /usr/local/bin/lmc
     echo "vmc removed."
-    exit 1
+    exit 0
 fi
 
 # misc. housekeeping ----------------------------------------------------------
@@ -112,7 +111,7 @@ fi
 
 if [ $4 = '-newrecordings' ] || [ $4 = '-addrecordings' ]; then # record audio into folder.
 
-    sudo python3 $libdir/getaudio.py \
+    sudo python3 /opt/vmc/lib/getaudio.py \
         $dict_file \
         $audio_folder \
         $reps \
@@ -122,12 +121,16 @@ if [ $4 = '-newrecordings' ] || [ $4 = '-addrecordings' ]; then # record audio i
     echo "Audio files saved into $audio_folder. They can be reused."
 fi
 
+# MOVE CONFLICTING LIB FILES INTO LIB TEMPORARILY -----------------------------
+
+bash /opt/vmc/lib/move_ps_files.sh -into_lib
+
 # PRODUCE DERIVATIVE FILES ----------------------------------------------------
 # These are recreated for all audio files, regardless of whether this is a new
 # collection of recordings or just adding to an old one.  Simpler that way.
 
 echo -e "\nProducing sentence file derivatives..."
-sudo python3 $libdir/format_text.py \
+sudo python3 /opt/vmc/lib/format_text.py \
     $dict_file \
     $model_name \
     $audio_folder \
@@ -135,7 +138,7 @@ sudo python3 $libdir/format_text.py \
     $current_number_of_recordings
 
 echo "Producing audio file derivatives..."
-sudo bash $libdir/acousticfiles.sh \
+sudo bash /opt/vmc/lib/acousticfiles.sh \
     "$audio_folder" \
     "$model_location" \
     "$audio_folder/$model_name.fileids"
@@ -143,7 +146,9 @@ sudo bash $libdir/acousticfiles.sh \
 # CREATE MODELS ---------------------------------------------------------------
 
 echo "Creating voice model..."
-sudo bash $libdir/voicemodel.sh $model_name $model_location $audio_folder 
+sudo bash /opt/vmc/lib/voicemodel.sh $model_name $model_location $audio_folder 
+
+echo "Conflicting CMU Sphinx files moved into backup."
 
 # FIX ROOT PERMISSIONS --------------------------------------------------------
 # I really hope this isn't horrible practice, but it's the most straightforward
@@ -151,7 +156,12 @@ sudo bash $libdir/voicemodel.sh $model_name $model_location $audio_folder
 sudo chown -R $USER: $model_location
 sudo chown -R $USER: $audio_folder
 
+# MOVE CONFLICTING LIBS OUT OF THE WAY ----------------------------------------
+
+bash /opt/vmc/lib/move_ps_files.sh -out_of_lib
+
 # DONE ------------------------------------------------------------------------
 echo "Process complete."
 echo -e "ACOUSTIC VOICE MODEL: \t $model_location"
 echo -e "VOICE RECORDINGS: \t $audio_folder"
+