Merge full_rewrite branch

Because a bug my Debian Setup : kernel doesn't have time to refresh partitions scheme then mkfs doesn't format partitions

.gitignore

@@ -1,7 +1,2 @@
 tmp/
-syslinux/*.c32
-syslinux/ldlinux.*
-EFI
-debian-8.5.0-amd64-netinst.iso
-vmlinuz
-initrd.gz
+repo/

README.md

@@ -1,44 +1,64 @@
 Debian Auto Installer
-====================
+=====================
 
-A set of tools to create automated Debian Installation Key. I created this tools
-for my work. You can create bios, efi64 key but prepare file in the folder
- whitout creating key.
+A script to create automated Debian Installation Key. I created it for my work.
+ You can create bios, efi64 key or prepare files into a folder whitout creating 
+key. 
  
  * init.sh : script to create USB key
- * preseed : textfile that contain all instructions for Debian Installer
- * partman : partition recipe for preseed
+ * preseeds : folder containing all preseeds files
+ * partman_recipes : folder containinf partman recipes
  * syslinux/syslinux.cfg : Syslinux configuration file
+ * conf/*.conf : "folder configuration" files
 
-# Howto create usb key
-
- 1. Plug your key
- 2. Make sure yout key is not mounted
- 3. Execute ``./init.sh -d /dev/sdb`` (replace /dev/sdb with your device)
- 4. Done!
 
 # Init.sh command line options
 
  * ``-a [amd64|i386]`` choose architecture for debian files (default amd64)
  * ``-b`` create key for bios boot architecture (default efi64)
+ * ``-c --conf`` set folder configuration files (default conf/archlinux.conf)
  * ``-d device`` USB device like /dev/sdb
+ * ``--debian-version`` debian iso version to download (default lastest stable)
+ * ``--repo`` repository folder (default ./repo)
+ * ``--tmp`` Temporary folder used to create USB key to create directory tree 
 
-Be careful : if ``-d`` parameter is define then the relative peripheral will 
-be erased
+Be careful : if ``-d`` parameter is define then the usb key / external drive 
+will be erased.
 
-For now, this script doesn't verify the state of your device (is mounted or not)
- but this is planned.
+Since syslinux files are placed differently depending on the distribution,
+ configuration file (``-c``) contain directory to syslinux files :
+ 
+ * __syslinux_mbr_bl_folder__ : folder containing mbr.bin
+ * __syslinux_efi_bl_folder__ : folder containing efi64 folder
+ * __syslinux_modules_folder__ : folder containing syslinux modules folders
 
-## Exemples
+## Examples
 
-Prepare file in directory for efi system without writing an usb key for AMD64 arch.
+Prepare file in directory for efi system without writing an usb key for AMD64.
 
 ```
 ./init.sh
 ```
 
-Prepare an usb key (/dev/sdb) for bios / i386 architecture
+Prepare an USB key (/dev/sdb) for bios / i386 architecture
 
 ```
 ./init.sh -a i386 -b -d /dev/sdb
 ```
+
+Prepare an USB key on Debian
+
+```
+./init.sh -d /dev/sdb -c conf/debian.conf
+```
+
+# How it work
+
+ 1. Create a repo folder then a subfolder named with debian version then 
+    download debian iso, vmlinuz and initrt.gz.
+ 2. Create a temporary folder and copy all necessary files into it (pressed,
+    partman recipes, syslinux modules...).
+ 4. If ``-d /dev/sdx``, unmount usb key( if mounted) and create partition scheme,
+    format and  mount partitions.
+ 5. Copy all content of temporary folder into mounted usb key.
+ 6. Unmount usb key.

conf/archlinux.conf

@@ -0,0 +1,4 @@
+# Archlinux syslinux folder configuration
+syslinux_mbr_bl_folder="/usr/lib/syslinux/bios"
+syslinux_efi_bl_folder="/usr/lib/syslinux"
+syslinux_modules_folder="/usr/lib/syslinux"

conf/debian.conf

@@ -0,0 +1,4 @@
+# Debian syslinux folder configuration
+syslinux_mbr_bl_folder="/usr/lib/SYSLINUX"
+syslinux_efi_bl_folder="/usr/lib/SYSLINUX.EFI"
+syslinux_modules_folder="/usr/lib/syslinux"

init.sh

@@ -6,225 +6,331 @@
 # Dépend : syslinux,wget
 #
 OPTIND=1
-DEBIAN_VERSION="9.2.0"
-DEBIAN_ARCH="amd64"
-BOOT_TYPE="efi64"
-SYSLINUX_EXE=`which syslinux`
+
+# Default Variables
+debian_arch="amd64"
+debian_version=$(wget -q --output-document - https://cdimage.debian.org/debian-cd/ | \
+    grep -o '[0-9]\{1,2\}\.[0-9]\{1,2\}\.[0.9]\{1,2\}' | head -1)
+archive=0
+boot_type="efi64"
+repo_dir="repo"
+tmp_dir="tmp"
+current_dir=`pwd`
+# In Archlinux, this is default files for syslinux...
+conf_file="conf/archlinux.conf"
+key_mountpoint="/mnt/usbstick"
+device=""
+# constant
+APP_NAME="Debian USB Creator"
+VERSION="1.99.0"
+SYSLINUX_EXE=$(which syslinux)
+PARTED_EXE=$(which parted)
 SYSLINUX_FILES="menu.c32 vesamenu.c32 libutil.c32 libcom32.c32"
 SYSLINUX_BIOS_FILES=""
 SYSLINUX_EFI_FILES="ldlinux.e64"
-TMP_DIR="tmp"
-REPO_DIR="repo"
-MOUNT_DIR="/mnt/usbstick"
-current_dir=`pwd`
+
+process_args() {
+    # While getops doesn't handle long parameters
+    # I need an personnal function
+    # Inspired by http://mywiki.wooledge.org/BashFAQ/035
+    while :; do
+        case $1 in
+            -h|-\?|--help)
+                usage
+                exit 0
+                ;;
+            -a|--arch)
+                if [[ $2 == "amd64" || $2 == "i386" ]]
+                then
+                    debian_arch=$2
+                    shift
+                else
+                    printf "Error : CPU architecture must be amd64 or i386"
+                    exit 1
+                fi
+                ;;
+            -b|--bios)
+                boot_type="bios"
+                ;;
+            -c|--conf)
+                if [ -f $2 ]
+                then
+                    conf_file=$2
+                else
+                    printf "%s file not found" $2
+                    exit 1
+                fi
+                shift
+                ;;
+            -d|--device)
+                if [ -b $2 ]
+                then
+                    device=$2
+                    shift
+                else
+                    printf "%s doesn't a block device, exiting" "$2"
+                    exit 1
+                fi
+                ;;
+            --debian-version)
+                if [[ $debian_version != $2 ]]
+                then
+                    archive=1
+                    debian_version=$2
+                fi
+                shift
+                ;;
+            --repo)
+                repo_dir=$2
+                shift
+                ;;
+            --tmp)
+                tmp_dir=$2
+                shift
+                ;;
+            -v|--version)
+                show_version
+                exit 0
+                ;;
+            *)
+            break
+        esac
+        shift
+    done
+}
 
 usage() {
-    cat << EOF
+cat << EOF
 USAGE: ${0} [-a architecture] [-b] 
-    -a  amd64 | i386 CPU architecture
-    -b bios boot type (default is efi64)
-    -d device_block USB key to write
-    
+-a | --arch         Select CPU Architecture amd64 | i386 CPU
+-b | --bios         Create a bios bott compatible key 
+-d | --device       Device_block USB key to write
+--debian-version    Choose Debian Version to download
+--repo              Repo directory (defaukt is ./repo)
+--temp              Temp directory (default is ./tmp)
+-v | --version      Show version
+
 ${0} initialize an usb key for Debian automatic install with preseed and a partman recipe.
 EOF
 }
 
+show_version () {
+printf "%s %s 2016-2017 ephase@xieme-art.org\n" "$APP_NAME" "$VERSION"
+}
+
 download (){
-    #Download function
-    # $1 : filename
-    # $2 : url
-    # $3 : 1 if abort download if file exist
-    echo -ne "Downloading $1 ... "
-    if [[ -f $1 && $3 == "1" ]]
+#Download function
+# $1 : filename
+# $2 : url
+# $3 : 1 if abort download if file exist
+printf "Downloading %s :    " "$1"
+local url
+local http_response
+url="$2/$1"
+http_response=$(wget --spider --server-response "$url" 2>&1 | grep HTTP/ | tail -1 |  awk ' { printf $2 }')
+if [ "$http_response" -eq 200 ]
+then
+    wget -c --progress=dot "$url" 2>&1 | grep --line-buffered "[0-9]\{1,3\}%" -o | awk '{printf ("\b\b\b\b%4s", $1)}'
+    if [ $? -eq 0 ]
     then
-        echo file already downloaded.
-        return
+        printf " \b\b\b\b\b done\n "
+    else
+        printf " error, exiting.\n"
+        exit 1
     fi
-    wget ‐‐no-clobber -4 $2/$1 &> /dev/null
-    echo "done!"
+else
+    printf "Error 404 : file not found, exiting\n"
+    exit 1
+fi
+}
+
+process_conf_file (){
+    while read -r p
+    do
+        var=$(echo "$p" | awk -F"=" '{print $1}')
+        param=$(echo "$p" | awk -F"=" '{print $2}')
+        eval "$var"="$param"
+    done < <( cat "$1" | grep '^[^#].*')
+
 }
 
 create_dir(){
-    # $1 directory to create
-    if [ ! -d $1 ]
+# $1 directory to create
+    if [ ! -d "$1" ]
     then 
-        mkdir -p $1
+        mkdir -p "$1"
     else
-        if [[ $2 == 1 ]]
+        if [ -n "$2" ] && [ "$2" -eq 1 ] 
         then
-            rm -rf $1
-            create_dir $1
+            rm -rf "$1"
+            create_dir "$1"
         fi
     fi
 }
 
+# unmount if device is mounted
+umount_device (){
+    local mounted_part=""
+    mounted_part=$(mount | grep "${1}[0-9]" | awk '{print $3}')
+    if [ !  "$mounted_part" == "" ]
+    then
+        for part  in $mounted_part
+        do
+            if [ -d "$part" ]
+            then
+                printf "Unmount %s\n " "$part"
+                umount "$part" --recursive
+            fi
+        done
+    fi
+}
+
+efi_create_key_structure () {
+    parted -s "$1" mklabel gpt mkpart EFS fat32 1MiB 128MiB set 1 boot on mkpart debian fat32 128MiB 100%
+    sleep 2
+    mkfs.vfat "${1}1" -n efi &> /dev/null
+    mkfs.vfat "${1}2" -n debian &> /dev/null
+    sync
+    mount "${1}2" "$2"
+    create_dir "$2/efi"
+    mount "${1}1" "$2/efi"
+}
+
+bios_create_key_structure() {
+    parted -s "$1" mklabel msdos mkpart primary fat32 1MiB 100% set 1 boot on
+    sleep 2
+    mkfs.vfat "${1}1" -n debian &> /dev/null
+    mount "${1}1" "$2"
+}
+
 #Stop script if syslinux is not installed
-if [[  $SYSLINUX_EXE == "" ]]
+if [ -z  "$SYSLINUX_EXE" ]
 then
-    echo "Syslinux not found, script can't continue…"
+    printf "Syslinux not found, script can't continue.\n"
     exit 1
 fi
 
-if [ -d /usr/lib/syslinux/efi64 ]
-then    
-    syslinux_mod="/usr/lib/syslinux"
-else
-    if [ -d /usr/lib/syslinux/modules/efi64 ]
-    then
-        syslinux_mod="/usr/lib/syslinux/modules"
-    else
-        echo "Syslinux modules folder not found, exiting"
-        exit 1
-    fi
+if [ -z "$PARTED_EXE" ]
+then 
+    printf "Parted not found, script can't continue.\n"
+    exit 1
 fi
 
 
-# Arguments ...
-while getopts "ha:bd:" opt; do
-    case "$opt" in
-        h)
-            usage
-            exit 0
-            ;;
-        a)
-            if [[ $OPTARG == "amd64" || $OPTARG == "i386" ]]
-            then
-                DEBIAN_ARCH=$OPTARG
-            else
-                echo "CPU architecture must be amd64 of i386, bye."
-                exit 1
-            fi
-            ;;
-        b)
-            BOOT_TYPE="bios"
-            ;;
-        d)
-            if [ -b $OPTARG ]
-            then
-                dest=$OPTARG
-            else
-                echo "Destination is not a block device, bye."
-                exit 1
-            fi
-            ;;
-    esac
-done
+process_args "$@"
+process_conf_file "$conf_file"
+if [[ ! -d "$syslinux_mbr_bl_folder" || ! -d "$syslinux_efi_bl_folder" || ! -d "$syslinux_modules_folder" ]]
+then
+    printf "Bad folder configuration in %s\n" "$conf_file"
+    exit 1
+fi
 
-bootfiles_Debian="http://ftp.debian.org/debian/dists/jessie/main/installer-${DEBIAN_ARCH}/current/images/hd-media"
-iso_url="http://cdimage.debian.org/debian-cd/${DEBIAN_VERSION}/${DEBIAN_ARCH}/iso-cd"
-iso_file="debian-${DEBIAN_VERSION}-${DEBIAN_ARCH}-netinst.iso"
-repo_path="$REPO_DIR/$DEBIAN_ARCH/$DEBIAN_VERSION"
+bootfiles_Debian="http://ftp.debian.org/debian/dists/stable/main/installer-${debian_arch}/current/images/hd-media"
+if [ $archive -eq 1 ]
+then
+    iso_url="http://cdimage.debian.org/cdimage/archive/${debian_version}/${debian_arch}/iso-cd"
+else
+    iso_url="http://cdimage.debian.org/debian-cd/${debian_version}/${debian_arch}/iso-cd"
+fi 
+iso_file="debian-${debian_version}-${debian_arch}-netinst.iso"
+repo_path="$repo_dir/$debian_arch/$debian_version"
 
 
 # Download Debian files (kernel, initrd and iso)
-echo -e "\nPrepare Debian files\n---"
+printf "Downloading Debian files\n---\n"
+
 # Création du répertoire temporaire
-create_dir $repo_path
-cd $repo_path
-download $iso_file $iso_url 1
+create_dir "$repo_path"
+cd "$repo_path"
+
+download "$iso_file" "$iso_url" 1
 for file in "vmlinuz" "initrd.gz" 
 do
-    download $file $bootfiles_Debian 1
+    download "$file" "$bootfiles_Debian" 1
 done
-cd $current_dir
+cd "$current_dir"
 
-if [[ -z $dest ]]
+if [ -z "$device" ]
 then
-    echo "No destination device specified, USB key will not be created."
+    printf "No destination device specified, USB key will not be created.\n"
 fi
-create_dir $TMP_DIR 1
-cp $repo_path/$iso_file $TMP_DIR
+create_dir "$tmp_dir" 1
+cp "$repo_path/$iso_file" "$tmp_dir"
 # Copy wanted syslinux file from archive
 
-case $BOOT_TYPE in
+case $boot_type in
     efi64)
-        if [ -f /usr/lib/syslinux/efi64/syslinux.efi ]
-        then    
-            syslinux_efi="/usr/lib/syslinux/efi64/syslinux.efi"
-        else
-            if [ -d /usr/lib/SYSLINUX.EFI/modules/efi64/syslinux.efi ]
-            then
-                 syslinux_efi="/usr/lib/SYSLINUX.EFI/efi64/syslinux.efi"
-            else
-                echo "Syslinux efi bootlader not found"
-                exit 1
-            fi
-        fi
-
-        syslinux_folder=${TMP_DIR}/efi/EFI/boot
-        create_dir $syslinux_folder
-        cp -T ${syslinux_efi} $syslinux_folder/bootx64.efi &> /dev/null
+        syslinux_folder=${tmp_dir}/efi/EFI/boot
+        create_dir "$syslinux_folder"
+        cp -T "$syslinux_efi_bl_folder/$boot_type/syslinux.efi" "$syslinux_folder/bootx64.efi" &> /dev/null
         for file in $SYSLINUX_EFI_FILES
         do
-            echo "copying $file ..."  
-            cp $syslinux_mod/$BOOT_TYPE/$file $syslinux_folder
+            printf "copying %s \n" "$file"  
+            cp "$syslinux_modules_folder/$boot_type/$file" "$syslinux_folder" 
         done
-        for file in $SYSLINUX_FILES
-        do
-            echo "copie de $file"  
-            cp ${syslinux_mod}/${BOOT_TYPE}/${file} $syslinux_folder
-        done
-        cp $repo_path/{vmlinuz,initrd.gz} $TMP_DIR/efi
-        cp -R syslinux/syslinux.cfg  $syslinux_folder &> /dev/null
+        cp "$repo_path"/{vmlinuz,initrd.gz} "$tmp_dir/efi"
+        cp -R syslinux/syslinux.cfg "$syslinux_folder" &> /dev/null
 
-        ;;
+        ;; 
     bios)
-        create_dir ${TMP_DIR}/syslinux 1
-        for file in $SYSLINUX_FILES
-        do
-            echo "copie de $file"  
-            cp ${syslinux_mod}/${BOOT_TYPE}/${file} ${TMP_DIR}/syslinux/
-        done
-
+        syslinux_folder=${tmp_dir}/syslinux
+        create_dir "$syslinux_folder"
         for file in $SYSLINUX_BIOS_FILES
         do
-            echo "copying $file ..."  
-            cp ${syslinux_mod}/${BOOT_TYPE}/${file} ${TMP_DIR}/syslinux/
+            printf "copying %s\n" "$file"   
+            cp "$syslinux_modules_folder/$boot_type/$file" "$syslinux_folder"
         done
-        cp -R syslinux  $TMP_DIR &> /dev/null
+        cp "$repo_path"/{vmlinuz,initrd.gz} "$tmp_dir"
+        cp -R syslinux  "$tmp_dir" &> /dev/null
         ;;
 esac
-echo Copying configurations files 
-create_dir $TMP_DIR/partman_recipes
-cp -R partman_recipes $TMP_DIR &> /dev/null
-create_dir $TMP_DIR/preseeds
-cp -R preseeds $TMP_DIR &> /dev/null
 
-if [[ -n $dest ]]
+for file in $SYSLINUX_FILES
+do
+    printf "copying %s\n" "$file"    
+    cp "$syslinux_modules_folder/$boot_type/$file" "$syslinux_folder"
+done
+
+echo Copying configurations files 
+create_dir "$tmp_dir/partman_recipes"
+cp -R partman_recipes "$tmp_dir" &> /dev/null
+create_dir "$tmp_dir/preseeds"
+cp -R preseeds "$tmp_dir" &> /dev/null
+
+if [ -n "$device" ]
 then
-    read -p "[WARNING] Disk $dest will be erased continue [Y/N]? " -n 1 -r
+    printf "Partitionning and formating %s\n" "$device"
+    read -p "[WARNING] Disk $device will be erased continue [Y/N]? " -n 1 -r
     if [[ ! $REPLY =~ ^[Yy]$ ]]
     then
        exit 1
     fi
+    printf "\n\n"
+    umount_device "$device"
+    create_dir "$key_mountpoint"
     #dd if=/dev/zero of=${dest} bs=512 count=1 conv=notrunc &> /dev/null
-    echo -e "\nCreating partition"
-    parted -s $dest mklabel gpt mkpart EFS fat32 1MiB 128MiB set 1 boot on
-    parted -s $dest mkpart EFS fat32 128MiB 100%
-    mkfs.vfat ${dest}1 -n efi &> /dev/null
-    mkfs.vfat ${dest}2 -n debian &> /dev/null
-
-    # mount and copy
-    create_dir $MOUNT_DIR
-    mount ${dest}2 $MOUNT_DIR
-    create_dir $MOUNT_DIR/efi
-    mount ${dest}1 $MOUNT_DIR/efi
-
-    echo "copying all file to USB drive"
-    cp -R ${TMP_DIR}/* $MOUNT_DIR
-    umount $MOUNT_DIR/efi
-    umount $MOUNT_DIR
-    echo Unmounting key, please wait ...
+    case "$boot_type" in
+        "efi64") 
+            efi_create_key_structure "$device" "$key_mountpoint"
+            ;; 
+        "bios")
+            bios_create_key_structure "$device" "$key_mountpoint"
+            ;;
+    esac
+    printf "copying all file to USB drive\n"
+    cp -R "$tmp_dir"/* "$key_mountpoint"
+    umount_device "$device" 
     sleep 5
     # Make key bootable if bios.
-    if [[ $BOOT_TYPE == "bios" ]]
+    if [ "$boot_type" = "bios" ]
     then
         # In this mode, we need to write syslinux MBR.
-        echo "Writing syslinux mbr.ini fo $dest"
-        dd bs=440 count=1 conv=notrunc if=${syslinux_mod}/${BOOT_TYPE}/mbr.bin of=${dest}
+        printf "Writing syslinux mbr.ini to %s\n"  "$device"
+        dd bs=440 count=1 conv=notrunc if="$syslinux_mbr_bl_folder"/mbr.bin of="$device"
         sleep 2
-        syslinux --directory /syslinux --install ${dest}1
+        "$SYSLINUX_EXE" --directory /syslinux --install "${device}1"
     fi 
-    exit 0
-    #rm -rf $TMP_DIR
+    rm -rf "$tmp_dir"
+    printf "\nBootable USB key created\n"
+    exit 0 
 fi