path: root/scripts/usr/bin/.drive-pull-unsynced

#!/bin/bash
###################################################################################################
# .drive-pull-unsynced
###################################################################################################
# Copyright 2017-2023 Jean-Christophe Manciot <jcmanciot@sdxlive.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 
# in compliance with the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# This script is using a project hosted at https://github.com/odeke-em/drive, originally developed 
# by Burcu Dogan while working on the Google Drive team and now maintained by Emmanuel T Odeke.
#
# Attribution — You must give appropriate credit, provide a link to the license, and indicate if 
#               changes were made. You may do so in any reasonable manner, but not in any way that 
#               suggests the licensor endorses you or your use.
###################################################################################################

###################################################################################################
#
# Pulling (downloading) recursively all the files/folders (objects) listed in .driveselect under 
# the section [pull] from the current directory's remote equivalent on your Google Drive.online 
# into the local current directory.
# This operation:
# - updates only unsynced objects that exist locally with their remote counterpart. 
# - downloads only unsynced objects that do not exist locally but do remotely, 
# - **does not** delete local objects that exist locally but are not present remotely, 
#   except when there are children folders.
# The scale of the operation depends on the depth setting in your <google_drive_folder>/.driverc.
# The list of objects affected by this operation can be optionnaly filtered by
# <google_drive_folder>/.driveignore.
#
###################################################################################################
#
# Requirements:
# ------------
#
# - We assume that:
#       + drive-google package is already installed.
#       + coreutils, perl-base, sed & zenity packages are already installed.
#
# - **This is an internal script called by drive-sync when necessary.**
#
# - <google_drive_folder>/.driverc:it must be properly setup, either manually or with drive-setup,
#                                  with the following settings:
#       * depth=
#               + -1: an infinite traversal depth, i.e recursive traversal from the current working  
#                     directory until there is no more child folder
#               + non-zero number: a fixed traversal depth
#       * ignore-checksum=
#               + true: files checksum is not verified 
#               + false: files checksum is verified 
#       * decryption-password="<your_secret>": drive optionally decrypts all pulled files which 
#                                              are encrypted at rest on your Google Drive online.
#       * ...: cf. https://github.com/odeke-em/drive for a complete list of available  
#              options. They can only be set manually through .driverc or through drive-setup for
#              the sake of simplicity.
#              Cf. /etc/drive-google/.driverc for some examples.
#
# - <google_drive_folder>/.driveignore: it may be used to exclude/include some objects from the 
#                                       scope of the current drive-<command> by using regular
#                                       expressions.
#                                       Cf. /etc/drive-google/.driveignore for its format & some 
#                                       examples.
#
# - ./.driveselect: it may be used to include/exclude the objects from the scope of the current 
#                   drive-<command>. 
#                   Cf. /etc/drive-google/.driveselect for its format & some examples.
#
###################################################################################################
#
# Parameters:
# ----------
#
# - <message> is used in the interactive GUI windows
#
# - -<anything> will display this help message
#
###################################################################################################
#
# Notes:
# -----
#
# - One must be **cautious** with this command: a list of potentially affected objects will be
#   printed before the actual operation.
#   I recommend to perform a backup of your local Google Drive folder before running this command.
#
# - This command must be used when **recursively found remote objects are newer or more important 
#   than their local counterparts.**
#   In other words, run this command when **your remote Google Drive is the master**.
#
# - Running this command pulls some remote objects locally; its speed is proportional to: 
#       + the number of different objects recursively found
#       + the size of those remote objects recursively pulled locally
#       + the end-to-end bandwidth, latency & quality of your current Internet connection to your 
#         Google Drive online.
#
###################################################################################################
if [[ $LOG_DRIVE_PULL_UNSYNCED == ON ]]; then
        # Getting the last existing file descriptor of ~/.drive-google/logs/drive-google.log
        fd=$(ls -al /proc/$$/fd | grep ~/.drive-google/logs/drive-google.log | sed -E 's|^.* ([0-9]+) -> .*$|\1|g' | sort -V | tail -n 1)

        # Case where fd is unset, i.e the file ~/.drive-google/logs/drive-google.log has not yet been opened
        if [[ -z $fd ]]; then
                lowest_unused_fd () 
                {
                        local fd=0
                        while [ -e /proc/$$/fd/${fd} ]; do
                                fd=$((fd+1))
                        done
                        echo $fd
                }
                fd=$(lowest_unused_fd)

                # Opening ~/.drive-google/logs/drive-google.log as file descriptor $fd for appending
                eval "exec $fd>> ~/.drive-google/logs/drive-google.log"
        fi

        # Writing the trace output generated when set -x is enabled to file descriptor $fd 
        eval "BASH_XTRACEFD=$fd"
        # Logging line numbers - We could also use ${0} for ${BASH_SOURCE}
        BASH_SOURCE_BASENAME=$(basename ${BASH_SOURCE})
        export PS4='${BASH_SOURCE_BASENAME}.${LINENO}+ '
        # Expanding all variables and prints the full commands before output of the command
        set -x
fi

# Preventing globbing (pathname expansion)
set -o noglob
# Allowing extended globbing
shopt -s extglob
# Expanding aliases
shopt -s expand_aliases

# Including alias definitions to sed/grep/awk/perl commands for readability sake
. /usr/include/drive-google/aliases

#------------------------------------pull-objects--------------------------------------------------
# Pulling all objects listed in "${filtered_objects[@]}"
# Usage: pull-objects | zenity --progress --percentage=0 --auto-close
# Return strings which feed 'zenity --progress'
# - "# Pulling $object at ${speed_eta[0]}/s ETA: ${speed_eta[1]}"
# - "$current_percentage"
# The function uses the global array variables:
# - "${filtered_objects[@]}"
function pull-objects ()
{
        # Initializing:
        # - starting point of the operation in seconds,
        # - elapsed time since the operation has started in seconds,
        # - current size of pulled objects so far,
        # - current percentage of pulled objects so far,
        # - total size of all objects to be pulled,
        # - current objects number to be pulled.
        start=0
        elapsed_time=0
        current_size=0
        current_percentage=0
        total_size=0
        current_objects_number=0

        # Computing total size of all objects
        for object in "${filtered_objects[@]}"
        do
                # Getting the object size in bytes
                object_size=$(.drive-compute-object-size "$object")

                let "total_size += $object_size"
        done

        start=$SECONDS

        for object in "${filtered_objects[@]}"
        do
                ((current_objects_number++))

                # Getting the object size in bytes
                object_size=$(.drive-compute-object-size "$object")

                # Removing leading object stats
                object=$(echo "$object" | rm_object_stats)

                # Feeding 'zenity --progress'
                if [[ $current_objects_number -ne 1 ]]; then
                        LC_ALL=C printf '%s %s%s %s\n' "# Pulling $object" "at ${speed_eta[0]}" "/s" "ETA: ${speed_eta[1]}"
                else
                        # $current_size is 0
                        LC_ALL=C printf '%s\n' "# Pulling $object"
                fi

                # Removing all occurrences of '&' in $object as a workaround to a markup ampersand issue with zenity
                noamp_object=$(echo "${object}" | rm_ampersand)

                # While there is a connection error and the user wants to continue trying
                while :
                do
                        # Pulling the $object
                        unset output
                        output=$(drive pull -force -no-prompt -depth 0 -verbose "$object" 2>&1)

                        # Checking $output
                        if [[ "${output[@]}" =~ 'set to be ignored yet is being processed' ]]; then
                                output="Ignored:\t\t${object}"
                        elif [[ "${output[@]}" =~ 'Get https://www.googleapis.com/drive/' ]]; then
                                # Connection error
                                # Displaying the error 
                                zenity --error --title "Connection error accessing your Google Drive online in ${noamp_local_folder}" --text="${noamp_object}\n\nCould not connect to https://www.googleapis.com/drive/v2/files or/nor to https://accounts.google.com/o/oauth2/token" --width 1400
                                zenity --question --title "Connection error accessing your Google Drive online in ${noamp_local_folder}" --text="${noamp_object}\n\nDo you want to abort current drive operation?" --width 1400

                                if [[ $? -eq 1 ]]; then
                                        # Continuing trying
                                        continue
                                elif [[ $? -eq 0 ]]; then
                                        # Aborting
                                        exit 9
                                fi
                        elif [[ "${output[@]}" =~ "doesn't exist" ]]; then
                                output="Absent:\t\t\t${object}"
                        elif [[ "${output[@]}" =~ "local is a directory while remote is a file" ]]; then
                                output="Inconsistent types:\t${object}"
                        elif [[ "${output[@]}" =~ "local is a file while remote is a directory" ]]; then
                                output="Inconsistent types:\t${object}"
                        elif [[ "${output[@]}" =~ 'Everything is up-to-date'\. ]]; then
                                output="Already synced:\t\t${object}"
                        elif [[ "${output[@]}" =~ Resolving\.\.\. ]]; then
                                output="Pulled:\t\t\t${object}"
                        else
                                output="??????:\t\t\t${object}"
                        fi

                        if [[ ("$output" =~ 'Pulled') || ("$output" =~ 'Already synced') ]]; then
                                # Adding the object size to the current size of pulled objects so far
                                let "current_size += $object_size"

                                elapsed_time=$(( SECONDS - start ))

                                # Computing speed & estimated time of arrival
                                readarray -t speed_eta < <(.drive-compute-speed-eta $current_size $total_size $elapsed_time)

                                # Computing the current percentage of pulled objects
                                current_percentage=$(echo "scale=4; (${current_size}/${total_size})*100" | bc)

                                # Feeding 'zenity --progress'
                                LC_ALL=C printf '%.2f\n' "$current_percentage"
                        fi

                        # We're done
                        break
                done

                # Saving $output in temporary .driveoutput.log file after having escaped all "escape sequences"
                printf '%b\n' "$output" | escape_escape_sequences >> .driveoutput.log
        done
}

#-----------------------------------.drive-pull-unsynced-------------------------------------------

message=$1

if [[ ! "$message" =~ ^- ]]; then
        local_folder="$(pwd)"

        # Removing all occurrences of '&' in $object as a workaround to a markup ampersand issue with zenity
        noamp_local_folder="$(echo "${local_folder}" | rm_ampersand)"

        # Finding local root Google Drive folder
        gd_root_folder="$(.drive-find-gd)"
        return_code=$?

        # Removing all occurrences of '&' in $gd_root_folder as a workaround to a markup ampersand issue with zenity
        noamp_gd_root_folder="$(echo "${gd_root_folder}" | rm_ampersand)"

        case $return_code in
                0)
                        # Everything is fine - we continue
                        ;;
                7)
                        zenity --warning --title "Drive context found in $noamp_gd_root_folder" --text="We have moved to $noamp_gd_root_folder\n\nIf you prefer to select another Google Drive folder:\n\t--> 'Change local folder' in drive-menu" --width 1400

                        cd "$gd_root_folder"

                        local_folder="$(pwd)"

                        # Removing all occurrences of '&' in $object as a workaround to a markup ampersand issue with zenity
                        noamp_local_folder="$(echo "${local_folder}" | rm_ampersand)"
                        ;;
                8)
                        zenity --error --title "No drive context found below and above $noamp_local_folder" --text="- either drive has not been initialized properly (at least one hop away from '/') with:\n\t--> '# drive init google_drive_folder' within a terminal\n- or we are not located below your google_drive_folder top level branch:\n\t--> 'Change local folder' in drive-menu" --width 1400
                        exit $return_code
                        ;;
                *)
                        # Something is fatally wrong
                        zenity --error --title "Fatal error in drive-menu" --text="\tUnknown return code from .drive-find-gd: $return_code\n\nPlease report that issue to: https://gitlab.com/jean-christophe-manciot/Drive/issues" --width 1200
                        exit 99
                        ;;
        esac

        # Filtering & sorting the objects list
        filtered_objects=()
        readarray -t filtered_objects < <(.drive-select-objects 'select_file' 'pull')

        if [[ ${#filtered_objects[@]} -eq 0 ]]; then
                zenity --warning --title "Pulling 0 $message" --text="None needs to be pulled" --width=1400
                exit 0
        fi

        # Purging the section [pull] in the file .driveselect
        .drive-write-driveselect pull "${null[@]}"

        printf '%s\n' "${filtered_objects[@]}" | zenity --text-info --title "Do you confirm pulling ${#filtered_objects[@]} $message?" --width 1800 --height 800
        if [[ ${PIPESTATUS[1]} -eq 0 ]]; then
                # OK has been selected
                > .driveoutput.log

                # Pulling all objects listed in ${filtered_objects[@]}
                pull-objects | zenity --progress --percentage=0 --auto-close --title="Pulling ${#filtered_objects[@]} $message" --width 1800

                readarray -t pulled_objects < <(cat .driveoutput.log | select_Pulled)

                cat .driveoutput.log | zenity --text-info --title "Pulled ${#pulled_objects[@]}/${#filtered_objects[@]} $message" --width 1800 --height 800
                rm .driveoutput.log
        else
                zenity --warning --title "Do you confirm pulling ${#filtered_objects[@]} $message?" --text="Operation cancelled" --width=1400
        fi

        exit 0
else
        # Any -parameter
        end_of_help_message_line_number=$(awk '/^if \[\[ \$LOG_/{ print NR; exit }' $(which drive-pull))
        ((end_of_help_message_line_number--))
        awk -v var=$end_of_help_message_line_number 'NR >= 20 && NR <= var' $(which drive-pull) | zenity --text-info --title "drive-pull help" --width 1125 --height 1000
        exit 1
fi