path: root/scripts/usr/bin/drive-check-duplicates

#!/bin/bash
###################################################################################################
# drive-check-duplicates
###################################################################################################
# Copyright 2017-2023 Jean-Christophe Manciot <jcmanciot@sdxlive.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 
# in compliance with the License. You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# This script is using a project hosted at https://github.com/odeke-em/drive, originally developed 
# by Burcu Dogan while working on the Google Drive team and now maintained by Emmanuel T Odeke.
#
# Attribution — You must give appropriate credit, provide a link to the license, and indicate if 
#               changes were made. You may do so in any reasonable manner, but not in any way that 
#               suggests the licensor endorses you or your use.
###################################################################################################

###################################################################################################
#
# Checking for duplicates by looking for "clashes", i.e objects (objects) with the same name in 
# the same location.
#
###################################################################################################
#
# Requirements:
# ------------
#
# - We assume that:
#       + drive-google package is already installed.
#       + coreutils, perl-base, sed & zenity packages are already installed.
#
# - This command must be called from within your local Google drive folder(s).
#
# - <google_drive_folder>/.driverc:it must be properly setup, either manually or with drive-setup,
#                                  with the following settings:
#       * depth=
#               + -1: an infinite traversal depth, i.e recursive traversal from the current working  
#                     directory until there is no more child folder
#               + non-zero number: a fixed traversal depth
#       * fix-mode=
#               + rename: default value telling drive to rename duplicate objects
#               + trash:  tells drive to trash duplicate objects instead
#       * ...: cf. https://github.com/odeke-em/drive for a complete list of available  
#              options. They can only be set manually through .driverc or through drive-setup for
#              the sake of simplicity.
#              Cf. /etc/drive-google/.driverc for some examples.
#
# - <google_drive_folder>/.driveignore: it may be used to exclude/include some objects from the 
#                                       scope of the current drive-<command> by using regular
#                                       expressions.
#                                       Cf. /etc/drive-google/.driveignore for its format & some 
#                                       examples.
#
# - ./.driveselect: it may be used to include/exclude the objects from the scope of the current 
#                   drive-<command>. 
#                   Cf. /etc/drive-google/.driveselect for its format & some examples.
#
###################################################################################################
#
# Parameters:
# ----------
#
# - <anything> will display this help message
#
###################################################################################################
#
# Usage example:
# -------------
#
# cd ~/<google_drive_folder>
# drive-check-duplicates
#
###################################################################################################
if [[ $LOG_DRIVE_CHECK_DUPLICATES == ON ]]; then
        # Getting the last existing file descriptor of ~/.drive-google/logs/drive-google.log
        fd=$(ls -al /proc/$$/fd | grep ~/.drive-google/logs/drive-google.log | sed -E 's|^.* ([0-9]+) -> .*$|\1|g' | sort -V | tail -n 1)

        # Case where fd is unset, i.e the file ~/.drive-google/logs/drive-google.log has not yet been opened
        if [[ -z $fd ]]; then
                lowest_unused_fd () 
                {
                        local fd=0
                        while [ -e /proc/$$/fd/${fd} ]; do
                                fd=$((fd+1))
                        done
                        echo $fd
                }
                fd=$(lowest_unused_fd)

                # Opening ~/.drive-google/logs/drive-google.log as file descriptor $fd for appending
                eval "exec $fd>> ~/.drive-google/logs/drive-google.log"
        fi

        # Writing the trace output generated when set -x is enabled to file descriptor $fd 
        eval "BASH_XTRACEFD=$fd"
        # Logging line numbers - We could also use ${0} for ${BASH_SOURCE}
        BASH_SOURCE_BASENAME=$(basename ${BASH_SOURCE})
        export PS4='${BASH_SOURCE_BASENAME}.${LINENO}+ '
        # Expanding all variables and prints the full commands before output of the command
        set -x
fi

# Preventing globbing (pathname expansion)
set -o noglob
# Allowing extended globbing
shopt -s extglob
# Expanding aliases
shopt -s expand_aliases

# Including alias definitions to sed/grep/awk/perl commands for readability sake
. /usr/include/drive-google/aliases

#------------------------------------check-duplicate-objects---------------------------------------
# Checking for duplicates among all objects listed in "${filtered_objects[@]}"
# Usage: check-duplicate-objects | zenity --progress --percentage=0 --auto-close
# Return strings which feed 'zenity --progress'
# - "# Checking for duplicate $object"
# - "$current_percentage"
# The function uses the global array variables:
# - "${filtered_objects[@]}"
function check-duplicate-objects ()
{
        # Initializing:
        # - current checked objects number so far,
        # - current percentage of checked objects so far,
        current_objects_number=0
        current_percentage=0

        # Reading fix_mode from .driverc
        fix_mode_value=$(.drive-read-driverc fix-mode clashes)

        if [[ "$fix_mode_value" != 'trash' ]]; then
                # Setting default value
                fix_mode_value=rename
        fi

        for object in "${filtered_objects[@]}"
        do
                # Removing leading object stats
                object=$(echo "$object" | rm_object_stats)

                # Feeding 'zenity --progress'
                LC_ALL=C printf '%s\n' "# Checking for duplicate $object"

                # Removing all occurrences of '&' in $object as a workaround to a markup ampersand issue with zenity
                noamp_object=$(echo "${object}" | rm_ampersand)

                fix=false

                # While there is a connection error and the user wants to continue trying
                while :
                do
                        unset output

                        if [[ $fix == false ]]; then
                                # Checking for duplicates
                                output=$(drive clashes -depth 1 "$object" 2>&1)
                        else
                                # Fixing duplicates by trashing or renaming them
                                output=$(drive clashes -fix -fix-mode $fix_mode_value -no-prompt -depth 1 "$object" 2>&1)
                        fi

                        # Checking $output
                        if [[ "${output[@]}" =~ 'no clashes exist!' ]]; then
                                output="No duplicate:\t\t${object}"
                        elif [[ "${output[@]}" =~ 'Get https://www.googleapis.com/drive/' ]]; then
                                # Connection error
                                # Displaying the error 
                                zenity --error --title "Connection error accessing your Google Drive online in ${noamp_local_folder}" --text="${noamp_object}\n\nCould not connect to https://www.googleapis.com/drive/v2/files or/nor to https://accounts.google.com/o/oauth2/token" --width 1400
                                zenity --question --title "Connection error accessing your Google Drive online in ${noamp_local_folder}" --text="${noamp_object}\n\nDo you want to abort current drive operation?" --width 1400

                                if [[ $? -eq 1 ]]; then
                                        # Continuing trying
                                        continue
                                elif [[ $? -eq 0 ]]; then
                                        # Aborting
                                        exit 9
                                fi
                        elif [[ "${output[@]}" =~ "doesn't exist" ]]; then
                                output="Absent:\t\t\t${object}"
                        else
                                if [[ $fix == false ]]; then
                                        fix=true
                                        continue
                                else
                                        unset output
                                        output="Fixed duplicate:\t${object}"
                                fi
                        fi

                        if [[ ("$output" =~ 'No duplicate') || ("$output" =~ 'Fixed duplicate') || ("$output" =~ 'Absent') ]]; then
                                (( current_objects_number++ ))

                                # Computing the current percentage of pulled objects
                                current_percentage=$(echo "scale=4; (${current_objects_number}/${#filtered_objects[@]})*100" | bc)

                                # Feeding 'zenity --progress'
                                LC_ALL=C printf '%.2f\n' "$current_percentage"
                        fi

                        # We're done
                        break
                done

                # Saving $output in temporary .driveoutput.log file after having escaped all "escape sequences"
                printf '%b\n' "$output" | escape_escape_sequences >> .driveoutput.log
        done
}

#------------------------------------drive-check-duplicates----------------------------------------

# Checking for no parameter
if [[ $# -eq 0 ]]; then
        local_folder="$(pwd)"

        # Removing all occurrences of '&' in $object as a workaround to a markup ampersand issue with zenity
        noamp_local_folder="$(echo "${local_folder}" | rm_ampersand)"

        # Finding local root Google Drive folder
        gd_root_folder="$(.drive-find-gd)"
        return_code=$?

        # Removing all occurrences of '&' in $gd_root_folder as a workaround to a markup ampersand issue with zenity
        noamp_gd_root_folder="$(echo "${gd_root_folder}" | rm_ampersand)"

        case $return_code in
                0)
                        # Everything is fine - we continue
                        ;;
                7)
                        zenity --warning --title "Drive context found in $noamp_gd_root_folder" --text="We have moved to $noamp_gd_root_folder\n\nIf you prefer to select another Google Drive folder:\n\t--> 'Change local folder' in drive-menu" --width 1400

                        cd "$gd_root_folder"

                        local_folder="$(pwd)"

                        # Removing all occurrences of '&' in $object as a workaround to a markup ampersand issue with zenity
                        noamp_local_folder="$(echo "${local_folder}" | rm_ampersand)"
                        ;;
                8)
                        zenity --error --title "No drive context found below and above $noamp_local_folder" --text="- either drive has not been initialized properly (at least one hop away from '/') with:\n\t--> '# drive init google_drive_folder' within a terminal\n- or we are not located below your google_drive_folder top level branch:\n\t--> 'Change local folder' in drive-menu" --width 1400
                        exit $return_code
                        ;;
                *)
                        # Something is fatally wrong
                        zenity --error --title "Fatal error in drive-menu" --text="\tUnknown return code from .drive-find-gd: $return_code\n\nPlease report that issue to: https://gitlab.com/jean-christophe-manciot/Drive/issues" --width 1200
                        exit 99
                        ;;
        esac

        # Filtering & sorting the objects list
        readarray -t filtered_objects < <(.drive-select-objects 'remote_fs' 'clashes')

        if [[ "${filtered_objects[0]}" =~ 'Connection error accessing your Google Drive online in' ]]; then
                # Nothing can be done
                zenity --error --title "Fatal Error" --text="Operation cancelled" --width 800
                exit 9
        elif [[ ${#filtered_objects[@]} -eq 0 ]]; then
                zenity --warning --title "Checking for duplicates among 0 remote Google Drive objects in ${noamp_local_folder}" --text="None needs to be checked" --width=1400
                exit 0
        fi

        printf '%s\n' "${filtered_objects[@]}" | zenity --text-info --title "Do you confirm checking for duplicates among the following ${#filtered_objects[@]} remote Google Drive objects in ${noamp_local_folder}?" --height 800 --width 1400 
        if [[ ${PIPESTATUS[1]} -eq 0 ]]; then
                # OK has been selected
                > .driveoutput.log

                # Checking for duplicates among all objects listed in "${filtered_objects[@]}"
                check-duplicate-objects | zenity --progress --percentage=0 --auto-close --title="Checking for duplicates among ${#filtered_objects[@]} remote Google Drive objects" --width 1800

                readarray -t unduplicate_objects < <(cat .driveoutput.log | select_duplicate)

                cat .driveoutput.log | zenity --text-info --title "Found unduplicated/deduplicated ${#unduplicate_objects[@]}/${#filtered_objects[@]} remote Google Drive objects in ${noamp_local_folder}" --width 1800 --height 800
                rm .driveoutput.log
        else
                zenity --warning --title "Do you confirm checking for duplicates among the following ${#filtered_objects[@]} remote Google Drive objects?" --text="Operation cancelled" --width=1200
                exit 3
        fi

        exit 0
else
        # Any parameter
        end_of_help_message_line_number=$(awk '/^if \[\[ \$LOG_/{ print NR; exit }' $(which drive-check-duplicates))
        ((end_of_help_message_line_number--))
        awk -v var=$end_of_help_message_line_number 'NR >= 20 && NR <= var' $(which drive-check-duplicates) | zenity --text-info --title "drive-check-duplicates help" --width 1150 --height 1000
        exit 1
fi