How to fix "You have to delete all the items"

I have this script that delete OneDrive folder but I'm alway getting this error so just wondering how can I fix this?.

I provided the parent folder and sometime it can delete multiple subfolder levels items but randomly I get this error and it stop executing.

"Error: You have to delete all the items in this folder before you can delete the folder." 

#Config Variables
$SiteURL = "https://companyName-my.sharepoint.com/personal/user_id/"
$ListName = "Documents"
$FolderServerRelativeURL = "/personal/parentfolderpath"
function Remove-Folder {
    Try {
        #Connect to PnP Online
        Connect-PnPOnline -Url $SiteURL -useWebLogin
        $List = Get-PnPList $ListName
    
            
        #Get All Items from Folder in Batch
        $global:counter = 0;
        $ListItems = Get-PnPListItem -List $ListName -PageSize 500 -Fields ID, FileDirRef, FileRef -ScriptBlock `
        { Param($items) $global:counter += $items.Count; Write-Progress -PercentComplete ($global:Counter / ($List.ItemCount) * 100) -Activity `
                "Getting Items from Folder '$FolderServerRelativeURL'" -Status "Getting Items $global:Counter of $($List.ItemCount)"; }
     
        #Get List Items from the folder and Sort List Items based on Server Relative URL - Descending
        $ItemsFromFolder = $ListItems | Where { $_.FieldValues["FileDirRef"] -match $FolderServerRelativeURL } | Select-Object @{Name = "ServerRelativeURL"; Expression = { $_.FieldValues.FileRef } }, ID | Sort-Object -Descending -Property ServerRelativeURL
        Write-host "Total Number of Items Found in the Folder:"$ItemsFromFolder.count
              
        #Delete List Items in Batch
        $Batch = New-PnPBatch
        ForEach ($Item in $ItemsFromFolder) {
            try {
                Remove-PnPListItem -List $ListName -Identity $Item.ID  -Batch $Batch
                Write-host "Item Queued for deletion:"$Item.ServerRelativeURL
                Invoke-PnPBatch -Batch $Batch -Details
                $ErrorActionPreference = "Continue"
            }
            catch {
                Write-host "Item not found " $Item.ServerRelativeURL
            }
        }
    }
    Catch {
        write-host "Error: $($_.Exception.Message)" -foregroundcolor Red
    }
}