Forum Discussion
SharePoint Migration Tool after failed copying
Hi Kevin (@kevinbull1720),
The reason the original script isn't seeing your existing SharePoint files is twofold:
- Path Resolution: Get-PnPFile requires the exact server-relative URL (e.g., /sites/yoursite/Shared Documents/...). If you pass "Documents/..." (display name) or omit the site path, SharePoint returns a 404. Because -ErrorAction SilentlyContinue was used, it returned $null, treating every file as missing/new.
- Upload Cmdlet: The original script also used Copy-PnPFile (which only works within SharePoint) instead of Add-PnPFile (for uploading local/file-share files).
I've rewritten the script to fix this and address your empty subfolders:
- Pre-indexes SPO (UseIndexCache = $true): Caches your SharePoint library inventory in memory once, making comparisons instant and eliminating API throttling/path lookup errors.
- Empty Subfolder Sync (SyncEmptyDirectories = $true): Recreates any missing or empty subfolder structures left behind by Migration Manager.
- Auto-resolves URLs: Dynamically detects the server-relative path whether you pass "Documents" or "Shared Documents".
- Correct Uploads: Uses Add-PnPFile and compares timestamps in UTC.
# Script: SharePoint-IncrementalSync.ps1
[CmdletBinding(SupportsShouldProcess = $true)]
param (
[Parameter(Mandatory = $false)]
[string]$SourcePath,
[Parameter(Mandatory = $false)]
[string]$SharePointSiteUrl,
[Parameter(Mandatory = $false)]
[string]$LibraryName,
[Parameter(Mandatory = $false)]
[switch]$WhatIf = $true,
[Parameter(Mandatory = $false)]
[switch]$SyncEmptyDirectories = $true,
[Parameter(Mandatory = $false)]
[int]$BatchSize = 25,
[Parameter(Mandatory = $false)]
[switch]$UseIndexCache = $true
)
# Self-elevate if not running as administrator (Windows PowerShell)
if ($PSVersionTable.PSVersion.Major -le 5 -and -not ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")) {
Write-Warning "Running as Administrator is recommended. Attempting to self-elevate..."
Start-Process powershell.exe "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`"" -Verb RunAs
exit
}
# Function to ensure PowerShell version compatibility
function Test-PowerShellRequirements {
$minVersion = "5.1"
$currentVersion = $PSVersionTable.PSVersion.ToString()
if ([version]$currentVersion -lt [version]$minVersion) {
throw "PowerShell version $minVersion or higher is required. Current version: $currentVersion"
}
}
# Function to install and import required modules
function Install-RequiredModules {
$moduleName = "PnP.PowerShell"
try {
if (-not (Get-Module -ListAvailable -Name $moduleName)) {
Write-Host "Installing $moduleName module for CurrentUser..." -ForegroundColor Yellow
Install-Module -Name $moduleName -Force -AllowClobber -Scope CurrentUser
}
Import-Module -Name $moduleName -Force
Write-Host "PnP.PowerShell module loaded successfully." -ForegroundColor Green
}
catch {
throw "Failed to install/import $($moduleName): $_"
}
}
# Main sync function
function Start-SharePointIncrementalSync {
param (
[string]$SourcePath,
[string]$SharePointSiteUrl,
[string]$LibraryName,
[bool]$WhatIf = $true,
[bool]$SyncEmptyDirectories = $true,
[int]$BatchSize = 25,
[bool]$UseIndexCache = $true
)
$script:processedCount = 0
$script:errorCount = 0
$script:copiedCount = 0
$script:skippedCount = 0
$script:folderCreatedCount = 0
$cleanSourcePath = $SourcePath.TrimEnd('\', '/')
if (-not (Test-Path -Path $cleanSourcePath)) {
throw "Source path '$cleanSourcePath' does not exist or is not accessible."
}
# Setup Logging
$logDirectory = if ($PSScriptRoot) { $PSScriptRoot } else { [System.IO.Directory]::GetCurrentDirectory() }
$logPath = Join-Path $logDirectory "SPSync_$(Get-Date -Format 'yyyyMMdd_HHmmss').log"
Start-Transcript -Path $logPath -Append
Write-Host "==========================================================" -ForegroundColor Cyan
Write-Host " SharePoint Online Incremental Sync & Comparison Tool" -ForegroundColor Cyan
Write-Host "==========================================================" -ForegroundColor Cyan
Write-Host "Source: $cleanSourcePath" -ForegroundColor White
Write-Host "SharePoint Site: $SharePointSiteUrl" -ForegroundColor White
Write-Host "Target Library: $LibraryName" -ForegroundColor White
Write-Host "WhatIf (Dry Run): $WhatIf" -ForegroundColor $(if ($WhatIf) { "Yellow" } else { "Magenta" })
Write-Host "Sync Empty Folders: $SyncEmptyDirectories" -ForegroundColor White
Write-Host "Use Index Cache: $UseIndexCache" -ForegroundColor White
Write-Host "Log file: $logPath" -ForegroundColor Gray
Write-Host "----------------------------------------------------------" -ForegroundColor Cyan
try {
# Connect to SharePoint Online
Write-Host "`nConnecting to SharePoint Online..." -ForegroundColor Cyan
Connect-PnPOnline -Url $SharePointSiteUrl -Interactive
$web = Get-PnPWeb -Includes ServerRelativeUrl
Write-Host "Successfully connected to $($web.ServerRelativeUrl)" -ForegroundColor Green
# Resolve Document Library and Root Folder
Write-Host "Resolving target document library '$LibraryName'..." -ForegroundColor Cyan
$library = Get-PnPList -Identity $LibraryName -Includes RootFolder -ErrorAction SilentlyContinue
if (-not $library) {
# Try finding by title or internal url
$allLists = Get-PnPList -Includes RootFolder
$library = $allLists | Where-Object { $_.Title -eq $LibraryName -or $_.RootFolder.Name -eq $LibraryName }
}
if (-not $library) {
throw "Could not locate library '$LibraryName' on site '$SharePointSiteUrl'. Please check library name."
}
$libraryServerRelativeUrl = $library.RootFolder.ServerRelativeUrl.TrimEnd('/')
$webServerRelativeUrl = $web.ServerRelativeUrl.TrimEnd('/')
# Site-relative path of the library root (e.g. 'Shared Documents' or 'sites/sub/Shared Documents')
$librarySiteRelativeUrl = if ($webServerRelativeUrl -eq "" -or $webServerRelativeUrl -eq "/") {
$libraryServerRelativeUrl.TrimStart('/')
} else {
$libraryServerRelativeUrl.Substring($webServerRelativeUrl.Length).TrimStart('/')
}
Write-Host "Document Library Title: $($library.Title)" -ForegroundColor Gray
Write-Host "Library Server-Relative URL: $libraryServerRelativeUrl" -ForegroundColor Gray
Write-Host "Library Site-Relative URL: $librarySiteRelativeUrl" -ForegroundColor Gray
# Cache existing SharePoint files to avoid thousands of slow API calls
$spFileCache = @{}
if ($UseIndexCache) {
Write-Host "`nScanning existing files in SharePoint library (this may take a moment)..." -ForegroundColor Cyan
try {
# Load all items from the library with necessary fields
$spItems = Get-PnPListItem -List $library -PageSize 2000 -Fields "FileRef", "FileLeafRef", "Modified", "FSObjType"
$spFileCount = 0
foreach ($item in $spItems) {
if ($item.FieldValues["FSObjType"] -eq 0 -or $item.FileSystemObjectType -eq "File") {
$fileRef = $item.FieldValues["FileRef"]
if ($fileRef -and $fileRef.StartsWith($libraryServerRelativeUrl, [System.StringComparison]::OrdinalIgnoreCase)) {
# Key: normalized relative path without leading slash, lowercased
$relItemKey = $fileRef.Substring($libraryServerRelativeUrl.Length).TrimStart('/').ToLower()
$spFileCache[$relItemKey] = @{
ServerRelativeUrl = $fileRef
ModifiedUtc = [DateTime]$item.FieldValues["Modified"]
}
$spFileCount++
}
}
}
Write-Host "Cataloged $spFileCount existing files from SharePoint library into memory." -ForegroundColor Green
}
catch {
Write-Warning "Could not batch index items via Get-PnPListItem ($($_.Exception.Message)). Falling back to direct query mode."
$UseIndexCache = $false
}
}
# Sync/Create missing directories (including empty subfolders)
if ($SyncEmptyDirectories) {
Write-Host "`nChecking directory structure and empty folders..." -ForegroundColor Cyan
$sourceDirs = Get-ChildItem -Path $cleanSourcePath -Recurse -Directory
foreach ($dir in $sourceDirs) {
$dirRelPath = $dir.FullName.Substring($cleanSourcePath.Length).TrimStart('\')
$targetDirSiteRelative = "$librarySiteRelativeUrl/" + $dirRelPath.Replace('\', '/')
if (-not $WhatIf) {
try {
Resolve-PnPFolder -SiteRelativePath $targetDirSiteRelative -ErrorAction SilentlyContinue | Out-Null
}
catch {
Write-Warning "Could not resolve folder '$targetDirSiteRelative': $($_.Exception.Message)"
}
}
}
}
# Enumerate source files
Write-Host "`nScanning source files in: $cleanSourcePath" -ForegroundColor Cyan
$sourceFiles = Get-ChildItem -Path $cleanSourcePath -Recurse -File
$totalFiles = $sourceFiles.Count
Write-Host "Found $totalFiles files to compare and process." -ForegroundColor Cyan
if ($totalFiles -eq 0) {
Write-Host "No files found at source path. Sync complete." -ForegroundColor Yellow
return
}
# Process source files in batches
for ($i = 0; $i -lt $totalFiles; $i += $BatchSize) {
$batch = $sourceFiles | Select-Object -Skip $i -First $BatchSize
foreach ($sourceFile in $batch) {
$script:processedCount++
$percent = [math]::Round(($script:processedCount / $totalFiles) * 100, 1)
Write-Progress -Activity "Comparing & Syncing Files" -Status "$script:processedCount of $totalFiles ($percent%)" -PercentComplete $percent
# Determine relative path in a case-insensitive, clean way
if ($sourceFile.FullName.StartsWith($cleanSourcePath, [System.StringComparison]::OrdinalIgnoreCase)) {
$relativePath = $sourceFile.FullName.Substring($cleanSourcePath.Length).TrimStart('\')
} else {
$relativePath = $sourceFile.FullName.Replace($cleanSourcePath, '').TrimStart('\')
}
$normalizedRelPath = $relativePath.Replace('\', '/')
$cacheKey = $normalizedRelPath.ToLower()
$parentDir = [System.IO.Path]::GetDirectoryName($relativePath)
$targetSiteRelativeFolder = if ([string]::IsNullOrWhiteSpace($parentDir)) {
$librarySiteRelativeUrl
} else {
"$librarySiteRelativeUrl/" + $parentDir.Replace('\', '/')
}
$serverRelativeFilePath = "$libraryServerRelativeUrl/$normalizedRelPath"
try {
$spFileFound = $false
$spLastModifiedUtc = $null
if ($UseIndexCache) {
if ($spFileCache.ContainsKey($cacheKey)) {
$spFileFound = $true
$spLastModifiedUtc = $spFileCache[$cacheKey].ModifiedUtc
}
}
else {
# Direct lookup using exact ServerRelativeUrl
$spFile = Get-PnPFile -Url $serverRelativeFilePath -ErrorAction SilentlyContinue
if ($spFile) {
$spFileFound = $true
$spLastModifiedUtc = [DateTime]$spFile.TimeLastModified
}
}
$sourceLastModifiedUtc = $sourceFile.LastWriteTimeUtc
if ($spFileFound) {
# Allow 2-second timestamp tolerance for filesystem precision differences
$diffSeconds = ($sourceLastModifiedUtc - $spLastModifiedUtc).TotalSeconds
if ($diffSeconds -gt 2) {
Write-Host "[$script:processedCount/$totalFiles] [NEWER SOURCE] Updating: $relativePath" -ForegroundColor Yellow
Write-Host " Source UTC: $sourceLastModifiedUtc | SPO UTC: $spLastModifiedUtc" -ForegroundColor DarkGray
if (-not $WhatIf) {
Resolve-PnPFolder -SiteRelativePath $targetSiteRelativeFolder -ErrorAction SilentlyContinue | Out-Null
Add-PnPFile -Path $sourceFile.FullName -Folder $targetSiteRelativeFolder -Overwrite -ErrorAction Stop | Out-Null
$script:copiedCount++
}
}
else {
Write-Host "[$script:processedCount/$totalFiles] [MATCHED/NEWER SPO] Skipping: $relativePath" -ForegroundColor Green
$script:skippedCount++
}
}
else {
Write-Host "[$script:processedCount/$totalFiles] [MISSING IN SPO] Uploading: $relativePath" -ForegroundColor Cyan
if (-not $WhatIf) {
Resolve-PnPFolder -SiteRelativePath $targetSiteRelativeFolder -ErrorAction SilentlyContinue | Out-Null
Add-PnPFile -Path $sourceFile.FullName -Folder $targetSiteRelativeFolder -Overwrite -ErrorAction Stop | Out-Null
$script:copiedCount++
}
}
}
catch {
Write-Host "[$script:processedCount/$totalFiles] [ERROR] Failed processing '$relativePath': $($_.Exception.Message)" -ForegroundColor Red
$script:errorCount++
Add-Content -Path $logPath -Value "ERROR [$relativePath]: $($_.Exception.Message)"
}
}
# Slight delay between batches to respect SharePoint API rate limits
Start-Sleep -Milliseconds 100
}
}
catch {
Write-Host "`nCRITICAL ERROR: $_" -ForegroundColor Red
Add-Content -Path $logPath -Value "CRITICAL: $_"
}
finally {
Write-Progress -Activity "Comparing & Syncing Files" -Completed
Write-Host "`n==========================================================" -ForegroundColor Cyan
Write-Host " Sync Process Completed" -ForegroundColor Cyan
Write-Host "==========================================================" -ForegroundColor Cyan
Write-Host "Total files evaluated: $script:processedCount" -ForegroundColor White
Write-Host "Files copied/updated: $script:copiedCount" -ForegroundColor $(if ($WhatIf) { "Yellow" } else { "Green" })
Write-Host "Files skipped (newer in SPO): $script:skippedCount" -ForegroundColor Green
Write-Host "Errors encountered: $script:errorCount" -ForegroundColor $(if ($script:errorCount -gt 0) { "Red" } else { "Gray" })
Write-Host "WhatIf Mode was: $WhatIf" -ForegroundColor $(if ($WhatIf) { "Yellow" } else { "White" })
Write-Host "Detailed Log: $logPath" -ForegroundColor Gray
Write-Host "==========================================================`n" -ForegroundColor Cyan
Disconnect-PnPOnline -ErrorAction SilentlyContinue
Stop-Transcript
}
}
# Execution Entry Point
try {
Test-PowerShellRequirements
Install-RequiredModules
# CONFIGURATION PARAMETERS: Modify these values for your environment
$syncParams = @{
SourcePath = "\\onpremserver\share\folder" # e.g. "D:\Files" or "\\server\share"
SharePointSiteUrl = "https://yourtenant.sharepoint.com/sites/yoursite"
LibraryName = "Documents" # Display name or URL name (e.g. "Documents" or "Shared Documents")
WhatIf = $true # Set to $false to perform the actual file transfer
SyncEmptyDirectories = $true # Set to $true to recreate missing/empty subfolders
BatchSize = 25 # Adjust batch size based on connection
UseIndexCache = $true # Recommended $true: loads SPO inventory once for high speed & zero throttling
}
# If parameters were passed directly to script via command line, use those instead
if ($PSBoundParameters.Count -gt 0) {
Start-SharePointIncrementalSync @PSBoundParameters
} else {
Start-SharePointIncrementalSync @syncParams
}
}
catch {
Write-Host "Script initialization failed: $_" -ForegroundColor Red
Write-Host "Stack Trace: $($_.ScriptStackTrace)" -ForegroundColor Red
exit 1
}