diff options
| -rw-r--r-- | README.md | 30 | ||||
| -rw-r--r-- | T2R.ps1 | 137 | ||||
| -rw-r--r-- | combine-template.conf | 3 | ||||
| -rw-r--r-- | generate_config.py | 206 | ||||
| -rw-r--r-- | remote-template.conf | 14 | ||||
| -rw-r--r-- | union-template.conf | 3 |
6 files changed, 237 insertions, 156 deletions
@@ -1,6 +1,6 @@ # Teams to rclone -powershell script to collect all the drive_ids in sharepoint associated to a team and create a union mount. Read only permissions for creating local backups and/or archives +Python script to collect all the drive_ids in sharepoint associated to a team and create a combine mount. Read only permissions for creating local backups and/or archives ## Why? @@ -13,9 +13,8 @@ Becuase microsoft dont like to make things easy and create sharepoint subsites f 3. Once the app has been made make note of the Application (client) ID and Directory (tennant) ID 4. Go to Certificates & Secrets > Client secrets and create a new client secret, set the expire and click add 5. Once it has been generated make a note of the Value, once you change page that will be stared out and you wont be able to see it again. -6. In your config.json set the `tennantId`, `clientID` and `client secret (Value)` -7. in the remote-template.conf set the `client ID`, `client secret (value)`, `tennantID` and replace `<tenant>` with your tenantID -8. Go to `API permissions > Add a permission > Microsoft Graph > Application permissions` and add the below permissions +6. In your config.json set the `tenantId`, `clientID` and `clientSecret` (Value) +7. Go to `API permissions > Add a permission > Microsoft Graph > Application permissions` and add the below permissions ```plain Channel.ReadBasic.All @@ -26,16 +25,29 @@ Sites.Read.All Team.ReadBasic.All ``` -9. Once those have been added go to `Add a permission > Sharepoint > Application permissions` and add the below permission +8. Once those have been added go to `Add a permission > Sharepoint > Application permissions` and add the below permission ```plain Sites.Read.All ``` +9. Grant admin consent. + +## Flags + +```txt +--team-name [TEAM_NAME ...] + Team display name prefix(es). Required unless --team-id +--team-id Team id (skips name lookup). Required unless --team-name +--exact Exact displayName match +--config Path to config.json +-o, --outdir Output directory for .conf files +-h, --help Show help +``` + ## Usage -- Run the script -- Enter your team name when it asks +- Make sure you have python 3.7+ and rclone in your PATH +- Run `python3 generate_config.py --team-name "Team One" "Team Two"` - Wait -- Run `rclone lsd --config ./rclone TEAM_NAME_union:` when the script is finished to confirm you can see everything -- If the team only contains standard channels it will create a config with 1 remote +- Run `rclone lsd --config ./Team_One.conf Team_One_combine:` when the script is finished to confirm you can see everything diff --git a/T2R.ps1 b/T2R.ps1 deleted file mode 100644 index 514551f..0000000 --- a/T2R.ps1 +++ /dev/null @@ -1,137 +0,0 @@ -param( - [Parameter(Mandatory = $true)] - [string[]]$Team, - [switch]$Exact -) - -$config = Get-Content -Path ./config.json -Raw | ConvertFrom-Json -$tenantId = $config.tenantId -$clientID = $config.clientID -$clientSecret = (ConvertTo-SecureString $config.clientSecret -AsPlainText -Force ) -$Scope = "https://graph.microsoft.com/.default" -$authToken = Get-MsalToken -ClientId $clientID -ClientSecret $clientSecret -TenantId $tenantId -Scopes $Scope - -Write-Host "Processing $($Team.Count) teams: $($Team -join ', ')" - -$Headers = @{ - "Authorization" = "Bearer $($authToken.AccessToken)" - "Content-type" = "application/json" -} - -$allTeams = @() -$teamsResponse = Invoke-WebRequest -Uri "https://graph.microsoft.com/v1.0/teams/" -Headers $Headers -Method GET -$aadTeams = $teamsResponse.Content | ConvertFrom-Json -$allTeams += $aadTeams.value -if ($aadTeams.'@odata.nextLink') { - do { - $teamsResponse = Invoke-WebRequest -Uri $aadTeams.'@odata.nextLink' -Headers $Headers -Method GET - $aadTeams = $teamsResponse.Content | ConvertFrom-Json - $allTeams += $aadTeams.value - } until (!$aadTeams.'@odata.nextLink') -} -$aadTeams = $allTeams - -foreach ($requestedTeam in $Team) { - Write-Host "`nProcessing team: $requestedTeam" - - $channels = [System.Collections.ArrayList]::New() - $foundTeam = $false - - foreach ($teams in $aadTeams) { - if ($foundTeam) { break } - - $teamMatches = if ($Exact) { - $teams.displayName -eq $requestedTeam - } else { - $teams.displayName -like "$($requestedTeam)*" - } - - if ($teamMatches) { - Write-Host "Found matching team: $($teams.displayName)" - $apiUri = "https://graph.microsoft.com/v1.0/teams/$($teams.id)/channels" - - $response = Invoke-WebRequest -Headers $Headers -Uri $apiUri -Method GET - - try { - $teamResponse = $response.Content | ConvertFrom-Json -ErrorAction Stop - } catch { - $teamResponse = [System.Web.HttpUtility]::JavaScriptStringDeserializer.Deserialize($response.Content, [System.Object]) - } - - $channelList = $null - - if ($teamResponse.value) { - $channelList = $teamResponse.value - } elseif ($teamResponse.PSObject.Properties['value']) { - $channelList = $teamResponse.PSObject.Properties['value'].Value - } else { - $rawContent = $response.Content - if ($rawContent -match '"value":\s*\[(.*?)\]') { - $jsonArray = "[$($matches[1])]" - $channelList = $jsonArray | ConvertFrom-Json - } - } - - if ($channelList) { - Write-Host "Processing $($channelList.Count) channels for team: $($teams.displayName)" - foreach ($channel in $channelList) { - Start-Sleep -Milliseconds 250 - $folderResponse = Invoke-WebRequest -Headers $Headers -Uri "https://graph.microsoft.com/v1.0/teams/$($teams.id)/channels/$($channel.id)/filesFolder" -Method GET - $channelFolder = $folderResponse.Content | ConvertFrom-Json - $DriveID = $channelFolder.parentReference.driveId - $displayName = $channelFolder.name -replace " ", "_" -replace ",", "" - $Result = [PSCustomObject]@{ - displayName = $displayName - driveID = $DriveID - membershipType = $channel.membershipType - teamName = $teams.displayName - } - [void]$channels.Add($Result) - Write-Host "Added channel: $displayName with DriveID: $DriveID" - } - } else { - Write-Host "WARNING: No channels found for team: $($teams.displayName)" - } - $foundTeam = $true - } - } - - if ($channels.Count -eq 0) { - Write-Host "ERROR: No channels were processed for team: $requestedTeam" - continue - } - - Write-Host "Total channels processed for $requestedTeam`: $($channels.Count)" - - $uniqueDrives = $channels | Sort-Object -Property driveID -Unique | Select-Object displayName, driveID, membershipType, teamName - Write-Host "Unique drives found for $requestedTeam`: $($uniqueDrives.Count)" - - $uniqueDrives | ForEach-Object { - if ($_.membershipType -eq 'standard') { - $_.displayName = $_.teamName -replace " ", "_" - } - } - - $UnionName = "$($channels[0].teamName)_union" -replace " ", "_" - $RcloneFileName = $UnionName -replace "_union", "" - - foreach ($drive in $uniqueDrives) { - $config = Get-Content -Path ./remote-template.conf -Raw - $remote = $config -f $drive.displayName, $drive.driveID - Add-Content -Value $remote -Path "./$RcloneFileName.conf" - Write-Host "Added remote config for: $($drive.displayName)" - } - - if ($uniqueDrives.Count -ge 2) { - $teamNames = $($uniqueDrives.displayName -join [Environment]::NewLine) -replace "\n", ": " - $teamNames = "$teamNames" + ":" - $union = Get-Content -Path ./union-template.conf -Raw - $unionConf = $union -f $UnionName, $teamNames - Add-Content -Value $unionConf -Path "./$RcloneFileName.conf" - Write-Host "Generated union config: $UnionName" - } else { - Write-Host "Skipping union creation - only $($uniqueDrives.Count) remote(s) found (union requires 2+)" - } - - Write-Host "Generated config file: $RcloneFileName.conf" -}
\ No newline at end of file diff --git a/combine-template.conf b/combine-template.conf new file mode 100644 index 0000000..dce9c84 --- /dev/null +++ b/combine-template.conf @@ -0,0 +1,3 @@ +[{0}] +type = combine +upstreams = {1} diff --git a/generate_config.py b/generate_config.py new file mode 100644 index 0000000..bba424e --- /dev/null +++ b/generate_config.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Generate rclone combine configs for Microsoft Teams channel drives.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +GRAPH = "https://graph.microsoft.com/v1.0" +HERE = Path(__file__).resolve().parent +DEFAULT_CONFIG = HERE / "config.json" +REMOTE_TMPL = (HERE / "remote-template.conf").read_text() +COMBINE_TMPL = (HERE / "combine-template.conf").read_text() + + +def sanitize_remote(name: str) -> str: + return re.sub(r"[^\w.-]+", "_", name).strip("_") + + +def get_token(tenant: str, client_id: str, client_secret: str) -> str: + data = urllib.parse.urlencode( + { + "client_id": client_id, + "client_secret": client_secret, + "scope": "https://graph.microsoft.com/.default", + "grant_type": "client_credentials", + } + ).encode() + req = urllib.request.Request( + f"https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token", + data=data, + method="POST", + ) + with urllib.request.urlopen(req) as resp: + return json.load(resp)["access_token"] + + +def graph_get(url: str, token: str) -> dict: + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + with urllib.request.urlopen(req) as resp: + return json.load(resp) + + +def graph_paged(url: str, token: str) -> list: + items: list = [] + while url: + payload = graph_get(url, token) + items.extend(payload.get("value", [])) + url = payload.get("@odata.nextLink") + return items + + +def find_team(teams: list, name: str, exact: bool) -> dict | None: + for team in teams: + dn = team.get("displayName") or "" + if exact and dn == name: + return team + if not exact and dn.lower().startswith(name.lower()): + return team + return None + + +def collect_channels(team_id: str, token: str) -> list[dict]: + channels = graph_paged(f"{GRAPH}/teams/{team_id}/channels", token) + results = [] + for ch in channels: + time.sleep(0.25) + folder = graph_get( + f"{GRAPH}/teams/{team_id}/channels/{ch['id']}/filesFolder", token + ) + results.append( + { + "displayName": ch["displayName"], + "driveId": folder["parentReference"]["driveId"], + "membershipType": ch.get("membershipType", "standard"), + } + ) + print(f" {ch['membershipType']}: {ch['displayName']} -> {folder['parentReference']['driveId']}") + return results + + +def build_config(team_name: str, channels: list[dict], creds: dict) -> str: + team_remote = sanitize_remote(team_name) + standard = [c for c in channels if c["membershipType"] == "standard"] + other = [c for c in channels if c["membershipType"] != "standard"] + + blocks: list[str] = [] + upstreams: list[str] = [] + written_remotes: set[str] = set() + + def remote_block(name: str, drive_id: str) -> str: + return REMOTE_TMPL.format( + name=name, + drive_id=drive_id, + client_id=creds["client_id"], + client_secret=creds["client_secret"], + tenant=creds["tenant"], + ) + + if standard: + blocks.append(remote_block(team_remote, standard[0]["driveId"])) + written_remotes.add(team_remote) + for ch in standard: + name = ch["displayName"] + upstreams.append(f'"{name}={team_remote}:{name}"') + + for ch in other: + remote = sanitize_remote(ch["displayName"]) + if remote in written_remotes: + remote = f"{remote}_{ch['membershipType']}" + blocks.append(remote_block(remote, ch["driveId"])) + written_remotes.add(remote) + name = ch["displayName"] + # private drives nest files under a folder named after the channel + if ch["membershipType"] == "private": + upstreams.append(f'"{name}={remote}:{name}"') + else: + # shared / unknownFutureValue: files sit at drive root + upstreams.append(f'"{name}={remote}:"') + + if upstreams: + blocks.append(COMBINE_TMPL.format(f"{team_remote}_combine", " ".join(upstreams))) + + return "\n".join(blocks) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--team-name", nargs="+", help="Team display name prefix(es)") + parser.add_argument("--team-id", help="Team id (skips name lookup)") + parser.add_argument("--exact", action="store_true", help="Exact displayName match") + parser.add_argument( + "--config", + type=Path, + default=DEFAULT_CONFIG, + help=f"Path to config.json (default: {DEFAULT_CONFIG})", + ) + parser.add_argument( + "-o", + "--outdir", + type=Path, + default=HERE, + help="Output directory for .conf files", + ) + args = parser.parse_args() + + if not args.team_id and not args.team_name: + parser.error("provide --team-name and/or --team-id") + + cfg = json.loads(args.config.read_text()) + creds = { + "tenant": cfg["tenantId"], + "client_id": cfg["clientID"], + "client_secret": cfg["clientSecret"], + } + + print("Authenticating...") + token = get_token(creds["tenant"], creds["client_id"], creds["client_secret"]) + args.outdir.mkdir(parents=True, exist_ok=True) + + targets = [] + if args.team_id: + team = graph_get(f"{GRAPH}/teams/{args.team_id}", token) + print(f"Team: {team['displayName']} ({args.team_id})") + targets.append(team) + + if args.team_name: + print("Listing teams...") + all_teams = graph_paged(f"{GRAPH}/teams/", token) + for requested in args.team_name: + team = find_team(all_teams, requested, args.exact) + if not team: + print(f"ERROR: no team matched '{requested}'", file=sys.stderr) + continue + print(f"Found: {team['displayName']} ({team['id']})") + targets.append(team) + + for team in targets: + print(f"\nChannels for {team['displayName']}:") + try: + channels = collect_channels(team["id"], token) + except urllib.error.HTTPError as e: + print(f"ERROR fetching channels: {e}", file=sys.stderr) + continue + + if not channels: + print(f"ERROR: no channels for {team['displayName']}", file=sys.stderr) + continue + + conf = build_config(team["displayName"], channels, creds) + out = args.outdir / f"{sanitize_remote(team['displayName'])}.conf" + out.write_text(conf) + print(f"Wrote {out}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/remote-template.conf b/remote-template.conf index 38e77fc..f80e76a 100644 --- a/remote-template.conf +++ b/remote-template.conf @@ -1,12 +1,12 @@ -[{0}] +[{name}] type = onedrive -drive_id = {1} +drive_id = {drive_id} drive_type = documentLibrary -client_id = -client_secret = -tenant = +client_id = {client_id} +client_secret = {client_secret} +tenant = {tenant} client_credentials = true access_scopes = Files.Read Files.Read.All Sites.Read.All offline_access link_scope = organization -auth_url = https://login.microsoftonline.com/<tenant>/oauth2/v2.0/authorize -token_url = https://login.microsoftonline.com/<tenant>/oauth2/v2.0/token +auth_url = https://login.microsoftonline.com/{tenant}/oauth2/v2.0/authorize +token_url = https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token diff --git a/union-template.conf b/union-template.conf deleted file mode 100644 index 5920179..0000000 --- a/union-template.conf +++ /dev/null @@ -1,3 +0,0 @@ -[{0}] -type = union -upstreams = {1}
\ No newline at end of file |