diff --git a/.github/workflows/ExternalData-VariousHostingServices.yml b/.github/workflows/ExternalData-VariousHostingServices.yml new file mode 100644 index 0000000..b995f8e --- /dev/null +++ b/.github/workflows/ExternalData-VariousHostingServices.yml @@ -0,0 +1,86 @@ +name: Pull external data source Various Hosting Services + +on: + workflow_dispatch: + + schedule: + - cron: "0 */6 * * *" + +jobs: + pull-external-data: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout main + uses: actions/checkout@v4 + + - name: Pull external data and modify if needed + shell: pwsh + run: | + if (-not (Test-Path -Path "${{ github.workspace }}/ExternalData/")) { New-Item -ItemType Directory -Path "${{ github.workspace }}/ExternalData/" -ErrorAction SilentlyContinue } + + function Get-AddressFamily { + param([string]$IPRange) + try { + return [IPAddress]($IPRange -replace '/.*') | Select-Object -ExpandProperty AddressFamily + } catch { + return $null + } + } + + # Vultr - JSON geofeed + $VultrRaw = Invoke-RestMethod -Uri 'https://geofeed.constant.com/?json' + $Vultr = foreach ($s in $VultrRaw.subnets) { + [PSCustomObject]@{ + Service = 'Vultr' + IPRange = $s.ip_prefix + Country = $s.alpha2code + LanguageCode = $s.region + City = $s.city + PostalCode = $s.postal_code + AddressFamily = Get-AddressFamily $s.ip_prefix + } + } + $Vultr | Select-Object IPRange, Country, LanguageCode, City, PostalCode, AddressFamily | Export-Csv -Path "${{ github.workspace }}/ExternalData/Vultr.csv" -Force + + # DigitalOcean - headerless CSV + $DORaw = Invoke-WebRequest -Uri 'https://digitalocean.com/geo/google.csv' | Select-Object -ExpandProperty Content | ConvertFrom-Csv -Header @('IPRange', 'Country', 'LanguageCode', 'City', 'PostalCode') + $DigitalOcean = foreach ($r in $DORaw) { + if ($r.IPRange -match '^#' -or [string]::IsNullOrWhiteSpace($r.IPRange)) { continue } + [PSCustomObject]@{ + Service = 'DigitalOcean' + IPRange = $r.IPRange + Country = $r.Country + LanguageCode = $r.LanguageCode + City = $r.City + PostalCode = $r.PostalCode + AddressFamily = Get-AddressFamily $r.IPRange + } + } + $DigitalOcean | Select-Object IPRange, Country, LanguageCode, City, PostalCode, AddressFamily | Export-Csv -Path "${{ github.workspace }}/ExternalData/DigitalOcean.csv" -Force + + # Linode - RFC 8805 geofeed CSV with comment lines + $LinodeContent = Invoke-WebRequest -Uri 'https://geoip.linode.com/' | Select-Object -ExpandProperty Content + $LinodeLines = $LinodeContent -split "`n" | Where-Object { $_ -and $_ -notmatch '^\s*#' } + $LinodeRaw = $LinodeLines | ConvertFrom-Csv -Header @('IPRange', 'Country', 'LanguageCode', 'City', 'PostalCode') + $Linode = foreach ($r in $LinodeRaw) { + if ([string]::IsNullOrWhiteSpace($r.IPRange)) { continue } + [PSCustomObject]@{ + Service = 'Linode' + IPRange = $r.IPRange + Country = $r.Country + LanguageCode = $r.LanguageCode + City = $r.City + PostalCode = $r.PostalCode + AddressFamily = Get-AddressFamily $r.IPRange + } + } + $Linode | Select-Object IPRange, Country, LanguageCode, City, PostalCode, AddressFamily | Export-Csv -Path "${{ github.workspace }}/ExternalData/Linode.csv" -Force + + # Consolidated CSV across all three services + $All = @($Vultr) + @($DigitalOcean) + @($Linode) + $All | Select-Object Service, IPRange, Country, LanguageCode, City, PostalCode, AddressFamily | Export-Csv -Path "${{ github.workspace }}/ExternalData/VariousHostingServices.csv" -Force + + - name: Commit updated data to repo + uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/ExternalData/README.md b/ExternalData/README.md index e17fb47..b780126 100644 --- a/ExternalData/README.md +++ b/ExternalData/README.md @@ -7,4 +7,9 @@ Some external data sources need additional modification or are not available thr | `https://mask-api.icloud.com/egress-ip-ranges.csv` | Current list of all IP addresses of the iCloud Private Relay service.
https://developer.apple.com/support/prepare-your-network-for-icloud-private-relay/ | Added column to distinguish between IPv4 and IPv6 | `externaldata` cannot fetch the CSV from Apple servers | | `https://www.gstatic.com/g1vpn/geofeed` | Current list of all IP addresses of the Google One VPN service.
https://one.google.com/about/vpn/howitworks | Added column to distinguish between IPv4 and IPv6 | `externaldata` cannot fetch the CSV from Google server | | `https://learn.microsoft.com/en-us/windows-hardware/drivers/ifs/allocated-altitudes` | Microsoft File systems driver allocated filter altitudes | Convert from markdown to csv | | -| `https://endpoints.office.com/endpoints/worldwide` | Current list of all IP ranges of M365 services in an easy to consume CSV | None | | \ No newline at end of file +| `https://endpoints.office.com/endpoints/worldwide` | Current list of all IP ranges of M365 services in an easy to consume CSV | None | | +| `https://geofeed.constant.com/?json` | Current list of all IP ranges of the Vultr hosting service. | Converted from JSON to CSV, added AddressFamily | Source is JSON; harmonized columns for consolidation | +| `https://digitalocean.com/geo/google.csv` | Current list of all IP ranges of the DigitalOcean hosting service. | Added header row and AddressFamily column | Source has no header row | +| `https://geoip.linode.com/` | Current list of all IP ranges of the Linode hosting service (RFC 8805 geofeed). | Stripped comments, added AddressFamily column | Source contains `#` comment lines | + +The three hosting services above are pulled by a single workflow that produces one CSV per source and a consolidated `VariousHostingServices.csv` containing all entries with an additional `Service` column. \ No newline at end of file