mirror of
https://github.com/aphrody-labs/msys2-devtools.git
synced 2026-10-14 04:19:12 +00:00
Christoph Reiter
ff551d2795
We want to count every attempt of pacman to fetch something only once. So for the repo, 200 and 206 (does that count?) and 304. For mirrorbits 302 when a redirect to a mirror happens and 304.
468 lines
16 KiB
Python
Executable File
468 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Expects traefik json access logs either by passing a log file as the first argument, or via stdin
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import argparse
|
|
from datetime import datetime
|
|
from collections import Counter
|
|
from typing import List, Tuple, Optional
|
|
from dataclasses import dataclass
|
|
|
|
from tabulate import tabulate
|
|
import requests_cache
|
|
from netaddr import IPSet, IPAddress
|
|
|
|
|
|
@dataclass
|
|
class LogEntry:
|
|
ClientHost: str
|
|
DownstreamStatus: int
|
|
RequestHost: str
|
|
RequestMethod: str
|
|
RequestPath: str
|
|
RequestProtocol: str
|
|
UserAgent: str
|
|
time: str
|
|
client_info: "Optional[ClientInfo]"
|
|
|
|
|
|
@dataclass
|
|
class UserAgent:
|
|
pacman_version: Tuple[int, int, int]
|
|
windows_version: Tuple[int, int]
|
|
build_number: int
|
|
cygwin_arch: str
|
|
host_arch: str
|
|
libalpm_version: Tuple[int, int, int]
|
|
|
|
|
|
@dataclass
|
|
class ClientInfo:
|
|
pacman_version: str
|
|
windows_edition: str
|
|
user_agent: UserAgent
|
|
ci: str
|
|
|
|
|
|
def is_valid_user_agent(ua: str) -> bool:
|
|
return ua.startswith("pacman") and "_NT-" in ua
|
|
|
|
|
|
def parse_user_agent(ua: str) -> UserAgent:
|
|
assert is_valid_user_agent(ua)
|
|
m = re.match(r"pacman/([^\s]+) \(([^-]+)-([^-]+)-?([0-9]+|)-?(.+|) ([^)]+)\) libalpm/(.+)", ua)
|
|
assert m is not None
|
|
pacman_version, _, windows_version, build_number, extra, arch, alpm_version = m.groups()
|
|
|
|
pacman_version = tuple(map(int, pacman_version.split("-")[0].split(".")))
|
|
windows_version = tuple(map(int, windows_version.split(".")))
|
|
build_number = int(build_number or "-1")
|
|
cygwin_arch = arch
|
|
if extra in ("WOW64", "WOW") and cygwin_arch == "i686":
|
|
host_arch = "x86_64"
|
|
elif extra == "ARM64":
|
|
host_arch = "aarch64"
|
|
elif extra:
|
|
host_arch = "???"
|
|
else:
|
|
host_arch = cygwin_arch
|
|
alpm_version = tuple(map(int, alpm_version.split(".")))
|
|
|
|
return UserAgent(
|
|
pacman_version,
|
|
windows_version,
|
|
build_number,
|
|
cygwin_arch,
|
|
host_arch,
|
|
alpm_version
|
|
)
|
|
|
|
|
|
def test_parse_user_agent():
|
|
ua = "pacman/6.0.1 (MSYS_NT-10.0-19042 x86_64) libalpm/13.0.1"
|
|
assert parse_user_agent(ua).pacman_version == (6, 0, 1)
|
|
ua = "pacman/6.0.1 (MSYS_NT-10.0-19042-WOW64 i686) libalpm/13.0.1"
|
|
assert parse_user_agent(ua).cygwin_arch == "i686"
|
|
assert parse_user_agent(ua).host_arch == "x86_64"
|
|
assert parse_user_agent(ua).build_number == 19042
|
|
ua = "pacman/5.2.1 (MSYS_NT-6.1-7601-WOW64 i686) libalpm/12.0.1"
|
|
assert parse_user_agent(ua).cygwin_arch == "i686"
|
|
assert parse_user_agent(ua).host_arch == "x86_64"
|
|
assert parse_user_agent(ua).build_number == 7601
|
|
ua = "pacman/5.1.0 (MINGW64_NT-10.0 x86_64) libalpm/11.0.0"
|
|
assert parse_user_agent(ua).build_number == -1
|
|
assert parse_user_agent(ua).cygwin_arch == "x86_64"
|
|
assert parse_user_agent(ua).host_arch == "x86_64"
|
|
ua = "pacman/5.0.1 (UCRT64_NT-10.0-WOW i686) libalpm/10.0.1"
|
|
assert parse_user_agent(ua).build_number == -1
|
|
assert parse_user_agent(ua).host_arch == "x86_64"
|
|
assert parse_user_agent(ua).cygwin_arch == "i686"
|
|
ua = "pacman/4.2.1-313-g5535-dirty (MINGW64_NT-6.1 x86_64) libalpm/9.0.1"
|
|
assert parse_user_agent(ua).pacman_version == (4, 2, 1)
|
|
ua = "pacman/6.0.1 (MSYS_NT-6.0-6002 x86_64) libalpm/13.0.1"
|
|
assert parse_user_agent(ua).windows_version == (6, 0)
|
|
ua = "pacman/6.1.0 (MSYS_NT-10.0-22631-ARM64 x86_64) libalpm/14.0.0"
|
|
assert parse_user_agent(ua).host_arch == "aarch64"
|
|
assert parse_user_agent(ua).cygwin_arch == "x86_64"
|
|
ua = "pacman/6.1.0 (MSYS_NT-10.0-22631-ARM64 i686) libalpm/14.0.0"
|
|
assert parse_user_agent(ua).host_arch == "aarch64"
|
|
assert parse_user_agent(ua).cygwin_arch == "i686"
|
|
|
|
|
|
def test_get_windows_edition():
|
|
ua = "pacman/6.0.1 (MSYS_NT-6.0-6002 x86_64) libalpm/13.0.1"
|
|
assert get_windows_edition(parse_user_agent(ua)) == "Vista"
|
|
ua = "pacman/4.2.1-463-gbb493-dirty (MINGW32_NT-5.0 i686) libalpm/10.0.0"
|
|
assert get_windows_edition(parse_user_agent(ua)) == "2000"
|
|
|
|
|
|
def get_windows_edition(ua: UserAgent):
|
|
if ua.windows_version >= (10, 0):
|
|
if ua.build_number >= 22000:
|
|
return "11"
|
|
else:
|
|
return "10"
|
|
elif ua.windows_version == (5, 0):
|
|
return "2000"
|
|
elif ua.windows_version == (5, 1):
|
|
return "XP"
|
|
elif ua.windows_version == (5, 2):
|
|
return "XP"
|
|
elif ua.windows_version == (6, 0):
|
|
return "Vista"
|
|
elif ua.windows_version == (6, 1):
|
|
return "7"
|
|
elif ua.windows_version == (6, 2):
|
|
return "8"
|
|
elif ua.windows_version == (6, 3):
|
|
return "8.1"
|
|
else:
|
|
assert 0, ua
|
|
|
|
|
|
def get_ci_networks():
|
|
session = requests_cache.CachedSession(
|
|
'msys2-logstats-ci', expire_after=360, use_cache_dir=True)
|
|
r = session.get("https://api.github.com/meta")
|
|
r.raise_for_status()
|
|
gha = IPSet(r.json()["actions"])
|
|
|
|
r = session.get("https://www.appveyor.com/ips.json")
|
|
r.raise_for_status()
|
|
appveyor = IPSet(r.json())
|
|
|
|
r = session.get("https://www.gstatic.com/ipranges/cloud.json")
|
|
r.raise_for_status()
|
|
prefixes = set()
|
|
for prefix in r.json()["prefixes"]:
|
|
prefixes.add(prefix.get("ipv4Prefix", prefix.get("ipv6Prefix")))
|
|
gcp = IPSet(prefixes)
|
|
|
|
r = session.get("https://ip-ranges.amazonaws.com/ip-ranges.json")
|
|
r.raise_for_status()
|
|
prefixes = set()
|
|
for prefix in r.json()["prefixes"]:
|
|
prefixes.add(prefix["ip_prefix"])
|
|
for prefix in r.json()["ipv6_prefixes"]:
|
|
prefixes.add(prefix["ipv6_prefix"])
|
|
aws = IPSet(prefixes)
|
|
|
|
r = session.get(
|
|
"https://www.microsoft.com/en-us/download/details.aspx?id=56519",
|
|
headers={
|
|
'User-Agent':
|
|
'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0'
|
|
}
|
|
)
|
|
r.raise_for_status()
|
|
match = re.search("(https://download.microsoft.com/.*?/ServiceTags_Public_.*?\\.json)", r.text)
|
|
r = session.get(match.groups(0)[0])
|
|
r.raise_for_status()
|
|
prefixes = set()
|
|
for value in r.json()["values"]:
|
|
for nw in value["properties"]["addressPrefixes"]:
|
|
prefixes.add(nw)
|
|
azure = IPSet(prefixes)
|
|
|
|
return {"GHA": gha, "APPV": appveyor, "GCP": gcp, "AWS": aws, "AZ": azure}
|
|
|
|
|
|
def get_repo_for_path(path: str) -> str:
|
|
repo = path.rsplit("/", 1)[0].lstrip("/")
|
|
if repo == "mingw/i686":
|
|
repo = "mingw/mingw32"
|
|
elif repo == "mingw/x86_64":
|
|
repo = "mingw/mingw64"
|
|
|
|
return repo
|
|
|
|
|
|
def get_type_for_path(path: str) -> str:
|
|
if path.endswith(".db") or ".db." in path:
|
|
return "db"
|
|
elif path.endswith(".files") or ".files." in path:
|
|
return "db"
|
|
else:
|
|
return "pkg"
|
|
|
|
|
|
def print_repos(entries, show_ci):
|
|
for request_type in ["pkg", "db"]:
|
|
type_requests = [e for e in entries if get_type_for_path(e.RequestPath) == request_type]
|
|
table = []
|
|
for (repo, type_, ci), count in Counter([
|
|
(get_repo_for_path(e.RequestPath), get_type_for_path(e.RequestPath),
|
|
e.client_info.ci) for e in type_requests]).most_common():
|
|
pcnt = count / len(type_requests) * 100
|
|
line = [repo, type_, ci, f"{pcnt:.2f}%", f"{count}"]
|
|
if not show_ci:
|
|
line.pop(2)
|
|
table.append(line)
|
|
headers = ["Repo", "Type", "CI", "% Requests", "Requests"]
|
|
if not show_ci:
|
|
headers.pop(2)
|
|
print()
|
|
print(tabulate(table, headers, stralign="right", numalign="right"))
|
|
|
|
|
|
def print_windows_major(clients, entries, show_ci):
|
|
per_request = {}
|
|
for (edition, ci), count in Counter([(e.client_info.windows_edition, e.client_info.ci) for e in entries]).most_common():
|
|
per_request[(edition, ci)] = count
|
|
table = []
|
|
for (edition, ci), count_clients in Counter([(u.windows_edition, u.ci) for u in clients]).most_common():
|
|
pcnt_clients = count_clients / len(clients) * 100
|
|
count_req = per_request[(edition, ci)]
|
|
pcnt_req = count_req / len(entries) * 100
|
|
line = [edition, ci, f"{pcnt_clients:.2f}%", f"{count_clients}", f"{pcnt_req:.2f}%", f"{count_req}"]
|
|
if not show_ci:
|
|
line.pop(1)
|
|
table.append(line)
|
|
headers = ["Windows", "CI", "% Clients", "Clients", "% Requests", "Requests"]
|
|
if not show_ci:
|
|
headers.pop(1)
|
|
print()
|
|
print(tabulate(table, headers, stralign="right", numalign="right"))
|
|
|
|
|
|
def print_ci_systems(clients, entries):
|
|
per_request = {}
|
|
for ci, count in Counter([e.client_info.ci for e in entries]).most_common():
|
|
per_request[ci] = count
|
|
table = []
|
|
for ci, count_clients in Counter([u.ci for u in clients]).most_common():
|
|
pcnt_clients = count_clients / len(clients) * 100
|
|
count_req = per_request[ci]
|
|
pcnt_req = count_req / len(entries) * 100
|
|
line = [ci, f"{pcnt_clients:.2f}%", f"{count_clients}", f"{pcnt_req:.2f}%", f"{count_req}"]
|
|
table.append(line)
|
|
headers = ["CI", "% Clients", "Clients", "% Requests", "Requests"]
|
|
print()
|
|
print(tabulate(table, headers, stralign="right", numalign="right"))
|
|
|
|
|
|
def print_windows_version_details(clients, show_ci):
|
|
table = []
|
|
for (windows_version, build_number, ci), count in Counter(
|
|
[(u.user_agent.windows_version, u.user_agent.build_number, u.ci) for u in clients]).most_common():
|
|
pcnt = count / len(clients) * 100
|
|
line = [".".join(map(str, windows_version)), build_number, ci, f"{pcnt:.2f}%", f"{count}"]
|
|
if not show_ci:
|
|
line.pop(2)
|
|
table.append(line)
|
|
headers = ["Win Ver", "Build Number", "CI", "% Clients", "Clients"]
|
|
if not show_ci:
|
|
headers.pop(2)
|
|
print()
|
|
print(tabulate(table, headers, stralign="right", numalign="right"))
|
|
|
|
|
|
def print_pacman(clients, show_ci):
|
|
table = []
|
|
for (version, ci), count in Counter([(u.pacman_version, u.ci) for u in clients]).most_common():
|
|
pcnt = count / len(clients) * 100
|
|
line = [version, ci, f"{pcnt:.2f}%", f"{count}"]
|
|
if not show_ci:
|
|
line.pop(1)
|
|
table.append(line)
|
|
headers = ["Pacman Ver", "CI", "% Clients", "Clients"]
|
|
if not show_ci:
|
|
headers.pop(1)
|
|
print()
|
|
print(tabulate(table, headers, stralign="right", numalign="right"))
|
|
|
|
|
|
def print_system_arch(clients, show_ci):
|
|
table = []
|
|
for (cygwin_arch, host_arch, ci), count in Counter([(u.user_agent.cygwin_arch, u.user_agent.host_arch, u.ci) for u in clients]).most_common():
|
|
pcnt = count / len(clients) * 100
|
|
line = [cygwin_arch, host_arch, ci, f"{pcnt:.2f}%", f"{count}"]
|
|
if not show_ci:
|
|
line.pop(2)
|
|
table.append(line)
|
|
headers = ["Cygwin Arch", "Host Arch", "CI", "% Clients", "Clients"]
|
|
if not show_ci:
|
|
headers.pop(2)
|
|
print()
|
|
print(tabulate(table, headers, stralign="right", numalign="right"))
|
|
|
|
|
|
def datetime_fromisoformat(value: str) -> datetime:
|
|
# For Python <3.11
|
|
if value.endswith("Z"):
|
|
value = value[:-1] + "+00:00"
|
|
return datetime.fromisoformat(value)
|
|
|
|
|
|
def main(argv):
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('infile', nargs='?', type=argparse.FileType('r', encoding="utf-8"), default=sys.stdin)
|
|
parser.add_argument('--show-ci', action='store_true', help='show CI/cloud providers')
|
|
parser.add_argument('--skip-ci', action='store_true', help='skip CI/cloud IP ranges')
|
|
parser.add_argument('--only-ci', action='store_true', help='only CI/cloud IP ranges')
|
|
parser.add_argument('--show-summary', action='store_true', help='show only a CI/cloud summary')
|
|
args = parser.parse_args(argv[1:])
|
|
|
|
assert not (args.skip_ci and args.only_ci)
|
|
|
|
detect_ci = False
|
|
|
|
if args.show_summary:
|
|
assert not args.skip_ci
|
|
assert not args.only_ci
|
|
args.show_ci = True
|
|
detect_ci = True
|
|
|
|
if args.skip_ci or args.only_ci:
|
|
detect_ci = True
|
|
|
|
if detect_ci:
|
|
ci_networks = get_ci_networks()
|
|
|
|
entries: List[LogEntry] = []
|
|
decoder = json.JSONDecoder()
|
|
with args.infile as h:
|
|
for line in h:
|
|
if not line.startswith("{"):
|
|
continue
|
|
decoded = decoder.decode(line)
|
|
if "RequestHost" not in decoded or "request_User-Agent" not in decoded:
|
|
continue
|
|
entry = LogEntry(
|
|
decoded["ClientHost"],
|
|
decoded["DownstreamStatus"],
|
|
decoded["RequestHost"],
|
|
decoded["RequestMethod"],
|
|
decoded["RequestPath"],
|
|
decoded["RequestProtocol"],
|
|
decoded["request_User-Agent"],
|
|
decoded["time"],
|
|
None,
|
|
)
|
|
if entry.RequestMethod != "GET":
|
|
continue
|
|
if entry.RequestHost == "repo.msys2.org":
|
|
if entry.DownstreamStatus not in [200, 206, 304]:
|
|
continue
|
|
elif entry.RequestHost == "mirror.msys2.org":
|
|
if entry.DownstreamStatus not in [302, 304]:
|
|
continue
|
|
else:
|
|
continue
|
|
if not is_valid_user_agent(entry.UserAgent):
|
|
continue
|
|
entries.append(entry)
|
|
|
|
def user_key(entry):
|
|
# only take the system part of the UA, so we don't count
|
|
# a client twice if it upgrades during a job for example
|
|
system_id = "".join(entry.UserAgent.split()[1:3])
|
|
return system_id + entry.ClientHost
|
|
|
|
first = entries[0].time
|
|
last = entries[0].time
|
|
grouped = {}
|
|
for entry in entries:
|
|
if entry.time < first:
|
|
first = entry.time
|
|
if entry.time > last:
|
|
last = entry.time
|
|
key = user_key(entry)
|
|
grouped.setdefault(key, []).append(entry)
|
|
|
|
ip_to_ci = {}
|
|
if detect_ci:
|
|
def get_ip_to_ci(ip_addr: str) -> str:
|
|
ip = IPAddress(ip_addr)
|
|
for name, ipset in ci_networks.items():
|
|
if ip in ipset:
|
|
return name
|
|
return ""
|
|
|
|
for group_entries in grouped.values():
|
|
ip = group_entries[0].ClientHost
|
|
ip_to_ci[ip] = get_ip_to_ci(ip)
|
|
|
|
clients: List[ClientInfo] = []
|
|
for group_entries in grouped.values():
|
|
user_agent = parse_user_agent(group_entries[0].UserAgent)
|
|
ci = ip_to_ci.get(group_entries[0].ClientHost, "")
|
|
client_info = ClientInfo(
|
|
".".join(map(str, user_agent.pacman_version)),
|
|
get_windows_edition(user_agent),
|
|
user_agent,
|
|
ci
|
|
)
|
|
clients.append(client_info)
|
|
for e in group_entries:
|
|
e.client_info = client_info
|
|
|
|
if args.skip_ci:
|
|
entries = [e for e in entries if not ip_to_ci.get(e.ClientHost, "")]
|
|
clients = [c for c in clients if not c.ci]
|
|
|
|
if args.only_ci:
|
|
entries = [e for e in entries if ip_to_ci.get(e.ClientHost, "")]
|
|
clients = [c for c in clients if c.ci]
|
|
|
|
# Log info
|
|
diff = datetime_fromisoformat(last) - datetime_fromisoformat(first)
|
|
duration = (diff).total_seconds()
|
|
requests_per_second = len(entries) / duration
|
|
print(tabulate([
|
|
["Duration", f"from {first} to {last} ({diff})"],
|
|
["Requests", f"{len(entries)} ({requests_per_second:.2f}/s)"],
|
|
["Clients", f"{len(clients)} (clients are grouped by IP+WinVer+Arch)"],
|
|
["Included", "CI only" if args.only_ci else "non-CI only" if args.skip_ci else "all"],
|
|
]))
|
|
|
|
# Repos
|
|
if not args.show_summary:
|
|
print_repos(entries, args.show_ci)
|
|
|
|
# CI Systems
|
|
if args.show_ci:
|
|
print_ci_systems(clients, entries)
|
|
|
|
# Windows versions
|
|
if not args.show_summary:
|
|
print_windows_major(clients, entries, args.show_ci)
|
|
|
|
# Windows versions detailed
|
|
if not args.show_summary:
|
|
print_windows_version_details(clients, args.show_ci)
|
|
|
|
# Pacman
|
|
if not args.show_summary:
|
|
print_pacman(clients, args.show_ci)
|
|
|
|
# CPU Arch
|
|
if not args.show_summary:
|
|
print_system_arch(clients, args.show_ci)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv)
|