diff --git a/autosurfer/ct.py b/autosurfer/ct.py new file mode 100644 index 0000000..2a65c61 --- /dev/null +++ b/autosurfer/ct.py @@ -0,0 +1,177 @@ +from datetime import UTC, datetime +import logging +import random +from functools import wraps +from json import JSONDecodeError +import asyncio +import base64 + +from cryptography import x509 +import httpx +import structlog + + +logger = structlog.stdlib.get_logger() +client = httpx.AsyncClient() + + +async def get_servers() -> set[str]: + """TODO.""" + # The format of these server lists are not part of the RFC. + # https://certificate.transparency.dev/useragents/ + server_lists = { + "https://www.gstatic.com/ct/log_list/v3/log_list.json", + "https://valid.apple.com/ct/log_list/current_log_list.json", + } + servers = set() + now = datetime.now(tz=UTC) + for server_list in server_lists: + try: + r = await client.get(server_list) + r.raise_for_status() + servers.update( + log["url"] + for operator in r.json()["operators"] + for log in operator["logs"] + if ("usable" in log["state"] + and datetime.fromisoformat(log["temporal_interval"]["start_inclusive"]) <= now + and datetime.fromisoformat(log["temporal_interval"]["end_exclusive"]) > now) + ) + except: + logger.exception("Error in log server list") + continue + if not servers: + raise ValueError("All log server lists failed") + return servers + + + +def decode_cert(leaf: bytes) -> x509.Certificate: + # MerkleTreeLeaf for timestamped entry containing an x509 certificate: + # + # +------+-----------------------+ + # | Byte | | + # +------+-----------------------+ + # | 0 | Version | + # +------+-----------------------+ + # | 1 | Leaf type | + # +------+-----------------------+ + # | 2 | | + # | 3 | | + # | 4 | | + # | 5 | Timestamp | + # | 6 | | + # | 7 | | + # | 8 | | + # | 9 | | + # +------+-----------------------+ + # | 10 | Entry type | + # | 11 | | + # +------+-----------------------+ + # | 12 | | + # | 13 | Cert length (n) | + # | 14 | | + # +------+-----------------------+ + # | 15 | | + # | .. | x509 DER cert | + # | n | | + # +------+-----------------------+ + # | n+1 | CT extensions | + # | .. | | + # +------+-----------------------+ + # + # https://www.rfc-editor.org/rfc/rfc6962.html#section-3.4 + # https://www.rfc-editor.org/rfc/rfc5246.html#section-4 + + # RFC 6962 only defines version 1 (0x00) of the merkle tree leaf and + # a single leaf type: timestamped entry (0x00). + if (version := leaf[0]) != 0: + raise ValueError(f"Unknown version {version}") + if (leaf_type := leaf[1]) != 0: + raise ValueError(f"Unknown leaf type {leaf_type}") + + if leaf[10:12] != b"\x00\x00": + # Timestamped entry type 0x0000 designates a x509 certificate. Type + # 0x001 is a precert, which we can not use, and therefore ignore. + raise TypeError("Not x509 entry") + + cert_length = int.from_bytes(leaf[12:15], "big") + cert_bytes = leaf[15 : 15 + cert_length] + cert = x509.load_der_x509_certificate(cert_bytes) + return cert + + +def forever(f): + @wraps(f) + async def wrapper(*args, **kwargs): + while True: + try: + await f(*args, **kwargs) + except Exception: + logger.exception("Retrying") + await asyncio.sleep(30) + except: + break + return wrapper + + +class Watcher: + page_size = 100 + + def __init__(self, server: str, queue: asyncio.Queue) -> None: + self.server = server + self.queue = queue + + self.log = logger.bind(server=server) + + self.tree_size = 0 + self.tree_watcher = asyncio.create_task(self.watch_tree_size()) + + self.start = 0 + self.end = 0 + + @forever + async def watch_tree_size(self) -> None: + r = await client.get(f"{self.server}ct/v1/get-sth") + self.tree_size = r.json()["tree_size"] + self.log.debug("Tree size", size=self.tree_size) + await asyncio.sleep(600) + + @forever + async def watcher(self) -> None: + index = random.randrange(self.start, self.tree_size - self.page_size) + r = await client.get(f"{self.server}ct/v1/get-entries", params={"start": index, "end": index + self.page_size,},) + entries = r.json()["entries"] + + now = datetime.now(tz=UTC) + expired = 0 + for entry in entries: + leaf = base64.b64decode(entry["leaf_input"]) + try: + cert = decode_cert(leaf) + except TypeError: + # Ignore precerts + continue + if cert.not_valid_before_utc > now: + continue + if cert.not_valid_after_utc < now: + expired += 1 + continue + await self.queue.put(cert) + + # All expired: move up + if len(entries) == expired > 5: + self.start = index + + +q = asyncio.Queue(maxsize=100) + + +async def asd(): + while True: + # await asyncio.sleep(10) + cert = await q.get() + print(cert) + + +asyncio.run(main(q)) diff --git a/autosurfer/main.py b/autosurfer/main.py index 0666453..41839c8 100644 --- a/autosurfer/main.py +++ b/autosurfer/main.py @@ -1,15 +1,11 @@ import asyncio -import json import math import os import random -import websockets from selenium import webdriver from selenium.common.exceptions import InvalidSessionIdException from selenium.common.exceptions import WebDriverException -from selenium.webdriver.firefox.service import Service -from selenium.webdriver.remote.webelement import WebElement service = webdriver.FirefoxService( # Selenium only checks /usr/bin/geckodriver by default @@ -27,42 +23,6 @@ driver = webdriver.Firefox(service=service, options=options) driver.set_page_load_timeout(3) -async def ct_stream(domains: asyncio.Queue) -> None: - """Watch Certificate Transparency (CT) logs for new certificates.""" - while True: - try: - async with websockets.connect("wss://certstream.calidog.io") as websocket: - async for message_data in websocket: - ct_handler(message_data, domains) - except (KeyboardInterrupt, asyncio.CancelledError): - return - except Exception as e: - print(e) - - -def ct_handler(data: websockets.Data, domains: asyncio.Queue) -> None: - """Save certificate's domain to queue if needed.""" - # There are A LOT of certificates coming through the transparency logs; - # immediately bail without spending time decoding the message if we have - # enough domains queued up already. - if domains.full(): - return - - message = json.loads(data) - if message["message_type"] != "certificate_update": - return - - # Certificates can verify multiple domains: We arbitrarily select the first - # non-wildcard one since we cannot connect to such host in the browser. - cert_domains = message["data"]["leaf_cert"]["all_domains"] - try: - cert_domain = next(d for d in cert_domains if "*" not in d) - except StopIteration: - return - - domains.put_nowait(cert_domain) - - async def surf(url: str) -> None: """Surf around URL for a bit.""" for i in range(math.ceil(random.expovariate(0.5))): @@ -77,7 +37,9 @@ async def surf(url: str) -> None: except InvalidSessionIdException: # Browser closed: no way to recover raise - except WebDriverException: + except WebDriverException as e: + print(e) + print(type(e)) # Timeout, network error, JavaScript failure etc. break try: diff --git a/autosurfer/test.py b/autosurfer/test.py new file mode 100644 index 0000000..5cb98ed --- /dev/null +++ b/autosurfer/test.py @@ -0,0 +1,71 @@ +import base64 +import json + +from cryptography import x509 + + +entry = json.loads(""" +{"entries":[{"leaf_input":"AAAAAAGKyn3LVQAAAAWaMIIFljCCBH6gAwIBAgISA1Ze6lz5tg9Vm1yFZ8CMFn59MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQswCQYDVQQDEwJSMzAeFw0yMzA5MjUwMjU5MTVaFw0yMzEyMjQwMjU5MTRaMCIxIDAeBgNVBAMTF2FjYWRlbWllLXJlcGFyYXRpb24uY29tMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA1/yZnBS5gqu6VeGZlAN0v4OiJVv5ou02edfaUBkWMwRZoTwGzCG1aVZ1T4Eds172YWlU7g2SnCnPBH0X6vg000gJCnrEAZxB9TOfOslcMHIuJkFH/wNhiUmeQfksdjdrQyyFzCnP/4LlMZeVYQteQMBLwjgwDzsuDJFxRAOoIJ/VTfJiacORSryRIeo8f1l5lC7gW8UGWCXj3kMIVhtZrnrWBRLU+d1wScNG88o2q1oWG223OxCyecQ8CmQEQfiwv4TrPlBL+PHclJztsmFEK6DzMgSjTkiiyeF5Z2/WcQBe3WFOPG79Esskr8zj7Bhc2xP+qaq7fY4ayPY8fb0/xIJPD6Tfz73QcbHuUVJAPa4msRjkb96M9EuhGALJhbxHCv5dodp2Wn8T2GqwZyTjIOjHsnE1naQcZf4PujZv1o1RXVeASV3PSQvRadzk4E3FI5WPxHHm5XSS/MuyMRtbqov5+4ZG5+XYgMZIiTjl+vwwaHJWQ83EMo1yYxb62Q/ZAgMBAAGjggI0MIICMDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFBv6Yowk5v1ZmFEIBOq87KPmUDe4MB8GA1UdIwQYMBaAFBQusxe3WFbLrlAJQOYfr52LFMLGMFUGCCsGAQUFBwEBBEkwRzAhBggrBgEFBQcwAYYVaHR0cDovL3IzLm8ubGVuY3Iub3JnMCIGCCsGAQUFBzAChhZodHRwOi8vcjMuaS5sZW5jci5vcmcvMD8GA1UdEQQ4MDaCF2FjYWRlbWllLXJlcGFyYXRpb24uY29tght3d3cuYWNhZGVtaWUtcmVwYXJhdGlvbi5jb20wEwYDVR0gBAwwCjAIBgZngQwBAgEwggECBgorBgEEAdZ5AgQCBIHzBIHwAO4AdQC3Pvsk35xNunXyOcW6WPRsXfxCz3qfNcSeHQmBJe20mQAAAYrKfcnyAAAEAwBGMEQCIED3KmO6PVRtmMhnBwZWYE5qgPUAm1hvaBXedvKrOfXBAiBbzgjjRJ93kscFUfuHOumxGmDAZpziUb7j7kWO22gNoAB1AHoyjFTYty22IOo44FIe6YQWcDIThU070ivBOlejUutSAAABisp9yf4AAAQDAEYwRAIgRJFMlpnrmKaAv8WdFuyusnS5o6zOuxFy2/p1qv9GSNYCIGj2NUTCeI0fiVPerNOt8TIa2HkPo2akOfmdhDnc72pHMA0GCSqGSIb3DQEBCwUAA4IBAQBR4OsZnweD+y2D4Gtq3k8ukX+eDjawSsecXLha/kmMA/NpwgKh7Wh548SROkbapHyqWZKTTCIMbFLiYqsORCfT4G4QZXpdQLVD0jik4nr1N44sbbK8dmbpWJUEHRVLV+2KUctKiBAFD/p6tWRfsyi60mdpYqqT2HL/KcNNvtcHIdma6dQHldqHAPUUjqmIt5zZcuCiEA4AK6Ma9ZJpDmCqMi+WbB4cUqVXZSW36Sg4OXgmwXHceZpeKSu3aNe7MWUXZzNoUmc8Ss5oK6Rl9t2oXaxmYROvrB2YNFrI0mVUXoe/6MeDg/IGK6lIPE9/jnfDvQgb3NyAJ+tC17Is773gAAA=","extra_data":"AAqPAAUaMIIFFjCCAv6gAwIBAgIRAJErCErPDBinU/bWLiWnX1owDQYJKoZIhvcNAQELBQAwTzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2VhcmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMjAwOTA0MDAwMDAwWhcNMjUwOTE1MTYwMDAwWjAyMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNTGV0J3MgRW5jcnlwdDELMAkGA1UEAxMCUjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC7AhUozPaglNMPEuyNVZLD+ILxmaZ6QoinXSaqtSu5xUyxr45r+XXIo9cPR5QUVTVXjJ6oojkZ9YI8QqlObvU7wy7bjcCwXPNZOOftz2nwWgsbvsCUJCWH+jdxsxPnHKzhm+/b5DtFUkWWqcFTzjTIUu61ru2P3mBw4qVUq7ZtDpelQDRrK9O8ZutmNHz6a4uPVymZ+DAXXbpyb/uBxa3Shlg9F8fnCbvxK/eG3MHacV3URuPMrSXBiLxgZ3Vms/EY96Jc5lP/Ooi2R6X/ExjqmAl3P51T+c8B5fWmcBcUr2Ok/5mzk53cU6cG/kiFHaFpriV1uxPMUgP17VGhi9sVAgMBAAGjggEIMIIBBDAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFBQusxe3WFbLrlAJQOYfr52LFMLGMB8GA1UdIwQYMBaAFHm0WeZ7tuXkAXOACIjIGlj26ZtuMDIGCCsGAQUFBwEBBCYwJDAiBggrBgEFBQcwAoYWaHR0cDovL3gxLmkubGVuY3Iub3JnLzAnBgNVHR8EIDAeMBygGqAYhhZodHRwOi8veDEuYy5sZW5jci5vcmcvMCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCFyk5HPqP3hUSFvNVneLKYY611TR6WPTNlclQtgaDqw+34IL9fzLdwALduO/ZelN7kIJ+m74uyA+eitRY8kc607TkC53wlikfmZW4/RvTZ8M6UK+5UzhK8jCdLuMGYL6KvzXGRSgi3yLgjewQtCPkIVz6D2QQzCkcheAmCJ8MqyJu5zlzyZMjAvnnAT45tRAxekrsu94sQ4egdRCnbWSDtY7kh+BImlJNXoB1lBMEKIq4QDUOXoRgffuDghje1WrG9ML+Hbisq/yFOGwXD9RiX8F6sw6W4avAuvDszue5L3sz85K+EC4Y/wFVDNvZo4TYXao6Z0f+lQKc0t8DQYzk1OXVu8rp2yJMC6alLbBfODALZvYH7n7do1AZls4I9d1P4jnkDrQoxB3UqQ9hVl3LEKQ73xF1OyK5GhDDX8oVfGKF5u+decIsH4YaTw7mP3GFxJSqv3+0lUFJoi5Lc5da149p90IdshCExroL1+7mryIkXPeFM5TgO9r0rvZaBFOvV2z0gp35Z0+L4WPlbuEjN/lxPFin+HlUjr8gRsI3qfJOQFy/9rKIJR0Y/8Omwt/8oTWgy1mdeHmmjk7j1nYsvC9JSQ6ZvMldlTTKB3zhThV1+XWYp6rjd5JW1zbVWEkLNxE7GJThEUG3szgBVGP7pSWTUTsqXnLRbwHOoq7hHwgAFbzCCBWswggNToAMCAQICEQCCEM+w0kDjWURj4LtjgosAMA0GCSqGSIb3DQEBCwUAME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBTZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgxMB4XDTE1MDYwNDExMDQzOFoXDTM1MDYwNDExMDQzOFowTzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2VhcmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCt6CRz9BQ385ueK1coHIe+3LffOJCMbjzmV6B493XCov71am72AE8o295ohmxEk7axY/0UEmu/H9LqMZshftEzPLpI9d1537O4/xLxIZpLwYqGcWlKZmZsj348cL+tKSIG8+TA5oCu4kuPt5l+lAOf00eXfJlII1PoOK5PCm+DLtFJV4yAdLbaL9A4jXsDcCEbdfIwPPqPrt3aY6vrFk/CjhFLfs8L6P+1dy70sntK4EwSJQxwjQMpoOFTJOwT2e4ZvxCzSow/iaNhUd6shweU9GNx7C7ib1uYgeGJXDR5bHbvO5BieebbpJovJsXQEOEO3tkQjhb7t/eo98flAgeYjzYIlefiN5YNNnWe+w5ysR2bvAP5SQXYgd0FtCrWQemsAXaVCg/Y39W9Eh81LygXbNKYwagJZHduRze6zqxZXmidf3LWicUGQSk+WT7dJvUkyRGnWqNMQB9GoZm1pzpRboY7nn1ypxIFeFntPlF4FQsDj43QLwWyPntKHEtzBRL8xurgUBN8Q5N0s8p0544fAQjQMNRbcTa0B7rBMDBcSLeCO5imfWCKoqMpgsy6vYMEG6KDA0Gh1gXxG8K28Kh8hjtGqEgqiNx2mna/H2qlPRmP6zjzZN7IKw0KKP/32+IVQtQi0Cdd4Xn+GOdwiK1O5tmLOsbdJ1Fu/7xk9TNDTwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUebRZ5nu25eQBc4AIiMgaWPbpm24wDQYJKoZIhvcNAQELBQADggIBAFUfWKm8sqhQ0Ayx2BppICcpCKxhdVyKbviC5Wkv1fZWS7m4cxBZ0yGXfudMcfuy0mCtOagL6hchVoXxUA5Z687gWem6yRXvhp2PhID25OmRkNwXm2IbRfBmldJ8b8LqO+8fz8vWrifxqbDIrv19fpr6IgTr/9l/6pErIrEXDo/yijRbWNj8AclUubgmzIqIM4lMLYQ8gt/ullcFuiy798S3x047gr4xyCJzc5LRwoCkOTkQMyOCTDyfhrJVmB2+KYaMIpue4ms7VzqCcE3cCceJywoHTWzoXY7J786rx7u1K05F1krQJszlcsoIaqWV4xWh96TtySxfpfv/rCgCLr7Xe7vjcXuQFtMHXkZTfDcHQozTxJac1Zm1KuCVGoBIrkw5B87MR6RSlSu6uPut0jNTfeUdTW3VobHHQm/mQCc1XKMotweN540zkOcjn/tQnHlsRtW0FbOWbn6bDJY6uFItP9Zb4fsIwoT+JKijidqsauEYKrGoQ2Fb0x/cO4128i3ojXXfFzNsPVP7e8tBX//cotBhOOGWuKxdizfXddUzwJkRrp1BwXJ1hL4CQUJfZyRIlNGbJ74HP7m4T4F0UeF6t+2dI+K+4NUoBBM8MQOe3Xpsj8YHGMZ/3keOPyieBAbPpVQ0d73siZvpF0PfW9tf/o4eV6LNQJ1+YiLa3hgn"},{"leaf_input":"AAAAAAGKyn2rQQABvxRon/El5KuI4vx5ey1DgmsYmRY0nDd5Cg4GfJ8S+bgAA38wggN7oAMCAQICEANHV5kRW5SzAVSiOiQxsS8wDQYJKoZIhvcNAQELBQAwPDELMAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEcMBoGA1UEAxMTQW1hem9uIFJTQSAyMDQ4IE0wMzAeFw0yMzA5MjUwMDAwMDBaFw0yMzA5MjcyMzU5NTlaMCwxKjAoBgNVBAMTITE2OTU2MTQzNDI5NjQuYW1kdi5keWluZ2JpcmRzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM6YMfbmf+oreseu/eSpgyhiKfCpONcTv5+saa2pT9N8zJGqfm2VCkJ8qld1yfto07UQ7XZtgr17GzroM3zExMCU+6J/F1YaWvHsAINRAXIdXBdJuv8WCuynNVP0W847hfC5MMlBo/rzjlck+hmd7T8Mhva7SqcLLuOndXWftfZj/ooyAOLJKZEgdKBSJ4kKz2EuWI8D9KIWlP+Gy1w1p+u7x3Z2kGYLa1EY4QhX8+RWBqt5ReCKx/98C/leZdYoDD11qwjJwW61h4PREC7/b1AHi/GpgxJ6JdX1RIYtQ+tK8Pz6YNtsBppECrKRrN1T+PAbhRqZfoBzeIGsX77dlh0CAwEAAaOCAZ8wggGbMB8GA1UdIwQYMBaAFFXZGF/SHMwB4Vi0vqvZVUIB1y4CMB0GA1UdDgQWBBSyCAseRw656y6h1A4vYL6otPZg9jBTBgNVHREETDBKgiExNjk1NjE0MzQyOTY0LmFtZHYuZHlpbmdiaXJkcy5jb22CJXNhbi4xNjk1NjE0MzQyOTY0LmFtZHYuZHlpbmdiaXJkcy5jb20wEwYDVR0gBAwwCjAIBgZngQwBAgEwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA7BgNVHR8ENDAyMDCgLqAshipodHRwOi8vY3JsLnIybTAzLmFtYXpvbnRydXN0LmNvbS9yMm0wMy5jcmwwdQYIKwYBBQUHAQEEaTBnMC0GCCsGAQUFBzABhiFodHRwOi8vb2NzcC5yMm0wMy5hbWF6b250cnVzdC5jb20wNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQucjJtMDMuYW1hem9udHJ1c3QuY29tL3IybTAzLmNlcjAMBgNVHRMBAf8EAjAAAAA=","extra_data":"AASsMIIEqDCCA5CgAwIBAgIQA0dXmRFblLMBVKI6JDGxLzANBgkqhkiG9w0BAQsFADA8MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRwwGgYDVQQDExNBbWF6b24gUlNBIDIwNDggTTAzMB4XDTIzMDkyNTAwMDAwMFoXDTIzMDkyNzIzNTk1OVowLDEqMCgGA1UEAxMhMTY5NTYxNDM0Mjk2NC5hbWR2LmR5aW5nYmlyZHMuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzpgx9uZ/6it6x6795KmDKGIp8Kk41xO/n6xpralP03zMkap+bZUKQnyqV3XJ+2jTtRDtdm2CvXsbOugzfMTEwJT7on8XVhpa8ewAg1EBch1cF0m6/xYK7Kc1U/RbzjuF8LkwyUGj+vOOVyT6GZ3tPwyG9rtKpwsu46d1dZ+19mP+ijIA4skpkSB0oFIniQrPYS5YjwP0ohaU/4bLXDWn67vHdnaQZgtrURjhCFfz5FYGq3lF4IrH/3wL+V5l1igMPXWrCMnBbrWHg9EQLv9vUAeL8amDEnol1fVEhi1D60rw/Ppg22wGmkQKspGs3VP48BuFGpl+gHN4gaxfvt2WHQIDAQABo4IBtDCCAbAwHwYDVR0jBBgwFoAUVdkYX9IczAHhWLS+q9lVQgHXLgIwHQYDVR0OBBYEFLIICx5HDrnrLqHUDi9gvqi09mD2MFMGA1UdEQRMMEqCITE2OTU2MTQzNDI5NjQuYW1kdi5keWluZ2JpcmRzLmNvbYIlc2FuLjE2OTU2MTQzNDI5NjQuYW1kdi5keWluZ2JpcmRzLmNvbTATBgNVHSAEDDAKMAgGBmeBDAECATAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMDsGA1UdHwQ0MDIwMKAuoCyGKmh0dHA6Ly9jcmwucjJtMDMuYW1hem9udHJ1c3QuY29tL3IybTAzLmNybDB1BggrBgEFBQcBAQRpMGcwLQYIKwYBBQUHMAGGIWh0dHA6Ly9vY3NwLnIybTAzLmFtYXpvbnRydXN0LmNvbTA2BggrBgEFBQcwAoYqaHR0cDovL2NydC5yMm0wMy5hbWF6b250cnVzdC5jb20vcjJtMDMuY2VyMAwGA1UdEwEB/wQCMAAwEwYKKwYBBAHWeQIEAwEB/wQCBQAwDQYJKoZIhvcNAQELBQADggEBAAW3Mm7KdTp3N/D012oBRsXPDDgSlEbNaEEKkI82WHH2lrTbCqcU58tnJ7zQ6HGSO/MAKRpfkKfGQwmyuLkt9ikzw3XvLx57GH21NJkS6CDg+2x07i5ED6nJRbVnox6Yn2NxjH+yqzZy3GHNEl2puAd2bi93CZgOiFJNbTe0LSiHReaLjwpa5HWXfFP/xawbue0kS+1rFPy0PLmuT4KO0pj/f6/ycw7V6X07Mhhcs6yW3ZEGIb2WVYVqT7+RyiIv86YgO2s8NBWsJ2Sy+Z6ANyKA8aey+wdIolEiBIJKlqDfJsD9UpmauvMxyDjPrP7XrIwnkYEVIskVHqKN85FNRwwAB60ABGIwggReMIIDRqADAgECAhMHcxJM1AbSZ8CZHN0pmp84MXmFMA0GCSqGSIb3DQEBCwUAMDkxCzAJBgNVBAYTAlVTMQ8wDQYDVQQKEwZBbWF6b24xGTAXBgNVBAMTEEFtYXpvbiBSb290IENBIDEwHhcNMjIwODIzMjIyNjA0WhcNMzAwODIzMjIyNjA0WjA8MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRwwGgYDVQQDExNBbWF6b24gUlNBIDIwNDggTTAzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt3+lWSjy+4zjvlN/jkdXcFsKX8H0nLz0bUJGQWNw9jTpYD2VL7t1ZtQDsa1ZQqiSyvr2EozBwyw2nGXDtveN5cWDEHT3+eZqVwBa2cvMX4toMoQDkNCa8rO6HWd8h+oSIoJBKZdeHT1ez507tybmptovaGE0kJIB6NyLBy84WAU2AQxHjbCboogUexBbbyOED7W78zSnp9XBF64SBl8t93HzY/HQslyTeVJPcQH8l9t2x07PPODliRjV1+6p/zLl9R5nsLNZdtOO6PBf9Oi+Z5YoDfpUsLPvlrNbq0M24OfroU6kANln2iZVoXoGo0mIsxWXo/f9zQ6JQwLpnk58pwIDAQABo4IBWjCCAVYwEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBRV2Rhf0hzMAeFYtL6r2VVCAdcuAjAfBgNVHSMEGDAWgBSEGMyFNOy8DJSULghZnMeyEE4KCDB7BggrBgEFBQcBAQRvMG0wLwYIKwYBBQUHMAGGI2h0dHA6Ly9vY3NwLnJvb3RjYTEuYW1hem9udHJ1c3QuY29tMDoGCCsGAQUFBzAChi5odHRwOi8vY3J0LnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jvb3RjYTEuY2VyMD8GA1UdHwQ4MDYwNKAyoDCGLmh0dHA6Ly9jcmwucm9vdGNhMS5hbWF6b250cnVzdC5jb20vcm9vdGNhMS5jcmwwEwYDVR0gBAwwCjAIBgZngQwBAgEwDQYJKoZIhvcNAQELBQADggEBAAaN5abZwL7fPYzNIKe17jr/slm+PeJBkNjsS84M11kXjWfvUuuT2Rt83B6srR1U3NbfpZ5+vuPtlLAFWDnRblrLrmM1I3vhNNBXXK3hadJtW83oqUyzWGLI5l2gcqwoNXZQY3Ft2gFgJA1rYZgXwVAhYu6A2nflpI6kzLpwWXKYVCiYiuc5AfArWe9VjirENIkxT+A0nPXsPwCs83eeCfJnZPB6knS4Eg3Pcbm17at0VQXiLhjcFvsVSUcNibi7xohEamTZml9ebFqjGdsgFlG8Ys9JS4/CfChQ4ywnyOKsjm+zZ29yE9vsy0jWuV6LTZ/aK8fa/GSMMuW1Kl7Z+IIAA0UwggNBMIICKaADAgECAhMGbJ/Pmb+MCjni8HiKQ+aWNlvKMA0GCSqGSIb3DQEBCwUAMDkxCzAJBgNVBAYTAlVTMQ8wDQYDVQQKEwZBbWF6b24xGTAXBgNVBAMTEEFtYXpvbiBSb290IENBIDEwHhcNMTUwNTI2MDAwMDAwWhcNMzgwMTE3MDAwMDAwWjA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24gUm9vdCBDQSAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsniAccp41eNxr0eAUHR9btjXiHb0mWj3WCFg+XSEAS+sAi2G06BDek6ypNA2ugG+jdtIyAcXNkz07ogjxz7rN/W1GfhJaLDe17l2OB1hnqT+gjal5UpW5EXh+f20Fvp02pybNTkv+rAgUAZsetCAsqb5r+xHGY9QOAfcooc5WPi61an5SGcwlu6UeF5viaNRwDCGZqFFZrpU66PDkflI3P/R6DAtfS10cDXXiCT3nsRZbrtzhxfyMkYouEP6tx2qyrTynyQOLUv3cVxeaf/qlQLLOIquUDhv2/stYhvFxx5U4XfgZ8gPnIcj1j9AIH8ggMSATD47JCaOBK5smsiqDQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUhBjMhTTsvAyUlC4IWZzHshBOCggwDQYJKoZIhvcNAQELBQADggEBAJjyN1pBkKEaxXZRKCA2Iw6u5ii7qviUrkikMH8b/CSNS7TIoZf2tvF6cMhTk8wIKOOYJc8jpPneIdN8hQmtTpp1OsILaol4dkRHGGVsjUGOO3+ay/S1p1DXBSw36ANLrelhoAJu9fLwxbLtW7fc+pRcd54TpX9SrZXy+JM73otcW8paUltgrxT3S++j+59AlW0xVPxC08dGHyOt2Q9IcJrZdXhx0XJDNHVuV1nCAlwmYCnPIxkWjohDpdTkywj7IxFD6EMpcmKhqV1eCNSQrrjYzhTC0FXyhvbEk0N3ZmHAuehB15d4YANuSnKupdF9uhCehmwbirlZM/jrxJC+8bk="}]} +""")["entries"][0] + + +def decode_cert(leaf: bytes) -> x509.Certificate: + # MerkleTreeLeaf for timestamped entry containing an x509 certificate: + # + # +------+-----------------------+ + # | Byte | | + # +------+-----------------------+ + # | 0 | Version | + # +------+-----------------------+ + # | 1 | Leaf type | + # +------+-----------------------+ + # | 2 | | + # | 3 | | + # | 4 | | + # | 5 | Timestamp | + # | 6 | | + # | 7 | | + # | 8 | | + # | 9 | | + # +------+-----------------------+ + # | 10 | Entry type | + # | 11 | | + # +------+-----------------------+ + # | 12 | | + # | 13 | Cert length (n) | + # | 14 | | + # +------+-----------------------+ + # | 15 | | + # | .. | x509 DER cert | + # | n | | + # +------+-----------------------+ + # | n+1 | CT extensions | + # | .. | | + # +------+-----------------------+ + # https://www.rfc-editor.org/rfc/rfc6962.html#section-3.4 + # https://www.rfc-editor.org/rfc/rfc5246.html#section-4 + + # RFC 6962 only defines version 1 (0x00) of the merkle tree leaf and + # a single leaf type: timestamped entry (0x00). + if (version := leaf[0]) != 0: + raise ValueError(f"Unknown version {version}") + if (leaf_type := leaf[1]) != 0: + raise ValueError(f"Unknown leaf type {leaf_type}") + + if leaf[10:12] != b"\x00\x00": + # Timestamped entry type 0x0000 designates a x509 certificate. Type + # 0x001 is a precert, which we can not use, and therefore ignore. + raise TypeError("Not x509 entry") + + cert_length = int.from_bytes(leaf[12:15], "big") + cert_bytes = leaf[15 : 15 + cert_length] + cert = x509.load_der_x509_certificate(cert_bytes) + + return cert + + +leaf = base64.b64decode(entry["leaf_input"]) + +cert = decode_cert(leaf) + +print(cert.subject) diff --git a/flake.nix b/flake.nix index 3c5b832..aa365f6 100644 --- a/flake.nix +++ b/flake.nix @@ -45,8 +45,10 @@ }) pkgs.geckodriver (pkgs.python3.withPackages (ps: [ + ps.cryptography + ps.httpx ps.selenium - ps.websockets + ps.structlog ])) # pkgs.bashInteractive # pkgs.coreutils