Facing issue with timeout for wildcard domain poll_and_finalize is just in timeout state

Hi this is my code : import os
import boto3
import logging
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.backends import default_backend
from cryptography import x509
from cryptography.x509.oid import NameOID
from acme import client, messages, challenges
from acme.errors import PollError
from josepy import jwk
from datetime import datetime, timedelta
import time

Set up logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(name)

S3 client to upload files

s3 = boto3.client('s3')

Constants

S3_BUCKET_NAME = "abc-test-abc-000-test-submit1"
S3_CERTIFICATE_PATH = "ssl_auto_certs/certificate.pem"
S3_PRIVATE_KEY_PATH = "ssl_auto_certs/private_key.pem"
DOMAIN = "testvar.hydra.sophos.com"
EMAIL = "varun.vikram@sophos.com" # Update with your email

ACME Directory URL (for Let's Encrypt)

ACME_DIRECTORY_URL = "https://acme-staging-v02.api.letsencrypt.org/directory" # Production URL (use staging for testing)

Generate CSR and private key for both server and client authentication

def generate_csr_and_key(domain):
private_key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)

# Create CSR with extensions for server and client authentication
csr_builder = x509.CertificateSigningRequestBuilder().subject_name(
    x509.Name([
        x509.NameAttribute(NameOID.COMMON_NAME, DOMAIN),
        x509.NameAttribute(NameOID.ORGANIZATION_NAME, "SOPHOS LIMITED"),
        x509.NameAttribute(NameOID.COUNTRY_NAME, "GB"),
        x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Oxfordshire"),
        x509.NameAttribute(NameOID.LOCALITY_NAME, "Abingdon"),
    ])
).add_extension(
    x509.KeyUsage(
        digital_signature=True,
        key_encipherment=True,
        content_commitment=False,
        key_agreement=False,
        data_encipherment=False,
        key_cert_sign=False,
        crl_sign=False,
        encipher_only=False,
        decipher_only=False
    ),
    critical=True
).add_extension(
    x509.ExtendedKeyUsage([
        x509.ExtendedKeyUsageOID.SERVER_AUTH,   # For Server Authentication
        x509.ExtendedKeyUsageOID.CLIENT_AUTH    # For Client Authentication
    ]),
    critical=True
).add_extension(
    x509.SubjectAlternativeName([
        x509.DNSName(f"*.{domain}"),
        x509.DNSName(domain),  # Wildcard for both server and client
    ]),
    critical=False
)

# Sign the CSR with the private key
csr = csr_builder.sign(private_key, hashes.SHA256(), default_backend())

# Save CSR and private key to /tmp directory (Lambda temporary storage)
csr_path = "/tmp/certificate.csr"
private_key_path = "/tmp/private_key.pem"

with open(csr_path, "wb") as csr_file:
    csr_file.write(csr.public_bytes(serialization.Encoding.PEM))

with open(private_key_path, "wb") as key_file:
    key_file.write(private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ))

logger.info(f"CSR and private key generated and saved to /tmp: {csr_path}, {private_key_path}")

return csr_path, private_key_path, csr, private_key

Upload to S3

def upload_to_s3(cert_path, private_key_path):
try:
# Upload CSR to S3
with open(cert_path, "rb") as csr_file:
s3.put_object(
Bucket=S3_BUCKET_NAME,
Key=S3_CERTIFICATE_PATH,
Body=csr_file,
ServerSideEncryption='AES256'
)

    # Upload Private Key to S3
    with open(private_key_path, "rb") as key_file:
        s3.put_object(
            Bucket=S3_BUCKET_NAME,
            Key=S3_PRIVATE_KEY_PATH,
            Body=key_file,
            ServerSideEncryption='AES256'
        )

    logger.info("CSR and private key successfully uploaded to S3.")

except Exception as e:
    logger.error(f"Failed to upload to S3: {e}")
    raise

Perform DNS-01 Challenge for domain validation

def dns01_challenge(orderr, domain, private_key_jwk, client_acme):
route53 = boto3.client('route53')
dns_challenge = None

# Loop through the authorizations to find the DNS-01 challenge
for auth in orderr.authorizations:
    logger.info(f"Authorization URL: {auth.uri}")
    for challenge in auth.body.challenges:
        logger.info(f"Challenge status: {challenge.status}, Challenge URL: {challenge.uri}")
        
        if isinstance(challenge.chall, challenges.DNS01):
            dns_challenge = challenge
            logger.info(f"Found DNS-01 challenge: {challenge}")
            break
    if dns_challenge:
        break

# If DNS-01 challenge is not found, raise an exception
if dns_challenge is None:
    raise Exception("No DNS-01 challenge found.")

# Create TXT record for DNS validation
response, validation = dns_challenge.response_and_validation(private_key_jwk)
change_batch = {
    'Changes': [{
        'Action': 'UPSERT',
        'ResourceRecordSet': {
            'Name':f"_acme-challenge.{domain}",
            'Type': 'TXT',
            'TTL': 60,
            'ResourceRecords': [{'Value': f'"{validation}"'}]
        }
    }]
}

# Update Route 53 DNS record
try:
    route53.change_resource_record_sets(HostedZoneId="Z07477012J1NZ3TLVGDW7", ChangeBatch=change_batch)
    logger.info(f"DNS TXT record added for {dns_challenge.validation_domain_name(domain)}")
except Exception as e:
    logger.error(f"Error adding DNS TXT record: {e}")
    raise

# Wait for DNS propagation and finalize the order
client_acme.answer_challenge(dns_challenge, response)
finalized_orderr = client_acme.poll_and_finalize(orderr)
return finalized_orderr.fullchain_pem
#retries = 30
#for i in range(retries):
    #try:
        #finalized_orderr = client_acme.poll_and_finalize(order)
        #logger.info(f"Certificate issued: {finalized_order.fullchain_pem}")
        #return finalized_order.fullchain_pem  # Return the certificate chain
    #except PollError:
        #wait_time = 30 * (i + 1)
        #ogger.info(f"Retrying in {wait_time} seconds...")
        #time.sleep(wait_time)

#raise Exception("DNS validation failed after retries.")

Request certificate from Let's Encrypt using CSR

def request_certificate_from_letsencrypt(csr, private_key):
# Create a client to interact with Let's Encrypt
private_key_pem = private_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption()
)

# Generate JWK from private key using jwk.RSAKey
private_key_jwk = jwk.JWKRSA(key=private_key)

# Initialize ACME client
net = client.ClientNetwork(private_key_jwk, user_agent="AWS Lambda/ACME")
directory = client.ClientV2.get_directory(ACME_DIRECTORY_URL, net)
client_acme = client.ClientV2(directory, net=net)

# Register account with Let's Encrypt
account = client_acme.new_account(messages.NewRegistration.from_data(email=EMAIL, terms_of_service_agreed=True))

# Create new order with the CSR
orderr = client_acme.new_order(csr.public_bytes(serialization.Encoding.PEM))

# Perform DNS-01 challenge for domain validation
certificate = dns01_challenge(orderr, DOMAIN, private_key_jwk, client_acme)

# Save certificate and private key to /tmp directory (Lambda storage)
cert_path = "/tmp/certificate.pem"
private_key_path = "/tmp/private_key.pem"
with open(cert_path, "wb") as cert_file:
    cert_file.write(certificate)

with open(private_key_path, "wb") as key_file:
    key_file.write(private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ))

logger.info(f"Certificate and private key saved to: {cert_path}, {private_key_path}")

return cert_path, private_key_path

Main handler function

def main(event, context):
try:
# Generate CSR and private key
csr_path, private_key_path, csr, private_key = generate_csr_and_key(DOMAIN)

    # Request certificate from Let's Encrypt
    cert_path, private_key_path = request_certificate_from_letsencrypt(csr, private_key)
    
    # Upload certificate and private key to S3
    upload_to_s3(cert_path, private_key_path)
    
except Exception as e:
    logger.error(f"Error during certificate generation and upload: {e}")
    raise

Lambda handler

def lambda_handler(event, context):
main(event, context)
I am trying to generate the certificate for my wildcard but all the time it is showing error.timeout can anyone help me on this , I am using route53 where the _acme-challenge. is created but don't know what I am missing .

Hello @Varunvik88, welcome to the Let's Encrypt community. :slightly_smiling_face:

Possibly because the FQDN doesn't seem to exist.

Checking with nslookup

$ nslookup -q=ns testvar.hydra.sophos.com ns-930.awsdns-52.net.
Server:         ns-930.awsdns-52.net.
Address:        205.251.195.162#53

** server can't find testvar.hydra.sophos.com: NXDOMAIN
$ nslookup -q=ns testvar.hydra.sophos.com ns-209.awsdns-26.com.
Server:         ns-209.awsdns-26.com.
Address:        205.251.192.209#53

** server can't find testvar.hydra.sophos.com: NXDOMAIN
$ nslookup -q=ns testvar.hydra.sophos.com ns-1592.awsdns-07.co.uk.
Server:         ns-1592.awsdns-07.co.uk.
Address:        205.251.198.56#53

** server can't find testvar.hydra.sophos.com: NXDOMAIN
$ nslookup -q=ns testvar.hydra.sophos.com ns-1378.awsdns-44.org.
Server:         ns-1378.awsdns-44.org.
Address:        205.251.197.98#53

** server can't find testvar.hydra.sophos.com: NXDOMAIN

And with dig

$ dig +trace testvar.hydra.sophos.com NS

; <<>> DiG 9.18.30-0ubuntu0.24.04.2-Ubuntu <<>> +trace testvar.hydra.sophos.com NS
;; global options: +cmd
.                       514486  IN      NS      j.root-servers.net.
.                       514486  IN      NS      k.root-servers.net.
.                       514486  IN      NS      l.root-servers.net.
.                       514486  IN      NS      m.root-servers.net.
.                       514486  IN      NS      a.root-servers.net.
.                       514486  IN      NS      b.root-servers.net.
.                       514486  IN      NS      c.root-servers.net.
.                       514486  IN      NS      d.root-servers.net.
.                       514486  IN      NS      e.root-servers.net.
.                       514486  IN      NS      f.root-servers.net.
.                       514486  IN      NS      g.root-servers.net.
.                       514486  IN      NS      h.root-servers.net.
.                       514486  IN      NS      i.root-servers.net.
;; Received 811 bytes from 127.0.0.53#53(127.0.0.53) in 9 ms

;; communications error to 192.36.148.17#53: timed out
;; communications error to 192.36.148.17#53: timed out
;; communications error to 192.36.148.17#53: timed out
;; UDP setup with 2001:503:c27::2:30#53(2001:503:c27::2:30) for testvar.hydra.sophos.com failed: network unreachable.
;; UDP setup with 2001:503:ba3e::2:30#53(2001:503:ba3e::2:30) for testvar.hydra.sophos.com failed: network unreachable.
;; UDP setup with 2001:7fd::1#53(2001:7fd::1) for testvar.hydra.sophos.com failed: network unreachable.
com.                    172800  IN      NS      a.gtld-servers.net.
com.                    172800  IN      NS      b.gtld-servers.net.
com.                    172800  IN      NS      c.gtld-servers.net.
com.                    172800  IN      NS      d.gtld-servers.net.
com.                    172800  IN      NS      e.gtld-servers.net.
com.                    172800  IN      NS      f.gtld-servers.net.
com.                    172800  IN      NS      g.gtld-servers.net.
com.                    172800  IN      NS      h.gtld-servers.net.
com.                    172800  IN      NS      i.gtld-servers.net.
com.                    172800  IN      NS      j.gtld-servers.net.
com.                    172800  IN      NS      k.gtld-servers.net.
com.                    172800  IN      NS      l.gtld-servers.net.
com.                    172800  IN      NS      m.gtld-servers.net.
com.                    86400   IN      DS      19718 13 2 8ACBB0CD28F41250A80A491389424D341522D946B0DA0C0291F2D3D7 71D7805A
com.                    86400   IN      RRSIG   DS 8 1 86400 20250221050000 20250208040000 26470 . KsDXOYofS9soPKLRWnQ2gQF01I5oO2G044LpNuQFJ596OFKgIUgkwFcn fDNhqlbWzpUZYZYM/FNk0xBPJp92BuyRFhNyPlrWLR4GJkL0W4YhyHfE fgDS+sr18nkmjMAUHyefGIHSlAtBQ/yfMA9xJ6dQ+5sRvES/SQQDAiKy mF4e7zw9k6X/VVScBwTGyBfjeJHnz6kgzUavryShBrktPqdOXstBZOkq Ydq1UGA9Gs642cTronr8+Ha3TYHWDzEoLvbfTNjqsmdgWhFSGapQJh2U h+S2FdUVBYeY7/bd77EMvmTAoLJE9xPafvba/Tm00+OCLDqX9oqNvuLY wPmtDQ==
;; Received 1184 bytes from 192.58.128.30#53(j.root-servers.net) in 13 ms

;; UDP setup with 2001:503:39c1::30#53(2001:503:39c1::30) for testvar.hydra.sophos.com failed: network unreachable.
sophos.com.             172800  IN      NS      a11-66.akam.net.
sophos.com.             172800  IN      NS      a10-65.akam.net.
sophos.com.             172800  IN      NS      a9-65.akam.net.
sophos.com.             172800  IN      NS      a1-100.akam.net.
sophos.com.             172800  IN      NS      a14-67.akam.net.
sophos.com.             172800  IN      NS      a18-64.akam.net.
CK0POJMG874LJREF7EFN8430QVIT8BSM.com. 900 IN NSEC3 1 1 0 - CK0Q3UDG8CEKKAE7RUKPGCT1DVSSH8LL NS SOA RRSIG DNSKEY NSEC3PARAM
CK0POJMG874LJREF7EFN8430QVIT8BSM.com. 900 IN RRSIG NSEC3 13 2 900 20250212002642 20250204231642 23202 com. R8ldMEiXxK+YJH03CPrVNsnsd/n7VucDXHI6rXm9xeEZH/DccZLHosj4 9MJzMIDC/xKypGz7hxIyrxifHQDHOw==
CMQR1MI4DRUNSVSVRCLBK17CEVMQLAS1.com. 900 IN NSEC3 1 1 0 - CMQRONPC55NG78U9E418QOQ1DQ0IUFT0 NS DS RRSIG
CMQR1MI4DRUNSVSVRCLBK17CEVMQLAS1.com. 900 IN RRSIG NSEC3 13 2 900 20250212022200 20250205011200 23202 com. H0bWQI7lw8+b8Dy/PqiL7IT1PH3k+/WiI3LmrN1NFiVO8W0g2G9rhvTB yh1nU2Op8vzuc6nOpMwDO9IWTXsyGA==
;; Received 543 bytes from 192.55.83.30#53(m.gtld-servers.net) in 93 ms

;; UDP setup with 2600:1480:d000::41#53(2600:1480:d000::41) for testvar.hydra.sophos.com failed: network unreachable.
hydra.sophos.com.       300     IN      NS      ns-930.awsdns-52.net.
hydra.sophos.com.       300     IN      NS      ns-1378.awsdns-44.org.
hydra.sophos.com.       300     IN      NS      ns-1592.awsdns-07.co.uk.
hydra.sophos.com.       300     IN      NS      ns-209.awsdns-26.com.
;; Received 190 bytes from 184.85.248.65#53(a9-65.akam.net) in 25 ms

hydra.sophos.com.       900     IN      SOA     ns-930.awsdns-52.net. awsdns-hostmaster.amazon.com. 1 7200 900 1209600 86400
;; Received 134 bytes from 205.251.195.162#53(ns-930.awsdns-52.net) in 24 ms

Edit

And this shows the same results as well testvar.hydra.sophos.com | DNSViz

Hi thanks I did change my domain my domain now is inf.hydra.sophos.com and the txt entry is done in this hosted zone as _acme-challenge. inf.hydra.sophos.com but now I am getting an error : Status: Failed
Test Event Name: ssl_test

Response:
{
"errorMessage": "",
"errorType": "ValidationError",
"requestId": "905a868d-9331-407b-835d-9c2fd89cd013",
"stackTrace": [
" File "/var/task/lambda_function.py", line 230, in main\n cert_path, private_key_path = request_certificate_from_letsencrypt(csr, private_key)\n",
" File "/var/task/lambda_function.py", line 204, in request_certificate_from_letsencrypt\n certificate = dns01_challenge(orderr, DOMAIN, private_key_jwk, client_acme)\n",
" File "/var/task/lambda_function.py", line 177, in dns01_challenge\n finalized_orderr = client_acme.poll_and_finalize(orderr)\n",
" File "/var/task/acme/client.py", line 184, in poll_and_finalize\n orderr = self.poll_authorizations(orderr, deadline)\n",
" File "/var/task/acme/client.py", line 209, in poll_authorizations\n raise errors.ValidationError(failed)\n"
]
}

Function Logs:
START RequestId: 905a868d-9331-407b-835d-9c2fd89cd013 Version: $LATEST
[ERROR] 2025-02-08T17:40:48.583Z 905a868d-9331-407b-835d-9c2fd89cd013 Error during certificate generation and upload:
LAMBDA_WARNING: Unhandled exception. The most likely cause is an issue in the function code. However, in rare cases, a Lambda runtime update can cause unexpected function behavior. For functions using managed runtimes, runtime updates can be triggered by a function change, or can be applied automatically. To determine if the runtime has been updated, check the runtime version in the INIT_START log entry. If this error correlates with a change in the runtime version, you may be able to mitigate this error by temporarily rolling back to the previous runtime version. For more information, see Understanding how Lambda manages runtime version updates - AWS Lambda
[ERROR] ValidationError
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 230, in main
cert_path, private_key_path = request_certificate_from_letsencrypt(csr, private_key)
File "/var/task/lambda_function.py", line 204, in request_certificate_from_letsencrypt
certificate = dns01_challenge(orderr, DOMAIN, private_key_jwk, client_acme)
File "/var/task/lambda_function.py", line 177, in dns01_challenge
finalized_orderr = client_acme.poll_and_finalize(orderr)
File "/var/task/acme/client.py", line 184, in poll_and_finalize
orderr = self.poll_authorizations(orderr, deadline)
File "/var/task/acme/client.py", line 209, in poll_authorizations
raise errors.ValidationError(failed)
END RequestId: 905a868d-9331-407b-835d-9c2fd89cd013
REPORT RequestId: 905a868d-9331-407b-835d-9c2fd89cd013 Duration: 4548.20 ms Billed Duration: 4549 ms Memory Size: 128 MB Max Memory Used: 98 MB Init Duration: 1276.69 ms

Request ID: 905a868d-9331-407b-835d-9c2fd89cd013
validationerror

Hi @Bruce5051 also I have add client_acme.answer_challenge(dns_challenge, response) which is already in there but now this validation error is coming , if you don't mind could you please run through the code once . Thanks for the help

I do see the DNS-01 challenge _acme-challenge.inf.hydra.sophos.com TXT record still present here https://unboundtest.com/m/TXT/_acme-challenge.inf.hydra.sophos.com/2ZCNPCUV

Query results for TXT _acme-challenge.inf.hydra.sophos.com

Response:
;; opcode: QUERY, status: NOERROR, id: 41869
;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1

;; OPT PSEUDOSECTION:
; EDNS: version 0; flags: do; udp: 1232

;; QUESTION SECTION:
;_acme-challenge.inf.hydra.sophos.com.	IN	 TXT

;; ANSWER SECTION:
_acme-challenge.inf.hydra.sophos.com.	0	IN	TXT	"DdcbBMCfdmeSxJFiIs_zTywoNr7B2WTiB7oXht_HF9A"

----- Unbound logs -----
Feb 08 18:20:34 unbound[16343:0] debug: creating udp6 socket ::1 1053

@Bruce5051 what improvement should I do , do you have any suggestion on this ??

No not really; however I use https://certbot.eff.org/ and GitHub - acmesh-official/acme.sh: A pure Unix shell script implementing ACME client protocol ACME Clients, there are other ACME Client Implementations - Let's Encrypt and libraries as well. https://acmeclients.com/

@Bruce5051 I am new to this community so don't have that many contacts , if you can help and also mention this issue somewhere, this could really help me . I want to achieve this through python .

Sorry, at this point in time patience is the best advice I can give. As this is a community of community volunteers.

Kindly wait to see if there are more knowledgeable Let's Encrypt community volunteers willing to assist.

Edit

Some clients to look at for examples https://letsencrypt.org/docs/client-options/#clients-python and libraries https://letsencrypt.org/docs/client-options/#libraries-python

1 Like

Hmm , totally understandable thanks @Bruce5051

1 Like

There is not much resource with wildcard and dns01 challenge handling using python , if anyone knows then pls comment on this .

Hi @rmbolger , I saw few or your solution on this community so I am reaching out to you , could you please provide your insight in validation error issue in the above code .

Most people rely on an existing ACME Client. Certbot is written in python, is open-source, and commonly used.

But, is there a reason you need to develop your own ACME Client? I have moved your thread into the Client Dev section since that seems to be your intent.

2 Likes

yes @MikeMcQ there is a reason , that is why I am looking for a solution .

thanks @MikeMcQ for moving the issue , if you are aware of this python way that I have used , you can provide your inputs on this .

@Nagendra I saw one of your issue where you where facing issue with the certificate fetching , I am facing validation issue so did you faced something like this or how you resolve your issue .

Hi @mcpherrinm I saw one of your comment in community post where to share the code of http-01 challenge handling and account creation and something like this , could you please help me with my code .

People will help you if they can, but spamming people with tags is not acceptable.

2 Likes

Hey Varun, I think the issue might be with storing csr. Instead of writing it to plain txt file, can you try writing to pickle file . This way the original object will be preserved and then read from pickle. Let me know if this works out.

1 Like