SERVFAIL only on staging environment (followup)

Help
1

following up on this:

Logs (sorry for the formatting) from the dehydrated client i use:

fatal: [localhost]: FAILED! => {"changed": true, "cmd": ["/usr/bin/dehydrated", "-c"], "delta": "0:00:06.760936", "end": "2019-01-15 11:24:22.586541", "msg": "non-zero return code", "rc": 1, "start": "2019-01-15 11:24:15.825605", "stderr": "ERROR: Challenge is invalid! (returned: invalid) (result: {\n "type": "http-01",\n "status": "invalid",\n "error": {\n "type": "urn:acme:error:dns",\n "detail": "DNS problem: SERVFAIL looking up CAA for iszt.hu",\n "status": 400\n },\n "uri": "https://acme-staging.api.letsencrypt.org/acme/challenge/pAhIkHlEHAS7VXPT1hkmjc6CbYauY5n2poK-EKVy8hg/222737824\",\n "token": "zIjJkw_JiAJXtyigZGlbbNnKJFrosN7HN5hiGsBUqj8",\n "validationRecord": [\n {\n "url": "http://lists2.iszt.hu/.well-known/acme-challenge/zIjJkw_JiAJXtyigZGlbbNnKJFrosN7HN5hiGsBUqj8\",\n "hostname": "lists2.iszt.hu",\n "port": "80",\n "addressesResolved": [\n "2a00:e6a0:2:4000::34"\n ],\n "addressUsed": "2a00:e6a0:2:4000::34"\n }\n ]\n})", "stderr_lines": ["ERROR: Challenge is invalid! (returned: invalid) (result: {", " "type": "http-01",", " "status": "invalid",", " "error": {", " "type": "urn:acme:error:dns",", " "detail": "DNS problem: SERVFAIL looking up CAA for iszt.hu",", " "status": 400", " },", " "uri": "https://acme-staging.api.letsencrypt.org/acme/challenge/pAhIkHlEHAS7VXPT1hkmjc6CbYauY5n2poK-EKVy8hg/222737824\",", " "token": "zIjJkw_JiAJXtyigZGlbbNnKJFrosN7HN5hiGsBUqj8",", " "validationRecord": [", " {", " "url": "http://lists2.iszt.hu/.well-known/acme-challenge/zIjJkw_JiAJXtyigZGlbbNnKJFrosN7HN5hiGsBUqj8\",", " "hostname": "lists2.iszt.hu",", " "port": "80",", " "addressesResolved": [", " "2a00:e6a0:2:4000::34"", " ],", " "addressUsed": "2a00:e6a0:2:4000::34"", " }", " ]", "})"], "stdout": "# INFO: Using main config file /etc/dehydrated/config\n# INFO: Using additional config file /etc/dehydrated/conf.d/staging-ca.sh\n + Creating chain cache directory /var/lib/dehydrated/chains\nProcessing lists2.iszt.hu\n + Creating new directory /var/lib/dehydrated/certs/lists2.iszt.hu ...\n + Signing domains...\n + Generating private key...\n + Generating signing request...\n + Requesting authorization for lists2.iszt.hu...\n + 1 pending challenge(s)\n + Deploying challenge tokens...\n + Responding to challenge for lists2.iszt.hu authorization...\n + Cleaning challenge tokens...\n + Challenge validation has failed :(", "stdout_lines": ["# INFO: Using main config file /etc/dehydrated/config", "# INFO: Using additional config file /etc/dehydrated/conf.d/staging-ca.sh", " + Creating chain cache directory /var/lib/dehydrated/chains", "Processing lists2.iszt.hu", " + Creating new directory /var/lib/dehydrated/certs/lists2.iszt.hu ...", " + Signing domains...", " + Generating private key...", " + Generating signing request...", " + Requesting authorization for lists2.iszt.hu...", " + 1 pending challenge(s)", " + Deploying challenge tokens...", " + Responding to challenge for lists2.iszt.hu authorization...", " + Cleaning challenge tokens...", " + Challenge validation has failed :("]}

apache logs:

root@lists2:~# less /var/log/apache2/access.log
2600:1f16:185:3210:fa10:3caa:9df7:9ce9 - - [15/Jan/2019:11:24:21 +0000] "GET /.well-known/acme-challenge/zIjJkw_JiAJXtyigZGlbbNnKJFrosN7HN5hiGsBUqj8 HTTP/1.1" 200 308 "-" "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)"
2600:3000:2710:300::1e - - [15/Jan/2019:11:24:21 +0000] "GET /.well-known/acme-challenge/zIjJkw_JiAJXtyigZGlbbNnKJFrosN7HN5hiGsBUqj8 HTTP/1.1" 200 308 "-" "Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)"
5.188.210.50 - - [15/Jan/2019:11:50:44 +0000] "GET http://5.188.210.50/echo.php HTTP/1.1" 404 464 "https://www.google.com/" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"

traceroute

root@lists2:~# traceroute 2600:1f16:185:3210:fa10:3caa:9df7:9ce9
traceroute to 2600:1f16:185:3210:fa10:3caa:9df7:9ce9 (2600:1f16:185:3210:fa10:3caa:9df7:9ce9), 30 hops max, 80 byte packets
1 r1.iszt.hu (2a00:e6a0:2:4000::253) 1.034 ms 1.002 ms 0.908 ms
2 2a01:6ee0:0:bb00:0:cc00:2:1 (2a01:6ee0:0:bb00:0:cc00:2:1) 8.933 ms 8.868 ms 8.783 ms
3 2001:7f8:35::6939:1 (2001:7f8:35::6939:1) 0.328 ms 0.255 ms 0.292 ms
4 100ge11-1.core1.vie1.he.net (2001:470:0:365::1) 4.771 ms 4.620 ms 4.633 ms
5 100ge13-1.core1.par2.he.net (2001:470:0:3f4::1) 20.109 ms 20.037 ms 20.028 ms
6 100ge14-1.core1.nyc4.he.net (2001:470:0:33b::1) 90.527 ms 90.527 ms 90.480 ms
7 2001:428:601:e::1 (2001:428:601:e::1) 90.602 ms 90.842 ms 90.790 ms
8 2001:428::205:171:8:67 (2001:428::205:171:8:67) 90.888 ms 90.787 ms 90.719 ms
9 2001:428:4402:10:0:2e:0:2 (2001:428:4402:10:0:2e:0:2) 95.752 ms 97.090 ms 95.794 ms
10 * * *
11 2620:107:4000:ff::32 (2620:107:4000:ff::32) 111.999 ms 2620:107:4000:ff::33 (2620:107:4000:ff::33) 111.931 ms 2620:107:4000:ff::31 (2620:107:4000:ff::31) 112.150 ms
12 * * *
13 * * *
14 * * *
15 2620:107:4000:9::2f (2620:107:4000:9::2f) 111.875 ms 2620:107:4000:9::2e (2620:107:4000:9::2e) 112.085 ms *
16 2620:107:4000:9::34 (2620:107:4000:9::34) 116.730 ms 2620:107:4000:9::37 (2620:107:4000:9::37) 118.379 ms 2620:107:4000:9::3a (2620:107:4000:9::3a) 118.219 ms
17 2620:107:4000:9::3b (2620:107:4000:9::3b) 112.278 ms 2620:107:4000:9::2e (2620:107:4000:9::2e) 112.119 ms 112.051 ms
18 * 2620:107:4000:9::38 (2620:107:4000:9::38) 121.835 ms 2620:107:4000:9::39 (2620:107:4000:9::39) 112.269 ms
19 * 2620:107:4000:9::3b (2620:107:4000:9::3b) 112.246 ms *
20 * * *
21 * * *
22 * * *
23 * * *
24 * * *
25 * * *
26 * * *
27 * * *
28 * * *
29 * * *
30 * * *
root@lists2:~# traceroute 2600:3000:2710:300::1e
traceroute to 2600:3000:2710:300::1e (2600:3000:2710:300::1e), 30 hops max, 80 byte packets
1 r1.iszt.hu (2a00:e6a0:2:4000::253) 2.427 ms 2.341 ms 2.322 ms
2 2a01:6ee0:0:bb00:0:cc00:2:1 (2a01:6ee0:0:bb00:0:cc00:2:1) 0.435 ms 0.472 ms 0.437 ms
3 2001:7f8:35::6939:1 (2001:7f8:35::6939:1) 0.321 ms 0.288 ms 0.233 ms
4 100ge11-1.core1.vie1.he.net (2001:470:0:365::1) 4.977 ms 4.790 ms 4.719 ms
5 100ge13-1.core1.par2.he.net (2001:470:0:3f4::1) 28.244 ms 28.202 ms 28.144 ms
6 100ge14-1.core1.nyc4.he.net (2001:470:0:33b::1) 90.652 ms 90.514 ms 90.664 ms
7 2001:428:601:e::1 (2001:428:601:e::1) 92.402 ms 92.349 ms 92.283 ms
8 2001:428::205:171:200:219 (2001:428::205:171:200:219) 129.735 ms 129.910 ms 129.864 ms
9 2001:428:3801:208::2 (2001:428:3801:208::2) 131.086 ms 131.071 ms 131.016 ms
10 2600:3000:2:300::1 (2600:3000:2:300::1) 130.937 ms 130.876 ms 130.795 ms
11 2600:3000:0:2::7d (2600:3000:0:2::7d) 130.765 ms 131.004 ms 130.937 ms
12 2600:3000:1:230::2 (2600:3000:1:230::2) 141.439 ms 141.389 ms 141.532 ms
13 2600:3000:0:2::416 (2600:3000:0:2::416) 141.479 ms 141.609 ms 141.727 ms
14 2600:3000:3:720::2 (2600:3000:3:720::2) 141.396 ms 141.330 ms 141.265 ms
15 2600:3000:2700:1073::4 (2600:3000:2700:1073::4) 141.025 ms 140.990 ms 140.928 ms
16 * * *
17 * * *
18 * * *
19 * * *
20 * * *
21 * * *
22 * * *
23 * * *
24 * * *
25 * * *
26 * * *
27 * * *
28 * * *
29 * * *
30 * * *
root@lists2:~#

1 Like
2

ping @JamesLE who I believe requested this information

2 Likes
3

Note: this was again a transient issue and worked an hour later (when I got to retry).

4

Thanks so much, @cstamas! Based on this, I was able to track down what I think is a minor problem with our staging environment. Production isn’t affected and we should get staging fixed shortly.

4 Likes
5

Glad to hear that I was able to help and thanks for tracking this down.

3 Likes
6

This topic was automatically closed 30 days after the last reply. New replies are no longer allowed.