@cpu could we please get your assistance?
I am not able to get to the bottom of a seemingly network-related issue with @luuvuong .
I have written a reproduction program, which is at the bottom of this post.
It seems like large request bodies are not making it to Boulder via Akamai in south east Asia.
Their output to the program is as follows, and I can’t repro on any other networks, and they can send large request bodies to other hosts (cpanel.com, example.org) just fine.
If we send with a small request body (e.g. 1K) it works OK.
[root@webhost-0104 ~]# ./network-test
Trying against https://cpanel.com with 50012 bytes body
[1515405940593163588] Got conn to 208.74.121.58:443
[1515405940593546450] Wrote request
[1515405941274465754] TTFB
[1515405941275096792] Response: 200 OK (24707 bytes): [PCFET0NUWVBFIGh0bWw+CjxodG1sIGxhbmc9ImVuIj4KCiAgPGhlYWQ+CiAgICA8bWV0YSBjaGFyc2V0PSJ1dGYtOCI+CiAgICA8bWV0YSBodHRwLWVxdWl2PSJYLVVBLUNvbQ==]...
Trying against https://example.org with 50012 bytes body
[1515405941862972818] Got conn to 93.184.216.34:443
[1515405941863449239] Wrote request
[1515405942413049915] TTFB
[1515405942413448127] Response: 200 OK (1270 bytes): [PCFkb2N0eXBlIGh0bWw+CjxodG1sPgo8aGVhZD4KICAgIDx0aXRsZT5FeGFtcGxlIERvbWFpbjwvdGl0bGU+CgogICAgPG1ldGEgY2hhcnNldD0idXRmLTgiIC8+CiAgICA8bQ==]...
Trying against https://127.0.0.1:2083 with 50012 bytes body
[1515405942421597395] Got conn to 127.0.0.1:2083
[1515405942421896672] Wrote request
[1515405942462921579] TTFB
[1515405942463802530] Response: 401 Access Denied (42225 bytes): [CjwhRE9DVFlQRSBodG1sPgo8aHRtbCBsYW5nPSJlbiIgZGlyPSJsdHIiPgo8aGVhZD4KICAgIDxtZXRhIGh0dHAtZXF1aXY9IkNvbnRlbnQtVHlwZSIgY29udGVudD0idGV4dA==]...
Trying against https://acme-v01.api.letsencrypt.org/acme/new-cert with 50012 bytes body
[1515405942604677618] Got conn to 104.111.200.137:443
[1515405942604864840] Wrote request
[1515405972463974115] Experienced request error: Post https://acme-v01.api.letsencrypt.org/acme/new-cert: net/http: request canceled (Client.Timeout exceeded while awaiting headers)
Trying against https://acme-staging.api.letsencrypt.org/acme/new-cert with 50012 bytes body
[1515405972590767451] Got conn to 104.111.200.137:443
[1515405972590974514] Wrote request
[1515406002464191275] Experienced request error: Post https://acme-staging.api.letsencrypt.org/acme/new-cert: net/http: request canceled (Client.Timeout exceeded while awaiting headers)
package main
import (
"bytes"
"crypto/tls"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptrace"
"os"
"strconv"
"strings"
"time"
)
func main() {
bufSize, _ := strconv.Atoi(os.Getenv("NETWORK_TEST_SIZE"))
if bufSize == 0 {
bufSize = 500
}
buf, _ := json.Marshal(map[string]string{
"dummy": strings.Repeat("x", bufSize*100),
})
urls := []string{
"https://cpanel.com",
"https://example.org",
"https://127.0.0.1:2083",
"https://acme-v01.api.letsencrypt.org/acme/new-cert",
"https://acme-staging.api.letsencrypt.org/acme/new-cert",
}
for _, u := range urls {
hit(u, buf)
}
}
func hit(u string, payload []byte) {
defer fmt.Println()
cl := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
req, err := http.NewRequest("POST", u, bytes.NewReader(payload))
if err != nil {
panic(err)
}
trace := &httptrace.ClientTrace{
GotConn: func(connInfo httptrace.GotConnInfo) {
fmt.Printf("\t[%d] Got conn to %s\n", time.Now().UnixNano(), connInfo.Conn.RemoteAddr())
},
WroteRequest: func(i httptrace.WroteRequestInfo) {
fmt.Printf("\t[%d] Wrote request\n", time.Now().UnixNano())
},
Got100Continue: func() {
fmt.Printf("\t[%d] Got 100-continue\n", time.Now().UnixNano())
},
GotFirstResponseByte: func() {
fmt.Printf("\t[%d] TTFB\n", time.Now().UnixNano())
},
}
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
fmt.Printf("Trying against %s with %d bytes body\n", u, len(payload))
resp, err := cl.Do(req)
if err != nil {
fmt.Printf("\t[%d] Experienced request error: %v\n", time.Now().UnixNano(), err)
return
}
defer resp.Body.Close()
var buf []byte
if buf, err = ioutil.ReadAll(resp.Body); err != nil {
fmt.Printf("\t[%d] Experienced read error: %v\n", time.Now().UnixNano(), err)
}
fmt.Printf("\t[%d] Response: %s (%d bytes): [%v]...\n", time.Now().UnixNano(),
resp.Status, len(buf), base64.StdEncoding.EncodeToString(buf[:100]))
}
!
Our operations team will be working on this problem and will reply on-thread if there is anything they need from you or if there are any updates to provide.