1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
// Package http defines higher level helpers for the net/http package
package http
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/go-errors/errors"
)
// URLParameters is a type used for the parameters in the URL.
type URLParameters map[string]string
// OptionalParams is a structure that defines the optional parameters that are given when making a HTTP call.
type OptionalParams struct {
Headers http.Header
URLParameters URLParameters
Body url.Values
Timeout time.Duration
}
// ConstructURL creates a URL with the included parameters.
func ConstructURL(baseURL string, params URLParameters) (string, error) {
u, err := url.Parse(baseURL)
if err != nil {
return "", errors.WrapPrefix(err,
fmt.Sprintf("failed to construct url '%s' with parameters: %v", u, params), 0)
}
q := u.Query()
for p, value := range params {
q.Set(p, value)
}
u.RawQuery = q.Encode()
return u.String(), nil
}
// optionalURL ensures that the URL contains the optional parameters
// it returns the url (with parameters if success) and an error indicating success.
func optionalURL(urlStr string, opts *OptionalParams) (string, error) {
if opts == nil {
return urlStr, nil
}
return ConstructURL(urlStr, opts.URLParameters)
}
// optionalHeaders ensures that the HTTP request uses the optional headers if defined.
func optionalHeaders(req *http.Request, opts *OptionalParams) {
// Add headers
if opts != nil && req != nil && opts.Headers != nil {
for k, v := range opts.Headers {
req.Header.Add(k, v[0])
}
}
}
// optionalBodyReader returns a HTTP body reader if there is a body, otherwise nil.
func optionalBodyReader(opts *OptionalParams) io.Reader {
if opts != nil && opts.Body != nil {
return strings.NewReader(opts.Body.Encode())
}
return nil
}
// Client is a wrapper around http.Client with some convenience features
// - A default timeout of 5 seconds
// - A read limiter to prevent servers from sending large amounts of data
// - Checking on http code with custom errors
type Client struct {
// Client is the HTTP Client that sends the request
Client *http.Client
// ReadLimit denotes the maximum amount of bytes that are read in HTTP responses
// This is used to prevent servers from sending huge amounts of data
// A limit of 16MB, although maybe much larger than needed, ensures that we do not run into problems
ReadLimit int64
// Timeout denotes the default timeout for each request
Timeout time.Duration
}
// Returns a HTTP client with some default settings
func NewClient() *Client {
c := &http.Client{}
// ReadLimit denotes the maximum amount of bytes that are read in HTTP responses
// This is used to prevent servers from sending huge amounts of data
// A limit of 16MB, although maybe much larger than needed, ensures that we do not run into problems
// The timeout is 10 seconds by default. We pass it here and not in the http client because we want to do it per request
return &Client{Client: c, ReadLimit: 16 << 20, Timeout: 10 * time.Second}
}
// Get creates a Get request and returns the headers, body and an error.
func (c *Client) Get(url string) (http.Header, []byte, error) {
return c.Do(http.MethodGet, url, nil)
}
// PostWithOpts creates a Post request with optional parameters and returns the headers, body and an error.
func (c *Client) PostWithOpts(url string, opts *OptionalParams) (http.Header, []byte, error) {
return c.Do(http.MethodPost, url, opts)
}
// MethodWithOpts Do send a HTTP request using a method (e.g. GET, POST), an url and optional parameters
// It returns the HTTP headers, the body and an error if there is one.
func (c *Client) Do(method string, urlStr string, opts *OptionalParams) (http.Header, []byte, error) {
// Make sure the url contains all the parameters
// This can return an error,
// it already has the right error, so we don't wrap it further
urlStr, err := optionalURL(urlStr, opts)
if err != nil {
// No further type wrapping is needed here
return nil, nil, err
}
// The timeout is configurable for each request
timeout := c.Timeout
if opts != nil && opts.Timeout.Seconds() > 0 {
timeout = opts.Timeout
}
ctx, cncl := context.WithTimeout(context.Background(), timeout)
defer cncl()
// Create request object with the body reader generated from the optional arguments
req, err := http.NewRequestWithContext(ctx, method, urlStr, optionalBodyReader(opts))
if err != nil {
return nil, nil, errors.WrapPrefix(err,
fmt.Sprintf("failed HTTP request with method %s and url %s", method, urlStr), 0)
}
// Make sure the headers contain all the parameters
optionalHeaders(req, opts)
// Do request
res, err := c.Client.Do(req)
if err != nil {
return nil, nil, errors.WrapPrefix(err,
fmt.Sprintf("failed HTTP request with method %s and url %s", method, urlStr), 0)
}
// Request successful, make sure body is closed at the end
defer func() {
_ = res.Body.Close()
}()
// Return a string
// A max bytes reader is normally used for request bodies with a writer
// However, this is still nice to use because unlike a limitreader, it returns an error if the body is too large
// We use this function without a writer so we pass nil
// We impose a limit because servers could be malicious and send huge amounts of data
r := http.MaxBytesReader(nil, res.Body, c.ReadLimit)
body, err := io.ReadAll(r)
if err != nil {
return res.Header, nil, errors.WrapPrefix(err,
fmt.Sprintf("failed HTTP request with method: %s, url: %s and max bytes size: %v", method, urlStr, c.ReadLimit), 0)
}
if res.StatusCode < 200 || res.StatusCode > 299 {
return res.Header, body, errors.Wrap(&StatusError{URL: urlStr, Body: string(body), Status: res.StatusCode}, 0)
}
// Return the body in bytes and signal the status error if there was one
return res.Header, body, nil
}
// StatusError indicates that we have received a HTTP status error.
type StatusError struct {
URL string
Body string
Status int
}
// Error returns the StatusError as an error string.
func (e *StatusError) Error() string {
return fmt.Sprintf(
"failed obtaining HTTP resource: %s as it gave an unsuccessful status code: %d. Body: %s",
e.URL,
e.Status,
e.Body,
)
}
|
