package apimate import ( "fmt" "github.com/rollicks-c/apimate/internal/client" "net/http" "strings" ) func WithAllPages(pageParam, pagesHeader string) client.RequestOption { return func(ctx *client.RequestContext) error { ctx.Paging.ConsumeAll = true ctx.Paging.PageParam = pageParam ctx.Paging.PageCountHeader = pagesHeader return nil } } func WithAcceptedErrors(codes ...int) client.RequestOption { checker := func(resp *http.Response) bool { for _, c := range codes { if c == resp.StatusCode { return true } } return false } return func(ctx *client.RequestContext) error { ctx.StatusChecker = checker return nil } } func WithStatusChecker(checker client.StatusChecker) client.RequestOption { return func(ctx *client.RequestContext) error { ctx.StatusChecker = checker return nil } } type Client struct { apiUrl string defaultOptions []client.RequestOption } func New(apiUrl string, defaults ...client.RequestOption) *Client { return &Client{ apiUrl: apiUrl, defaultOptions: defaults, } } func (c Client) Request(method, ep string, options ...client.RequestOption) error { // create context with default options ctx := &client.RequestContext{ ApiUrl: c.apiUrl, Method: method, Endpoint: fmt.Sprintf("%s/%s", strings.TrimSuffix(c.apiUrl, "/"), strings.TrimPrefix(ep, "/")), AutoThrottle: true, AutoRetries: 3, Paging: client.PagingConfig{ConsumeAll: false}, DefaultOptions: c.defaultOptions, } // apply defaults options defaults := []client.RequestOption{ WithDefaultRequest(), WithNullReceiver(), WithAcceptedErrors(), } // apply custom options options = append(defaults, options...) for _, option := range options { if err := option(ctx); err != nil { return err } } // execute runner := client.NewRunner(*ctx) if err := runner.DoRequest(); err != nil { return err } return nil }
package client import ( "encoding/json" "fmt" "io" "net/http" "net/url" ) type AuthentikAuth struct { url string clientID string username string password string } func NewAuthentikAuth(url, clientID, username, password string) *AuthentikAuth { return &AuthentikAuth{ url: url, clientID: clientID, username: username, password: password, } } func (a AuthentikAuth) Authenticate() (string, error) { // gather data endpoint := fmt.Sprintf("%s/application/o/token/", a.url) payload := url.Values{ "grant_type": {"client_credentials"}, "client_id": {a.clientID}, "username": {a.username}, "password": {a.password}, } // send request resp, err := http.PostForm(endpoint, payload) if err != nil { return "", err } defer resp.Body.Close() // read response if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) } body, err := io.ReadAll(resp.Body) if err != nil { return "", err } var data map[string]interface{} if err := json.Unmarshal(body, &data); err != nil { return "", err } accessToken := data["access_token"].(string) return accessToken, nil }
package client import ( "bytes" "encoding/json" "io" ) func ParseArrayList(raw []byte) ([]byte, error) { buf := bytes.NewReader(raw) decoder := json.NewDecoder(buf) var all []map[string]interface{} // Decode JSON arrays one by one for { var page []map[string]interface{} if err := decoder.Decode(&page); err == io.EOF { break } else if err != nil { return nil, err } all = append(all, page...) } // merged, err := json.Marshal(all) if err != nil { return nil, err } return merged, nil }
package client import ( "bytes" "fmt" "net/http" ) type responseProcessor struct { ctx RequestContext } func (rp responseProcessor) process(resp *http.Response, data [][]byte) error { // check status if rp.ctx.StatusChecker(resp) { return nil } if rp.isErrorCode(resp.StatusCode) { httpError := fmt.Errorf("unexpected status code: %d", resp.StatusCode) if data != nil { httpError = fmt.Errorf("unexpected status code: %d - %s", resp.StatusCode, string(data[0])) } return httpError } // process body if err := rp.ctx.Receiver(data); err != nil { return err } return nil } func (rp responseProcessor) isErrorCode(code int) bool { if code < 200 { return true } if code > 299 { return true } return false } func MergePages(pages [][]byte) []byte { var buffer bytes.Buffer for _, b := range pages { buffer.Write(b) } return buffer.Bytes() }
package client import ( "fmt" "io" "net/http" "strconv" "time" ) type RequestRunner struct { ctx RequestContext } type requester func(req *http.Request) (*http.Response, error) func NewRunner(ctx RequestContext) *RequestRunner { return &RequestRunner{ ctx: ctx, } } func (r RequestRunner) DoRequest() error { // apply defaults for _, opt := range r.ctx.DefaultOptions { if err := opt(&r.ctx); err != nil { return err } } // prepare client := &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } exe := func(req *http.Request) (*http.Response, error) { return client.Do(r.ctx.Req) } // runner middleware exe = r.autoThrottle(exe) exe = r.autoRetry(exe) // execute resp, data, err := r.pagedConsume(exe) if err != nil { return err } // process response rp := responseProcessor{ ctx: r.ctx, } if err := rp.process(resp, data); err != nil { return err } return nil } func (r RequestRunner) autoThrottle(rq requester) requester { if !r.ctx.AutoThrottle { return rq } at := func(req *http.Request) (*http.Response, error) { attempts := 3 for { // run request resp, err := rq(req) if err != nil { return nil, err } // check rate limit if resp.StatusCode == http.StatusTooManyRequests { // limit attempts attempts-- if attempts <= 0 { return nil, fmt.Errorf("too many attempts") } // wait and retry after := resp.Header.Get("Retry-After") seconds, err := strconv.Atoi(after) if err != nil { seconds = 1 } else if seconds <= 0 { seconds = 1 } time.Sleep(time.Duration(seconds) * time.Second) continue } // done return resp, nil } } return at } func (r RequestRunner) autoRetry(rq requester) requester { if r.ctx.AutoRetries <= 0 { return rq } mw := func(req *http.Request) (*http.Response, error) { attempts := r.ctx.AutoRetries for { // run request resp, err := rq(req) if err != nil { return nil, err } // check if failed (can be rate limit) if resp.StatusCode == http.StatusBadRequest { // limit attempts attempts-- if attempts <= 0 { err := fmt.Errorf("too many failed attempts for [%s]", req.URL.String()) data, bodyErr := io.ReadAll(resp.Body) if bodyErr == nil { err = fmt.Errorf("%s: %s", err, string(data)) } return nil, err } // wait and retry time.Sleep(time.Duration(1) * time.Second) continue } // done return resp, nil } } return mw } func (r RequestRunner) directConsume(rq requester) (*http.Response, [][]byte, error) { // read response res, err := rq(r.ctx.Req) if err != nil { return nil, nil, err } data, err := io.ReadAll(res.Body) if err != nil { return nil, nil, err } defer res.Body.Close() //pack pack := [][]byte{data} return res, pack, nil } func (r RequestRunner) pagedConsume(rq requester) (*http.Response, [][]byte, error) { // no paging if !r.ctx.Paging.ConsumeAll { return r.directConsume(rq) } // start consuming at first page page := 1 var res *http.Response var combinedData [][]byte for { // set page param values := r.ctx.Req.URL.Query() values.Set(r.ctx.Paging.PageParam, fmt.Sprintf("%d", page)) r.ctx.Req.URL.RawQuery = values.Encode() // read response pageRes, err := rq(r.ctx.Req) if err != nil { return nil, nil, err } body, err := io.ReadAll(pageRes.Body) if err != nil { return nil, nil, err } if err := pageRes.Body.Close(); err != nil { return nil, nil, err } // combine combinedData = append(combinedData, body) res = pageRes // handle paging totalPages, ok, err := r.getPageCount(pageRes) if !ok { break } if err != nil { return nil, nil, err } if page >= totalPages { break } page++ } return res, combinedData, nil } func (r RequestRunner) getPageCount(res *http.Response) (int, bool, error) { exp := res.Header.Get(r.ctx.Paging.PageCountHeader) if exp == "" { return 0, false, nil } pageCount, err := strconv.Atoi(exp) if err != nil { return 0, false, err } return pageCount, true, nil }
package apimate import ( "github.com/rollicks-c/apimate/internal/client" ) func WithBearerAuth(apiToken string) client.RequestOption { return func(ctx *client.RequestContext) error { ctx.Req.Header.Set("Authorization", "Bearer "+apiToken) return nil } } func WithHeaderAuth(headerKey, token string) client.RequestOption { return func(ctx *client.RequestContext) error { ctx.Req.Header.Set(headerKey, token) return nil } } func WithBasicAuth(user, pass string) client.RequestOption { return func(ctx *client.RequestContext) error { ctx.Req.SetBasicAuth(user, pass) return nil } } func WithAuthentikAuth(url, clientID, username, password string) client.RequestOption { return func(ctx *client.RequestContext) error { token, err := client.NewAuthentikAuth(url, clientID, username, password).Authenticate() if err != nil { return err } ctx.Req.Header.Set("Authorization", "Bearer "+token) return nil } }
package apimate import ( "bytes" "encoding/json" "github.com/rollicks-c/apimate/internal/client" "io" "mime/multipart" "net/http" "net/url" "strings" ) func WithDefaultRequest() client.RequestOption { return func(ctx *client.RequestContext) error { req, err := http.NewRequest(ctx.Method, ctx.Endpoint, nil) if err != nil { return err } ctx.Req = req return nil } } func WithPayload(body io.Reader) client.RequestOption { return func(ctx *client.RequestContext) error { req, err := http.NewRequest(ctx.Method, ctx.Endpoint, body) if err != nil { return err } ctx.Req = req return nil } } func WithFormPayload(body io.Reader) client.RequestOption { return func(ctx *client.RequestContext) error { req, err := http.NewRequest(ctx.Method, ctx.Endpoint, body) if err != nil { return err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") ctx.Req = req return nil } } func WithJSONPayload(data any) client.RequestOption { return func(ctx *client.RequestContext) error { raw, err := json.Marshal(data) if err != nil { return err } body := bytes.NewReader(raw) req, err := http.NewRequest(ctx.Method, ctx.Endpoint, body) if err != nil { return err } req.Header.Set("Content-Type", "application/json") ctx.Req = req return nil } } func WithValues(values url.Values) client.RequestOption { return func(ctx *client.RequestContext) error { req, err := http.NewRequest(ctx.Method, ctx.Endpoint, strings.NewReader(values.Encode())) if err != nil { return err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") ctx.Req = req return nil } } func WithQuery(values url.Values) client.RequestOption { return func(ctx *client.RequestContext) error { // parse url parsedURL, err := url.Parse(ctx.Endpoint) if err != nil { return err } // add values parsedURL.RawQuery = values.Encode() ctx.Endpoint = parsedURL.String() ctx.Req.URL = parsedURL return nil } } func WithFilePayload(name string, data []byte) client.RequestOption { return func(ctx *client.RequestContext) error { // write file to form field body := &bytes.Buffer{} writer := multipart.NewWriter(body) formFile, err := writer.CreateFormFile("file", name) if err != nil { return err } if _, err = io.Copy(formFile, bytes.NewReader(data)); err != nil { return err } _ = writer.Close() // create request req, err := http.NewRequest(ctx.Method, ctx.Endpoint, body) if err != nil { return err } req.Header.Set("Content-Type", writer.FormDataContentType()) ctx.Req = req return nil } }
package apimate import ( "bytes" "encoding/json" "github.com/rollicks-c/apimate/internal/client" ) func WithJSONReceiver(receiver interface{}) client.RequestOption { return func(ctx *client.RequestContext) error { ctx.Req.Header.Set("Content-Type", "application/json") ctx.Receiver = func(payload [][]byte) error { // empty if len(payload) == 0 { return nil } // paged if len(payload) > 1 { merged := bytes.Join(payload, []byte("\n")) data, err := client.ParseArrayList(merged) if err != nil { return err } payload = [][]byte{data} } // decode if err := json.Unmarshal(payload[0], receiver); err != nil { return err } return nil } return nil } } func WithRawReceiver(receiver *[]byte) client.RequestOption { return func(ctx *client.RequestContext) error { ctx.Receiver = func(payload [][]byte) error { *receiver = client.MergePages(payload) return nil } return nil } } func WithCustomReceiver(receiver client.Receiver) client.RequestOption { return func(ctx *client.RequestContext) error { ctx.Receiver = receiver return nil } } func WithNullReceiver() client.RequestOption { return func(ctx *client.RequestContext) error { ctx.Receiver = func(bytes [][]byte) error { return nil } return nil } }