-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathcrossrefapi.go
243 lines (218 loc) · 6.1 KB
/
crossrefapi.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
package crossrefapi
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"time"
// Caltech Library Packages
"github.com/caltechlibrary/doitools"
)
type CrossRefClient struct {
AppName string
MailTo string `json:"mailto"`
API string `json:"api"`
RateLimitLimit int `json:"limit"`
RateLimitInterval int `json:"interval"`
LimitCount int `json:"limit_count"`
Status string
StatusCode int
LastRequest time.Time `json:"last_request"`
}
// Object is the general holder of what get back after unmarshaling json
type Object = map[string]interface{}
// Custom JSON decoder so we can treat numbers easier
func JsonDecode(src []byte, obj interface{}) error {
dec := json.NewDecoder(bytes.NewReader(src))
dec.UseNumber()
err := dec.Decode(&obj)
if err != nil && err != io.EOF {
return err
}
return nil
}
// MarshalObject provide a custom json encoder to solve a an issue with
// HTML entities getting converted to UTF-8 code points by json.Marshal()
// in recent versions of go (~= go1.21).
func MarshalObject(obj interface{}, prefix string, indent string) ([]byte, error) {
buf := []byte{}
w := bytes.NewBuffer(buf)
enc := json.NewEncoder(w)
enc.SetEscapeHTML(false)
enc.SetIndent(prefix, indent)
err := enc.Encode(obj)
if err != nil {
return nil, err
}
return w.Bytes(), err
}
// NewCrossRefClient creates a client and makes a request
// and returns the JSON source as a []byte or error if their is
// a problem.
func NewCrossRefClient(appName string, mailTo string) (*CrossRefClient, error) {
if strings.TrimSpace(mailTo) == "" {
return nil, fmt.Errorf("An mailto value is required for politeness")
}
client := new(CrossRefClient)
client.AppName = appName
client.API = `https://api.crossref.org`
client.MailTo = mailTo
return client, nil
}
func (c *CrossRefClient) calcDelay() time.Duration {
if c.RateLimitLimit == 0 {
return time.Duration(0)
}
return time.Duration(int64(math.Ceil(float64(c.RateLimitInterval) / float64(c.RateLimitLimit))))
}
// mergeQueries merges key-value pairs from src into dst, modifying dst in-place,
// appending values for existing keys.
// Returns modified dst struct.
func mergeQueries(dst *url.Values, src url.Values) *url.Values {
for k, vs := range src {
for _, v := range vs {
dst.Add(k, v)
}
}
return dst
}
// getJSON retrieves the path from the CrossRef API maintaining politeness.
// It returns a []byte of JSON source or an error
func (c *CrossRefClient) getJSON(p string, query *url.Values) ([]byte, error) {
var src []byte
u, err := url.Parse(c.API)
if err != nil {
return nil, err
}
q := u.Query()
q.Set("mailto", c.MailTo)
// Add additional query parameters, if provided
if query != nil {
mergeQueries(&q, *query)
}
u.RawQuery = q.Encode()
u.Path = p
client := http.Client{}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Add("User-Agent", fmt.Sprintf("%s, based on crossrefapi/%s (github.com/caltechlibrary/crossrefapi/; mailto: %s), A golang cli based on https://github.com/CrossRef/rest-api-doc", c.AppName, Version, c.MailTo))
// NOTE: Next request can be made based on last request time plus
// the duration suggested by X-Rate-Limit-Interval / X-Rate-Limit-Limit
if delay := c.calcDelay(); delay.Seconds() > 0 {
time.Sleep(delay)
}
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
// Save the response status
c.Status = res.Status
c.StatusCode = res.StatusCode
// Process the body buffer
if c.StatusCode == 200 {
src, err = ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
}
// NOTE: we want to track the current values for any limits
// `X-Rate-Limit-Limit` and `X-Rate-Limit-Interval` as well
// as LastRequest time
if s := res.Header.Get("X-Rate-Limit-Limit"); s != "" {
if i, err := strconv.Atoi(s); err == nil {
c.RateLimitLimit = i
}
} else if c.RateLimitLimit == 0 {
c.RateLimitLimit = 1
}
if s := res.Header.Get("X-Rate-Limit-Interval"); s != "" {
if i, err := strconv.Atoi(strings.TrimSuffix(s, "s")); err == nil {
c.RateLimitInterval = i
}
} else if c.RateLimitInterval == 0 {
c.RateLimitInterval = 1
}
c.LastRequest = time.Now()
return src, nil
}
// TypesJSON return a list of types in JSON source
func (c *CrossRefClient) TypesJSON() ([]byte, error) {
return c.getJSON("types", nil)
}
// Types returns the list of supported types as a Object
func (c *CrossRefClient) Types() (Object, error) {
src, err := c.TypesJSON()
if err != nil {
return nil, err
}
object := make(Object)
err = JsonDecode(src, &object)
if err != nil {
return nil, err
}
return object, nil
}
// WorksJSON return the work JSON source or error for a client and DOI
func (c *CrossRefClient) WorksJSON(doi string) ([]byte, error) {
s, err := doitools.NormalizeDOI(doi)
if err != nil {
return nil, err
}
return c.getJSON(path.Join("works", s), nil)
}
// Works return the Work unmarshaled into a Object (i.e. map[string]interface{})
func (c *CrossRefClient) Works(doi string) (*Works, error) {
src, err := c.WorksJSON(doi)
if err != nil {
return nil, err
}
if len(src) > 0 {
work := &Works{}
err = JsonDecode(src, &work)
if err != nil {
return nil, err
}
return work, nil
}
return nil, nil
}
type WorksQueryResponse WorksResponse[WorksQueryMessage]
type WorksQueryMessage struct {
ItemsPerPage int64 `json:"items-per-page"`
Query struct {
StartIndex int64 `json:"start-index"`
SearchTerms string `json:"search-terms"`
} `json:"query"`
TotalResults int64 `json:"total-results"`
NextCursor string `json:"next-cursor,omitempty"`
Items []Message `json:"items,omitempty"`
}
func (c *CrossRefClient) QueryWorks(query WorksQuery) (*WorksQueryResponse, error) {
q, err := query.Encode()
if err != nil {
return nil, err
}
src, err := c.getJSON("works", &q)
if err != nil {
return nil, err
}
if len(src) > 0 {
work := &WorksQueryResponse{}
err = JsonDecode(src, work)
if err != nil {
return nil, err
}
return work, nil
}
return nil, nil
}