httpclient.nim 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323
  1. #
  2. #
  3. # Nim's Runtime Library
  4. # (c) Copyright 2019 Nim Contributors
  5. #
  6. # See the file "copying.txt", included in this
  7. # distribution, for details about the copyright.
  8. #
  9. ## This module implements a simple HTTP client that can be used to retrieve
  10. ## webpages and other data.
  11. ##
  12. ## Retrieving a website
  13. ## ====================
  14. ##
  15. ## This example uses HTTP GET to retrieve
  16. ## `http://google.com`:
  17. ##
  18. ## .. code-block:: Nim
  19. ## import std/httpclient
  20. ## var client = newHttpClient()
  21. ## try:
  22. ## echo client.getContent("http://google.com")
  23. ## finally:
  24. ## client.close()
  25. ##
  26. ## The same action can also be performed asynchronously, simply use the
  27. ## `AsyncHttpClient`:
  28. ##
  29. ## .. code-block:: Nim
  30. ## import std/[asyncdispatch, httpclient]
  31. ##
  32. ## proc asyncProc(): Future[string] {.async.} =
  33. ## var client = newAsyncHttpClient()
  34. ## try:
  35. ## return await client.getContent("http://google.com")
  36. ## finally:
  37. ## client.close()
  38. ##
  39. ## echo waitFor asyncProc()
  40. ##
  41. ## The functionality implemented by `HttpClient` and `AsyncHttpClient`
  42. ## is the same, so you can use whichever one suits you best in the examples
  43. ## shown here.
  44. ##
  45. ## **Note:** You need to run asynchronous examples in an async proc
  46. ## otherwise you will get an `Undeclared identifier: 'await'` error.
  47. ##
  48. ## Using HTTP POST
  49. ## ===============
  50. ##
  51. ## This example demonstrates the usage of the W3 HTML Validator, it
  52. ## uses `multipart/form-data` as the `Content-Type` to send the HTML to be
  53. ## validated to the server.
  54. ##
  55. ## .. code-block:: Nim
  56. ## var client = newHttpClient()
  57. ## var data = newMultipartData()
  58. ## data["output"] = "soap12"
  59. ## data["uploaded_file"] = ("test.html", "text/html",
  60. ## "<html><head></head><body><p>test</p></body></html>")
  61. ## try:
  62. ## echo client.postContent("http://validator.w3.org/check", multipart=data)
  63. ## finally:
  64. ## client.close()
  65. ##
  66. ## To stream files from disk when performing the request, use `addFiles`.
  67. ##
  68. ## **Note:** This will allocate a new `Mimetypes` database every time you call
  69. ## it, you can pass your own via the `mimeDb` parameter to avoid this.
  70. ##
  71. ## .. code-block:: Nim
  72. ## let mimes = newMimetypes()
  73. ## var client = newHttpClient()
  74. ## var data = newMultipartData()
  75. ## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes)
  76. ## try:
  77. ## echo client.postContent("http://validator.w3.org/check", multipart=data)
  78. ## finally:
  79. ## client.close()
  80. ##
  81. ## You can also make post requests with custom headers.
  82. ## This example sets `Content-Type` to `application/json`
  83. ## and uses a json object for the body
  84. ##
  85. ## .. code-block:: Nim
  86. ## import std/[httpclient, json]
  87. ##
  88. ## let client = newHttpClient()
  89. ## client.headers = newHttpHeaders({ "Content-Type": "application/json" })
  90. ## let body = %*{
  91. ## "data": "some text"
  92. ## }
  93. ## try:
  94. ## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body)
  95. ## echo response.status
  96. ## finally:
  97. ## client.close()
  98. ##
  99. ## Progress reporting
  100. ## ==================
  101. ##
  102. ## You may specify a callback procedure to be called during an HTTP request.
  103. ## This callback will be executed every second with information about the
  104. ## progress of the HTTP request.
  105. ##
  106. ## .. code-block:: Nim
  107. ## import std/[asyncdispatch, httpclient]
  108. ##
  109. ## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} =
  110. ## echo("Downloaded ", progress, " of ", total)
  111. ## echo("Current rate: ", speed div 1000, "kb/s")
  112. ##
  113. ## proc asyncProc() {.async.} =
  114. ## var client = newAsyncHttpClient()
  115. ## client.onProgressChanged = onProgressChanged
  116. ## try:
  117. ## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test")
  118. ## finally:
  119. ## client.close()
  120. ##
  121. ## waitFor asyncProc()
  122. ##
  123. ## If you would like to remove the callback simply set it to `nil`.
  124. ##
  125. ## .. code-block:: Nim
  126. ## client.onProgressChanged = nil
  127. ##
  128. ## .. warning:: The `total` reported by httpclient may be 0 in some cases.
  129. ##
  130. ##
  131. ## SSL/TLS support
  132. ## ===============
  133. ## This requires the OpenSSL library. Fortunately it's widely used and installed
  134. ## on many operating systems. httpclient will use SSL automatically if you give
  135. ## any of the functions a url with the `https` schema, for example:
  136. ## `https://github.com/`.
  137. ##
  138. ## You will also have to compile with `ssl` defined like so:
  139. ## `nim c -d:ssl ...`.
  140. ##
  141. ## Certificate validation is performed by default.
  142. ##
  143. ## A set of directories and files from the `ssl_certs <ssl_certs.html>`_
  144. ## module are scanned to locate CA certificates.
  145. ##
  146. ## Example of setting SSL verification parameters in a new client:
  147. ##
  148. ## .. code-block:: Nim
  149. ## import httpclient
  150. ## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer))
  151. ##
  152. ## There are three options for verify mode:
  153. ##
  154. ## * ``CVerifyNone``: certificates are not verified;
  155. ## * ``CVerifyPeer``: certificates are verified;
  156. ## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional
  157. ## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to
  158. ## locate certificates
  159. ##
  160. ## See `newContext <net.html#newContext.string,string,string,string>`_ to tweak or disable certificate validation.
  161. ##
  162. ## Timeouts
  163. ## ========
  164. ##
  165. ## Currently only the synchronous functions support a timeout.
  166. ## The timeout is
  167. ## measured in milliseconds, once it is set any call on a socket which may
  168. ## block will be susceptible to this timeout.
  169. ##
  170. ## It may be surprising but the
  171. ## function as a whole can take longer than the specified timeout, only
  172. ## individual internal calls on the socket are affected. In practice this means
  173. ## that as long as the server is sending data an exception will not be raised,
  174. ## if however data does not reach the client within the specified timeout a
  175. ## `TimeoutError` exception will be raised.
  176. ##
  177. ## Here is how to set a timeout when creating an `HttpClient` instance:
  178. ##
  179. ## .. code-block:: Nim
  180. ## import std/httpclient
  181. ##
  182. ## let client = newHttpClient(timeout = 42)
  183. ##
  184. ## Proxy
  185. ## =====
  186. ##
  187. ## A proxy can be specified as a param to any of the procedures defined in
  188. ## this module. To do this, use the `newProxy` constructor. Unfortunately,
  189. ## only basic authentication is supported at the moment.
  190. ##
  191. ## Some examples on how to configure a Proxy for `HttpClient`:
  192. ##
  193. ## .. code-block:: Nim
  194. ## import std/httpclient
  195. ##
  196. ## let myProxy = newProxy("http://myproxy.network")
  197. ## let client = newHttpClient(proxy = myProxy)
  198. ##
  199. ## Use proxies with basic authentication:
  200. ##
  201. ## .. code-block:: Nim
  202. ## import std/httpclient
  203. ##
  204. ## let myProxy = newProxy("http://myproxy.network", auth="user:password")
  205. ## let client = newHttpClient(proxy = myProxy)
  206. ##
  207. ## Get Proxy URL from environment variables:
  208. ##
  209. ## .. code-block:: Nim
  210. ## import std/httpclient
  211. ##
  212. ## var url = ""
  213. ## try:
  214. ## if existsEnv("http_proxy"):
  215. ## url = getEnv("http_proxy")
  216. ## elif existsEnv("https_proxy"):
  217. ## url = getEnv("https_proxy")
  218. ## except ValueError:
  219. ## echo "Unable to parse proxy from environment variables."
  220. ##
  221. ## let myProxy = newProxy(url = url)
  222. ## let client = newHttpClient(proxy = myProxy)
  223. ##
  224. ## Redirects
  225. ## =========
  226. ##
  227. ## The maximum redirects can be set with the `maxRedirects` of `int` type,
  228. ## it specifies the maximum amount of redirects to follow,
  229. ## it defaults to `5`, you can set it to `0` to disable redirects.
  230. ##
  231. ## Here you can see an example about how to set the `maxRedirects` of `HttpClient`:
  232. ##
  233. ## .. code-block:: Nim
  234. ## import std/httpclient
  235. ##
  236. ## let client = newHttpClient(maxRedirects = 0)
  237. ##
  238. import std/private/since
  239. import std/[
  240. net, strutils, uri, parseutils, base64, os, mimetypes,
  241. math, random, httpcore, times, tables, streams, monotimes,
  242. asyncnet, asyncdispatch, asyncfile, nativesockets,
  243. ]
  244. when defined(nimPreviewSlimSystem):
  245. import std/[assertions, syncio]
  246. export httpcore except parseHeader # TODO: The `except` doesn't work
  247. type
  248. Response* = ref object
  249. version*: string
  250. status*: string
  251. headers*: HttpHeaders
  252. body: string
  253. bodyStream*: Stream
  254. AsyncResponse* = ref object
  255. version*: string
  256. status*: string
  257. headers*: HttpHeaders
  258. body: string
  259. bodyStream*: FutureStream[string]
  260. proc code*(response: Response | AsyncResponse): HttpCode
  261. {.raises: [ValueError, OverflowDefect].} =
  262. ## Retrieves the specified response's `HttpCode`.
  263. ##
  264. ## Raises a `ValueError` if the response's `status` does not have a
  265. ## corresponding `HttpCode`.
  266. return response.status[0 .. 2].parseInt.HttpCode
  267. proc contentType*(response: Response | AsyncResponse): string {.inline.} =
  268. ## Retrieves the specified response's content type.
  269. ##
  270. ## This is effectively the value of the "Content-Type" header.
  271. response.headers.getOrDefault("content-type")
  272. proc contentLength*(response: Response | AsyncResponse): int =
  273. ## Retrieves the specified response's content length.
  274. ##
  275. ## This is effectively the value of the "Content-Length" header.
  276. ##
  277. ## A `ValueError` exception will be raised if the value is not an integer.
  278. ## If the Content-Length header is not set in the response, ContentLength is set to the value -1.
  279. var contentLengthHeader = response.headers.getOrDefault("Content-Length", @["-1"])
  280. result = contentLengthHeader.parseInt()
  281. proc lastModified*(response: Response | AsyncResponse): DateTime =
  282. ## Retrieves the specified response's last modified time.
  283. ##
  284. ## This is effectively the value of the "Last-Modified" header.
  285. ##
  286. ## Raises a `ValueError` if the parsing fails or the value is not a correctly
  287. ## formatted time.
  288. var lastModifiedHeader = response.headers.getOrDefault("last-modified")
  289. result = parse(lastModifiedHeader, "ddd, dd MMM yyyy HH:mm:ss 'GMT'", utc())
  290. proc body*(response: Response): string =
  291. ## Retrieves the specified response's body.
  292. ##
  293. ## The response's body stream is read synchronously.
  294. if response.body.len == 0:
  295. response.body = response.bodyStream.readAll()
  296. return response.body
  297. proc body*(response: AsyncResponse): Future[string] {.async.} =
  298. ## Reads the response's body and caches it. The read is performed only
  299. ## once.
  300. if response.body.len == 0:
  301. response.body = await readAll(response.bodyStream)
  302. return response.body
  303. type
  304. Proxy* = ref object
  305. url*: Uri
  306. auth*: string
  307. MultipartEntry = object
  308. name, content: string
  309. case isFile: bool
  310. of true:
  311. filename, contentType: string
  312. fileSize: int64
  313. isStream: bool
  314. else: discard
  315. MultipartEntries* = openArray[tuple[name, content: string]]
  316. MultipartData* = ref object
  317. content: seq[MultipartEntry]
  318. ProtocolError* = object of IOError ## exception that is raised when server
  319. ## does not conform to the implemented
  320. ## protocol
  321. HttpRequestError* = object of IOError ## Thrown in the `getContent` proc
  322. ## and `postContent` proc,
  323. ## when the server returns an error
  324. const defUserAgent* = "Nim httpclient/" & NimVersion
  325. proc httpError(msg: string) =
  326. var e: ref ProtocolError
  327. new(e)
  328. e.msg = msg
  329. raise e
  330. proc fileError(msg: string) =
  331. var e: ref IOError
  332. new(e)
  333. e.msg = msg
  334. raise e
  335. when not defined(ssl):
  336. type SslContext = ref object
  337. var defaultSslContext {.threadvar.}: SslContext
  338. proc getDefaultSSL(): SslContext =
  339. result = defaultSslContext
  340. when defined(ssl):
  341. if result == nil:
  342. defaultSslContext = newContext(verifyMode = CVerifyPeer)
  343. result = defaultSslContext
  344. doAssert result != nil, "failure to initialize the SSL context"
  345. proc newProxy*(url: string; auth = ""): Proxy =
  346. ## Constructs a new `TProxy` object.
  347. result = Proxy(url: parseUri(url), auth: auth)
  348. proc newProxy*(url: Uri; auth = ""): Proxy =
  349. ## Constructs a new `TProxy` object.
  350. result = Proxy(url: url, auth: auth)
  351. proc newMultipartData*: MultipartData {.inline.} =
  352. ## Constructs a new `MultipartData` object.
  353. MultipartData()
  354. proc `$`*(data: MultipartData): string {.since: (1, 1).} =
  355. ## convert MultipartData to string so it's human readable when echo
  356. ## see https://github.com/nim-lang/Nim/issues/11863
  357. const sep = "-".repeat(30)
  358. for pos, entry in data.content:
  359. result.add(sep & center($pos, 3) & sep)
  360. result.add("\nname=\"" & entry.name & "\"")
  361. if entry.isFile:
  362. result.add("; filename=\"" & entry.filename & "\"\n")
  363. result.add("Content-Type: " & entry.contentType)
  364. result.add("\n\n" & entry.content & "\n")
  365. proc add*(p: MultipartData, name, content: string, filename: string = "",
  366. contentType: string = "", useStream = true) =
  367. ## Add a value to the multipart data.
  368. ##
  369. ## When `useStream` is `false`, the file will be read into memory.
  370. ##
  371. ## Raises a `ValueError` exception if
  372. ## `name`, `filename` or `contentType` contain newline characters.
  373. if {'\c', '\L'} in name:
  374. raise newException(ValueError, "name contains a newline character")
  375. if {'\c', '\L'} in filename:
  376. raise newException(ValueError, "filename contains a newline character")
  377. if {'\c', '\L'} in contentType:
  378. raise newException(ValueError, "contentType contains a newline character")
  379. var entry = MultipartEntry(
  380. name: name,
  381. content: content,
  382. isFile: filename.len > 0
  383. )
  384. if entry.isFile:
  385. entry.isStream = useStream
  386. entry.filename = filename
  387. entry.contentType = contentType
  388. p.content.add(entry)
  389. proc add*(p: MultipartData, xs: MultipartEntries): MultipartData
  390. {.discardable.} =
  391. ## Add a list of multipart entries to the multipart data `p`. All values are
  392. ## added without a filename and without a content type.
  393. ##
  394. ## .. code-block:: Nim
  395. ## data.add({"action": "login", "format": "json"})
  396. for name, content in xs.items:
  397. p.add(name, content)
  398. result = p
  399. proc newMultipartData*(xs: MultipartEntries): MultipartData =
  400. ## Create a new multipart data object and fill it with the entries `xs`
  401. ## directly.
  402. ##
  403. ## .. code-block:: Nim
  404. ## var data = newMultipartData({"action": "login", "format": "json"})
  405. result = MultipartData()
  406. for entry in xs:
  407. result.add(entry.name, entry.content)
  408. proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]],
  409. mimeDb = newMimetypes(), useStream = true):
  410. MultipartData {.discardable.} =
  411. ## Add files to a multipart data object. The files will be streamed from disk
  412. ## when the request is being made. When `stream` is `false`, the files are
  413. ## instead read into memory, but beware this is very memory ineffecient even
  414. ## for small files. The MIME types will automatically be determined.
  415. ## Raises an `IOError` if the file cannot be opened or reading fails. To
  416. ## manually specify file content, filename and MIME type, use `[]=` instead.
  417. ##
  418. ## .. code-block:: Nim
  419. ## data.addFiles({"uploaded_file": "public/test.html"})
  420. for name, file in xs.items:
  421. var contentType: string
  422. let (_, fName, ext) = splitFile(file)
  423. if ext.len > 0:
  424. contentType = mimeDb.getMimetype(ext[1..ext.high], "")
  425. let content = if useStream: file else: readFile(file)
  426. p.add(name, content, fName & ext, contentType, useStream = useStream)
  427. result = p
  428. proc `[]=`*(p: MultipartData, name, content: string) {.inline.} =
  429. ## Add a multipart entry to the multipart data `p`. The value is added
  430. ## without a filename and without a content type.
  431. ##
  432. ## .. code-block:: Nim
  433. ## data["username"] = "NimUser"
  434. p.add(name, content)
  435. proc `[]=`*(p: MultipartData, name: string,
  436. file: tuple[name, contentType, content: string]) {.inline.} =
  437. ## Add a file to the multipart data `p`, specifying filename, contentType
  438. ## and content manually.
  439. ##
  440. ## .. code-block:: Nim
  441. ## data["uploaded_file"] = ("test.html", "text/html",
  442. ## "<html><head></head><body><p>test</p></body></html>")
  443. p.add(name, file.content, file.name, file.contentType, useStream = false)
  444. proc getBoundary(p: MultipartData): string =
  445. if p == nil or p.content.len == 0: return
  446. while true:
  447. result = $rand(int.high)
  448. for i, entry in p.content:
  449. if result in entry.content: break
  450. elif i == p.content.high: return
  451. proc sendFile(socket: Socket | AsyncSocket,
  452. entry: MultipartEntry) {.multisync.} =
  453. const chunkSize = 2^18
  454. let file =
  455. when socket is AsyncSocket: openAsync(entry.content)
  456. else: newFileStream(entry.content, fmRead)
  457. var buffer: string
  458. while true:
  459. buffer =
  460. when socket is AsyncSocket: (await read(file, chunkSize))
  461. else: readStr(file, chunkSize)
  462. if buffer.len == 0: break
  463. await socket.send(buffer)
  464. file.close()
  465. proc getNewLocation(lastURL: Uri, headers: HttpHeaders): Uri =
  466. let newLocation = headers.getOrDefault"Location"
  467. if newLocation == "": httpError("location header expected")
  468. # Relative URLs. (Not part of the spec, but soon will be.)
  469. let parsedLocation = parseUri(newLocation)
  470. if parsedLocation.hostname == "" and parsedLocation.path != "":
  471. result = lastURL
  472. result.path = parsedLocation.path
  473. result.query = parsedLocation.query
  474. result.anchor = parsedLocation.anchor
  475. else:
  476. result = parsedLocation
  477. proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeaders,
  478. proxy: Proxy): string =
  479. # GET
  480. result = $httpMethod
  481. result.add ' '
  482. if proxy.isNil or requestUrl.scheme == "https":
  483. # /path?query
  484. if not requestUrl.path.startsWith("/"): result.add '/'
  485. result.add(requestUrl.path)
  486. if requestUrl.query.len > 0:
  487. result.add("?" & requestUrl.query)
  488. else:
  489. # Remove the 'http://' from the URL for CONNECT requests for TLS connections.
  490. var modifiedUrl = requestUrl
  491. if requestUrl.scheme == "https": modifiedUrl.scheme = ""
  492. result.add($modifiedUrl)
  493. # HTTP/1.1\c\l
  494. result.add(" HTTP/1.1" & httpNewLine)
  495. # Host header.
  496. if not headers.hasKey("Host"):
  497. if requestUrl.port == "":
  498. add(result, "Host: " & requestUrl.hostname & httpNewLine)
  499. else:
  500. add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & httpNewLine)
  501. # Connection header.
  502. if not headers.hasKey("Connection"):
  503. add(result, "Connection: Keep-Alive" & httpNewLine)
  504. # Proxy auth header.
  505. if not proxy.isNil and proxy.auth != "":
  506. let auth = base64.encode(proxy.auth)
  507. add(result, "Proxy-Authorization: Basic " & auth & httpNewLine)
  508. for key, val in headers:
  509. add(result, key & ": " & val & httpNewLine)
  510. add(result, httpNewLine)
  511. type
  512. ProgressChangedProc*[ReturnType] =
  513. proc (total, progress, speed: BiggestInt):
  514. ReturnType {.closure, gcsafe.}
  515. HttpClientBase*[SocketType] = ref object
  516. socket: SocketType
  517. connected: bool
  518. currentURL: Uri ## Where we are currently connected.
  519. headers*: HttpHeaders ## Headers to send in requests.
  520. maxRedirects: Natural ## Maximum redirects, set to `0` to disable.
  521. userAgent: string
  522. timeout*: int ## Only used for blocking HttpClient for now.
  523. proxy: Proxy
  524. ## `nil` or the callback to call when request progress changes.
  525. when SocketType is Socket:
  526. onProgressChanged*: ProgressChangedProc[void]
  527. else:
  528. onProgressChanged*: ProgressChangedProc[Future[void]]
  529. when defined(ssl):
  530. sslContext: net.SslContext
  531. contentTotal: BiggestInt
  532. contentProgress: BiggestInt
  533. oneSecondProgress: BiggestInt
  534. lastProgressReport: MonoTime
  535. when SocketType is AsyncSocket:
  536. bodyStream: FutureStream[string]
  537. parseBodyFut: Future[void]
  538. else:
  539. bodyStream: Stream
  540. getBody: bool ## When `false`, the body is never read in requestAux.
  541. type
  542. HttpClient* = HttpClientBase[Socket]
  543. proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
  544. sslContext = getDefaultSSL(), proxy: Proxy = nil,
  545. timeout = -1, headers = newHttpHeaders()): HttpClient =
  546. ## Creates a new HttpClient instance.
  547. ##
  548. ## `userAgent` specifies the user agent that will be used when making
  549. ## requests.
  550. ##
  551. ## `maxRedirects` specifies the maximum amount of redirects to follow,
  552. ## default is 5.
  553. ##
  554. ## `sslContext` specifies the SSL context to use for HTTPS requests.
  555. ## See `SSL/TLS support <#sslslashtls-support>`_
  556. ##
  557. ## `proxy` specifies an HTTP proxy to use for this HTTP client's
  558. ## connections.
  559. ##
  560. ## `timeout` specifies the number of milliseconds to allow before a
  561. ## `TimeoutError` is raised.
  562. ##
  563. ## `headers` specifies the HTTP Headers.
  564. runnableExamples:
  565. import std/strutils
  566. let exampleHtml = newHttpClient().getContent("http://example.com")
  567. assert "Example Domain" in exampleHtml
  568. assert "Pizza" notin exampleHtml
  569. new result
  570. result.headers = headers
  571. result.userAgent = userAgent
  572. result.maxRedirects = maxRedirects
  573. result.proxy = proxy
  574. result.timeout = timeout
  575. result.onProgressChanged = nil
  576. result.bodyStream = newStringStream()
  577. result.getBody = true
  578. when defined(ssl):
  579. result.sslContext = sslContext
  580. type
  581. AsyncHttpClient* = HttpClientBase[AsyncSocket]
  582. proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
  583. sslContext = getDefaultSSL(), proxy: Proxy = nil,
  584. headers = newHttpHeaders()): AsyncHttpClient =
  585. ## Creates a new AsyncHttpClient instance.
  586. ##
  587. ## `userAgent` specifies the user agent that will be used when making
  588. ## requests.
  589. ##
  590. ## `maxRedirects` specifies the maximum amount of redirects to follow,
  591. ## default is 5.
  592. ##
  593. ## `sslContext` specifies the SSL context to use for HTTPS requests.
  594. ##
  595. ## `proxy` specifies an HTTP proxy to use for this HTTP client's
  596. ## connections.
  597. ##
  598. ## `headers` specifies the HTTP Headers.
  599. runnableExamples:
  600. import std/[asyncdispatch, strutils]
  601. proc asyncProc(): Future[string] {.async.} =
  602. let client = newAsyncHttpClient()
  603. result = await client.getContent("http://example.com")
  604. let exampleHtml = waitFor asyncProc()
  605. assert "Example Domain" in exampleHtml
  606. assert "Pizza" notin exampleHtml
  607. new result
  608. result.headers = headers
  609. result.userAgent = userAgent
  610. result.maxRedirects = maxRedirects
  611. result.proxy = proxy
  612. result.timeout = -1 # TODO
  613. result.onProgressChanged = nil
  614. result.bodyStream = newFutureStream[string]("newAsyncHttpClient")
  615. result.getBody = true
  616. when defined(ssl):
  617. result.sslContext = sslContext
  618. proc close*(client: HttpClient | AsyncHttpClient) =
  619. ## Closes any connections held by the HTTP client.
  620. if client.connected:
  621. client.socket.close()
  622. client.connected = false
  623. proc getSocket*(client: HttpClient): Socket {.inline.} =
  624. ## Get network socket, useful if you want to find out more details about the connection
  625. ##
  626. ## this example shows info about local and remote endpoints
  627. ##
  628. ## .. code-block:: Nim
  629. ## if client.connected:
  630. ## echo client.getSocket.getLocalAddr
  631. ## echo client.getSocket.getPeerAddr
  632. ##
  633. return client.socket
  634. proc getSocket*(client: AsyncHttpClient): AsyncSocket {.inline.} =
  635. return client.socket
  636. proc reportProgress(client: HttpClient | AsyncHttpClient,
  637. progress: BiggestInt) {.multisync.} =
  638. client.contentProgress += progress
  639. client.oneSecondProgress += progress
  640. if (getMonoTime() - client.lastProgressReport).inSeconds >= 1:
  641. if not client.onProgressChanged.isNil:
  642. await client.onProgressChanged(client.contentTotal,
  643. client.contentProgress,
  644. client.oneSecondProgress)
  645. client.oneSecondProgress = 0
  646. client.lastProgressReport = getMonoTime()
  647. proc recvFull(client: HttpClient | AsyncHttpClient, size: int, timeout: int,
  648. keep: bool): Future[int] {.multisync.} =
  649. ## Ensures that all the data requested is read and returned.
  650. var readLen = 0
  651. while true:
  652. if size == readLen: break
  653. let remainingSize = size - readLen
  654. let sizeToRecv = min(remainingSize, net.BufferSize)
  655. when client.socket is Socket:
  656. let data = client.socket.recv(sizeToRecv, timeout)
  657. else:
  658. let data = await client.socket.recv(sizeToRecv)
  659. if data == "":
  660. client.close()
  661. break # We've been disconnected.
  662. readLen.inc(data.len)
  663. if keep:
  664. await client.bodyStream.write(data)
  665. await reportProgress(client, data.len)
  666. return readLen
  667. proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void]
  668. {.multisync.} =
  669. while true:
  670. var chunkSize = 0
  671. var chunkSizeStr = await client.socket.recvLine()
  672. var i = 0
  673. if chunkSizeStr == "":
  674. httpError("Server terminated connection prematurely")
  675. while i < chunkSizeStr.len:
  676. case chunkSizeStr[i]
  677. of '0'..'9':
  678. chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0'))
  679. of 'a'..'f':
  680. chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10)
  681. of 'A'..'F':
  682. chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10)
  683. of ';':
  684. # http://tools.ietf.org/html/rfc2616#section-3.6.1
  685. # We don't care about chunk-extensions.
  686. break
  687. else:
  688. httpError("Invalid chunk size: " & chunkSizeStr)
  689. inc(i)
  690. if chunkSize <= 0:
  691. discard await recvFull(client, 2, client.timeout, false) # Skip \c\L
  692. break
  693. var bytesRead = await recvFull(client, chunkSize, client.timeout, true)
  694. if bytesRead != chunkSize:
  695. httpError("Server terminated connection prematurely")
  696. bytesRead = await recvFull(client, 2, client.timeout, false) # Skip \c\L
  697. if bytesRead != 2:
  698. httpError("Server terminated connection prematurely")
  699. # Trailer headers will only be sent if the request specifies that we want
  700. # them: http://tools.ietf.org/html/rfc2616#section-3.6.1
  701. proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders,
  702. httpVersion: string): Future[void] {.multisync.} =
  703. # Reset progress from previous requests.
  704. client.contentTotal = 0
  705. client.contentProgress = 0
  706. client.oneSecondProgress = 0
  707. client.lastProgressReport = MonoTime()
  708. when client is AsyncHttpClient:
  709. assert(not client.bodyStream.finished)
  710. if headers.getOrDefault"Transfer-Encoding" == "chunked":
  711. await parseChunks(client)
  712. else:
  713. # -REGION- Content-Length
  714. # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.3
  715. var contentLengthHeader = headers.getOrDefault"Content-Length"
  716. if contentLengthHeader != "":
  717. var length = contentLengthHeader.parseInt()
  718. client.contentTotal = length
  719. if length > 0:
  720. let recvLen = await client.recvFull(length, client.timeout, true)
  721. if recvLen == 0:
  722. client.close()
  723. httpError("Got disconnected while trying to read body.")
  724. if recvLen != length:
  725. httpError("Received length doesn't match expected length. Wanted " &
  726. $length & " got: " & $recvLen)
  727. else:
  728. # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO
  729. # -REGION- Connection: Close
  730. # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5
  731. let implicitConnectionClose =
  732. httpVersion == "1.0" or
  733. # This doesn't match the HTTP spec, but it fixes issues for non-conforming servers.
  734. (httpVersion == "1.1" and headers.getOrDefault"Connection" == "")
  735. if headers.getOrDefault"Connection" == "close" or implicitConnectionClose:
  736. while true:
  737. let recvLen = await client.recvFull(4000, client.timeout, true)
  738. if recvLen != 4000:
  739. client.close()
  740. break
  741. when client is AsyncHttpClient:
  742. client.bodyStream.complete()
  743. else:
  744. client.bodyStream.setPosition(0)
  745. # If the server will close our connection, then no matter the method of
  746. # reading the body, we need to close our socket.
  747. if headers.getOrDefault"Connection" == "close":
  748. client.close()
  749. proc parseResponse(client: HttpClient | AsyncHttpClient,
  750. getBody: bool): Future[Response | AsyncResponse]
  751. {.multisync.} =
  752. new result
  753. var parsedStatus = false
  754. var linei = 0
  755. var fullyRead = false
  756. var line = ""
  757. result.headers = newHttpHeaders()
  758. while true:
  759. linei = 0
  760. when client is HttpClient:
  761. line = await client.socket.recvLine(client.timeout)
  762. else:
  763. line = await client.socket.recvLine()
  764. if line == "":
  765. # We've been disconnected.
  766. client.close()
  767. break
  768. if line == httpNewLine:
  769. fullyRead = true
  770. break
  771. if not parsedStatus:
  772. # Parse HTTP version info and status code.
  773. var le = skipIgnoreCase(line, "HTTP/", linei)
  774. if le <= 0:
  775. httpError("invalid http version, `" & line & "`")
  776. inc(linei, le)
  777. le = skipIgnoreCase(line, "1.1", linei)
  778. if le > 0: result.version = "1.1"
  779. else:
  780. le = skipIgnoreCase(line, "1.0", linei)
  781. if le <= 0: httpError("unsupported http version")
  782. result.version = "1.0"
  783. inc(linei, le)
  784. # Status code
  785. linei.inc skipWhitespace(line, linei)
  786. result.status = line[linei .. ^1]
  787. parsedStatus = true
  788. else:
  789. # Parse headers
  790. var name = ""
  791. var le = parseUntil(line, name, ':', linei)
  792. if le <= 0: httpError("invalid headers")
  793. inc(linei, le)
  794. if line[linei] != ':': httpError("invalid headers")
  795. inc(linei) # Skip :
  796. result.headers.add(name, line[linei .. ^1].strip())
  797. if result.headers.len > headerLimit:
  798. httpError("too many headers")
  799. if not fullyRead:
  800. httpError("Connection was closed before full request has been made")
  801. when client is HttpClient:
  802. result.bodyStream = newStringStream()
  803. else:
  804. result.bodyStream = newFutureStream[string]("parseResponse")
  805. if getBody and result.code != Http204:
  806. client.bodyStream = result.bodyStream
  807. when client is HttpClient:
  808. parseBody(client, result.headers, result.version)
  809. else:
  810. assert(client.parseBodyFut.isNil or client.parseBodyFut.finished)
  811. # do not wait here for the body request to complete
  812. client.parseBodyFut = parseBody(client, result.headers, result.version)
  813. client.parseBodyFut.addCallback do():
  814. if client.parseBodyFut.failed:
  815. client.bodyStream.fail(client.parseBodyFut.error)
  816. else:
  817. when client is AsyncHttpClient:
  818. result.bodyStream.complete()
  819. proc newConnection(client: HttpClient | AsyncHttpClient,
  820. url: Uri) {.multisync.} =
  821. if client.currentURL.hostname != url.hostname or
  822. client.currentURL.scheme != url.scheme or
  823. client.currentURL.port != url.port or
  824. (not client.connected):
  825. # Connect to proxy if specified
  826. let connectionUrl =
  827. if client.proxy.isNil: url else: client.proxy.url
  828. let isSsl = connectionUrl.scheme.toLowerAscii() == "https"
  829. if isSsl and not defined(ssl):
  830. raise newException(HttpRequestError,
  831. "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.")
  832. if client.connected:
  833. client.close()
  834. client.connected = false
  835. # TODO: I should be able to write 'net.Port' here...
  836. let port =
  837. if connectionUrl.port == "":
  838. if isSsl:
  839. nativesockets.Port(443)
  840. else:
  841. nativesockets.Port(80)
  842. else: nativesockets.Port(connectionUrl.port.parseInt)
  843. when client is HttpClient:
  844. client.socket = await net.dial(connectionUrl.hostname, port)
  845. elif client is AsyncHttpClient:
  846. client.socket = await asyncnet.dial(connectionUrl.hostname, port)
  847. else: {.fatal: "Unsupported client type".}
  848. when defined(ssl):
  849. if isSsl:
  850. try:
  851. client.sslContext.wrapConnectedSocket(
  852. client.socket, handshakeAsClient, connectionUrl.hostname)
  853. except:
  854. client.socket.close()
  855. raise getCurrentException()
  856. # If need to CONNECT through proxy
  857. if url.scheme == "https" and not client.proxy.isNil:
  858. when defined(ssl):
  859. # Pass only host:port for CONNECT
  860. var connectUrl = initUri()
  861. connectUrl.hostname = url.hostname
  862. connectUrl.port = if url.port != "": url.port else: "443"
  863. let proxyHeaderString = generateHeaders(connectUrl, HttpConnect,
  864. newHttpHeaders(), client.proxy)
  865. await client.socket.send(proxyHeaderString)
  866. let proxyResp = await parseResponse(client, false)
  867. if not proxyResp.status.startsWith("200"):
  868. raise newException(HttpRequestError,
  869. "The proxy server rejected a CONNECT request, " &
  870. "so a secure connection could not be established.")
  871. client.sslContext.wrapConnectedSocket(
  872. client.socket, handshakeAsClient, url.hostname)
  873. else:
  874. raise newException(HttpRequestError,
  875. "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.")
  876. # May be connected through proxy but remember actual URL being accessed
  877. client.currentURL = url
  878. client.connected = true
  879. proc readFileSizes(client: HttpClient | AsyncHttpClient,
  880. multipart: MultipartData) {.multisync.} =
  881. for entry in multipart.content.mitems():
  882. if not entry.isFile: continue
  883. if not entry.isStream:
  884. entry.fileSize = entry.content.len
  885. continue
  886. # TODO: look into making getFileSize work with async
  887. let fileSize = getFileSize(entry.content)
  888. entry.fileSize = fileSize
  889. proc format(entry: MultipartEntry, boundary: string): string =
  890. result = "--" & boundary & httpNewLine
  891. result.add("Content-Disposition: form-data; name=\"" & entry.name & "\"")
  892. if entry.isFile:
  893. result.add("; filename=\"" & entry.filename & "\"" & httpNewLine)
  894. result.add("Content-Type: " & entry.contentType & httpNewLine)
  895. else:
  896. result.add(httpNewLine & httpNewLine & entry.content)
  897. proc format(client: HttpClient | AsyncHttpClient,
  898. multipart: MultipartData): Future[seq[string]] {.multisync.} =
  899. let bound = getBoundary(multipart)
  900. client.headers["Content-Type"] = "multipart/form-data; boundary=" & bound
  901. await client.readFileSizes(multipart)
  902. var length: int64
  903. for entry in multipart.content:
  904. result.add(format(entry, bound) & httpNewLine)
  905. if entry.isFile:
  906. length += entry.fileSize + httpNewLine.len
  907. result.add "--" & bound & "--" & httpNewLine
  908. for s in result: length += s.len
  909. client.headers["Content-Length"] = $length
  910. proc override(fallback, override: HttpHeaders): HttpHeaders =
  911. # Right-biased map union for `HttpHeaders`
  912. result = newHttpHeaders()
  913. # Copy by value
  914. result.table[] = fallback.table[]
  915. if override.isNil:
  916. # Return the copy of fallback so it does not get modified
  917. return result
  918. for k, vs in override.table:
  919. result[k] = vs
  920. proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri,
  921. httpMethod: HttpMethod, body = "", headers: HttpHeaders = nil,
  922. multipart: MultipartData = nil): Future[Response | AsyncResponse]
  923. {.multisync.} =
  924. # Helper that actually makes the request. Does not handle redirects.
  925. if url.scheme == "":
  926. raise newException(ValueError, "No uri scheme supplied.")
  927. when client is AsyncHttpClient:
  928. if not client.parseBodyFut.isNil:
  929. # let the current operation finish before making another request
  930. await client.parseBodyFut
  931. client.parseBodyFut = nil
  932. await newConnection(client, url)
  933. var newHeaders: HttpHeaders
  934. var data: seq[string]
  935. if multipart != nil and multipart.content.len > 0:
  936. # `format` modifies `client.headers`, see
  937. # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979
  938. data = await client.format(multipart)
  939. newHeaders = client.headers.override(headers)
  940. else:
  941. newHeaders = client.headers.override(headers)
  942. # Only change headers if they have not been specified already
  943. if not newHeaders.hasKey("Content-Length"):
  944. if body.len != 0:
  945. newHeaders["Content-Length"] = $body.len
  946. elif httpMethod notin {HttpGet, HttpHead}:
  947. newHeaders["Content-Length"] = "0"
  948. if not newHeaders.hasKey("user-agent") and client.userAgent.len > 0:
  949. newHeaders["User-Agent"] = client.userAgent
  950. let headerString = generateHeaders(url, httpMethod, newHeaders,
  951. client.proxy)
  952. await client.socket.send(headerString)
  953. if data.len > 0:
  954. var buffer: string
  955. for i, entry in multipart.content:
  956. buffer.add data[i]
  957. if not entry.isFile: continue
  958. if buffer.len > 0:
  959. await client.socket.send(buffer)
  960. buffer.setLen(0)
  961. if entry.isStream:
  962. await client.socket.sendFile(entry)
  963. else:
  964. await client.socket.send(entry.content)
  965. buffer.add httpNewLine
  966. # send the rest and the last boundary
  967. await client.socket.send(buffer & data[^1])
  968. elif body.len > 0:
  969. await client.socket.send(body)
  970. let getBody = httpMethod notin {HttpHead, HttpConnect} and
  971. client.getBody
  972. result = await parseResponse(client, getBody)
  973. proc request*(client: HttpClient | AsyncHttpClient, url: Uri | string,
  974. httpMethod: HttpMethod | string = HttpGet, body = "",
  975. headers: HttpHeaders = nil,
  976. multipart: MultipartData = nil): Future[Response | AsyncResponse]
  977. {.multisync.} =
  978. ## Connects to the hostname specified by the URL and performs a request
  979. ## using the custom method string specified by `httpMethod`.
  980. ##
  981. ## Connection will be kept alive. Further requests on the same `client` to
  982. ## the same hostname will not require a new connection to be made. The
  983. ## connection can be closed by using the `close` procedure.
  984. ##
  985. ## This procedure will follow redirects up to a maximum number of redirects
  986. ## specified in `client.maxRedirects`.
  987. ##
  988. ## You need to make sure that the `url` doesn't contain any newline
  989. ## characters. Failing to do so will raise `AssertionDefect`.
  990. ##
  991. ## `headers` are HTTP headers that override the `client.headers` for
  992. ## this specific request only and will not be persisted.
  993. ##
  994. ## **Deprecated since v1.5**: use HttpMethod enum instead; string parameter httpMethod is deprecated
  995. when url is string:
  996. doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters")
  997. let url = parseUri(url)
  998. when httpMethod is string:
  999. {.warning:
  1000. "Deprecated since v1.5; use HttpMethod enum instead; string parameter httpMethod is deprecated".}
  1001. let httpMethod = case httpMethod
  1002. of "HEAD":
  1003. HttpHead
  1004. of "GET":
  1005. HttpGet
  1006. of "POST":
  1007. HttpPost
  1008. of "PUT":
  1009. HttpPut
  1010. of "DELETE":
  1011. HttpDelete
  1012. of "TRACE":
  1013. HttpTrace
  1014. of "OPTIONS":
  1015. HttpOptions
  1016. of "CONNECT":
  1017. HttpConnect
  1018. of "PATCH":
  1019. HttpPatch
  1020. else:
  1021. raise newException(ValueError, "Invalid HTTP method name: " & httpMethod)
  1022. result = await client.requestAux(url, httpMethod, body, headers, multipart)
  1023. var lastURL = url
  1024. for i in 1..client.maxRedirects:
  1025. let statusCode = result.code
  1026. if statusCode notin {Http301, Http302, Http303, Http307, Http308}:
  1027. break
  1028. let redirectTo = getNewLocation(lastURL, result.headers)
  1029. var redirectMethod: HttpMethod
  1030. var redirectBody: string
  1031. # For more informations about the redirect methods see:
  1032. # https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
  1033. case statusCode
  1034. of Http301, Http302, Http303:
  1035. # The method is changed to GET unless it is GET or HEAD (RFC2616)
  1036. if httpMethod notin {HttpGet, HttpHead}:
  1037. redirectMethod = HttpGet
  1038. else:
  1039. redirectMethod = httpMethod
  1040. # The body is stripped away
  1041. redirectBody = ""
  1042. # Delete any header value associated with the body
  1043. if not headers.isNil():
  1044. headers.del("Content-Length")
  1045. headers.del("Content-Type")
  1046. headers.del("Transfer-Encoding")
  1047. of Http307, Http308:
  1048. # The method and the body are unchanged
  1049. redirectMethod = httpMethod
  1050. redirectBody = body
  1051. else:
  1052. # Unreachable
  1053. doAssert(false)
  1054. # Check if the redirection is to the same domain or a sub-domain (foo.com
  1055. # -> sub.foo.com)
  1056. if redirectTo.hostname != lastURL.hostname and
  1057. not redirectTo.hostname.endsWith("." & lastURL.hostname):
  1058. # Perform some cleanup of the header values
  1059. if headers != nil:
  1060. # Delete the Host header
  1061. headers.del("Host")
  1062. # Do not send any sensitive info to a unknown host
  1063. headers.del("Authorization")
  1064. result = await client.requestAux(redirectTo, redirectMethod, redirectBody,
  1065. headers, multipart)
  1066. lastURL = redirectTo
  1067. proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync.} =
  1068. ## Returns the content of a response as a string.
  1069. ##
  1070. ## A `HttpRequestError` will be raised if the server responds with a
  1071. ## client error (status code 4xx) or a server error (status code 5xx).
  1072. if resp.code.is4xx or resp.code.is5xx:
  1073. raise newException(HttpRequestError, resp.status)
  1074. else:
  1075. return await resp.bodyStream.readAll()
  1076. proc head*(client: HttpClient | AsyncHttpClient,
  1077. url: Uri | string): Future[Response | AsyncResponse] {.multisync.} =
  1078. ## Connects to the hostname specified by the URL and performs a HEAD request.
  1079. ##
  1080. ## This procedure uses httpClient values such as `client.maxRedirects`.
  1081. result = await client.request(url, HttpHead)
  1082. proc get*(client: HttpClient | AsyncHttpClient,
  1083. url: Uri | string): Future[Response | AsyncResponse] {.multisync.} =
  1084. ## Connects to the hostname specified by the URL and performs a GET request.
  1085. ##
  1086. ## This procedure uses httpClient values such as `client.maxRedirects`.
  1087. result = await client.request(url, HttpGet)
  1088. proc getContent*(client: HttpClient | AsyncHttpClient,
  1089. url: Uri | string): Future[string] {.multisync.} =
  1090. ## Connects to the hostname specified by the URL and returns the content of a GET request.
  1091. let resp = await get(client, url)
  1092. return await responseContent(resp)
  1093. proc delete*(client: HttpClient | AsyncHttpClient,
  1094. url: Uri | string): Future[Response | AsyncResponse] {.multisync.} =
  1095. ## Connects to the hostname specified by the URL and performs a DELETE request.
  1096. ## This procedure uses httpClient values such as `client.maxRedirects`.
  1097. result = await client.request(url, HttpDelete)
  1098. proc deleteContent*(client: HttpClient | AsyncHttpClient,
  1099. url: Uri | string): Future[string] {.multisync.} =
  1100. ## Connects to the hostname specified by the URL and returns the content of a DELETE request.
  1101. let resp = await delete(client, url)
  1102. return await responseContent(resp)
  1103. proc post*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
  1104. multipart: MultipartData = nil): Future[Response | AsyncResponse]
  1105. {.multisync.} =
  1106. ## Connects to the hostname specified by the URL and performs a POST request.
  1107. ## This procedure uses httpClient values such as `client.maxRedirects`.
  1108. result = await client.request(url, HttpPost, body, multipart=multipart)
  1109. proc postContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
  1110. multipart: MultipartData = nil): Future[string]
  1111. {.multisync.} =
  1112. ## Connects to the hostname specified by the URL and returns the content of a POST request.
  1113. let resp = await post(client, url, body, multipart)
  1114. return await responseContent(resp)
  1115. proc put*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
  1116. multipart: MultipartData = nil): Future[Response | AsyncResponse]
  1117. {.multisync.} =
  1118. ## Connects to the hostname specified by the URL and performs a PUT request.
  1119. ## This procedure uses httpClient values such as `client.maxRedirects`.
  1120. result = await client.request(url, HttpPut, body, multipart=multipart)
  1121. proc putContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
  1122. multipart: MultipartData = nil): Future[string] {.multisync.} =
  1123. ## Connects to the hostname specified by the URL andreturns the content of a PUT request.
  1124. let resp = await put(client, url, body, multipart)
  1125. return await responseContent(resp)
  1126. proc patch*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
  1127. multipart: MultipartData = nil): Future[Response | AsyncResponse]
  1128. {.multisync.} =
  1129. ## Connects to the hostname specified by the URL and performs a PATCH request.
  1130. ## This procedure uses httpClient values such as `client.maxRedirects`.
  1131. result = await client.request(url, HttpPatch, body, multipart=multipart)
  1132. proc patchContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
  1133. multipart: MultipartData = nil): Future[string]
  1134. {.multisync.} =
  1135. ## Connects to the hostname specified by the URL and returns the content of a PATCH request.
  1136. let resp = await patch(client, url, body, multipart)
  1137. return await responseContent(resp)
  1138. proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) =
  1139. ## Downloads `url` and saves it to `filename`.
  1140. client.getBody = false
  1141. defer:
  1142. client.getBody = true
  1143. let resp = client.get(url)
  1144. if resp.code.is4xx or resp.code.is5xx:
  1145. raise newException(HttpRequestError, resp.status)
  1146. client.bodyStream = newFileStream(filename, fmWrite)
  1147. if client.bodyStream.isNil:
  1148. fileError("Unable to open file")
  1149. parseBody(client, resp.headers, resp.version)
  1150. client.bodyStream.close()
  1151. proc downloadFileEx(client: AsyncHttpClient,
  1152. url: Uri | string, filename: string): Future[void] {.async.} =
  1153. ## Downloads `url` and saves it to `filename`.
  1154. client.getBody = false
  1155. let resp = await client.get(url)
  1156. if resp.code.is4xx or resp.code.is5xx:
  1157. raise newException(HttpRequestError, resp.status)
  1158. client.bodyStream = newFutureStream[string]("downloadFile")
  1159. var file = openAsync(filename, fmWrite)
  1160. defer: file.close()
  1161. # Let `parseBody` write response data into client.bodyStream in the
  1162. # background.
  1163. let parseBodyFut = parseBody(client, resp.headers, resp.version)
  1164. parseBodyFut.addCallback do():
  1165. if parseBodyFut.failed:
  1166. client.bodyStream.fail(parseBodyFut.error)
  1167. # The `writeFromStream` proc will complete once all the data in the
  1168. # `bodyStream` has been written to the file.
  1169. await file.writeFromStream(client.bodyStream)
  1170. proc downloadFile*(client: AsyncHttpClient, url: Uri | string,
  1171. filename: string): Future[void] =
  1172. result = newFuture[void]("downloadFile")
  1173. try:
  1174. result = downloadFileEx(client, url, filename)
  1175. except Exception as exc:
  1176. result.fail(exc)
  1177. finally:
  1178. result.addCallback(
  1179. proc () = client.getBody = true
  1180. )