123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323 |
- #
- #
- # Nim's Runtime Library
- # (c) Copyright 2019 Nim Contributors
- #
- # See the file "copying.txt", included in this
- # distribution, for details about the copyright.
- #
- ## This module implements a simple HTTP client that can be used to retrieve
- ## webpages and other data.
- ##
- ## Retrieving a website
- ## ====================
- ##
- ## This example uses HTTP GET to retrieve
- ## `http://google.com`:
- ##
- ## .. code-block:: Nim
- ## import std/httpclient
- ## var client = newHttpClient()
- ## try:
- ## echo client.getContent("http://google.com")
- ## finally:
- ## client.close()
- ##
- ## The same action can also be performed asynchronously, simply use the
- ## `AsyncHttpClient`:
- ##
- ## .. code-block:: Nim
- ## import std/[asyncdispatch, httpclient]
- ##
- ## proc asyncProc(): Future[string] {.async.} =
- ## var client = newAsyncHttpClient()
- ## try:
- ## return await client.getContent("http://google.com")
- ## finally:
- ## client.close()
- ##
- ## echo waitFor asyncProc()
- ##
- ## The functionality implemented by `HttpClient` and `AsyncHttpClient`
- ## is the same, so you can use whichever one suits you best in the examples
- ## shown here.
- ##
- ## **Note:** You need to run asynchronous examples in an async proc
- ## otherwise you will get an `Undeclared identifier: 'await'` error.
- ##
- ## Using HTTP POST
- ## ===============
- ##
- ## This example demonstrates the usage of the W3 HTML Validator, it
- ## uses `multipart/form-data` as the `Content-Type` to send the HTML to be
- ## validated to the server.
- ##
- ## .. code-block:: Nim
- ## var client = newHttpClient()
- ## var data = newMultipartData()
- ## data["output"] = "soap12"
- ## data["uploaded_file"] = ("test.html", "text/html",
- ## "<html><head></head><body><p>test</p></body></html>")
- ## try:
- ## echo client.postContent("http://validator.w3.org/check", multipart=data)
- ## finally:
- ## client.close()
- ##
- ## To stream files from disk when performing the request, use `addFiles`.
- ##
- ## **Note:** This will allocate a new `Mimetypes` database every time you call
- ## it, you can pass your own via the `mimeDb` parameter to avoid this.
- ##
- ## .. code-block:: Nim
- ## let mimes = newMimetypes()
- ## var client = newHttpClient()
- ## var data = newMultipartData()
- ## data.addFiles({"uploaded_file": "test.html"}, mimeDb = mimes)
- ## try:
- ## echo client.postContent("http://validator.w3.org/check", multipart=data)
- ## finally:
- ## client.close()
- ##
- ## You can also make post requests with custom headers.
- ## This example sets `Content-Type` to `application/json`
- ## and uses a json object for the body
- ##
- ## .. code-block:: Nim
- ## import std/[httpclient, json]
- ##
- ## let client = newHttpClient()
- ## client.headers = newHttpHeaders({ "Content-Type": "application/json" })
- ## let body = %*{
- ## "data": "some text"
- ## }
- ## try:
- ## let response = client.request("http://some.api", httpMethod = HttpPost, body = $body)
- ## echo response.status
- ## finally:
- ## client.close()
- ##
- ## Progress reporting
- ## ==================
- ##
- ## You may specify a callback procedure to be called during an HTTP request.
- ## This callback will be executed every second with information about the
- ## progress of the HTTP request.
- ##
- ## .. code-block:: Nim
- ## import std/[asyncdispatch, httpclient]
- ##
- ## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} =
- ## echo("Downloaded ", progress, " of ", total)
- ## echo("Current rate: ", speed div 1000, "kb/s")
- ##
- ## proc asyncProc() {.async.} =
- ## var client = newAsyncHttpClient()
- ## client.onProgressChanged = onProgressChanged
- ## try:
- ## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test")
- ## finally:
- ## client.close()
- ##
- ## waitFor asyncProc()
- ##
- ## If you would like to remove the callback simply set it to `nil`.
- ##
- ## .. code-block:: Nim
- ## client.onProgressChanged = nil
- ##
- ## .. warning:: The `total` reported by httpclient may be 0 in some cases.
- ##
- ##
- ## SSL/TLS support
- ## ===============
- ## This requires the OpenSSL library. Fortunately it's widely used and installed
- ## on many operating systems. httpclient will use SSL automatically if you give
- ## any of the functions a url with the `https` schema, for example:
- ## `https://github.com/`.
- ##
- ## You will also have to compile with `ssl` defined like so:
- ## `nim c -d:ssl ...`.
- ##
- ## Certificate validation is performed by default.
- ##
- ## A set of directories and files from the `ssl_certs <ssl_certs.html>`_
- ## module are scanned to locate CA certificates.
- ##
- ## Example of setting SSL verification parameters in a new client:
- ##
- ## .. code-block:: Nim
- ## import httpclient
- ## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer))
- ##
- ## There are three options for verify mode:
- ##
- ## * ``CVerifyNone``: certificates are not verified;
- ## * ``CVerifyPeer``: certificates are verified;
- ## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional
- ## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to
- ## locate certificates
- ##
- ## See `newContext <net.html#newContext.string,string,string,string>`_ to tweak or disable certificate validation.
- ##
- ## Timeouts
- ## ========
- ##
- ## Currently only the synchronous functions support a timeout.
- ## The timeout is
- ## measured in milliseconds, once it is set any call on a socket which may
- ## block will be susceptible to this timeout.
- ##
- ## It may be surprising but the
- ## function as a whole can take longer than the specified timeout, only
- ## individual internal calls on the socket are affected. In practice this means
- ## that as long as the server is sending data an exception will not be raised,
- ## if however data does not reach the client within the specified timeout a
- ## `TimeoutError` exception will be raised.
- ##
- ## Here is how to set a timeout when creating an `HttpClient` instance:
- ##
- ## .. code-block:: Nim
- ## import std/httpclient
- ##
- ## let client = newHttpClient(timeout = 42)
- ##
- ## Proxy
- ## =====
- ##
- ## A proxy can be specified as a param to any of the procedures defined in
- ## this module. To do this, use the `newProxy` constructor. Unfortunately,
- ## only basic authentication is supported at the moment.
- ##
- ## Some examples on how to configure a Proxy for `HttpClient`:
- ##
- ## .. code-block:: Nim
- ## import std/httpclient
- ##
- ## let myProxy = newProxy("http://myproxy.network")
- ## let client = newHttpClient(proxy = myProxy)
- ##
- ## Use proxies with basic authentication:
- ##
- ## .. code-block:: Nim
- ## import std/httpclient
- ##
- ## let myProxy = newProxy("http://myproxy.network", auth="user:password")
- ## let client = newHttpClient(proxy = myProxy)
- ##
- ## Get Proxy URL from environment variables:
- ##
- ## .. code-block:: Nim
- ## import std/httpclient
- ##
- ## var url = ""
- ## try:
- ## if existsEnv("http_proxy"):
- ## url = getEnv("http_proxy")
- ## elif existsEnv("https_proxy"):
- ## url = getEnv("https_proxy")
- ## except ValueError:
- ## echo "Unable to parse proxy from environment variables."
- ##
- ## let myProxy = newProxy(url = url)
- ## let client = newHttpClient(proxy = myProxy)
- ##
- ## Redirects
- ## =========
- ##
- ## The maximum redirects can be set with the `maxRedirects` of `int` type,
- ## it specifies the maximum amount of redirects to follow,
- ## it defaults to `5`, you can set it to `0` to disable redirects.
- ##
- ## Here you can see an example about how to set the `maxRedirects` of `HttpClient`:
- ##
- ## .. code-block:: Nim
- ## import std/httpclient
- ##
- ## let client = newHttpClient(maxRedirects = 0)
- ##
- import std/private/since
- import std/[
- net, strutils, uri, parseutils, base64, os, mimetypes,
- math, random, httpcore, times, tables, streams, monotimes,
- asyncnet, asyncdispatch, asyncfile, nativesockets,
- ]
- when defined(nimPreviewSlimSystem):
- import std/[assertions, syncio]
- export httpcore except parseHeader # TODO: The `except` doesn't work
- type
- Response* = ref object
- version*: string
- status*: string
- headers*: HttpHeaders
- body: string
- bodyStream*: Stream
- AsyncResponse* = ref object
- version*: string
- status*: string
- headers*: HttpHeaders
- body: string
- bodyStream*: FutureStream[string]
- proc code*(response: Response | AsyncResponse): HttpCode
- {.raises: [ValueError, OverflowDefect].} =
- ## Retrieves the specified response's `HttpCode`.
- ##
- ## Raises a `ValueError` if the response's `status` does not have a
- ## corresponding `HttpCode`.
- return response.status[0 .. 2].parseInt.HttpCode
- proc contentType*(response: Response | AsyncResponse): string {.inline.} =
- ## Retrieves the specified response's content type.
- ##
- ## This is effectively the value of the "Content-Type" header.
- response.headers.getOrDefault("content-type")
- proc contentLength*(response: Response | AsyncResponse): int =
- ## Retrieves the specified response's content length.
- ##
- ## This is effectively the value of the "Content-Length" header.
- ##
- ## A `ValueError` exception will be raised if the value is not an integer.
- ## If the Content-Length header is not set in the response, ContentLength is set to the value -1.
- var contentLengthHeader = response.headers.getOrDefault("Content-Length", @["-1"])
- result = contentLengthHeader.parseInt()
- proc lastModified*(response: Response | AsyncResponse): DateTime =
- ## Retrieves the specified response's last modified time.
- ##
- ## This is effectively the value of the "Last-Modified" header.
- ##
- ## Raises a `ValueError` if the parsing fails or the value is not a correctly
- ## formatted time.
- var lastModifiedHeader = response.headers.getOrDefault("last-modified")
- result = parse(lastModifiedHeader, "ddd, dd MMM yyyy HH:mm:ss 'GMT'", utc())
- proc body*(response: Response): string =
- ## Retrieves the specified response's body.
- ##
- ## The response's body stream is read synchronously.
- if response.body.len == 0:
- response.body = response.bodyStream.readAll()
- return response.body
- proc body*(response: AsyncResponse): Future[string] {.async.} =
- ## Reads the response's body and caches it. The read is performed only
- ## once.
- if response.body.len == 0:
- response.body = await readAll(response.bodyStream)
- return response.body
- type
- Proxy* = ref object
- url*: Uri
- auth*: string
- MultipartEntry = object
- name, content: string
- case isFile: bool
- of true:
- filename, contentType: string
- fileSize: int64
- isStream: bool
- else: discard
- MultipartEntries* = openArray[tuple[name, content: string]]
- MultipartData* = ref object
- content: seq[MultipartEntry]
- ProtocolError* = object of IOError ## exception that is raised when server
- ## does not conform to the implemented
- ## protocol
- HttpRequestError* = object of IOError ## Thrown in the `getContent` proc
- ## and `postContent` proc,
- ## when the server returns an error
- const defUserAgent* = "Nim httpclient/" & NimVersion
- proc httpError(msg: string) =
- var e: ref ProtocolError
- new(e)
- e.msg = msg
- raise e
- proc fileError(msg: string) =
- var e: ref IOError
- new(e)
- e.msg = msg
- raise e
- when not defined(ssl):
- type SslContext = ref object
- var defaultSslContext {.threadvar.}: SslContext
- proc getDefaultSSL(): SslContext =
- result = defaultSslContext
- when defined(ssl):
- if result == nil:
- defaultSslContext = newContext(verifyMode = CVerifyPeer)
- result = defaultSslContext
- doAssert result != nil, "failure to initialize the SSL context"
- proc newProxy*(url: string; auth = ""): Proxy =
- ## Constructs a new `TProxy` object.
- result = Proxy(url: parseUri(url), auth: auth)
- proc newProxy*(url: Uri; auth = ""): Proxy =
- ## Constructs a new `TProxy` object.
- result = Proxy(url: url, auth: auth)
- proc newMultipartData*: MultipartData {.inline.} =
- ## Constructs a new `MultipartData` object.
- MultipartData()
- proc `$`*(data: MultipartData): string {.since: (1, 1).} =
- ## convert MultipartData to string so it's human readable when echo
- ## see https://github.com/nim-lang/Nim/issues/11863
- const sep = "-".repeat(30)
- for pos, entry in data.content:
- result.add(sep & center($pos, 3) & sep)
- result.add("\nname=\"" & entry.name & "\"")
- if entry.isFile:
- result.add("; filename=\"" & entry.filename & "\"\n")
- result.add("Content-Type: " & entry.contentType)
- result.add("\n\n" & entry.content & "\n")
- proc add*(p: MultipartData, name, content: string, filename: string = "",
- contentType: string = "", useStream = true) =
- ## Add a value to the multipart data.
- ##
- ## When `useStream` is `false`, the file will be read into memory.
- ##
- ## Raises a `ValueError` exception if
- ## `name`, `filename` or `contentType` contain newline characters.
- if {'\c', '\L'} in name:
- raise newException(ValueError, "name contains a newline character")
- if {'\c', '\L'} in filename:
- raise newException(ValueError, "filename contains a newline character")
- if {'\c', '\L'} in contentType:
- raise newException(ValueError, "contentType contains a newline character")
- var entry = MultipartEntry(
- name: name,
- content: content,
- isFile: filename.len > 0
- )
- if entry.isFile:
- entry.isStream = useStream
- entry.filename = filename
- entry.contentType = contentType
- p.content.add(entry)
- proc add*(p: MultipartData, xs: MultipartEntries): MultipartData
- {.discardable.} =
- ## Add a list of multipart entries to the multipart data `p`. All values are
- ## added without a filename and without a content type.
- ##
- ## .. code-block:: Nim
- ## data.add({"action": "login", "format": "json"})
- for name, content in xs.items:
- p.add(name, content)
- result = p
- proc newMultipartData*(xs: MultipartEntries): MultipartData =
- ## Create a new multipart data object and fill it with the entries `xs`
- ## directly.
- ##
- ## .. code-block:: Nim
- ## var data = newMultipartData({"action": "login", "format": "json"})
- result = MultipartData()
- for entry in xs:
- result.add(entry.name, entry.content)
- proc addFiles*(p: MultipartData, xs: openArray[tuple[name, file: string]],
- mimeDb = newMimetypes(), useStream = true):
- MultipartData {.discardable.} =
- ## Add files to a multipart data object. The files will be streamed from disk
- ## when the request is being made. When `stream` is `false`, the files are
- ## instead read into memory, but beware this is very memory ineffecient even
- ## for small files. The MIME types will automatically be determined.
- ## Raises an `IOError` if the file cannot be opened or reading fails. To
- ## manually specify file content, filename and MIME type, use `[]=` instead.
- ##
- ## .. code-block:: Nim
- ## data.addFiles({"uploaded_file": "public/test.html"})
- for name, file in xs.items:
- var contentType: string
- let (_, fName, ext) = splitFile(file)
- if ext.len > 0:
- contentType = mimeDb.getMimetype(ext[1..ext.high], "")
- let content = if useStream: file else: readFile(file)
- p.add(name, content, fName & ext, contentType, useStream = useStream)
- result = p
- proc `[]=`*(p: MultipartData, name, content: string) {.inline.} =
- ## Add a multipart entry to the multipart data `p`. The value is added
- ## without a filename and without a content type.
- ##
- ## .. code-block:: Nim
- ## data["username"] = "NimUser"
- p.add(name, content)
- proc `[]=`*(p: MultipartData, name: string,
- file: tuple[name, contentType, content: string]) {.inline.} =
- ## Add a file to the multipart data `p`, specifying filename, contentType
- ## and content manually.
- ##
- ## .. code-block:: Nim
- ## data["uploaded_file"] = ("test.html", "text/html",
- ## "<html><head></head><body><p>test</p></body></html>")
- p.add(name, file.content, file.name, file.contentType, useStream = false)
- proc getBoundary(p: MultipartData): string =
- if p == nil or p.content.len == 0: return
- while true:
- result = $rand(int.high)
- for i, entry in p.content:
- if result in entry.content: break
- elif i == p.content.high: return
- proc sendFile(socket: Socket | AsyncSocket,
- entry: MultipartEntry) {.multisync.} =
- const chunkSize = 2^18
- let file =
- when socket is AsyncSocket: openAsync(entry.content)
- else: newFileStream(entry.content, fmRead)
- var buffer: string
- while true:
- buffer =
- when socket is AsyncSocket: (await read(file, chunkSize))
- else: readStr(file, chunkSize)
- if buffer.len == 0: break
- await socket.send(buffer)
- file.close()
- proc getNewLocation(lastURL: Uri, headers: HttpHeaders): Uri =
- let newLocation = headers.getOrDefault"Location"
- if newLocation == "": httpError("location header expected")
- # Relative URLs. (Not part of the spec, but soon will be.)
- let parsedLocation = parseUri(newLocation)
- if parsedLocation.hostname == "" and parsedLocation.path != "":
- result = lastURL
- result.path = parsedLocation.path
- result.query = parsedLocation.query
- result.anchor = parsedLocation.anchor
- else:
- result = parsedLocation
- proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeaders,
- proxy: Proxy): string =
- # GET
- result = $httpMethod
- result.add ' '
- if proxy.isNil or requestUrl.scheme == "https":
- # /path?query
- if not requestUrl.path.startsWith("/"): result.add '/'
- result.add(requestUrl.path)
- if requestUrl.query.len > 0:
- result.add("?" & requestUrl.query)
- else:
- # Remove the 'http://' from the URL for CONNECT requests for TLS connections.
- var modifiedUrl = requestUrl
- if requestUrl.scheme == "https": modifiedUrl.scheme = ""
- result.add($modifiedUrl)
- # HTTP/1.1\c\l
- result.add(" HTTP/1.1" & httpNewLine)
- # Host header.
- if not headers.hasKey("Host"):
- if requestUrl.port == "":
- add(result, "Host: " & requestUrl.hostname & httpNewLine)
- else:
- add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & httpNewLine)
- # Connection header.
- if not headers.hasKey("Connection"):
- add(result, "Connection: Keep-Alive" & httpNewLine)
- # Proxy auth header.
- if not proxy.isNil and proxy.auth != "":
- let auth = base64.encode(proxy.auth)
- add(result, "Proxy-Authorization: Basic " & auth & httpNewLine)
- for key, val in headers:
- add(result, key & ": " & val & httpNewLine)
- add(result, httpNewLine)
- type
- ProgressChangedProc*[ReturnType] =
- proc (total, progress, speed: BiggestInt):
- ReturnType {.closure, gcsafe.}
- HttpClientBase*[SocketType] = ref object
- socket: SocketType
- connected: bool
- currentURL: Uri ## Where we are currently connected.
- headers*: HttpHeaders ## Headers to send in requests.
- maxRedirects: Natural ## Maximum redirects, set to `0` to disable.
- userAgent: string
- timeout*: int ## Only used for blocking HttpClient for now.
- proxy: Proxy
- ## `nil` or the callback to call when request progress changes.
- when SocketType is Socket:
- onProgressChanged*: ProgressChangedProc[void]
- else:
- onProgressChanged*: ProgressChangedProc[Future[void]]
- when defined(ssl):
- sslContext: net.SslContext
- contentTotal: BiggestInt
- contentProgress: BiggestInt
- oneSecondProgress: BiggestInt
- lastProgressReport: MonoTime
- when SocketType is AsyncSocket:
- bodyStream: FutureStream[string]
- parseBodyFut: Future[void]
- else:
- bodyStream: Stream
- getBody: bool ## When `false`, the body is never read in requestAux.
- type
- HttpClient* = HttpClientBase[Socket]
- proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
- sslContext = getDefaultSSL(), proxy: Proxy = nil,
- timeout = -1, headers = newHttpHeaders()): HttpClient =
- ## Creates a new HttpClient instance.
- ##
- ## `userAgent` specifies the user agent that will be used when making
- ## requests.
- ##
- ## `maxRedirects` specifies the maximum amount of redirects to follow,
- ## default is 5.
- ##
- ## `sslContext` specifies the SSL context to use for HTTPS requests.
- ## See `SSL/TLS support <#sslslashtls-support>`_
- ##
- ## `proxy` specifies an HTTP proxy to use for this HTTP client's
- ## connections.
- ##
- ## `timeout` specifies the number of milliseconds to allow before a
- ## `TimeoutError` is raised.
- ##
- ## `headers` specifies the HTTP Headers.
- runnableExamples:
- import std/strutils
- let exampleHtml = newHttpClient().getContent("http://example.com")
- assert "Example Domain" in exampleHtml
- assert "Pizza" notin exampleHtml
- new result
- result.headers = headers
- result.userAgent = userAgent
- result.maxRedirects = maxRedirects
- result.proxy = proxy
- result.timeout = timeout
- result.onProgressChanged = nil
- result.bodyStream = newStringStream()
- result.getBody = true
- when defined(ssl):
- result.sslContext = sslContext
- type
- AsyncHttpClient* = HttpClientBase[AsyncSocket]
- proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5,
- sslContext = getDefaultSSL(), proxy: Proxy = nil,
- headers = newHttpHeaders()): AsyncHttpClient =
- ## Creates a new AsyncHttpClient instance.
- ##
- ## `userAgent` specifies the user agent that will be used when making
- ## requests.
- ##
- ## `maxRedirects` specifies the maximum amount of redirects to follow,
- ## default is 5.
- ##
- ## `sslContext` specifies the SSL context to use for HTTPS requests.
- ##
- ## `proxy` specifies an HTTP proxy to use for this HTTP client's
- ## connections.
- ##
- ## `headers` specifies the HTTP Headers.
- runnableExamples:
- import std/[asyncdispatch, strutils]
- proc asyncProc(): Future[string] {.async.} =
- let client = newAsyncHttpClient()
- result = await client.getContent("http://example.com")
- let exampleHtml = waitFor asyncProc()
- assert "Example Domain" in exampleHtml
- assert "Pizza" notin exampleHtml
-
- new result
- result.headers = headers
- result.userAgent = userAgent
- result.maxRedirects = maxRedirects
- result.proxy = proxy
- result.timeout = -1 # TODO
- result.onProgressChanged = nil
- result.bodyStream = newFutureStream[string]("newAsyncHttpClient")
- result.getBody = true
- when defined(ssl):
- result.sslContext = sslContext
- proc close*(client: HttpClient | AsyncHttpClient) =
- ## Closes any connections held by the HTTP client.
- if client.connected:
- client.socket.close()
- client.connected = false
- proc getSocket*(client: HttpClient): Socket {.inline.} =
- ## Get network socket, useful if you want to find out more details about the connection
- ##
- ## this example shows info about local and remote endpoints
- ##
- ## .. code-block:: Nim
- ## if client.connected:
- ## echo client.getSocket.getLocalAddr
- ## echo client.getSocket.getPeerAddr
- ##
- return client.socket
- proc getSocket*(client: AsyncHttpClient): AsyncSocket {.inline.} =
- return client.socket
- proc reportProgress(client: HttpClient | AsyncHttpClient,
- progress: BiggestInt) {.multisync.} =
- client.contentProgress += progress
- client.oneSecondProgress += progress
- if (getMonoTime() - client.lastProgressReport).inSeconds >= 1:
- if not client.onProgressChanged.isNil:
- await client.onProgressChanged(client.contentTotal,
- client.contentProgress,
- client.oneSecondProgress)
- client.oneSecondProgress = 0
- client.lastProgressReport = getMonoTime()
- proc recvFull(client: HttpClient | AsyncHttpClient, size: int, timeout: int,
- keep: bool): Future[int] {.multisync.} =
- ## Ensures that all the data requested is read and returned.
- var readLen = 0
- while true:
- if size == readLen: break
- let remainingSize = size - readLen
- let sizeToRecv = min(remainingSize, net.BufferSize)
- when client.socket is Socket:
- let data = client.socket.recv(sizeToRecv, timeout)
- else:
- let data = await client.socket.recv(sizeToRecv)
- if data == "":
- client.close()
- break # We've been disconnected.
- readLen.inc(data.len)
- if keep:
- await client.bodyStream.write(data)
- await reportProgress(client, data.len)
- return readLen
- proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void]
- {.multisync.} =
- while true:
- var chunkSize = 0
- var chunkSizeStr = await client.socket.recvLine()
- var i = 0
- if chunkSizeStr == "":
- httpError("Server terminated connection prematurely")
- while i < chunkSizeStr.len:
- case chunkSizeStr[i]
- of '0'..'9':
- chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0'))
- of 'a'..'f':
- chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10)
- of 'A'..'F':
- chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10)
- of ';':
- # http://tools.ietf.org/html/rfc2616#section-3.6.1
- # We don't care about chunk-extensions.
- break
- else:
- httpError("Invalid chunk size: " & chunkSizeStr)
- inc(i)
- if chunkSize <= 0:
- discard await recvFull(client, 2, client.timeout, false) # Skip \c\L
- break
- var bytesRead = await recvFull(client, chunkSize, client.timeout, true)
- if bytesRead != chunkSize:
- httpError("Server terminated connection prematurely")
- bytesRead = await recvFull(client, 2, client.timeout, false) # Skip \c\L
- if bytesRead != 2:
- httpError("Server terminated connection prematurely")
- # Trailer headers will only be sent if the request specifies that we want
- # them: http://tools.ietf.org/html/rfc2616#section-3.6.1
- proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders,
- httpVersion: string): Future[void] {.multisync.} =
- # Reset progress from previous requests.
- client.contentTotal = 0
- client.contentProgress = 0
- client.oneSecondProgress = 0
- client.lastProgressReport = MonoTime()
- when client is AsyncHttpClient:
- assert(not client.bodyStream.finished)
- if headers.getOrDefault"Transfer-Encoding" == "chunked":
- await parseChunks(client)
- else:
- # -REGION- Content-Length
- # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.3
- var contentLengthHeader = headers.getOrDefault"Content-Length"
- if contentLengthHeader != "":
- var length = contentLengthHeader.parseInt()
- client.contentTotal = length
- if length > 0:
- let recvLen = await client.recvFull(length, client.timeout, true)
- if recvLen == 0:
- client.close()
- httpError("Got disconnected while trying to read body.")
- if recvLen != length:
- httpError("Received length doesn't match expected length. Wanted " &
- $length & " got: " & $recvLen)
- else:
- # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.4 TODO
- # -REGION- Connection: Close
- # (http://tools.ietf.org/html/rfc2616#section-4.4) NR.5
- let implicitConnectionClose =
- httpVersion == "1.0" or
- # This doesn't match the HTTP spec, but it fixes issues for non-conforming servers.
- (httpVersion == "1.1" and headers.getOrDefault"Connection" == "")
- if headers.getOrDefault"Connection" == "close" or implicitConnectionClose:
- while true:
- let recvLen = await client.recvFull(4000, client.timeout, true)
- if recvLen != 4000:
- client.close()
- break
- when client is AsyncHttpClient:
- client.bodyStream.complete()
- else:
- client.bodyStream.setPosition(0)
- # If the server will close our connection, then no matter the method of
- # reading the body, we need to close our socket.
- if headers.getOrDefault"Connection" == "close":
- client.close()
- proc parseResponse(client: HttpClient | AsyncHttpClient,
- getBody: bool): Future[Response | AsyncResponse]
- {.multisync.} =
- new result
- var parsedStatus = false
- var linei = 0
- var fullyRead = false
- var line = ""
- result.headers = newHttpHeaders()
- while true:
- linei = 0
- when client is HttpClient:
- line = await client.socket.recvLine(client.timeout)
- else:
- line = await client.socket.recvLine()
- if line == "":
- # We've been disconnected.
- client.close()
- break
- if line == httpNewLine:
- fullyRead = true
- break
- if not parsedStatus:
- # Parse HTTP version info and status code.
- var le = skipIgnoreCase(line, "HTTP/", linei)
- if le <= 0:
- httpError("invalid http version, `" & line & "`")
- inc(linei, le)
- le = skipIgnoreCase(line, "1.1", linei)
- if le > 0: result.version = "1.1"
- else:
- le = skipIgnoreCase(line, "1.0", linei)
- if le <= 0: httpError("unsupported http version")
- result.version = "1.0"
- inc(linei, le)
- # Status code
- linei.inc skipWhitespace(line, linei)
- result.status = line[linei .. ^1]
- parsedStatus = true
- else:
- # Parse headers
- var name = ""
- var le = parseUntil(line, name, ':', linei)
- if le <= 0: httpError("invalid headers")
- inc(linei, le)
- if line[linei] != ':': httpError("invalid headers")
- inc(linei) # Skip :
- result.headers.add(name, line[linei .. ^1].strip())
- if result.headers.len > headerLimit:
- httpError("too many headers")
- if not fullyRead:
- httpError("Connection was closed before full request has been made")
- when client is HttpClient:
- result.bodyStream = newStringStream()
- else:
- result.bodyStream = newFutureStream[string]("parseResponse")
- if getBody and result.code != Http204:
- client.bodyStream = result.bodyStream
- when client is HttpClient:
- parseBody(client, result.headers, result.version)
- else:
- assert(client.parseBodyFut.isNil or client.parseBodyFut.finished)
- # do not wait here for the body request to complete
- client.parseBodyFut = parseBody(client, result.headers, result.version)
- client.parseBodyFut.addCallback do():
- if client.parseBodyFut.failed:
- client.bodyStream.fail(client.parseBodyFut.error)
- else:
- when client is AsyncHttpClient:
- result.bodyStream.complete()
- proc newConnection(client: HttpClient | AsyncHttpClient,
- url: Uri) {.multisync.} =
- if client.currentURL.hostname != url.hostname or
- client.currentURL.scheme != url.scheme or
- client.currentURL.port != url.port or
- (not client.connected):
- # Connect to proxy if specified
- let connectionUrl =
- if client.proxy.isNil: url else: client.proxy.url
- let isSsl = connectionUrl.scheme.toLowerAscii() == "https"
- if isSsl and not defined(ssl):
- raise newException(HttpRequestError,
- "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.")
- if client.connected:
- client.close()
- client.connected = false
- # TODO: I should be able to write 'net.Port' here...
- let port =
- if connectionUrl.port == "":
- if isSsl:
- nativesockets.Port(443)
- else:
- nativesockets.Port(80)
- else: nativesockets.Port(connectionUrl.port.parseInt)
- when client is HttpClient:
- client.socket = await net.dial(connectionUrl.hostname, port)
- elif client is AsyncHttpClient:
- client.socket = await asyncnet.dial(connectionUrl.hostname, port)
- else: {.fatal: "Unsupported client type".}
- when defined(ssl):
- if isSsl:
- try:
- client.sslContext.wrapConnectedSocket(
- client.socket, handshakeAsClient, connectionUrl.hostname)
- except:
- client.socket.close()
- raise getCurrentException()
- # If need to CONNECT through proxy
- if url.scheme == "https" and not client.proxy.isNil:
- when defined(ssl):
- # Pass only host:port for CONNECT
- var connectUrl = initUri()
- connectUrl.hostname = url.hostname
- connectUrl.port = if url.port != "": url.port else: "443"
- let proxyHeaderString = generateHeaders(connectUrl, HttpConnect,
- newHttpHeaders(), client.proxy)
- await client.socket.send(proxyHeaderString)
- let proxyResp = await parseResponse(client, false)
- if not proxyResp.status.startsWith("200"):
- raise newException(HttpRequestError,
- "The proxy server rejected a CONNECT request, " &
- "so a secure connection could not be established.")
- client.sslContext.wrapConnectedSocket(
- client.socket, handshakeAsClient, url.hostname)
- else:
- raise newException(HttpRequestError,
- "SSL support is not available. Cannot connect over SSL. Compile with -d:ssl to enable.")
- # May be connected through proxy but remember actual URL being accessed
- client.currentURL = url
- client.connected = true
- proc readFileSizes(client: HttpClient | AsyncHttpClient,
- multipart: MultipartData) {.multisync.} =
- for entry in multipart.content.mitems():
- if not entry.isFile: continue
- if not entry.isStream:
- entry.fileSize = entry.content.len
- continue
- # TODO: look into making getFileSize work with async
- let fileSize = getFileSize(entry.content)
- entry.fileSize = fileSize
- proc format(entry: MultipartEntry, boundary: string): string =
- result = "--" & boundary & httpNewLine
- result.add("Content-Disposition: form-data; name=\"" & entry.name & "\"")
- if entry.isFile:
- result.add("; filename=\"" & entry.filename & "\"" & httpNewLine)
- result.add("Content-Type: " & entry.contentType & httpNewLine)
- else:
- result.add(httpNewLine & httpNewLine & entry.content)
- proc format(client: HttpClient | AsyncHttpClient,
- multipart: MultipartData): Future[seq[string]] {.multisync.} =
- let bound = getBoundary(multipart)
- client.headers["Content-Type"] = "multipart/form-data; boundary=" & bound
- await client.readFileSizes(multipart)
- var length: int64
- for entry in multipart.content:
- result.add(format(entry, bound) & httpNewLine)
- if entry.isFile:
- length += entry.fileSize + httpNewLine.len
- result.add "--" & bound & "--" & httpNewLine
- for s in result: length += s.len
- client.headers["Content-Length"] = $length
- proc override(fallback, override: HttpHeaders): HttpHeaders =
- # Right-biased map union for `HttpHeaders`
- result = newHttpHeaders()
- # Copy by value
- result.table[] = fallback.table[]
- if override.isNil:
- # Return the copy of fallback so it does not get modified
- return result
- for k, vs in override.table:
- result[k] = vs
- proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri,
- httpMethod: HttpMethod, body = "", headers: HttpHeaders = nil,
- multipart: MultipartData = nil): Future[Response | AsyncResponse]
- {.multisync.} =
- # Helper that actually makes the request. Does not handle redirects.
- if url.scheme == "":
- raise newException(ValueError, "No uri scheme supplied.")
- when client is AsyncHttpClient:
- if not client.parseBodyFut.isNil:
- # let the current operation finish before making another request
- await client.parseBodyFut
- client.parseBodyFut = nil
- await newConnection(client, url)
- var newHeaders: HttpHeaders
- var data: seq[string]
- if multipart != nil and multipart.content.len > 0:
- # `format` modifies `client.headers`, see
- # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979
- data = await client.format(multipart)
- newHeaders = client.headers.override(headers)
- else:
- newHeaders = client.headers.override(headers)
- # Only change headers if they have not been specified already
- if not newHeaders.hasKey("Content-Length"):
- if body.len != 0:
- newHeaders["Content-Length"] = $body.len
- elif httpMethod notin {HttpGet, HttpHead}:
- newHeaders["Content-Length"] = "0"
- if not newHeaders.hasKey("user-agent") and client.userAgent.len > 0:
- newHeaders["User-Agent"] = client.userAgent
- let headerString = generateHeaders(url, httpMethod, newHeaders,
- client.proxy)
- await client.socket.send(headerString)
- if data.len > 0:
- var buffer: string
- for i, entry in multipart.content:
- buffer.add data[i]
- if not entry.isFile: continue
- if buffer.len > 0:
- await client.socket.send(buffer)
- buffer.setLen(0)
- if entry.isStream:
- await client.socket.sendFile(entry)
- else:
- await client.socket.send(entry.content)
- buffer.add httpNewLine
- # send the rest and the last boundary
- await client.socket.send(buffer & data[^1])
- elif body.len > 0:
- await client.socket.send(body)
- let getBody = httpMethod notin {HttpHead, HttpConnect} and
- client.getBody
- result = await parseResponse(client, getBody)
- proc request*(client: HttpClient | AsyncHttpClient, url: Uri | string,
- httpMethod: HttpMethod | string = HttpGet, body = "",
- headers: HttpHeaders = nil,
- multipart: MultipartData = nil): Future[Response | AsyncResponse]
- {.multisync.} =
- ## Connects to the hostname specified by the URL and performs a request
- ## using the custom method string specified by `httpMethod`.
- ##
- ## Connection will be kept alive. Further requests on the same `client` to
- ## the same hostname will not require a new connection to be made. The
- ## connection can be closed by using the `close` procedure.
- ##
- ## This procedure will follow redirects up to a maximum number of redirects
- ## specified in `client.maxRedirects`.
- ##
- ## You need to make sure that the `url` doesn't contain any newline
- ## characters. Failing to do so will raise `AssertionDefect`.
- ##
- ## `headers` are HTTP headers that override the `client.headers` for
- ## this specific request only and will not be persisted.
- ##
- ## **Deprecated since v1.5**: use HttpMethod enum instead; string parameter httpMethod is deprecated
- when url is string:
- doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters")
- let url = parseUri(url)
- when httpMethod is string:
- {.warning:
- "Deprecated since v1.5; use HttpMethod enum instead; string parameter httpMethod is deprecated".}
- let httpMethod = case httpMethod
- of "HEAD":
- HttpHead
- of "GET":
- HttpGet
- of "POST":
- HttpPost
- of "PUT":
- HttpPut
- of "DELETE":
- HttpDelete
- of "TRACE":
- HttpTrace
- of "OPTIONS":
- HttpOptions
- of "CONNECT":
- HttpConnect
- of "PATCH":
- HttpPatch
- else:
- raise newException(ValueError, "Invalid HTTP method name: " & httpMethod)
- result = await client.requestAux(url, httpMethod, body, headers, multipart)
- var lastURL = url
- for i in 1..client.maxRedirects:
- let statusCode = result.code
- if statusCode notin {Http301, Http302, Http303, Http307, Http308}:
- break
- let redirectTo = getNewLocation(lastURL, result.headers)
- var redirectMethod: HttpMethod
- var redirectBody: string
- # For more informations about the redirect methods see:
- # https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
- case statusCode
- of Http301, Http302, Http303:
- # The method is changed to GET unless it is GET or HEAD (RFC2616)
- if httpMethod notin {HttpGet, HttpHead}:
- redirectMethod = HttpGet
- else:
- redirectMethod = httpMethod
- # The body is stripped away
- redirectBody = ""
- # Delete any header value associated with the body
- if not headers.isNil():
- headers.del("Content-Length")
- headers.del("Content-Type")
- headers.del("Transfer-Encoding")
- of Http307, Http308:
- # The method and the body are unchanged
- redirectMethod = httpMethod
- redirectBody = body
- else:
- # Unreachable
- doAssert(false)
- # Check if the redirection is to the same domain or a sub-domain (foo.com
- # -> sub.foo.com)
- if redirectTo.hostname != lastURL.hostname and
- not redirectTo.hostname.endsWith("." & lastURL.hostname):
- # Perform some cleanup of the header values
- if headers != nil:
- # Delete the Host header
- headers.del("Host")
- # Do not send any sensitive info to a unknown host
- headers.del("Authorization")
- result = await client.requestAux(redirectTo, redirectMethod, redirectBody,
- headers, multipart)
- lastURL = redirectTo
- proc responseContent(resp: Response | AsyncResponse): Future[string] {.multisync.} =
- ## Returns the content of a response as a string.
- ##
- ## A `HttpRequestError` will be raised if the server responds with a
- ## client error (status code 4xx) or a server error (status code 5xx).
- if resp.code.is4xx or resp.code.is5xx:
- raise newException(HttpRequestError, resp.status)
- else:
- return await resp.bodyStream.readAll()
- proc head*(client: HttpClient | AsyncHttpClient,
- url: Uri | string): Future[Response | AsyncResponse] {.multisync.} =
- ## Connects to the hostname specified by the URL and performs a HEAD request.
- ##
- ## This procedure uses httpClient values such as `client.maxRedirects`.
- result = await client.request(url, HttpHead)
- proc get*(client: HttpClient | AsyncHttpClient,
- url: Uri | string): Future[Response | AsyncResponse] {.multisync.} =
- ## Connects to the hostname specified by the URL and performs a GET request.
- ##
- ## This procedure uses httpClient values such as `client.maxRedirects`.
- result = await client.request(url, HttpGet)
- proc getContent*(client: HttpClient | AsyncHttpClient,
- url: Uri | string): Future[string] {.multisync.} =
- ## Connects to the hostname specified by the URL and returns the content of a GET request.
- let resp = await get(client, url)
- return await responseContent(resp)
- proc delete*(client: HttpClient | AsyncHttpClient,
- url: Uri | string): Future[Response | AsyncResponse] {.multisync.} =
- ## Connects to the hostname specified by the URL and performs a DELETE request.
- ## This procedure uses httpClient values such as `client.maxRedirects`.
- result = await client.request(url, HttpDelete)
- proc deleteContent*(client: HttpClient | AsyncHttpClient,
- url: Uri | string): Future[string] {.multisync.} =
- ## Connects to the hostname specified by the URL and returns the content of a DELETE request.
- let resp = await delete(client, url)
- return await responseContent(resp)
- proc post*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
- multipart: MultipartData = nil): Future[Response | AsyncResponse]
- {.multisync.} =
- ## Connects to the hostname specified by the URL and performs a POST request.
- ## This procedure uses httpClient values such as `client.maxRedirects`.
- result = await client.request(url, HttpPost, body, multipart=multipart)
- proc postContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
- multipart: MultipartData = nil): Future[string]
- {.multisync.} =
- ## Connects to the hostname specified by the URL and returns the content of a POST request.
- let resp = await post(client, url, body, multipart)
- return await responseContent(resp)
- proc put*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
- multipart: MultipartData = nil): Future[Response | AsyncResponse]
- {.multisync.} =
- ## Connects to the hostname specified by the URL and performs a PUT request.
- ## This procedure uses httpClient values such as `client.maxRedirects`.
- result = await client.request(url, HttpPut, body, multipart=multipart)
- proc putContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
- multipart: MultipartData = nil): Future[string] {.multisync.} =
- ## Connects to the hostname specified by the URL andreturns the content of a PUT request.
- let resp = await put(client, url, body, multipart)
- return await responseContent(resp)
- proc patch*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
- multipart: MultipartData = nil): Future[Response | AsyncResponse]
- {.multisync.} =
- ## Connects to the hostname specified by the URL and performs a PATCH request.
- ## This procedure uses httpClient values such as `client.maxRedirects`.
- result = await client.request(url, HttpPatch, body, multipart=multipart)
- proc patchContent*(client: HttpClient | AsyncHttpClient, url: Uri | string, body = "",
- multipart: MultipartData = nil): Future[string]
- {.multisync.} =
- ## Connects to the hostname specified by the URL and returns the content of a PATCH request.
- let resp = await patch(client, url, body, multipart)
- return await responseContent(resp)
- proc downloadFile*(client: HttpClient, url: Uri | string, filename: string) =
- ## Downloads `url` and saves it to `filename`.
- client.getBody = false
- defer:
- client.getBody = true
- let resp = client.get(url)
-
- if resp.code.is4xx or resp.code.is5xx:
- raise newException(HttpRequestError, resp.status)
- client.bodyStream = newFileStream(filename, fmWrite)
- if client.bodyStream.isNil:
- fileError("Unable to open file")
- parseBody(client, resp.headers, resp.version)
- client.bodyStream.close()
- proc downloadFileEx(client: AsyncHttpClient,
- url: Uri | string, filename: string): Future[void] {.async.} =
- ## Downloads `url` and saves it to `filename`.
- client.getBody = false
- let resp = await client.get(url)
-
- if resp.code.is4xx or resp.code.is5xx:
- raise newException(HttpRequestError, resp.status)
- client.bodyStream = newFutureStream[string]("downloadFile")
- var file = openAsync(filename, fmWrite)
- defer: file.close()
- # Let `parseBody` write response data into client.bodyStream in the
- # background.
- let parseBodyFut = parseBody(client, resp.headers, resp.version)
- parseBodyFut.addCallback do():
- if parseBodyFut.failed:
- client.bodyStream.fail(parseBodyFut.error)
- # The `writeFromStream` proc will complete once all the data in the
- # `bodyStream` has been written to the file.
- await file.writeFromStream(client.bodyStream)
- proc downloadFile*(client: AsyncHttpClient, url: Uri | string,
- filename: string): Future[void] =
- result = newFuture[void]("downloadFile")
- try:
- result = downloadFileEx(client, url, filename)
- except Exception as exc:
- result.fail(exc)
- finally:
- result.addCallback(
- proc () = client.getBody = true
- )
|