3 # Client-side HTTP for GET, POST, and HEAD commands.
4 # These routines can be used in untrusted code that uses
5 # the Safesock security policy. These procedures use a
6 # callback interface to avoid using vwait, which is not
7 # defined in the safe base.
9 # See the file "license.terms" for information on usage and
10 # redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES.
12 # RCS: @(#) $Id: http.tcl,v 1.43 2002/10/03 13:34:32 dkf Exp $
14 # Rough version history:
15 # 1.0 Old http_get interface
16 # 2.0 http:: namespace and http::geturl
17 # 2.1 Added callbacks to handle arriving data, and timeouts
18 # 2.2 Added ability to fetch into a channel
19 # 2.3 Added SSL support, and ability to post from a channel
20 # This version also cleans up error cases and eliminates the
21 # "ioerror" status in favor of raising an error
22 # 2.4 Added -binary option to http::geturl and charset element
25 package require
Tcl 8.2
26 # keep this in sync with pkgIndex.tcl
27 # and with the install directories in Makefiles
28 package provide
http 2.4.2
36 -proxyfilter http::ProxyRequired
38 set http(-useragent) "Tcl http client package [package provide http]"
42 variable alphanumeric a-zA-Z0-9
43 for {set i
0} {$i <= 256} {incr i
} {
45 if {![string match
\[$alphanumeric\] $c]} {
46 set formMap
($c) %[format %.2x
$i]
49 # These are handled specially
50 array set formMap
{ " " + \n %0d
%0a
}
59 variable encodings
[string tolower
[encoding names
]]
60 # This can be changed, but iso8859-1 is the RFC standard.
61 variable defaultCharset
"iso8859-1"
63 namespace export geturl config reset wait formatQuery register unregister
64 # Useful, but not exported: data size status code
69 # See documentaion for details.
72 # proto URL protocol prefix, e.g. https
73 # port Default port for protocol
74 # command Command to use to create socket
76 # list of port and command that was registered.
78 proc http::register {proto port command
} {
80 set urlTypes
($proto) [list $port $command]
85 # Unregisters URL protocol handler
88 # proto URL protocol prefix, e.g. https
90 # list of port and command that was unregistered.
92 proc http::unregister {proto
} {
94 if {![info exists urlTypes
($proto)]} {
95 return -code error "unsupported url type \"$proto\""
97 set old
$urlTypes($proto)
98 unset urlTypes
($proto)
104 # See documentaion for details.
107 # args Options parsed by the procedure.
111 proc http::config {args
} {
113 set options [lsort [array names
http -*]]
114 set usage
[join $options ", "]
115 if {[llength $args] == 0} {
117 foreach name
$options {
118 lappend result
$name $http($name)
122 regsub -all -- - $options {} options
123 set pat ^
-([join $options |
])$
124 if {[llength $args] == 1} {
125 set flag
[lindex $args 0]
126 if {[regexp -- $pat $flag]} {
129 return -code error "Unknown option $flag, must be: $usage"
132 foreach {flag value
} $args {
133 if {[regexp -- $pat $flag]} {
134 set http($flag) $value
136 return -code error "Unknown option $flag, must be: $usage"
144 # Clean up the socket and eval close time callbacks
147 # token Connection token.
148 # errormsg (optional) If set, forces status to error.
149 # skipCB (optional) If set, don't call the -command callback. This
150 # is useful when geturl wants to throw an exception instead
151 # of calling the callback. That way, the same error isn't
152 # reported to two places.
157 proc http::Finish { token
{errormsg
""} {skipCB
0}} {
160 global errorInfo errorCode
161 if {[string length
$errormsg] != 0} {
162 set state
(error) [list $errormsg $errorInfo $errorCode]
163 set state
(status
) error
165 catch {close $state(sock
)}
166 catch {after cancel
$state(after)}
167 if {[info exists state
(-command)] && !$skipCB} {
168 if {[catch {eval $state(-command) {$token}} err
]} {
169 if {[string length
$errormsg] == 0} {
170 set state
(error) [list $err $errorInfo $errorCode]
171 set state
(status
) error
174 if {[info exists state
(-command)]} {
175 # Command callback may already have unset our state
176 unset state
(-command)
183 # See documentaion for details.
186 # token Connection token.
192 proc http::reset { token
{why reset
} } {
195 set state
(status
) $why
196 catch {fileevent $state(sock
) readable
{}}
197 catch {fileevent $state(sock
) writable
{}}
199 if {[info exists state
(error)]} {
200 set errorlist
$state(error)
202 eval ::error $errorlist
208 # Establishes a connection to a remote url via http.
211 # url The http URL to goget.
212 # args Option value pairs. Valid options include:
213 # -blocksize, -validate, -headers, -timeout
215 # Returns a token for this connection.
216 # This token is the name of an array that the caller should
217 # unset to garbage collect the state.
219 proc http::geturl { url args
} {
222 variable defaultCharset
224 # Initialize the state variable, an array. We'll return the
225 # name of this array as the token for the transaction.
227 if {![info exists
http(uid
)]} {
230 set token
[namespace current
]::[incr http(uid
)]
235 # Process command options.
244 -type application
/x-www-form-urlencoded
258 set state
(charset
) $defaultCharset
259 set options {-binary -blocksize -channel -command -handler -headers \
260 -progress -query -queryblocksize -querychannel -queryprogress\
261 -validate -timeout -type}
262 set usage
[join $options ", "]
263 regsub -all -- - $options {} options
264 set pat ^
-([join $options |
])$
265 foreach {flag value
} $args {
266 if {[regexp $pat $flag]} {
268 if {[info exists state
($flag)] && \
269 [string is integer
-strict $state($flag)] && \
270 ![string is integer
-strict $value]} {
272 return -code error "Bad value for $flag ($value), must be integer"
274 set state
($flag) $value
277 return -code error "Unknown option $flag, can be: $usage"
281 # Make sure -query and -querychannel aren't both specified
283 set isQueryChannel
[info exists state
(-querychannel)]
284 set isQuery
[info exists state
(-query)]
285 if {$isQuery && $isQueryChannel} {
287 return -code error "Can't combine -query and -querychannel options!"
290 # Validate URL, determine the server host and port, and check proxy case
292 if {![regexp -nocase {^
(([^
:]*)://)?
([^
/:]+)(:([0-9]+))?
(/.
*)?
$} $url \
293 x prefix proto host y port srvurl
]} {
295 return -code error "Unsupported URL: $url"
297 if {[string length
$proto] == 0} {
299 set url
${proto
}://$url
301 if {![info exists urlTypes
($proto)]} {
303 return -code error "Unsupported URL type \"$proto\""
305 set defport
[lindex $urlTypes($proto) 0]
306 set defcmd
[lindex $urlTypes($proto) 1]
308 if {[string length
$port] == 0} {
311 if {[string length
$srvurl] == 0} {
314 if {[string length
$proto] == 0} {
318 if {![catch {$http(-proxyfilter) $host} proxy
]} {
319 set phost
[lindex $proxy 0]
320 set pport
[lindex $proxy 1]
323 # If a timeout is specified we set up the after event
324 # and arrange for an asynchronous socket connection.
326 if {$state(-timeout) > 0} {
327 set state
(after) [after $state(-timeout) \
328 [list http::reset $token timeout
]]
334 # If we are using the proxy, we must pass in the full URL that
335 # includes the server name.
337 if {[info exists phost
] && [string length
$phost]} {
339 set conStat
[catch {eval $defcmd $async {$phost $pport}} s
]
341 set conStat
[catch {eval $defcmd $async {$host $port}} s
]
345 # something went wrong while trying to establish the connection
346 # Clean up after events and such, but DON'T call the command callback
347 # (if available) because we're going to throw an exception from here
351 return -code error $s
355 # Wait for the connection to complete
357 if {$state(-timeout) > 0} {
358 fileevent $s writable
[list http::Connect $token]
361 if {[string equal
$state(status
) "error"]} {
362 # something went wrong while trying to establish the connection
363 # Clean up after events and such, but DON'T call the command
364 # callback (if available) because we're going to throw an
365 # exception from here instead.
366 set err
[lindex $state(error) 0]
368 return -code error $err
369 } elseif
{![string equal
$state(status
) "connect"]} {
370 # Likely to be connection timeout
376 # Send data in cr-lf format, but accept any line terminators
378 fconfigure $s -translation {auto crlf
} -buffersize $state(-blocksize)
380 # The following is disallowed in safe interpreters, but the socket
381 # is already in non-blocking mode in that case.
383 catch {fconfigure $s -blocking off
}
386 set state
(querylength
) [string length
$state(-query)]
387 if {$state(querylength
) > 0} {
391 # there's no query data
395 } elseif
{$state(-validate)} {
397 } elseif
{$isQueryChannel} {
399 # The query channel must be blocking for the async Write to
401 fconfigure $state(-querychannel) -blocking 1 -translation binary
406 puts $s "$how $srvurl HTTP/1.0"
407 puts $s "Accept: $http(-accept)"
408 if {$port == $defport} {
409 # Don't add port in this case, to handle broken servers.
411 puts $s "Host: $host"
413 puts $s "Host: $host:$port"
415 puts $s "User-Agent: $http(-useragent)"
416 foreach {key value
} $state(-headers) {
417 regsub -all \[\n\r\] $value {} value
418 set key
[string trim
$key]
419 if {[string equal
$key "Content-Length"]} {
421 set state
(querylength
) $value
423 if {[string length
$key]} {
424 puts $s "$key: $value"
427 if {$isQueryChannel && $state(querylength
) == 0} {
428 # Try to determine size of data in channel
429 # If we cannot seek, the surrounding catch will trap us
431 set start
[tell $state(-querychannel)]
432 seek $state(-querychannel) 0 end
433 set state
(querylength
) \
434 [expr {[tell $state(-querychannel)] - $start}]
435 seek $state(-querychannel) $start
438 # Flush the request header and set up the fileevent that will
439 # either push the POST data or read the response.
443 # It is possible to have both the read and write fileevents active
444 # at this point. The only scenario it seems to affect is a server
445 # that closes the connection without reading the POST data.
446 # (e.g., early versions TclHttpd in various error cases).
447 # Depending on the platform, the client may or may not be able to
448 # get the response from the server because of the error it will
449 # get trying to write the post data. Having both fileevents active
450 # changes the timing and the behavior, but no two platforms
451 # (among Solaris, Linux, and NT) behave the same, and none
452 # behave all that well in any case. Servers should always read thier
453 # POST data if they expect the client to read their response.
455 if {$isQuery ||
$isQueryChannel} {
456 puts $s "Content-Type: $state(-type)"
458 puts $s "Content-Length: $state(querylength)"
461 fconfigure $s -translation {auto
binary}
462 fileevent $s writable
[list http::Write $token]
466 fileevent $s readable
[list http::Event $token]
469 if {! [info exists state
(-command)]} {
471 # geturl does EVERYTHING asynchronously, so if the user
472 # calls it synchronously, we just do a wait here.
475 if {[string equal
$state(status
) "error"]} {
476 # Something went wrong, so throw the exception, and the
477 # enclosing catch will do cleanup.
478 return -code error [lindex $state(error) 0]
482 # The socket probably was never connected,
483 # or the connection dropped later.
485 # Clean up after events and such, but DON'T call the command callback
486 # (if available) because we're going to throw an exception from here
489 # if state(status) is error, it means someone's already called Finish
490 # to do the above-described clean up.
491 if {[string equal
$state(status
) "error"]} {
495 return -code error $err
501 # Data access functions:
502 # Data - the URL data
503 # Status - the transaction status: ok, reset, eof, timeout
504 # Code - the HTTP transaction code, e.g., 200
505 # Size - the size of the URL data
507 proc http::data {token
} {
512 proc http::status {token
} {
515 return $state(status
)
517 proc http::code {token
} {
522 proc http::ncode {token
} {
525 if {[regexp {[0-9]{3}} $state(http) numeric_code
]} {
531 proc http::size {token
} {
534 return $state(currentsize
)
537 proc http::error {token
} {
540 if {[info exists state
(error)]} {
548 # Garbage collect the state associated with a transaction
551 # token The token returned from http::geturl
554 # unsets the state array
556 proc http::cleanup {token
} {
559 if {[info exists state
]} {
566 # This callback is made when an asyncronous connection completes.
569 # token The token returned from http::geturl
572 # Sets the status of the connection, which unblocks
573 # the waiting geturl call
575 proc http::Connect {token
} {
578 global errorInfo errorCode
579 if {[eof $state(sock
)] ||
580 [string length
[fconfigure $state(sock
) -error]]} {
581 Finish
$token "connect failed [fconfigure $state(sock) -error]" 1
583 set state
(status
) connect
584 fileevent $state(sock
) writable
{}
591 # Write POST query data to the socket
594 # token The token for the connection
597 # Write the socket and handle callbacks.
599 proc http::Write {token
} {
604 # Output a block. Tcl will buffer this if the socket blocks
609 # Catch I/O errors on dead sockets
611 if {[info exists state
(-query)]} {
613 # Chop up large query strings so queryprogress callback
614 # can give smooth feedback
617 [string range
$state(-query) $state(queryoffset
) \
618 [expr {$state(queryoffset
) + $state(-queryblocksize) - 1}]]
619 incr state
(queryoffset
) $state(-queryblocksize)
620 if {$state(queryoffset
) >= $state(querylength
)} {
621 set state
(queryoffset
) $state(querylength
)
626 # Copy blocks from the query channel
628 set outStr
[read $state(-querychannel) $state(-queryblocksize)]
629 puts -nonewline $s $outStr
630 incr state
(queryoffset
) [string length
$outStr]
631 if {[eof $state(-querychannel)]} {
636 # Do not call Finish here, but instead let the read half of
637 # the socket process whatever server reply there is to get.
639 set state
(posterror
) $err
644 fileevent $s writable
{}
645 fileevent $s readable
[list http::Event $token]
648 # Callback to the client after we've completely handled everything
650 if {[string length
$state(-queryprogress)]} {
651 eval $state(-queryprogress) [list $token $state(querylength
)\
658 # Handle input on the socket
661 # token The token returned from http::geturl
664 # Read the socket and handle callbacks.
666 proc http::Event {token
} {
675 if {[string equal
$state(state
) "header"]} {
676 if {[catch {gets $s line
} n
]} {
680 set state
(state
) body
681 if {$state(-binary) ||
![regexp -nocase ^
text $state(type
)] ||
\
682 [regexp gzip|compress
$state(coding
)]} {
683 # Turn off conversions for non-text data
684 fconfigure $s -translation binary
685 if {[info exists state
(-channel)]} {
686 fconfigure $state(-channel) -translation binary
689 # If we are getting text, set the incoming channel's
690 # encoding correctly. iso8859-1 is the RFC default, but
691 # this could be any IANA charset. However, we only know
692 # how to convert what we have encodings for.
693 set idx
[lsearch -exact $encodings \
694 [string tolower
$state(charset
)]]
696 fconfigure $s -encoding [lindex $encodings $idx]
699 if {[info exists state
(-channel)] && \
700 ![info exists state
(-handler)]} {
701 # Initiate a sequence of background fcopies
702 fileevent $s readable
{}
706 if {[regexp -nocase {^content-type
:(.
+)$} $line x type
]} {
707 set state
(type
) [string trim
$type]
708 # grab the optional charset information
709 regexp -nocase {charset
\s
*=\s
*(\S
+)} $type x state
(charset
)
711 if {[regexp -nocase {^content-length
:(.
+)$} $line x length
]} {
712 set state
(totalsize
) [string trim
$length]
714 if {[regexp -nocase {^content-encoding
:(.
+)$} $line x coding
]} {
715 set state
(coding
) [string trim
$coding]
717 if {[regexp -nocase {^
([^
:]+):(.
+)$} $line x key value
]} {
718 lappend state
(meta
) $key [string trim
$value]
719 } elseif
{[regexp ^HTTP
$line]} {
720 set state
(http) $line
725 if {[info exists state
(-handler)]} {
726 set n
[eval $state(-handler) {$s $token}]
728 set block
[read $s $state(-blocksize)]
729 set n
[string length
$block]
731 append state
(body
) $block
735 incr state
(currentsize
) $n
740 if {[info exists state
(-progress)]} {
741 eval $state(-progress) \
742 {$token $state(totalsize
) $state(currentsize
)}
750 # Error handling wrapper around fcopy
753 # s The socket to copy from
754 # token The token returned from http::geturl
757 # This closes the connection upon error
759 proc http::CopyStart {s token
} {
763 fcopy $s $state(-channel) -size $state(-blocksize) -command \
764 [list http::CopyDone $token]
772 # fcopy completion callback
775 # token The token returned from http::geturl
776 # count The amount transfered
781 proc http::CopyDone {token count
{error {}}} {
785 incr state
(currentsize
) $count
786 if {[info exists state
(-progress)]} {
787 eval $state(-progress) {$token $state(totalsize
) $state(currentsize
)}
789 # At this point the token may have been reset
790 if {[string length
$error]} {
792 } elseif
{[catch {eof $s} iseof
] ||
$iseof} {
801 # Handle eof on the socket
804 # token The token returned from http::geturl
807 # Clean up the socket
809 proc http::Eof {token
} {
812 if {[string equal
$state(state
) "header"]} {
814 set state
(status
) eof
824 # See documentaion for details.
827 # token Connection token.
830 # The status after the wait.
832 proc http::wait {token
} {
836 if {![info exists state
(status
)] ||
[string length
$state(status
)] == 0} {
837 # We must wait on the original variable name, not the upvar alias
838 vwait $token\(status
)
841 return $state(status
)
844 # http::formatQuery --
846 # See documentaion for details.
847 # Call http::formatQuery with an even number of arguments, where
848 # the first is a name, the second is a value, the third is another
852 # args A list of name-value pairs.
857 proc http::formatQuery {args
} {
861 append result
$sep [mapReply
$i]
862 if {[string equal
$sep "="]} {
873 # Do x-www-urlencoded character mapping
876 # string The string the needs to be encoded
881 proc http::mapReply {string} {
883 variable alphanumeric
885 # The spec says: "non-alphanumeric characters are replaced by '%HH'"
886 # 1 leave alphanumerics characters alone
887 # 2 Convert every other character to an array lookup
888 # 3 Escape constructs that are "special" to the tcl parser
889 # 4 "subst" the result, doing all the array substitutions
891 regsub -all \[^
$alphanumeric\] $string {$formMap(&)} string
892 regsub -all {[][{})\\]\)} $string {\\&} string
893 return [subst -nocommand $string]
896 # http::ProxyRequired --
897 # Default proxy filter.
900 # host The destination host
903 # The current proxy settings
905 proc http::ProxyRequired {host
} {
907 if {[info exists
http(-proxyhost)] && [string length
$http(-proxyhost)]} {
908 if {![info exists
http(-proxyport)] ||
\
909 ![string length
$http(-proxyport)]} {
910 set http(-proxyport) 8080
912 return [list $http(-proxyhost) $http(-proxyport)]