9c2d43de2bbc811ecb2332bf35ccd77c7bc9401a
| blob | 9c2d43de2bbc811ecb2332bf35ccd77c7bc9401a |
1 # http.tcl --
2 #
3 # Client-side HTTP for GET, POST, and HEAD commands. These routines can
4 # be used in untrusted code that uses the Safesock security policy.
5 # These procedures use a callback interface to avoid using vwait, which
6 # is not defined in the safe base.
7 #
8 # See the file "license.terms" for information on usage and redistribution of
9 # this file, and for a DISCLAIMER OF ALL WARRANTIES.
11 package require Tcl 8.4
12 # Keep this in sync with pkgIndex.tcl and with the install directories in
13 # Makefiles
14 package provide http 2.7.7
16 namespace eval http {
17 # Allow resourcing to not clobber existing data
19 variable http
20 if {![info exists http]} {
21 array set http {
22 -accept */*
23 -proxyhost {}
24 -proxyport {}
25 -proxyfilter http::ProxyRequired
26 -urlencoding utf-8
27 }
28 set http(-useragent) "Tcl http client package [package provide http]"
29 }
31 proc init {} {
32 # Set up the map for quoting chars. RFC3986 Section 2.3 say percent
33 # encode all except: "... percent-encoded octets in the ranges of
34 # ALPHA (%41-%5A and %61-%7A), DIGIT (%30-%39), hyphen (%2D), period
35 # (%2E), underscore (%5F), or tilde (%7E) should not be created by URI
36 # producers ..."
37 for {set i 0} {$i <= 256} {incr i} {
38 set c [format %c $i]
39 if {![string match {[-._~a-zA-Z0-9]} $c]} {
40 set map($c) %[format %.2x $i]
41 }
42 }
43 # These are handled specially
44 set map(\n) %0d%0a
45 variable formMap [array get map]
47 # Create a map for HTTP/1.1 open sockets
48 variable socketmap
49 if {[info exists socketmap]} {
50 # Close but don't remove open sockets on re-init
51 foreach {url sock} [array get socketmap] {
52 catch {close $sock}
53 }
54 }
55 array set socketmap {}
56 }
57 init
59 variable urlTypes
60 if {![info exists urlTypes]} {
61 set urlTypes(http) [list 80 ::socket]
62 }
64 variable encodings [string tolower [encoding names]]
65 # This can be changed, but iso8859-1 is the RFC standard.
66 variable defaultCharset
67 if {![info exists defaultCharset]} {
68 set defaultCharset "iso8859-1"
69 }
71 # Force RFC 3986 strictness in geturl url verification?
72 variable strict
73 if {![info exists strict]} {
74 set strict 1
75 }
77 # Let user control default keepalive for compatibility
78 variable defaultKeepalive
79 if {![info exists defaultKeepalive]} {
80 set defaultKeepalive 0
81 }
83 namespace export geturl config reset wait formatQuery register unregister
84 # Useful, but not exported: data size status code
85 }
87 # http::Log --
88 #
89 # Debugging output -- define this to observe HTTP/1.1 socket usage.
90 # Should echo any args received.
91 #
92 # Arguments:
93 # msg Message to output
94 #
95 proc http::Log {args} {}
97 # http::register --
98 #
99 # See documentation for details.
100 #
101 # Arguments:
102 # proto URL protocol prefix, e.g. https
103 # port Default port for protocol
104 # command Command to use to create socket
105 # Results:
106 # list of port and command that was registered.
108 proc http::register {proto port command} {
109 variable urlTypes
110 set urlTypes($proto) [list $port $command]
111 }
113 # http::unregister --
114 #
115 # Unregisters URL protocol handler
116 #
117 # Arguments:
118 # proto URL protocol prefix, e.g. https
119 # Results:
120 # list of port and command that was unregistered.
122 proc http::unregister {proto} {
123 variable urlTypes
124 if {![info exists urlTypes($proto)]} {
125 return -code error "unsupported url type \"$proto\""
126 }
127 set old $urlTypes($proto)
128 unset urlTypes($proto)
129 return $old
130 }
132 # http::config --
133 #
134 # See documentation for details.
135 #
136 # Arguments:
137 # args Options parsed by the procedure.
138 # Results:
139 # TODO
141 proc http::config {args} {
142 variable http
143 set options [lsort [array names http -*]]
144 set usage [join $options ", "]
145 if {[llength $args] == 0} {
146 set result {}
147 foreach name $options {
148 lappend result $name $http($name)
149 }
150 return $result
151 }
152 set options [string map {- ""} $options]
153 set pat ^-(?:[join $options |])$
154 if {[llength $args] == 1} {
155 set flag [lindex $args 0]
156 if {![regexp -- $pat $flag]} {
157 return -code error "Unknown option $flag, must be: $usage"
158 }
159 return $http($flag)
160 } else {
161 foreach {flag value} $args {
162 if {![regexp -- $pat $flag]} {
163 return -code error "Unknown option $flag, must be: $usage"
164 }
165 set http($flag) $value
166 }
167 }
168 }
170 # http::Finish --
171 #
172 # Clean up the socket and eval close time callbacks
173 #
174 # Arguments:
175 # token Connection token.
176 # errormsg (optional) If set, forces status to error.
177 # skipCB (optional) If set, don't call the -command callback. This
178 # is useful when geturl wants to throw an exception instead
179 # of calling the callback. That way, the same error isn't
180 # reported to two places.
181 #
182 # Side Effects:
183 # Closes the socket
185 proc http::Finish {token {errormsg ""} {skipCB 0}} {
186 variable $token
187 upvar 0 $token state
188 global errorInfo errorCode
189 if {$errormsg ne ""} {
190 set state(error) [list $errormsg $errorInfo $errorCode]
191 set state(status) "error"
192 }
193 if {
194 ($state(status) eq "timeout") || ($state(status) eq "error") ||
195 ([info exists state(connection)] && ($state(connection) eq "close"))
196 } then {
197 CloseSocket $state(sock) $token
198 }
199 if {[info exists state(after)]} {
200 after cancel $state(after)
201 }
202 if {[info exists state(-command)] && !$skipCB} {
203 if {[catch {eval $state(-command) {$token}} err]} {
204 if {$errormsg eq ""} {
205 set state(error) [list $err $errorInfo $errorCode]
206 set state(status) error
207 }
208 }
209 # Command callback may already have unset our state
210 unset -nocomplain state(-command)
211 }
212 }
214 # http::CloseSocket -
215 #
216 # Close a socket and remove it from the persistent sockets table. If
217 # possible an http token is included here but when we are called from a
218 # fileevent on remote closure we need to find the correct entry - hence
219 # the second section.
221 proc ::http::CloseSocket {s {token {}}} {
222 variable socketmap
223 catch {fileevent $s readable {}}
224 set conn_id {}
225 if {$token ne ""} {
226 variable $token
227 upvar 0 $token state
228 if {[info exists state(socketinfo)]} {
229 set conn_id $state(socketinfo)
230 }
231 } else {
232 set map [array get socketmap]
233 set ndx [lsearch -exact $map $s]
234 if {$ndx != -1} {
235 incr ndx -1
236 set conn_id [lindex $map $ndx]
237 }
238 }
239 if {$conn_id eq {} || ![info exists socketmap($conn_id)]} {
240 Log "Closing socket $s (no connection info)"
241 if {[catch {close $s} err]} {
242 Log "Error: $err"
243 }
244 } else {
245 if {[info exists socketmap($conn_id)]} {
246 Log "Closing connection $conn_id (sock $socketmap($conn_id))"
247 if {[catch {close $socketmap($conn_id)} err]} {
248 Log "Error: $err"
249 }
250 unset socketmap($conn_id)
251 } else {
252 Log "Cannot close connection $conn_id - no socket in socket map"
253 }
254 }
255 }
257 # http::reset --
258 #
259 # See documentation for details.
260 #
261 # Arguments:
262 # token Connection token.
263 # why Status info.
264 #
265 # Side Effects:
266 # See Finish
268 proc http::reset {token {why reset}} {
269 variable $token
270 upvar 0 $token state
271 set state(status) $why
272 catch {fileevent $state(sock) readable {}}
273 catch {fileevent $state(sock) writable {}}
274 Finish $token
275 if {[info exists state(error)]} {
276 set errorlist $state(error)
277 unset state
278 eval ::error $errorlist
279 }
280 }
282 # http::geturl --
283 #
284 # Establishes a connection to a remote url via http.
285 #
286 # Arguments:
287 # url The http URL to goget.
288 # args Option value pairs. Valid options include:
289 # -blocksize, -validate, -headers, -timeout
290 # Results:
291 # Returns a token for this connection. This token is the name of an
292 # array that the caller should unset to garbage collect the state.
294 proc http::geturl {url args} {
295 variable http
296 variable urlTypes
297 variable defaultCharset
298 variable defaultKeepalive
299 variable strict
301 # Initialize the state variable, an array. We'll return the name of this
302 # array as the token for the transaction.
304 if {![info exists http(uid)]} {
305 set http(uid) 0
306 }
307 set token [namespace current]::[incr http(uid)]
308 variable $token
309 upvar 0 $token state
310 reset $token
312 # Process command options.
314 array set state {
315 -binary false
316 -blocksize 8192
317 -queryblocksize 8192
318 -validate 0
319 -headers {}
320 -timeout 0
321 -type application/x-www-form-urlencoded
322 -queryprogress {}
323 -protocol 1.1
324 binary 0
325 state connecting
326 meta {}
327 coding {}
328 currentsize 0
329 totalsize 0
330 querylength 0
331 queryoffset 0
332 type text/html
333 body {}
334 status ""
335 http ""
336 connection close
337 }
338 set state(-keepalive) $defaultKeepalive
339 set state(-strict) $strict
340 # These flags have their types verified [Bug 811170]
341 array set type {
342 -binary boolean
343 -blocksize integer
344 -queryblocksize integer
345 -strict boolean
346 -timeout integer
347 -validate boolean
348 }
349 set state(charset) $defaultCharset
350 set options {
351 -binary -blocksize -channel -command -handler -headers -keepalive
352 -method -myaddr -progress -protocol -query -queryblocksize
353 -querychannel -queryprogress -strict -timeout -type -validate
354 }
355 set usage [join [lsort $options] ", "]
356 set options [string map {- ""} $options]
357 set pat ^-(?:[join $options |])$
358 foreach {flag value} $args {
359 if {[regexp -- $pat $flag]} {
360 # Validate numbers
361 if {
362 [info exists type($flag)] &&
363 ![string is $type($flag) -strict $value]
364 } then {
365 unset $token
366 return -code error \
367 "Bad value for $flag ($value), must be $type($flag)"
368 }
369 set state($flag) $value
370 } else {
371 unset $token
372 return -code error "Unknown option $flag, can be: $usage"
373 }
374 }
376 # Make sure -query and -querychannel aren't both specified
378 set isQueryChannel [info exists state(-querychannel)]
379 set isQuery [info exists state(-query)]
380 if {$isQuery && $isQueryChannel} {
381 unset $token
382 return -code error "Can't combine -query and -querychannel options!"
383 }
385 # Validate URL, determine the server host and port, and check proxy case
386 # Recognize user:pass@host URLs also, although we do not do anything with
387 # that info yet.
389 # URLs have basically four parts.
390 # First, before the colon, is the protocol scheme (e.g. http)
391 # Second, for HTTP-like protocols, is the authority
392 # The authority is preceded by // and lasts up to (but not including)
393 # the following / and it identifies up to four parts, of which only one,
394 # the host, is required (if an authority is present at all). All other
395 # parts of the authority (user name, password, port number) are optional.
396 # Third is the resource name, which is split into two parts at a ?
397 # The first part (from the single "/" up to "?") is the path, and the
398 # second part (from that "?" up to "#") is the query. *HOWEVER*, we do
399 # not need to separate them; we send the whole lot to the server.
400 # Fourth is the fragment identifier, which is everything after the first
401 # "#" in the URL. The fragment identifier MUST NOT be sent to the server
402 # and indeed, we don't bother to validate it (it could be an error to
403 # pass it in here, but it's cheap to strip).
404 #
405 # An example of a URL that has all the parts:
406 #
407 # http://jschmoe:xyzzy@www.bogus.net:8000/foo/bar.tml?q=foo#changes
408 #
409 # The "http" is the protocol, the user is "jschmoe", the password is
410 # "xyzzy", the host is "www.bogus.net", the port is "8000", the path is
411 # "/foo/bar.tml", the query is "q=foo", and the fragment is "changes".
412 #
413 # Note that the RE actually combines the user and password parts, as
414 # recommended in RFC 3986. Indeed, that RFC states that putting passwords
415 # in URLs is a Really Bad Idea, something with which I would agree utterly.
416 # Also note that we do not currently support IPv6 addresses.
417 #
418 # From a validation perspective, we need to ensure that the parts of the
419 # URL that are going to the server are correctly encoded. This is only
420 # done if $state(-strict) is true (inherited from $::http::strict).
422 set URLmatcher {(?x) # this is _expanded_ syntax
423 ^
424 (?: (\w+) : ) ? # <protocol scheme>
425 (?: //
426 (?:
427 (
428 [^@/\#?]+ # <userinfo part of authority>
429 ) @
430 )?
431 ( [^/:\#?]+ ) # <host part of authority>
432 (?: : (\d+) )? # <port part of authority>
433 )?
434 ( / [^\#]*)? # <path> (including query)
435 (?: \# (.*) )? # <fragment>
436 $
437 }
439 # Phase one: parse
440 if {![regexp -- $URLmatcher $url -> proto user host port srvurl]} {
441 unset $token
442 return -code error "Unsupported URL: $url"
443 }
444 # Phase two: validate
445 if {$host eq ""} {
446 # Caller has to provide a host name; we do not have a "default host"
447 # that would enable us to handle relative URLs.
448 unset $token
449 return -code error "Missing host part: $url"
450 # Note that we don't check the hostname for validity here; if it's
451 # invalid, we'll simply fail to resolve it later on.
452 }
453 if {$port ne "" && $port > 65535} {
454 unset $token
455 return -code error "Invalid port number: $port"
456 }
457 # The user identification and resource identification parts of the URL can
458 # have encoded characters in them; take care!
459 if {$user ne ""} {
460 # Check for validity according to RFC 3986, Appendix A
461 set validityRE {(?xi)
462 ^
463 (?: [-\w.~!$&'()*+,;=:] | %[0-9a-f][0-9a-f] )+
464 $
465 }
466 if {$state(-strict) && ![regexp -- $validityRE $user]} {
467 unset $token
468 # Provide a better error message in this error case
469 if {[regexp {(?i)%(?![0-9a-f][0-9a-f]).?.?} $user bad]} {
470 return -code error \
471 "Illegal encoding character usage \"$bad\" in URL user"
472 }
473 return -code error "Illegal characters in URL user"
474 }
475 }
476 if {$srvurl ne ""} {
477 # Check for validity according to RFC 3986, Appendix A
478 set validityRE {(?xi)
479 ^
480 # Path part (already must start with / character)
481 (?: [-\w.~!$&'()*+,;=:@/] | %[0-9a-f][0-9a-f] )*
482 # Query part (optional, permits ? characters)
483 (?: \? (?: [-\w.~!$&'()*+,;=:@/?] | %[0-9a-f][0-9a-f] )* )?
484 $
485 }
486 if {$state(-strict) && ![regexp -- $validityRE $srvurl]} {
487 unset $token
488 # Provide a better error message in this error case
489 if {[regexp {(?i)%(?![0-9a-f][0-9a-f])..} $srvurl bad]} {
490 return -code error \
491 "Illegal encoding character usage \"$bad\" in URL path"
492 }
493 return -code error "Illegal characters in URL path"
494 }
495 } else {
496 set srvurl /
497 }
498 if {$proto eq ""} {
499 set proto http
500 }
501 if {![info exists urlTypes($proto)]} {
502 unset $token
503 return -code error "Unsupported URL type \"$proto\""
504 }
505 set defport [lindex $urlTypes($proto) 0]
506 set defcmd [lindex $urlTypes($proto) 1]
508 if {$port eq ""} {
509 set port $defport
510 }
511 if {![catch {$http(-proxyfilter) $host} proxy]} {
512 set phost [lindex $proxy 0]
513 set pport [lindex $proxy 1]
514 }
516 # OK, now reassemble into a full URL
517 set url ${proto}://
518 if {$user ne ""} {
519 append url $user
520 append url @
521 }
522 append url $host
523 if {$port != $defport} {
524 append url : $port
525 }
526 append url $srvurl
527 # Don't append the fragment!
528 set state(url) $url
530 # If a timeout is specified we set up the after event and arrange for an
531 # asynchronous socket connection.
533 set sockopts [list]
534 if {$state(-timeout) > 0} {
535 set state(after) [after $state(-timeout) \
536 [list http::reset $token timeout]]
537 lappend sockopts -async
538 }
540 # If we are using the proxy, we must pass in the full URL that includes
541 # the server name.
543 if {[info exists phost] && ($phost ne "")} {
544 set srvurl $url
545 set targetAddr [list $phost $pport]
546 } else {
547 set targetAddr [list $host $port]
548 }
549 # Proxy connections aren't shared among different hosts.
550 set state(socketinfo) $host:$port
552 # See if we are supposed to use a previously opened channel.
553 if {$state(-keepalive)} {
554 variable socketmap
555 if {[info exists socketmap($state(socketinfo))]} {
556 if {[catch {fconfigure $socketmap($state(socketinfo))}]} {
557 Log "WARNING: socket for $state(socketinfo) was closed"
558 unset socketmap($state(socketinfo))
559 } else {
560 set sock $socketmap($state(socketinfo))
561 Log "reusing socket $sock for $state(socketinfo)"
562 catch {fileevent $sock writable {}}
563 catch {fileevent $sock readable {}}
564 }
565 }
566 # don't automatically close this connection socket
567 set state(connection) {}
568 }
569 if {![info exists sock]} {
570 # Pass -myaddr directly to the socket command
571 if {[info exists state(-myaddr)]} {
572 lappend sockopts -myaddr $state(-myaddr)
573 }
574 if {[catch {eval $defcmd $sockopts $targetAddr} sock]} {
575 # something went wrong while trying to establish the connection.
576 # Clean up after events and such, but DON'T call the command
577 # callback (if available) because we're going to throw an
578 # exception from here instead.
580 set state(sock) $sock
581 Finish $token "" 1
582 cleanup $token
583 return -code error $sock
584 }
585 }
586 set state(sock) $sock
587 Log "Using $sock for $state(socketinfo)" \
588 [expr {$state(-keepalive)?"keepalive":""}]
589 if {$state(-keepalive)} {
590 set socketmap($state(socketinfo)) $sock
591 }
593 # Wait for the connection to complete.
595 if {$state(-timeout) > 0} {
596 fileevent $sock writable [list http::Connect $token]
597 http::wait $token
599 if {![info exists state]} {
600 # If we timed out then Finish has been called and the users
601 # command callback may have cleaned up the token. If so we end up
602 # here with nothing left to do.
603 return $token
604 } elseif {$state(status) eq "error"} {
605 # Something went wrong while trying to establish the connection.
606 # Clean up after events and such, but DON'T call the command
607 # callback (if available) because we're going to throw an
608 # exception from here instead.
609 set err [lindex $state(error) 0]
610 cleanup $token
611 return -code error $err
612 } elseif {$state(status) ne "connect"} {
613 # Likely to be connection timeout
614 return $token
615 }
616 set state(status) ""
617 }
619 # Send data in cr-lf format, but accept any line terminators
621 fconfigure $sock -translation {auto crlf} -buffersize $state(-blocksize)
623 # The following is disallowed in safe interpreters, but the socket is
624 # already in non-blocking mode in that case.
626 catch {fconfigure $sock -blocking off}
627 set how GET
628 if {$isQuery} {
629 set state(querylength) [string length $state(-query)]
630 if {$state(querylength) > 0} {
631 set how POST
632 set contDone 0
633 } else {
634 # There's no query data.
635 unset state(-query)
636 set isQuery 0
637 }
638 } elseif {$state(-validate)} {
639 set how HEAD
640 } elseif {$isQueryChannel} {
641 set how POST
642 # The query channel must be blocking for the async Write to
643 # work properly.
644 fconfigure $state(-querychannel) -blocking 1 -translation binary
645 set contDone 0
646 }
647 if {[info exists state(-method)] && $state(-method) ne ""} {
648 set how $state(-method)
649 }
651 if {[catch {
652 puts $sock "$how $srvurl HTTP/$state(-protocol)"
653 puts $sock "Accept: $http(-accept)"
654 array set hdrs $state(-headers)
655 if {[info exists hdrs(Host)]} {
656 # Allow Host spoofing. [Bug 928154]
657 puts $sock "Host: $hdrs(Host)"
658 } elseif {$port == $defport} {
659 # Don't add port in this case, to handle broken servers. [Bug
660 # #504508]
661 puts $sock "Host: $host"
662 } else {
663 puts $sock "Host: $host:$port"
664 }
665 unset hdrs
666 puts $sock "User-Agent: $http(-useragent)"
667 if {$state(-protocol) == 1.0 && $state(-keepalive)} {
668 puts $sock "Connection: keep-alive"
669 }
670 if {$state(-protocol) > 1.0 && !$state(-keepalive)} {
671 puts $sock "Connection: close" ;# RFC2616 sec 8.1.2.1
672 }
673 if {[info exists phost] && ($phost ne "") && $state(-keepalive)} {
674 puts $sock "Proxy-Connection: Keep-Alive"
675 }
676 set accept_encoding_seen 0
677 set content_type_seen 0
678 foreach {key value} $state(-headers) {
679 if {[string equal -nocase $key "host"]} {
680 continue
681 }
682 if {[string equal -nocase $key "accept-encoding"]} {
683 set accept_encoding_seen 1
684 }
685 if {[string equal -nocase $key "content-type"]} {
686 set content_type_seen 1
687 }
688 set value [string map [list \n "" \r ""] $value]
689 set key [string trim $key]
690 if {[string equal -nocase $key "content-length"]} {
691 set contDone 1
692 set state(querylength) $value
693 }
694 if {[string length $key]} {
695 puts $sock "$key: $value"
696 }
697 }
698 # Soft zlib dependency check - no package require
699 if {
700 !$accept_encoding_seen &&
701 ([package vsatisfies [package provide Tcl] 8.6]
702 || [llength [package provide zlib]]) &&
703 !([info exists state(-channel)] || [info exists state(-handler)])
704 } then {
705 puts $sock "Accept-Encoding: gzip, identity, *;q=0.1"
706 }
707 if {$isQueryChannel && $state(querylength) == 0} {
708 # Try to determine size of data in channel. If we cannot seek, the
709 # surrounding catch will trap us
711 set start [tell $state(-querychannel)]
712 seek $state(-querychannel) 0 end
713 set state(querylength) \
714 [expr {[tell $state(-querychannel)] - $start}]
715 seek $state(-querychannel) $start
716 }
718 # Flush the request header and set up the fileevent that will either
719 # push the POST data or read the response.
720 #
721 # fileevent note:
722 #
723 # It is possible to have both the read and write fileevents active at
724 # this point. The only scenario it seems to affect is a server that
725 # closes the connection without reading the POST data. (e.g., early
726 # versions TclHttpd in various error cases). Depending on the
727 # platform, the client may or may not be able to get the response from
728 # the server because of the error it will get trying to write the post
729 # data. Having both fileevents active changes the timing and the
730 # behavior, but no two platforms (among Solaris, Linux, and NT) behave
731 # the same, and none behave all that well in any case. Servers should
732 # always read their POST data if they expect the client to read their
733 # response.
735 if {$isQuery || $isQueryChannel} {
736 if {!$content_type_seen} {
737 puts $sock "Content-Type: $state(-type)"
738 }
739 if {!$contDone} {
740 puts $sock "Content-Length: $state(querylength)"
741 }
742 puts $sock ""
743 fconfigure $sock -translation {auto binary}
744 fileevent $sock writable [list http::Write $token]
745 } else {
746 puts $sock ""
747 flush $sock
748 fileevent $sock readable [list http::Event $sock $token]
749 }
751 if {![info exists state(-command)]} {
752 # geturl does EVERYTHING asynchronously, so if the user calls it
753 # synchronously, we just do a wait here.
755 wait $token
756 if {$state(status) eq "error"} {
757 # Something went wrong, so throw the exception, and the
758 # enclosing catch will do cleanup.
759 return -code error [lindex $state(error) 0]
760 }
761 }
762 } err]} then {
763 # The socket probably was never connected, or the connection dropped
764 # later.
766 # Clean up after events and such, but DON'T call the command callback
767 # (if available) because we're going to throw an exception from here
768 # instead.
770 # if state(status) is error, it means someone's already called Finish
771 # to do the above-described clean up.
772 if {$state(status) ne "error"} {
773 Finish $token $err 1
774 }
775 cleanup $token
776 return -code error $err
777 }
779 return $token
780 }
782 # Data access functions:
783 # Data - the URL data
784 # Status - the transaction status: ok, reset, eof, timeout
785 # Code - the HTTP transaction code, e.g., 200
786 # Size - the size of the URL data
788 proc http::data {token} {
789 variable $token
790 upvar 0 $token state
791 return $state(body)
792 }
793 proc http::status {token} {
794 if {![info exists $token]} {
795 return "error"
796 }
797 variable $token
798 upvar 0 $token state
799 return $state(status)
800 }
801 proc http::code {token} {
802 variable $token
803 upvar 0 $token state
804 return $state(http)
805 }
806 proc http::ncode {token} {
807 variable $token
808 upvar 0 $token state
809 if {[regexp {[0-9]{3}} $state(http) numeric_code]} {
810 return $numeric_code
811 } else {
812 return $state(http)
813 }
814 }
815 proc http::size {token} {
816 variable $token
817 upvar 0 $token state
818 return $state(currentsize)
819 }
820 proc http::meta {token} {
821 variable $token
822 upvar 0 $token state
823 return $state(meta)
824 }
825 proc http::error {token} {
826 variable $token
827 upvar 0 $token state
828 if {[info exists state(error)]} {
829 return $state(error)
830 }
831 return ""
832 }
834 # http::cleanup
835 #
836 # Garbage collect the state associated with a transaction
837 #
838 # Arguments
839 # token The token returned from http::geturl
840 #
841 # Side Effects
842 # unsets the state array
844 proc http::cleanup {token} {
845 variable $token
846 upvar 0 $token state
847 if {[info exists state]} {
848 unset state
849 }
850 }
852 # http::Connect
853 #
854 # This callback is made when an asyncronous connection completes.
855 #
856 # Arguments
857 # token The token returned from http::geturl
858 #
859 # Side Effects
860 # Sets the status of the connection, which unblocks
861 # the waiting geturl call
863 proc http::Connect {token} {
864 variable $token
865 upvar 0 $token state
866 global errorInfo errorCode
867 if {
868 [eof $state(sock)] ||
869 [string length [fconfigure $state(sock) -error]]
870 } then {
871 Finish $token "connect failed [fconfigure $state(sock) -error]" 1
872 } else {
873 set state(status) connect
874 fileevent $state(sock) writable {}
875 }
876 return
877 }
879 # http::Write
880 #
881 # Write POST query data to the socket
882 #
883 # Arguments
884 # token The token for the connection
885 #
886 # Side Effects
887 # Write the socket and handle callbacks.
889 proc http::Write {token} {
890 variable $token
891 upvar 0 $token state
892 set sock $state(sock)
894 # Output a block. Tcl will buffer this if the socket blocks
895 set done 0
896 if {[catch {
897 # Catch I/O errors on dead sockets
899 if {[info exists state(-query)]} {
900 # Chop up large query strings so queryprogress callback can give
901 # smooth feedback.
903 puts -nonewline $sock \
904 [string range $state(-query) $state(queryoffset) \
905 [expr {$state(queryoffset) + $state(-queryblocksize) - 1}]]
906 incr state(queryoffset) $state(-queryblocksize)
907 if {$state(queryoffset) >= $state(querylength)} {
908 set state(queryoffset) $state(querylength)
909 set done 1
910 }
911 } else {
912 # Copy blocks from the query channel
914 set outStr [read $state(-querychannel) $state(-queryblocksize)]
915 puts -nonewline $sock $outStr
916 incr state(queryoffset) [string length $outStr]
917 if {[eof $state(-querychannel)]} {
918 set done 1
919 }
920 }
921 } err]} then {
922 # Do not call Finish here, but instead let the read half of the socket
923 # process whatever server reply there is to get.
925 set state(posterror) $err
926 set done 1
927 }
928 if {$done} {
929 catch {flush $sock}
930 fileevent $sock writable {}
931 fileevent $sock readable [list http::Event $sock $token]
932 }
934 # Callback to the client after we've completely handled everything.
936 if {[string length $state(-queryprogress)]} {
937 eval $state(-queryprogress) \
938 [list $token $state(querylength) $state(queryoffset)]
939 }
940 }
942 # http::Event
943 #
944 # Handle input on the socket
945 #
946 # Arguments
947 # sock The socket receiving input.
948 # token The token returned from http::geturl
949 #
950 # Side Effects
951 # Read the socket and handle callbacks.
953 proc http::Event {sock token} {
954 variable $token
955 upvar 0 $token state
957 if {![info exists state]} {
958 Log "Event $sock with invalid token '$token' - remote close?"
959 if {![eof $sock]} {
960 if {[set d [read $sock]] ne ""} {
961 Log "WARNING: additional data left on closed socket"
962 }
963 }
964 CloseSocket $sock
965 return
966 }
967 if {$state(state) eq "connecting"} {
968 if {[catch {gets $sock state(http)} n]} {
969 return [Finish $token $n]
970 } elseif {$n >= 0} {
971 set state(state) "header"
972 }
973 } elseif {$state(state) eq "header"} {
974 if {[catch {gets $sock line} n]} {
975 return [Finish $token $n]
976 } elseif {$n == 0} {
977 # We have now read all headers
978 # We ignore HTTP/1.1 100 Continue returns. RFC2616 sec 8.2.3
979 if {$state(http) == "" || [lindex $state(http) 1] == 100} {
980 return
981 }
983 set state(state) body
985 # If doing a HEAD, then we won't get any body
986 if {$state(-validate)} {
987 Eof $token
988 return
989 }
991 # For non-chunked transfer we may have no body - in this case we
992 # may get no further file event if the connection doesn't close
993 # and no more data is sent. We can tell and must finish up now -
994 # not later.
995 if {
996 !(([info exists state(connection)]
997 && ($state(connection) eq "close"))
998 || [info exists state(transfer)])
999 && ($state(totalsize) == 0)
1000 } then {
1001 Log "body size is 0 and no events likely - complete."
1002 Eof $token
1003 return
1004 }
1006 # We have to use binary translation to count bytes properly.
1007 fconfigure $sock -translation binary
1009 if {
1010 $state(-binary) || ![string match -nocase text* $state(type)]
1011 } then {
1012 # Turn off conversions for non-text data
1013 set state(binary) 1
1014 }
1015 if {
1016 $state(binary) || [string match *gzip* $state(coding)] ||
1017 [string match *compress* $state(coding)]
1018 } then {
1019 if {[info exists state(-channel)]} {
1020 fconfigure $state(-channel) -translation binary
1021 }
1022 }
1023 if {
1024 [info exists state(-channel)] &&
1025 ![info exists state(-handler)]
1026 } then {
1027 # Initiate a sequence of background fcopies
1028 fileevent $sock readable {}
1029 CopyStart $sock $token
1030 return
1031 }
1032 } elseif {$n > 0} {
1033 # Process header lines
1034 if {[regexp -nocase {^([^:]+):(.+)$} $line x key value]} {
1035 switch -- [string tolower $key] {
1036 content-type {
1037 set state(type) [string trim [string tolower $value]]
1038 # grab the optional charset information
1039 if {[regexp -nocase \
1040 {charset\s*=\s*\"((?:[^""]|\\\")*)\"} \
1041 $state(type) -> cs]} {
1042 set state(charset) [string map {{\"} \"} $cs]
1043 } else {
1044 regexp -nocase {charset\s*=\s*(\S+?);?} \
1045 $state(type) -> state(charset)
1046 }
1047 }
1048 content-length {
1049 set state(totalsize) [string trim $value]
1050 }
1051 content-encoding {
1052 set state(coding) [string trim $value]
1053 }
1054 transfer-encoding {
1055 set state(transfer) \
1056 [string trim [string tolower $value]]
1057 }
1058 proxy-connection -
1059 connection {
1060 set state(connection) \
1061 [string trim [string tolower $value]]
1062 }
1063 }
1064 lappend state(meta) $key [string trim $value]
1065 }
1066 }
1067 } else {
1068 # Now reading body
1069 if {[catch {
1070 if {[info exists state(-handler)]} {
1071 set n [eval $state(-handler) [list $sock $token]]
1072 } elseif {[info exists state(transfer_final)]} {
1073 set line [getTextLine $sock]
1074 set n [string length $line]
1075 if {$n > 0} {
1076 Log "found $n bytes following final chunk"
1077 append state(transfer_final) $line
1078 } else {
1079 Log "final chunk part"
1080 Eof $token
1081 }
1082 } elseif {
1083 [info exists state(transfer)]
1084 && $state(transfer) eq "chunked"
1085 } then {
1086 set size 0
1087 set chunk [getTextLine $sock]
1088 set n [string length $chunk]
1089 if {[string trim $chunk] ne ""} {
1090 scan $chunk %x size
1091 if {$size != 0} {
1092 set bl [fconfigure $sock -blocking]
1093 fconfigure $sock -blocking 1
1094 set chunk [read $sock $size]
1095 fconfigure $sock -blocking $bl
1096 set n [string length $chunk]
1097 if {$n >= 0} {
1098 append state(body) $chunk
1099 }
1100 if {$size != [string length $chunk]} {
1101 Log "WARNING: mis-sized chunk:\
1102 was [string length $chunk], should be $size"
1103 }
1104 getTextLine $sock
1105 } else {
1106 set state(transfer_final) {}
1107 }
1108 }
1109 } else {
1110 #Log "read non-chunk $state(currentsize) of $state(totalsize)"
1111 set block [read $sock $state(-blocksize)]
1112 set n [string length $block]
1113 if {$n >= 0} {
1114 append state(body) $block
1115 }
1116 }
1117 if {[info exists state]} {
1118 if {$n >= 0} {
1119 incr state(currentsize) $n
1120 }
1121 # If Content-Length - check for end of data.
1122 if {
1123 ($state(totalsize) > 0)
1124 && ($state(currentsize) >= $state(totalsize))
1125 } then {
1126 Eof $token
1127 }
1128 }
1129 } err]} then {
1130 return [Finish $token $err]
1131 } else {
1132 if {[info exists state(-progress)]} {
1133 eval $state(-progress) \
1134 [list $token $state(totalsize) $state(currentsize)]
1135 }
1136 }
1137 }
1139 # catch as an Eof above may have closed the socket already
1140 if {![catch {eof $sock} eof] && $eof} {
1141 if {[info exists $token]} {
1142 set state(connection) close
1143 Eof $token
1144 } else {
1145 # open connection closed on a token that has been cleaned up.
1146 CloseSocket $sock
1147 }
1148 return
1149 }
1150 }
1152 # http::getTextLine --
1153 #
1154 # Get one line with the stream in blocking crlf mode
1155 #
1156 # Arguments
1157 # sock The socket receiving input.
1158 #
1159 # Results:
1160 # The line of text, without trailing newline
1162 proc http::getTextLine {sock} {
1163 set tr [fconfigure $sock -translation]
1164 set bl [fconfigure $sock -blocking]
1165 fconfigure $sock -translation crlf -blocking 1
1166 set r [gets $sock]
1167 fconfigure $sock -translation $tr -blocking $bl
1168 return $r
1169 }
1171 # http::CopyStart
1172 #
1173 # Error handling wrapper around fcopy
1174 #
1175 # Arguments
1176 # sock The socket to copy from
1177 # token The token returned from http::geturl
1178 #
1179 # Side Effects
1180 # This closes the connection upon error
1182 proc http::CopyStart {sock token} {
1183 variable $token
1184 upvar 0 $token state
1185 if {[catch {
1186 fcopy $sock $state(-channel) -size $state(-blocksize) -command \
1187 [list http::CopyDone $token]
1188 } err]} then {
1189 Finish $token $err
1190 }
1191 }
1193 # http::CopyDone
1194 #
1195 # fcopy completion callback
1196 #
1197 # Arguments
1198 # token The token returned from http::geturl
1199 # count The amount transfered
1200 #
1201 # Side Effects
1202 # Invokes callbacks
1204 proc http::CopyDone {token count {error {}}} {
1205 variable $token
1206 upvar 0 $token state
1207 set sock $state(sock)
1208 incr state(currentsize) $count
1209 if {[info exists state(-progress)]} {
1210 eval $state(-progress) \
1211 [list $token $state(totalsize) $state(currentsize)]
1212 }
1213 # At this point the token may have been reset
1214 if {[string length $error]} {
1215 Finish $token $error
1216 } elseif {[catch {eof $sock} iseof] || $iseof} {
1217 Eof $token
1218 } else {
1219 CopyStart $sock $token
1220 }
1221 }
1223 # http::Eof
1224 #
1225 # Handle eof on the socket
1226 #
1227 # Arguments
1228 # token The token returned from http::geturl
1229 #
1230 # Side Effects
1231 # Clean up the socket
1233 proc http::Eof {token {force 0}} {
1234 variable $token
1235 upvar 0 $token state
1236 if {$state(state) eq "header"} {
1237 # Premature eof
1238 set state(status) eof
1239 } else {
1240 set state(status) ok
1241 }
1243 if {($state(coding) eq "gzip") && [string length $state(body)] > 0} {
1244 if {[catch {
1245 if {[package vsatisfies [package present Tcl] 8.6]} {
1246 # The zlib integration into 8.6 includes proper gzip support
1247 set state(body) [zlib gunzip $state(body)]
1248 } else {
1249 set state(body) [Gunzip $state(body)]
1250 }
1251 } err]} then {
1252 return [Finish $token $err]
1253 }
1254 }
1256 if {!$state(binary)} {
1257 # If we are getting text, set the incoming channel's encoding
1258 # correctly. iso8859-1 is the RFC default, but this could be any IANA
1259 # charset. However, we only know how to convert what we have
1260 # encodings for.
1262 set enc [CharsetToEncoding $state(charset)]
1263 if {$enc ne "binary"} {
1264 set state(body) [encoding convertfrom $enc $state(body)]
1265 }
1267 # Translate text line endings.
1268 set state(body) [string map {\r\n \n \r \n} $state(body)]
1269 }
1271 Finish $token
1272 }
1274 # http::wait --
1275 #
1276 # See documentation for details.
1277 #
1278 # Arguments:
1279 # token Connection token.
1280 #
1281 # Results:
1282 # The status after the wait.
1284 proc http::wait {token} {
1285 variable $token
1286 upvar 0 $token state
1288 if {![info exists state(status)] || $state(status) eq ""} {
1289 # We must wait on the original variable name, not the upvar alias
1290 vwait ${token}(status)
1291 }
1293 return [status $token]
1294 }
1296 # http::formatQuery --
1297 #
1298 # See documentation for details. Call http::formatQuery with an even
1299 # number of arguments, where the first is a name, the second is a value,
1300 # the third is another name, and so on.
1301 #
1302 # Arguments:
1303 # args A list of name-value pairs.
1304 #
1305 # Results:
1306 # TODO
1308 proc http::formatQuery {args} {
1309 set result ""
1310 set sep ""
1311 foreach i $args {
1312 append result $sep [mapReply $i]
1313 if {$sep eq "="} {
1314 set sep &
1315 } else {
1316 set sep =
1317 }
1318 }
1319 return $result
1320 }
1322 # http::mapReply --
1323 #
1324 # Do x-www-urlencoded character mapping
1325 #
1326 # Arguments:
1327 # string The string the needs to be encoded
1328 #
1329 # Results:
1330 # The encoded string
1332 proc http::mapReply {string} {
1333 variable http
1334 variable formMap
1336 # The spec says: "non-alphanumeric characters are replaced by '%HH'". Use
1337 # a pre-computed map and [string map] to do the conversion (much faster
1338 # than [regsub]/[subst]). [Bug 1020491]
1340 if {$http(-urlencoding) ne ""} {
1341 set string [encoding convertto $http(-urlencoding) $string]
1342 return [string map $formMap $string]
1343 }
1344 set converted [string map $formMap $string]
1345 if {[string match "*\[\u0100-\uffff\]*" $converted]} {
1346 regexp {[\u0100-\uffff]} $converted badChar
1347 # Return this error message for maximum compatability... :^/
1348 return -code error \
1349 "can't read \"formMap($badChar)\": no such element in array"
1350 }
1351 return $converted
1352 }
1354 # http::ProxyRequired --
1355 # Default proxy filter.
1356 #
1357 # Arguments:
1358 # host The destination host
1359 #
1360 # Results:
1361 # The current proxy settings
1363 proc http::ProxyRequired {host} {
1364 variable http
1365 if {[info exists http(-proxyhost)] && [string length $http(-proxyhost)]} {
1366 if {
1367 ![info exists http(-proxyport)] ||
1368 ![string length $http(-proxyport)]
1369 } then {
1370 set http(-proxyport) 8080
1371 }
1372 return [list $http(-proxyhost) $http(-proxyport)]
1373 }
1374 }
1376 # http::CharsetToEncoding --
1377 #
1378 # Tries to map a given IANA charset to a tcl encoding. If no encoding
1379 # can be found, returns binary.
1380 #
1382 proc http::CharsetToEncoding {charset} {
1383 variable encodings
1385 set charset [string tolower $charset]
1386 if {[regexp {iso-?8859-([0-9]+)} $charset -> num]} {
1387 set encoding "iso8859-$num"
1388 } elseif {[regexp {iso-?2022-(jp|kr)} $charset -> ext]} {
1389 set encoding "iso2022-$ext"
1390 } elseif {[regexp {shift[-_]?js} $charset]} {
1391 set encoding "shiftjis"
1392 } elseif {[regexp {(?:windows|cp)-?([0-9]+)} $charset -> num]} {
1393 set encoding "cp$num"
1394 } elseif {$charset eq "us-ascii"} {
1395 set encoding "ascii"
1396 } elseif {[regexp {(?:iso-?)?lat(?:in)?-?([0-9]+)} $charset -> num]} {
1397 switch -- $num {
1398 5 {set encoding "iso8859-9"}
1399 1 - 2 - 3 {
1400 set encoding "iso8859-$num"
1401 }
1402 }
1403 } else {
1404 # other charset, like euc-xx, utf-8,... may directly map to encoding
1405 set encoding $charset
1406 }
1407 set idx [lsearch -exact $encodings $encoding]
1408 if {$idx >= 0} {
1409 return $encoding
1410 } else {
1411 return "binary"
1412 }
1413 }
1415 # http::Gunzip --
1416 #
1417 # Decompress data transmitted using the gzip transfer coding.
1418 #
1420 # FIX ME: redo using zlib sinflate
1421 proc http::Gunzip {data} {
1422 binary scan $data Scb5icc magic method flags time xfl os
1423 set pos 10
1424 if {$magic != 0x1f8b} {
1425 return -code error "invalid data: supplied data is not in gzip format"
1426 }
1427 if {$method != 8} {
1428 return -code error "invalid compression method"
1429 }
1431 # lassign [split $flags ""] f_text f_crc f_extra f_name f_comment
1432 foreach {f_text f_crc f_extra f_name f_comment} [split $flags ""] break
1433 set extra ""
1434 if {$f_extra} {
1435 binary scan $data @${pos}S xlen
1436 incr pos 2
1437 set extra [string range $data $pos $xlen]
1438 set pos [incr xlen]
1439 }
1441 set name ""
1442 if {$f_name} {
1443 set ndx [string first \0 $data $pos]
1444 set name [string range $data $pos $ndx]
1445 set pos [incr ndx]
1446 }
1448 set comment ""
1449 if {$f_comment} {
1450 set ndx [string first \0 $data $pos]
1451 set comment [string range $data $pos $ndx]
1452 set pos [incr ndx]
1453 }
1455 set fcrc ""
1456 if {$f_crc} {
1457 set fcrc [string range $data $pos [incr pos]]
1458 incr pos
1459 }
1461 binary scan [string range $data end-7 end] ii crc size
1462 set inflated [zlib inflate [string range $data $pos end-8]]
1463 set chk [zlib crc32 $inflated]
1464 if {($crc & 0xffffffff) != ($chk & 0xffffffff)} {
1465 return -code error "invalid data: checksum mismatch $crc != $chk"
1466 }
1467 return $inflated
1468 }
1470 # Local variables:
1471 # indent-tabs-mode: t
1472 # End: