1.\"- 2.\" Copyright (c) 1998-2004 Dag-Erling Co�dan Sm�rgrav 3.\" All rights reserved. 4.\" 5.\" Redistribution and use in source and binary forms, with or without 6.\" modification, are permitted provided that the following conditions 7.\" are met: 8.\" 1. Redistributions of source code must retain the above copyright 9.\" notice, this list of conditions and the following disclaimer. 10.\" 2. Redistributions in binary form must reproduce the above copyright 11.\" notice, this list of conditions and the following disclaimer in the 12.\" documentation and/or other materials provided with the distribution. 13.\" 14.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24.\" SUCH DAMAGE. 25.\" 26.\" $FreeBSD$ 27.\" 28.Dd April 22, 2007 29.Dt FETCH 3 30.Os 31.Sh NAME 32.Nm fetchMakeURL , 33.Nm fetchParseURL , 34.Nm fetchFreeURL , 35.Nm fetchXGetURL , 36.Nm fetchGetURL , 37.Nm fetchPutURL , 38.Nm fetchStatURL , 39.Nm fetchListURL , 40.Nm fetchXGet , 41.Nm fetchGet , 42.Nm fetchPut , 43.Nm fetchStat , 44.Nm fetchList , 45.Nm fetchXGetFile , 46.Nm fetchGetFile , 47.Nm fetchPutFile , 48.Nm fetchStatFile , 49.Nm fetchListFile , 50.Nm fetchXGetHTTP , 51.Nm fetchGetHTTP , 52.Nm fetchPutHTTP , 53.Nm fetchStatHTTP , 54.Nm fetchListHTTP , 55.Nm fetchXGetFTP , 56.Nm fetchGetFTP , 57.Nm fetchPutFTP , 58.Nm fetchStatFTP , 59.Nm fetchListFTP 60.Nd file transfer functions 61.Sh LIBRARY 62.Lb libfetch 63.Sh SYNOPSIS 64.In sys/param.h 65.In stdio.h 66.In fetch.h 67.Ft struct url * 68.Fn fetchMakeURL "const char *scheme" "const char *host" "int port" "const char *doc" "const char *user" "const char *pwd" 69.Ft struct url * 70.Fn fetchParseURL "const char *URL" 71.Ft void 72.Fn fetchFreeURL "struct url *u" 73.Ft FILE * 74.Fn fetchXGetURL "const char *URL" "struct url_stat *us" "const char *flags" 75.Ft FILE * 76.Fn fetchGetURL "const char *URL" "const char *flags" 77.Ft FILE * 78.Fn fetchPutURL "const char *URL" "const char *flags" 79.Ft int 80.Fn fetchStatURL "const char *URL" "struct url_stat *us" "const char *flags" 81.Ft struct url_ent * 82.Fn fetchListURL "const char *URL" "const char *flags" 83.Ft FILE * 84.Fn fetchXGet "struct url *u" "struct url_stat *us" "const char *flags" 85.Ft FILE * 86.Fn fetchGet "struct url *u" "const char *flags" 87.Ft FILE * 88.Fn fetchPut "struct url *u" "const char *flags" 89.Ft int 90.Fn fetchStat "struct url *u" "struct url_stat *us" "const char *flags" 91.Ft struct url_ent * 92.Fn fetchList "struct url *u" "const char *flags" 93.Ft FILE * 94.Fn fetchXGetFile "struct url *u" "struct url_stat *us" "const char *flags" 95.Ft FILE * 96.Fn fetchGetFile "struct url *u" "const char *flags" 97.Ft FILE * 98.Fn fetchPutFile "struct url *u" "const char *flags" 99.Ft int 100.Fn fetchStatFile "struct url *u" "struct url_stat *us" "const char *flags" 101.Ft struct url_ent * 102.Fn fetchListFile "struct url *u" "const char *flags" 103.Ft FILE * 104.Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "const char *flags" 105.Ft FILE * 106.Fn fetchGetHTTP "struct url *u" "const char *flags" 107.Ft FILE * 108.Fn fetchPutHTTP "struct url *u" "const char *flags" 109.Ft int 110.Fn fetchStatHTTP "struct url *u" "struct url_stat *us" "const char *flags" 111.Ft struct url_ent * 112.Fn fetchListHTTP "struct url *u" "const char *flags" 113.Ft FILE * 114.Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "const char *flags" 115.Ft FILE * 116.Fn fetchGetFTP "struct url *u" "const char *flags" 117.Ft FILE * 118.Fn fetchPutFTP "struct url *u" "const char *flags" 119.Ft int 120.Fn fetchStatFTP "struct url *u" "struct url_stat *us" "const char *flags" 121.Ft struct url_ent * 122.Fn fetchListFTP "struct url *u" "const char *flags" 123.Sh DESCRIPTION 124These functions implement a high-level library for retrieving and 125uploading files using Uniform Resource Locators (URLs). 126.Pp 127.Fn fetchParseURL 128takes a URL in the form of a null-terminated string and splits it into 129its components function according to the Common Internet Scheme Syntax 130detailed in RFC1738. 131A regular expression which produces this syntax is: 132.Bd -literal 133 <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)? 134.Ed 135.Pp 136If the URL does not seem to begin with a scheme name, the following 137syntax is assumed: 138.Bd -literal 139 ((<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)? 140.Ed 141.Pp 142Note that some components of the URL are not necessarily relevant to 143all URL schemes. 144For instance, the file scheme only needs the <scheme> and <document> 145components. 146.Pp 147.Fn fetchMakeURL 148and 149.Fn fetchParseURL 150return a pointer to a 151.Vt url 152structure, which is defined as follows in 153.In fetch.h : 154.Bd -literal 155#define URL_SCHEMELEN 16 156#define URL_USERLEN 256 157#define URL_PWDLEN 256 158 159struct url { 160 char scheme[URL_SCHEMELEN+1]; 161 char user[URL_USERLEN+1]; 162 char pwd[URL_PWDLEN+1]; 163 char host[MAXHOSTNAMELEN+1]; 164 int port; 165 char *doc; 166 off_t offset; 167 size_t length; 168}; 169.Ed 170.Pp 171The pointer returned by 172.Fn fetchMakeURL 173or 174.Fn fetchParseURL 175should be freed using 176.Fn fetchFreeURL . 177.Pp 178.Fn fetchXGetURL , 179.Fn fetchGetURL , 180and 181.Fn fetchPutURL 182constitute the recommended interface to the 183.Nm fetch 184library. 185They examine the URL passed to them to determine the transfer 186method, and call the appropriate lower-level functions to perform the 187actual transfer. 188.Fn fetchXGetURL 189also returns the remote document's metadata in the 190.Vt url_stat 191structure pointed to by the 192.Fa us 193argument. 194.Pp 195The 196.Fa flags 197argument is a string of characters which specify transfer options. 198The 199meaning of the individual flags is scheme-dependent, and is detailed 200in the appropriate section below. 201.Pp 202.Fn fetchStatURL 203attempts to obtain the requested document's metadata and fill in the 204structure pointed to by its second argument. 205The 206.Vt url_stat 207structure is defined as follows in 208.In fetch.h : 209.Bd -literal 210struct url_stat { 211 off_t size; 212 time_t atime; 213 time_t mtime; 214}; 215.Ed 216.Pp 217If the size could not be obtained from the server, the 218.Fa size 219field is set to -1. 220If the modification time could not be obtained from the server, the 221.Fa mtime 222field is set to the epoch. 223If the access time could not be obtained from the server, the 224.Fa atime 225field is set to the modification time. 226.Pp 227.Fn fetchListURL 228attempts to list the contents of the directory pointed to by the URL 229provided. 230If successful, it returns a malloced array of 231.Vt url_ent 232structures. 233The 234.Vt url_ent 235structure is defined as follows in 236.In fetch.h : 237.Bd -literal 238struct url_ent { 239 char name[MAXPATHLEN]; 240 struct url_stat stat; 241}; 242.Ed 243.Pp 244The list is terminated by an entry with an empty name. 245.Pp 246The pointer returned by 247.Fn fetchListURL 248should be freed using 249.Fn free . 250.Pp 251.Fn fetchXGet , 252.Fn fetchGet , 253.Fn fetchPut 254and 255.Fn fetchStat 256are similar to 257.Fn fetchXGetURL , 258.Fn fetchGetURL , 259.Fn fetchPutURL 260and 261.Fn fetchStatURL , 262except that they expect a pre-parsed URL in the form of a pointer to 263a 264.Vt struct url 265rather than a string. 266.Pp 267All of the 268.Fn fetchXGetXXX , 269.Fn fetchGetXXX 270and 271.Fn fetchPutXXX 272functions return a pointer to a stream which can be used to read or 273write data from or to the requested document, respectively. 274Note that 275although the implementation details of the individual access methods 276vary, it can generally be assumed that a stream returned by one of the 277.Fn fetchXGetXXX 278or 279.Fn fetchGetXXX 280functions is read-only, and that a stream returned by one of the 281.Fn fetchPutXXX 282functions is write-only. 283.Sh FILE SCHEME 284.Fn fetchXGetFile , 285.Fn fetchGetFile 286and 287.Fn fetchPutFile 288provide access to documents which are files in a locally mounted file 289system. 290Only the <document> component of the URL is used. 291.Pp 292.Fn fetchXGetFile 293and 294.Fn fetchGetFile 295do not accept any flags. 296.Pp 297.Fn fetchPutFile 298accepts the 299.Ql a 300(append to file) flag. 301If that flag is specified, the data written to 302the stream returned by 303.Fn fetchPutFile 304will be appended to the previous contents of the file, instead of 305replacing them. 306.Sh FTP SCHEME 307.Fn fetchXGetFTP , 308.Fn fetchGetFTP 309and 310.Fn fetchPutFTP 311implement the FTP protocol as described in RFC959. 312.Pp 313If the 314.Ql p 315(passive) flag is specified, a passive (rather than active) connection 316will be attempted. 317.Pp 318If the 319.Ql l 320(low) flag is specified, data sockets will be allocated in the low (or 321default) port range instead of the high port range (see 322.Xr ip 4 ) . 323.Pp 324If the 325.Ql d 326(direct) flag is specified, 327.Fn fetchXGetFTP , 328.Fn fetchGetFTP 329and 330.Fn fetchPutFTP 331will use a direct connection even if a proxy server is defined. 332.Pp 333If no user name or password is given, the 334.Nm fetch 335library will attempt an anonymous login, with user name "anonymous" 336and password "anonymous@<hostname>". 337.Pp 338By default, 339.Nm fetch 340changes directories one at a time for the best compatibility with a wide 341variety of servers. 342If it is compiled with the 343.Ql FTP_COMBINE_CWDS 344flag, it will first try a single change directory command for the entire 345path and fall back to the default scheme if an error occurred. 346This can save latency when used with a slow network link. 347.Sh HTTP SCHEME 348The 349.Fn fetchXGetHTTP , 350.Fn fetchGetHTTP 351and 352.Fn fetchPutHTTP 353functions implement the HTTP/1.1 protocol. 354With a little luck, there is 355even a chance that they comply with RFC2616 and RFC2617. 356.Pp 357If the 358.Ql d 359(direct) flag is specified, 360.Fn fetchXGetHTTP , 361.Fn fetchGetHTTP 362and 363.Fn fetchPutHTTP 364will use a direct connection even if a proxy server is defined. 365.Pp 366Since there seems to be no good way of implementing the HTTP PUT 367method in a manner consistent with the rest of the 368.Nm fetch 369library, 370.Fn fetchPutHTTP 371is currently unimplemented. 372.Sh AUTHENTICATION 373Apart from setting the appropriate environment variables and 374specifying the user name and password in the URL or the 375.Vt struct url , 376the calling program has the option of defining an authentication 377function with the following prototype: 378.Pp 379.Ft int 380.Fn myAuthMethod "struct url *u" 381.Pp 382The callback function should fill in the 383.Fa user 384and 385.Fa pwd 386fields in the provided 387.Vt struct url 388and return 0 on success, or any other value to indicate failure. 389.Pp 390To register the authentication callback, simply set 391.Va fetchAuthMethod 392to point at it. 393The callback will be used whenever a site requires authentication and 394the appropriate environment variables are not set. 395.Pp 396This interface is experimental and may be subject to change. 397.Sh RETURN VALUES 398.Fn fetchParseURL 399returns a pointer to a 400.Vt struct url 401containing the individual components of the URL. 402If it is 403unable to allocate memory, or the URL is syntactically incorrect, 404.Fn fetchParseURL 405returns a NULL pointer. 406.Pp 407The 408.Fn fetchStat 409functions return 0 on success and -1 on failure. 410.Pp 411All other functions return a stream pointer which may be used to 412access the requested document, or NULL if an error occurred. 413.Pp 414The following error codes are defined in 415.In fetch.h : 416.Bl -tag -width 18n 417.It Bq Er FETCH_ABORT 418Operation aborted 419.It Bq Er FETCH_AUTH 420Authentication failed 421.It Bq Er FETCH_DOWN 422Service unavailable 423.It Bq Er FETCH_EXISTS 424File exists 425.It Bq Er FETCH_FULL 426File system full 427.It Bq Er FETCH_INFO 428Informational response 429.It Bq Er FETCH_MEMORY 430Insufficient memory 431.It Bq Er FETCH_MOVED 432File has moved 433.It Bq Er FETCH_NETWORK 434Network error 435.It Bq Er FETCH_OK 436No error 437.It Bq Er FETCH_PROTO 438Protocol error 439.It Bq Er FETCH_RESOLV 440Resolver error 441.It Bq Er FETCH_SERVER 442Server error 443.It Bq Er FETCH_TEMP 444Temporary error 445.It Bq Er FETCH_TIMEOUT 446Operation timed out 447.It Bq Er FETCH_UNAVAIL 448File is not available 449.It Bq Er FETCH_UNKNOWN 450Unknown error 451.It Bq Er FETCH_URL 452Invalid URL 453.El 454.Pp 455The accompanying error message includes a protocol-specific error code 456and message, e.g.\& "File is not available (404 Not Found)" 457.Sh ENVIRONMENT 458.Bl -tag -width ".Ev FETCH_BIND_ADDRESS" 459.It Ev FETCH_BIND_ADDRESS 460Specifies a hostname or IP address to which sockets used for outgoing 461connections will be bound. 462.It Ev FTP_LOGIN 463Default FTP login if none was provided in the URL. 464.It Ev FTP_PASSIVE_MODE 465If set to anything but 466.Ql no , 467forces the FTP code to use passive mode. 468.It Ev FTP_PASSWORD 469Default FTP password if the remote server requests one and none was 470provided in the URL. 471.It Ev FTP_PROXY 472URL of the proxy to use for FTP requests. 473The document part is ignored. 474FTP and HTTP proxies are supported; if no scheme is specified, FTP is 475assumed. 476If the proxy is an FTP proxy, 477.Nm libfetch 478will send 479.Ql user@host 480as user name to the proxy, where 481.Ql user 482is the real user name, and 483.Ql host 484is the name of the FTP server. 485.Pp 486If this variable is set to an empty string, no proxy will be used for 487FTP requests, even if the 488.Ev HTTP_PROXY 489variable is set. 490.It Ev ftp_proxy 491Same as 492.Ev FTP_PROXY , 493for compatibility. 494.It Ev HTTP_AUTH 495Specifies HTTP authorization parameters as a colon-separated list of 496items. 497The first and second item are the authorization scheme and realm 498respectively; further items are scheme-dependent. 499Currently, only basic authorization is supported. 500.Pp 501Basic authorization requires two parameters: the user name and 502password, in that order. 503.Pp 504This variable is only used if the server requires authorization and 505no user name or password was specified in the URL. 506.It Ev HTTP_PROXY 507URL of the proxy to use for HTTP requests. 508The document part is ignored. 509Only HTTP proxies are supported for HTTP requests. 510If no port number is specified, the default is 3128. 511.Pp 512Note that this proxy will also be used for FTP documents, unless the 513.Ev FTP_PROXY 514variable is set. 515.It Ev http_proxy 516Same as 517.Ev HTTP_PROXY , 518for compatibility. 519.It Ev HTTP_PROXY_AUTH 520Specifies authorization parameters for the HTTP proxy in the same 521format as the 522.Ev HTTP_AUTH 523variable. 524.Pp 525This variable is used if and only if connected to an HTTP proxy, and 526is ignored if a user and/or a password were specified in the proxy 527URL. 528.It Ev HTTP_REFERER 529Specifies the referrer URL to use for HTTP requests. 530If set to 531.Dq auto , 532the document URL will be used as referrer URL. 533.It Ev HTTP_USER_AGENT 534Specifies the User-Agent string to use for HTTP requests. 535This can be useful when working with HTTP origin or proxy servers that 536differentiate between user agents. 537.It Ev NETRC 538Specifies a file to use instead of 539.Pa ~/.netrc 540to look up login names and passwords for FTP sites. 541See 542.Xr ftp 1 543for a description of the file format. 544This feature is experimental. 545.El 546.Sh EXAMPLES 547To access a proxy server on 548.Pa proxy.example.com 549port 8080, set the 550.Ev HTTP_PROXY 551environment variable in a manner similar to this: 552.Pp 553.Dl HTTP_PROXY=http://proxy.example.com:8080 554.Pp 555If the proxy server requires authentication, there are 556two options available for passing the authentication data. 557The first method is by using the proxy URL: 558.Pp 559.Dl HTTP_PROXY=http://<user>:<pwd>@proxy.example.com:8080 560.Pp 561The second method is by using the 562.Ev HTTP_PROXY_AUTH 563environment variable: 564.Bd -literal -offset indent 565HTTP_PROXY=http://proxy.example.com:8080 566HTTP_PROXY_AUTH=basic:*:<user>:<pwd> 567.Ed 568.Sh SEE ALSO 569.Xr fetch 1 , 570.Xr ftpio 3 , 571.Xr ip 4 572.Rs 573.%A J. Postel 574.%A J. K. Reynolds 575.%D October 1985 576.%B File Transfer Protocol 577.%O RFC959 578.Re 579.Rs 580.%A P. Deutsch 581.%A A. Emtage 582.%A A. Marine. 583.%D May 1994 584.%T How to Use Anonymous FTP 585.%O RFC1635 586.Re 587.Rs 588.%A T. Berners-Lee 589.%A L. Masinter 590.%A M. McCahill 591.%D December 1994 592.%T Uniform Resource Locators (URL) 593.%O RFC1738 594.Re 595.Rs 596.%A R. Fielding 597.%A J. Gettys 598.%A J. Mogul 599.%A H. Frystyk 600.%A L. Masinter 601.%A P. Leach 602.%A T. Berners-Lee 603.%D January 1999 604.%B Hypertext Transfer Protocol -- HTTP/1.1 605.%O RFC2616 606.Re 607.Rs 608.%A J. Franks 609.%A P. Hallam-Baker 610.%A J. Hostetler 611.%A S. Lawrence 612.%A P. Leach 613.%A A. Luotonen 614.%A L. Stewart 615.%D June 1999 616.%B HTTP Authentication: Basic and Digest Access Authentication 617.%O RFC2617 618.Re 619.Sh HISTORY 620The 621.Nm fetch 622library first appeared in 623.Fx 3.0 . 624.Sh AUTHORS 625.An -nosplit 626The 627.Nm fetch 628library was mostly written by 629.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org 630with numerous suggestions from 631.An Jordan K. Hubbard Aq jkh@FreeBSD.org , 632.An Eugene Skepner Aq eu@qub.com 633and other 634.Fx 635developers. 636It replaces the older 637.Nm ftpio 638library written by 639.An Poul-Henning Kamp Aq phk@FreeBSD.org 640and 641.An Jordan K. Hubbard Aq jkh@FreeBSD.org . 642.Pp 643This manual page was written by 644.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org . 645.Sh BUGS 646Some parts of the library are not yet implemented. 647The most notable 648examples of this are 649.Fn fetchPutHTTP , 650.Fn fetchListHTTP , 651.Fn fetchListFTP 652and FTP proxy support. 653.Pp 654There is no way to select a proxy at run-time other than setting the 655.Ev HTTP_PROXY 656or 657.Ev FTP_PROXY 658environment variables as appropriate. 659.Pp 660.Nm libfetch 661does not understand or obey 305 (Use Proxy) replies. 662.Pp 663Error numbers are unique only within a certain context; the error 664codes used for FTP and HTTP overlap, as do those used for resolver and 665system errors. 666For instance, error code 202 means "Command not 667implemented, superfluous at this site" in an FTP context and 668"Accepted" in an HTTP context. 669.Pp 670.Fn fetchStatFTP 671does not check that the result of an MDTM command is a valid date. 672.Pp 673The man page is incomplete, poorly written and produces badly 674formatted text. 675.Pp 676The error reporting mechanism is unsatisfactory. 677.Pp 678Some parts of the code are not fully reentrant. 679