1.\" Copyright (c) 1998 Dag-Erling Co�dan Sm�rgrav 2.\" All rights reserved. 3.\" 4.\" Redistribution and use in source and binary forms, with or without 5.\" modification, are permitted provided that the following conditions 6.\" are met: 7.\" 1. Redistributions of source code must retain the above copyright 8.\" notice, this list of conditions and the following disclaimer. 9.\" 2. Redistributions in binary form must reproduce the above copyright 10.\" notice, this list of conditions and the following disclaimer in the 11.\" documentation and/or other materials provided with the distribution. 12.\" 13.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23.\" SUCH DAMAGE. 24.\" 25.\" $FreeBSD$ 26.\" 27.Dd July 1, 1998 28.Dt FETCH 3 29.Os 30.Sh NAME 31.Nm fetchMakeURL , 32.Nm fetchParseURL , 33.Nm fetchFreeURL , 34.Nm fetchXGetURL , 35.Nm fetchGetURL , 36.Nm fetchPutURL , 37.Nm fetchStatURL , 38.Nm fetchListURL , 39.Nm fetchXGet , 40.Nm fetchGet , 41.Nm fetchPut , 42.Nm fetchStat , 43.Nm fetchList , 44.Nm fetchXGetFile , 45.Nm fetchGetFile , 46.Nm fetchPutFile , 47.Nm fetchStatFile , 48.Nm fetchListFile , 49.Nm fetchXGetHTTP , 50.Nm fetchGetHTTP , 51.Nm fetchPutHTTP , 52.Nm fetchStatHTTP , 53.Nm fetchListHTTP , 54.Nm fetchXGetFTP , 55.Nm fetchGetFTP , 56.Nm fetchPutFTP , 57.Nm fetchStatFTP , 58.Nm fetchListFTP 59.Nd file transfer functions 60.Sh LIBRARY 61.Lb libfetch 62.Sh SYNOPSIS 63.Fd #include <sys/param.h> 64.Fd #include <stdio.h> 65.Fd #include <fetch.h> 66.Ft struct url * 67.Fn fetchMakeURL "char *scheme" "char *host" "int port" "char *doc" "char *user" "char *pwd" 68.Ft struct url * 69.Fn fetchParseURL "char *URL" 70.Ft void 71.Fn fetchFreeURL "struct url *URL" 72.Ft FILE * 73.Fn fetchXGetURL "char *URL" "struct url_stat *us" "char *flags" 74.Ft FILE * 75.Fn fetchGetURL "char *URL" "char *flags" 76.Ft FILE * 77.Fn fetchPutURL "char *URL" "char *flags" 78.Ft int 79.Fn fetchStatURL "char *URL" "struct url_stat *us" "char *flags" 80.Ft struct url_ent * 81.Fn fetchListURL "char *URL" "char *flags" 82.Ft FILE * 83.Fn fetchXGet "struct url *URL" "struct url_stat *us" "char *flags" 84.Ft FILE * 85.Fn fetchGet "struct url *URL" "char *flags" 86.Ft FILE * 87.Fn fetchPut "struct url *URL" "char *flags" 88.Ft int 89.Fn fetchStat "struct url *URL" "struct url_stat *us" "char *flags" 90.Ft struct url_ent * 91.Fn fetchList "struct url *" "char *flags" 92.Ft FILE * 93.Fn fetchXGetFile "struct url *u" "struct url_stat *us" "char *flags" 94.Ft FILE * 95.Fn fetchGetFile "struct url *u" "char *flags" 96.Ft FILE * 97.Fn fetchPutFile "struct url *u" "char *flags" 98.Ft int 99.Fn fetchStatFile "struct url *URL" "struct url_stat *us" "char *flags" 100.Ft struct url_ent * 101.Fn fetchListFile "struct url *" "char *flags" 102.Ft FILE * 103.Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "char *flags" 104.Ft FILE * 105.Fn fetchGetHTTP "struct url *u" "char *flags" 106.Ft FILE * 107.Fn fetchPutHTTP "struct url *u" "char *flags" 108.Ft int 109.Fn fetchStatHTTP "struct url *URL" "struct url_stat *us" "char *flags" 110.Ft struct url_ent * 111.Fn fetchListHTTP "struct url *" "char *flags" 112.Ft FILE * 113.Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "char *flags" 114.Ft FILE * 115.Fn fetchGetFTP "struct url *u" "char *flags" 116.Ft FILE * 117.Fn fetchPutFTP "struct url *u" "char *flags" 118.Ft int 119.Fn fetchStatFTP "struct url *URL" "struct url_stat *us" "char *flags" 120.Ft struct url_ent * 121.Fn fetchListFTP "struct url *" "char *flags" 122.Sh DESCRIPTION 123.Pp 124These functions implement a high-level library for retrieving and 125uploading files using Uniform Resource Locators (URLs). 126.Pp 127.Fn fetchParseURL 128takes a URL in the form of a null-terminated string and splits it into 129its components function according to the Common Internet Scheme Syntax 130detailed in RFC1738. 131A regular expression which produces this syntax is: 132.Bd -literal 133 <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)? 134.Ed 135.Pp 136If the URL does not seem to begin with a scheme name, the following 137syntax is assumed: 138.Bd -literal 139 ((<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)? 140.Ed 141.Pp 142Note that some components of the URL are not necessarily relevant to 143all URL schemes. 144For instance, the file scheme only needs the <scheme> and <document> 145components. 146.Pp 147.Fn fetchMakeURL 148and 149.Fn fetchParseURL 150return a pointer to a 151.Fa url 152structure, which is defined as follows in 153.Aq Pa fetch.h : 154.Bd -literal 155#define URL_SCHEMELEN 16 156#define URL_USERLEN 256 157#define URL_PWDLEN 256 158 159struct url { 160 char scheme[URL_SCHEMELEN+1]; 161 char user[URL_USERLEN+1]; 162 char pwd[URL_PWDLEN+1]; 163 char host[MAXHOSTNAMELEN+1]; 164 int port; 165 char *doc; 166 off_t offset; 167 size_t length; 168}; 169.Ed 170.Pp 171The pointer returned by 172.Fn fetchMakeURL 173or 174.Fn fetchParseURL 175should be freed using 176.Fn fetchFreeURL . 177.Pp 178.Fn fetchXGetURL , 179.Fn fetchGetURL , 180and 181.Fn fetchPutURL 182constitute the recommended interface to the 183.Nm fetch 184library. 185They examine the URL passed to them to determine the transfer 186method, and call the appropriate lower-level functions to perform the 187actual transfer. 188.Fn fetchXGetURL 189also returns the remote document's metadata in the 190.Fa url_stat 191structure pointed to by the 192.Fa us 193argument. 194.Pp 195The 196.Fa flags 197argument is a string of characters which specify transfer options. 198The 199meaning of the individual flags is scheme-dependent, and is detailed 200in the appropriate section below. 201.Pp 202.Fn fetchStatURL 203attempts to obtain the requested document's metadata and fill in the 204structure pointed to by it's second argument. 205The 206.Fa url_stat 207structure is defined as follows in 208.Aq Pa fetch.h : 209.Bd -literal 210struct url_stat { 211 off_t size; 212 time_t atime; 213 time_t mtime; 214}; 215.Ed 216.Pp 217If the size could not be obtained from the server, the 218.Fa size 219field is set to -1. 220If the modification time could not be obtained from the server, the 221.Fa mtime 222field is set to the epoch. 223If the access time could not be obtained from the server, the 224.Fa atime 225field is set to the modification time. 226.Pp 227.Fn fetchListURL 228attempts to list the contents of the directory pointed to by the URL 229provided. 230If successful, it returns a malloced array of 231.Fa url_ent 232structures. 233The 234.Fa url_ent 235structure is defined as follows in 236.Aq Pa fetch.h : 237.Bd -literal 238struct url_ent { 239 char name[MAXPATHLEN]; 240 struct url_stat stat; 241}; 242.Ed 243.Pp 244The list is terminated by an entry with an empty name. 245.Pp 246The pointer returned by 247.Fn fetchListURL 248should be freed using 249.Fn free . 250.Pp 251.Fn fetchXGet , 252.Fn fetchGet , 253.Fn fetchPut 254and 255.Fn fetchStat 256are similar to 257.Fn fetchXGetURL , 258.Fn fetchGetURL , 259.Fn fetchPutURL 260and 261.Fn fetchStatURL , 262except that they expect a pre-parsed URL in the form of a pointer to 263a 264.Fa struct url 265rather than a string. 266.Pp 267All of the 268.Fn fetchXGetXXX , 269.Fn fetchGetXXX 270and 271.Fn fetchPutXXX 272functions return a pointer to a stream which can be used to read or 273write data from or to the requested document, respectively. 274Note that 275although the implementation details of the individual access methods 276vary, it can generally be assumed that a stream returned by one of the 277.Fn fetchXGetXXX 278or 279.Fn fetchGetXXX 280functions is read-only, and that a stream returned by one of the 281.Fn fetchPutXXX 282functions is write-only. 283.Sh FILE SCHEME 284.Fn fetchXGetFile , 285.Fn fetchGetFile 286and 287.Fn fetchPutFile 288provide access to documents which are files in a locally mounted file 289system. 290Only the <document> component of the URL is used. 291.Pp 292.Fn fetchXGetFile 293and 294.Fn fetchGetFile 295do not accept any flags. 296.Pp 297.Fn fetchPutFile 298accepts the 299.Fa a 300(append to file) flag. 301If that flag is specified, the data written to 302the stream returned by 303.Fn fetchPutFile 304will be appended to the previous contents of the file, instead of 305replacing them. 306.Sh FTP SCHEME 307.Fn fetchXGetFTP , 308.Fn fetchGetFTP 309and 310.Fn fetchPutFTP 311implement the FTP protocol as described in RFC959. 312.Pp 313If the 314.Fa p 315(passive) flag is specified, a passive (rather than active) connection 316will be attempted. 317.Pp 318If the 319.Fa h 320(high) flag is specified, data sockets will be allocated in the high 321port range (see 322.Xr ip 4 ). 323.Pp 324If the 325.Fa d 326(direct) flag is specified, 327.Fn fetchXGetFTP , 328.Fn fetchGetFTP 329and 330.Fn fetchPutFTP 331will use a direct connection even if a proxy server is defined. 332.Pp 333If no user name or password is given, the 334.Nm fetch 335library will attempt an anonymous login, with user name "ftp" and 336password "ftp". 337.Sh HTTP SCHEME 338The 339.Fn fetchXGetHTTP , 340.Fn fetchGetHTTP 341and 342.Fn fetchPutHTTP 343functions implement the HTTP/1.1 protocol. 344With a little luck, there's 345even a chance that they comply with RFC2068. 346.Pp 347If the 348.Fa d 349(direct) flag is specified, 350.Fn fetchXGetHTTP , 351.Fn fetchGetHTTP 352and 353.Fn fetchPutHTTP 354will use a direct connection even if a proxy server is defined. 355.Pp 356Since there seems to be no good way of implementing the HTTP PUT 357method in a manner consistent with the rest of the 358.Nm fetch 359library, 360.Fn fetchPutHTTP 361is currently unimplemented. 362.Sh RETURN VALUES 363.Fn fetchParseURL 364returns a pointer to a 365.Fa struct url 366containing the individual components of the URL. 367If it is 368unable to allocate memory, or the URL is syntactically incorrect, 369.Fn fetchParseURL 370returns a NULL pointer. 371.Pp 372The 373.Fn fetchStat 374functions return 0 on success and -1 on failure. 375.Pp 376All other functions return a stream pointer which may be used to 377access the requested document, or NULL if an error occurred. 378.Pp 379The following error codes are defined in 380.Aq Pa fetch.h : 381.Bl -tag -width 18n 382.It Bq Er FETCH_ABORT 383Operation aborted 384.It Bq Er FETCH_AUTH 385Authentication failed 386.It Bq Er FETCH_DOWN 387Service unavailable 388.It Bq Er FETCH_EXISTS 389File exists 390.It Bq Er FETCH_FULL 391File system full 392.It Bq Er FETCH_INFO 393Informational response 394.It Bq Er FETCH_MEMORY 395Insufficient memory 396.It Bq Er FETCH_MOVED 397File has moved 398.It Bq Er FETCH_NETWORK 399Network error 400.It Bq Er FETCH_OK 401No error 402.It Bq Er FETCH_PROTO 403Protocol error 404.It Bq Er FETCH_RESOLV 405Resolver error 406.It Bq Er FETCH_SERVER 407Server error 408.It Bq Er FETCH_TEMP 409Temporary error 410.It Bq Er FETCH_TIMEOUT 411Operation timed out 412.It Bq Er FETCH_UNAVAIL 413File is not available 414.It Bq Er FETCH_UNKNOWN 415Unknown error 416.It Bq Er FETCH_URL 417Invalid URL 418.El 419.Pp 420The accompanying error message includes a protocol-specific error code 421and message, e.g. "File is not available (404 Not Found)" 422.Sh ENVIRONMENT 423.Bl -tag -width FTP_PASSIVE_MODE 424.It Ev FTP_PASSIVE_MODE 425The FTP code selects passive mode by default. 426To force active mode FTP, set this variable to 427.Ql no . 428.It Ev FTP_PASSWORD 429Default FTP password if the remote server requests one and none was 430provided in the URL. 431.It Ev FTP_PROXY 432URL of the proxy to use for FTP requests. 433The document part is ignored. 434FTP and HTTP proxies are supported; if no scheme is specified, FTP is 435assumed. 436If the proxy is an FTP proxy, 437.Nm libfetch 438will send 439.Ql user@host 440as user name to the proxy, where 441.Ql user 442is the real user name, and 443.Ql host 444is the name of the FTP server. 445.Pp 446If this variable is set to an empty string, no proxy will be used for 447FTP requests, even if the 448.Ev HTTP_PROXY 449variable is set. 450.It Ev HTTP_AUTH 451Specifies HTTP authorization parameters as a colon-separated list of 452items. 453The first and second item are the authorization scheme and realm 454respectively; further items are scheme-dependent. 455Currently, only basic authorization is supported. 456.Pp 457Basic authorization requires two parameters: the user name and 458password, in that order. 459.Pp 460This variable is only used if the server requires authorization and 461no user name or password was specified in the URL. 462.It Ev HTTP_PROXY 463URL of the proxy to use for HTTP requests. 464The document part is ignored. 465Only HTTP proxies are supported for HTTP requests. 466If no port number is specified, the default is 3128. 467.Pp 468Note that this proxy will also be used for FTP documents, unless the 469.Ev FTP_PROXY 470variable is set. 471.It Ev HTTP_PROXY_AUTH 472Specifies authorization parameters for the HTTP proxy in the same 473format as the 474.Ev HTTP_AUTH 475variable. 476.Pp 477This variable is used if and only if connected to an HTTP proxy, and 478is ignored if a user and/or a password were specified in the proxy 479URL. 480.El 481.Sh SEE ALSO 482.Xr fetch 1 , 483.Xr ftpio 3 , 484.Xr ip 4 . 485.Rs 486.%A T. Berners-Lee 487.%A L. Masinter 488.%A M. McCahill 489.%D December 1994 490.%T Uniform Resource Locators (URL) 491.%O RFC1738 492.Re 493.Rs 494.%A R. Fielding 495.%A J. Gettys 496.%A J. Mogul 497.%A H. Frystyk 498.%A T. Berners-Lee 499.%D Januray 1997 500.%B Hypertext Transfer Protocol -- HTTP/1.1 501.%O RFC2068 502.Re 503.Rs 504.%A J. Postel 505.%A J. K. Reynolds 506.%D October 1985 507.%B File Transfer Protocol 508.%O RFC959 509.Re 510.Sh HISTORY 511The 512.Nm fetch 513library first appeared in 514.Fx 3.0 . 515.Sh AUTHORS 516.An -nosplit 517The 518.Nm fetch 519library was mostly written by 520.An Dag-Erling Co\(:idan Sm\(/orgrav Aq des@FreeBSD.org 521with numerous suggestions from 522.An Jordan K. Hubbard Aq jkh@FreeBSD.org , 523.An Eugene Skepner Aq eu@qub.com 524and other 525.Fx 526developers. 527It replaces the older 528.Nm ftpio 529library written by 530.An Poul-Henning Kamp Aq pkh@FreeBSD.org 531and 532.An Jordan K. Hubbard Aq jkh@FreeBSD.org . 533.Pp 534This manual page was written by 535.An Dag-Erling Co\(:idan Sm\(/orgrav Aq des@FreeBSD.org . 536.Sh BUGS 537Some parts of the library are not yet implemented. 538The most notable 539examples of this are 540.Fn fetchPutHTTP , 541.Fn fetchListHTTP , 542.Fn fetchListFTP 543and FTP proxy support. 544.Pp 545There's no way to select a proxy at run-time other than setting the 546.Ev HTTP_PROXY 547or 548.Ev FTP_PROXY 549environment variables as appropriate. 550.Pp 551.Nm libfetch 552does not attempt to interpret and respond to authentication requests 553from the HTTP server or proxy (code 401 and 407 respectively). 554.Pp 555.Nm libfetch 556does not understand or obey 305 (Use Proxy) replies. 557.Pp 558No attempt is made to encode spaces etc. within URLs. 559Spaces in the 560document part of an URLshould be replaced with "%20" in HTTP URLs and 561"\\ " in FTP URLs. 562.Pp 563Error numbers are unique only within a certain context; the error 564codes used for FTP and HTTP overlap, as do those used for resolver and 565system errors. 566For instance, error code 202 means "Command not 567implemented, superfluous at this site" in an FTP context and 568"Accepted" in an HTTP context. 569.Pp 570.Fn fetchStatFTP 571does not check that the result of an MDTM command is a valid date. 572.Pp 573The HTTP code needs a complete rewrite, or at least a serious cleanup. 574.Pp 575The man page is incomplete, poorly written and produces badly 576formatted text. 577.Pp 578The error reporting mechanism is unsatisfactory. 579.Pp 580Some parts of the code are not fully reentrant. 581.Pp 582Tons of other stuff. 583