1.\" Copyright (c) 1998 Dag-Erling Co�dan Sm�rgrav 2.\" All rights reserved. 3.\" 4.\" Redistribution and use in source and binary forms, with or without 5.\" modification, are permitted provided that the following conditions 6.\" are met: 7.\" 1. Redistributions of source code must retain the above copyright 8.\" notice, this list of conditions and the following disclaimer. 9.\" 2. Redistributions in binary form must reproduce the above copyright 10.\" notice, this list of conditions and the following disclaimer in the 11.\" documentation and/or other materials provided with the distribution. 12.\" 13.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23.\" SUCH DAMAGE. 24.\" 25.\" $FreeBSD$ 26.\" 27.Dd July 1, 1998 28.Dt FETCH 3 29.Os 30.Sh NAME 31.Nm fetchMakeURL , 32.Nm fetchParseURL , 33.Nm fetchFreeURL , 34.Nm fetchXGetURL , 35.Nm fetchGetURL , 36.Nm fetchPutURL , 37.Nm fetchStatURL , 38.Nm fetchListURL , 39.Nm fetchXGet , 40.Nm fetchGet , 41.Nm fetchPut , 42.Nm fetchStat , 43.Nm fetchList , 44.Nm fetchXGetFile , 45.Nm fetchGetFile , 46.Nm fetchPutFile , 47.Nm fetchStatFile , 48.Nm fetchListFile , 49.Nm fetchXGetHTTP , 50.Nm fetchGetHTTP , 51.Nm fetchPutHTTP , 52.Nm fetchStatHTTP , 53.Nm fetchListHTTP , 54.Nm fetchXGetFTP , 55.Nm fetchGetFTP , 56.Nm fetchPutFTP , 57.Nm fetchStatFTP , 58.Nm fetchListFTP 59.Nd file transfer functions 60.Sh LIBRARY 61.Lb libfetch 62.Sh SYNOPSIS 63.Fd #include <sys/param.h> 64.Fd #include <stdio.h> 65.Fd #include <fetch.h> 66.Ft struct url * 67.Fn fetchMakeURL "char *scheme" "char *host" "int port" "char *doc" "char *user" "char *pwd" 68.Ft struct url * 69.Fn fetchParseURL "char *URL" 70.Ft void 71.Fn fetchFreeURL "struct url *URL" 72.Ft FILE * 73.Fn fetchXGetURL "char *URL" "struct url_stat *us" "char *flags" 74.Ft FILE * 75.Fn fetchGetURL "char *URL" "char *flags" 76.Ft FILE * 77.Fn fetchPutURL "char *URL" "char *flags" 78.Ft int 79.Fn fetchStatURL "char *URL" "struct url_stat *us" "char *flags" 80.Ft struct url_ent * 81.Fn fetchListURL "char *URL" "char *flags" 82.Ft FILE * 83.Fn fetchXGet "struct url *URL" "struct url_stat *us" "char *flags" 84.Ft FILE * 85.Fn fetchGet "struct url *URL" "char *flags" 86.Ft FILE * 87.Fn fetchPut "struct url *URL" "char *flags" 88.Ft int 89.Fn fetchStat "struct url *URL" "struct url_stat *us" "char *flags" 90.Ft struct url_ent * 91.Fn fetchList "struct url *" "char *flags" 92.Ft FILE * 93.Fn fetchXGetFile "struct url *u" "struct url_stat *us" "char *flags" 94.Ft FILE * 95.Fn fetchGetFile "struct url *u" "char *flags" 96.Ft FILE * 97.Fn fetchPutFile "struct url *u" "char *flags" 98.Ft int 99.Fn fetchStatFile "struct url *URL" "struct url_stat *us" "char *flags" 100.Ft struct url_ent * 101.Fn fetchListFile "struct url *" "char *flags" 102.Ft FILE * 103.Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "char *flags" 104.Ft FILE * 105.Fn fetchGetHTTP "struct url *u" "char *flags" 106.Ft FILE * 107.Fn fetchPutHTTP "struct url *u" "char *flags" 108.Ft int 109.Fn fetchStatHTTP "struct url *URL" "struct url_stat *us" "char *flags" 110.Ft struct url_ent * 111.Fn fetchListHTTP "struct url *" "char *flags" 112.Ft FILE * 113.Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "char *flags" 114.Ft FILE * 115.Fn fetchGetFTP "struct url *u" "char *flags" 116.Ft FILE * 117.Fn fetchPutFTP "struct url *u" "char *flags" 118.Ft int 119.Fn fetchStatFTP "struct url *URL" "struct url_stat *us" "char *flags" 120.Ft struct url_ent * 121.Fn fetchListFTP "struct url *" "char *flags" 122.Sh DESCRIPTION 123.Pp 124These functions implement a high-level library for retrieving and 125uploading files using Uniform Resource Locators (URLs). 126.Pp 127.Fn fetchParseURL 128takes a URL in the form of a null-terminated string and splits it into 129its components function according to the Common Internet Scheme Syntax 130detailed in RFC1738. 131A regular expression which produces this syntax is: 132.Bd -literal 133 <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)? 134.Ed 135.Pp 136Note that some components of the URL are not necessarily relevant to 137all URL schemes. 138For instance, the file scheme only needs the <scheme> 139and <document> components. 140.Pp 141.Fn fetchMakeURL 142and 143.Fn fetchParseURL 144return a pointer to a 145.Fa url 146structure, which is defined as follows in 147.Aq Pa fetch.h : 148.Bd -literal 149#define URL_SCHEMELEN 16 150#define URL_USERLEN 256 151#define URL_PWDLEN 256 152 153struct url { 154 char scheme[URL_SCHEMELEN+1]; 155 char user[URL_USERLEN+1]; 156 char pwd[URL_PWDLEN+1]; 157 char host[MAXHOSTNAMELEN+1]; 158 int port; 159 char *doc; 160 off_t offset; 161 size_t length; 162}; 163.Ed 164.Pp 165The pointer returned by 166.Fn fetchMakeURL 167or 168.Fn fetchParseURL 169should be freed using 170.Fn fetchFreeURL . 171.Pp 172.Fn fetchXGetURL , 173.Fn fetchGetURL , 174and 175.Fn fetchPutURL 176constitute the recommended interface to the 177.Nm fetch 178library. 179They examine the URL passed to them to determine the transfer 180method, and call the appropriate lower-level functions to perform the 181actual transfer. 182.Fn fetchXGetURL 183also returns the remote document's metadata in the 184.Fa url_stat 185structure pointed to by the 186.Fa us 187argument. 188.Pp 189The 190.Fa flags 191argument is a string of characters which specify transfer options. 192The 193meaning of the individual flags is scheme-dependent, and is detailed 194in the appropriate section below. 195.Pp 196.Fn fetchStatURL 197attempts to obtain the requested document's metadata and fill in the 198structure pointed to by it's second argument. 199The 200.Fa url_stat 201structure is defined as follows in 202.Aq Pa fetch.h : 203.Bd -literal 204struct url_stat { 205 off_t size; 206 time_t atime; 207 time_t mtime; 208}; 209.Ed 210.Pp 211If the size could not be obtained from the server, the 212.Fa size 213field is set to -1. 214If the modification time could not be obtained from the server, the 215.Fa mtime 216field is set to the epoch. 217If the access time could not be obtained from the server, the 218.Fa atime 219field is set to the modification time. 220.Pp 221.Fn fetchListURL 222attempts to list the contents of the directory pointed to by the URL 223provided. 224If successful, it returns a malloced array of 225.Fa url_ent 226structures. 227The 228.Fa url_ent 229structure is defined as follows in 230.Aq Pa fetch.h : 231.Bd -literal 232struct url_ent { 233 char name[MAXPATHLEN]; 234 struct url_stat stat; 235}; 236.Ed 237.Pp 238The list is terminated by an entry with an empty name. 239.Pp 240The pointer returned by 241.Fn fetchListURL 242should be freed using 243.Fn free . 244.Pp 245.Fn fetchXGet , 246.Fn fetchGet , 247.Fn fetchPut 248and 249.Fn fetchStat 250are similar to 251.Fn fetchXGetURL , 252.Fn fetchGetURL , 253.Fn fetchPutURL 254and 255.Fn fetchStatURL , 256except that they expect a pre-parsed URL in the form of a pointer to 257a 258.Fa struct url 259rather than a string. 260.Pp 261All of the 262.Fn fetchXGetXXX , 263.Fn fetchGetXXX 264and 265.Fn fetchPutXXX 266functions return a pointer to a stream which can be used to read or 267write data from or to the requested document, respectively. 268Note that 269although the implementation details of the individual access methods 270vary, it can generally be assumed that a stream returned by one of the 271.Fn fetchXGetXXX 272or 273.Fn fetchGetXXX 274functions is read-only, and that a stream returned by one of the 275.Fn fetchPutXXX 276functions is write-only. 277.Sh FILE SCHEME 278.Fn fetchXGetFile , 279.Fn fetchGetFile 280and 281.Fn fetchPutFile 282provide access to documents which are files in a locally mounted file 283system. 284Only the <document> component of the URL is used. 285.Pp 286.Fn fetchXGetFile 287and 288.Fn fetchGetFile 289do not accept any flags. 290.Pp 291.Fn fetchPutFile 292accepts the 293.Fa a 294(append to file) flag. 295If that flag is specified, the data written to 296the stream returned by 297.Fn fetchPutFile 298will be appended to the previous contents of the file, instead of 299replacing them. 300.Sh FTP SCHEME 301.Fn fetchXGetFTP , 302.Fn fetchGetFTP 303and 304.Fn fetchPutFTP 305implement the FTP protocol as described in RFC959. 306.Pp 307If the 308.Fa p 309(passive) flag is specified, a passive (rather than active) connection 310will be attempted. 311.Pp 312If the 313.Fa h 314(high) flag is specified, data sockets will be allocated in the high 315port range (see 316.Xr ip 4 ). 317.Pp 318If the 319.Fa d 320(direct) flag is specified, 321.Fn fetchXGetFTP , 322.Fn fetchGetFTP 323and 324.Fn fetchPutFTP 325will use a direct connection even if a proxy server is defined. 326.Pp 327If no user name or password is given, the 328.Nm fetch 329library will attempt an anonymous login, with user name "ftp" and 330password "ftp". 331.Sh HTTP SCHEME 332The 333.Fn fetchXGetHTTP , 334.Fn fetchGetHTTP 335and 336.Fn fetchPutHTTP 337functions implement the HTTP/1.1 protocol. 338With a little luck, there's 339even a chance that they comply with RFC2068. 340.Pp 341If the 342.Fa d 343(direct) flag is specified, 344.Fn fetchXGetHTTP , 345.Fn fetchGetHTTP 346and 347.Fn fetchPutHTTP 348will use a direct connection even if a proxy server is defined. 349.Pp 350Since there seems to be no good way of implementing the HTTP PUT 351method in a manner consistent with the rest of the 352.Nm fetch 353library, 354.Fn fetchPutHTTP 355is currently unimplemented. 356.Sh RETURN VALUES 357.Fn fetchParseURL 358returns a pointer to a 359.Fa struct url 360containing the individual components of the URL. 361If it is 362unable to allocate memory, or the URL is syntactically incorrect, 363.Fn fetchParseURL 364returns a NULL pointer. 365.Pp 366The 367.Fn fetchStat 368functions return 0 on success and -1 on failure. 369.Pp 370All other functions return a stream pointer which may be used to 371access the requested document, or NULL if an error occurred. 372.Pp 373The following error codes are defined in 374.Aq Pa fetch.h : 375.Bl -tag -width 18n 376.It Bq Er FETCH_ABORT 377Operation aborted 378.It Bq Er FETCH_AUTH 379Authentication failed 380.It Bq Er FETCH_DOWN 381Service unavailable 382.It Bq Er FETCH_EXISTS 383File exists 384.It Bq Er FETCH_FULL 385File system full 386.It Bq Er FETCH_INFO 387Informational response 388.It Bq Er FETCH_MEMORY 389Insufficient memory 390.It Bq Er FETCH_MOVED 391File has moved 392.It Bq Er FETCH_NETWORK 393Network error 394.It Bq Er FETCH_OK 395No error 396.It Bq Er FETCH_PROTO 397Protocol error 398.It Bq Er FETCH_RESOLV 399Resolver error 400.It Bq Er FETCH_SERVER 401Server error 402.It Bq Er FETCH_TEMP 403Temporary error 404.It Bq Er FETCH_TIMEOUT 405Operation timed out 406.It Bq Er FETCH_UNAVAIL 407File is not available 408.It Bq Er FETCH_UNKNOWN 409Unknown error 410.It Bq Er FETCH_URL 411Invalid URL 412.El 413.Pp 414The accompanying error message includes a protocol-specific error code 415and message, e.g. "File is not available (404 Not Found)" 416.Sh ENVIRONMENT 417.Bl -tag -width HTTP_PROXY_AUTH 418.It Ev FTP_PROXY 419host name of the FTP proxy to use, optionally followed by a port 420number separated from the host name by a colon. 421.It Ev HTTP_AUTH 422Specifies HTTP authorization parameters, used only if the server 423requires authorization and no user name or password was specified in 424the URL. 425The first and second item are the authorization scheme and realm 426respectively; further items are scheme-dependent. 427Currently, only basic authorization is supported. 428Basic authorization requires two parameters: the user name and 429password, in that order. 430.It Ev HTTP_PROXY 431host name of the HTTP proxy to use, optionally followed by a port 432number separated from the host name by a colon. 433If no port number is specified, the default is 3128. 434.It Ev HTTP_PROXY_AUTH 435Specifies authorization parameters for the HTTP proxy in the same 436format as 437.Ev HTTP_AUTH . 438The value of this variable is used if and only if connected to an HTTP 439proxy. 440.El 441.Sh SEE ALSO 442.Xr fetch 1 , 443.Xr ftpio 3 , 444.Xr ip 4 . 445.Rs 446.%A T. Berners-Lee 447.%A L. Masinter 448.%A M. McCahill 449.%D December 1994 450.%T Uniform Resource Locators (URL) 451.%O RFC1738 452.Re 453.Rs 454.%A R. Fielding 455.%A J. Gettys 456.%A J. Mogul 457.%A H. Frystyk 458.%A T. Berners-Lee 459.%D Januray 1997 460.%B Hypertext Transfer Protocol -- HTTP/1.1 461.%O RFC2068 462.Re 463.Rs 464.%A J. Postel 465.%A J. K. Reynolds 466.%D October 1985 467.%B File Transfer Protocol 468.%O RFC959 469.Re 470.Sh HISTORY 471The 472.Nm fetch 473library first appeared in 474.Fx 3.0 . 475.Sh AUTHORS 476The 477.Nm fetch 478library was mostly written by 479.An Dag-Erling Co�dan Sm�rgrav Aq des@FreeBSD.org 480with numerous suggestions from 481.An Jordan K. Hubbard Aq jkh@FreeBSD.org , 482.An Eugene Skepner Aq eu@qub.com 483and other FreeBSD developers. 484It replaces the older 485.Nm ftpio 486library written by 487.An Poul-Henning Kamp Aq pkh@FreeBSD.org 488and 489.An Jordan K. Hubbard Aq jkh@FreeBSD.org . 490.Pp 491This manual page was written by 492.An Dag-Erling Co�dan Sm�rgrav Aq des@FreeBSD.org 493.Sh BUGS 494Some parts of the library are not yet implemented. 495The most notable 496examples of this are 497.Fn fetchPutHTTP , 498.Fn fetchListHTTP , 499.Fn fetchListFTP 500and FTP proxy support. 501.Pp 502There's no way to select a proxy at run-time other than setting the 503.Ev HTTP_PROXY 504or 505.Ev FTP_PROXY 506environment variables as appropriate. 507.Pp 508.Nm libfetch 509does not attempt to interpret and respond to authentication requests 510from the HTTP server or proxy (code 401 and 407 respectively). 511.Pp 512.Nm libfetch 513does not understand or obey 305 (Use Proxy) replies. 514.Pp 515No attempt is made to encode spaces etc. within URLs. 516Spaces in the 517document part of an URLshould be replaced with "%20" in HTTP URLs and 518"\\ " in FTP URLs. 519.Pp 520Error numbers are unique only within a certain context; the error 521codes used for FTP and HTTP overlap, as do those used for resolver and 522system errors. 523For instance, error code 202 means "Command not 524implemented, superfluous at this site" in an FTP context and 525"Accepted" in an HTTP context. 526.Pp 527.Fn fetchStatFTP 528does not check that the result of an MDTM command is a valid date. 529.Pp 530The HTTP code needs a complete rewrite, or at least a serious cleanup. 531.Pp 532The man page is incomplete, poorly written and produces badly 533formatted text. 534.Pp 535The error reporting mechanism is unsatisfactory. 536.Pp 537Some parts of the code are not fully reentrant. 538.Pp 539Tons of other stuff. 540