xref: /freebsd/lib/libfetch/fetch.3 (revision 6990ffd8a95caaba6858ad44ff1b3157d1efba8f)
1.\" Copyright (c) 1998 Dag-Erling Co�dan Sm�rgrav
2.\" All rights reserved.
3.\"
4.\" Redistribution and use in source and binary forms, with or without
5.\" modification, are permitted provided that the following conditions
6.\" are met:
7.\" 1. Redistributions of source code must retain the above copyright
8.\"    notice, this list of conditions and the following disclaimer.
9.\" 2. Redistributions in binary form must reproduce the above copyright
10.\"    notice, this list of conditions and the following disclaimer in the
11.\"    documentation and/or other materials provided with the distribution.
12.\"
13.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23.\" SUCH DAMAGE.
24.\"
25.\" $FreeBSD$
26.\"
27.Dd July 1, 1998
28.Dt FETCH 3
29.Os
30.Sh NAME
31.Nm fetchMakeURL ,
32.Nm fetchParseURL ,
33.Nm fetchFreeURL ,
34.Nm fetchXGetURL ,
35.Nm fetchGetURL ,
36.Nm fetchPutURL ,
37.Nm fetchStatURL ,
38.Nm fetchListURL ,
39.Nm fetchXGet ,
40.Nm fetchGet ,
41.Nm fetchPut ,
42.Nm fetchStat ,
43.Nm fetchList ,
44.Nm fetchXGetFile ,
45.Nm fetchGetFile ,
46.Nm fetchPutFile ,
47.Nm fetchStatFile ,
48.Nm fetchListFile ,
49.Nm fetchXGetHTTP ,
50.Nm fetchGetHTTP ,
51.Nm fetchPutHTTP ,
52.Nm fetchStatHTTP ,
53.Nm fetchListHTTP ,
54.Nm fetchXGetFTP ,
55.Nm fetchGetFTP ,
56.Nm fetchPutFTP ,
57.Nm fetchStatFTP ,
58.Nm fetchListFTP
59.Nd file transfer functions
60.Sh LIBRARY
61.Lb libfetch
62.Sh SYNOPSIS
63.Fd #include <sys/param.h>
64.Fd #include <stdio.h>
65.Fd #include <fetch.h>
66.Ft struct url *
67.Fn fetchMakeURL "const char *scheme" "const char *host" "int port" "const char *doc" "const char *user" "const char *pwd"
68.Ft struct url *
69.Fn fetchParseURL "const char *URL"
70.Ft void
71.Fn fetchFreeURL "struct url *u"
72.Ft FILE *
73.Fn fetchXGetURL "const char *URL" "struct url_stat *us" "const char *flags"
74.Ft FILE *
75.Fn fetchGetURL "const char *URL" "const char *flags"
76.Ft FILE *
77.Fn fetchPutURL "const char *URL" "const char *flags"
78.Ft int
79.Fn fetchStatURL "const char *URL" "struct url_stat *us" "const char *flags"
80.Ft struct url_ent *
81.Fn fetchListURL "const char *URL" "const char *flags"
82.Ft FILE *
83.Fn fetchXGet "struct url *u" "struct url_stat *us" "const char *flags"
84.Ft FILE *
85.Fn fetchGet "struct url *u" "const char *flags"
86.Ft FILE *
87.Fn fetchPut "struct url *u" "const char *flags"
88.Ft int
89.Fn fetchStat "struct url *u" "struct url_stat *us" "const char *flags"
90.Ft struct url_ent *
91.Fn fetchList "struct url *u" "const char *flags"
92.Ft FILE *
93.Fn fetchXGetFile "struct url *u" "struct url_stat *us" "const char *flags"
94.Ft FILE *
95.Fn fetchGetFile "struct url *u" "const char *flags"
96.Ft FILE *
97.Fn fetchPutFile "struct url *u" "const char *flags"
98.Ft int
99.Fn fetchStatFile "struct url *u" "struct url_stat *us" "const char *flags"
100.Ft struct url_ent *
101.Fn fetchListFile "struct url *u" "const char *flags"
102.Ft FILE *
103.Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "const char *flags"
104.Ft FILE *
105.Fn fetchGetHTTP "struct url *u" "const char *flags"
106.Ft FILE *
107.Fn fetchPutHTTP "struct url *u" "const char *flags"
108.Ft int
109.Fn fetchStatHTTP "struct url *u" "struct url_stat *us" "const char *flags"
110.Ft struct url_ent *
111.Fn fetchListHTTP "struct url *u" "const char *flags"
112.Ft FILE *
113.Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "const char *flags"
114.Ft FILE *
115.Fn fetchGetFTP "struct url *u" "const char *flags"
116.Ft FILE *
117.Fn fetchPutFTP "struct url *u" "const char *flags"
118.Ft int
119.Fn fetchStatFTP "struct url *u" "struct url_stat *us" "const char *flags"
120.Ft struct url_ent *
121.Fn fetchListFTP "struct url *u" "const char *flags"
122.Sh DESCRIPTION
123These functions implement a high-level library for retrieving and
124uploading files using Uniform Resource Locators (URLs).
125.Pp
126.Fn fetchParseURL
127takes a URL in the form of a null-terminated string and splits it into
128its components function according to the Common Internet Scheme Syntax
129detailed in RFC1738.
130A regular expression which produces this syntax is:
131.Bd -literal
132    <scheme>:(//(<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
133.Ed
134.Pp
135If the URL does not seem to begin with a scheme name, the following
136syntax is assumed:
137.Bd -literal
138    ((<user>(:<pwd>)?@)?<host>(:<port>)?)?/(<document>)?
139.Ed
140.Pp
141Note that some components of the URL are not necessarily relevant to
142all URL schemes.
143For instance, the file scheme only needs the <scheme> and <document>
144components.
145.Pp
146.Fn fetchMakeURL
147and
148.Fn fetchParseURL
149return a pointer to a
150.Vt url
151structure, which is defined as follows in
152.Aq Pa fetch.h :
153.Bd -literal
154#define URL_SCHEMELEN 16
155#define URL_USERLEN 256
156#define URL_PWDLEN 256
157
158struct url {
159    char	 scheme[URL_SCHEMELEN+1];
160    char	 user[URL_USERLEN+1];
161    char	 pwd[URL_PWDLEN+1];
162    char	 host[MAXHOSTNAMELEN+1];
163    int		 port;
164    char	*doc;
165    off_t	 offset;
166    size_t	 length;
167};
168.Ed
169.Pp
170The pointer returned by
171.Fn fetchMakeURL
172or
173.Fn fetchParseURL
174should be freed using
175.Fn fetchFreeURL .
176.Pp
177.Fn fetchXGetURL ,
178.Fn fetchGetURL ,
179and
180.Fn fetchPutURL
181constitute the recommended interface to the
182.Nm fetch
183library.
184They examine the URL passed to them to determine the transfer
185method, and call the appropriate lower-level functions to perform the
186actual transfer.
187.Fn fetchXGetURL
188also returns the remote document's metadata in the
189.Vt url_stat
190structure pointed to by the
191.Fa us
192argument.
193.Pp
194The
195.Fa flags
196argument is a string of characters which specify transfer options.
197The
198meaning of the individual flags is scheme-dependent, and is detailed
199in the appropriate section below.
200.Pp
201.Fn fetchStatURL
202attempts to obtain the requested document's metadata and fill in the
203structure pointed to by its second argument.
204The
205.Vt url_stat
206structure is defined as follows in
207.Aq Pa fetch.h :
208.Bd -literal
209struct url_stat {
210    off_t	 size;
211    time_t	 atime;
212    time_t	 mtime;
213};
214.Ed
215.Pp
216If the size could not be obtained from the server, the
217.Fa size
218field is set to -1.
219If the modification time could not be obtained from the server, the
220.Fa mtime
221field is set to the epoch.
222If the access time could not be obtained from the server, the
223.Fa atime
224field is set to the modification time.
225.Pp
226.Fn fetchListURL
227attempts to list the contents of the directory pointed to by the URL
228provided.
229If successful, it returns a malloced array of
230.Vt url_ent
231structures.
232The
233.Vt url_ent
234structure is defined as follows in
235.Aq Pa fetch.h :
236.Bd -literal
237struct url_ent {
238    char         name[MAXPATHLEN];
239    struct url_stat stat;
240};
241.Ed
242.Pp
243The list is terminated by an entry with an empty name.
244.Pp
245The pointer returned by
246.Fn fetchListURL
247should be freed using
248.Fn free .
249.Pp
250.Fn fetchXGet ,
251.Fn fetchGet ,
252.Fn fetchPut
253and
254.Fn fetchStat
255are similar to
256.Fn fetchXGetURL ,
257.Fn fetchGetURL ,
258.Fn fetchPutURL
259and
260.Fn fetchStatURL ,
261except that they expect a pre-parsed URL in the form of a pointer to
262a
263.Vt struct url
264rather than a string.
265.Pp
266All of the
267.Fn fetchXGetXXX ,
268.Fn fetchGetXXX
269and
270.Fn fetchPutXXX
271functions return a pointer to a stream which can be used to read or
272write data from or to the requested document, respectively.
273Note that
274although the implementation details of the individual access methods
275vary, it can generally be assumed that a stream returned by one of the
276.Fn fetchXGetXXX
277or
278.Fn fetchGetXXX
279functions is read-only, and that a stream returned by one of the
280.Fn fetchPutXXX
281functions is write-only.
282.Sh FILE SCHEME
283.Fn fetchXGetFile ,
284.Fn fetchGetFile
285and
286.Fn fetchPutFile
287provide access to documents which are files in a locally mounted file
288system.
289Only the <document> component of the URL is used.
290.Pp
291.Fn fetchXGetFile
292and
293.Fn fetchGetFile
294do not accept any flags.
295.Pp
296.Fn fetchPutFile
297accepts the
298.Ql a
299(append to file) flag.
300If that flag is specified, the data written to
301the stream returned by
302.Fn fetchPutFile
303will be appended to the previous contents of the file, instead of
304replacing them.
305.Sh FTP SCHEME
306.Fn fetchXGetFTP ,
307.Fn fetchGetFTP
308and
309.Fn fetchPutFTP
310implement the FTP protocol as described in RFC959.
311.Pp
312If the
313.Ql p
314(passive) flag is specified, a passive (rather than active) connection
315will be attempted.
316.Pp
317If the
318.Ql l
319(low) flag is specified, data sockets will be allocated in the low (or
320default) port range instead of the hifh port range (see
321.Xr ip 4 ) .
322.Pp
323If the
324.Ql d
325(direct) flag is specified,
326.Fn fetchXGetFTP ,
327.Fn fetchGetFTP
328and
329.Fn fetchPutFTP
330will use a direct connection even if a proxy server is defined.
331.Pp
332If no user name or password is given, the
333.Nm fetch
334library will attempt an anonymous login, with user name "anonymous"
335and password "anonymous@<hostname>".
336.Sh HTTP SCHEME
337The
338.Fn fetchXGetHTTP ,
339.Fn fetchGetHTTP
340and
341.Fn fetchPutHTTP
342functions implement the HTTP/1.1 protocol.
343With a little luck, there's
344even a chance that they comply with RFC2616 and RFC2617.
345.Pp
346If the
347.Ql d
348(direct) flag is specified,
349.Fn fetchXGetHTTP ,
350.Fn fetchGetHTTP
351and
352.Fn fetchPutHTTP
353will use a direct connection even if a proxy server is defined.
354.Pp
355Since there seems to be no good way of implementing the HTTP PUT
356method in a manner consistent with the rest of the
357.Nm fetch
358library,
359.Fn fetchPutHTTP
360is currently unimplemented.
361.Sh AUTHENTICATION
362Apart from setting the appropriate environment variables and
363specifying the user name and password in the URL or the
364.Vt struct url ,
365the calling program has the option of defining an authentication
366function with the following prototype:
367.Pp
368.Ft int
369.Fn myAuthMethod "struct url *u"
370.Pp
371The callback function should fill in the
372.Fa user
373and
374.Fa pwd
375fields in the provided
376.Vt struct url
377and return 0 on success, or any other value to indicate failure.
378.Pp
379To register the authentication callback, simply set
380.Va fetchAuthMethod
381to point at it.
382The callback will be used whenever a site requires authentication and
383the appropriate environment variables aren't set.
384.Pp
385This interface is experimental and may be subject to change.
386.Sh RETURN VALUES
387.Fn fetchParseURL
388returns a pointer to a
389.Vt struct url
390containing the individual components of the URL.
391If it is
392unable to allocate memory, or the URL is syntactically incorrect,
393.Fn fetchParseURL
394returns a NULL pointer.
395.Pp
396The
397.Fn fetchStat
398functions return 0 on success and -1 on failure.
399.Pp
400All other functions return a stream pointer which may be used to
401access the requested document, or NULL if an error occurred.
402.Pp
403The following error codes are defined in
404.Aq Pa fetch.h :
405.Bl -tag -width 18n
406.It Bq Er FETCH_ABORT
407Operation aborted
408.It Bq Er FETCH_AUTH
409Authentication failed
410.It Bq Er FETCH_DOWN
411Service unavailable
412.It Bq Er FETCH_EXISTS
413File exists
414.It Bq Er FETCH_FULL
415File system full
416.It Bq Er FETCH_INFO
417Informational response
418.It Bq Er FETCH_MEMORY
419Insufficient memory
420.It Bq Er FETCH_MOVED
421File has moved
422.It Bq Er FETCH_NETWORK
423Network error
424.It Bq Er FETCH_OK
425No error
426.It Bq Er FETCH_PROTO
427Protocol error
428.It Bq Er FETCH_RESOLV
429Resolver error
430.It Bq Er FETCH_SERVER
431Server error
432.It Bq Er FETCH_TEMP
433Temporary error
434.It Bq Er FETCH_TIMEOUT
435Operation timed out
436.It Bq Er FETCH_UNAVAIL
437File is not available
438.It Bq Er FETCH_UNKNOWN
439Unknown error
440.It Bq Er FETCH_URL
441Invalid URL
442.El
443.Pp
444The accompanying error message includes a protocol-specific error code
445and message, e.g. "File is not available (404 Not Found)"
446.Sh ENVIRONMENT
447.Bl -tag -width FTP_PASSIVE_MODE
448.It Ev FTP_LOGIN
449Default FTP login if none was provided in the URL.
450.It Ev FTP_PASSIVE_MODE
451If set to anything but
452.Ql no ,
453forces the FTP code to use passive mode.
454.It Ev FTP_PASSWORD
455Default FTP password if the remote server requests one and none was
456provided in the URL.
457.It Ev FTP_PROXY
458URL of the proxy to use for FTP requests.
459The document part is ignored.
460FTP and HTTP proxies are supported; if no scheme is specified, FTP is
461assumed.
462If the proxy is an FTP proxy,
463.Nm libfetch
464will send
465.Ql user@host
466as user name to the proxy, where
467.Ql user
468is the real user name, and
469.Ql host
470is the name of the FTP server.
471.Pp
472If this variable is set to an empty string, no proxy will be used for
473FTP requests, even if the
474.Ev HTTP_PROXY
475variable is set.
476.It Ev ftp_proxy
477Same as
478.Ev FTP_PROXY ,
479for compatibility.
480.It Ev HTTP_AUTH
481Specifies HTTP authorization parameters as a colon-separated list of
482items.
483The first and second item are the authorization scheme and realm
484respectively; further items are scheme-dependent.
485Currently, only basic authorization is supported.
486.Pp
487Basic authorization requires two parameters: the user name and
488password, in that order.
489.Pp
490This variable is only used if the server requires authorization and
491no user name or password was specified in the URL.
492.It Ev HTTP_PROXY
493URL of the proxy to use for HTTP requests.
494The document part is ignored.
495Only HTTP proxies are supported for HTTP requests.
496If no port number is specified, the default is 3128.
497.Pp
498Note that this proxy will also be used for FTP documents, unless the
499.Ev FTP_PROXY
500variable is set.
501.It Ev http_proxy
502Same as
503.Ev HTTP_PROXY ,
504for compatibility.
505.It Ev HTTP_PROXY_AUTH
506Specifies authorization parameters for the HTTP proxy in the same
507format as the
508.Ev HTTP_AUTH
509variable.
510.Pp
511This variable is used if and only if connected to an HTTP proxy, and
512is ignored if a user and/or a password were specified in the proxy
513URL.
514.It Ev HTTP_USER_AGENT
515Specifies the User-Agent string to use for HTTP requests.
516This can be useful when working with HTTP origin or proxy servers that
517differentiate between user agents.
518.El
519.Sh SEE ALSO
520.Xr fetch 1 ,
521.Xr ftpio 3 ,
522.Xr ip 4
523.Rs
524.%A J. Postel
525.%A J. K. Reynolds
526.%D October 1985
527.%B File Transfer Protocol
528.%O RFC959
529.Re
530.Rs
531.%A P. Deutsch
532.%A A. Emtage
533.%A A. Marine.
534.%D May 1994
535.%T How to Use Anonymous FTP
536.%O RFC1635
537.Re
538.Rs
539.%A T. Berners-Lee
540.%A L. Masinter
541.%A M. McCahill
542.%D December 1994
543.%T Uniform Resource Locators (URL)
544.%O RFC1738
545.Re
546.Rs
547.%A R. Fielding
548.%A J. Gettys
549.%A J. Mogul
550.%A H. Frystyk
551.%A L. Masinter
552.%A P. Leach
553.%A T. Berners-Lee
554.%D Januray 1999
555.%B Hypertext Transfer Protocol -- HTTP/1.1
556.%O RFC2616
557.Re
558.Rs
559.%A J. Franks
560.%A P. Hallam-Baker
561.%A J. Hostetler
562.%A S. Lawrence
563.%A P. Leach
564.%A A. Luotonen
565.%A L. Stewart
566.%D June 1999
567.%B HTTP Authentication: Basic and Digest Access Authentication
568.%O RFC2617
569.Re
570.Sh HISTORY
571The
572.Nm fetch
573library first appeared in
574.Fx 3.0 .
575.Sh AUTHORS
576.An -nosplit
577The
578.Nm fetch
579library was mostly written by
580.An Dag-Erling Co\(:idan Sm\(/orgrav Aq des@FreeBSD.org
581with numerous suggestions from
582.An Jordan K. Hubbard Aq jkh@FreeBSD.org ,
583.An Eugene Skepner Aq eu@qub.com
584and other
585.Fx
586developers.
587It replaces the older
588.Nm ftpio
589library written by
590.An Poul-Henning Kamp Aq pkh@FreeBSD.org
591and
592.An Jordan K. Hubbard Aq jkh@FreeBSD.org .
593.Pp
594This manual page was written by
595.An Dag-Erling Co\(:idan Sm\(/orgrav Aq des@FreeBSD.org .
596.Sh BUGS
597Some parts of the library are not yet implemented.
598The most notable
599examples of this are
600.Fn fetchPutHTTP ,
601.Fn fetchListHTTP ,
602.Fn fetchListFTP
603and FTP proxy support.
604.Pp
605There's no way to select a proxy at run-time other than setting the
606.Ev HTTP_PROXY
607or
608.Ev FTP_PROXY
609environment variables as appropriate.
610.Pp
611.Nm libfetch
612does not understand or obey 305 (Use Proxy) replies.
613.Pp
614Error numbers are unique only within a certain context; the error
615codes used for FTP and HTTP overlap, as do those used for resolver and
616system errors.
617For instance, error code 202 means "Command not
618implemented, superfluous at this site" in an FTP context and
619"Accepted" in an HTTP context.
620.Pp
621.Fn fetchStatFTP
622does not check that the result of an MDTM command is a valid date.
623.Pp
624The man page is incomplete, poorly written and produces badly
625formatted text.
626.Pp
627The error reporting mechanism is unsatisfactory.
628.Pp
629Some parts of the code are not fully reentrant.
630