1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <stdio.h>
26 #include <stdlib.h>
27 #include <string.h>
28 #include <strings.h>
29 #include <sys/types.h>
30 #include <errno.h>
31 #include <syslog.h>
32 #include <unistd.h>
33 #include <sys/types.h>
34 #include <sys/socket.h>
35 #include <sys/time.h>
36 #include <netinet/in.h>
37 #include <arpa/inet.h>
38 #include <netdb.h>
39 #include <sys/stat.h>
40 #include <sys/sdt.h>
41 #include <signal.h>
42 #include <fcntl.h>
43 #include <libnvpair.h>
44 #include <libstmf.h>
45 #include <door.h>
46 #include <pthread.h>
47 #include <libscf.h>
48 #include <locale.h>
49 #include <sys/stmf_ioctl.h>
50 #include <sys/pppt_ioctl.h>
51 #include <libstmfproxy.h>
52
53 #define PPPT_NODE "/devices/pseudo/pppt@0:pppt"
54 #define USAGE "Usage: %s [-d][-f][-n nodeid] nodename\n" \
55 "Note: nodename must be the same on both nodes\n"
56
57
58 /*
59 * static functions
60 */
61 static void daemonInit(void);
62 static void killHandler();
63 static int postMsg(uint_t nelem, uchar_t *aluaMsg);
64
65
66 /*
67 * globals
68 */
69 void *t_handle; /* transport handle */
70 char aluaNode[256]; /* one of the two alua peers */
71 char myNode[256]; /* this hostname */
72 int log_debug = 0;
73 int fore_ground = 0;
74 int proxy_hdl;
75 pt_ops_t *pt_ops;
76 pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER;
77
78 /*
79 * killHandler
80 *
81 * Terminates this process on SIGQUIT, SIGINT, SIGTERM
82 */
83 /* ARGSUSED */
84 static void
killHandler(int sig)85 killHandler(int sig)
86 {
87 exit(0);
88 }
89
90 /*
91 * doorHandler
92 *
93 * Recieve data from the local proxy port provider and relay
94 * it to the peer node.
95 */
96 /* ARGSUSED */
97 void
doorHandler(void * cookie,char * args,size_t alen,door_desc_t * ddp,uint_t ndid)98 doorHandler(
99 void *cookie,
100 char *args,
101 size_t alen,
102 door_desc_t *ddp,
103 uint_t ndid)
104 {
105 uint32_t result = 0;
106
107 if (ddp != NULL || ndid != 0) {
108 syslog(LOG_DAEMON|LOG_WARNING,
109 "descriptor passed to door %p %d", ddp, ndid);
110 result = EINVAL;
111 }
112
113 if (args == NULL || alen == 0) {
114 syslog(LOG_DAEMON|LOG_WARNING,
115 "empty message passed to door %p %d", args, alen);
116 result = EFAULT;
117 }
118
119 if (result == 0)
120 result = postMsg((uint_t)alen, (uchar_t *)args);
121 (void) door_return((char *)&result, sizeof (result), NULL, 0);
122
123 syslog(LOG_DAEMON|LOG_WARNING, "door_return FAILED %d", errno);
124 exit(errno);
125 }
126
127 static int
postMsg(uint_t nelem,uchar_t * aluaMsg)128 postMsg(uint_t nelem, uchar_t *aluaMsg)
129 {
130 uint32_t buflen;
131 uchar_t *buf;
132 int ret = 0;
133 int ns;
134
135 if (t_handle == NULL) {
136 syslog(LOG_DAEMON|LOG_WARNING,
137 "postMsg() no transport handle");
138 exit(1);
139 }
140
141 buf = malloc(nelem + sizeof (buflen));
142
143 buflen = htonl(nelem); /* length in network byte order */
144 bcopy(&buflen, buf, sizeof (buflen));
145 bcopy(aluaMsg, buf + sizeof (buflen), nelem);
146
147 (void) pthread_mutex_lock(&send_mutex);
148 ns = pt_ops->stmf_proxy_send(t_handle, buf, nelem + sizeof (buflen));
149 (void) pthread_mutex_unlock(&send_mutex);
150 if (ns != nelem + sizeof (buflen)) {
151 ret = errno;
152 if (ret == 0)
153 ret = ENOTTY; /* something bogus */
154 syslog(LOG_DAEMON|LOG_CRIT, "send() call failed: %d", ret);
155 }
156 free(buf);
157 return (ret);
158 }
159
160 /*
161 * Multi-thread the data path from the peer node to the local
162 * proxy port provider. During discover, there can be a large
163 * burst of messages from the peer node proportional to the number
164 * of LUs. Multiple threads allow these messages to be processed
165 * simultaneously.
166 */
167 typedef struct pppt_drv_queue {
168 struct pppt_drv_queue *next;
169 uint32_t buflen;
170 uchar_t *buf;
171 } pppt_drv_queue_t;
172
173 pppt_drv_queue_t *pq_head = NULL;
174 pthread_mutex_t pq_mutex = PTHREAD_MUTEX_INITIALIZER;
175 pthread_cond_t pq_cond = PTHREAD_COND_INITIALIZER;
176 int pq_num_threads = 0;
177 int pq_avail_threads = 0;
178
179 /*ARGSUSED*/
180 void *
push_to_drv(void * arg)181 push_to_drv(void *arg)
182 {
183 pppt_drv_queue_t *pq;
184 int rc;
185
186 (void) pthread_mutex_lock(&pq_mutex);
187 pq_num_threads++;
188 (void) pthread_mutex_unlock(&pq_mutex);
189 for (;;) {
190 (void) pthread_mutex_lock(&pq_mutex);
191 while (pq_head == NULL) {
192 pq_avail_threads++;
193 (void) pthread_cond_wait(&pq_cond, &pq_mutex);
194 pq_avail_threads--;
195 }
196 pq = pq_head;
197 pq_head = pq->next;
198 pq->next = NULL;
199 (void) pthread_mutex_unlock(&pq_mutex);
200 /* Relay the message to the local kernel */
201 rc = stmfPostProxyMsg(proxy_hdl, (void *)pq->buf, pq->buflen);
202 if (rc != STMF_STATUS_SUCCESS) {
203 /* XXX die ? */
204 syslog(LOG_DAEMON|LOG_CRIT, "ioctl failed - %d", errno);
205 }
206 free(pq->buf);
207 free(pq);
208 }
209 /*NOTREACHED*/
210 return (NULL);
211 }
212
213 /*
214 * Receive data from peer and queue it up for the proxy driver.
215 */
216 int message_count = 0;
217 static void
relay_peer_msg()218 relay_peer_msg()
219 {
220 uint32_t buflen;
221 pppt_drv_queue_t *pq, *tmpq;
222 pthread_t tid;
223 int rc;
224
225
226 /* first receive the length of the message */
227 if ((pt_ops->stmf_proxy_recv(t_handle, (uchar_t *)&buflen,
228 sizeof (buflen))) != sizeof (buflen)) {
229 syslog(LOG_DAEMON|LOG_WARNING, "recv() call failed: %d",
230 errno);
231 exit(1);
232 }
233
234 pq = malloc(sizeof (*pq));
235 pq->next = NULL;
236 pq->buflen = ntohl(buflen);
237 pq->buf = malloc(pq->buflen+4);
238 if (log_debug) {
239 syslog(LOG_DAEMON|LOG_DEBUG,
240 "recvMsg: size of buffer - %d", (int)pq->buflen);
241 }
242
243 if ((pt_ops->stmf_proxy_recv(t_handle, pq->buf, pq->buflen)) !=
244 pq->buflen) {
245 syslog(LOG_DAEMON|LOG_WARNING, "recv() call failed: %d",
246 errno);
247 exit(1);
248 }
249
250 /* Eat the first message from peer */
251 if (message_count++ == 0) {
252 *(pq->buf+pq->buflen) = 0;
253 free(pq->buf);
254 free(pq);
255 return;
256 }
257
258 /* Queue the message to the driver */
259 (void) pthread_mutex_lock(&pq_mutex);
260 if (pq_head == NULL) {
261 pq_head = pq;
262 } else {
263 /* add to the tail */
264 tmpq = pq_head;
265 while (tmpq->next != NULL)
266 tmpq = tmpq->next;
267 tmpq->next = pq;
268 }
269
270 /* Make sure there is a thread to service this message */
271 if (pq_avail_threads) {
272 /* wake an available thread */
273 (void) pthread_cond_signal(&pq_cond);
274 (void) pthread_mutex_unlock(&pq_mutex);
275 } else {
276 /* no threads available, create a new thread */
277 (void) pthread_mutex_unlock(&pq_mutex);
278 rc = pthread_create(&tid, NULL, push_to_drv, NULL);
279 if (rc != 0) {
280 syslog(LOG_DAEMON|LOG_WARNING,
281 "pthread_create() call failed: %d", rc);
282 if (pq_num_threads == 0) {
283 /* never created a thread */
284 exit(rc);
285 }
286 }
287 }
288 }
289
290 /*
291 * Initialization for a daemon process
292 */
293 static void
daemonInit(void)294 daemonInit(void)
295 {
296 pid_t pid;
297 int devnull;
298
299 if (fore_ground)
300 return;
301
302 if ((pid = fork()) < 0) {
303 syslog(LOG_DAEMON|LOG_CRIT, "Could not fork(). Exiting");
304 exit(1);
305 } else if (pid != 0) {
306 /*
307 * XXX
308 * Simple approach for now - let the service go online.
309 * Later, set-up a pipe to the child and wait until the
310 * child indicates service is setup.
311 */
312 exit(SMF_EXIT_OK);
313 }
314
315 (void) setsid();
316
317 (void) chdir("/");
318
319 (void) umask(0);
320
321
322 devnull = open("/dev/null", O_RDWR);
323 if (devnull < 0) {
324 syslog(LOG_DAEMON|LOG_CRIT,
325 "Failed to open /dev/null. Exiting");
326 exit(1);
327 }
328
329 (void) dup2(devnull, STDIN_FILENO);
330 (void) dup2(devnull, STDOUT_FILENO);
331 (void) dup2(devnull, STDERR_FILENO);
332 (void) close(devnull);
333 }
334
335 void
daemon_fini(int rc)336 daemon_fini(int rc)
337 {
338 /*
339 * XXX inform the parent about the service state
340 * For now, just exit on error.
341 */
342 if (rc != 0)
343 exit(rc);
344 }
345
346 static int
open_proxy_driver()347 open_proxy_driver()
348 {
349 int drv_door_fd;
350 int stmf_ret;
351
352 /*
353 * Create communication channel for the driver.
354 */
355 if ((drv_door_fd = door_create(doorHandler, NULL, 0)) < 0) {
356 perror("door_create");
357 syslog(LOG_DAEMON|LOG_DEBUG,
358 "could not create door: errno %d", errno);
359 return (SMF_EXIT_ERR_FATAL);
360 }
361
362 stmf_ret = stmfInitProxyDoor(&proxy_hdl, drv_door_fd);
363 if (stmf_ret != STMF_STATUS_SUCCESS) {
364 perror("pppt ioctl: door install");
365 syslog(LOG_DAEMON|LOG_DEBUG,
366 "could not install door: errno %d", errno);
367 return (SMF_EXIT_ERR_FATAL);
368 }
369
370 return (SMF_EXIT_OK);
371 }
372
373 /*
374 * daemon entry
375 *
376 * parse arguments
377 * create resources to talk to child
378 * if !foreground
379 * daemonize, run as child
380 * open proxy driver
381 * install door in proxy driver
382 * create socket
383 * if server-side
384 * bind socket
385 * if !foreground
386 * inform parent things aok
387 * if parent
388 * exit(SMF_EXIT_OK)
389 * if server-side
390 * accept
391 * if client-side
392 * connect
393 * send hello
394 * recv hello
395 * loop on recieve
396 * XXX anyway to check in envp that we are started by SMF?
397 */
398 int
main(int argc,char * argv[])399 main(int argc, char *argv[])
400 {
401 struct sockaddr_in sin;
402 int rc;
403 struct sigaction act;
404 sigset_t sigmask;
405 int c;
406 int node = 0;
407 int node_override = 0;
408 int server_node = 0;
409 int server_match = 0;
410 extern char *optarg;
411 int stmf_ret;
412
413 (void) setlocale(LC_ALL, "");
414 openlog("stmfproxy", LOG_PID, LOG_DAEMON);
415 (void) setlogmask(LOG_UPTO(LOG_INFO));
416
417 while ((c = getopt(argc, argv, "dfn:")) != -1) {
418 switch (c) {
419 case 'd':
420 (void) setlogmask(LOG_UPTO(LOG_DEBUG));
421 log_debug = 1;
422 break;
423 case 'f':
424 fore_ground = 1;
425 break;
426 case 'n':
427 node_override = 1;
428 node = atoi(optarg);
429 break;
430 default:
431 /*
432 * Should never happen from smf
433 */
434 (void) fprintf(stderr, USAGE, argv[0]);
435 exit(SMF_EXIT_ERR_CONFIG);
436 break;
437 }
438 }
439 /*
440 * After the options, only the server argument should remain.
441 */
442 if (optind != argc-1) {
443 (void) fprintf(stderr, USAGE, argv[0]);
444 exit(SMF_EXIT_ERR_CONFIG);
445 }
446 (void) strcpy(aluaNode, argv[optind]);
447 syslog(LOG_DAEMON|LOG_DEBUG, "aluaNode %s", aluaNode);
448 if (gethostname(myNode, 255)) {
449 perror("gethostname");
450 exit(1);
451 }
452 if ((inet_aton(aluaNode, &sin.sin_addr)) == 0) {
453 /*
454 * Not ipaddr, try hostname match.
455 */
456 server_match = (strcmp(aluaNode, myNode)) ? 0 : 1;
457 } else {
458 /*
459 * see if this is our ip address
460 */
461 (void) fprintf(stderr, "Sorry, cannot use ip adress format\n");
462 }
463 if (server_match) {
464 server_node = 1;
465 if (!node_override)
466 node = 1;
467 }
468
469
470 /*
471 * Allow SIGQUIT, SIGINT and SIGTERM signals to terminate us
472 */
473 act.sa_handler = killHandler;
474 (void) sigemptyset(&act.sa_mask);
475 act.sa_flags = 0;
476
477 /* Install the signal handler */
478 (void) sigaction(SIGQUIT, &act, NULL);
479 (void) sigaction(SIGINT, &act, NULL);
480 (void) sigaction(SIGTERM, &act, NULL);
481 (void) sigaction(SIGHUP, &act, NULL);
482
483 /* block all signals */
484 (void) sigfillset(&sigmask);
485
486 /* unblock SIGQUIT, SIGINT, SIGTERM */
487 (void) sigdelset(&sigmask, SIGQUIT);
488 (void) sigdelset(&sigmask, SIGINT);
489 (void) sigdelset(&sigmask, SIGTERM);
490 (void) sigdelset(&sigmask, SIGHUP);
491
492 (void) sigprocmask(SIG_SETMASK, &sigmask, NULL);
493
494 /* time to go backstage */
495 daemonInit();
496
497 if ((rc = open_proxy_driver()) != 0)
498 daemon_fini(rc);
499
500 if ((rc = stmf_proxy_transport_init("sockets", &pt_ops)) != 0)
501 daemon_fini(rc);
502
503 /*
504 * Establish connection
505 *
506 * At this point, the parent has exited and the service
507 * is online. But there are no real proxy services until
508 * this connect call succeeds. That could take a long time if
509 * the peer node is down.
510 */
511 t_handle = pt_ops->stmf_proxy_connect(server_node, aluaNode);
512 if (t_handle == NULL) {
513 syslog(LOG_DAEMON|LOG_WARNING,
514 "socket() call failed: %d", errno);
515 exit(1);
516 }
517
518 /* The first message is a greeting */
519 (void) postMsg((uint_t)strlen(myNode)+1, (uchar_t *)myNode);
520 /* Read the greeting from peer node */
521 relay_peer_msg();
522 /*
523 * Set the alua state in stmf. No need to keep
524 * the device open since the proxy driver has a reference.
525 */
526 stmf_ret = stmfSetAluaState(B_TRUE, node);
527 if (stmf_ret != STMF_STATUS_SUCCESS) {
528 syslog(LOG_DAEMON|LOG_CRIT, "stmf ioctl failed - %x", stmf_ret);
529 exit(1);
530 }
531
532 /* service is online */
533 daemon_fini(0);
534
535 /*
536 * Loop relaying data from the peer daemon to the local kernel.
537 * Data coming from the local kernel is handled asynchronously
538 * by the door server.
539 */
540 for (;;) { /* loop forever */
541 relay_peer_msg();
542 }
543 }
544