1 /*-
2 * Copyright (c) 2012, 2013, 2014 Spectra Logic Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions, and the following disclaimer,
10 * without modification.
11 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
12 * substantially similar to the "NO WARRANTY" disclaimer below
13 * ("Disclaimer") and any redistribution must be conditioned upon
14 * including a substantially similar Disclaimer requirement for further
15 * binary redistribution.
16 *
17 * NO WARRANTY
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGES.
29 *
30 * Authors: Alan Somers (Spectra Logic Corporation)
31 */
32 #include <sys/cdefs.h>
33 #include <sys/byteorder.h>
34
35 #include <stdarg.h>
36 #include <syslog.h>
37
38 #include <libnvpair.h>
39 #include <libzfs.h>
40
41 #include <list>
42 #include <map>
43 #include <sstream>
44 #include <string>
45
46 #include <gmock/gmock.h>
47 #include <gtest/gtest.h>
48
49 #include <devdctl/guid.h>
50 #include <devdctl/event.h>
51 #include <devdctl/event_factory.h>
52 #include <devdctl/exception.h>
53 #include <devdctl/consumer.h>
54
55 #include <zfsd/callout.h>
56 #include <zfsd/vdev_iterator.h>
57 #include <zfsd/zfsd_event.h>
58 #include <zfsd/case_file.h>
59 #include <zfsd/vdev.h>
60 #include <zfsd/zfsd.h>
61 #include <zfsd/zfsd_exception.h>
62 #include <zfsd/zpool_list.h>
63
64 #include "libmocks.h"
65 /*================================== Macros ==================================*/
66 #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
67
68 /*============================ Namespace Control =============================*/
69 using std::string;
70 using std::stringstream;
71
72 using DevdCtl::Event;
73 using DevdCtl::EventFactory;
74 using DevdCtl::EventList;
75 using DevdCtl::Guid;
76 using DevdCtl::NVPairMap;
77
78 /* redefine zpool_handle here because libzfs_impl.h is not includable */
79 struct zpool_handle
80 {
81 libzfs_handle_t *zpool_hdl;
82 zpool_handle_t *zpool_next;
83 char zpool_name[ZFS_MAX_DATASET_NAME_LEN];
84 int zpool_state;
85 unsigned int zpool_n_propnames;
86 #define ZHP_MAX_PROPNAMES 4
87 const char *zpool_propnames[ZHP_MAX_PROPNAMES];
88 size_t zpool_config_size;
89 nvlist_t *zpool_config;
90 nvlist_t *zpool_old_config;
91 nvlist_t *zpool_props;
92 diskaddr_t zpool_start_block;
93 };
94
95 class MockZfsEvent : public ZfsEvent
96 {
97 public:
98 MockZfsEvent(Event::Type, NVPairMap&, const string&);
~MockZfsEvent()99 virtual ~MockZfsEvent() {}
100
101 static BuildMethod MockZfsEventBuilder;
102
103 MOCK_CONST_METHOD0(ProcessPoolEvent, void());
104
105 static EventFactory::Record s_buildRecords[];
106 };
107
108 EventFactory::Record MockZfsEvent::s_buildRecords[] =
109 {
110 { Event::NOTIFY, "ZFS", &MockZfsEvent::MockZfsEventBuilder }
111 };
112
MockZfsEvent(Event::Type type,NVPairMap & map,const string & str)113 MockZfsEvent::MockZfsEvent(Event::Type type, NVPairMap& map,
114 const string& str)
115 : ZfsEvent(type, map, str)
116 {
117 }
118
119 Event *
MockZfsEventBuilder(Event::Type type,NVPairMap & nvpairs,const string & eventString)120 MockZfsEvent::MockZfsEventBuilder(Event::Type type,
121 NVPairMap &nvpairs,
122 const string &eventString)
123 {
124 return (new MockZfsEvent(type, nvpairs, eventString));
125 }
126
127 /*
128 * A dummy Vdev class used for testing other classes
129 */
130 class MockVdev : public Vdev
131 {
132 public:
133 MockVdev(nvlist_t *vdevConfig);
~MockVdev()134 virtual ~MockVdev() {}
135
136 MOCK_CONST_METHOD0(GUID, Guid());
137 MOCK_CONST_METHOD0(PoolGUID, Guid());
138 MOCK_CONST_METHOD0(State, vdev_state());
139 MOCK_CONST_METHOD0(PhysicalPath, string());
140 MOCK_CONST_METHOD2(Name, string(zpool_handle_t * zhp, bool verbose));
141 };
142
MockVdev(nvlist_t * vdevConfig)143 MockVdev::MockVdev(nvlist_t *vdevConfig)
144 : Vdev(vdevConfig)
145 {
146 }
147
148 /*
149 * A CaseFile class with side effects removed, for testing
150 */
151 class TestableCaseFile : public CaseFile
152 {
153 public:
154 static TestableCaseFile &Create(Vdev &vdev);
155 TestableCaseFile(Vdev &vdev);
~TestableCaseFile()156 virtual ~TestableCaseFile() {}
157
158 MOCK_METHOD0(Close, void());
159 MOCK_METHOD1(RegisterCallout, void(const Event &event));
160 MOCK_METHOD0(RefreshVdevState, bool());
161 MOCK_METHOD1(ReEvaluate, bool(const ZfsEvent &event));
162
RealReEvaluate(const ZfsEvent & event)163 bool RealReEvaluate(const ZfsEvent &event)
164 {
165 return (CaseFile::ReEvaluate(event));
166 }
167
168 /*
169 * This splices the event lists, a procedure that would normally be done
170 * by OnGracePeriodEnded, but we don't necessarily call that in the
171 * unit tests
172 */
173 void SpliceEvents();
174
175 /*
176 * Used by some of our expectations. CaseFile does not publicize this
177 */
getActiveCases()178 static int getActiveCases()
179 {
180 return (s_activeCases.size());
181 }
182 };
183
TestableCaseFile(Vdev & vdev)184 TestableCaseFile::TestableCaseFile(Vdev &vdev)
185 : CaseFile(vdev)
186 {
187 }
188
189 TestableCaseFile &
Create(Vdev & vdev)190 TestableCaseFile::Create(Vdev &vdev)
191 {
192 TestableCaseFile *newCase;
193 newCase = new TestableCaseFile(vdev);
194 return (*newCase);
195 }
196
197 void
SpliceEvents()198 TestableCaseFile::SpliceEvents()
199 {
200 m_events.splice(m_events.begin(), m_tentativeEvents);
201 }
202
203
204 /*
205 * Test class ZfsdException
206 */
207 class ZfsdExceptionTest : public ::testing::Test
208 {
209 protected:
SetUp()210 virtual void SetUp()
211 {
212 ASSERT_EQ(0, nvlist_alloc(&poolConfig, NV_UNIQUE_NAME, 0));
213 ASSERT_EQ(0, nvlist_add_string(poolConfig,
214 ZPOOL_CONFIG_POOL_NAME, "unit_test_pool"));
215 ASSERT_EQ(0, nvlist_add_uint64(poolConfig,
216 ZPOOL_CONFIG_POOL_GUID, 0x1234));
217
218 ASSERT_EQ(0, nvlist_alloc(&vdevConfig, NV_UNIQUE_NAME, 0));
219 ASSERT_EQ(0, nvlist_add_uint64(vdevConfig,
220 ZPOOL_CONFIG_GUID, 0x5678));
221 bzero(&poolHandle, sizeof(poolHandle));
222 poolHandle.zpool_config = poolConfig;
223 }
224
TearDown()225 virtual void TearDown()
226 {
227 nvlist_free(poolConfig);
228 nvlist_free(vdevConfig);
229 }
230
231 nvlist_t *poolConfig;
232 nvlist_t *vdevConfig;
233 zpool_handle_t poolHandle;
234 };
235
TEST_F(ZfsdExceptionTest,StringConstructorNull)236 TEST_F(ZfsdExceptionTest, StringConstructorNull)
237 {
238 ZfsdException ze("");
239 EXPECT_STREQ("", ze.GetString().c_str());
240 }
241
TEST_F(ZfsdExceptionTest,StringConstructorFormatted)242 TEST_F(ZfsdExceptionTest, StringConstructorFormatted)
243 {
244 ZfsdException ze(" %d %s", 55, "hello world");
245 EXPECT_STREQ(" 55 hello world", ze.GetString().c_str());
246 }
247
TEST_F(ZfsdExceptionTest,LogSimple)248 TEST_F(ZfsdExceptionTest, LogSimple)
249 {
250 ZfsdException ze("unit test w/o vdev or pool");
251 ze.Log();
252 EXPECT_EQ(LOG_ERR, syslog_last_priority);
253 EXPECT_STREQ("unit test w/o vdev or pool\n", syslog_last_message);
254 }
255
TEST_F(ZfsdExceptionTest,Pool)256 TEST_F(ZfsdExceptionTest, Pool)
257 {
258 const char msg[] = "Exception with pool name";
259 char expected[4096];
260 sprintf(expected, "Pool unit_test_pool: %s\n", msg);
261 ZfsdException ze(poolConfig, msg);
262 ze.Log();
263 EXPECT_STREQ(expected, syslog_last_message);
264 }
265
TEST_F(ZfsdExceptionTest,PoolHandle)266 TEST_F(ZfsdExceptionTest, PoolHandle)
267 {
268 const char msg[] = "Exception with pool handle";
269 char expected[4096];
270 sprintf(expected, "Pool unit_test_pool: %s\n", msg);
271 ZfsdException ze(&poolHandle, msg);
272 ze.Log();
273 EXPECT_STREQ(expected, syslog_last_message);
274 }
275
276 /*
277 * Test class Vdev
278 */
279 class VdevTest : public ::testing::Test
280 {
281 protected:
SetUp()282 virtual void SetUp()
283 {
284 ASSERT_EQ(0, nvlist_alloc(&m_poolConfig, NV_UNIQUE_NAME, 0));
285 ASSERT_EQ(0, nvlist_add_uint64(m_poolConfig,
286 ZPOOL_CONFIG_POOL_GUID,
287 0x1234));
288
289 ASSERT_EQ(0, nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0));
290 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_GUID,
291 0x5678));
292 }
293
TearDown()294 virtual void TearDown()
295 {
296 nvlist_free(m_poolConfig);
297 nvlist_free(m_vdevConfig);
298 }
299
300 nvlist_t *m_poolConfig;
301 nvlist_t *m_vdevConfig;
302 };
303
304
TEST_F(VdevTest,StateFromConfig)305 TEST_F(VdevTest, StateFromConfig)
306 {
307 vdev_stat_t vs;
308
309 vs.vs_state = VDEV_STATE_OFFLINE;
310
311 ASSERT_EQ(0, nvlist_add_uint64_array(m_vdevConfig,
312 ZPOOL_CONFIG_VDEV_STATS,
313 (uint64_t*)&vs,
314 sizeof(vs) / sizeof(uint64_t)));
315
316 Vdev vdev(m_poolConfig, m_vdevConfig);
317
318 EXPECT_EQ(VDEV_STATE_OFFLINE, vdev.State());
319 }
320
TEST_F(VdevTest,StateFaulted)321 TEST_F(VdevTest, StateFaulted)
322 {
323 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_FAULTED, 1));
324
325 Vdev vdev(m_poolConfig, m_vdevConfig);
326
327 EXPECT_EQ(VDEV_STATE_FAULTED, vdev.State());
328 }
329
330 /*
331 * Test that we can construct a Vdev from the label information that is stored
332 * on an available spare drive
333 */
TEST_F(VdevTest,ConstructAvailSpare)334 TEST_F(VdevTest, ConstructAvailSpare)
335 {
336 nvlist_t *labelConfig;
337
338 ASSERT_EQ(0, nvlist_alloc(&labelConfig, NV_UNIQUE_NAME, 0));
339 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_GUID,
340 1948339428197961030));
341 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_POOL_STATE,
342 POOL_STATE_SPARE));
343
344 EXPECT_NO_THROW(Vdev vdev(labelConfig));
345
346 nvlist_free(labelConfig);
347 }
348
349 /* Available spares will always show the HEALTHY state */
TEST_F(VdevTest,AvailSpareState)350 TEST_F(VdevTest, AvailSpareState) {
351 nvlist_t *labelConfig;
352
353 ASSERT_EQ(0, nvlist_alloc(&labelConfig, NV_UNIQUE_NAME, 0));
354 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_GUID,
355 1948339428197961030));
356 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_POOL_STATE,
357 POOL_STATE_SPARE));
358
359 Vdev vdev(labelConfig);
360 EXPECT_EQ(VDEV_STATE_HEALTHY, vdev.State());
361
362 nvlist_free(labelConfig);
363 }
364
365 /* Test the Vdev::IsSpare method */
TEST_F(VdevTest,IsSpare)366 TEST_F(VdevTest, IsSpare) {
367 Vdev notSpare(m_poolConfig, m_vdevConfig);
368 EXPECT_EQ(false, notSpare.IsSpare());
369
370 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_IS_SPARE, 1));
371 Vdev isSpare(m_poolConfig, m_vdevConfig);
372 EXPECT_EQ(true, isSpare.IsSpare());
373 }
374
375 /*
376 * Test class ZFSEvent
377 */
378 class ZfsEventTest : public ::testing::Test
379 {
380 protected:
SetUp()381 virtual void SetUp()
382 {
383 m_eventFactory = new EventFactory();
384 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
385 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
386
387 m_event = NULL;
388 }
389
TearDown()390 virtual void TearDown()
391 {
392 delete m_eventFactory;
393 delete m_event;
394 }
395
396 EventFactory *m_eventFactory;
397 Event *m_event;
398 };
399
TEST_F(ZfsEventTest,ProcessPoolEventGetsCalled)400 TEST_F(ZfsEventTest, ProcessPoolEventGetsCalled)
401 {
402 string evString("!system=ZFS "
403 "subsystem=ZFS "
404 "type=sysevent.fs.zfs.vdev_remove "
405 "pool_name=foo "
406 "pool_guid=9756779504028057996 "
407 "vdev_guid=1631193447431603339 "
408 "vdev_path=/dev/da1 "
409 "timestamp=1348871594");
410 m_event = Event::CreateEvent(*m_eventFactory, evString);
411 MockZfsEvent *mock_event = static_cast<MockZfsEvent*>(m_event);
412
413 EXPECT_CALL(*mock_event, ProcessPoolEvent()).Times(1);
414 mock_event->Process();
415 }
416
417 /*
418 * Test class CaseFile
419 */
420
421 class CaseFileTest : public ::testing::Test
422 {
423 protected:
SetUp()424 virtual void SetUp()
425 {
426 m_eventFactory = new EventFactory();
427 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
428 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
429
430 m_event = NULL;
431
432 nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0);
433 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig,
434 ZPOOL_CONFIG_GUID, 0xbeef));
435 m_vdev = new MockVdev(m_vdevConfig);
436 ON_CALL(*m_vdev, GUID())
437 .WillByDefault(::testing::Return(Guid(123)));
438 ON_CALL(*m_vdev, Name(::testing::_, ::testing::_))
439 .WillByDefault(::testing::Return(string("/dev/da999")));
440 ON_CALL(*m_vdev, PoolGUID())
441 .WillByDefault(::testing::Return(Guid(456)));
442 ON_CALL(*m_vdev, State())
443 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
444 m_caseFile = &TestableCaseFile::Create(*m_vdev);
445 ON_CALL(*m_caseFile, ReEvaluate(::testing::_))
446 .WillByDefault(::testing::Invoke(m_caseFile, &TestableCaseFile::RealReEvaluate));
447 return;
448 }
449
TearDown()450 virtual void TearDown()
451 {
452 delete m_caseFile;
453 nvlist_free(m_vdevConfig);
454 delete m_vdev;
455 delete m_event;
456 delete m_eventFactory;
457 }
458
459 nvlist_t *m_vdevConfig;
460 MockVdev *m_vdev;
461 TestableCaseFile *m_caseFile;
462 Event *m_event;
463 EventFactory *m_eventFactory;
464 };
465
466 /*
467 * A Vdev with no events should not be degraded or faulted
468 */
TEST_F(CaseFileTest,HealthyVdev)469 TEST_F(CaseFileTest, HealthyVdev)
470 {
471 EXPECT_FALSE(m_caseFile->ShouldDegrade());
472 EXPECT_FALSE(m_caseFile->ShouldFault());
473 }
474
475 /*
476 * A Vdev with only one event should not be degraded or faulted
477 * For performance reasons, RefreshVdevState should not be called.
478 */
TEST_F(CaseFileTest,HealthyishVdev)479 TEST_F(CaseFileTest, HealthyishVdev)
480 {
481 string evString("!system=ZFS "
482 "class=ereport.fs.zfs.io "
483 "ena=12091638756982918145 "
484 "parent_guid=13237004955564865395 "
485 "parent_type=raidz "
486 "pool=testpool.4415 "
487 "pool_context=0 "
488 "pool_failmode=wait "
489 "pool_guid=456 "
490 "subsystem=ZFS "
491 "timestamp=1348867914 "
492 "type=ereport.fs.zfs.io "
493 "vdev_guid=123 "
494 "vdev_path=/dev/da400 "
495 "vdev_type=disk "
496 "zio_blkid=622 "
497 "zio_err=1 "
498 "zio_level=-2 "
499 "zio_object=0 "
500 "zio_objset=37 "
501 "zio_offset=25598976 "
502 "zio_size=1024");
503 m_event = Event::CreateEvent(*m_eventFactory, evString);
504 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
505
506 EXPECT_CALL(*m_caseFile, RefreshVdevState())
507 .Times(::testing::Exactly(0));
508 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
509 EXPECT_FALSE(m_caseFile->ShouldDegrade());
510 EXPECT_FALSE(m_caseFile->ShouldFault());
511 }
512
513 /* The case file should be closed when its pool is destroyed */
TEST_F(CaseFileTest,PoolDestroy)514 TEST_F(CaseFileTest, PoolDestroy)
515 {
516 string evString("!system=ZFS "
517 "pool_name=testpool.4415 "
518 "pool_guid=456 "
519 "subsystem=ZFS "
520 "timestamp=1348867914 "
521 "type=sysevent.fs.zfs.pool_destroy ");
522 m_event = Event::CreateEvent(*m_eventFactory, evString);
523 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
524 EXPECT_CALL(*m_caseFile, Close());
525 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
526 }
527
528 /*
529 * A Vdev with a very large number of Delay errors should fault
530 * For performance reasons, RefreshVdevState should be called at most once
531 */
TEST_F(CaseFileTest,VeryManyDelayErrors)532 TEST_F(CaseFileTest, VeryManyDelayErrors)
533 {
534 EXPECT_CALL(*m_caseFile, RefreshVdevState())
535 .Times(::testing::AtMost(1))
536 .WillRepeatedly(::testing::Return(true));
537
538 for(int i=0; i<100; i++) {
539 stringstream evStringStream;
540 evStringStream <<
541 "!system=ZFS "
542 "class=ereport.fs.zfs.delay "
543 "ena=12091638756982918145 "
544 "parent_guid=13237004955564865395 "
545 "parent_type=raidz "
546 "pool=testpool.4415 "
547 "pool_context=0 "
548 "pool_failmode=wait "
549 "pool_guid=456 "
550 "pool_state= 0"
551 "subsystem=ZFS "
552 "time=";
553 evStringStream << i << "0000000000000000 ";
554 evStringStream << "timestamp=" << i << " ";
555 evStringStream <<
556 "type=ereport.fs.zfs.delay "
557 "vdev_ashift=12 "
558 "vdev_cksum_errors=0 "
559 "vdev_complete_ts=948336226469 "
560 "vdev_delays=77 "
561 "vdev_delta_ts=123998485899 "
562 "vdev_guid=123 "
563 "vdev_path=/dev/da400 "
564 "vdev_read_errors=0 "
565 "vdev_spare_guids= "
566 "vdev_type=disk "
567 "vdev_write_errors=0 "
568 "zio_blkid=622 "
569 "zio_delay=31000041101 "
570 "zio_delta=123998485899 "
571 "zio_err=0 "
572 "zio_flags=1572992 "
573 "zio_level=-2 "
574 "zio_object=0 "
575 "zio_objset=37 "
576 "zio_offset=25598976 "
577 "zio_pipeline=48234496 "
578 "zio_priority=3 "
579 "zio_size=1024"
580 "zio_stage=33554432 "
581 "zio_timestamp=824337740570 ";
582 Event *event(Event::CreateEvent(*m_eventFactory,
583 evStringStream.str()));
584 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
585 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
586 delete event;
587 }
588
589 m_caseFile->SpliceEvents();
590 EXPECT_FALSE(m_caseFile->ShouldDegrade());
591 EXPECT_TRUE(m_caseFile->ShouldFault());
592 }
593
594 /*
595 * A Vdev with a very large number of IO errors should fault
596 * For performance reasons, RefreshVdevState should be called at most once
597 */
TEST_F(CaseFileTest,VeryManyIOErrors)598 TEST_F(CaseFileTest, VeryManyIOErrors)
599 {
600 EXPECT_CALL(*m_caseFile, RefreshVdevState())
601 .Times(::testing::AtMost(1))
602 .WillRepeatedly(::testing::Return(true));
603
604 for(int i=0; i<100; i++) {
605 stringstream evStringStream;
606 evStringStream <<
607 "!system=ZFS "
608 "class=ereport.fs.zfs.io "
609 "ena=12091638756982918145 "
610 "parent_guid=13237004955564865395 "
611 "parent_type=raidz "
612 "pool=testpool.4415 "
613 "pool_context=0 "
614 "pool_failmode=wait "
615 "pool_guid=456 "
616 "subsystem=ZFS "
617 "timestamp=";
618 evStringStream << i << " ";
619 evStringStream <<
620 "type=ereport.fs.zfs.io "
621 "vdev_guid=123 "
622 "vdev_path=/dev/da400 "
623 "vdev_type=disk "
624 "zio_blkid=622 "
625 "zio_err=1 "
626 "zio_level=-2 "
627 "zio_object=0 "
628 "zio_objset=37 "
629 "zio_offset=25598976 "
630 "zio_size=1024";
631 Event *event(Event::CreateEvent(*m_eventFactory,
632 evStringStream.str()));
633 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
634 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
635 delete event;
636 }
637
638 m_caseFile->SpliceEvents();
639 EXPECT_FALSE(m_caseFile->ShouldDegrade());
640 EXPECT_TRUE(m_caseFile->ShouldFault());
641 }
642
643 /*
644 * A Vdev with a very large number of checksum errors should degrade
645 * For performance reasons, RefreshVdevState should be called at most once
646 */
TEST_F(CaseFileTest,VeryManyChecksumErrors)647 TEST_F(CaseFileTest, VeryManyChecksumErrors)
648 {
649 EXPECT_CALL(*m_caseFile, RefreshVdevState())
650 .Times(::testing::AtMost(1))
651 .WillRepeatedly(::testing::Return(true));
652
653 for(int i=0; i<100; i++) {
654 stringstream evStringStream;
655 evStringStream <<
656 "!system=ZFS "
657 "bad_cleared_bits=03000000000000803f50b00000000000 "
658 "bad_range_clears=0000000e "
659 "bad_range_sets=00000000 "
660 "bad_ranges=0000000000000010 "
661 "bad_ranges_min_gap=8 "
662 "bad_set_bits=00000000000000000000000000000000 "
663 "class=ereport.fs.zfs.checksum "
664 "ena=12272856582652437505 "
665 "parent_guid=5838204195352909894 "
666 "parent_type=raidz pool=testpool.7640 "
667 "pool_context=0 "
668 "pool_failmode=wait "
669 "pool_guid=456 "
670 "subsystem=ZFS timestamp=";
671 evStringStream << i << " ";
672 evStringStream <<
673 "type=ereport.fs.zfs.checksum "
674 "vdev_guid=123 "
675 "vdev_path=/mnt/tmp/file1.7702 "
676 "vdev_type=file "
677 "zio_blkid=0 "
678 "zio_err=0 "
679 "zio_level=0 "
680 "zio_object=3 "
681 "zio_objset=0 "
682 "zio_offset=16896 "
683 "zio_size=512";
684 Event *event(Event::CreateEvent(*m_eventFactory,
685 evStringStream.str()));
686 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
687 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
688 delete event;
689 }
690
691 m_caseFile->SpliceEvents();
692 EXPECT_TRUE(m_caseFile->ShouldDegrade());
693 EXPECT_FALSE(m_caseFile->ShouldFault());
694 }
695
696 /*
697 * Test CaseFile::ReEvaluateByGuid
698 */
699 class ReEvaluateByGuidTest : public ::testing::Test
700 {
701 protected:
SetUp()702 virtual void SetUp()
703 {
704 m_eventFactory = new EventFactory();
705 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
706 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
707 m_event = Event::CreateEvent(*m_eventFactory, s_evString);
708 nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0);
709 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig,
710 ZPOOL_CONFIG_GUID, 0xbeef));
711 m_vdev456 = new ::testing::NiceMock<MockVdev>(m_vdevConfig);
712 m_vdev789 = new ::testing::NiceMock<MockVdev>(m_vdevConfig);
713 ON_CALL(*m_vdev456, GUID())
714 .WillByDefault(::testing::Return(Guid(123)));
715 ON_CALL(*m_vdev456, PoolGUID())
716 .WillByDefault(::testing::Return(Guid(456)));
717 ON_CALL(*m_vdev456, State())
718 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
719 ON_CALL(*m_vdev789, GUID())
720 .WillByDefault(::testing::Return(Guid(123)));
721 ON_CALL(*m_vdev789, PoolGUID())
722 .WillByDefault(::testing::Return(Guid(789)));
723 ON_CALL(*m_vdev789, State())
724 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
725 m_caseFile456 = NULL;
726 m_caseFile789 = NULL;
727 return;
728 }
729
TearDown()730 virtual void TearDown()
731 {
732 delete m_caseFile456;
733 delete m_caseFile789;
734 nvlist_free(m_vdevConfig);
735 delete m_vdev456;
736 delete m_vdev789;
737 delete m_event;
738 delete m_eventFactory;
739 }
740
741 static string s_evString;
742 nvlist_t *m_vdevConfig;
743 ::testing::NiceMock<MockVdev> *m_vdev456;
744 ::testing::NiceMock<MockVdev> *m_vdev789;
745 TestableCaseFile *m_caseFile456;
746 TestableCaseFile *m_caseFile789;
747 Event *m_event;
748 EventFactory *m_eventFactory;
749 };
750
751 string ReEvaluateByGuidTest::s_evString(
752 "!system=ZFS "
753 "pool_guid=16271873792808333580 "
754 "pool_name=foo "
755 "subsystem=ZFS "
756 "timestamp=1360620391 "
757 "type=sysevent.fs.zfs.config_sync");
758
759
760 /*
761 * Test the ReEvaluateByGuid method on an empty list of casefiles.
762 * We must create one event, even though it never gets used, because it will
763 * be passed by reference to ReEvaluateByGuid
764 */
TEST_F(ReEvaluateByGuidTest,ReEvaluateByGuid_empty)765 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_empty)
766 {
767 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
768
769 EXPECT_EQ(0, TestableCaseFile::getActiveCases());
770 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
771 EXPECT_EQ(0, TestableCaseFile::getActiveCases());
772 }
773
774 /*
775 * Test the ReEvaluateByGuid method on a list of CaseFiles that contains only
776 * one CaseFile, which doesn't match the criteria
777 */
TEST_F(ReEvaluateByGuidTest,ReEvaluateByGuid_oneFalse)778 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_oneFalse)
779 {
780 m_caseFile456 = &TestableCaseFile::Create(*m_vdev456);
781 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
782
783 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
784 EXPECT_CALL(*m_caseFile456, ReEvaluate(::testing::_))
785 .Times(::testing::Exactly(0));
786 CaseFile::ReEvaluateByGuid(Guid(789), *zfs_event);
787 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
788 }
789
790 /*
791 * Test the ReEvaluateByGuid method on a list of CaseFiles that contains only
792 * one CaseFile, which does match the criteria
793 */
TEST_F(ReEvaluateByGuidTest,ReEvaluateByGuid_oneTrue)794 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_oneTrue)
795 {
796 m_caseFile456 = &TestableCaseFile::Create(*m_vdev456);
797 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
798
799 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
800 EXPECT_CALL(*m_caseFile456, ReEvaluate(::testing::_))
801 .Times(::testing::Exactly(1))
802 .WillRepeatedly(::testing::Return(false));
803 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
804 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
805 }
806
807 /*
808 * Test the ReEvaluateByGuid method on a long list of CaseFiles that contains a
809 * few cases which meet the criteria
810 */
TEST_F(ReEvaluateByGuidTest,ReEvaluateByGuid_five)811 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_five)
812 {
813 TestableCaseFile *CaseFile1 = &TestableCaseFile::Create(*m_vdev456);
814 TestableCaseFile *CaseFile2 = &TestableCaseFile::Create(*m_vdev789);
815 TestableCaseFile *CaseFile3 = &TestableCaseFile::Create(*m_vdev456);
816 TestableCaseFile *CaseFile4 = &TestableCaseFile::Create(*m_vdev789);
817 TestableCaseFile *CaseFile5 = &TestableCaseFile::Create(*m_vdev789);
818 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
819
820 EXPECT_EQ(5, TestableCaseFile::getActiveCases());
821 EXPECT_CALL(*CaseFile1, ReEvaluate(::testing::_))
822 .Times(::testing::Exactly(1))
823 .WillRepeatedly(::testing::Return(false));
824 EXPECT_CALL(*CaseFile3, ReEvaluate(::testing::_))
825 .Times(::testing::Exactly(1))
826 .WillRepeatedly(::testing::Return(false));
827 EXPECT_CALL(*CaseFile2, ReEvaluate(::testing::_))
828 .Times(::testing::Exactly(0));
829 EXPECT_CALL(*CaseFile4, ReEvaluate(::testing::_))
830 .Times(::testing::Exactly(0));
831 EXPECT_CALL(*CaseFile5, ReEvaluate(::testing::_))
832 .Times(::testing::Exactly(0));
833 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
834 EXPECT_EQ(5, TestableCaseFile::getActiveCases());
835 delete CaseFile1;
836 delete CaseFile2;
837 delete CaseFile3;
838 delete CaseFile4;
839 delete CaseFile5;
840 }
841
842 /*
843 * Test VdevIterator
844 */
845 class VdevIteratorTest : public ::testing::Test
846 {
847 };
848
VdevIteratorTestCB(Vdev & vdev,void * cbArg)849 bool VdevIteratorTestCB(Vdev &vdev, void *cbArg) {
850 return (false);
851 }
852
853 /*
854 * VdevIterator::Next should not crash when run on a pool that has a previously
855 * removed vdev. Regression for
856 * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=273663
857 */
TEST_F(VdevIteratorTest,VdevRemoval)858 TEST_F(VdevIteratorTest, VdevRemoval)
859 {
860 nvlist_t* poolConfig, *rootVdev;
861
862 ASSERT_EQ(0, nvlist_alloc(&rootVdev, NV_UNIQUE_NAME, 0));
863 ASSERT_EQ(0, nvlist_add_uint64(rootVdev, ZPOOL_CONFIG_GUID, 0x5678));
864 /*
865 * Note: pools with previously-removed top-level VDEVs will contain a
866 * TLV in their labels that has 0 children.
867 */
868 ASSERT_EQ(0, nvlist_add_nvlist_array(rootVdev, ZPOOL_CONFIG_CHILDREN,
869 NULL, 0));
870 ASSERT_EQ(0, nvlist_alloc(&poolConfig, NV_UNIQUE_NAME, 0));
871 ASSERT_EQ(0, nvlist_add_uint64(poolConfig,
872 ZPOOL_CONFIG_POOL_GUID, 0x1234));
873 ASSERT_EQ(0, nvlist_add_nvlist(poolConfig, ZPOOL_CONFIG_VDEV_TREE,
874 rootVdev));
875
876 VdevIterator(poolConfig).Each(VdevIteratorTestCB, NULL);
877 }
878