117 static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn)  in fsnotify_conn_mask_p()  argument
119 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)  in fsnotify_conn_mask_p()
120 		return &fsnotify_conn_inode(conn)->i_fsnotify_mask;  in fsnotify_conn_mask_p()
121 	else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT)  in fsnotify_conn_mask_p()
122 		return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask;  in fsnotify_conn_mask_p()
123 	else if (conn->type == FSNOTIFY_OBJ_TYPE_SB)  in fsnotify_conn_mask_p()
124 		return &fsnotify_conn_sb(conn)->s_fsnotify_mask;  in fsnotify_conn_mask_p()
125 	else if (conn->type == FSNOTIFY_OBJ_TYPE_MNTNS)  in fsnotify_conn_mask_p()
126 		return &fsnotify_conn_mntns(conn)->n_fsnotify_mask;  in fsnotify_conn_mask_p()
130 __u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn)  in fsnotify_conn_mask()  argument
132 	if (WARN_ON(!fsnotify_valid_obj_type(conn->type)))  in fsnotify_conn_mask()
135 	return READ_ONCE(*fsnotify_conn_mask_p(conn));  in fsnotify_conn_mask()
172 					struct fsnotify_mark_connector *conn)  in fsnotify_update_sb_watchers()  argument
175 	bool is_watched = conn->flags & FSNOTIFY_CONN_FLAG_IS_WATCHED;  in fsnotify_update_sb_watchers()
179 	if (conn->obj)  in fsnotify_update_sb_watchers()
180 		first_mark = hlist_entry_safe(conn->list.first,  in fsnotify_update_sb_watchers()
192 	for (unsigned int p = conn->prio + 1; p <= highest_prio; p++)  in fsnotify_update_sb_watchers()
194 	for (unsigned int p = conn->prio; p > highest_prio; p--)  in fsnotify_update_sb_watchers()
196 	conn->prio = highest_prio;  in fsnotify_update_sb_watchers()
201 		conn->flags |= FSNOTIFY_CONN_FLAG_IS_WATCHED;  in fsnotify_update_sb_watchers()
204 		conn->flags &= ~FSNOTIFY_CONN_FLAG_IS_WATCHED;  in fsnotify_update_sb_watchers()
217 static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn,  in fsnotify_update_iref()  argument
220 	bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;  in fsnotify_update_iref()
223 	if (conn->type != FSNOTIFY_OBJ_TYPE_INODE ||  in fsnotify_update_iref()
229 		fsnotify_get_inode_ref(fsnotify_conn_inode(conn));  in fsnotify_update_iref()
230 		conn->flags |= FSNOTIFY_CONN_FLAG_HAS_IREF;  in fsnotify_update_iref()
233 		inode = fsnotify_conn_inode(conn);  in fsnotify_update_iref()
234 		conn->flags &= ~FSNOTIFY_CONN_FLAG_HAS_IREF;  in fsnotify_update_iref()
240 static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)  in __fsnotify_recalc_mask()  argument
246 	assert_spin_locked(&conn->lock);  in __fsnotify_recalc_mask()
248 	if (!fsnotify_valid_obj_type(conn->type))  in __fsnotify_recalc_mask()
250 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in __fsnotify_recalc_mask()
254 		if (conn->type == FSNOTIFY_OBJ_TYPE_INODE &&  in __fsnotify_recalc_mask()
260 	 * confusing readers not holding conn->lock with partial updates.  in __fsnotify_recalc_mask()
262 	WRITE_ONCE(*fsnotify_conn_mask_p(conn), new_mask);  in __fsnotify_recalc_mask()
264 	return fsnotify_update_iref(conn, want_iref);  in __fsnotify_recalc_mask()
268 					struct fsnotify_mark_connector *conn)  in fsnotify_conn_watches_children()  argument
270 	if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)  in fsnotify_conn_watches_children()
273 	return fsnotify_inode_watches_children(fsnotify_conn_inode(conn));  in fsnotify_conn_watches_children()
277 					struct fsnotify_mark_connector *conn)  in fsnotify_conn_set_children_dentry_flags()  argument
279 	if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)  in fsnotify_conn_set_children_dentry_flags()
282 	fsnotify_set_children_dentry_flags(fsnotify_conn_inode(conn));  in fsnotify_conn_set_children_dentry_flags()
291 void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)  in fsnotify_recalc_mask()  argument
295 	if (!conn)  in fsnotify_recalc_mask()
298 	spin_lock(&conn->lock);  in fsnotify_recalc_mask()
299 	update_children = !fsnotify_conn_watches_children(conn);  in fsnotify_recalc_mask()
300 	__fsnotify_recalc_mask(conn);  in fsnotify_recalc_mask()
301 	update_children &= fsnotify_conn_watches_children(conn);  in fsnotify_recalc_mask()
302 	spin_unlock(&conn->lock);  in fsnotify_recalc_mask()
309 		fsnotify_conn_set_children_dentry_flags(conn);  in fsnotify_recalc_mask()
315 	struct fsnotify_mark_connector *conn, *free;  in fsnotify_connector_destroy_workfn()  local
318 	conn = connector_destroy_list;  in fsnotify_connector_destroy_workfn()
323 	while (conn) {  in fsnotify_connector_destroy_workfn()
324 		free = conn;  in fsnotify_connector_destroy_workfn()
325 		conn = conn->destroy_next;  in fsnotify_connector_destroy_workfn()
331 					struct fsnotify_mark_connector *conn,  in fsnotify_detach_connector_from_object()  argument
334 	fsnotify_connp_t *connp = fsnotify_object_connp(conn->obj, conn->type);  in fsnotify_detach_connector_from_object()
335 	struct super_block *sb = fsnotify_connector_sb(conn);  in fsnotify_detach_connector_from_object()
338 	*type = conn->type;  in fsnotify_detach_connector_from_object()
339 	if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED)  in fsnotify_detach_connector_from_object()
342 	if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {  in fsnotify_detach_connector_from_object()
343 		inode = fsnotify_conn_inode(conn);  in fsnotify_detach_connector_from_object()
347 		if (!(conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF))  in fsnotify_detach_connector_from_object()
349 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {  in fsnotify_detach_connector_from_object()
350 		fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0;  in fsnotify_detach_connector_from_object()
351 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) {  in fsnotify_detach_connector_from_object()
352 		fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;  in fsnotify_detach_connector_from_object()
353 	} else if (conn->type == FSNOTIFY_OBJ_TYPE_MNTNS) {  in fsnotify_detach_connector_from_object()
354 		fsnotify_conn_mntns(conn)->n_fsnotify_mask = 0;  in fsnotify_detach_connector_from_object()
358 	conn->obj = NULL;  in fsnotify_detach_connector_from_object()
359 	conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;  in fsnotify_detach_connector_from_object()
361 		fsnotify_update_sb_watchers(sb, conn);  in fsnotify_detach_connector_from_object()
389 	struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);  in fsnotify_put_mark()  local
395 	if (!conn) {  in fsnotify_put_mark()
405 	if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))  in fsnotify_put_mark()
409 	if (hlist_empty(&conn->list)) {  in fsnotify_put_mark()
410 		objp = fsnotify_detach_connector_from_object(conn, &type);  in fsnotify_put_mark()
413 		struct super_block *sb = fsnotify_connector_sb(conn);  in fsnotify_put_mark()
417 			fsnotify_update_sb_watchers(sb, conn);  in fsnotify_put_mark()
418 		objp = __fsnotify_recalc_mask(conn);  in fsnotify_put_mark()
419 		type = conn->type;  in fsnotify_put_mark()
422 	spin_unlock(&conn->lock);  in fsnotify_put_mark()
428 		conn->destroy_next = connector_destroy_list;  in fsnotify_put_mark()
429 		connector_destroy_list = conn;  in fsnotify_put_mark()
661 	struct fsnotify_mark_connector *conn;  in fsnotify_attach_connector_to_object()  local
663 	conn = kmem_cache_alloc(fsnotify_mark_connector_cachep, GFP_KERNEL);  in fsnotify_attach_connector_to_object()
664 	if (!conn)  in fsnotify_attach_connector_to_object()
666 	spin_lock_init(&conn->lock);  in fsnotify_attach_connector_to_object()
667 	INIT_HLIST_HEAD(&conn->list);  in fsnotify_attach_connector_to_object()
668 	conn->flags = 0;  in fsnotify_attach_connector_to_object()
669 	conn->prio = 0;  in fsnotify_attach_connector_to_object()
670 	conn->type = obj_type;  in fsnotify_attach_connector_to_object()
671 	conn->obj = obj;  in fsnotify_attach_connector_to_object()
677 	if (cmpxchg(connp, NULL, conn)) {  in fsnotify_attach_connector_to_object()
679 		kmem_cache_free(fsnotify_mark_connector_cachep, conn);  in fsnotify_attach_connector_to_object()
693 	struct fsnotify_mark_connector *conn;  in fsnotify_grab_connector()  local
697 	conn = srcu_dereference(*connp, &fsnotify_mark_srcu);  in fsnotify_grab_connector()
698 	if (!conn)  in fsnotify_grab_connector()
700 	spin_lock(&conn->lock);  in fsnotify_grab_connector()
701 	if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) {  in fsnotify_grab_connector()
702 		spin_unlock(&conn->lock);  in fsnotify_grab_connector()
708 	return conn;  in fsnotify_grab_connector()
722 	struct fsnotify_mark_connector *conn;  in fsnotify_add_mark_list()  local
743 	conn = fsnotify_grab_connector(connp);  in fsnotify_add_mark_list()
744 	if (!conn) {  in fsnotify_add_mark_list()
753 	if (hlist_empty(&conn->list)) {  in fsnotify_add_mark_list()
754 		hlist_add_head_rcu(&mark->obj_list, &conn->list);  in fsnotify_add_mark_list()
759 	hlist_for_each_entry(lmark, &conn->list, obj_list) {  in fsnotify_add_mark_list()
781 		fsnotify_update_sb_watchers(sb, conn);  in fsnotify_add_mark_list()
787 	WRITE_ONCE(mark->connector, conn);  in fsnotify_add_mark_list()
789 	spin_unlock(&conn->lock);  in fsnotify_add_mark_list()
860 	struct fsnotify_mark_connector *conn;  in fsnotify_find_mark()  local
866 	conn = fsnotify_grab_connector(connp);  in fsnotify_find_mark()
867 	if (!conn)  in fsnotify_find_mark()
870 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_find_mark()
874 			spin_unlock(&conn->lock);  in fsnotify_find_mark()
878 	spin_unlock(&conn->lock);  in fsnotify_find_mark()
931 	struct fsnotify_mark_connector *conn;  in fsnotify_destroy_marks()  local
936 	conn = fsnotify_grab_connector(connp);  in fsnotify_destroy_marks()
937 	if (!conn)  in fsnotify_destroy_marks()
941 	 * fsnotify_clear_marks_by_group() and once we drop the conn->lock, the  in fsnotify_destroy_marks()
944 	 * iteration after regaining conn->lock.  in fsnotify_destroy_marks()
946 	hlist_for_each_entry(mark, &conn->list, obj_list) {  in fsnotify_destroy_marks()
948 		spin_unlock(&conn->lock);  in fsnotify_destroy_marks()
953 		spin_lock(&conn->lock);  in fsnotify_destroy_marks()
960 	objp = fsnotify_detach_connector_from_object(conn, &type);  in fsnotify_destroy_marks()
961 	spin_unlock(&conn->lock);  in fsnotify_destroy_marks()