99 #define shm_unlock(shp)			\  argument
100 	ipc_unlock(&(shp)->shm_perm)
105 static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp);
126 	struct shmid_kernel *shp;  in do_shm_rmid()  local
128 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in do_shm_rmid()
129 	WARN_ON(ns != shp->ns);  in do_shm_rmid()
131 	if (shp->shm_nattch) {  in do_shm_rmid()
132 		shp->shm_perm.mode |= SHM_DEST;  in do_shm_rmid()
134 		ipc_set_key_private(&shm_ids(ns), &shp->shm_perm);  in do_shm_rmid()
135 		shm_unlock(shp);  in do_shm_rmid()
137 		shm_destroy(ns, shp);  in do_shm_rmid()
235 	struct shmid_kernel *shp = container_of(ptr, struct shmid_kernel,  in shm_rcu_free()  local
237 	security_shm_free(&shp->shm_perm);  in shm_rcu_free()
238 	kfree(shp);  in shm_rcu_free()
245 static inline void shm_clist_rm(struct shmid_kernel *shp)  in shm_clist_rm()  argument
256 	if (!list_empty(&shp->shm_clist)) {  in shm_clist_rm()
261 		creator = shp->shm_creator;  in shm_clist_rm()
268 		list_del_init(&shp->shm_clist);  in shm_clist_rm()
283 	struct shmid_kernel *shp;  in __shm_open()  local
285 	shp = shm_lock(sfd->ns, sfd->id);  in __shm_open()
287 	if (IS_ERR(shp))  in __shm_open()
288 		return PTR_ERR(shp);  in __shm_open()
290 	if (shp->shm_file != sfd->file) {  in __shm_open()
292 		shm_unlock(shp);  in __shm_open()
296 	shp->shm_atim = ktime_get_real_seconds();  in __shm_open()
297 	ipc_update_pid(&shp->shm_lprid, task_tgid(current));  in __shm_open()
298 	shp->shm_nattch++;  in __shm_open()
299 	shm_unlock(shp);  in __shm_open()
331 static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)  in shm_destroy()  argument
335 	shm_file = shp->shm_file;  in shm_destroy()
336 	shp->shm_file = NULL;  in shm_destroy()
337 	ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;  in shm_destroy()
338 	shm_rmid(shp);  in shm_destroy()
339 	shm_unlock(shp);  in shm_destroy()
341 		shmem_lock(shm_file, 0, shp->mlock_ucounts);  in shm_destroy()
343 	ipc_update_pid(&shp->shm_cprid, NULL);  in shm_destroy()
344 	ipc_update_pid(&shp->shm_lprid, NULL);  in shm_destroy()
345 	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);  in shm_destroy()
358 static bool shm_may_destroy(struct shmid_kernel *shp)  in shm_may_destroy()  argument
360 	return (shp->shm_nattch == 0) &&  in shm_may_destroy()
361 	       (shp->ns->shm_rmid_forced ||  in shm_may_destroy()
362 		(shp->shm_perm.mode & SHM_DEST));  in shm_may_destroy()
373 	struct shmid_kernel *shp;  in __shm_close()  local
378 	shp = shm_lock(ns, sfd->id);  in __shm_close()
384 	if (WARN_ON_ONCE(IS_ERR(shp)))  in __shm_close()
387 	ipc_update_pid(&shp->shm_lprid, task_tgid(current));  in __shm_close()
388 	shp->shm_dtim = ktime_get_real_seconds();  in __shm_close()
389 	shp->shm_nattch--;  in __shm_close()
390 	if (shm_may_destroy(shp))  in __shm_close()
391 		shm_destroy(ns, shp);  in __shm_close()
393 		shm_unlock(shp);  in __shm_close()
415 	struct shmid_kernel *shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in shm_try_destroy_orphaned()  local
423 	if (!list_empty(&shp->shm_clist))  in shm_try_destroy_orphaned()
426 	if (shm_may_destroy(shp)) {  in shm_try_destroy_orphaned()
427 		shm_lock_by_ptr(shp);  in shm_try_destroy_orphaned()
428 		shm_destroy(ns, shp);  in shm_try_destroy_orphaned()
448 		struct shmid_kernel *shp;  in exit_shm()  local
458 		shp = list_first_entry(&task->sysvshm.shm_clist, struct shmid_kernel,  in exit_shm()
468 		ns = shp->ns;  in exit_shm()
486 			list_del_init(&shp->shm_clist);  in exit_shm()
496 		WARN_ON(!ipc_rcu_getref(&shp->shm_perm));  in exit_shm()
505 		list_del_init(&shp->shm_clist);  in exit_shm()
514 		shm_lock_by_ptr(shp);  in exit_shm()
519 		ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);  in exit_shm()
521 		if (ipc_valid_object(&shp->shm_perm)) {  in exit_shm()
522 			if (shm_may_destroy(shp))  in exit_shm()
523 				shm_destroy(ns, shp);  in exit_shm()
525 				shm_unlock(shp);  in exit_shm()
532 			shm_unlock(shp);  in exit_shm()
708 	struct shmid_kernel *shp;  in newseg()  local
725 	shp = kmalloc_obj(*shp, GFP_KERNEL_ACCOUNT);  in newseg()
726 	if (unlikely(!shp))  in newseg()
729 	shp->shm_perm.key = key;  in newseg()
730 	shp->shm_perm.mode = (shmflg & S_IRWXUGO);  in newseg()
731 	shp->mlock_ucounts = NULL;  in newseg()
733 	shp->shm_perm.security = NULL;  in newseg()
734 	error = security_shm_alloc(&shp->shm_perm);  in newseg()
736 		kfree(shp);  in newseg()
770 	shp->shm_cprid = get_pid(task_tgid(current));  in newseg()
771 	shp->shm_lprid = NULL;  in newseg()
772 	shp->shm_atim = shp->shm_dtim = 0;  in newseg()
773 	shp->shm_ctim = ktime_get_real_seconds();  in newseg()
774 	shp->shm_segsz = size;  in newseg()
775 	shp->shm_nattch = 0;  in newseg()
776 	shp->shm_file = file;  in newseg()
777 	shp->shm_creator = current;  in newseg()
780 	error = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);  in newseg()
784 	shp->ns = ns;  in newseg()
787 	list_add(&shp->shm_clist, &current->sysvshm.shm_clist);  in newseg()
794 	file_inode(file)->i_ino = shp->shm_perm.id;  in newseg()
797 	error = shp->shm_perm.id;  in newseg()
799 	ipc_unlock_object(&shp->shm_perm);  in newseg()
804 	ipc_update_pid(&shp->shm_cprid, NULL);  in newseg()
805 	ipc_update_pid(&shp->shm_lprid, NULL);  in newseg()
807 	ipc_rcu_putref(&shp->shm_perm, shm_rcu_free);  in newseg()
810 	call_rcu(&shp->shm_perm.rcu, shm_rcu_free);  in newseg()
819 	struct shmid_kernel *shp;  in shm_more_checks()  local
821 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in shm_more_checks()
822 	if (shp->shm_segsz < params->u.size)  in shm_more_checks()
934 static void shm_add_rss_swap(struct shmid_kernel *shp,  in shm_add_rss_swap()  argument
939 	inode = file_inode(shp->shm_file);  in shm_add_rss_swap()
941 	if (is_file_hugepages(shp->shm_file)) {  in shm_add_rss_swap()
943 		struct hstate *h = hstate_file(shp->shm_file);  in shm_add_rss_swap()
975 		struct shmid_kernel *shp;  in shm_get_stat()  local
980 		shp = container_of(ipc, struct shmid_kernel, shm_perm);  in shm_get_stat()
982 		shm_add_rss_swap(shp, rss, swp);  in shm_get_stat()
997 	struct shmid_kernel *shp;  in shmctl_down()  local
1010 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in shmctl_down()
1012 	err = security_shm_shmctl(&shp->shm_perm, cmd);  in shmctl_down()
1018 		ipc_lock_object(&shp->shm_perm);  in shmctl_down()
1023 		ipc_lock_object(&shp->shm_perm);  in shmctl_down()
1027 		shp->shm_ctim = ktime_get_real_seconds();  in shmctl_down()
1035 	ipc_unlock_object(&shp->shm_perm);  in shmctl_down()
1085 	struct shmid_kernel *shp;  in shmctl_stat()  local
1092 		shp = shm_obtain_object(ns, shmid);  in shmctl_stat()
1093 		if (IS_ERR(shp)) {  in shmctl_stat()
1094 			err = PTR_ERR(shp);  in shmctl_stat()
1098 		shp = shm_obtain_object_check(ns, shmid);  in shmctl_stat()
1099 		if (IS_ERR(shp)) {  in shmctl_stat()
1100 			err = PTR_ERR(shp);  in shmctl_stat()
1113 		audit_ipc_obj(&shp->shm_perm);  in shmctl_stat()
1116 		if (ipcperms(ns, &shp->shm_perm, S_IRUGO))  in shmctl_stat()
1120 	err = security_shm_shmctl(&shp->shm_perm, cmd);  in shmctl_stat()
1124 	ipc_lock_object(&shp->shm_perm);  in shmctl_stat()
1126 	if (!ipc_valid_object(&shp->shm_perm)) {  in shmctl_stat()
1127 		ipc_unlock_object(&shp->shm_perm);  in shmctl_stat()
1132 	kernel_to_ipc64_perm(&shp->shm_perm, &tbuf->shm_perm);  in shmctl_stat()
1133 	tbuf->shm_segsz	= shp->shm_segsz;  in shmctl_stat()
1134 	tbuf->shm_atime	= shp->shm_atim;  in shmctl_stat()
1135 	tbuf->shm_dtime	= shp->shm_dtim;  in shmctl_stat()
1136 	tbuf->shm_ctime	= shp->shm_ctim;  in shmctl_stat()
1138 	tbuf->shm_atime_high = shp->shm_atim >> 32;  in shmctl_stat()
1139 	tbuf->shm_dtime_high = shp->shm_dtim >> 32;  in shmctl_stat()
1140 	tbuf->shm_ctime_high = shp->shm_ctim >> 32;  in shmctl_stat()
1142 	tbuf->shm_cpid	= pid_vnr(shp->shm_cprid);  in shmctl_stat()
1143 	tbuf->shm_lpid	= pid_vnr(shp->shm_lprid);  in shmctl_stat()
1144 	tbuf->shm_nattch = shp->shm_nattch;  in shmctl_stat()
1157 		err = shp->shm_perm.id;  in shmctl_stat()
1160 	ipc_unlock_object(&shp->shm_perm);  in shmctl_stat()
1168 	struct shmid_kernel *shp;  in shmctl_do_lock()  local
1173 	shp = shm_obtain_object_check(ns, shmid);  in shmctl_do_lock()
1174 	if (IS_ERR(shp)) {  in shmctl_do_lock()
1175 		err = PTR_ERR(shp);  in shmctl_do_lock()
1179 	audit_ipc_obj(&(shp->shm_perm));  in shmctl_do_lock()
1180 	err = security_shm_shmctl(&shp->shm_perm, cmd);  in shmctl_do_lock()
1184 	ipc_lock_object(&shp->shm_perm);  in shmctl_do_lock()
1187 	if (!ipc_valid_object(&shp->shm_perm)) {  in shmctl_do_lock()
1195 		if (!uid_eq(euid, shp->shm_perm.uid) &&  in shmctl_do_lock()
1196 		    !uid_eq(euid, shp->shm_perm.cuid)) {  in shmctl_do_lock()
1206 	shm_file = shp->shm_file;  in shmctl_do_lock()
1214 		if (!err && !(shp->shm_perm.mode & SHM_LOCKED)) {  in shmctl_do_lock()
1215 			shp->shm_perm.mode |= SHM_LOCKED;  in shmctl_do_lock()
1216 			shp->mlock_ucounts = ucounts;  in shmctl_do_lock()
1222 	if (!(shp->shm_perm.mode & SHM_LOCKED))  in shmctl_do_lock()
1224 	shmem_lock(shm_file, 0, shp->mlock_ucounts);  in shmctl_do_lock()
1225 	shp->shm_perm.mode &= ~SHM_LOCKED;  in shmctl_do_lock()
1226 	shp->mlock_ucounts = NULL;  in shmctl_do_lock()
1228 	ipc_unlock_object(&shp->shm_perm);  in shmctl_do_lock()
1236 	ipc_unlock_object(&shp->shm_perm);  in shmctl_do_lock()
1522 	struct shmid_kernel *shp;  in do_shmat()  local
1582 	shp = shm_obtain_object_check(ns, shmid);  in do_shmat()
1583 	if (IS_ERR(shp)) {  in do_shmat()
1584 		err = PTR_ERR(shp);  in do_shmat()
1589 	if (ipcperms(ns, &shp->shm_perm, acc_mode))  in do_shmat()
1592 	err = security_shm_shmat(&shp->shm_perm, shmaddr, shmflg);  in do_shmat()
1596 	ipc_lock_object(&shp->shm_perm);  in do_shmat()
1599 	if (!ipc_valid_object(&shp->shm_perm)) {  in do_shmat()
1600 		ipc_unlock_object(&shp->shm_perm);  in do_shmat()
1614 	base = get_file(shp->shm_file);  in do_shmat()
1615 	shp->shm_nattch++;  in do_shmat()
1617 	ipc_unlock_object(&shp->shm_perm);  in do_shmat()
1638 	sfd->id = shp->shm_perm.id;  in do_shmat()
1677 	shp = shm_lock(ns, shmid);  in do_shmat()
1678 	shp->shm_nattch--;  in do_shmat()
1680 	if (shm_may_destroy(shp))  in do_shmat()
1681 		shm_destroy(ns, shp);  in do_shmat()
1683 		shm_unlock(shp);  in do_shmat()
1845 	struct shmid_kernel *shp;  in sysvipc_shm_proc_show()  local
1848 	shp = container_of(ipcp, struct shmid_kernel, shm_perm);  in sysvipc_shm_proc_show()
1849 	shm_add_rss_swap(shp, &rss, &swp);  in sysvipc_shm_proc_show()
1861 		   shp->shm_perm.key,  in sysvipc_shm_proc_show()
1862 		   shp->shm_perm.id,  in sysvipc_shm_proc_show()
1863 		   shp->shm_perm.mode,  in sysvipc_shm_proc_show()
1864 		   shp->shm_segsz,  in sysvipc_shm_proc_show()
1865 		   pid_nr_ns(shp->shm_cprid, pid_ns),  in sysvipc_shm_proc_show()
1866 		   pid_nr_ns(shp->shm_lprid, pid_ns),  in sysvipc_shm_proc_show()
1867 		   shp->shm_nattch,  in sysvipc_shm_proc_show()
1868 		   from_kuid_munged(user_ns, shp->shm_perm.uid),  in sysvipc_shm_proc_show()
1869 		   from_kgid_munged(user_ns, shp->shm_perm.gid),  in sysvipc_shm_proc_show()
1870 		   from_kuid_munged(user_ns, shp->shm_perm.cuid),  in sysvipc_shm_proc_show()
1871 		   from_kgid_munged(user_ns, shp->shm_perm.cgid),  in sysvipc_shm_proc_show()
1872 		   shp->shm_atim,  in sysvipc_shm_proc_show()
1873 		   shp->shm_dtim,  in sysvipc_shm_proc_show()
1874 		   shp->shm_ctim,  in sysvipc_shm_proc_show()