1 // SPDX-License-Identifier: GPL-2.0-only
26 	pr_err("Request R=%08x r=%d fl=%lx or=%x e=%ld\n",  in netfs_dump_request()
27 	       rreq->debug_id, refcount_read(&rreq->ref), rreq->flags,  in netfs_dump_request()
28 	       rreq->origin, rreq->error);  in netfs_dump_request()
30 	       rreq->start, rreq->transferred, rreq->submitted, rreq->len);  in netfs_dump_request()
32 	       rreq->cleaned_to, rreq->collected_to, atomic64_read(&rreq->issued_to));  in netfs_dump_request()
33 	pr_err("  iw=%pSR\n", rreq->netfs_ops->issue_write);  in netfs_dump_request()
36 		const struct netfs_io_stream *s = &rreq->io_streams[i];  in netfs_dump_request()
38 		pr_err("  str[%x] s=%x e=%d acnf=%u,%u,%u,%u\n",  in netfs_dump_request()
39 		       s->stream_nr, s->source, s->error,  in netfs_dump_request()
40 		       s->avail, s->active, s->need_retry, s->failed);  in netfs_dump_request()
41 		pr_err("  str[%x] ct=%llx t=%zx\n",  in netfs_dump_request()
42 		       s->stream_nr, s->collected_to, s->transferred);  in netfs_dump_request()
43 		list_for_each_entry(sreq, &s->subrequests, rreq_link) {  in netfs_dump_request()
44 			pr_err("  sreq[%x:%x] sc=%u s=%llx t=%zx/%zx r=%d f=%lx\n",  in netfs_dump_request()
45 			       sreq->stream_nr, sreq->debug_index, sreq->source,  in netfs_dump_request()
46 			       sreq->start, sreq->transferred, sreq->len,  in netfs_dump_request()
47 			       refcount_read(&sreq->ref), sreq->flags);  in netfs_dump_request()
60 	struct netfs_inode *ictx = netfs_inode(folio->mapping->host);  in netfs_folio_written_back()
71 		fend = folio_pos(folio) + finfo->dirty_offset + finfo->dirty_len;  in netfs_folio_written_back()
72 		if (fend > ictx->zero_point)  in netfs_folio_written_back()
73 			ictx->zero_point = fend;  in netfs_folio_written_back()
76 		group = finfo->netfs_group;  in netfs_folio_written_back()
114 	struct folio_queue *folioq = wreq->buffer.tail;  in netfs_writeback_unlock_folios()
115 	unsigned long long collected_to = wreq->collected_to;  in netfs_writeback_unlock_folios()
116 	unsigned int slot = wreq->buffer.first_tail_slot;  in netfs_writeback_unlock_folios()
124 	if (wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) {  in netfs_writeback_unlock_folios()
131 		folioq = rolling_buffer_delete_spent(&wreq->buffer);  in netfs_writeback_unlock_folios()
145 			      "R=%08x: folio %lx is not under writeback\n",  in netfs_writeback_unlock_folios()
146 			      wreq->debug_id, folio->index))  in netfs_writeback_unlock_folios()
152 		flen = finfo ? finfo->dirty_offset + finfo->dirty_len : fsize;  in netfs_writeback_unlock_folios()
154 		fend = min_t(unsigned long long, fpos + flen, wreq->i_size);  in netfs_writeback_unlock_folios()
162 		wreq->nr_group_rel += netfs_folio_written_back(folio);  in netfs_writeback_unlock_folios()
163 		wreq->cleaned_to = fpos + fsize;  in netfs_writeback_unlock_folios()
173 			folioq = rolling_buffer_delete_spent(&wreq->buffer);  in netfs_writeback_unlock_folios()
183 	wreq->buffer.tail = folioq;  in netfs_writeback_unlock_folios()
185 	wreq->buffer.first_tail_slot = slot;  in netfs_writeback_unlock_folios()
190  * retry some of the results - or even do an RMW cycle for content crypto.
205 	_enter("%llx-%llx", wreq->start, wreq->start + wreq->len);  in netfs_collect_write_results()
210 	issued_to = atomic64_read(&wreq->issued_to);  in netfs_collect_write_results()
213 	if (wreq->origin == NETFS_WRITEBACK ||  in netfs_collect_write_results()
214 	    wreq->origin == NETFS_WRITETHROUGH ||  in netfs_collect_write_results()
215 	    wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE)  in netfs_collect_write_results()
226 		stream = &wreq->io_streams[s];  in netfs_collect_write_results()
228 		if (!smp_load_acquire(&stream->active))  in netfs_collect_write_results()
231 		front = stream->front;  in netfs_collect_write_results()
234 			//_debug("sreq [%x] %llx %zx/%zx",  in netfs_collect_write_results()
235 			//       front->debug_index, front->start, front->transferred, front->len);  in netfs_collect_write_results()
237 			if (stream->collected_to < front->start) {  in netfs_collect_write_results()
239 				stream->collected_to = front->start;  in netfs_collect_write_results()
247 			smp_rmb(); /* Read counters after I-P flag. */  in netfs_collect_write_results()
249 			if (stream->failed) {  in netfs_collect_write_results()
250 				stream->collected_to = front->start + front->len;  in netfs_collect_write_results()
254 			if (front->start + front->transferred > stream->collected_to) {  in netfs_collect_write_results()
255 				stream->collected_to = front->start + front->transferred;  in netfs_collect_write_results()
256 				stream->transferred = stream->collected_to - wreq->start;  in netfs_collect_write_results()
257 				stream->transferred_valid = true;  in netfs_collect_write_results()
260 			if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {  in netfs_collect_write_results()
261 				stream->failed = true;  in netfs_collect_write_results()
262 				stream->error = front->error;  in netfs_collect_write_results()
263 				if (stream->source == NETFS_UPLOAD_TO_SERVER)  in netfs_collect_write_results()
264 					mapping_set_error(wreq->mapping, front->error);  in netfs_collect_write_results()
268 			if (front->transferred < front->len) {  in netfs_collect_write_results()
269 				stream->need_retry = true;  in netfs_collect_write_results()
276 			spin_lock(&wreq->lock);  in netfs_collect_write_results()
279 			list_del_init(&front->rreq_link);  in netfs_collect_write_results()
280 			front = list_first_entry_or_null(&stream->subrequests,  in netfs_collect_write_results()
282 			stream->front = front;  in netfs_collect_write_results()
283 			spin_unlock(&wreq->lock);  in netfs_collect_write_results()
293 		if (!front && issued_to > stream->collected_to) {  in netfs_collect_write_results()
295 			stream->collected_to = issued_to;  in netfs_collect_write_results()
298 		if (stream->collected_to < collected_to)  in netfs_collect_write_results()
299 			collected_to = stream->collected_to;  in netfs_collect_write_results()
302 	if (collected_to != ULLONG_MAX && collected_to > wreq->collected_to)  in netfs_collect_write_results()
303 		wreq->collected_to = collected_to;  in netfs_collect_write_results()
306 		stream = &wreq->io_streams[s];  in netfs_collect_write_results()
307 		if (stream->active)  in netfs_collect_write_results()
311 	trace_netfs_collect_state(wreq, wreq->collected_to, notes);  in netfs_collect_write_results()
315 		if (wreq->cleaned_to < wreq->collected_to)  in netfs_collect_write_results()
318 		wreq->cleaned_to = wreq->collected_to;  in netfs_collect_write_results()
338 	netfs_put_group_many(wreq->group, wreq->nr_group_rel);  in netfs_collect_write_results()
339 	wreq->nr_group_rel = 0;  in netfs_collect_write_results()
340 	_leave(" = %x", notes);  in netfs_collect_write_results()
358 	struct netfs_inode *ictx = netfs_inode(wreq->inode);  in netfs_write_collection()
363 	_enter("R=%x", wreq->debug_id);  in netfs_write_collection()
370 	if (!test_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags))  in netfs_write_collection()
376 		struct netfs_io_stream *stream = &wreq->io_streams[s];  in netfs_write_collection()
377 		if (!stream->active)  in netfs_write_collection()
379 		if (!list_empty(&stream->subrequests))  in netfs_write_collection()
381 		if (stream->transferred_valid &&  in netfs_write_collection()
382 		    stream->transferred < transferred) {  in netfs_write_collection()
383 			transferred = stream->transferred;  in netfs_write_collection()
390 		wreq->transferred = transferred;  in netfs_write_collection()
393 	if (wreq->io_streams[1].active &&  in netfs_write_collection()
394 	    wreq->io_streams[1].failed &&  in netfs_write_collection()
395 	    ictx->ops->invalidate_cache) {  in netfs_write_collection()
399 		ictx->ops->invalidate_cache(wreq);  in netfs_write_collection()
402 	if ((wreq->origin == NETFS_UNBUFFERED_WRITE ||  in netfs_write_collection()
403 	     wreq->origin == NETFS_DIO_WRITE) &&  in netfs_write_collection()
404 	    !wreq->error)  in netfs_write_collection()
405 		netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred);  in netfs_write_collection()
407 	if (wreq->origin == NETFS_DIO_WRITE &&  in netfs_write_collection()
408 	    wreq->mapping->nrpages) {  in netfs_write_collection()
412 		 * ->write_iter() is prevented from interfering by the DIO  in netfs_write_collection()
415 		pgoff_t first = wreq->start >> PAGE_SHIFT;  in netfs_write_collection()
416 		pgoff_t last = (wreq->start + wreq->transferred - 1) >> PAGE_SHIFT;  in netfs_write_collection()
417 		invalidate_inode_pages2_range(wreq->mapping, first, last);  in netfs_write_collection()
420 	if (wreq->origin == NETFS_DIO_WRITE)  in netfs_write_collection()
421 		inode_dio_end(wreq->inode);  in netfs_write_collection()
427 	if (wreq->iocb) {  in netfs_write_collection()
428 		size_t written = min(wreq->transferred, wreq->len);  in netfs_write_collection()
429 		wreq->iocb->ki_pos += written;  in netfs_write_collection()
430 		if (wreq->iocb->ki_complete) {  in netfs_write_collection()
432 			wreq->iocb->ki_complete(  in netfs_write_collection()
433 				wreq->iocb, wreq->error ? wreq->error : written);  in netfs_write_collection()
435 		wreq->iocb = VFS_PTR_POISON;  in netfs_write_collection()
457  * netfs_write_subrequest_terminated - Note the termination of a write operation.
478 	struct netfs_io_request *wreq = subreq->rreq;  in netfs_write_subrequest_terminated()
480 	_enter("%x[%x] %zd", wreq->debug_id, subreq->debug_index, transferred_or_error);  in netfs_write_subrequest_terminated()
482 	switch (subreq->source) {  in netfs_write_subrequest_terminated()
494 		subreq->error = transferred_or_error;  in netfs_write_subrequest_terminated()
495 		if (subreq->error == -EAGAIN)  in netfs_write_subrequest_terminated()
496 			set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);  in netfs_write_subrequest_terminated()
498 			set_bit(NETFS_SREQ_FAILED, &subreq->flags);  in netfs_write_subrequest_terminated()
501 		switch (subreq->source) {  in netfs_write_subrequest_terminated()
512 		set_bit(NETFS_RREQ_PAUSE, &wreq->flags);  in netfs_write_subrequest_terminated()
514 		if (WARN(transferred_or_error > subreq->len - subreq->transferred,  in netfs_write_subrequest_terminated()
515 			 "Subreq excess write: R=%x[%x] %zd > %zu - %zu",  in netfs_write_subrequest_terminated()
516 			 wreq->debug_id, subreq->debug_index,  in netfs_write_subrequest_terminated()
517 			 transferred_or_error, subreq->len, subreq->transferred))  in netfs_write_subrequest_terminated()
518 			transferred_or_error = subreq->len - subreq->transferred;  in netfs_write_subrequest_terminated()
520 		subreq->error = 0;  in netfs_write_subrequest_terminated()
521 		subreq->transferred += transferred_or_error;  in netfs_write_subrequest_terminated()
523 		if (subreq->transferred < subreq->len)  in netfs_write_subrequest_terminated()
524 			set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);  in netfs_write_subrequest_terminated()