#include <linux/blk-crypto.h>
#include <linux/fs.h>
#include <linux/time.h>
#include <linux/highuid.h>
#include <linux/pagemap.h>
#include <linux/quotaops.h>
#include <linux/string.h>
#include <linux/buffer_head.h>
#include <linux/writeback.h>
#include <linux/mpage.h>
#include <linux/namei.h>
#include <linux/uio.h>
#include <linux/bio.h>
#include <linux/workqueue.h>
#include <linux/kernel.h>
#include <linux/iomap.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include "ext4_jbd2.h"
#include "xattr.h"
#include "acl.h"
#include <trace/events/ext4.h>
static struct kmem_cache *io_end_cachep;
static struct kmem_cache *io_end_vec_cachep;
int __init ext4_init_pageio(void)
{
io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT);
if (io_end_cachep == NULL)
return -ENOMEM;
io_end_vec_cachep = KMEM_CACHE(ext4_io_end_vec, 0);
if (io_end_vec_cachep == NULL) {
kmem_cache_destroy(io_end_cachep);
return -ENOMEM;
}
return 0;
}
void ext4_exit_pageio(void)
{
kmem_cache_destroy(io_end_cachep);
kmem_cache_destroy(io_end_vec_cachep);
}
struct ext4_io_end_vec *ext4_alloc_io_end_vec(ext4_io_end_t *io_end)
{
struct ext4_io_end_vec *io_end_vec;
io_end_vec = kmem_cache_zalloc(io_end_vec_cachep, GFP_NOFS);
if (!io_end_vec)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&io_end_vec->list);
list_add_tail(&io_end_vec->list, &io_end->list_vec);
return io_end_vec;
}
static void ext4_free_io_end_vec(ext4_io_end_t *io_end)
{
struct ext4_io_end_vec *io_end_vec, *tmp;
if (list_empty(&io_end->list_vec))
return;
list_for_each_entry_safe(io_end_vec, tmp, &io_end->list_vec, list) {
list_del(&io_end_vec->list);
kmem_cache_free(io_end_vec_cachep, io_end_vec);
}
}
struct ext4_io_end_vec *ext4_last_io_end_vec(ext4_io_end_t *io_end)
{
BUG_ON(list_empty(&io_end->list_vec));
return list_last_entry(&io_end->list_vec, struct ext4_io_end_vec, list);
}
static void buffer_io_error(struct buffer_head *bh)
{
printk_ratelimited(KERN_ERR "Buffer I/O error on device %pg, logical block %llu\n",
bh->b_bdev,
(unsigned long long)bh->b_blocknr);
}
static void ext4_finish_bio(struct bio *bio)
{
struct folio_iter fi;
bio_for_each_folio_all(fi, bio) {
struct folio *folio = fi.folio;
struct buffer_head *bh, *head;
size_t bio_start = fi.offset;
size_t bio_end = bio_start + fi.length;
unsigned under_io = 0;
unsigned long flags;
if (bio->bi_status) {
int err = blk_status_to_errno(bio->bi_status);
mapping_set_error(folio->mapping, err);
}
bh = head = folio_buffers(folio);
spin_lock_irqsave(&head->b_uptodate_lock, flags);
do {
if (bh_offset(bh) < bio_start ||
bh_offset(bh) + bh->b_size > bio_end) {
if (buffer_async_write(bh))
under_io++;
continue;
}
clear_buffer_async_write(bh);
if (bio->bi_status) {
set_buffer_write_io_error(bh);
buffer_io_error(bh);
}
} while ((bh = bh->b_this_page) != head);
spin_unlock_irqrestore(&head->b_uptodate_lock, flags);
if (!under_io)
folio_end_writeback(folio);
}
}
static void ext4_release_io_end(ext4_io_end_t *io_end)
{
struct bio *bio, *next_bio;
BUG_ON(!list_empty(&io_end->list));
BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN);
WARN_ON(io_end->handle);
for (bio = io_end->bio; bio; bio = next_bio) {
next_bio = bio->bi_private;
ext4_finish_bio(bio);
bio_put(bio);
}
ext4_free_io_end_vec(io_end);
kmem_cache_free(io_end_cachep, io_end);
}
static int ext4_end_io_end(ext4_io_end_t *io_end)
{
struct inode *inode = io_end->inode;
handle_t *handle = io_end->handle;
struct super_block *sb = inode->i_sb;
int ret = 0;
ext4_debug("ext4_end_io_nolock: io_end 0x%p from inode %llu,list->next 0x%p,"
"list->prev 0x%p\n",
io_end, inode->i_ino, io_end->list.next, io_end->list.prev);
io_end->handle = NULL;
if (unlikely(io_end->flag & EXT4_IO_END_FAILED)) {
ret = -EIO;
if (handle)
jbd2_journal_free_reserved(handle);
if (test_opt(sb, DATA_ERR_ABORT))
jbd2_journal_abort(EXT4_SB(sb)->s_journal, ret);
} else {
ret = ext4_convert_unwritten_io_end_vec(handle, io_end);
}
if (ret < 0 && !ext4_emergency_state(sb) &&
io_end->flag & EXT4_IO_END_UNWRITTEN) {
ext4_msg(sb, KERN_EMERG,
"failed to convert unwritten extents to written "
"extents -- potential data loss! "
"(inode %llu, error %d)", inode->i_ino, ret);
}
ext4_clear_io_unwritten_flag(io_end);
ext4_release_io_end(io_end);
return ret;
}
static void dump_completed_IO(struct inode *inode, struct list_head *head)
{
#ifdef EXT4FS_DEBUG
struct list_head *cur, *before, *after;
ext4_io_end_t *io_end, *io_end0, *io_end1;
if (list_empty(head))
return;
ext4_debug("Dump inode %llu completed io list\n", inode->i_ino);
list_for_each_entry(io_end, head, list) {
cur = &io_end->list;
before = cur->prev;
io_end0 = container_of(before, ext4_io_end_t, list);
after = cur->next;
io_end1 = container_of(after, ext4_io_end_t, list);
ext4_debug("io 0x%p from inode %llu,prev 0x%p,next 0x%p\n",
io_end, inode->i_ino, io_end0, io_end1);
}
#endif
}
static bool ext4_io_end_defer_completion(ext4_io_end_t *io_end)
{
if (io_end->flag & EXT4_IO_END_UNWRITTEN &&
!list_empty(&io_end->list_vec))
return true;
if (test_opt(io_end->inode->i_sb, DATA_ERR_ABORT) &&
io_end->flag & EXT4_IO_END_FAILED &&
!ext4_emergency_state(io_end->inode->i_sb))
return true;
return false;
}
static void ext4_add_complete_io(ext4_io_end_t *io_end)
{
struct ext4_inode_info *ei = EXT4_I(io_end->inode);
struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb);
struct workqueue_struct *wq;
unsigned long flags;
WARN_ON(!(io_end->flag & EXT4_IO_END_DEFER_COMPLETION));
WARN_ON(io_end->flag & EXT4_IO_END_UNWRITTEN &&
!io_end->handle && sbi->s_journal);
WARN_ON(!io_end->bio);
spin_lock_irqsave(&ei->i_completed_io_lock, flags);
wq = sbi->rsv_conversion_wq;
if (list_empty(&ei->i_rsv_conversion_list))
queue_work(wq, &ei->i_rsv_conversion_work);
list_add_tail(&io_end->list, &ei->i_rsv_conversion_list);
spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
}
static int ext4_do_flush_completed_IO(struct inode *inode,
struct list_head *head)
{
ext4_io_end_t *io_end;
struct list_head unwritten;
unsigned long flags;
struct ext4_inode_info *ei = EXT4_I(inode);
int err, ret = 0;
spin_lock_irqsave(&ei->i_completed_io_lock, flags);
dump_completed_IO(inode, head);
list_replace_init(head, &unwritten);
spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
while (!list_empty(&unwritten)) {
io_end = list_entry(unwritten.next, ext4_io_end_t, list);
BUG_ON(!(io_end->flag & EXT4_IO_END_DEFER_COMPLETION));
list_del_init(&io_end->list);
err = ext4_end_io_end(io_end);
if (unlikely(!ret && err))
ret = err;
}
return ret;
}
void ext4_end_io_rsv_work(struct work_struct *work)
{
struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
i_rsv_conversion_work);
ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list);
}
ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags)
{
ext4_io_end_t *io_end = kmem_cache_zalloc(io_end_cachep, flags);
if (io_end) {
io_end->inode = inode;
INIT_LIST_HEAD(&io_end->list);
INIT_LIST_HEAD(&io_end->list_vec);
refcount_set(&io_end->count, 1);
}
return io_end;
}
void ext4_put_io_end_defer(ext4_io_end_t *io_end)
{
if (refcount_dec_and_test(&io_end->count)) {
if (ext4_io_end_defer_completion(io_end))
return ext4_add_complete_io(io_end);
ext4_release_io_end(io_end);
}
}
int ext4_put_io_end(ext4_io_end_t *io_end)
{
if (refcount_dec_and_test(&io_end->count)) {
if (ext4_io_end_defer_completion(io_end))
return ext4_end_io_end(io_end);
ext4_release_io_end(io_end);
}
return 0;
}
ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end)
{
refcount_inc(&io_end->count);
return io_end;
}
static void ext4_end_bio(struct bio *bio)
{
ext4_io_end_t *io_end = bio->bi_private;
sector_t bi_sector = bio->bi_iter.bi_sector;
if (WARN_ONCE(!io_end, "io_end is NULL: %pg: sector %Lu len %u err %d\n",
bio->bi_bdev,
(long long) bio->bi_iter.bi_sector,
(unsigned) bio_sectors(bio),
bio->bi_status)) {
ext4_finish_bio(bio);
bio_put(bio);
return;
}
bio->bi_end_io = NULL;
if (bio->bi_status) {
struct inode *inode = io_end->inode;
ext4_warning(inode->i_sb, "I/O error %d writing to inode %llu "
"starting block %llu)",
bio->bi_status, inode->i_ino,
(unsigned long long)
bi_sector >> (inode->i_blkbits - 9));
io_end->flag |= EXT4_IO_END_FAILED;
mapping_set_error(inode->i_mapping,
blk_status_to_errno(bio->bi_status));
}
if (ext4_io_end_defer_completion(io_end)) {
bio->bi_private = xchg(&io_end->bio, bio);
ext4_put_io_end_defer(io_end);
} else {
ext4_put_io_end_defer(io_end);
ext4_finish_bio(bio);
bio_put(bio);
}
}
void ext4_io_submit(struct ext4_io_submit *io)
{
struct bio *bio = io->io_bio;
if (bio) {
if (io->io_wbc->sync_mode == WB_SYNC_ALL)
io->io_bio->bi_opf |= REQ_SYNC;
blk_crypto_submit_bio(io->io_bio);
}
io->io_bio = NULL;
}
void ext4_io_submit_init(struct ext4_io_submit *io,
struct writeback_control *wbc)
{
io->io_wbc = wbc;
io->io_bio = NULL;
io->io_end = NULL;
}
static void io_submit_init_bio(struct ext4_io_submit *io,
struct inode *inode,
struct folio *folio,
struct buffer_head *bh)
{
struct bio *bio;
bio = bio_alloc(bh->b_bdev, BIO_MAX_VECS, REQ_OP_WRITE, GFP_NOIO);
fscrypt_set_bio_crypt_ctx(bio, inode, folio_pos(folio) + bh_offset(bh),
GFP_NOIO);
bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
bio->bi_end_io = ext4_end_bio;
bio->bi_private = ext4_get_io_end(io->io_end);
bio->bi_write_hint = inode->i_write_hint;
io->io_bio = bio;
io->io_next_block = bh->b_blocknr;
wbc_init_bio(io->io_wbc, bio);
}
static bool io_submit_need_new_bio(struct ext4_io_submit *io,
struct inode *inode,
struct folio *folio,
struct buffer_head *bh)
{
if (bh->b_blocknr != io->io_next_block)
return true;
if (!fscrypt_mergeable_bio(io->io_bio, inode,
folio_pos(folio) + bh_offset(bh)))
return true;
return false;
}
static void io_submit_add_bh(struct ext4_io_submit *io,
struct inode *inode,
struct folio *folio,
struct buffer_head *bh)
{
if (io->io_bio && io_submit_need_new_bio(io, inode, folio, bh)) {
submit_and_retry:
ext4_io_submit(io);
}
if (io->io_bio == NULL)
io_submit_init_bio(io, inode, folio, bh);
if (!bio_add_folio(io->io_bio, folio, bh->b_size, bh_offset(bh)))
goto submit_and_retry;
wbc_account_cgroup_owner(io->io_wbc, folio, bh->b_size);
io->io_next_block++;
}
void ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio,
size_t len)
{
struct inode *inode = folio->mapping->host;
unsigned block_start;
struct buffer_head *bh, *head;
int nr_to_submit = 0;
struct writeback_control *wbc = io->io_wbc;
bool keep_towrite = false;
BUG_ON(!folio_test_locked(folio));
BUG_ON(folio_test_writeback(folio));
if (len < folio_size(folio))
folio_zero_segment(folio, len, folio_size(folio));
bh = head = folio_buffers(folio);
do {
block_start = bh_offset(bh);
if (block_start >= len) {
clear_buffer_dirty(bh);
set_buffer_uptodate(bh);
continue;
}
if (!buffer_dirty(bh) || buffer_delay(bh) ||
!buffer_mapped(bh) || buffer_unwritten(bh)) {
if (!buffer_mapped(bh))
clear_buffer_dirty(bh);
if (buffer_dirty(bh) ||
(buffer_jbd(bh) && buffer_jbddirty(bh))) {
if (!folio_test_dirty(folio))
folio_redirty_for_writepage(wbc, folio);
keep_towrite = true;
}
continue;
}
if (buffer_new(bh))
clear_buffer_new(bh);
set_buffer_async_write(bh);
clear_buffer_dirty(bh);
nr_to_submit++;
} while ((bh = bh->b_this_page) != head);
if (!nr_to_submit) {
__folio_start_writeback(folio, keep_towrite);
folio_end_writeback(folio);
return;
}
bh = head = folio_buffers(folio);
__folio_start_writeback(folio, keep_towrite);
do {
if (!buffer_async_write(bh))
continue;
io_submit_add_bh(io, inode, folio, bh);
} while ((bh = bh->b_this_page) != head);
}
static void ext4_iomap_wb_disksize_pending_wait(struct inode *inode,
loff_t pos, size_t size)
{
loff_t disksize = READ_ONCE(EXT4_I(inode)->i_disksize);
loff_t pstart, plen;
if (pos < round_up(disksize, i_blocksize(inode)))
return;
plen = ext4_iomap_get_disksize_pending_range(inode, &pstart);
if (!plen || pos < pstart + plen)
return;
trace_ext4_iomap_wb_disksize_pending_wait(inode, pos, size);
ext4_iomap_wait_disksize_pending(inode);
}
static int ext4_iomap_wb_update_disksize(handle_t *handle, struct inode *inode,
loff_t end, bool is_disksize_grow)
{
loff_t new_disksize, i_size;
struct ext4_inode_info *ei = EXT4_I(inode);
int ret;
down_write(&ei->i_data_sem);
i_size = i_size_read(inode);
if (is_disksize_grow &&
ext4_test_inode_state(inode, EXT4_STATE_DISKSIZE_GROW_PENDING))
is_disksize_grow = false;
new_disksize = is_disksize_grow ? i_size : min(end, i_size);
if (new_disksize > ei->i_disksize) {
trace_ext4_iomap_wb_update_disksize(inode, end, i_size,
ei->i_disksize, new_disksize, is_disksize_grow);
WRITE_ONCE(ei->i_disksize, new_disksize);
}
up_write(&ei->i_data_sem);
ret = ext4_mark_inode_dirty(handle, inode);
if (ret)
EXT4_ERROR_INODE_ERR(inode, -ret, "Failed to mark inode dirty");
return ret;
}
static void ext4_iomap_finish_ioend(struct iomap_ioend *ioend)
{
struct inode *inode = ioend->io_inode;
struct super_block *sb = inode->i_sb;
loff_t pos = ioend->io_offset;
size_t size = ioend->io_size;
loff_t end = pos + size;
unsigned long io_mode = (unsigned long)ioend->io_private;
bool is_disksize_grow = (io_mode == EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO);
handle_t *handle;
int credits;
int ret, err;
ret = blk_status_to_errno(ioend->io_bio.bi_status);
if (is_disksize_grow)
trace_ext4_iomap_wb_disksize_pending_complete(ioend->io_inode,
ioend->io_offset, ioend->io_size, ret);
if (unlikely(ret)) {
if (test_opt(sb, DATA_ERR_ABORT) && !ext4_emergency_state(sb))
jbd2_journal_abort(EXT4_SB(sb)->s_journal, ret);
goto out;
}
if (!(ioend->io_flags & IOMAP_IOEND_UNWRITTEN) &&
end <= READ_ONCE(EXT4_I(inode)->i_disksize) && !is_disksize_grow)
goto out;
ext4_iomap_wb_disksize_pending_wait(inode, pos, size);
credits = ext4_chunk_trans_blocks(inode,
EXT4_MAX_BLOCKS(size, pos, inode->i_blkbits));
handle = ext4_journal_start(inode, EXT4_HT_EXT_CONVERT, credits);
if (IS_ERR(handle)) {
ret = PTR_ERR(handle);
goto out_err;
}
if (end > READ_ONCE(EXT4_I(inode)->i_disksize) || is_disksize_grow) {
ret = ext4_iomap_wb_update_disksize(handle, inode, end,
is_disksize_grow);
if (ret)
goto out_journal;
}
if (ioend->io_flags & IOMAP_IOEND_UNWRITTEN)
ret = ext4_convert_unwritten_extents(handle, inode, pos,
size, NULL);
out_journal:
err = ext4_journal_stop(handle);
if (!ret)
ret = err;
out_err:
if (ret < 0 && !ext4_emergency_state(sb)) {
ext4_msg(sb, KERN_EMERG,
"failed to convert unwritten extents to written extents or update inode size -- potential data loss! (inode %llu, error %d)",
inode->i_ino, ret);
}
out:
iomap_finish_ioends(ioend, ret);
}
void ext4_iomap_end_io(struct work_struct *work)
{
struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
i_rsv_conversion_work);
struct iomap_ioend *ioend;
struct list_head ioend_list;
unsigned long flags;
spin_lock_irqsave(&ei->i_completed_io_lock, flags);
list_replace_init(&ei->i_rsv_conversion_list, &ioend_list);
spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
iomap_sort_ioends(&ioend_list);
while (!list_empty(&ioend_list)) {
ioend = list_entry(ioend_list.next, struct iomap_ioend, io_list);
list_del_init(&ioend->io_list);
iomap_ioend_try_merge(ioend, &ioend_list);
ext4_iomap_finish_ioend(ioend);
}
}
void ext4_iomap_end_bio(struct bio *bio)
{
struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
struct ext4_inode_info *ei = EXT4_I(ioend->io_inode);
unsigned long io_mode = (unsigned long)ioend->io_private;
unsigned long flags;
if (io_mode == EXT4_IOMAP_IOEND_DISKSIZE_GROW_IO)
ext4_iomap_clear_disksize_pending(ioend->io_inode);
spin_lock_irqsave(&ei->i_completed_io_lock, flags);
if (list_empty(&ei->i_rsv_conversion_list))
queue_work(EXT4_SB(ioend->io_inode->i_sb)->rsv_conversion_wq,
&ei->i_rsv_conversion_work);
list_add_tail(&ioend->io_list, &ei->i_rsv_conversion_list);
spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
}