summaryrefslogtreecommitdiff
path: root/fs/f2fs
diff options
context:
space:
mode:
Diffstat (limited to 'fs/f2fs')
-rw-r--r--fs/f2fs/checkpoint.c2
-rw-r--r--fs/f2fs/data.c203
-rw-r--r--fs/f2fs/f2fs.h21
-rw-r--r--fs/f2fs/file.c4
-rw-r--r--fs/f2fs/gc.c6
-rw-r--r--fs/f2fs/inline.c2
-rw-r--r--fs/f2fs/namei.c8
-rw-r--r--fs/f2fs/node.c2
-rw-r--r--fs/f2fs/super.c6
9 files changed, 160 insertions, 94 deletions
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index 04c608646fd5..760d1ad22722 100644
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -386,7 +386,7 @@ static int f2fs_set_meta_page_dirty(struct page *page)
if (!PageUptodate(page))
SetPageUptodate(page);
if (!PageDirty(page)) {
- f2fs_set_page_dirty_nobuffers(page);
+ __set_page_dirty_nobuffers(page);
inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_META);
SetPagePrivate(page);
f2fs_trace_pid(page);
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index ad44712bb902..a670702cf3ff 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -19,8 +19,6 @@
#include <linux/bio.h>
#include <linux/prefetch.h>
#include <linux/uio.h>
-#include <linux/mm.h>
-#include <linux/memcontrol.h>
#include <linux/cleancache.h>
#include "f2fs.h"
@@ -30,6 +28,11 @@
#include <trace/events/f2fs.h>
#include <trace/events/android_fs.h>
+#define NUM_PREALLOC_POST_READ_CTXS 128
+
+static struct kmem_cache *bio_post_read_ctx_cache;
+static mempool_t *bio_post_read_ctx_pool;
+
static bool __is_cp_guaranteed(struct page *page)
{
struct address_space *mapping = page->mapping;
@@ -50,11 +53,77 @@ static bool __is_cp_guaranteed(struct page *page)
return false;
}
-static void f2fs_read_end_io(struct bio *bio)
+/* postprocessing steps for read bios */
+enum bio_post_read_step {
+ STEP_INITIAL = 0,
+ STEP_DECRYPT,
+};
+
+struct bio_post_read_ctx {
+ struct bio *bio;
+ struct work_struct work;
+ unsigned int cur_step;
+ unsigned int enabled_steps;
+};
+
+static void __read_end_io(struct bio *bio)
{
- struct bio_vec *bvec;
+ struct page *page;
+ struct bio_vec *bv;
int i;
+ bio_for_each_segment_all(bv, bio, i) {
+ page = bv->bv_page;
+
+ /* PG_error was set if any post_read step failed */
+ if (bio->bi_error || PageError(page)) {
+ ClearPageUptodate(page);
+ SetPageError(page);
+ } else {
+ SetPageUptodate(page);
+ }
+ unlock_page(page);
+ }
+ if (bio->bi_private)
+ mempool_free(bio->bi_private, bio_post_read_ctx_pool);
+ bio_put(bio);
+}
+
+static void bio_post_read_processing(struct bio_post_read_ctx *ctx);
+
+static void decrypt_work(struct work_struct *work)
+{
+ struct bio_post_read_ctx *ctx =
+ container_of(work, struct bio_post_read_ctx, work);
+
+ fscrypt_decrypt_bio(ctx->bio);
+
+ bio_post_read_processing(ctx);
+}
+
+static void bio_post_read_processing(struct bio_post_read_ctx *ctx)
+{
+ switch (++ctx->cur_step) {
+ case STEP_DECRYPT:
+ if (ctx->enabled_steps & (1 << STEP_DECRYPT)) {
+ INIT_WORK(&ctx->work, decrypt_work);
+ fscrypt_enqueue_decrypt_work(&ctx->work);
+ return;
+ }
+ ctx->cur_step++;
+ /* fall-through */
+ default:
+ __read_end_io(ctx->bio);
+ }
+}
+
+static bool f2fs_bio_post_read_required(struct bio *bio)
+{
+ return bio->bi_private && !bio->bi_error;
+}
+
+static void f2fs_read_end_io(struct bio *bio)
+{
#ifdef CONFIG_F2FS_FAULT_INJECTION
if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) {
f2fs_show_injection_info(FAULT_IO);
@@ -62,28 +131,15 @@ static void f2fs_read_end_io(struct bio *bio)
}
#endif
- if (f2fs_bio_encrypted(bio)) {
- if (bio->bi_error) {
- fscrypt_release_ctx(bio->bi_private);
- } else {
- fscrypt_decrypt_bio_pages(bio->bi_private, bio);
- return;
- }
- }
+ if (f2fs_bio_post_read_required(bio)) {
+ struct bio_post_read_ctx *ctx = bio->bi_private;
- bio_for_each_segment_all(bvec, bio, i) {
- struct page *page = bvec->bv_page;
-
- if (!bio->bi_error) {
- if (!PageUptodate(page))
- SetPageUptodate(page);
- } else {
- ClearPageUptodate(page);
- SetPageError(page);
- }
- unlock_page(page);
+ ctx->cur_step = STEP_INITIAL;
+ bio_post_read_processing(ctx);
+ return;
}
- bio_put(bio);
+
+ __read_end_io(bio);
}
static void f2fs_write_end_io(struct bio *bio)
@@ -480,29 +536,33 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
unsigned nr_pages)
{
struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
- struct fscrypt_ctx *ctx = NULL;
struct bio *bio;
-
- if (f2fs_encrypted_file(inode)) {
- ctx = fscrypt_get_ctx(inode, GFP_NOFS);
- if (IS_ERR(ctx))
- return ERR_CAST(ctx);
-
- /* wait the page to be moved by cleaning */
- f2fs_wait_on_block_writeback(sbi, blkaddr);
- }
+ struct bio_post_read_ctx *ctx;
+ unsigned int post_read_steps = 0;
bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false);
- if (!bio) {
- if (ctx)
- fscrypt_release_ctx(ctx);
+ if (!bio)
return ERR_PTR(-ENOMEM);
- }
f2fs_target_device(sbi, blkaddr, bio);
bio->bi_end_io = f2fs_read_end_io;
- bio->bi_private = ctx;
bio_set_op_attrs(bio, REQ_OP_READ, 0);
+ if (f2fs_encrypted_file(inode))
+ post_read_steps |= 1 << STEP_DECRYPT;
+ if (post_read_steps) {
+ ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
+ if (!ctx) {
+ bio_put(bio);
+ return ERR_PTR(-ENOMEM);
+ }
+ ctx->bio = bio;
+ ctx->enabled_steps = post_read_steps;
+ bio->bi_private = ctx;
+
+ /* wait the page to be moved by cleaning */
+ f2fs_wait_on_block_writeback(sbi, blkaddr);
+ }
+
return bio;
}
@@ -1523,7 +1583,7 @@ static int encrypt_one_page(struct f2fs_io_info *fio)
if (!f2fs_encrypted_file(inode))
return 0;
- /* wait for GCed encrypted page writeback */
+ /* wait for GCed page writeback via META_MAPPING */
f2fs_wait_on_block_writeback(fio->sbi, fio->old_blkaddr);
retry_encrypt:
@@ -1673,6 +1733,7 @@ got_it:
goto out_writepage;
set_page_writeback(page);
+ ClearPageError(page);
f2fs_put_dnode(&dn);
if (fio->need_lock == LOCK_REQ)
f2fs_unlock_op(fio->sbi);
@@ -1695,6 +1756,7 @@ got_it:
goto out_writepage;
set_page_writeback(page);
+ ClearPageError(page);
/* LFS mode write path */
write_data_page(&dn, fio);
@@ -2235,8 +2297,8 @@ repeat:
f2fs_wait_on_page_writeback(page, DATA, false);
- /* wait for GCed encrypted page writeback */
- if (f2fs_encrypted_file(inode))
+ /* wait for GCed page writeback via META_MAPPING */
+ if (f2fs_post_read_required(inode))
f2fs_wait_on_block_writeback(sbi, blkaddr);
if (len == PAGE_SIZE || PageUptodate(page))
@@ -2449,37 +2511,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)
return 1;
}
-/*
- * This was copied from __set_page_dirty_buffers which gives higher performance
- * in very high speed storages. (e.g., pmem)
- */
-void f2fs_set_page_dirty_nobuffers(struct page *page)
-{
- struct address_space *mapping = page->mapping;
- struct mem_cgroup *memcg;
- unsigned long flags;
-
- if (unlikely(!mapping))
- return;
-
- spin_lock(&mapping->private_lock);
- memcg = mem_cgroup_begin_page_stat(page);
- SetPageDirty(page);
- spin_unlock(&mapping->private_lock);
-
- spin_lock_irqsave(&mapping->tree_lock, flags);
- WARN_ON_ONCE(!PageUptodate(page));
- account_page_dirtied(page, mapping, memcg);
- radix_tree_tag_set(&mapping->page_tree,
- page_index(page), PAGECACHE_TAG_DIRTY);
- spin_unlock_irqrestore(&mapping->tree_lock, flags);
-
- mem_cgroup_end_page_stat(memcg);
-
- __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
- return;
-}
-
static int f2fs_set_data_page_dirty(struct page *page)
{
struct address_space *mapping = page->mapping;
@@ -2503,7 +2534,7 @@ static int f2fs_set_data_page_dirty(struct page *page)
}
if (!PageDirty(page)) {
- f2fs_set_page_dirty_nobuffers(page);
+ __set_page_dirty_nobuffers(page);
update_dirty_page(inode, page);
return 1;
}
@@ -2596,3 +2627,27 @@ const struct address_space_operations f2fs_dblock_aops = {
.migratepage = f2fs_migrate_page,
#endif
};
+
+int __init f2fs_init_post_read_processing(void)
+{
+ bio_post_read_ctx_cache = KMEM_CACHE(bio_post_read_ctx, 0);
+ if (!bio_post_read_ctx_cache)
+ goto fail;
+ bio_post_read_ctx_pool =
+ mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
+ bio_post_read_ctx_cache);
+ if (!bio_post_read_ctx_pool)
+ goto fail_free_cache;
+ return 0;
+
+fail_free_cache:
+ kmem_cache_destroy(bio_post_read_ctx_cache);
+fail:
+ return -ENOMEM;
+}
+
+void __exit f2fs_destroy_post_read_processing(void)
+{
+ mempool_destroy(bio_post_read_ctx_pool);
+ kmem_cache_destroy(bio_post_read_ctx_cache);
+}
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index e258f6536b56..d0bfcfed35e2 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -1673,7 +1673,7 @@ static inline bool f2fs_has_xattr_block(unsigned int ofs)
}
static inline bool __allow_reserved_blocks(struct f2fs_sb_info *sbi,
- struct inode *inode)
+ struct inode *inode, bool cap)
{
if (!inode)
return true;
@@ -1686,7 +1686,7 @@ static inline bool __allow_reserved_blocks(struct f2fs_sb_info *sbi,
if (!gid_eq(F2FS_OPTION(sbi).s_resgid, GLOBAL_ROOT_GID) &&
in_group_p(F2FS_OPTION(sbi).s_resgid))
return true;
- if (capable(CAP_SYS_RESOURCE))
+ if (cap && capable(CAP_SYS_RESOURCE))
return true;
return false;
}
@@ -1721,7 +1721,7 @@ static inline int inc_valid_block_count(struct f2fs_sb_info *sbi,
avail_user_block_count = sbi->user_block_count -
sbi->current_reserved_blocks;
- if (!__allow_reserved_blocks(sbi, inode))
+ if (!__allow_reserved_blocks(sbi, inode, true))
avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks;
if (unlikely(sbi->total_valid_block_count > avail_user_block_count)) {
@@ -1928,7 +1928,7 @@ static inline int inc_valid_node_count(struct f2fs_sb_info *sbi,
valid_block_count = sbi->total_valid_block_count +
sbi->current_reserved_blocks + 1;
- if (!__allow_reserved_blocks(sbi, inode))
+ if (!__allow_reserved_blocks(sbi, inode, false))
valid_block_count += F2FS_OPTION(sbi).root_reserved_blocks;
if (unlikely(valid_block_count > sbi->user_block_count)) {
@@ -2937,6 +2937,8 @@ void destroy_checkpoint_caches(void);
/*
* data.c
*/
+int f2fs_init_post_read_processing(void);
+void f2fs_destroy_post_read_processing(void);
void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
struct inode *inode, nid_t ino, pgoff_t idx,
@@ -2968,7 +2970,6 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
u64 start, u64 len);
bool should_update_inplace(struct inode *inode, struct f2fs_io_info *fio);
bool should_update_outplace(struct inode *inode, struct f2fs_io_info *fio);
-void f2fs_set_page_dirty_nobuffers(struct page *page);
int __f2fs_write_data_pages(struct address_space *mapping,
struct writeback_control *wbc,
enum iostat_type io_type);
@@ -3297,9 +3298,13 @@ static inline void f2fs_set_encrypted_inode(struct inode *inode)
#endif
}
-static inline bool f2fs_bio_encrypted(struct bio *bio)
+/*
+ * Returns true if the reads of the inode's data need to undergo some
+ * postprocessing step, like decryption or authenticity verification.
+ */
+static inline bool f2fs_post_read_required(struct inode *inode)
{
- return bio->bi_private != NULL;
+ return f2fs_encrypted_file(inode);
}
#define F2FS_FEATURE_FUNCS(name, flagname) \
@@ -3367,7 +3372,7 @@ static inline bool f2fs_may_encrypt(struct inode *inode)
static inline bool f2fs_force_buffered_io(struct inode *inode, int rw)
{
- return (f2fs_encrypted_file(inode) ||
+ return (f2fs_post_read_required(inode) ||
(rw == WRITE && test_opt(F2FS_I_SB(inode), LFS)) ||
F2FS_I_SB(inode)->s_ndevs);
}
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index 39c3acb454a3..7587758a285f 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -113,8 +113,8 @@ mapped:
/* fill the page */
f2fs_wait_on_page_writeback(page, DATA, false);
- /* wait for GCed encrypted page writeback */
- if (f2fs_encrypted_file(inode))
+ /* wait for GCed page writeback via META_MAPPING */
+ if (f2fs_post_read_required(inode))
f2fs_wait_on_block_writeback(sbi, dn.data_blkaddr);
out_sem:
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index 54f51a990794..c009b50d69f5 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -850,8 +850,8 @@ next_step:
if (IS_ERR(inode) || is_bad_inode(inode))
continue;
- /* if encrypted inode, let's go phase 3 */
- if (f2fs_encrypted_file(inode)) {
+ /* if inode uses special I/O path, let's go phase 3 */
+ if (f2fs_post_read_required(inode)) {
add_gc_inode(gc_list, inode);
continue;
}
@@ -899,7 +899,7 @@ next_step:
start_bidx = start_bidx_of_node(nofs, inode)
+ ofs_in_node;
- if (f2fs_encrypted_file(inode))
+ if (f2fs_post_read_required(inode))
move_data_block(inode, start_bidx, segno, off);
else
move_data_page(inode, start_bidx, gc_type,
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
index 37ab2159506a..922a213693c1 100644
--- a/fs/f2fs/inline.c
+++ b/fs/f2fs/inline.c
@@ -26,7 +26,7 @@ bool f2fs_may_inline_data(struct inode *inode)
if (i_size_read(inode) > MAX_INLINE_DATA(inode))
return false;
- if (f2fs_encrypted_file(inode))
+ if (f2fs_post_read_required(inode))
return false;
return true;
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index 5ec20f077629..fecae8685d2a 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -294,8 +294,8 @@ static int f2fs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
alloc_nid_done(sbi, ino);
- d_instantiate(dentry, inode);
unlock_new_inode(inode);
+ d_instantiate(dentry, inode);
if (IS_DIRSYNC(dir))
f2fs_sync_fs(sbi->sb, 1);
@@ -594,8 +594,8 @@ static int f2fs_symlink(struct inode *dir, struct dentry *dentry,
err = page_symlink(inode, disk_link.name, disk_link.len);
err_out:
- d_instantiate(dentry, inode);
unlock_new_inode(inode);
+ d_instantiate(dentry, inode);
/*
* Let's flush symlink data in order to avoid broken symlink as much as
@@ -658,8 +658,8 @@ static int f2fs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
alloc_nid_done(sbi, inode->i_ino);
- d_instantiate(dentry, inode);
unlock_new_inode(inode);
+ d_instantiate(dentry, inode);
if (IS_DIRSYNC(dir))
f2fs_sync_fs(sbi->sb, 1);
@@ -710,8 +710,8 @@ static int f2fs_mknod(struct inode *dir, struct dentry *dentry,
alloc_nid_done(sbi, inode->i_ino);
- d_instantiate(dentry, inode);
unlock_new_inode(inode);
+ d_instantiate(dentry, inode);
if (IS_DIRSYNC(dir))
f2fs_sync_fs(sbi->sb, 1);
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index 157d768c7b31..3871e7d3f69e 100644
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -1775,7 +1775,7 @@ static int f2fs_set_node_page_dirty(struct page *page)
if (!PageUptodate(page))
SetPageUptodate(page);
if (!PageDirty(page)) {
- f2fs_set_page_dirty_nobuffers(page);
+ __set_page_dirty_nobuffers(page);
inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_NODES);
SetPagePrivate(page);
f2fs_trace_pid(page);
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index a622eb4f59f2..55b2bad55671 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -3100,8 +3100,13 @@ static int __init init_f2fs_fs(void)
err = f2fs_create_root_stats();
if (err)
goto free_filesystem;
+ err = f2fs_init_post_read_processing();
+ if (err)
+ goto free_root_stats;
return 0;
+free_root_stats:
+ f2fs_destroy_root_stats();
free_filesystem:
unregister_filesystem(&f2fs_fs_type);
free_shrinker:
@@ -3124,6 +3129,7 @@ fail:
static void __exit exit_f2fs_fs(void)
{
+ f2fs_destroy_post_read_processing();
f2fs_destroy_root_stats();
unregister_filesystem(&f2fs_fs_type);
unregister_shrinker(&f2fs_shrinker_info);
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-23 09:53:15 +0000