#include <linux/writeback.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/completion.h>
#include <linux/iomap.h>
#include <linux/math64.h>
#include "bitmap.h"
#include "lcnalloc.h"
#include "mft.h"
#include "ntfs.h"
int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m,
u64 mft_no)
{
struct attr_record *a;
struct super_block *sb = vol->sb;
u16 attrs_offset;
u32 bytes_in_use;
if (!ntfs_is_file_record(m->magic)) {
ntfs_error(sb, "Record %llu has no FILE magic (0x%x)\n",
mft_no, le32_to_cpu(*(__le32 *)m));
goto err_out;
}
if (le16_to_cpu(m->usa_ofs) & 0x1 ||
(vol->mft_record_size >> NTFS_BLOCK_SIZE_BITS) + 1 != le16_to_cpu(m->usa_count) ||
le16_to_cpu(m->usa_ofs) + le16_to_cpu(m->usa_count) * 2 > vol->mft_record_size) {
ntfs_error(sb, "Record %llu has corrupt fix-up values fields\n",
mft_no);
goto err_out;
}
if (le32_to_cpu(m->bytes_allocated) != vol->mft_record_size) {
ntfs_error(sb, "Record %llu has corrupt allocation size (%u <> %u)\n",
mft_no, vol->mft_record_size,
le32_to_cpu(m->bytes_allocated));
goto err_out;
}
if (le32_to_cpu(m->bytes_in_use) > vol->mft_record_size) {
ntfs_error(sb, "Record %llu has corrupt in-use size (%u > %u)\n",
mft_no, le32_to_cpu(m->bytes_in_use),
vol->mft_record_size);
goto err_out;
}
if (le16_to_cpu(m->attrs_offset) & 7) {
ntfs_error(sb, "Attributes badly aligned in record %llu\n",
mft_no);
goto err_out;
}
attrs_offset = le16_to_cpu(m->attrs_offset);
bytes_in_use = le32_to_cpu(m->bytes_in_use);
if (attrs_offset > bytes_in_use ||
bytes_in_use - attrs_offset < sizeof_field(struct attr_record, type)) {
ntfs_error(sb, "Record %llu has corrupt attribute offset\n", mft_no);
goto err_out;
}
a = (struct attr_record *)((char *)m + attrs_offset);
if ((char *)a < (char *)m || (char *)a > (char *)m + vol->mft_record_size) {
ntfs_error(sb, "Record %llu is corrupt\n", mft_no);
goto err_out;
}
return 0;
err_out:
return -EIO;
}
static inline struct mft_record *map_mft_record_folio(struct ntfs_inode *ni)
{
loff_t i_size;
struct ntfs_volume *vol = ni->vol;
struct inode *mft_vi = vol->mft_ino;
struct folio *folio;
unsigned long index, end_index;
unsigned int ofs;
WARN_ON(ni->folio);
index = NTFS_MFT_NR_TO_PIDX(vol, ni->mft_no);
ofs = NTFS_MFT_NR_TO_POFS(vol, ni->mft_no);
i_size = i_size_read(mft_vi);
end_index = i_size >> PAGE_SHIFT;
if (unlikely(index >= end_index)) {
if (index > end_index || (i_size & ~PAGE_MASK) < ofs +
vol->mft_record_size) {
folio = ERR_PTR(-ENOENT);
ntfs_error(vol->sb,
"Attempt to read mft record 0x%llx, which is beyond the end of the mft. This is probably a bug in the ntfs driver.",
ni->mft_no);
goto err_out;
}
}
folio = read_mapping_folio(mft_vi->i_mapping, index, NULL);
if (!IS_ERR(folio)) {
u8 *addr;
ni->mrec = kmalloc(vol->mft_record_size, GFP_NOFS);
if (!ni->mrec) {
folio_put(folio);
folio = ERR_PTR(-ENOMEM);
goto err_out;
}
addr = kmap_local_folio(folio, 0);
memcpy(ni->mrec, addr + ofs, vol->mft_record_size);
post_read_mst_fixup((struct ntfs_record *)ni->mrec, vol->mft_record_size);
if (!ntfs_mft_record_check(vol, (struct mft_record *)ni->mrec, ni->mft_no)) {
kunmap_local(addr);
ni->folio = folio;
ni->folio_ofs = ofs;
return ni->mrec;
}
kunmap_local(addr);
folio_put(folio);
kfree(ni->mrec);
ni->mrec = NULL;
folio = ERR_PTR(-EIO);
NVolSetErrors(vol);
}
err_out:
ni->folio = NULL;
ni->folio_ofs = 0;
return (struct mft_record *)folio;
}
struct mft_record *map_mft_record(struct ntfs_inode *ni)
{
struct mft_record *m;
if (!ni)
return ERR_PTR(-EINVAL);
ntfs_debug("Entering for mft_no 0x%llx.", ni->mft_no);
atomic_inc(&ni->count);
if (ni->folio)
return (struct mft_record *)ni->mrec;
m = map_mft_record_folio(ni);
if (!IS_ERR(m))
return m;
atomic_dec(&ni->count);
if (PTR_ERR(m) != -EINTR && PTR_ERR(m) != -ERESTARTSYS)
ntfs_error(ni->vol->sb, "Failed with error code %lu.", -PTR_ERR(m));
return m;
}
void unmap_mft_record(struct ntfs_inode *ni)
{
struct folio *folio;
if (!ni)
return;
ntfs_debug("Entering for mft_no 0x%llx.", ni->mft_no);
folio = ni->folio;
if (atomic_dec_return(&ni->count) > 1)
return;
WARN_ON(!folio);
}
struct mft_record *map_extent_mft_record(struct ntfs_inode *base_ni, u64 mref,
struct ntfs_inode **ntfs_ino)
{
struct mft_record *m;
struct ntfs_inode *ni = NULL;
struct ntfs_inode **extent_nis = NULL;
int i;
u64 mft_no = MREF(mref);
u16 seq_no = MSEQNO(mref);
bool destroy_ni = false;
ntfs_debug("Mapping extent mft record 0x%llx (base mft record 0x%llx).",
mft_no, base_ni->mft_no);
atomic_inc(&base_ni->count);
retry:
mutex_lock(&base_ni->extent_lock);
if (base_ni->nr_extents > 0) {
extent_nis = base_ni->ext.extent_ntfs_inos;
for (i = 0; i < base_ni->nr_extents; i++) {
if (mft_no != extent_nis[i]->mft_no)
continue;
ni = extent_nis[i];
atomic_inc(&ni->count);
break;
}
}
if (likely(ni != NULL)) {
mutex_unlock(&base_ni->extent_lock);
atomic_dec(&base_ni->count);
m = map_mft_record(ni);
atomic_dec(&ni->count);
if (!IS_ERR(m)) {
if (likely(le16_to_cpu(m->sequence_number) == seq_no)) {
ntfs_debug("Done 1.");
*ntfs_ino = ni;
return m;
}
unmap_mft_record(ni);
ntfs_error(base_ni->vol->sb,
"Found stale extent mft reference! Corrupt filesystem. Run chkdsk.");
return ERR_PTR(-EIO);
}
map_err_out:
ntfs_error(base_ni->vol->sb,
"Failed to map extent mft record, error code %ld.",
-PTR_ERR(m));
return m;
}
mutex_unlock(&base_ni->extent_lock);
ni = ntfs_new_extent_inode(base_ni->vol->sb, mft_no);
if (unlikely(!ni)) {
atomic_dec(&base_ni->count);
return ERR_PTR(-ENOMEM);
}
ni->vol = base_ni->vol;
ni->seq_no = seq_no;
ni->nr_extents = -1;
ni->ext.base_ntfs_ino = base_ni;
m = map_mft_record(ni);
if (IS_ERR(m)) {
atomic_dec(&base_ni->count);
ntfs_clear_extent_inode(ni);
goto map_err_out;
}
if (seq_no && (le16_to_cpu(m->sequence_number) != seq_no)) {
ntfs_error(base_ni->vol->sb,
"Found stale extent mft reference! Corrupt filesystem. Run chkdsk.");
destroy_ni = true;
m = ERR_PTR(-EIO);
goto unm_nolock_err_out;
}
mutex_lock(&base_ni->extent_lock);
for (i = 0; i < base_ni->nr_extents; i++) {
if (mft_no == extent_nis[i]->mft_no) {
mutex_unlock(&base_ni->extent_lock);
ntfs_clear_extent_inode(ni);
goto retry;
}
}
if (!(base_ni->nr_extents & 3)) {
struct ntfs_inode **tmp;
int new_size = (base_ni->nr_extents + 4) * sizeof(struct ntfs_inode *);
tmp = kvzalloc(new_size, GFP_NOFS);
if (unlikely(!tmp)) {
ntfs_error(base_ni->vol->sb, "Failed to allocate internal buffer.");
destroy_ni = true;
m = ERR_PTR(-ENOMEM);
goto unm_err_out;
}
if (base_ni->nr_extents) {
WARN_ON(!base_ni->ext.extent_ntfs_inos);
memcpy(tmp, base_ni->ext.extent_ntfs_inos, new_size -
4 * sizeof(struct ntfs_inode *));
kvfree(base_ni->ext.extent_ntfs_inos);
}
base_ni->ext.extent_ntfs_inos = tmp;
}
base_ni->ext.extent_ntfs_inos[base_ni->nr_extents++] = ni;
mutex_unlock(&base_ni->extent_lock);
atomic_dec(&base_ni->count);
ntfs_debug("Done 2.");
*ntfs_ino = ni;
return m;
unm_err_out:
mutex_unlock(&base_ni->extent_lock);
unm_nolock_err_out:
unmap_mft_record(ni);
atomic_dec(&base_ni->count);
if (destroy_ni)
ntfs_clear_extent_inode(ni);
return m;
}
void __mark_mft_record_dirty(struct ntfs_inode *ni)
{
struct ntfs_inode *base_ni;
ntfs_debug("Entering for inode 0x%llx.", ni->mft_no);
WARN_ON(NInoAttr(ni));
if (likely(ni->nr_extents >= 0))
base_ni = ni;
else
base_ni = ni->ext.base_ntfs_ino;
__mark_inode_dirty(VFS_I(base_ni), I_DIRTY_DATASYNC);
}
struct ntfs_mft_io_unit {
sector_t sector;
unsigned int folio_ofs;
unsigned int len;
};
struct ntfs_mft_write_ctx {
struct folio *folio;
struct address_space *mapping;
struct ntfs_volume *vol;
struct completion *done;
int error;
bool writeback_started;
struct bio bio;
};
static struct bio_set ntfs_mft_bioset;
int ntfs_mft_bioset_init(void)
{
return bioset_init(&ntfs_mft_bioset, BIO_POOL_SIZE,
offsetof(struct ntfs_mft_write_ctx, bio),
BIOSET_NEED_BVECS);
}
void ntfs_mft_bioset_exit(void)
{
bioset_exit(&ntfs_mft_bioset);
}
static void ntfs_mft_end_io(struct bio *bio)
{
struct ntfs_mft_write_ctx *ctx =
container_of(bio, struct ntfs_mft_write_ctx, bio);
struct folio *folio = ctx->folio;
struct completion *done = ctx->done;
int err;
if (bio->bi_status)
err = blk_status_to_errno(bio->bi_status);
else
err = ctx->error;
if (err) {
mapping_set_error(ctx->mapping, err);
NVolSetErrors(ctx->vol);
ntfs_error(ctx->vol->sb, "I/O error while writing MFT: %d",
err);
}
if (!done)
bio_put(bio);
folio_end_writeback(folio);
folio_put(folio);
if (done)
complete(done);
}
static void ntfs_start_mft_writeback(struct ntfs_mft_write_ctx *ctx)
{
if (ctx->writeback_started)
return;
inode_attach_wb(ctx->mapping->host, NULL);
folio_get(ctx->folio);
folio_start_writeback(ctx->folio);
ctx->writeback_started = true;
}
static struct bio *ntfs_alloc_mft_parent_bio(struct ntfs_volume *vol,
struct folio *folio,
struct completion *done)
{
struct ntfs_mft_write_ctx *ctx;
struct bio *parent;
parent = bio_alloc_bioset(vol->sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOIO,
&ntfs_mft_bioset);
if (!parent)
return NULL;
ctx = container_of(parent, struct ntfs_mft_write_ctx, bio);
ctx->folio = folio;
ctx->mapping = folio->mapping;
ctx->vol = vol;
ctx->done = done;
ctx->error = 0;
ctx->writeback_started = false;
parent->bi_end_io = ntfs_mft_end_io;
return parent;
}
static int ntfs_sync_mft_mirror_unit(struct ntfs_volume *vol,
struct folio *source, u64 mirror_file_ofs,
const struct ntfs_mft_io_unit *unit)
{
struct folio *mirror;
struct bio_vec bvec;
struct bio bio;
u64 mirror_size;
unsigned int mirror_ofs;
u8 *src, *dst;
int err;
if (unlikely(!vol->mftmirr_ino))
return -EIO;
mirror_size = (u64)vol->mftmirr_size * vol->mft_record_size;
if (mirror_file_ofs >= mirror_size ||
unit->len > mirror_size - mirror_file_ofs)
return -EIO;
if (unit->folio_ofs + unit->len > folio_size(source))
return -EIO;
mirror = read_mapping_folio(vol->mftmirr_ino->i_mapping,
mirror_file_ofs >> PAGE_SHIFT, NULL);
if (IS_ERR(mirror))
return PTR_ERR(mirror);
folio_lock(mirror);
if (folio_test_writeback(mirror))
folio_wait_writeback(mirror);
mirror_ofs = mirror_file_ofs - folio_pos(mirror);
if (mirror_ofs + unit->len > folio_size(mirror)) {
err = -EIO;
goto out_unlock;
}
folio_clear_uptodate(mirror);
src = kmap_local_folio(source, unit->folio_ofs);
dst = kmap_local_folio(mirror, mirror_ofs);
memcpy(dst, src, unit->len);
kunmap_local(dst);
kunmap_local(src);
bio_init(&bio, vol->sb->s_bdev, &bvec, 1, REQ_OP_WRITE);
bio.bi_iter.bi_sector = ntfs_bytes_to_bio_sector(
NTFS_CLU_TO_B(vol, vol->mftmirr_lcn) + mirror_file_ofs);
if (!bio_add_folio(&bio, mirror, unit->len, mirror_ofs))
err = -EIO;
else
err = submit_bio_wait(&bio);
bio_uninit(&bio);
folio_mark_uptodate(mirror);
out_unlock:
folio_unlock(mirror);
folio_put(mirror);
return err;
}
static int ntfs_sync_mft_mirror_record(struct ntfs_volume *vol,
struct folio *source, const u64 mft_no)
{
u64 mirror_file_ofs = (u64)mft_no * vol->mft_record_size;
struct ntfs_mft_io_unit unit = {
.folio_ofs = NTFS_MFT_NR_TO_POFS(vol, mft_no),
.len = vol->mft_record_size,
};
mirror_file_ofs =
round_down(mirror_file_ofs, (u64)vol->mft_io_unit_size);
unit.folio_ofs = round_down(unit.folio_ofs, vol->mft_io_unit_size);
unit.len = vol->mft_io_unit_size;
return ntfs_sync_mft_mirror_unit(vol, source, mirror_file_ofs, &unit);
}
static int ntfs_prepare_mft_record_io_units(struct ntfs_inode *ni,
struct ntfs_mft_io_unit units[2])
{
struct ntfs_volume *vol = ni->vol;
u64 record_byte = (u64)ni->mft_no * vol->mft_record_size;
u64 disk_byte;
unsigned int nr_units = ni->mft_lcn_count;
unsigned int cluster_ofs;
unsigned int i;
if (!nr_units || nr_units > ARRAY_SIZE(ni->mft_lcn))
return -EIO;
for (i = 0; i < nr_units; i++)
if (ni->mft_lcn[i] < 0)
return -EIO;
cluster_ofs = ntfs_bytes_to_cluster_off(vol, record_byte);
if (vol->mft_io_unit_size > vol->mft_record_size) {
cluster_ofs = round_down(cluster_ofs, vol->mft_io_unit_size);
disk_byte = NTFS_CLU_TO_B(vol, ni->mft_lcn[0]) + cluster_ofs;
units[0] = (struct ntfs_mft_io_unit){
.sector = ntfs_bytes_to_bio_sector(disk_byte),
.folio_ofs = round_down(ni->folio_ofs,
vol->mft_io_unit_size),
.len = vol->mft_io_unit_size,
};
return 1;
}
disk_byte = NTFS_CLU_TO_B(vol, ni->mft_lcn[0]) + cluster_ofs;
units[0] = (struct ntfs_mft_io_unit){
.sector = ntfs_bytes_to_bio_sector(disk_byte),
.folio_ofs = ni->folio_ofs,
.len = vol->mft_record_size,
};
if (nr_units == 1)
return 1;
units[0].len = vol->cluster_size;
disk_byte = NTFS_CLU_TO_B(vol, ni->mft_lcn[1]);
units[1] = (struct ntfs_mft_io_unit){
.sector = ntfs_bytes_to_bio_sector(disk_byte),
.folio_ofs = ni->folio_ofs + vol->cluster_size,
.len = vol->mft_record_size - vol->cluster_size,
};
return 2;
}
int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int sync)
{
struct ntfs_volume *vol = ni->vol;
struct folio *folio = ni->folio;
DECLARE_COMPLETION_ONSTACK(done);
struct ntfs_mft_io_unit units[2];
struct ntfs_mft_write_ctx *ctx;
struct bio *parent, *child = NULL;
unsigned int nr_units;
int err = 0;
u8 *kaddr;
struct mft_record *fixup_m;
ntfs_debug("Entering for inode 0x%llx.", ni->mft_no);
WARN_ON(NInoAttr(ni));
WARN_ON(!folio_test_locked(folio));
if (folio_test_writeback(folio))
folio_wait_writeback(folio);
if (!NInoTestClearDirty(ni))
goto done;
err = ntfs_prepare_mft_record_io_units(ni, units);
if (err < 0)
goto err_out;
nr_units = err;
kaddr = kmap_local_folio(folio, 0);
fixup_m = (struct mft_record *)(kaddr + ni->folio_ofs);
memcpy(fixup_m, m, vol->mft_record_size);
err = pre_write_mst_fixup((struct ntfs_record *)fixup_m, vol->mft_record_size);
if (err) {
ntfs_error(vol->sb, "Failed to apply mst fixups!");
goto unmap_err_out;
}
parent = ntfs_alloc_mft_parent_bio(vol, folio, sync ? &done : NULL);
if (!parent) {
err = -ENOMEM;
goto unmap_err_out;
}
ctx = container_of(parent, struct ntfs_mft_write_ctx, bio);
parent->bi_iter.bi_sector = units[0].sector;
if (!bio_add_folio(parent, folio, units[0].len, units[0].folio_ofs)) {
err = -EIO;
goto put_bios_out;
}
if (nr_units == 2) {
if (bio_end_sector(parent) != units[1].sector ||
!bio_add_folio(parent, folio, units[1].len,
units[1].folio_ofs)) {
child = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE,
GFP_NOIO);
if (!child) {
err = -ENOMEM;
goto put_bios_out;
}
child->bi_iter.bi_sector = units[1].sector;
if (!bio_add_folio(child, folio, units[1].len,
units[1].folio_ofs)) {
err = -EIO;
goto put_bios_out;
}
}
}
if (ni->mft_no < vol->mftmirr_size) {
err = ntfs_sync_mft_mirror_record(vol, folio, ni->mft_no);
if (err)
ctx->error = err;
}
kunmap_local(kaddr);
ntfs_start_mft_writeback(ctx);
if (child) {
bio_chain(child, parent);
submit_bio(child);
}
submit_bio(parent);
if (sync) {
wait_for_completion(&done);
if (!err && parent->bi_status)
err = blk_status_to_errno(parent->bi_status);
bio_put(parent);
}
if (unlikely(err)) {
ntfs_error(vol->sb,
"I/O error while writing mft record 0x%llx! Marking base inode as bad. You should unmount the volume and run chkdsk.",
ni->mft_no);
return err;
}
done:
ntfs_debug("Done.");
return 0;
put_bios_out:
if (child)
bio_put(child);
bio_put(parent);
unmap_err_out:
kunmap_local(kaddr);
err_out:
if (err == -ENOMEM) {
ntfs_error(vol->sb,
"Not enough memory to write mft record. Redirtying so the write is retried later.");
mark_mft_record_dirty(ni);
if (!sync)
err = 0;
} else
NVolSetErrors(vol);
return err;
}
static int ntfs_test_inode_wb(struct inode *vi, u64 ino, void *data)
{
struct ntfs_attr *na = data;
if (!ntfs_test_inode(vi, na))
return 0;
if (NInoBeingDeleted(NTFS_I(vi))) {
na->state = NI_BeingDeleted;
return -1;
}
spin_lock(&vi->i_lock);
if (inode_state_read_once(vi) & I_CREATING) {
spin_unlock(&vi->i_lock);
na->state = NI_BeingCreated;
return -1;
}
spin_unlock(&vi->i_lock);
return igrab(vi) ? 1 : -1;
}
static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no,
const struct mft_record *m, struct ntfs_inode **locked_ni,
struct inode **ref_vi)
{
struct super_block *sb = vol->sb;
struct inode *mft_vi = vol->mft_ino;
struct inode *vi;
struct ntfs_inode *ni;
struct ntfs_attr na = {0};
ntfs_debug("Entering for inode 0x%llx.", mft_no);
*locked_ni = NULL;
*ref_vi = NULL;
ntfs_debug("Looking for inode 0x%llx in icache.", mft_no);
na.mft_no = mft_no;
na.type = AT_UNUSED;
if (!mft_no) {
vi = igrab(mft_vi);
WARN_ON(vi != mft_vi);
} else {
vi = find_inode_nowait(sb, mft_no, ntfs_test_inode_wb, &na);
if (na.state == NI_BeingDeleted || na.state == NI_BeingCreated)
return false;
}
if (vi) {
ntfs_debug("Base inode 0x%llx is in icache.", mft_no);
ni = NTFS_I(vi);
atomic_inc(&ni->count);
if (NInoDirty(ni)) {
ntfs_debug("Inode 0x%llx is dirty, do not write it.",
mft_no);
atomic_dec(&ni->count);
*ref_vi = vi;
return false;
}
ntfs_debug("Inode 0x%llx is not dirty.", mft_no);
if (unlikely(!mutex_trylock(&ni->mrec_lock))) {
ntfs_debug("Mft record 0x%llx is already locked, do not write it.", mft_no);
atomic_dec(&ni->count);
*ref_vi = vi;
return false;
}
ntfs_debug("Managed to lock mft record 0x%llx, write it.",
mft_no);
*locked_ni = ni;
*ref_vi = vi;
return true;
}
ntfs_debug("Inode 0x%llx is not in icache.", mft_no);
if (!ntfs_is_mft_record(m->magic)) {
ntfs_debug("Mft record 0x%llx is not a FILE record, write it.",
mft_no);
return true;
}
if (!m->base_mft_record) {
ntfs_debug("Mft record 0x%llx is a base record, write it.",
mft_no);
return true;
}
ntfs_debug("Mft record 0x%llx is an extent record, skip it.",
mft_no);
return false;
}
static const char *es = " Leaving inconsistent metadata. Unmount and run chkdsk.";
#define FIRST_NORMAL_MFT_RECORD 24
#define MFT_RECORD_RESERVE 4
static bool mft_reserved_is_free(struct ntfs_volume *vol,
struct ntfs_inode *mft_ni, s64 mft_no)
{
struct attr_record *a;
struct mft_record *m;
struct folio *folio;
void *mapped;
pgoff_t index = NTFS_MFT_NR_TO_PIDX(vol, mft_no);
unsigned int ofs = NTFS_MFT_NR_TO_POFS(vol, mft_no);
u32 attrs_offset, bytes_in_use;
bool available = false, have_std = false;
int i;
for (i = 0; i < mft_ni->nr_extents; i++) {
if (mft_ni->ext.extent_ntfs_inos[i] &&
mft_ni->ext.extent_ntfs_inos[i]->mft_no == mft_no)
return false;
}
m = kmalloc(vol->mft_record_size, GFP_NOFS);
if (!m)
return false;
folio = read_mapping_folio(vol->mft_ino->i_mapping, index, NULL);
if (IS_ERR(folio))
goto free_m;
folio_lock(folio);
mapped = kmap_local_folio(folio, 0);
memcpy(m, (u8 *)mapped + ofs, vol->mft_record_size);
kunmap_local(mapped);
folio_unlock(folio);
folio_put(folio);
if (post_read_mst_fixup((struct ntfs_record *)m, vol->mft_record_size))
goto free_m;
if (!ntfs_is_mft_record(m->magic) ||
!(m->flags & MFT_RECORD_IN_USE) || m->base_mft_record ||
m->link_count)
goto out;
attrs_offset = le16_to_cpu(m->attrs_offset);
bytes_in_use = le32_to_cpu(m->bytes_in_use);
if (attrs_offset > bytes_in_use || bytes_in_use > vol->mft_record_size ||
bytes_in_use - attrs_offset < sizeof(a->type))
goto out;
for (a = (struct attr_record *)((u8 *)m + attrs_offset);
(u8 *)a + sizeof(a->type) <= (u8 *)m + bytes_in_use;) {
u32 len;
if (a->type == AT_END) {
if ((u8 *)a + sizeof(a->type) + sizeof(a->length) >
(u8 *)m + bytes_in_use)
break;
available = have_std ||
(u8 *)a == (u8 *)m + attrs_offset;
break;
}
if (a->type == AT_FILE_NAME)
break;
len = le32_to_cpu(a->length);
if (len < offsetof(struct attr_record, data) ||
(u8 *)a + len > (u8 *)m + bytes_in_use)
break;
if (a->type == AT_STANDARD_INFORMATION) {
u32 value_len, value_ofs;
if (have_std || a->non_resident ||
len < offsetof(struct attr_record,
data.resident.reserved) + 1)
break;
value_len = le32_to_cpu(a->data.resident.value_length);
value_ofs = le16_to_cpu(a->data.resident.value_offset);
if (value_ofs > len || value_len > len - value_ofs)
break;
have_std = true;
}
a = (struct attr_record *)((u8 *)a + len);
}
out:
kfree(m);
return available;
free_m:
kfree(m);
return false;
}
static s64 mft_reserve_end(const u8 *buf, s64 buf_start, s64 buf_end,
s64 start, s64 pass_end, s64 initialized_mft_records)
{
s64 end = start + 1;
s64 limit = min_t(s64, start + MFT_RECORD_RESERVE, pass_end);
if (limit > initialized_mft_records)
limit = initialized_mft_records;
if (limit > buf_end)
limit = buf_end;
while (end < limit &&
!(buf[(end - buf_start) >> 3] &
(1 << ((end - buf_start) & 7))))
end++;
return end;
}
static s64 mft_bitmap_alloc_free_rec(struct ntfs_volume *vol,
struct ntfs_inode *base_ni,
s64 max_mft_no, s64 *new_reserve_end)
{
s64 pass_end, ll, data_pos, pass_start, ofs, bit;
s64 initialized_mft_records;
unsigned long flags;
struct address_space *mftbmp_mapping;
u8 *buf = NULL, *byte;
struct folio *folio;
unsigned int folio_ofs, size;
u8 pass, b;
ntfs_debug("Searching for free mft record in the currently initialized mft bitmap.");
mftbmp_mapping = vol->mftbmp_ino->i_mapping;
read_lock_irqsave(&NTFS_I(vol->mft_ino)->size_lock, flags);
pass_end = NTFS_I(vol->mft_ino)->allocated_size >>
vol->mft_record_size_bits;
initialized_mft_records = NTFS_I(vol->mft_ino)->initialized_size >>
vol->mft_record_size_bits;
read_unlock_irqrestore(&NTFS_I(vol->mft_ino)->size_lock, flags);
read_lock_irqsave(&NTFS_I(vol->mftbmp_ino)->size_lock, flags);
ll = NTFS_I(vol->mftbmp_ino)->initialized_size << 3;
read_unlock_irqrestore(&NTFS_I(vol->mftbmp_ino)->size_lock, flags);
if (pass_end > ll)
pass_end = ll;
if (max_mft_no >= 0 && pass_end > max_mft_no)
pass_end = max_mft_no;
if (base_ni && base_ni->mft_no == FILE_MFT) {
data_pos = FILE_first_user;
pass = 2;
if (data_pos >= pass_end)
return -ENOSPC;
} else {
pass = 1;
if (!base_ni)
data_pos = vol->mft_data_pos;
else
data_pos = base_ni->mft_no + 1;
if (data_pos < FIRST_NORMAL_MFT_RECORD)
data_pos = FIRST_NORMAL_MFT_RECORD;
if (data_pos >= pass_end) {
data_pos = FIRST_NORMAL_MFT_RECORD;
pass = 2;
if (data_pos >= pass_end)
return -ENOSPC;
}
}
pass_start = data_pos;
ntfs_debug("Starting bitmap search: pass %u, pass_start 0x%llx, pass_end 0x%llx, data_pos 0x%llx.",
pass, pass_start, pass_end, data_pos);
for (; pass <= 2;) {
ofs = data_pos >> 3;
folio_ofs = ofs & ~PAGE_MASK;
size = PAGE_SIZE - folio_ofs;
ll = ((pass_end + 7) >> 3) - ofs;
if (size > ll)
size = ll;
size <<= 3;
if (size) {
folio = read_mapping_folio(mftbmp_mapping,
ofs >> PAGE_SHIFT, NULL);
if (IS_ERR(folio)) {
ntfs_error(vol->sb, "Failed to read mft bitmap, aborting.");
return PTR_ERR(folio);
}
folio_lock(folio);
buf = (u8 *)kmap_local_folio(folio, 0) + folio_ofs;
bit = data_pos & 7;
data_pos &= ~7ull;
ntfs_debug("Before inner for loop: size 0x%x, data_pos 0x%llx, bit 0x%llx",
size, data_pos, bit);
for (; bit < size && data_pos + bit < pass_end;
bit &= ~7ull, bit += 8) {
byte = buf + (bit >> 3);
if (*byte == 0xff)
continue;
b = bit & 7;
for (; b < 8; b++) {
if (*byte & (1 << b))
continue;
ll = data_pos + (bit & ~7ull) + b;
if (ll >= pass_end)
break;
if ((!base_ni || base_ni->mft_no != FILE_MFT) &&
ll >= vol->mft_record_reserve_pos &&
ll < vol->mft_record_reserve_end)
continue;
if (unlikely(ll >= (1ll << 32))) {
folio_unlock(folio);
kunmap_local(buf);
folio_put(folio);
return -ENOSPC;
}
goto found;
}
}
ntfs_debug("After inner for loop: size 0x%x, data_pos 0x%llx, bit 0x%llx",
size, data_pos, bit);
data_pos += size;
folio_unlock(folio);
kunmap_local(buf);
folio_put(folio);
if (data_pos < pass_end)
continue;
}
if (++pass == 2) {
pass_end = pass_start;
data_pos = FIRST_NORMAL_MFT_RECORD;
pass_start = FIRST_NORMAL_MFT_RECORD;
ntfs_debug("pass %i, pass_start 0x%llx, pass_end 0x%llx.",
pass, pass_start, pass_end);
if (data_pos >= pass_end)
break;
}
}
ntfs_debug("Done. (No free mft records left in currently initialized mft bitmap.)");
return -ENOSPC;
found:
if (new_reserve_end)
*new_reserve_end = mft_reserve_end(buf, data_pos,
data_pos + size, ll, pass_end,
initialized_mft_records);
*byte |= 1 << b;
folio_mark_dirty(folio);
folio_unlock(folio);
kunmap_local(buf);
folio_put(folio);
ntfs_debug("Done. (Found and allocated mft record 0x%llx.)", ll);
return ll;
}
static int ntfs_mft_attr_extend(struct ntfs_inode *ni,
struct ntfs_inode *locked_ni)
{
int ret = 0;
struct ntfs_inode *base_ni;
if (NInoAttr(ni))
base_ni = ni->ext.base_ntfs_ino;
else
base_ni = ni;
if (!NInoAttrList(base_ni)) {
ret = ntfs_inode_add_attrlist(base_ni);
if (ret) {
pr_err("Can not add attrlist\n");
goto out;
} else {
ret = -EAGAIN;
goto out;
}
}
ret = ntfs_attr_update_mapping_pairs_locked(ni, 0, locked_ni);
if (ret)
pr_err("MP update failed\n");
out:
return ret;
}
static int ntfs_mft_bitmap_extend_allocation_nolock(struct ntfs_volume *vol)
{
s64 lcn;
s64 ll;
unsigned long flags;
struct folio *folio;
struct ntfs_inode *mft_ni, *mftbmp_ni;
struct runlist_element *rl, *rl2 = NULL;
struct ntfs_attr_search_ctx *ctx = NULL;
struct mft_record *mrec;
struct attr_record *a = NULL;
int ret, mp_size;
u32 old_alen = 0;
u8 *b, tb;
struct {
u8 added_cluster:1;
u8 added_run:1;
u8 mp_rebuilt:1;
u8 mp_extended:1;
} status = { 0, 0, 0, 0 };
size_t new_rl_count;
ntfs_debug("Extending mft bitmap allocation.");
if (!NVolFreeClusterKnown(vol))
wait_event(vol->free_waitq, NVolFreeClusterKnown(vol));
mft_ni = NTFS_I(vol->mft_ino);
mftbmp_ni = NTFS_I(vol->mftbmp_ino);
down_write(&mftbmp_ni->runlist.lock);
read_lock_irqsave(&mftbmp_ni->size_lock, flags);
ll = mftbmp_ni->allocated_size;
read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
rl = ntfs_attr_find_vcn_nolock(mftbmp_ni,
NTFS_B_TO_CLU(vol, ll - 1), NULL);
if (IS_ERR(rl) || unlikely(!rl->length || rl->lcn < 0)) {
up_write(&mftbmp_ni->runlist.lock);
ntfs_error(vol->sb,
"Failed to determine last allocated cluster of mft bitmap attribute.");
if (!IS_ERR(rl))
ret = -EIO;
else
ret = PTR_ERR(rl);
return ret;
}
lcn = rl->lcn + rl->length;
ntfs_debug("Last lcn of mft bitmap attribute is 0x%llx.",
(long long)lcn);
if (lcn >= vol->nr_clusters) {
lcn = -1;
goto alloc_cluster;
}
ll = lcn >> 3;
folio = read_mapping_folio(vol->lcnbmp_ino->i_mapping,
ll >> PAGE_SHIFT, NULL);
if (IS_ERR(folio)) {
up_write(&mftbmp_ni->runlist.lock);
ntfs_error(vol->sb, "Failed to read from lcn bitmap.");
return PTR_ERR(folio);
}
down_write(&vol->lcnbmp_lock);
folio_lock(folio);
b = (u8 *)kmap_local_folio(folio, 0) + (ll & ~PAGE_MASK);
tb = 1 << (lcn & 7ull);
if (*b != 0xff && !(*b & tb) &&
vol->lcn_empty_bits_per_page[ll >> PAGE_SHIFT] &&
ntfs_available_clusters_count(vol, 1) > 0) {
*b |= tb;
folio_mark_dirty(folio);
ntfs_dec_free_clusters(vol, 1);
ntfs_set_lcn_empty_bits(vol, ll >> PAGE_SHIFT, 1, 1);
folio_unlock(folio);
kunmap_local(b);
folio_put(folio);
up_write(&vol->lcnbmp_lock);
rl->length++;
rl[1].vcn++;
status.added_cluster = 1;
ntfs_debug("Appending one cluster to mft bitmap.");
} else {
folio_unlock(folio);
kunmap_local(b);
folio_put(folio);
up_write(&vol->lcnbmp_lock);
alloc_cluster:
rl2 = ntfs_cluster_alloc(vol, rl[1].vcn, 1, lcn, DATA_ZONE,
true, false, false);
if (IS_ERR(rl2)) {
up_write(&mftbmp_ni->runlist.lock);
ntfs_error(vol->sb,
"Failed to allocate a cluster for the mft bitmap.");
return PTR_ERR(rl2);
}
status.added_cluster = rl2->lcn == lcn;
rl = ntfs_runlists_merge(&mftbmp_ni->runlist, rl2, 0, &new_rl_count);
if (IS_ERR(rl)) {
up_write(&mftbmp_ni->runlist.lock);
ntfs_error(vol->sb, "Failed to merge runlists for mft bitmap.");
if (ntfs_cluster_free_from_rl(vol, rl2)) {
ntfs_error(vol->sb, "Failed to deallocate allocated cluster.%s",
es);
NVolSetErrors(vol);
}
kvfree(rl2);
return PTR_ERR(rl);
}
mftbmp_ni->runlist.rl = rl;
mftbmp_ni->runlist.count = new_rl_count;
status.added_run = !status.added_cluster;
ntfs_debug("Allocated one cluster for mft bitmap.");
for (; rl[1].length; rl++)
;
}
mrec = map_mft_record(mft_ni);
if (IS_ERR(mrec)) {
ntfs_error(vol->sb, "Failed to map mft record.");
ret = PTR_ERR(mrec);
goto undo_alloc;
}
ctx = ntfs_attr_get_search_ctx(mft_ni, mrec);
if (unlikely(!ctx)) {
ntfs_error(vol->sb, "Failed to get search context.");
ret = -ENOMEM;
goto undo_alloc;
}
ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
mftbmp_ni->name_len, CASE_SENSITIVE, rl[1].vcn, NULL,
0, ctx);
if (unlikely(ret)) {
ntfs_error(vol->sb,
"Failed to find last attribute extent of mft bitmap attribute.");
if (ret == -ENOENT)
ret = -EIO;
goto undo_alloc;
}
a = ctx->attr;
ll = le64_to_cpu(a->data.non_resident.lowest_vcn);
for (rl2 = rl; rl2 > mftbmp_ni->runlist.rl; rl2--) {
if (ll >= rl2->vcn)
break;
}
WARN_ON(ll < rl2->vcn);
WARN_ON(ll >= rl2->vcn + rl2->length);
mp_size = ntfs_get_size_for_mapping_pairs(vol, rl2, ll, -1, -1);
if (unlikely(mp_size <= 0)) {
ntfs_error(vol->sb,
"Get size for mapping pairs failed for mft bitmap attribute extent.");
ret = mp_size;
if (!ret)
ret = -EIO;
goto undo_alloc;
}
old_alen = le32_to_cpu(a->length);
ret = ntfs_attr_record_resize(ctx->mrec, a, mp_size +
le16_to_cpu(a->data.non_resident.mapping_pairs_offset));
if (unlikely(ret)) {
ret = ntfs_mft_attr_extend(mftbmp_ni, mftbmp_ni);
if (!ret)
goto extended_ok;
if (ret != -EAGAIN)
status.mp_extended = 1;
goto undo_alloc;
}
status.mp_rebuilt = 1;
ret = ntfs_mapping_pairs_build(vol, (u8 *)a +
le16_to_cpu(a->data.non_resident.mapping_pairs_offset),
mp_size, rl2, ll, -1, NULL, NULL, NULL);
if (unlikely(ret)) {
ntfs_error(vol->sb,
"Failed to build mapping pairs array for mft bitmap attribute.");
goto undo_alloc;
}
a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 1);
if (a->data.non_resident.lowest_vcn) {
mark_mft_record_dirty(ctx->ntfs_ino);
extended_ok:
ntfs_attr_reinit_search_ctx(ctx);
ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL,
0, ctx);
if (unlikely(ret)) {
ntfs_error(vol->sb,
"Failed to find first attribute extent of mft bitmap attribute.");
goto restore_undo_alloc;
}
a = ctx->attr;
}
write_lock_irqsave(&mftbmp_ni->size_lock, flags);
mftbmp_ni->allocated_size += vol->cluster_size;
a->data.non_resident.allocated_size =
cpu_to_le64(mftbmp_ni->allocated_size);
write_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
mark_mft_record_dirty(ctx->ntfs_ino);
ntfs_attr_put_search_ctx(ctx);
unmap_mft_record(mft_ni);
up_write(&mftbmp_ni->runlist.lock);
ntfs_debug("Done.");
return 0;
restore_undo_alloc:
ntfs_attr_reinit_search_ctx(ctx);
if (ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
mftbmp_ni->name_len, CASE_SENSITIVE, rl[1].vcn, NULL,
0, ctx)) {
ntfs_error(vol->sb,
"Failed to find last attribute extent of mft bitmap attribute.%s", es);
write_lock_irqsave(&mftbmp_ni->size_lock, flags);
mftbmp_ni->allocated_size += vol->cluster_size;
write_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
ntfs_attr_put_search_ctx(ctx);
unmap_mft_record(mft_ni);
up_write(&mftbmp_ni->runlist.lock);
NVolSetErrors(vol);
return ret;
}
a = ctx->attr;
a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 2);
undo_alloc:
if (status.added_cluster) {
rl->length--;
rl[1].vcn--;
} else if (status.added_run) {
lcn = rl->lcn;
rl->lcn = rl[1].lcn;
rl->length = 0;
mftbmp_ni->runlist.count--;
}
down_write(&vol->lcnbmp_lock);
if (ntfs_bitmap_clear_bit(vol->lcnbmp_ino, lcn)) {
ntfs_error(vol->sb, "Failed to free allocated cluster.%s", es);
NVolSetErrors(vol);
} else
ntfs_inc_free_clusters(vol, 1);
up_write(&vol->lcnbmp_lock);
if (status.mp_rebuilt) {
if (ntfs_mapping_pairs_build(vol, (u8 *)a + le16_to_cpu(
a->data.non_resident.mapping_pairs_offset),
old_alen - le16_to_cpu(
a->data.non_resident.mapping_pairs_offset),
rl2, ll, -1, NULL, NULL, NULL)) {
ntfs_error(vol->sb, "Failed to restore mapping pairs array.%s", es);
NVolSetErrors(vol);
}
if (ntfs_attr_record_resize(ctx->mrec, a, old_alen)) {
ntfs_error(vol->sb, "Failed to restore attribute record.%s", es);
NVolSetErrors(vol);
}
mark_mft_record_dirty(ctx->ntfs_ino);
} else if (status.mp_extended &&
ntfs_attr_update_mapping_pairs_locked(mftbmp_ni, 0,
mftbmp_ni)) {
ntfs_error(vol->sb, "Failed to restore mapping pairs.%s", es);
NVolSetErrors(vol);
}
if (ctx)
ntfs_attr_put_search_ctx(ctx);
if (!IS_ERR(mrec))
unmap_mft_record(mft_ni);
up_write(&mftbmp_ni->runlist.lock);
return ret;
}
static int ntfs_mft_bitmap_extend_initialized_nolock(struct ntfs_volume *vol)
{
s64 old_data_size, old_initialized_size;
unsigned long flags;
struct inode *mftbmp_vi;
struct ntfs_inode *mft_ni, *mftbmp_ni;
struct ntfs_attr_search_ctx *ctx;
struct mft_record *mrec;
struct attr_record *a;
int ret;
ntfs_debug("Extending mft bitmap initialized (and data) size.");
mft_ni = NTFS_I(vol->mft_ino);
mftbmp_vi = vol->mftbmp_ino;
mftbmp_ni = NTFS_I(mftbmp_vi);
mrec = map_mft_record(mft_ni);
if (IS_ERR(mrec)) {
ntfs_error(vol->sb, "Failed to map mft record.");
return PTR_ERR(mrec);
}
ctx = ntfs_attr_get_search_ctx(mft_ni, mrec);
if (unlikely(!ctx)) {
ntfs_error(vol->sb, "Failed to get search context.");
ret = -ENOMEM;
goto unm_err_out;
}
ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL, 0, ctx);
if (unlikely(ret)) {
ntfs_error(vol->sb,
"Failed to find first attribute extent of mft bitmap attribute.");
if (ret == -ENOENT)
ret = -EIO;
goto put_err_out;
}
a = ctx->attr;
write_lock_irqsave(&mftbmp_ni->size_lock, flags);
old_data_size = i_size_read(mftbmp_vi);
old_initialized_size = mftbmp_ni->initialized_size;
mftbmp_ni->initialized_size += 8;
a->data.non_resident.initialized_size =
cpu_to_le64(mftbmp_ni->initialized_size);
if (mftbmp_ni->initialized_size > old_data_size) {
i_size_write(mftbmp_vi, mftbmp_ni->initialized_size);
a->data.non_resident.data_size =
cpu_to_le64(mftbmp_ni->initialized_size);
}
write_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
mark_mft_record_dirty(ctx->ntfs_ino);
ntfs_attr_put_search_ctx(ctx);
unmap_mft_record(mft_ni);
ret = ntfs_attr_set(mftbmp_ni, old_initialized_size, 8, 0);
if (likely(!ret)) {
ntfs_debug("Done. (Wrote eight initialized bytes to mft bitmap.");
return 0;
}
ntfs_error(vol->sb, "Failed to write to mft bitmap.");
mrec = map_mft_record(mft_ni);
if (IS_ERR(mrec)) {
ntfs_error(vol->sb, "Failed to map mft record.%s", es);
NVolSetErrors(vol);
return ret;
}
ctx = ntfs_attr_get_search_ctx(mft_ni, mrec);
if (unlikely(!ctx)) {
ntfs_error(vol->sb, "Failed to get search context.%s", es);
NVolSetErrors(vol);
goto unm_err_out;
}
if (ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name,
mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL, 0, ctx)) {
ntfs_error(vol->sb,
"Failed to find first attribute extent of mft bitmap attribute.%s", es);
NVolSetErrors(vol);
put_err_out:
ntfs_attr_put_search_ctx(ctx);
unm_err_out:
unmap_mft_record(mft_ni);
goto err_out;
}
a = ctx->attr;
write_lock_irqsave(&mftbmp_ni->size_lock, flags);
mftbmp_ni->initialized_size = old_initialized_size;
a->data.non_resident.initialized_size =
cpu_to_le64(old_initialized_size);
if (i_size_read(mftbmp_vi) != old_data_size) {
i_size_write(mftbmp_vi, old_data_size);
a->data.non_resident.data_size = cpu_to_le64(old_data_size);
}
write_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
mark_mft_record_dirty(ctx->ntfs_ino);
ntfs_attr_put_search_ctx(ctx);
unmap_mft_record(mft_ni);
#ifdef DEBUG
read_lock_irqsave(&mftbmp_ni->size_lock, flags);
ntfs_debug("Restored status of mftbmp: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
mftbmp_ni->allocated_size, i_size_read(mftbmp_vi),
mftbmp_ni->initialized_size);
read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
#endif /* DEBUG */
err_out:
return ret;
}
static int ntfs_mft_data_extend_allocation_nolock(struct ntfs_volume *vol)
{
s64 lcn;
s64 old_last_vcn;
s64 min_nr, nr, ll;
unsigned long flags;
struct ntfs_inode *mft_ni;
struct runlist_element *rl, *rl2, *alloc_rl;
struct ntfs_attr_search_ctx *ctx = NULL;
struct mft_record *mrec;
struct attr_record *a = NULL;
int ret, mp_size;
u32 old_alen = 0;
bool mp_rebuilt = false, mp_extended = false;
size_t new_rl_count;
ntfs_debug("Extending mft data allocation.");
mft_ni = NTFS_I(vol->mft_ino);
down_write(&mft_ni->runlist.lock);
read_lock_irqsave(&mft_ni->size_lock, flags);
ll = mft_ni->allocated_size;
read_unlock_irqrestore(&mft_ni->size_lock, flags);
rl = ntfs_attr_find_vcn_nolock(mft_ni,
NTFS_B_TO_CLU(vol, ll - 1), NULL);
if (IS_ERR(rl) || unlikely(!rl->length || rl->lcn < 0)) {
up_write(&mft_ni->runlist.lock);
ntfs_error(vol->sb,
"Failed to determine last allocated cluster of mft data attribute.");
if (!IS_ERR(rl))
ret = -EIO;
else
ret = PTR_ERR(rl);
return ret;
}
lcn = rl->lcn + rl->length;
ntfs_debug("Last lcn of mft data attribute is 0x%llx.", lcn);
min_nr = DIV_ROUND_UP_ULL((u64)vol->mft_record_size * 2, vol->cluster_size);
nr = vol->mft_record_size << 4 >> vol->cluster_size_bits;
if (!nr)
nr = min_nr;
read_lock_irqsave(&mft_ni->size_lock, flags);
ll = mft_ni->allocated_size;
read_unlock_irqrestore(&mft_ni->size_lock, flags);
if (unlikely((ll + NTFS_CLU_TO_B(vol, nr)) >>
vol->mft_record_size_bits >= (1ll << 32))) {
nr = min_nr;
if (unlikely((ll + NTFS_CLU_TO_B(vol, nr)) >>
vol->mft_record_size_bits >= (1ll << 32))) {
ntfs_warning(vol->sb,
"Cannot allocate mft record because the maximum number of inodes (2^32) has already been reached.");
up_write(&mft_ni->runlist.lock);
return -ENOSPC;
}
}
ntfs_debug("Trying mft data allocation with %s cluster count %lli.",
nr > min_nr ? "default" : "minimal", (long long)nr);
old_last_vcn = rl[1].vcn;
up_write(&mft_ni->runlist.lock);
do {
alloc_rl = ntfs_cluster_alloc(vol, old_last_vcn, nr, lcn,
MFT_ZONE, true, false, false);
if (!IS_ERR(alloc_rl))
break;
if (PTR_ERR(alloc_rl) != -ENOSPC || nr == min_nr) {
ntfs_error(vol->sb,
"Failed to allocate the minimal number of clusters (%lli) for the mft data attribute.",
nr);
return PTR_ERR(alloc_rl);
}
nr = min_nr;
ntfs_debug("Retrying mft data allocation with minimal cluster count %lli.", nr);
} while (1);
down_write(&mft_ni->runlist.lock);
rl = ntfs_runlists_merge_keep_src(&mft_ni->runlist, alloc_rl, 0,
&new_rl_count);
if (IS_ERR(rl)) {
up_write(&mft_ni->runlist.lock);
ntfs_error(vol->sb, "Failed to merge runlists for mft data attribute.");
if (ntfs_cluster_free_from_rl(vol, alloc_rl)) {
ntfs_error(vol->sb,
"Failed to deallocate clusters from the mft data attribute.%s", es);
NVolSetErrors(vol);
}
kvfree(alloc_rl);
return PTR_ERR(rl);
}
mft_ni->runlist.rl = rl;
mft_ni->runlist.count = new_rl_count;
ntfs_debug("Allocated %lli clusters.", (long long)nr);
for (; rl[1].length; rl++)
;
up_write(&mft_ni->runlist.lock);
mrec = map_mft_record(mft_ni);
if (IS_ERR(mrec)) {
ntfs_error(vol->sb, "Failed to map mft record.");
ret = PTR_ERR(mrec);
goto undo_alloc;
}
ctx = ntfs_attr_get_search_ctx(mft_ni, mrec);
if (unlikely(!ctx)) {
ntfs_error(vol->sb, "Failed to get search context.");
ret = -ENOMEM;
goto undo_alloc;
}
ret = ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len,
CASE_SENSITIVE, rl[1].vcn, NULL, 0, ctx);
if (unlikely(ret)) {
ntfs_error(vol->sb, "Failed to find last attribute extent of mft data attribute.");
if (ret == -ENOENT)
ret = -EIO;
goto undo_alloc;
}
a = ctx->attr;
ll = le64_to_cpu(a->data.non_resident.lowest_vcn);
down_write(&mft_ni->runlist.lock);
for (rl2 = rl; rl2 > mft_ni->runlist.rl; rl2--) {
if (ll >= rl2->vcn)
break;
}
WARN_ON(ll < rl2->vcn);
WARN_ON(ll >= rl2->vcn + rl2->length);
mp_size = ntfs_get_size_for_mapping_pairs(vol, rl2, ll, -1, -1);
if (unlikely(mp_size <= 0)) {
ntfs_error(vol->sb,
"Get size for mapping pairs failed for mft data attribute extent.");
ret = mp_size;
if (!ret)
ret = -EIO;
up_write(&mft_ni->runlist.lock);
goto undo_alloc;
}
up_write(&mft_ni->runlist.lock);
old_alen = le32_to_cpu(a->length);
ret = ntfs_attr_record_resize(ctx->mrec, a, mp_size +
le16_to_cpu(a->data.non_resident.mapping_pairs_offset));
if (unlikely(ret)) {
ret = ntfs_mft_attr_extend(mft_ni, NULL);
if (!ret)
goto extended_ok;
if (ret != -EAGAIN)
mp_extended = true;
goto undo_alloc;
}
mp_rebuilt = true;
ret = ntfs_mapping_pairs_build(vol, (u8 *)a +
le16_to_cpu(a->data.non_resident.mapping_pairs_offset),
mp_size, rl2, ll, -1, NULL, NULL, NULL);
if (unlikely(ret)) {
ntfs_error(vol->sb, "Failed to build mapping pairs array of mft data attribute.");
goto undo_alloc;
}
a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 1);
if (a->data.non_resident.lowest_vcn) {
mark_mft_record_dirty(ctx->ntfs_ino);
extended_ok:
ntfs_attr_reinit_search_ctx(ctx);
ret = ntfs_attr_lookup(mft_ni->type, mft_ni->name,
mft_ni->name_len, CASE_SENSITIVE, 0, NULL, 0,
ctx);
if (unlikely(ret)) {
ntfs_error(vol->sb,
"Failed to find first attribute extent of mft data attribute.");
goto restore_undo_alloc;
}
a = ctx->attr;
}
write_lock_irqsave(&mft_ni->size_lock, flags);
mft_ni->allocated_size += NTFS_CLU_TO_B(vol, nr);
a->data.non_resident.allocated_size =
cpu_to_le64(mft_ni->allocated_size);
write_unlock_irqrestore(&mft_ni->size_lock, flags);
mark_mft_record_dirty(ctx->ntfs_ino);
ntfs_attr_put_search_ctx(ctx);
unmap_mft_record(mft_ni);
kvfree(alloc_rl);
ntfs_debug("Done.");
return 0;
restore_undo_alloc:
ntfs_attr_reinit_search_ctx(ctx);
if (ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len,
CASE_SENSITIVE, rl[1].vcn, NULL, 0, ctx)) {
ntfs_error(vol->sb,
"Failed to find last attribute extent of mft data attribute.%s", es);
write_lock_irqsave(&mft_ni->size_lock, flags);
mft_ni->allocated_size += NTFS_CLU_TO_B(vol, nr);
write_unlock_irqrestore(&mft_ni->size_lock, flags);
ntfs_attr_put_search_ctx(ctx);
unmap_mft_record(mft_ni);
kvfree(alloc_rl);
NVolSetErrors(vol);
return ret;
}
ctx->attr->data.non_resident.highest_vcn =
cpu_to_le64(old_last_vcn - 1);
undo_alloc:
down_write(&mft_ni->runlist.lock);
if (ntfs_rl_truncate_nolock(vol, &mft_ni->runlist, old_last_vcn)) {
ntfs_error(vol->sb, "Failed to truncate mft data attribute runlist.%s", es);
NVolSetErrors(vol);
kvfree(alloc_rl);
alloc_rl = NULL;
}
up_write(&mft_ni->runlist.lock);
if (ntfs_cluster_free_from_rl(vol, alloc_rl)) {
ntfs_error(vol->sb, "Failed to free clusters from mft data attribute.%s", es);
NVolSetErrors(vol);
}
kvfree(alloc_rl);
if (mp_extended && ntfs_attr_update_mapping_pairs(mft_ni, 0)) {
ntfs_error(vol->sb, "Failed to restore mapping pairs.%s",
es);
NVolSetErrors(vol);
}
if (ctx) {
a = ctx->attr;
if (mp_rebuilt && !IS_ERR(ctx->mrec)) {
int err;
down_read(&mft_ni->runlist.lock);
err = ntfs_mapping_pairs_build(vol, (u8 *)a + le16_to_cpu(
a->data.non_resident.mapping_pairs_offset),
old_alen - le16_to_cpu(
a->data.non_resident.mapping_pairs_offset),
mft_ni->runlist.rl, ll, -1, NULL, NULL, NULL);
up_read(&mft_ni->runlist.lock);
if (err) {
ntfs_error(vol->sb, "Failed to restore mapping pairs array.%s", es);
NVolSetErrors(vol);
}
if (ntfs_attr_record_resize(ctx->mrec, a, old_alen)) {
ntfs_error(vol->sb, "Failed to restore attribute record.%s", es);
NVolSetErrors(vol);
}
mark_mft_record_dirty(ctx->ntfs_ino);
} else if (IS_ERR(ctx->mrec)) {
ntfs_error(vol->sb, "Failed to restore attribute search context.%s", es);
NVolSetErrors(vol);
}
ntfs_attr_put_search_ctx(ctx);
}
if (!IS_ERR(mrec))
unmap_mft_record(mft_ni);
return ret;
}
static int ntfs_mft_record_layout(const struct ntfs_volume *vol, const s64 mft_no,
struct mft_record *m)
{
struct attr_record *a;
ntfs_debug("Entering for mft record 0x%llx.", (long long)mft_no);
if (mft_no >= (1ll << 32)) {
ntfs_error(vol->sb, "Mft record number 0x%llx exceeds maximum of 2^32.",
(long long)mft_no);
return -ERANGE;
}
memset(m, 0, vol->mft_record_size);
if (vol->major_ver < 3 || (vol->major_ver == 3 && !vol->minor_ver))
m->usa_ofs = cpu_to_le16((sizeof(struct mft_record_old) + 1) & ~1);
else {
m->usa_ofs = cpu_to_le16((sizeof(struct mft_record) + 1) & ~1);
m->reserved = 0;
m->mft_record_number = cpu_to_le32((u32)mft_no);
}
m->magic = magic_FILE;
if (vol->mft_record_size >= NTFS_BLOCK_SIZE)
m->usa_count = cpu_to_le16(vol->mft_record_size /
NTFS_BLOCK_SIZE + 1);
else {
m->usa_count = cpu_to_le16(1);
ntfs_warning(vol->sb,
"Sector size is bigger than mft record size. Setting usa_count to 1. If chkdsk reports this as corruption");
}
*(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)) = cpu_to_le16(1);
m->lsn = 0;
m->sequence_number = cpu_to_le16(1);
m->link_count = 0;
m->attrs_offset = cpu_to_le16((le16_to_cpu(m->usa_ofs) +
(le16_to_cpu(m->usa_count) << 1) + 7) & ~7);
m->flags = 0;
m->bytes_in_use = cpu_to_le32(le16_to_cpu(m->attrs_offset) + 8);
m->bytes_allocated = cpu_to_le32(vol->mft_record_size);
m->base_mft_record = 0;
m->next_attr_instance = 0;
a = (struct attr_record *)((u8 *)m + le16_to_cpu(m->attrs_offset));
a->type = AT_END;
a->length = 0;
ntfs_debug("Done.");
return 0;
}
static int ntfs_mft_record_format(const struct ntfs_volume *vol, const s64 mft_no)
{
loff_t i_size;
struct inode *mft_vi = vol->mft_ino;
struct folio *folio;
struct mft_record *m;
pgoff_t index, end_index;
unsigned int ofs;
int err;
ntfs_debug("Entering for mft record 0x%llx.", (long long)mft_no);
index = NTFS_MFT_NR_TO_PIDX(vol, mft_no);
ofs = NTFS_MFT_NR_TO_POFS(vol, mft_no);
i_size = i_size_read(mft_vi);
end_index = i_size >> PAGE_SHIFT;
if (unlikely(index >= end_index)) {
if (unlikely(index > end_index ||
ofs + vol->mft_record_size > (i_size & ~PAGE_MASK))) {
ntfs_error(vol->sb, "Tried to format non-existing mft record 0x%llx.",
(long long)mft_no);
return -ENOENT;
}
}
folio = read_mapping_folio(mft_vi->i_mapping, index, NULL);
if (IS_ERR(folio)) {
ntfs_error(vol->sb, "Failed to map page containing mft record to format 0x%llx.",
(long long)mft_no);
return PTR_ERR(folio);
}
folio_lock(folio);
if (folio_test_writeback(folio))
folio_wait_writeback(folio);
folio_clear_uptodate(folio);
m = (struct mft_record *)((u8 *)kmap_local_folio(folio, 0) + ofs);
err = ntfs_mft_record_layout(vol, mft_no, m);
if (unlikely(err)) {
ntfs_error(vol->sb, "Failed to layout mft record 0x%llx.",
(long long)mft_no);
folio_mark_uptodate(folio);
folio_unlock(folio);
kunmap_local(m);
folio_put(folio);
return err;
}
pre_write_mst_fixup((struct ntfs_record *)m, vol->mft_record_size);
folio_mark_uptodate(folio);
ntfs_mft_mark_dirty(folio);
folio_unlock(folio);
kunmap_local(m);
folio_put(folio);
ntfs_debug("Done.");
return 0;
}
int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode,
struct ntfs_inode **ni, struct ntfs_inode *base_ni,
struct mft_record **ni_mrec, const s64 mft_data_vcn)
{
s64 ll, bit, old_data_initialized, old_data_size;
s64 nr_new_mft_records = 0;
s64 max_mft_no = -1, reserve_start = -1, reserve_end = -1;
s64 candidate_reserve_end = -1;
s64 *reserve_endp;
unsigned long flags;
struct folio *folio;
struct ntfs_inode *mft_ni, *mftbmp_ni;
struct ntfs_attr_search_ctx *ctx;
struct mft_record *m = NULL;
struct attr_record *a;
pgoff_t index;
unsigned int ofs;
int err;
__le16 seq_no, usn;
bool record_formatted = false, from_reserve = false, tail_alloc = false;
bool reserve_created = false;
bool forced_reserved_record = false;
unsigned int memalloc_flags;
if (base_ni && *ni)
return -EINVAL;
if (mode && base_ni)
return -EINVAL;
if (mft_data_vcn >= 0 &&
(!base_ni || base_ni->mft_no != FILE_MFT))
return -EINVAL;
if (mft_data_vcn >= 0) {
u64 vbo;
if ((u64)mft_data_vcn > (U64_MAX >> vol->cluster_size_bits))
return -EOVERFLOW;
vbo = (u64)mft_data_vcn << vol->cluster_size_bits;
max_mft_no = vbo >> vol->mft_record_size_bits;
}
if (base_ni)
ntfs_debug("Entering (allocating an extent mft record for base mft record 0x%llx).",
(long long)base_ni->mft_no);
else
ntfs_debug("Entering (allocating a base mft record).");
memalloc_flags = memalloc_nofs_save();
mft_ni = NTFS_I(vol->mft_ino);
if (!base_ni || base_ni->mft_no != FILE_MFT)
mutex_lock(&mft_ni->mrec_lock);
mftbmp_ni = NTFS_I(vol->mftbmp_ino);
search_free_rec:
from_reserve = false;
reserve_created = false;
candidate_reserve_end = -1;
if (!base_ni || base_ni->mft_no != FILE_MFT)
down_write(&vol->mftbmp_lock);
if (base_ni && base_ni->mft_no == FILE_MFT &&
vol->mft_record_reserve_pos < vol->mft_record_reserve_end &&
(max_mft_no < 0 || vol->mft_record_reserve_pos < max_mft_no)) {
bit = vol->mft_record_reserve_pos;
err = ntfs_bitmap_set_bit(vol->mftbmp_ino, bit);
if (unlikely(err)) {
ntfs_error(vol->sb,
"Failed to allocate reserved MFT record 0x%llx.",
bit);
goto err_out;
}
vol->mft_record_reserve_pos++;
from_reserve = true;
ntfs_debug("Allocated MFT metadata record 0x%llx from tail reserve.",
bit);
goto have_alloc_rec;
}
reserve_endp = vol->mft_record_reserve_pos >=
vol->mft_record_reserve_end ? &candidate_reserve_end : NULL;
bit = mft_bitmap_alloc_free_rec(vol, base_ni, max_mft_no, reserve_endp);
if (bit >= 0) {
if (candidate_reserve_end > bit + 1) {
vol->mft_record_reserve_pos = bit + 1;
vol->mft_record_reserve_end = candidate_reserve_end;
reserve_created = true;
ntfs_debug("Reserved free MFT records [0x%llx, 0x%llx) for metadata.",
bit + 1, candidate_reserve_end);
}
ntfs_debug("Found and allocated free record (#1), bit 0x%llx.",
(long long)bit);
goto have_alloc_rec;
}
if (bit != -ENOSPC) {
if (!base_ni || base_ni->mft_no != FILE_MFT) {
up_write(&vol->mftbmp_lock);
mutex_unlock(&mft_ni->mrec_lock);
}
memalloc_nofs_restore(memalloc_flags);
return bit;
}
if (base_ni && base_ni->mft_no == FILE_MFT) {
static const u8 bootstrap_records[] = {
FILE_reserved15, FILE_reserved12, FILE_reserved13,
FILE_reserved14,
};
int i;
for (i = 0; i < ARRAY_SIZE(bootstrap_records); i++) {
if (max_mft_no >= 0 && bootstrap_records[i] >= max_mft_no)
continue;
if (!mft_reserved_is_free(vol, mft_ni,
bootstrap_records[i]))
continue;
bit = bootstrap_records[i];
forced_reserved_record = true;
ntfs_debug("Using reserved MFT record %lld to bootstrap metadata extension.",
bit);
goto have_alloc_rec;
}
memalloc_nofs_restore(memalloc_flags);
return bit;
}
read_lock_irqsave(&mft_ni->size_lock, flags);
ll = mft_ni->initialized_size >> vol->mft_record_size_bits;
read_unlock_irqrestore(&mft_ni->size_lock, flags);
read_lock_irqsave(&mftbmp_ni->size_lock, flags);
old_data_initialized = mftbmp_ni->initialized_size;
read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
if (old_data_initialized << 3 > ll &&
old_data_initialized << 3 > FIRST_NORMAL_MFT_RECORD) {
bit = ll;
if (bit < FIRST_NORMAL_MFT_RECORD)
bit = FIRST_NORMAL_MFT_RECORD;
if (unlikely(bit >= (1ll << 32)))
goto max_err_out;
ntfs_debug("Found free record (#2), bit 0x%llx.",
(long long)bit);
goto found_free_rec;
}
bit = old_data_initialized << 3;
if (unlikely(bit >= (1ll << 32)))
goto max_err_out;
read_lock_irqsave(&mftbmp_ni->size_lock, flags);
old_data_size = mftbmp_ni->allocated_size;
ntfs_debug("Status of mftbmp before extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
old_data_size, i_size_read(vol->mftbmp_ino),
old_data_initialized);
read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
if (old_data_initialized + 8 > old_data_size) {
ntfs_debug("mftbmp: initialized_size + 8 > allocated_size.");
err = ntfs_mft_bitmap_extend_allocation_nolock(vol);
if (err == -EAGAIN)
err = ntfs_mft_bitmap_extend_allocation_nolock(vol);
if (unlikely(err)) {
if (!base_ni || base_ni->mft_no != FILE_MFT)
up_write(&vol->mftbmp_lock);
goto err_out;
}
#ifdef DEBUG
read_lock_irqsave(&mftbmp_ni->size_lock, flags);
ntfs_debug("Status of mftbmp after allocation extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
mftbmp_ni->allocated_size,
i_size_read(vol->mftbmp_ino),
mftbmp_ni->initialized_size);
read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
#endif /* DEBUG */
}
err = ntfs_mft_bitmap_extend_initialized_nolock(vol);
if (unlikely(err)) {
if (!base_ni || base_ni->mft_no != FILE_MFT)
up_write(&vol->mftbmp_lock);
goto err_out;
}
#ifdef DEBUG
read_lock_irqsave(&mftbmp_ni->size_lock, flags);
ntfs_debug("Status of mftbmp after initialized extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
mftbmp_ni->allocated_size,
i_size_read(vol->mftbmp_ino),
mftbmp_ni->initialized_size);
read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
#endif /* DEBUG */
ntfs_debug("Found free record (#3), bit 0x%llx.", (long long)bit);
found_free_rec:
ntfs_debug("At found_free_rec.");
err = ntfs_bitmap_set_bit(vol->mftbmp_ino, bit);
if (unlikely(err)) {
ntfs_error(vol->sb, "Failed to allocate bit in mft bitmap.");
if (!base_ni || base_ni->mft_no != FILE_MFT)
up_write(&vol->mftbmp_lock);
goto err_out;
}
ntfs_debug("Set bit 0x%llx in mft bitmap.", (long long)bit);
have_alloc_rec:
ll = (bit + 1) << vol->mft_record_size_bits;
read_lock_irqsave(&mft_ni->size_lock, flags);
old_data_initialized = mft_ni->initialized_size;
read_unlock_irqrestore(&mft_ni->size_lock, flags);
tail_alloc = (!base_ni || base_ni->mft_no != FILE_MFT) &&
bit >= (old_data_initialized >> vol->mft_record_size_bits) &&
vol->mft_record_reserve_pos >= vol->mft_record_reserve_end;
if (tail_alloc)
ll = (bit + 2) << vol->mft_record_size_bits;
if (ll <= old_data_initialized) {
ntfs_debug("Allocated mft record already initialized.");
goto mft_rec_already_initialized;
}
ntfs_debug("Initializing allocated mft record.");
if (!base_ni || base_ni->mft_no != FILE_MFT) {
read_lock_irqsave(&mft_ni->size_lock, flags);
ntfs_debug("Status of mft data before extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
mft_ni->allocated_size, i_size_read(vol->mft_ino),
mft_ni->initialized_size);
while (ll > mft_ni->allocated_size) {
read_unlock_irqrestore(&mft_ni->size_lock, flags);
err = ntfs_mft_data_extend_allocation_nolock(vol);
if (err == -EAGAIN)
err = ntfs_mft_data_extend_allocation_nolock(vol);
if (unlikely(err)) {
ntfs_error(vol->sb, "Failed to extend mft data allocation.");
goto undo_mftbmp_alloc_nolock;
}
read_lock_irqsave(&mft_ni->size_lock, flags);
ntfs_debug("Status of mft data after allocation extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
mft_ni->allocated_size, i_size_read(vol->mft_ino),
mft_ni->initialized_size);
}
read_unlock_irqrestore(&mft_ni->size_lock, flags);
if (tail_alloc) {
s64 bitmap_records;
read_lock_irqsave(&mft_ni->size_lock, flags);
reserve_end = mft_ni->allocated_size >>
vol->mft_record_size_bits;
read_unlock_irqrestore(&mft_ni->size_lock, flags);
read_lock_irqsave(&mftbmp_ni->size_lock, flags);
bitmap_records = mftbmp_ni->initialized_size << 3;
read_unlock_irqrestore(&mftbmp_ni->size_lock, flags);
if (reserve_end > bitmap_records)
reserve_end = bitmap_records;
if (reserve_end > bit + 1 + MFT_RECORD_RESERVE)
reserve_end = bit + 1 + MFT_RECORD_RESERVE;
reserve_start = bit + 1;
if (reserve_end > reserve_start) {
ll = reserve_end << vol->mft_record_size_bits;
} else {
reserve_start = -1;
reserve_end = -1;
ll = (bit + 1) << vol->mft_record_size_bits;
}
}
} else if (ll > mft_ni->allocated_size) {
err = -ENOSPC;
goto undo_mftbmp_alloc_nolock;
}
write_lock_irqsave(&mft_ni->size_lock, flags);
old_data_initialized = mft_ni->initialized_size;
old_data_size = vol->mft_ino->i_size;
while (ll > mft_ni->initialized_size) {
s64 new_initialized_size, mft_no;
new_initialized_size = mft_ni->initialized_size +
vol->mft_record_size;
mft_no = mft_ni->initialized_size >> vol->mft_record_size_bits;
if (new_initialized_size > i_size_read(vol->mft_ino))
i_size_write(vol->mft_ino, new_initialized_size);
write_unlock_irqrestore(&mft_ni->size_lock, flags);
ntfs_debug("Initializing mft record 0x%llx.",
(long long)mft_no);
err = ntfs_mft_record_format(vol, mft_no);
if (unlikely(err)) {
ntfs_error(vol->sb, "Failed to format mft record.");
goto undo_data_init;
}
write_lock_irqsave(&mft_ni->size_lock, flags);
mft_ni->initialized_size = new_initialized_size;
}
write_unlock_irqrestore(&mft_ni->size_lock, flags);
record_formatted = true;
m = map_mft_record(mft_ni);
if (IS_ERR(m)) {
ntfs_error(vol->sb, "Failed to map mft record.");
err = PTR_ERR(m);
goto undo_data_init;
}
ctx = ntfs_attr_get_search_ctx(mft_ni, m);
if (unlikely(!ctx)) {
ntfs_error(vol->sb, "Failed to get search context.");
err = -ENOMEM;
unmap_mft_record(mft_ni);
goto undo_data_init;
}
err = ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len,
CASE_SENSITIVE, 0, NULL, 0, ctx);
if (unlikely(err)) {
ntfs_error(vol->sb, "Failed to find first attribute extent of mft data attribute.");
ntfs_attr_put_search_ctx(ctx);
unmap_mft_record(mft_ni);
goto undo_data_init;
}
a = ctx->attr;
read_lock_irqsave(&mft_ni->size_lock, flags);
a->data.non_resident.initialized_size =
cpu_to_le64(mft_ni->initialized_size);
a->data.non_resident.data_size =
cpu_to_le64(i_size_read(vol->mft_ino));
read_unlock_irqrestore(&mft_ni->size_lock, flags);
mark_mft_record_dirty(ctx->ntfs_ino);
ntfs_attr_put_search_ctx(ctx);
unmap_mft_record(mft_ni);
if (reserve_start >= 0 && reserve_end > reserve_start) {
vol->mft_record_reserve_pos = reserve_start;
vol->mft_record_reserve_end = reserve_end;
ntfs_debug("Reserved MFT records [0x%llx, 0x%llx) for metadata.",
reserve_start, reserve_end);
}
read_lock_irqsave(&mft_ni->size_lock, flags);
ntfs_debug("Status of mft data after mft record initialization: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.",
mft_ni->allocated_size, i_size_read(vol->mft_ino),
mft_ni->initialized_size);
WARN_ON(i_size_read(vol->mft_ino) > mft_ni->allocated_size);
WARN_ON(mft_ni->initialized_size > i_size_read(vol->mft_ino));
nr_new_mft_records = (i_size_read(vol->mft_ino) - old_data_size) >>
vol->mft_record_size_bits;
read_unlock_irqrestore(&mft_ni->size_lock, flags);
mft_rec_already_initialized:
if (nr_new_mft_records > 0)
ntfs_inc_free_mft_records(vol, nr_new_mft_records);
if (!base_ni || base_ni->mft_no != FILE_MFT)
up_write(&vol->mftbmp_lock);
index = NTFS_MFT_NR_TO_PIDX(vol, bit);
ofs = NTFS_MFT_NR_TO_POFS(vol, bit);
folio = read_mapping_folio(vol->mft_ino->i_mapping, index, NULL);
if (IS_ERR(folio)) {
ntfs_error(vol->sb, "Failed to map page containing allocated mft record 0x%llx.",
bit);
err = PTR_ERR(folio);
goto undo_mftbmp_alloc;
}
folio_lock(folio);
if (folio_test_writeback(folio))
folio_wait_writeback(folio);
folio_clear_uptodate(folio);
m = (struct mft_record *)((u8 *)kmap_local_folio(folio, 0) + ofs);
if (!record_formatted) {
if (!forced_reserved_record && ntfs_is_file_record(m->magic) &&
(m->flags & MFT_RECORD_IN_USE)) {
ntfs_warning(vol->sb,
"Mft record 0x%llx was marked free in mft bitmap but is marked used itself. Unmount and run chkdsk.",
bit);
folio_mark_uptodate(folio);
folio_unlock(folio);
kunmap_local(m);
folio_put(folio);
NVolSetErrors(vol);
goto search_free_rec;
}
seq_no = m->sequence_number;
if (!(le16_to_cpu(m->usa_ofs) & 1) &&
le16_to_cpu(m->usa_ofs) + sizeof(usn) <= vol->mft_record_size)
usn = *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs));
else
usn = 0;
err = ntfs_mft_record_layout(vol, bit, m);
if (unlikely(err)) {
ntfs_error(vol->sb, "Failed to layout allocated mft record 0x%llx.",
bit);
folio_mark_uptodate(folio);
folio_unlock(folio);
kunmap_local(m);
folio_put(folio);
goto undo_mftbmp_alloc;
}
if (seq_no)
m->sequence_number = seq_no;
if (usn && le16_to_cpu(usn) != 0xffff)
*(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)) = usn;
pre_write_mst_fixup((struct ntfs_record *)m, vol->mft_record_size);
}
m->flags |= MFT_RECORD_IN_USE;
if (S_ISDIR(mode))
m->flags |= MFT_RECORD_IS_DIRECTORY;
folio_mark_uptodate(folio);
if (base_ni) {
struct mft_record *m_tmp;
m->base_mft_record = MK_LE_MREF(base_ni->mft_no,
base_ni->seq_no);
m_tmp = map_extent_mft_record(base_ni,
MK_MREF(bit, le16_to_cpu(m->sequence_number)),
ni);
if (IS_ERR(m_tmp)) {
ntfs_error(vol->sb, "Failed to map allocated extent mft record 0x%llx.",
bit);
err = PTR_ERR(m_tmp);
if (forced_reserved_record) {
m->base_mft_record = 0;
m->flags |= MFT_RECORD_IN_USE;
} else {
m->flags &= cpu_to_le16(~le16_to_cpu(MFT_RECORD_IN_USE));
}
ntfs_mft_mark_dirty(folio);
folio_unlock(folio);
kunmap_local(m);
folio_put(folio);
goto undo_mftbmp_alloc;
}
ntfs_mft_mark_dirty(folio);
folio_unlock(folio);
kunmap_local(m);
folio_put(folio);
} else {
(*ni)->seq_no = le16_to_cpu(m->sequence_number);
(*ni)->mrec = kmemdup(m, vol->mft_record_size, GFP_NOFS);
if (!(*ni)->mrec) {
folio_unlock(folio);
kunmap_local(m);
folio_put(folio);
err = -ENOMEM;
goto undo_mftbmp_alloc;
}
post_read_mst_fixup((struct ntfs_record *)(*ni)->mrec, vol->mft_record_size);
ntfs_mft_mark_dirty(folio);
folio_unlock(folio);
(*ni)->folio = folio;
(*ni)->folio_ofs = ofs;
atomic_inc(&(*ni)->count);
vol->mft_data_pos = bit + 1;
}
if (!base_ni || base_ni->mft_no != FILE_MFT)
mutex_unlock(&mft_ni->mrec_lock);
memalloc_nofs_restore(memalloc_flags);
ntfs_debug("Returning opened, allocated %sinode 0x%llx.",
base_ni ? "extent " : "", bit);
(*ni)->mft_no = bit;
if (ni_mrec)
*ni_mrec = (*ni)->mrec;
if (!forced_reserved_record)
ntfs_dec_free_mft_records(vol, 1);
return 0;
undo_data_init:
write_lock_irqsave(&mft_ni->size_lock, flags);
mft_ni->initialized_size = old_data_initialized;
i_size_write(vol->mft_ino, old_data_size);
write_unlock_irqrestore(&mft_ni->size_lock, flags);
goto undo_mftbmp_alloc_nolock;
undo_mftbmp_alloc:
if (!base_ni || base_ni->mft_no != FILE_MFT)
down_write(&vol->mftbmp_lock);
undo_mftbmp_alloc_nolock:
if (!forced_reserved_record && ntfs_bitmap_clear_bit(vol->mftbmp_ino, bit)) {
ntfs_error(vol->sb, "Failed to clear bit in mft bitmap.%s", es);
NVolSetErrors(vol);
}
if ((from_reserve || reserve_created) &&
vol->mft_record_reserve_pos == bit + 1)
vol->mft_record_reserve_pos = bit;
if (!base_ni || base_ni->mft_no != FILE_MFT)
up_write(&vol->mftbmp_lock);
err_out:
if (!base_ni || base_ni->mft_no != FILE_MFT)
mutex_unlock(&mft_ni->mrec_lock);
memalloc_nofs_restore(memalloc_flags);
return err;
max_err_out:
ntfs_warning(vol->sb,
"Cannot allocate mft record because the maximum number of inodes (2^32) has already been reached.");
if (!base_ni || base_ni->mft_no != FILE_MFT) {
up_write(&vol->mftbmp_lock);
mutex_unlock(&mft_ni->mrec_lock);
}
memalloc_nofs_restore(memalloc_flags);
return -ENOSPC;
}
int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni)
{
u64 mft_no;
int err;
u16 seq_no;
__le16 old_seq_no;
__le64 old_base_mft_record;
struct mft_record *ni_mrec;
unsigned int memalloc_flags;
struct ntfs_inode *base_ni;
bool keep_reserved;
if (!vol || !ni)
return -EINVAL;
ntfs_debug("Entering for inode 0x%llx.\n", (long long)ni->mft_no);
ni_mrec = map_mft_record(ni);
if (IS_ERR(ni_mrec))
return -EIO;
mft_no = ni->mft_no;
if (likely(ni->nr_extents >= 0))
base_ni = ni;
else
base_ni = ni->ext.base_ntfs_ino;
keep_reserved = mft_no >= FILE_reserved12 &&
mft_no <= FILE_reserved15 &&
base_ni->mft_no == FILE_MFT;
old_base_mft_record = ni_mrec->base_mft_record;
if (keep_reserved) {
ni_mrec->base_mft_record = 0;
ni_mrec->flags |= MFT_RECORD_IN_USE;
} else {
ni_mrec->flags &= ~MFT_RECORD_IN_USE;
}
old_seq_no = ni_mrec->sequence_number;
seq_no = le16_to_cpu(old_seq_no);
if (seq_no == 0xffff)
seq_no = 1;
else if (seq_no)
seq_no++;
ni_mrec->sequence_number = cpu_to_le16(seq_no);
down_read(&NTFS_I(vol->mft_ino)->runlist.lock);
err = ntfs_get_block_mft_record(NTFS_I(vol->mft_ino), ni);
up_read(&NTFS_I(vol->mft_ino)->runlist.lock);
if (err) {
unmap_mft_record(ni);
return err;
}
NInoSetDirty(ni);
err = write_mft_record(ni, ni_mrec, 1);
if (err)
goto sync_rollback;
if (keep_reserved) {
unmap_mft_record(ni);
return 0;
}
memalloc_flags = memalloc_nofs_save();
if (base_ni->mft_no != FILE_MFT)
down_write(&vol->mftbmp_lock);
err = ntfs_bitmap_clear_bit(vol->mftbmp_ino, mft_no);
if (!err)
ntfs_inc_free_mft_records(vol, 1);
if (!err && base_ni->mft_no == FILE_MFT &&
mft_no + 1 == vol->mft_record_reserve_pos &&
mft_no < vol->mft_record_reserve_end)
vol->mft_record_reserve_pos = mft_no;
if (base_ni->mft_no != FILE_MFT)
up_write(&vol->mftbmp_lock);
memalloc_nofs_restore(memalloc_flags);
if (err)
goto bitmap_rollback;
unmap_mft_record(ni);
return 0;
bitmap_rollback:
memalloc_flags = memalloc_nofs_save();
if (base_ni->mft_no != FILE_MFT)
down_write(&vol->mftbmp_lock);
if (ntfs_bitmap_set_bit(vol->mftbmp_ino, mft_no))
ntfs_error(vol->sb, "ntfs_bitmap_set_bit failed in bitmap_rollback\n");
if (base_ni->mft_no != FILE_MFT)
up_write(&vol->mftbmp_lock);
memalloc_nofs_restore(memalloc_flags);
sync_rollback:
ntfs_error(vol->sb,
"Eeek! Rollback failed in %s. Leaving inconsistent metadata!\n", __func__);
ni_mrec->flags |= MFT_RECORD_IN_USE;
ni_mrec->sequence_number = old_seq_no;
ni_mrec->base_mft_record = old_base_mft_record;
NInoSetDirty(ni);
write_mft_record(ni, ni_mrec, 0);
unmap_mft_record(ni);
return err;
}
static bool ntfs_add_mft_io_unit(struct bio *bio, struct folio *folio,
const struct ntfs_mft_io_unit *unit)
{
if (!bio->bi_iter.bi_size)
bio->bi_iter.bi_sector = unit->sector;
else if (bio_end_sector(bio) != unit->sector)
return false;
return bio_add_folio(bio, folio, unit->len, unit->folio_ofs);
}
static void ntfs_release_mft_write_refs(struct ntfs_inode **locked_nis,
unsigned int nr_locked_nis,
struct inode **ref_inos,
unsigned int nr_ref_inos)
{
while (nr_locked_nis-- > 0) {
struct ntfs_inode *tni = locked_nis[nr_locked_nis];
mutex_unlock(&tni->mrec_lock);
atomic_dec(&tni->count);
}
while (nr_ref_inos-- > 0)
iput(ref_inos[nr_ref_inos]);
}
static int ntfs_map_mft_io_locked(struct ntfs_inode *ni, u64 folio_byte,
struct ntfs_mft_io_unit *unit)
{
struct ntfs_volume *vol = ni->vol;
u64 file_ofs = folio_byte + unit->folio_ofs;
s64 vcn, lcn;
lockdep_assert_held(&ni->runlist.lock);
if (!ni->runlist.rl)
return -EIO;
vcn = ntfs_bytes_to_cluster(vol, file_ofs);
lcn = ntfs_rl_vcn_to_lcn(ni->runlist.rl, vcn);
if (lcn == LCN_RL_NOT_MAPPED)
return -EAGAIN;
if (lcn < 0)
return -EIO;
if (ntfs_bytes_to_cluster_off(vol, file_ofs) + unit->len >
vol->cluster_size)
return -EIO;
unit->sector = ntfs_bytes_to_bio_sector(
NTFS_CLU_TO_B(vol, lcn) +
ntfs_bytes_to_cluster_off(vol, file_ofs));
return 0;
}
static int ntfs_map_mft_io_for_folio(struct ntfs_inode *ni, u64 folio_byte,
struct ntfs_mft_io_unit *unit)
{
int err;
if (!down_read_trylock(&ni->runlist.lock))
return -EAGAIN;
err = ntfs_map_mft_io_locked(ni, folio_byte, unit);
up_read(&ni->runlist.lock);
return err;
}
static int ntfs_prepare_mft_folio_units(struct ntfs_inode *ni, u64 folio_byte,
u64 file_limit,
const unsigned long *record_selected,
struct ntfs_mft_io_unit *units,
unsigned int *nr_units,
unsigned int max_units, bool *defer)
{
struct ntfs_volume *vol = ni->vol;
u64 folio_end = folio_byte + PAGE_SIZE;
u64 unit_byte = folio_byte;
while (unit_byte < folio_end && unit_byte < file_limit) {
struct ntfs_mft_io_unit unit;
u64 record_byte;
u64 unit_end;
u64 io_unit_end;
u64 cluster_end;
bool selected = false;
int err;
io_unit_end =
round_down(unit_byte, (u64)vol->mft_io_unit_size) +
vol->mft_io_unit_size;
cluster_end = ntfs_cluster_to_bytes(
vol, ntfs_bytes_to_cluster(vol, unit_byte) + 1);
unit_end = min3(io_unit_end, cluster_end, folio_end);
record_byte = round_down(unit_byte, (u64)vol->mft_record_size);
while (record_byte < min(unit_end, file_limit)) {
unsigned int record;
record = div_u64(record_byte - folio_byte,
vol->mft_record_size);
if (test_bit(record, record_selected)) {
selected = true;
break;
}
record_byte += vol->mft_record_size;
}
if (!selected)
goto next;
unit = (struct ntfs_mft_io_unit){
.folio_ofs = unit_byte - folio_byte,
.len = unit_end - unit_byte,
};
err = ntfs_map_mft_io_for_folio(ni, folio_byte, &unit);
if (err == -EAGAIN) {
*defer = true;
goto next;
}
if (err)
return err;
if (*nr_units >= max_units)
return -EOVERFLOW;
units[(*nr_units)++] = unit;
next:
unit_byte = unit_end;
}
return 0;
}
static void ntfs_mft_write_error(struct ntfs_volume *vol,
struct address_space *mapping, int err)
{
mapping_set_error(mapping, err);
NVolSetErrors(vol);
ntfs_error(vol->sb, "Error while writing MFT folio: %d", err);
}
static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *wbc)
{
struct address_space *mapping = folio->mapping;
struct inode *vi = mapping->host;
struct ntfs_inode *ni = NTFS_I(vi);
struct ntfs_volume *vol = ni->vol;
struct ntfs_inode **locked_nis __free(kfree) =
kmalloc_objs(struct ntfs_inode *, PAGE_SIZE / NTFS_BLOCK_SIZE,
GFP_NOFS);
struct inode **ref_inos __free(kfree) =
kmalloc_objs(struct inode *, PAGE_SIZE / NTFS_BLOCK_SIZE, GFP_NOFS);
struct ntfs_mft_io_unit *units __free(kfree) = NULL;
struct bio *parent = NULL, *child = NULL;
struct ntfs_mft_write_ctx *ctx = NULL;
u8 *kaddr = NULL;
DECLARE_BITMAP(record_selected, PAGE_SIZE / NTFS_BLOCK_SIZE) = {};
u64 folio_byte, file_limit, folio_end, mirror_size;
unsigned int nr_records, nr_units = 0, max_units;
unsigned int nr_locked_nis = 0, nr_ref_inos = 0;
unsigned int record, unit_idx;
unsigned long flags;
loff_t i_size;
s64 allocated_size;
bool defer = false, redirty = false;
int err = 0;
read_lock_irqsave(&ni->size_lock, flags);
i_size = i_size_read(vi);
allocated_size = ni->allocated_size;
read_unlock_irqrestore(&ni->size_lock, flags);
ntfs_debug("Entering for inode 0x%llx, folio index 0x%lx.", ni->mft_no,
folio->index);
WARN_ON(!folio_test_locked(folio));
folio_byte = folio_pos(folio);
folio_end = folio_byte + PAGE_SIZE;
mirror_size = (u64)vol->mftmirr_size * vol->mft_record_size;
if (i_size <= folio_byte)
folio_zero_segment(folio, 0, PAGE_SIZE);
else if ((u64)i_size < folio_end)
folio_zero_segment(folio, i_size - folio_byte, PAGE_SIZE);
file_limit = min_t(u64, i_size, allocated_size);
nr_records = PAGE_SIZE / vol->mft_record_size;
max_units = PAGE_SIZE / min(vol->cluster_size, vol->mft_record_size);
units = kmalloc_objs(*units, max_units, GFP_NOFS);
if (!locked_nis || !ref_inos || !units) {
err = -ENOMEM;
goto out_noio;
}
kaddr = kmap_local_folio(folio, 0);
folio_clear_uptodate(folio);
for (record = 0; record < nr_records; record++) {
struct ntfs_inode *tni = NULL;
struct inode *ref_vi = NULL;
u64 record_byte =
folio_byte + (u64)record * vol->mft_record_size;
u64 mft_no;
if (record_byte >= file_limit)
continue;
mft_no = record_byte >> vol->mft_record_size_bits;
if (!ntfs_may_write_mft_record(
vol, mft_no,
(struct mft_record *)(kaddr +
record * vol->mft_record_size),
&tni, &ref_vi)) {
if (ref_vi)
ref_inos[nr_ref_inos++] = ref_vi;
continue;
}
if (ref_vi)
ref_inos[nr_ref_inos++] = ref_vi;
if (tni) {
locked_nis[nr_locked_nis++] = tni;
if (tni->nr_extents < 0 &&
tni->ext.base_ntfs_ino == NTFS_I(vol->mft_ino))
continue;
}
__set_bit(record, record_selected);
}
err = ntfs_prepare_mft_folio_units(ni, folio_byte, file_limit,
record_selected, units, &nr_units,
max_units, &defer);
if (err)
goto out_noio;
if (!nr_units) {
if (defer)
goto out_noio;
folio_mark_uptodate(folio);
kunmap_local(kaddr);
folio_start_writeback(folio);
folio_unlock(folio);
folio_end_writeback(folio);
ntfs_release_mft_write_refs(locked_nis, nr_locked_nis, ref_inos,
nr_ref_inos);
return 0;
}
parent = ntfs_alloc_mft_parent_bio(vol, folio, NULL);
if (!parent) {
err = -ENOMEM;
goto out_noio;
}
ctx = container_of(parent, struct ntfs_mft_write_ctx, bio);
if (!ntfs_add_mft_io_unit(parent, folio, &units[0])) {
err = -EIO;
goto out_noio;
}
for (unit_idx = 0; unit_idx < nr_units; unit_idx++) {
struct ntfs_mft_io_unit *unit = &units[unit_idx];
u64 file_ofs = folio_byte + unit->folio_ofs;
if (unit_idx) {
struct bio *target = child ? child : parent;
if (!ntfs_add_mft_io_unit(target, folio, unit)) {
if (child) {
ntfs_start_mft_writeback(ctx);
bio_chain(child, parent);
submit_bio(child);
child = NULL;
}
child = bio_alloc(vol->sb->s_bdev, 1,
REQ_OP_WRITE, GFP_NOIO);
if (!child) {
defer = true;
break;
}
if (!ntfs_add_mft_io_unit(child, folio, unit)) {
bio_put(child);
child = NULL;
if (!ctx->error)
ctx->error = -EIO;
redirty = true;
err = -EIO;
break;
}
}
}
if (file_ofs < mirror_size) {
int mirror_err = ntfs_sync_mft_mirror_unit(
vol, folio, file_ofs, unit);
if (mirror_err && !ctx->error)
ctx->error = mirror_err;
}
}
if (child) {
ntfs_start_mft_writeback(ctx);
bio_chain(child, parent);
submit_bio(child);
}
ntfs_start_mft_writeback(ctx);
folio_mark_uptodate(folio);
if (defer || redirty)
folio_redirty_for_writepage(wbc, folio);
kunmap_local(kaddr);
kaddr = NULL;
folio_unlock(folio);
submit_bio(parent);
ntfs_release_mft_write_refs(locked_nis, nr_locked_nis, ref_inos,
nr_ref_inos);
return 0;
out_noio:
if (kaddr) {
folio_mark_uptodate(folio);
kunmap_local(kaddr);
}
if (err && err != -ENOMEM)
ntfs_mft_write_error(vol, mapping, err);
if (parent)
bio_put(parent);
if (redirty || defer || err == -ENOMEM)
folio_redirty_for_writepage(wbc, folio);
folio_unlock(folio);
ntfs_release_mft_write_refs(locked_nis, nr_locked_nis, ref_inos,
nr_ref_inos);
if (err == -ENOMEM || (defer && !err))
return 0;
return err;
}
int ntfs_mft_writepages(struct address_space *mapping,
struct writeback_control *wbc)
{
struct folio *folio = NULL;
int error;
if (NVolShutdown(NTFS_I(mapping->host)->vol))
return -EIO;
while ((folio = writeback_iter(mapping, wbc, folio, &error)))
error = ntfs_write_mft_block(folio, wbc);
return error;
}
void ntfs_mft_mark_dirty(struct folio *folio)
{
iomap_dirty_folio(folio->mapping, folio);
}