Compare commits
43 Commits
f83ec76bf2
...
8b789f2b76
Author | SHA1 | Date |
---|---|---|
|
8b789f2b76 | |
|
592a93fea1 | |
|
992d4e481e | |
|
dc3382fffd | |
|
37889ceadd | |
|
05950213a9 | |
|
d4b779985a | |
|
b6f456a76f | |
|
a865562646 | |
|
a1eab4d813 | |
|
5aca7966d2 | |
|
0b47b6c354 | |
|
ce4be9e430 | |
|
20c9ccffcc | |
|
46a51f4f5e | |
|
80eb65ccf6 | |
|
8679d2687c | |
|
5b8d296475 | |
|
b62fd63ade | |
|
e1868ba37f | |
|
5282491fc4 | |
|
d162694037 | |
|
025e87f8ea | |
|
c62cff4048 | |
|
e6b733ca2f | |
|
f826edeb88 | |
|
72291a5a0e | |
|
615cd3705d | |
|
04a06b139e | |
|
e6a0deb6fa | |
|
2da6de30e6 | |
|
8d79ed36bf | |
|
afb99e9f50 | |
|
a09a8a1fbb | |
|
98c6d25931 | |
|
7947ad1561 | |
|
a5edf3550f | |
|
77b8e6fbf9 | |
|
1e451977e1 | |
|
2c334d0384 | |
|
94a4acfec1 | |
|
79f919a89c | |
|
1071d560af |
|
@ -16196,6 +16196,7 @@ R: Rik van Riel <riel@surriel.com>
|
||||||
R: Liam R. Howlett <Liam.Howlett@oracle.com>
|
R: Liam R. Howlett <Liam.Howlett@oracle.com>
|
||||||
R: Vlastimil Babka <vbabka@suse.cz>
|
R: Vlastimil Babka <vbabka@suse.cz>
|
||||||
R: Harry Yoo <harry.yoo@oracle.com>
|
R: Harry Yoo <harry.yoo@oracle.com>
|
||||||
|
R: Jann Horn <jannh@google.com>
|
||||||
L: linux-mm@kvack.org
|
L: linux-mm@kvack.org
|
||||||
S: Maintained
|
S: Maintained
|
||||||
F: include/linux/rmap.h
|
F: include/linux/rmap.h
|
||||||
|
@ -16240,6 +16241,7 @@ R: Nico Pache <npache@redhat.com>
|
||||||
R: Ryan Roberts <ryan.roberts@arm.com>
|
R: Ryan Roberts <ryan.roberts@arm.com>
|
||||||
R: Dev Jain <dev.jain@arm.com>
|
R: Dev Jain <dev.jain@arm.com>
|
||||||
R: Barry Song <baohua@kernel.org>
|
R: Barry Song <baohua@kernel.org>
|
||||||
|
R: Lance Yang <lance.yang@linux.dev>
|
||||||
L: linux-mm@kvack.org
|
L: linux-mm@kvack.org
|
||||||
S: Maintained
|
S: Maintained
|
||||||
W: http://www.linux-mm.org
|
W: http://www.linux-mm.org
|
||||||
|
|
|
@ -1795,6 +1795,7 @@ static int write_same_filled_page(struct zram *zram, unsigned long fill,
|
||||||
u32 index)
|
u32 index)
|
||||||
{
|
{
|
||||||
zram_slot_lock(zram, index);
|
zram_slot_lock(zram, index);
|
||||||
|
zram_free_page(zram, index);
|
||||||
zram_set_flag(zram, index, ZRAM_SAME);
|
zram_set_flag(zram, index, ZRAM_SAME);
|
||||||
zram_set_handle(zram, index, fill);
|
zram_set_handle(zram, index, fill);
|
||||||
zram_slot_unlock(zram, index);
|
zram_slot_unlock(zram, index);
|
||||||
|
@ -1832,6 +1833,7 @@ static int write_incompressible_page(struct zram *zram, struct page *page,
|
||||||
kunmap_local(src);
|
kunmap_local(src);
|
||||||
|
|
||||||
zram_slot_lock(zram, index);
|
zram_slot_lock(zram, index);
|
||||||
|
zram_free_page(zram, index);
|
||||||
zram_set_flag(zram, index, ZRAM_HUGE);
|
zram_set_flag(zram, index, ZRAM_HUGE);
|
||||||
zram_set_handle(zram, index, handle);
|
zram_set_handle(zram, index, handle);
|
||||||
zram_set_obj_size(zram, index, PAGE_SIZE);
|
zram_set_obj_size(zram, index, PAGE_SIZE);
|
||||||
|
@ -1855,11 +1857,6 @@ static int zram_write_page(struct zram *zram, struct page *page, u32 index)
|
||||||
unsigned long element;
|
unsigned long element;
|
||||||
bool same_filled;
|
bool same_filled;
|
||||||
|
|
||||||
/* First, free memory allocated to this slot (if any) */
|
|
||||||
zram_slot_lock(zram, index);
|
|
||||||
zram_free_page(zram, index);
|
|
||||||
zram_slot_unlock(zram, index);
|
|
||||||
|
|
||||||
mem = kmap_local_page(page);
|
mem = kmap_local_page(page);
|
||||||
same_filled = page_same_filled(mem, &element);
|
same_filled = page_same_filled(mem, &element);
|
||||||
kunmap_local(mem);
|
kunmap_local(mem);
|
||||||
|
@ -1901,6 +1898,7 @@ static int zram_write_page(struct zram *zram, struct page *page, u32 index)
|
||||||
zcomp_stream_put(zstrm);
|
zcomp_stream_put(zstrm);
|
||||||
|
|
||||||
zram_slot_lock(zram, index);
|
zram_slot_lock(zram, index);
|
||||||
|
zram_free_page(zram, index);
|
||||||
zram_set_handle(zram, index, handle);
|
zram_set_handle(zram, index, handle);
|
||||||
zram_set_obj_size(zram, index, comp_len);
|
zram_set_obj_size(zram, index, comp_len);
|
||||||
zram_slot_unlock(zram, index);
|
zram_slot_unlock(zram, index);
|
||||||
|
|
|
@ -133,7 +133,7 @@ struct journal_sector {
|
||||||
commit_id_t commit_id;
|
commit_id_t commit_id;
|
||||||
};
|
};
|
||||||
|
|
||||||
#define MAX_TAG_SIZE (JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK]))
|
#define MAX_TAG_SIZE 255
|
||||||
|
|
||||||
#define METADATA_PADDING_SECTORS 8
|
#define METADATA_PADDING_SECTORS 8
|
||||||
|
|
||||||
|
|
|
@ -3813,8 +3813,10 @@ static void raid_io_hints(struct dm_target *ti, struct queue_limits *limits)
|
||||||
struct raid_set *rs = ti->private;
|
struct raid_set *rs = ti->private;
|
||||||
unsigned int chunk_size_bytes = to_bytes(rs->md.chunk_sectors);
|
unsigned int chunk_size_bytes = to_bytes(rs->md.chunk_sectors);
|
||||||
|
|
||||||
limits->io_min = chunk_size_bytes;
|
if (chunk_size_bytes) {
|
||||||
limits->io_opt = chunk_size_bytes * mddev_data_stripes(rs);
|
limits->io_min = chunk_size_bytes;
|
||||||
|
limits->io_opt = chunk_size_bytes * mddev_data_stripes(rs);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void raid_presuspend(struct dm_target *ti)
|
static void raid_presuspend(struct dm_target *ti)
|
||||||
|
|
|
@ -456,11 +456,15 @@ static void stripe_io_hints(struct dm_target *ti,
|
||||||
struct queue_limits *limits)
|
struct queue_limits *limits)
|
||||||
{
|
{
|
||||||
struct stripe_c *sc = ti->private;
|
struct stripe_c *sc = ti->private;
|
||||||
unsigned int chunk_size = sc->chunk_size << SECTOR_SHIFT;
|
unsigned int io_min, io_opt;
|
||||||
|
|
||||||
limits->chunk_sectors = sc->chunk_size;
|
limits->chunk_sectors = sc->chunk_size;
|
||||||
limits->io_min = chunk_size;
|
|
||||||
limits->io_opt = chunk_size * sc->stripes;
|
if (!check_shl_overflow(sc->chunk_size, SECTOR_SHIFT, &io_min) &&
|
||||||
|
!check_mul_overflow(io_min, sc->stripes, &io_opt)) {
|
||||||
|
limits->io_min = io_min;
|
||||||
|
limits->io_opt = io_opt;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct target_type stripe_target = {
|
static struct target_type stripe_target = {
|
||||||
|
|
|
@ -1919,8 +1919,8 @@ static void bq27xxx_battery_update_unlocked(struct bq27xxx_device_info *di)
|
||||||
bool has_singe_flag = di->opts & BQ27XXX_O_ZERO;
|
bool has_singe_flag = di->opts & BQ27XXX_O_ZERO;
|
||||||
|
|
||||||
cache.flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, has_singe_flag);
|
cache.flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, has_singe_flag);
|
||||||
if ((cache.flags & 0xff) == 0xff)
|
if (di->chip == BQ27000 && (cache.flags & 0xff) == 0xff)
|
||||||
cache.flags = -1; /* read error */
|
cache.flags = -ENODEV; /* bq27000 hdq read error */
|
||||||
if (cache.flags >= 0) {
|
if (cache.flags >= 0) {
|
||||||
cache.capacity = bq27xxx_battery_read_soc(di);
|
cache.capacity = bq27xxx_battery_read_soc(di);
|
||||||
|
|
||||||
|
|
|
@ -1795,7 +1795,14 @@ static int reclaim_bgs_cmp(void *unused, const struct list_head *a,
|
||||||
bg1 = list_entry(a, struct btrfs_block_group, bg_list);
|
bg1 = list_entry(a, struct btrfs_block_group, bg_list);
|
||||||
bg2 = list_entry(b, struct btrfs_block_group, bg_list);
|
bg2 = list_entry(b, struct btrfs_block_group, bg_list);
|
||||||
|
|
||||||
return bg1->used > bg2->used;
|
/*
|
||||||
|
* Some other task may be updating the ->used field concurrently, but it
|
||||||
|
* is not serious if we get a stale value or load/store tearing issues,
|
||||||
|
* as sorting the list of block groups to reclaim is not critical and an
|
||||||
|
* occasional imperfect order is ok. So silence KCSAN and avoid the
|
||||||
|
* overhead of locking or any other synchronization.
|
||||||
|
*/
|
||||||
|
return data_race(bg1->used > bg2->used);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline bool btrfs_should_reclaim(const struct btrfs_fs_info *fs_info)
|
static inline bool btrfs_should_reclaim(const struct btrfs_fs_info *fs_info)
|
||||||
|
|
|
@ -1843,7 +1843,6 @@ static void fill_stack_inode_item(struct btrfs_trans_handle *trans,
|
||||||
|
|
||||||
int btrfs_fill_inode(struct btrfs_inode *inode, u32 *rdev)
|
int btrfs_fill_inode(struct btrfs_inode *inode, u32 *rdev)
|
||||||
{
|
{
|
||||||
struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
|
||||||
struct btrfs_delayed_node *delayed_node;
|
struct btrfs_delayed_node *delayed_node;
|
||||||
struct btrfs_inode_item *inode_item;
|
struct btrfs_inode_item *inode_item;
|
||||||
struct inode *vfs_inode = &inode->vfs_inode;
|
struct inode *vfs_inode = &inode->vfs_inode;
|
||||||
|
@ -1864,8 +1863,6 @@ int btrfs_fill_inode(struct btrfs_inode *inode, u32 *rdev)
|
||||||
i_uid_write(vfs_inode, btrfs_stack_inode_uid(inode_item));
|
i_uid_write(vfs_inode, btrfs_stack_inode_uid(inode_item));
|
||||||
i_gid_write(vfs_inode, btrfs_stack_inode_gid(inode_item));
|
i_gid_write(vfs_inode, btrfs_stack_inode_gid(inode_item));
|
||||||
btrfs_i_size_write(inode, btrfs_stack_inode_size(inode_item));
|
btrfs_i_size_write(inode, btrfs_stack_inode_size(inode_item));
|
||||||
btrfs_inode_set_file_extent_range(inode, 0,
|
|
||||||
round_up(i_size_read(vfs_inode), fs_info->sectorsize));
|
|
||||||
vfs_inode->i_mode = btrfs_stack_inode_mode(inode_item);
|
vfs_inode->i_mode = btrfs_stack_inode_mode(inode_item);
|
||||||
set_nlink(vfs_inode, btrfs_stack_inode_nlink(inode_item));
|
set_nlink(vfs_inode, btrfs_stack_inode_nlink(inode_item));
|
||||||
inode_set_bytes(vfs_inode, btrfs_stack_inode_nbytes(inode_item));
|
inode_set_bytes(vfs_inode, btrfs_stack_inode_nbytes(inode_item));
|
||||||
|
|
|
@ -3885,10 +3885,6 @@ static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path
|
||||||
bool filled = false;
|
bool filled = false;
|
||||||
int first_xattr_slot;
|
int first_xattr_slot;
|
||||||
|
|
||||||
ret = btrfs_init_file_extent_tree(inode);
|
|
||||||
if (ret)
|
|
||||||
goto out;
|
|
||||||
|
|
||||||
ret = btrfs_fill_inode(inode, &rdev);
|
ret = btrfs_fill_inode(inode, &rdev);
|
||||||
if (!ret)
|
if (!ret)
|
||||||
filled = true;
|
filled = true;
|
||||||
|
@ -3920,8 +3916,6 @@ static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path
|
||||||
i_uid_write(vfs_inode, btrfs_inode_uid(leaf, inode_item));
|
i_uid_write(vfs_inode, btrfs_inode_uid(leaf, inode_item));
|
||||||
i_gid_write(vfs_inode, btrfs_inode_gid(leaf, inode_item));
|
i_gid_write(vfs_inode, btrfs_inode_gid(leaf, inode_item));
|
||||||
btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item));
|
btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item));
|
||||||
btrfs_inode_set_file_extent_range(inode, 0,
|
|
||||||
round_up(i_size_read(vfs_inode), fs_info->sectorsize));
|
|
||||||
|
|
||||||
inode_set_atime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->atime),
|
inode_set_atime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->atime),
|
||||||
btrfs_timespec_nsec(leaf, &inode_item->atime));
|
btrfs_timespec_nsec(leaf, &inode_item->atime));
|
||||||
|
@ -3953,6 +3947,11 @@ static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path
|
||||||
btrfs_set_inode_mapping_order(inode);
|
btrfs_set_inode_mapping_order(inode);
|
||||||
|
|
||||||
cache_index:
|
cache_index:
|
||||||
|
ret = btrfs_init_file_extent_tree(inode);
|
||||||
|
if (ret)
|
||||||
|
goto out;
|
||||||
|
btrfs_inode_set_file_extent_range(inode, 0,
|
||||||
|
round_up(i_size_read(vfs_inode), fs_info->sectorsize));
|
||||||
/*
|
/*
|
||||||
* If we were modified in the current generation and evicted from memory
|
* If we were modified in the current generation and evicted from memory
|
||||||
* and then re-read we need to do a full sync since we don't have any
|
* and then re-read we need to do a full sync since we don't have any
|
||||||
|
|
|
@ -1964,7 +1964,7 @@ static noinline int replay_one_name(struct btrfs_trans_handle *trans,
|
||||||
|
|
||||||
search_key.objectid = log_key.objectid;
|
search_key.objectid = log_key.objectid;
|
||||||
search_key.type = BTRFS_INODE_EXTREF_KEY;
|
search_key.type = BTRFS_INODE_EXTREF_KEY;
|
||||||
search_key.offset = key->objectid;
|
search_key.offset = btrfs_extref_hash(key->objectid, name.name, name.len);
|
||||||
ret = backref_in_log(root->log_root, &search_key, key->objectid, &name);
|
ret = backref_in_log(root->log_root, &search_key, key->objectid, &name);
|
||||||
if (ret < 0) {
|
if (ret < 0) {
|
||||||
goto out;
|
goto out;
|
||||||
|
|
|
@ -2582,9 +2582,9 @@ again:
|
||||||
spin_lock(&space_info->lock);
|
spin_lock(&space_info->lock);
|
||||||
space_info->total_bytes -= bg->length;
|
space_info->total_bytes -= bg->length;
|
||||||
space_info->disk_total -= bg->length * factor;
|
space_info->disk_total -= bg->length * factor;
|
||||||
|
space_info->disk_total -= bg->zone_unusable;
|
||||||
/* There is no allocation ever happened. */
|
/* There is no allocation ever happened. */
|
||||||
ASSERT(bg->used == 0);
|
ASSERT(bg->used == 0);
|
||||||
ASSERT(bg->zone_unusable == 0);
|
|
||||||
/* No super block in a block group on the zoned setup. */
|
/* No super block in a block group on the zoned setup. */
|
||||||
ASSERT(bg->bytes_super == 0);
|
ASSERT(bg->bytes_super == 0);
|
||||||
spin_unlock(&space_info->lock);
|
spin_unlock(&space_info->lock);
|
||||||
|
|
|
@ -1075,7 +1075,7 @@ void nilfs_sysfs_delete_device_group(struct the_nilfs *nilfs)
|
||||||
************************************************************************/
|
************************************************************************/
|
||||||
|
|
||||||
static ssize_t nilfs_feature_revision_show(struct kobject *kobj,
|
static ssize_t nilfs_feature_revision_show(struct kobject *kobj,
|
||||||
struct attribute *attr, char *buf)
|
struct kobj_attribute *attr, char *buf)
|
||||||
{
|
{
|
||||||
return sysfs_emit(buf, "%d.%d\n",
|
return sysfs_emit(buf, "%d.%d\n",
|
||||||
NILFS_CURRENT_REV, NILFS_MINOR_REV);
|
NILFS_CURRENT_REV, NILFS_MINOR_REV);
|
||||||
|
@ -1087,7 +1087,7 @@ static const char features_readme_str[] =
|
||||||
"(1) revision\n\tshow current revision of NILFS file system driver.\n";
|
"(1) revision\n\tshow current revision of NILFS file system driver.\n";
|
||||||
|
|
||||||
static ssize_t nilfs_feature_README_show(struct kobject *kobj,
|
static ssize_t nilfs_feature_README_show(struct kobject *kobj,
|
||||||
struct attribute *attr,
|
struct kobj_attribute *attr,
|
||||||
char *buf)
|
char *buf)
|
||||||
{
|
{
|
||||||
return sysfs_emit(buf, features_readme_str);
|
return sysfs_emit(buf, features_readme_str);
|
||||||
|
|
|
@ -50,16 +50,16 @@ struct nilfs_sysfs_dev_subgroups {
|
||||||
struct completion sg_segments_kobj_unregister;
|
struct completion sg_segments_kobj_unregister;
|
||||||
};
|
};
|
||||||
|
|
||||||
#define NILFS_COMMON_ATTR_STRUCT(name) \
|
#define NILFS_KOBJ_ATTR_STRUCT(name) \
|
||||||
struct nilfs_##name##_attr { \
|
struct nilfs_##name##_attr { \
|
||||||
struct attribute attr; \
|
struct attribute attr; \
|
||||||
ssize_t (*show)(struct kobject *, struct attribute *, \
|
ssize_t (*show)(struct kobject *, struct kobj_attribute *, \
|
||||||
char *); \
|
char *); \
|
||||||
ssize_t (*store)(struct kobject *, struct attribute *, \
|
ssize_t (*store)(struct kobject *, struct kobj_attribute *, \
|
||||||
const char *, size_t); \
|
const char *, size_t); \
|
||||||
}
|
}
|
||||||
|
|
||||||
NILFS_COMMON_ATTR_STRUCT(feature);
|
NILFS_KOBJ_ATTR_STRUCT(feature);
|
||||||
|
|
||||||
#define NILFS_DEV_ATTR_STRUCT(name) \
|
#define NILFS_DEV_ATTR_STRUCT(name) \
|
||||||
struct nilfs_##name##_attr { \
|
struct nilfs_##name##_attr { \
|
||||||
|
|
|
@ -554,7 +554,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
|
||||||
case SMB_DIRECT_MSG_DATA_TRANSFER: {
|
case SMB_DIRECT_MSG_DATA_TRANSFER: {
|
||||||
struct smb_direct_data_transfer *data_transfer =
|
struct smb_direct_data_transfer *data_transfer =
|
||||||
(struct smb_direct_data_transfer *)recvmsg->packet;
|
(struct smb_direct_data_transfer *)recvmsg->packet;
|
||||||
unsigned int data_length;
|
u32 remaining_data_length, data_offset, data_length;
|
||||||
int avail_recvmsg_count, receive_credits;
|
int avail_recvmsg_count, receive_credits;
|
||||||
|
|
||||||
if (wc->byte_len <
|
if (wc->byte_len <
|
||||||
|
@ -564,15 +564,25 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length);
|
||||||
data_length = le32_to_cpu(data_transfer->data_length);
|
data_length = le32_to_cpu(data_transfer->data_length);
|
||||||
if (data_length) {
|
data_offset = le32_to_cpu(data_transfer->data_offset);
|
||||||
if (wc->byte_len < sizeof(struct smb_direct_data_transfer) +
|
if (wc->byte_len < data_offset ||
|
||||||
(u64)data_length) {
|
wc->byte_len < (u64)data_offset + data_length) {
|
||||||
put_recvmsg(t, recvmsg);
|
put_recvmsg(t, recvmsg);
|
||||||
smb_direct_disconnect_rdma_connection(t);
|
smb_direct_disconnect_rdma_connection(t);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (remaining_data_length > t->max_fragmented_recv_size ||
|
||||||
|
data_length > t->max_fragmented_recv_size ||
|
||||||
|
(u64)remaining_data_length + (u64)data_length >
|
||||||
|
(u64)t->max_fragmented_recv_size) {
|
||||||
|
put_recvmsg(t, recvmsg);
|
||||||
|
smb_direct_disconnect_rdma_connection(t);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data_length) {
|
||||||
if (t->full_packet_received)
|
if (t->full_packet_received)
|
||||||
recvmsg->first_segment = true;
|
recvmsg->first_segment = true;
|
||||||
|
|
||||||
|
@ -1209,78 +1219,130 @@ static int smb_direct_writev(struct ksmbd_transport *t,
|
||||||
bool need_invalidate, unsigned int remote_key)
|
bool need_invalidate, unsigned int remote_key)
|
||||||
{
|
{
|
||||||
struct smb_direct_transport *st = smb_trans_direct_transfort(t);
|
struct smb_direct_transport *st = smb_trans_direct_transfort(t);
|
||||||
int remaining_data_length;
|
size_t remaining_data_length;
|
||||||
int start, i, j;
|
size_t iov_idx;
|
||||||
int max_iov_size = st->max_send_size -
|
size_t iov_ofs;
|
||||||
|
size_t max_iov_size = st->max_send_size -
|
||||||
sizeof(struct smb_direct_data_transfer);
|
sizeof(struct smb_direct_data_transfer);
|
||||||
int ret;
|
int ret;
|
||||||
struct kvec vec;
|
|
||||||
struct smb_direct_send_ctx send_ctx;
|
struct smb_direct_send_ctx send_ctx;
|
||||||
|
int error = 0;
|
||||||
|
|
||||||
if (st->status != SMB_DIRECT_CS_CONNECTED)
|
if (st->status != SMB_DIRECT_CS_CONNECTED)
|
||||||
return -ENOTCONN;
|
return -ENOTCONN;
|
||||||
|
|
||||||
//FIXME: skip RFC1002 header..
|
//FIXME: skip RFC1002 header..
|
||||||
|
if (WARN_ON_ONCE(niovs <= 1 || iov[0].iov_len != 4))
|
||||||
|
return -EINVAL;
|
||||||
buflen -= 4;
|
buflen -= 4;
|
||||||
|
iov_idx = 1;
|
||||||
|
iov_ofs = 0;
|
||||||
|
|
||||||
remaining_data_length = buflen;
|
remaining_data_length = buflen;
|
||||||
ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen);
|
ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen);
|
||||||
|
|
||||||
smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key);
|
smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key);
|
||||||
start = i = 1;
|
while (remaining_data_length) {
|
||||||
buflen = 0;
|
struct kvec vecs[SMB_DIRECT_MAX_SEND_SGES - 1]; /* minus smbdirect hdr */
|
||||||
while (true) {
|
size_t possible_bytes = max_iov_size;
|
||||||
buflen += iov[i].iov_len;
|
size_t possible_vecs;
|
||||||
if (buflen > max_iov_size) {
|
size_t bytes = 0;
|
||||||
if (i > start) {
|
size_t nvecs = 0;
|
||||||
remaining_data_length -=
|
|
||||||
(buflen - iov[i].iov_len);
|
|
||||||
ret = smb_direct_post_send_data(st, &send_ctx,
|
|
||||||
&iov[start], i - start,
|
|
||||||
remaining_data_length);
|
|
||||||
if (ret)
|
|
||||||
goto done;
|
|
||||||
} else {
|
|
||||||
/* iov[start] is too big, break it */
|
|
||||||
int nvec = (buflen + max_iov_size - 1) /
|
|
||||||
max_iov_size;
|
|
||||||
|
|
||||||
for (j = 0; j < nvec; j++) {
|
/*
|
||||||
vec.iov_base =
|
* For the last message remaining_data_length should be
|
||||||
(char *)iov[start].iov_base +
|
* have been 0 already!
|
||||||
j * max_iov_size;
|
*/
|
||||||
vec.iov_len =
|
if (WARN_ON_ONCE(iov_idx >= niovs)) {
|
||||||
min_t(int, max_iov_size,
|
error = -EINVAL;
|
||||||
buflen - max_iov_size * j);
|
goto done;
|
||||||
remaining_data_length -= vec.iov_len;
|
}
|
||||||
ret = smb_direct_post_send_data(st, &send_ctx, &vec, 1,
|
|
||||||
remaining_data_length);
|
/*
|
||||||
if (ret)
|
* We have 2 factors which limit the arguments we pass
|
||||||
goto done;
|
* to smb_direct_post_send_data():
|
||||||
}
|
*
|
||||||
i++;
|
* 1. The number of supported sges for the send,
|
||||||
if (i == niovs)
|
* while one is reserved for the smbdirect header.
|
||||||
break;
|
* And we currently need one SGE per page.
|
||||||
}
|
* 2. The number of negotiated payload bytes per send.
|
||||||
start = i;
|
*/
|
||||||
buflen = 0;
|
possible_vecs = min_t(size_t, ARRAY_SIZE(vecs), niovs - iov_idx);
|
||||||
} else {
|
|
||||||
i++;
|
while (iov_idx < niovs && possible_vecs && possible_bytes) {
|
||||||
if (i == niovs) {
|
struct kvec *v = &vecs[nvecs];
|
||||||
/* send out all remaining vecs */
|
int page_count;
|
||||||
remaining_data_length -= buflen;
|
|
||||||
ret = smb_direct_post_send_data(st, &send_ctx,
|
v->iov_base = ((u8 *)iov[iov_idx].iov_base) + iov_ofs;
|
||||||
&iov[start], i - start,
|
v->iov_len = min_t(size_t,
|
||||||
remaining_data_length);
|
iov[iov_idx].iov_len - iov_ofs,
|
||||||
if (ret)
|
possible_bytes);
|
||||||
|
page_count = get_buf_page_count(v->iov_base, v->iov_len);
|
||||||
|
if (page_count > possible_vecs) {
|
||||||
|
/*
|
||||||
|
* If the number of pages in the buffer
|
||||||
|
* is to much (because we currently require
|
||||||
|
* one SGE per page), we need to limit the
|
||||||
|
* length.
|
||||||
|
*
|
||||||
|
* We know possible_vecs is at least 1,
|
||||||
|
* so we always keep the first page.
|
||||||
|
*
|
||||||
|
* We need to calculate the number extra
|
||||||
|
* pages (epages) we can also keep.
|
||||||
|
*
|
||||||
|
* We calculate the number of bytes in the
|
||||||
|
* first page (fplen), this should never be
|
||||||
|
* larger than v->iov_len because page_count is
|
||||||
|
* at least 2, but adding a limitation feels
|
||||||
|
* better.
|
||||||
|
*
|
||||||
|
* Then we calculate the number of bytes (elen)
|
||||||
|
* we can keep for the extra pages.
|
||||||
|
*/
|
||||||
|
size_t epages = possible_vecs - 1;
|
||||||
|
size_t fpofs = offset_in_page(v->iov_base);
|
||||||
|
size_t fplen = min_t(size_t, PAGE_SIZE - fpofs, v->iov_len);
|
||||||
|
size_t elen = min_t(size_t, v->iov_len - fplen, epages*PAGE_SIZE);
|
||||||
|
|
||||||
|
v->iov_len = fplen + elen;
|
||||||
|
page_count = get_buf_page_count(v->iov_base, v->iov_len);
|
||||||
|
if (WARN_ON_ONCE(page_count > possible_vecs)) {
|
||||||
|
/*
|
||||||
|
* Something went wrong in the above
|
||||||
|
* logic...
|
||||||
|
*/
|
||||||
|
error = -EINVAL;
|
||||||
goto done;
|
goto done;
|
||||||
break;
|
}
|
||||||
}
|
}
|
||||||
|
possible_vecs -= page_count;
|
||||||
|
nvecs += 1;
|
||||||
|
possible_bytes -= v->iov_len;
|
||||||
|
bytes += v->iov_len;
|
||||||
|
|
||||||
|
iov_ofs += v->iov_len;
|
||||||
|
if (iov_ofs >= iov[iov_idx].iov_len) {
|
||||||
|
iov_idx += 1;
|
||||||
|
iov_ofs = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
remaining_data_length -= bytes;
|
||||||
|
|
||||||
|
ret = smb_direct_post_send_data(st, &send_ctx,
|
||||||
|
vecs, nvecs,
|
||||||
|
remaining_data_length);
|
||||||
|
if (unlikely(ret)) {
|
||||||
|
error = ret;
|
||||||
|
goto done;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
done:
|
done:
|
||||||
ret = smb_direct_flush_send_list(st, &send_ctx, true);
|
ret = smb_direct_flush_send_list(st, &send_ctx, true);
|
||||||
|
if (unlikely(!ret && error))
|
||||||
|
ret = error;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* As an optimization, we don't wait for individual I/O to finish
|
* As an optimization, we don't wait for individual I/O to finish
|
||||||
|
@ -1744,6 +1806,11 @@ static int smb_direct_init_params(struct smb_direct_transport *t,
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (device->attrs.max_send_sge < SMB_DIRECT_MAX_SEND_SGES) {
|
||||||
|
pr_err("warning: device max_send_sge = %d too small\n",
|
||||||
|
device->attrs.max_send_sge);
|
||||||
|
return -EINVAL;
|
||||||
|
}
|
||||||
if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) {
|
if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) {
|
||||||
pr_err("warning: device max_recv_sge = %d too small\n",
|
pr_err("warning: device max_recv_sge = %d too small\n",
|
||||||
device->attrs.max_recv_sge);
|
device->attrs.max_recv_sge);
|
||||||
|
@ -1767,7 +1834,7 @@ static int smb_direct_init_params(struct smb_direct_transport *t,
|
||||||
|
|
||||||
cap->max_send_wr = max_send_wrs;
|
cap->max_send_wr = max_send_wrs;
|
||||||
cap->max_recv_wr = t->recv_credit_max;
|
cap->max_recv_wr = t->recv_credit_max;
|
||||||
cap->max_send_sge = max_sge_per_wr;
|
cap->max_send_sge = SMB_DIRECT_MAX_SEND_SGES;
|
||||||
cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES;
|
cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES;
|
||||||
cap->max_inline_data = 0;
|
cap->max_inline_data = 0;
|
||||||
cap->max_rdma_ctxs = t->max_rw_credits;
|
cap->max_rdma_ctxs = t->max_rw_credits;
|
||||||
|
|
|
@ -636,6 +636,7 @@ struct damon_operations {
|
||||||
* @data: Data that will be passed to @fn.
|
* @data: Data that will be passed to @fn.
|
||||||
* @repeat: Repeat invocations.
|
* @repeat: Repeat invocations.
|
||||||
* @return_code: Return code from @fn invocation.
|
* @return_code: Return code from @fn invocation.
|
||||||
|
* @dealloc_on_cancel: De-allocate when canceled.
|
||||||
*
|
*
|
||||||
* Control damon_call(), which requests specific kdamond to invoke a given
|
* Control damon_call(), which requests specific kdamond to invoke a given
|
||||||
* function. Refer to damon_call() for more details.
|
* function. Refer to damon_call() for more details.
|
||||||
|
@ -645,6 +646,7 @@ struct damon_call_control {
|
||||||
void *data;
|
void *data;
|
||||||
bool repeat;
|
bool repeat;
|
||||||
int return_code;
|
int return_code;
|
||||||
|
bool dealloc_on_cancel;
|
||||||
/* private: internal use only */
|
/* private: internal use only */
|
||||||
/* informs if the kdamond finished handling of the request */
|
/* informs if the kdamond finished handling of the request */
|
||||||
struct completion completion;
|
struct completion completion;
|
||||||
|
|
|
@ -385,6 +385,16 @@ void folio_add_lru_vma(struct folio *, struct vm_area_struct *);
|
||||||
void mark_page_accessed(struct page *);
|
void mark_page_accessed(struct page *);
|
||||||
void folio_mark_accessed(struct folio *);
|
void folio_mark_accessed(struct folio *);
|
||||||
|
|
||||||
|
static inline bool folio_may_be_lru_cached(struct folio *folio)
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
* Holding PMD-sized folios in per-CPU LRU cache unbalances accounting.
|
||||||
|
* Holding small numbers of low-order mTHP folios in per-CPU LRU cache
|
||||||
|
* will be sensible, but nobody has implemented and tested that yet.
|
||||||
|
*/
|
||||||
|
return !folio_test_large(folio);
|
||||||
|
}
|
||||||
|
|
||||||
extern atomic_t lru_disable_count;
|
extern atomic_t lru_disable_count;
|
||||||
|
|
||||||
static inline bool lru_cache_disabled(void)
|
static inline bool lru_cache_disabled(void)
|
||||||
|
|
|
@ -126,8 +126,31 @@ DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem);
|
||||||
* of concurrent destructions. Use a separate workqueue so that cgroup
|
* of concurrent destructions. Use a separate workqueue so that cgroup
|
||||||
* destruction work items don't end up filling up max_active of system_wq
|
* destruction work items don't end up filling up max_active of system_wq
|
||||||
* which may lead to deadlock.
|
* which may lead to deadlock.
|
||||||
|
*
|
||||||
|
* A cgroup destruction should enqueue work sequentially to:
|
||||||
|
* cgroup_offline_wq: use for css offline work
|
||||||
|
* cgroup_release_wq: use for css release work
|
||||||
|
* cgroup_free_wq: use for free work
|
||||||
|
*
|
||||||
|
* Rationale for using separate workqueues:
|
||||||
|
* The cgroup root free work may depend on completion of other css offline
|
||||||
|
* operations. If all tasks were enqueued to a single workqueue, this could
|
||||||
|
* create a deadlock scenario where:
|
||||||
|
* - Free work waits for other css offline work to complete.
|
||||||
|
* - But other css offline work is queued after free work in the same queue.
|
||||||
|
*
|
||||||
|
* Example deadlock scenario with single workqueue (cgroup_destroy_wq):
|
||||||
|
* 1. umount net_prio
|
||||||
|
* 2. net_prio root destruction enqueues work to cgroup_destroy_wq (CPUx)
|
||||||
|
* 3. perf_event CSS A offline enqueues work to same cgroup_destroy_wq (CPUx)
|
||||||
|
* 4. net_prio cgroup_destroy_root->cgroup_lock_and_drain_offline.
|
||||||
|
* 5. net_prio root destruction blocks waiting for perf_event CSS A offline,
|
||||||
|
* which can never complete as it's behind in the same queue and
|
||||||
|
* workqueue's max_active is 1.
|
||||||
*/
|
*/
|
||||||
static struct workqueue_struct *cgroup_destroy_wq;
|
static struct workqueue_struct *cgroup_offline_wq;
|
||||||
|
static struct workqueue_struct *cgroup_release_wq;
|
||||||
|
static struct workqueue_struct *cgroup_free_wq;
|
||||||
|
|
||||||
/* generate an array of cgroup subsystem pointers */
|
/* generate an array of cgroup subsystem pointers */
|
||||||
#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
|
#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
|
||||||
|
@ -4159,6 +4182,7 @@ static void cgroup_file_release(struct kernfs_open_file *of)
|
||||||
cft->release(of);
|
cft->release(of);
|
||||||
put_cgroup_ns(ctx->ns);
|
put_cgroup_ns(ctx->ns);
|
||||||
kfree(ctx);
|
kfree(ctx);
|
||||||
|
of->priv = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
|
static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf,
|
||||||
|
@ -5558,7 +5582,7 @@ static void css_release_work_fn(struct work_struct *work)
|
||||||
cgroup_unlock();
|
cgroup_unlock();
|
||||||
|
|
||||||
INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);
|
INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);
|
||||||
queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);
|
queue_rcu_work(cgroup_free_wq, &css->destroy_rwork);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void css_release(struct percpu_ref *ref)
|
static void css_release(struct percpu_ref *ref)
|
||||||
|
@ -5567,7 +5591,7 @@ static void css_release(struct percpu_ref *ref)
|
||||||
container_of(ref, struct cgroup_subsys_state, refcnt);
|
container_of(ref, struct cgroup_subsys_state, refcnt);
|
||||||
|
|
||||||
INIT_WORK(&css->destroy_work, css_release_work_fn);
|
INIT_WORK(&css->destroy_work, css_release_work_fn);
|
||||||
queue_work(cgroup_destroy_wq, &css->destroy_work);
|
queue_work(cgroup_release_wq, &css->destroy_work);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void init_and_link_css(struct cgroup_subsys_state *css,
|
static void init_and_link_css(struct cgroup_subsys_state *css,
|
||||||
|
@ -5701,7 +5725,7 @@ err_list_del:
|
||||||
list_del_rcu(&css->sibling);
|
list_del_rcu(&css->sibling);
|
||||||
err_free_css:
|
err_free_css:
|
||||||
INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);
|
INIT_RCU_WORK(&css->destroy_rwork, css_free_rwork_fn);
|
||||||
queue_rcu_work(cgroup_destroy_wq, &css->destroy_rwork);
|
queue_rcu_work(cgroup_free_wq, &css->destroy_rwork);
|
||||||
return ERR_PTR(err);
|
return ERR_PTR(err);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5939,7 +5963,7 @@ static void css_killed_ref_fn(struct percpu_ref *ref)
|
||||||
|
|
||||||
if (atomic_dec_and_test(&css->online_cnt)) {
|
if (atomic_dec_and_test(&css->online_cnt)) {
|
||||||
INIT_WORK(&css->destroy_work, css_killed_work_fn);
|
INIT_WORK(&css->destroy_work, css_killed_work_fn);
|
||||||
queue_work(cgroup_destroy_wq, &css->destroy_work);
|
queue_work(cgroup_offline_wq, &css->destroy_work);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -6325,8 +6349,14 @@ static int __init cgroup_wq_init(void)
|
||||||
* We would prefer to do this in cgroup_init() above, but that
|
* We would prefer to do this in cgroup_init() above, but that
|
||||||
* is called before init_workqueues(): so leave this until after.
|
* is called before init_workqueues(): so leave this until after.
|
||||||
*/
|
*/
|
||||||
cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1);
|
cgroup_offline_wq = alloc_workqueue("cgroup_offline", 0, 1);
|
||||||
BUG_ON(!cgroup_destroy_wq);
|
BUG_ON(!cgroup_offline_wq);
|
||||||
|
|
||||||
|
cgroup_release_wq = alloc_workqueue("cgroup_release", 0, 1);
|
||||||
|
BUG_ON(!cgroup_release_wq);
|
||||||
|
|
||||||
|
cgroup_free_wq = alloc_workqueue("cgroup_free", 0, 1);
|
||||||
|
BUG_ON(!cgroup_free_wq);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
core_initcall(cgroup_wq_init);
|
core_initcall(cgroup_wq_init);
|
||||||
|
|
|
@ -9551,7 +9551,7 @@ static unsigned long tg_weight(struct task_group *tg)
|
||||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||||
return scale_load_down(tg->shares);
|
return scale_load_down(tg->shares);
|
||||||
#else
|
#else
|
||||||
return sched_weight_from_cgroup(tg->scx_weight);
|
return sched_weight_from_cgroup(tg->scx.weight);
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6788,12 +6788,8 @@ __bpf_kfunc u32 scx_bpf_reenqueue_local(void)
|
||||||
* CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
|
* CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
|
||||||
* the current local DSQ for running tasks and thus are not
|
* the current local DSQ for running tasks and thus are not
|
||||||
* visible to the BPF scheduler.
|
* visible to the BPF scheduler.
|
||||||
*
|
|
||||||
* Also skip re-enqueueing tasks that can only run on this
|
|
||||||
* CPU, as they would just be re-added to the same local
|
|
||||||
* DSQ without any benefit.
|
|
||||||
*/
|
*/
|
||||||
if (p->migration_pending || is_migration_disabled(p) || p->nr_cpus_allowed == 1)
|
if (p->migration_pending)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
dispatch_dequeue(rq, p);
|
dispatch_dequeue(rq, p);
|
||||||
|
|
|
@ -908,6 +908,8 @@ static int trace_kprobe_create_internal(int argc, const char *argv[],
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
}
|
}
|
||||||
buf = kmemdup(&argv[0][1], len + 1, GFP_KERNEL);
|
buf = kmemdup(&argv[0][1], len + 1, GFP_KERNEL);
|
||||||
|
if (!buf)
|
||||||
|
return -ENOMEM;
|
||||||
buf[len] = '\0';
|
buf[len] = '\0';
|
||||||
ret = kstrtouint(buf, 0, &maxactive);
|
ret = kstrtouint(buf, 0, &maxactive);
|
||||||
if (ret || !maxactive) {
|
if (ret || !maxactive) {
|
||||||
|
|
|
@ -2479,10 +2479,14 @@ static void kdamond_call(struct damon_ctx *ctx, bool cancel)
|
||||||
mutex_lock(&ctx->call_controls_lock);
|
mutex_lock(&ctx->call_controls_lock);
|
||||||
list_del(&control->list);
|
list_del(&control->list);
|
||||||
mutex_unlock(&ctx->call_controls_lock);
|
mutex_unlock(&ctx->call_controls_lock);
|
||||||
if (!control->repeat)
|
if (!control->repeat) {
|
||||||
complete(&control->completion);
|
complete(&control->completion);
|
||||||
else
|
} else if (control->canceled && control->dealloc_on_cancel) {
|
||||||
|
kfree(control);
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
list_add(&control->list, &repeat_controls);
|
list_add(&control->list, &repeat_controls);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
control = list_first_entry_or_null(&repeat_controls,
|
control = list_first_entry_or_null(&repeat_controls,
|
||||||
struct damon_call_control, list);
|
struct damon_call_control, list);
|
||||||
|
|
|
@ -1534,14 +1534,10 @@ static int damon_sysfs_repeat_call_fn(void *data)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct damon_call_control damon_sysfs_repeat_call_control = {
|
|
||||||
.fn = damon_sysfs_repeat_call_fn,
|
|
||||||
.repeat = true,
|
|
||||||
};
|
|
||||||
|
|
||||||
static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond)
|
static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond)
|
||||||
{
|
{
|
||||||
struct damon_ctx *ctx;
|
struct damon_ctx *ctx;
|
||||||
|
struct damon_call_control *repeat_call_control;
|
||||||
int err;
|
int err;
|
||||||
|
|
||||||
if (damon_sysfs_kdamond_running(kdamond))
|
if (damon_sysfs_kdamond_running(kdamond))
|
||||||
|
@ -1554,18 +1550,29 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond)
|
||||||
damon_destroy_ctx(kdamond->damon_ctx);
|
damon_destroy_ctx(kdamond->damon_ctx);
|
||||||
kdamond->damon_ctx = NULL;
|
kdamond->damon_ctx = NULL;
|
||||||
|
|
||||||
|
repeat_call_control = kmalloc(sizeof(*repeat_call_control),
|
||||||
|
GFP_KERNEL);
|
||||||
|
if (!repeat_call_control)
|
||||||
|
return -ENOMEM;
|
||||||
|
|
||||||
ctx = damon_sysfs_build_ctx(kdamond->contexts->contexts_arr[0]);
|
ctx = damon_sysfs_build_ctx(kdamond->contexts->contexts_arr[0]);
|
||||||
if (IS_ERR(ctx))
|
if (IS_ERR(ctx)) {
|
||||||
|
kfree(repeat_call_control);
|
||||||
return PTR_ERR(ctx);
|
return PTR_ERR(ctx);
|
||||||
|
}
|
||||||
err = damon_start(&ctx, 1, false);
|
err = damon_start(&ctx, 1, false);
|
||||||
if (err) {
|
if (err) {
|
||||||
|
kfree(repeat_call_control);
|
||||||
damon_destroy_ctx(ctx);
|
damon_destroy_ctx(ctx);
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
kdamond->damon_ctx = ctx;
|
kdamond->damon_ctx = ctx;
|
||||||
|
|
||||||
damon_sysfs_repeat_call_control.data = kdamond;
|
repeat_call_control->fn = damon_sysfs_repeat_call_fn;
|
||||||
damon_call(ctx, &damon_sysfs_repeat_call_control);
|
repeat_call_control->data = kdamond;
|
||||||
|
repeat_call_control->repeat = true;
|
||||||
|
repeat_call_control->dealloc_on_cancel = true;
|
||||||
|
damon_call(ctx, repeat_call_control);
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
14
mm/gup.c
14
mm/gup.c
|
@ -2287,8 +2287,8 @@ static unsigned long collect_longterm_unpinnable_folios(
|
||||||
struct pages_or_folios *pofs)
|
struct pages_or_folios *pofs)
|
||||||
{
|
{
|
||||||
unsigned long collected = 0;
|
unsigned long collected = 0;
|
||||||
bool drain_allow = true;
|
|
||||||
struct folio *folio;
|
struct folio *folio;
|
||||||
|
int drained = 0;
|
||||||
long i = 0;
|
long i = 0;
|
||||||
|
|
||||||
for (folio = pofs_get_folio(pofs, i); folio;
|
for (folio = pofs_get_folio(pofs, i); folio;
|
||||||
|
@ -2307,9 +2307,17 @@ static unsigned long collect_longterm_unpinnable_folios(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!folio_test_lru(folio) && drain_allow) {
|
if (drained == 0 && folio_may_be_lru_cached(folio) &&
|
||||||
|
folio_ref_count(folio) !=
|
||||||
|
folio_expected_ref_count(folio) + 1) {
|
||||||
|
lru_add_drain();
|
||||||
|
drained = 1;
|
||||||
|
}
|
||||||
|
if (drained == 1 && folio_may_be_lru_cached(folio) &&
|
||||||
|
folio_ref_count(folio) !=
|
||||||
|
folio_expected_ref_count(folio) + 1) {
|
||||||
lru_add_drain_all();
|
lru_add_drain_all();
|
||||||
drain_allow = false;
|
drained = 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!folio_isolate_lru(folio))
|
if (!folio_isolate_lru(folio))
|
||||||
|
|
|
@ -255,7 +255,7 @@ void mlock_folio(struct folio *folio)
|
||||||
|
|
||||||
folio_get(folio);
|
folio_get(folio);
|
||||||
if (!folio_batch_add(fbatch, mlock_lru(folio)) ||
|
if (!folio_batch_add(fbatch, mlock_lru(folio)) ||
|
||||||
folio_test_large(folio) || lru_cache_disabled())
|
!folio_may_be_lru_cached(folio) || lru_cache_disabled())
|
||||||
mlock_folio_batch(fbatch);
|
mlock_folio_batch(fbatch);
|
||||||
local_unlock(&mlock_fbatch.lock);
|
local_unlock(&mlock_fbatch.lock);
|
||||||
}
|
}
|
||||||
|
@ -278,7 +278,7 @@ void mlock_new_folio(struct folio *folio)
|
||||||
|
|
||||||
folio_get(folio);
|
folio_get(folio);
|
||||||
if (!folio_batch_add(fbatch, mlock_new(folio)) ||
|
if (!folio_batch_add(fbatch, mlock_new(folio)) ||
|
||||||
folio_test_large(folio) || lru_cache_disabled())
|
!folio_may_be_lru_cached(folio) || lru_cache_disabled())
|
||||||
mlock_folio_batch(fbatch);
|
mlock_folio_batch(fbatch);
|
||||||
local_unlock(&mlock_fbatch.lock);
|
local_unlock(&mlock_fbatch.lock);
|
||||||
}
|
}
|
||||||
|
@ -299,7 +299,7 @@ void munlock_folio(struct folio *folio)
|
||||||
*/
|
*/
|
||||||
folio_get(folio);
|
folio_get(folio);
|
||||||
if (!folio_batch_add(fbatch, folio) ||
|
if (!folio_batch_add(fbatch, folio) ||
|
||||||
folio_test_large(folio) || lru_cache_disabled())
|
!folio_may_be_lru_cached(folio) || lru_cache_disabled())
|
||||||
mlock_folio_batch(fbatch);
|
mlock_folio_batch(fbatch);
|
||||||
local_unlock(&mlock_fbatch.lock);
|
local_unlock(&mlock_fbatch.lock);
|
||||||
}
|
}
|
||||||
|
|
50
mm/swap.c
50
mm/swap.c
|
@ -164,6 +164,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
|
||||||
for (i = 0; i < folio_batch_count(fbatch); i++) {
|
for (i = 0; i < folio_batch_count(fbatch); i++) {
|
||||||
struct folio *folio = fbatch->folios[i];
|
struct folio *folio = fbatch->folios[i];
|
||||||
|
|
||||||
|
/* block memcg migration while the folio moves between lru */
|
||||||
|
if (move_fn != lru_add && !folio_test_clear_lru(folio))
|
||||||
|
continue;
|
||||||
|
|
||||||
folio_lruvec_relock_irqsave(folio, &lruvec, &flags);
|
folio_lruvec_relock_irqsave(folio, &lruvec, &flags);
|
||||||
move_fn(lruvec, folio);
|
move_fn(lruvec, folio);
|
||||||
|
|
||||||
|
@ -176,14 +180,10 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
|
||||||
}
|
}
|
||||||
|
|
||||||
static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
|
static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
|
||||||
struct folio *folio, move_fn_t move_fn,
|
struct folio *folio, move_fn_t move_fn, bool disable_irq)
|
||||||
bool on_lru, bool disable_irq)
|
|
||||||
{
|
{
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
|
|
||||||
if (on_lru && !folio_test_clear_lru(folio))
|
|
||||||
return;
|
|
||||||
|
|
||||||
folio_get(folio);
|
folio_get(folio);
|
||||||
|
|
||||||
if (disable_irq)
|
if (disable_irq)
|
||||||
|
@ -191,8 +191,8 @@ static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
|
||||||
else
|
else
|
||||||
local_lock(&cpu_fbatches.lock);
|
local_lock(&cpu_fbatches.lock);
|
||||||
|
|
||||||
if (!folio_batch_add(this_cpu_ptr(fbatch), folio) || folio_test_large(folio) ||
|
if (!folio_batch_add(this_cpu_ptr(fbatch), folio) ||
|
||||||
lru_cache_disabled())
|
!folio_may_be_lru_cached(folio) || lru_cache_disabled())
|
||||||
folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn);
|
folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn);
|
||||||
|
|
||||||
if (disable_irq)
|
if (disable_irq)
|
||||||
|
@ -201,13 +201,13 @@ static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
|
||||||
local_unlock(&cpu_fbatches.lock);
|
local_unlock(&cpu_fbatches.lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
#define folio_batch_add_and_move(folio, op, on_lru) \
|
#define folio_batch_add_and_move(folio, op) \
|
||||||
__folio_batch_add_and_move( \
|
__folio_batch_add_and_move( \
|
||||||
&cpu_fbatches.op, \
|
&cpu_fbatches.op, \
|
||||||
folio, \
|
folio, \
|
||||||
op, \
|
op, \
|
||||||
on_lru, \
|
offsetof(struct cpu_fbatches, op) >= \
|
||||||
offsetof(struct cpu_fbatches, op) >= offsetof(struct cpu_fbatches, lock_irq) \
|
offsetof(struct cpu_fbatches, lock_irq) \
|
||||||
)
|
)
|
||||||
|
|
||||||
static void lru_move_tail(struct lruvec *lruvec, struct folio *folio)
|
static void lru_move_tail(struct lruvec *lruvec, struct folio *folio)
|
||||||
|
@ -231,10 +231,10 @@ static void lru_move_tail(struct lruvec *lruvec, struct folio *folio)
|
||||||
void folio_rotate_reclaimable(struct folio *folio)
|
void folio_rotate_reclaimable(struct folio *folio)
|
||||||
{
|
{
|
||||||
if (folio_test_locked(folio) || folio_test_dirty(folio) ||
|
if (folio_test_locked(folio) || folio_test_dirty(folio) ||
|
||||||
folio_test_unevictable(folio))
|
folio_test_unevictable(folio) || !folio_test_lru(folio))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
folio_batch_add_and_move(folio, lru_move_tail, true);
|
folio_batch_add_and_move(folio, lru_move_tail);
|
||||||
}
|
}
|
||||||
|
|
||||||
void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
|
void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
|
||||||
|
@ -328,10 +328,11 @@ static void folio_activate_drain(int cpu)
|
||||||
|
|
||||||
void folio_activate(struct folio *folio)
|
void folio_activate(struct folio *folio)
|
||||||
{
|
{
|
||||||
if (folio_test_active(folio) || folio_test_unevictable(folio))
|
if (folio_test_active(folio) || folio_test_unevictable(folio) ||
|
||||||
|
!folio_test_lru(folio))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
folio_batch_add_and_move(folio, lru_activate, true);
|
folio_batch_add_and_move(folio, lru_activate);
|
||||||
}
|
}
|
||||||
|
|
||||||
#else
|
#else
|
||||||
|
@ -507,7 +508,7 @@ void folio_add_lru(struct folio *folio)
|
||||||
lru_gen_in_fault() && !(current->flags & PF_MEMALLOC))
|
lru_gen_in_fault() && !(current->flags & PF_MEMALLOC))
|
||||||
folio_set_active(folio);
|
folio_set_active(folio);
|
||||||
|
|
||||||
folio_batch_add_and_move(folio, lru_add, false);
|
folio_batch_add_and_move(folio, lru_add);
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(folio_add_lru);
|
EXPORT_SYMBOL(folio_add_lru);
|
||||||
|
|
||||||
|
@ -685,13 +686,13 @@ void lru_add_drain_cpu(int cpu)
|
||||||
void deactivate_file_folio(struct folio *folio)
|
void deactivate_file_folio(struct folio *folio)
|
||||||
{
|
{
|
||||||
/* Deactivating an unevictable folio will not accelerate reclaim */
|
/* Deactivating an unevictable folio will not accelerate reclaim */
|
||||||
if (folio_test_unevictable(folio))
|
if (folio_test_unevictable(folio) || !folio_test_lru(folio))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
if (lru_gen_enabled() && lru_gen_clear_refs(folio))
|
if (lru_gen_enabled() && lru_gen_clear_refs(folio))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
folio_batch_add_and_move(folio, lru_deactivate_file, true);
|
folio_batch_add_and_move(folio, lru_deactivate_file);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -704,13 +705,13 @@ void deactivate_file_folio(struct folio *folio)
|
||||||
*/
|
*/
|
||||||
void folio_deactivate(struct folio *folio)
|
void folio_deactivate(struct folio *folio)
|
||||||
{
|
{
|
||||||
if (folio_test_unevictable(folio))
|
if (folio_test_unevictable(folio) || !folio_test_lru(folio))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio))
|
if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
folio_batch_add_and_move(folio, lru_deactivate, true);
|
folio_batch_add_and_move(folio, lru_deactivate);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -723,10 +724,11 @@ void folio_deactivate(struct folio *folio)
|
||||||
void folio_mark_lazyfree(struct folio *folio)
|
void folio_mark_lazyfree(struct folio *folio)
|
||||||
{
|
{
|
||||||
if (!folio_test_anon(folio) || !folio_test_swapbacked(folio) ||
|
if (!folio_test_anon(folio) || !folio_test_swapbacked(folio) ||
|
||||||
|
!folio_test_lru(folio) ||
|
||||||
folio_test_swapcache(folio) || folio_test_unevictable(folio))
|
folio_test_swapcache(folio) || folio_test_unevictable(folio))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
folio_batch_add_and_move(folio, lru_lazyfree, true);
|
folio_batch_add_and_move(folio, lru_lazyfree);
|
||||||
}
|
}
|
||||||
|
|
||||||
void lru_add_drain(void)
|
void lru_add_drain(void)
|
||||||
|
|
|
@ -4507,7 +4507,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ineligible */
|
/* ineligible */
|
||||||
if (!folio_test_lru(folio) || zone > sc->reclaim_idx) {
|
if (zone > sc->reclaim_idx) {
|
||||||
gen = folio_inc_gen(lruvec, folio, false);
|
gen = folio_inc_gen(lruvec, folio, false);
|
||||||
list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
|
list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
|
||||||
return true;
|
return true;
|
||||||
|
|
|
@ -208,6 +208,9 @@ static int damon_sample_mtier_enable_store(
|
||||||
if (enabled == is_enabled)
|
if (enabled == is_enabled)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
|
if (!init_called)
|
||||||
|
return 0;
|
||||||
|
|
||||||
if (enabled) {
|
if (enabled) {
|
||||||
err = damon_sample_mtier_start();
|
err = damon_sample_mtier_start();
|
||||||
if (err)
|
if (err)
|
||||||
|
|
|
@ -137,6 +137,9 @@ static int damon_sample_prcl_enable_store(
|
||||||
if (enabled == is_enabled)
|
if (enabled == is_enabled)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
|
if (!init_called)
|
||||||
|
return 0;
|
||||||
|
|
||||||
if (enabled) {
|
if (enabled) {
|
||||||
err = damon_sample_prcl_start();
|
err = damon_sample_prcl_start();
|
||||||
if (err)
|
if (err)
|
||||||
|
|
|
@ -118,6 +118,9 @@ static int damon_sample_wsse_enable_store(
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
if (enabled) {
|
if (enabled) {
|
||||||
|
if (!init_called)
|
||||||
|
return 0;
|
||||||
|
|
||||||
err = damon_sample_wsse_start();
|
err = damon_sample_wsse_start();
|
||||||
if (err)
|
if (err)
|
||||||
enabled = false;
|
enabled = false;
|
||||||
|
|
|
@ -75,6 +75,9 @@ void exclude_cmds(struct cmdnames *cmds, struct cmdnames *excludes)
|
||||||
size_t ci, cj, ei;
|
size_t ci, cj, ei;
|
||||||
int cmp;
|
int cmp;
|
||||||
|
|
||||||
|
if (!excludes->cnt)
|
||||||
|
return;
|
||||||
|
|
||||||
ci = cj = ei = 0;
|
ci = cj = ei = 0;
|
||||||
while (ci < cmds->cnt && ei < excludes->cnt) {
|
while (ci < cmds->cnt && ei < excludes->cnt) {
|
||||||
cmp = strcmp(cmds->names[ci]->name, excludes->names[ei]->name);
|
cmp = strcmp(cmds->names[ci]->name, excludes->names[ei]->name);
|
||||||
|
|
|
@ -2009,6 +2009,7 @@ static int __cmd_contention(int argc, const char **argv)
|
||||||
.owner = show_lock_owner,
|
.owner = show_lock_owner,
|
||||||
.cgroups = RB_ROOT,
|
.cgroups = RB_ROOT,
|
||||||
};
|
};
|
||||||
|
struct perf_env host_env;
|
||||||
|
|
||||||
lockhash_table = calloc(LOCKHASH_SIZE, sizeof(*lockhash_table));
|
lockhash_table = calloc(LOCKHASH_SIZE, sizeof(*lockhash_table));
|
||||||
if (!lockhash_table)
|
if (!lockhash_table)
|
||||||
|
@ -2024,7 +2025,10 @@ static int __cmd_contention(int argc, const char **argv)
|
||||||
eops.mmap = perf_event__process_mmap;
|
eops.mmap = perf_event__process_mmap;
|
||||||
eops.tracing_data = perf_event__process_tracing_data;
|
eops.tracing_data = perf_event__process_tracing_data;
|
||||||
|
|
||||||
session = perf_session__new(use_bpf ? NULL : &data, &eops);
|
perf_env__init(&host_env);
|
||||||
|
session = __perf_session__new(use_bpf ? NULL : &data, &eops,
|
||||||
|
/*trace_event_repipe=*/false, &host_env);
|
||||||
|
|
||||||
if (IS_ERR(session)) {
|
if (IS_ERR(session)) {
|
||||||
pr_err("Initializing perf session failed\n");
|
pr_err("Initializing perf session failed\n");
|
||||||
err = PTR_ERR(session);
|
err = PTR_ERR(session);
|
||||||
|
@ -2142,6 +2146,7 @@ out_delete:
|
||||||
evlist__delete(con.evlist);
|
evlist__delete(con.evlist);
|
||||||
lock_contention_finish(&con);
|
lock_contention_finish(&con);
|
||||||
perf_session__delete(session);
|
perf_session__delete(session);
|
||||||
|
perf_env__exit(&host_env);
|
||||||
zfree(&lockhash_table);
|
zfree(&lockhash_table);
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
|
@ -477,6 +477,7 @@ static int __maps__insert(struct maps *maps, struct map *new)
|
||||||
}
|
}
|
||||||
/* Insert the value at the end. */
|
/* Insert the value at the end. */
|
||||||
maps_by_address[nr_maps] = map__get(new);
|
maps_by_address[nr_maps] = map__get(new);
|
||||||
|
map__set_kmap_maps(new, maps);
|
||||||
if (maps_by_name)
|
if (maps_by_name)
|
||||||
maps_by_name[nr_maps] = map__get(new);
|
maps_by_name[nr_maps] = map__get(new);
|
||||||
|
|
||||||
|
@ -502,8 +503,6 @@ static int __maps__insert(struct maps *maps, struct map *new)
|
||||||
if (map__end(new) < map__start(new))
|
if (map__end(new) < map__start(new))
|
||||||
RC_CHK_ACCESS(maps)->ends_broken = true;
|
RC_CHK_ACCESS(maps)->ends_broken = true;
|
||||||
|
|
||||||
map__set_kmap_maps(new, maps);
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -891,6 +890,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new)
|
||||||
if (before) {
|
if (before) {
|
||||||
map__put(maps_by_address[i]);
|
map__put(maps_by_address[i]);
|
||||||
maps_by_address[i] = before;
|
maps_by_address[i] = before;
|
||||||
|
map__set_kmap_maps(before, maps);
|
||||||
|
|
||||||
if (maps_by_name) {
|
if (maps_by_name) {
|
||||||
map__put(maps_by_name[ni]);
|
map__put(maps_by_name[ni]);
|
||||||
|
@ -918,6 +918,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new)
|
||||||
*/
|
*/
|
||||||
map__put(maps_by_address[i]);
|
map__put(maps_by_address[i]);
|
||||||
maps_by_address[i] = map__get(new);
|
maps_by_address[i] = map__get(new);
|
||||||
|
map__set_kmap_maps(new, maps);
|
||||||
|
|
||||||
if (maps_by_name) {
|
if (maps_by_name) {
|
||||||
map__put(maps_by_name[ni]);
|
map__put(maps_by_name[ni]);
|
||||||
|
@ -942,14 +943,13 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new)
|
||||||
*/
|
*/
|
||||||
map__put(maps_by_address[i]);
|
map__put(maps_by_address[i]);
|
||||||
maps_by_address[i] = map__get(new);
|
maps_by_address[i] = map__get(new);
|
||||||
|
map__set_kmap_maps(new, maps);
|
||||||
|
|
||||||
if (maps_by_name) {
|
if (maps_by_name) {
|
||||||
map__put(maps_by_name[ni]);
|
map__put(maps_by_name[ni]);
|
||||||
maps_by_name[ni] = map__get(new);
|
maps_by_name[ni] = map__get(new);
|
||||||
}
|
}
|
||||||
|
|
||||||
map__set_kmap_maps(new, maps);
|
|
||||||
|
|
||||||
check_invariants(maps);
|
check_invariants(maps);
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
@ -1019,6 +1019,7 @@ int maps__copy_from(struct maps *dest, struct maps *parent)
|
||||||
err = unwind__prepare_access(dest, new, NULL);
|
err = unwind__prepare_access(dest, new, NULL);
|
||||||
if (!err) {
|
if (!err) {
|
||||||
dest_maps_by_address[i] = new;
|
dest_maps_by_address[i] = new;
|
||||||
|
map__set_kmap_maps(new, dest);
|
||||||
if (dest_maps_by_name)
|
if (dest_maps_by_name)
|
||||||
dest_maps_by_name[i] = map__get(new);
|
dest_maps_by_name[i] = map__get(new);
|
||||||
RC_CHK_ACCESS(dest)->nr_maps = i + 1;
|
RC_CHK_ACCESS(dest)->nr_maps = i + 1;
|
||||||
|
|
Loading…
Reference in New Issue