diff options
Diffstat (limited to 'fs/ext4/fast_commit.c')
| -rw-r--r-- | fs/ext4/fast_commit.c | 210 | 
1 files changed, 124 insertions, 86 deletions
| diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 2af962cbb835..ef05bfa87798 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -229,6 +229,12 @@ __releases(&EXT4_SB(inode->i_sb)->s_fc_lock)  	finish_wait(wq, &wait.wq_entry);  } +static bool ext4_fc_disabled(struct super_block *sb) +{ +	return (!test_opt2(sb, JOURNAL_FAST_COMMIT) || +		(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)); +} +  /*   * Inform Ext4's fast about start of an inode update   * @@ -240,8 +246,7 @@ void ext4_fc_start_update(struct inode *inode)  {  	struct ext4_inode_info *ei = EXT4_I(inode); -	if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || -	    (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) +	if (ext4_fc_disabled(inode->i_sb))  		return;  restart: @@ -265,8 +270,7 @@ void ext4_fc_stop_update(struct inode *inode)  {  	struct ext4_inode_info *ei = EXT4_I(inode); -	if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || -	    (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) +	if (ext4_fc_disabled(inode->i_sb))  		return;  	if (atomic_dec_and_test(&ei->i_fc_updates)) @@ -283,8 +287,7 @@ void ext4_fc_del(struct inode *inode)  	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);  	struct ext4_fc_dentry_update *fc_dentry; -	if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || -	    (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) +	if (ext4_fc_disabled(inode->i_sb))  		return;  restart: @@ -337,8 +340,7 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl  	struct ext4_sb_info *sbi = EXT4_SB(sb);  	tid_t tid; -	if (!test_opt2(sb, JOURNAL_FAST_COMMIT) || -	    (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) +	if (ext4_fc_disabled(sb))  		return;  	ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); @@ -493,10 +495,8 @@ void __ext4_fc_track_unlink(handle_t *handle,  void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)  {  	struct inode *inode = d_inode(dentry); -	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); -	if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || -	    (sbi->s_mount_state & EXT4_FC_REPLAY)) +	if (ext4_fc_disabled(inode->i_sb))  		return;  	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) @@ -522,10 +522,8 @@ void __ext4_fc_track_link(handle_t *handle,  void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)  {  	struct inode *inode = d_inode(dentry); -	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); -	if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || -	    (sbi->s_mount_state & EXT4_FC_REPLAY)) +	if (ext4_fc_disabled(inode->i_sb))  		return;  	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) @@ -551,10 +549,8 @@ void __ext4_fc_track_create(handle_t *handle, struct inode *inode,  void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)  {  	struct inode *inode = d_inode(dentry); -	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); -	if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || -	    (sbi->s_mount_state & EXT4_FC_REPLAY)) +	if (ext4_fc_disabled(inode->i_sb))  		return;  	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) @@ -576,22 +572,20 @@ static int __track_inode(struct inode *inode, void *arg, bool update)  void ext4_fc_track_inode(handle_t *handle, struct inode *inode)  { -	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);  	int ret;  	if (S_ISDIR(inode->i_mode))  		return; +	if (ext4_fc_disabled(inode->i_sb)) +		return; +  	if (ext4_should_journal_data(inode)) {  		ext4_fc_mark_ineligible(inode->i_sb,  					EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);  		return;  	} -	if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || -	    (sbi->s_mount_state & EXT4_FC_REPLAY)) -		return; -  	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))  		return; @@ -634,15 +628,13 @@ static int __track_range(struct inode *inode, void *arg, bool update)  void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,  			 ext4_lblk_t end)  { -	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);  	struct __track_range_args args;  	int ret;  	if (S_ISDIR(inode->i_mode))  		return; -	if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || -	    (sbi->s_mount_state & EXT4_FC_REPLAY)) +	if (ext4_fc_disabled(inode->i_sb))  		return;  	if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) @@ -710,10 +702,10 @@ static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)  	 * After allocating len, we should have space at least for a 0 byte  	 * padding.  	 */ -	if (len + sizeof(struct ext4_fc_tl) > bsize) +	if (len + EXT4_FC_TAG_BASE_LEN > bsize)  		return NULL; -	if (bsize - off - 1 > len + sizeof(struct ext4_fc_tl)) { +	if (bsize - off - 1 > len + EXT4_FC_TAG_BASE_LEN) {  		/*  		 * Only allocate from current buffer if we have enough space for  		 * this request AND we have space to add a zero byte padding. @@ -730,10 +722,10 @@ static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)  	/* Need to add PAD tag */  	tl = (struct ext4_fc_tl *)(sbi->s_fc_bh->b_data + off);  	tl->fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD); -	pad_len = bsize - off - 1 - sizeof(struct ext4_fc_tl); +	pad_len = bsize - off - 1 - EXT4_FC_TAG_BASE_LEN;  	tl->fc_len = cpu_to_le16(pad_len);  	if (crc) -		*crc = ext4_chksum(sbi, *crc, tl, sizeof(*tl)); +		*crc = ext4_chksum(sbi, *crc, tl, EXT4_FC_TAG_BASE_LEN);  	if (pad_len > 0)  		ext4_fc_memzero(sb, tl + 1, pad_len, crc);  	ext4_fc_submit_bh(sb, false); @@ -775,7 +767,7 @@ static int ext4_fc_write_tail(struct super_block *sb, u32 crc)  	 * ext4_fc_reserve_space takes care of allocating an extra block if  	 * there's no enough space on this block for accommodating this tail.  	 */ -	dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(tail), &crc); +	dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc);  	if (!dst)  		return -ENOSPC; @@ -785,8 +777,8 @@ static int ext4_fc_write_tail(struct super_block *sb, u32 crc)  	tl.fc_len = cpu_to_le16(bsize - off - 1 + sizeof(struct ext4_fc_tail));  	sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize); -	ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), &crc); -	dst += sizeof(tl); +	ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, &crc); +	dst += EXT4_FC_TAG_BASE_LEN;  	tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);  	ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc);  	dst += sizeof(tail.fc_tid); @@ -808,15 +800,15 @@ static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,  	struct ext4_fc_tl tl;  	u8 *dst; -	dst = ext4_fc_reserve_space(sb, sizeof(tl) + len, crc); +	dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc);  	if (!dst)  		return false;  	tl.fc_tag = cpu_to_le16(tag);  	tl.fc_len = cpu_to_le16(len); -	ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc); -	ext4_fc_memcpy(sb, dst + sizeof(tl), val, len, crc); +	ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc); +	ext4_fc_memcpy(sb, dst + EXT4_FC_TAG_BASE_LEN, val, len, crc);  	return true;  } @@ -828,8 +820,8 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,  	struct ext4_fc_dentry_info fcd;  	struct ext4_fc_tl tl;  	int dlen = fc_dentry->fcd_name.len; -	u8 *dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(fcd) + dlen, -					crc); +	u8 *dst = ext4_fc_reserve_space(sb, +			EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);  	if (!dst)  		return false; @@ -838,8 +830,8 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,  	fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino);  	tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op);  	tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen); -	ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc); -	dst += sizeof(tl); +	ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc); +	dst += EXT4_FC_TAG_BASE_LEN;  	ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc);  	dst += sizeof(fcd);  	ext4_fc_memcpy(sb, dst, fc_dentry->fcd_name.name, dlen, crc); @@ -874,22 +866,25 @@ static int ext4_fc_write_inode(struct inode *inode, u32 *crc)  	tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);  	tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino)); +	ret = -ECANCELED;  	dst = ext4_fc_reserve_space(inode->i_sb, -			sizeof(tl) + inode_len + sizeof(fc_inode.fc_ino), crc); +		EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc);  	if (!dst) -		return -ECANCELED; +		goto err; -	if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, sizeof(tl), crc)) -		return -ECANCELED; -	dst += sizeof(tl); +	if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc)) +		goto err; +	dst += EXT4_FC_TAG_BASE_LEN;  	if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc)) -		return -ECANCELED; +		goto err;  	dst += sizeof(fc_inode);  	if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc),  					inode_len, crc)) -		return -ECANCELED; - -	return 0; +		goto err; +	ret = 0; +err: +	brelse(iloc.bh); +	return ret;  }  /* @@ -1343,7 +1338,7 @@ struct dentry_info_args {  };  static inline void tl_to_darg(struct dentry_info_args *darg, -			      struct  ext4_fc_tl *tl, u8 *val) +			      struct ext4_fc_tl *tl, u8 *val)  {  	struct ext4_fc_dentry_info fcd; @@ -1352,8 +1347,14 @@ static inline void tl_to_darg(struct dentry_info_args *darg,  	darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);  	darg->ino = le32_to_cpu(fcd.fc_ino);  	darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname); -	darg->dname_len = le16_to_cpu(tl->fc_len) - -		sizeof(struct ext4_fc_dentry_info); +	darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info); +} + +static inline void ext4_fc_get_tl(struct ext4_fc_tl *tl, u8 *val) +{ +	memcpy(tl, val, EXT4_FC_TAG_BASE_LEN); +	tl->fc_len = le16_to_cpu(tl->fc_len); +	tl->fc_tag = le16_to_cpu(tl->fc_tag);  }  /* Unlink replay function */ @@ -1491,13 +1492,15 @@ static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)  		if (state->fc_modified_inodes[i] == ino)  			return 0;  	if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) { -		state->fc_modified_inodes = krealloc( -				state->fc_modified_inodes, +		int *fc_modified_inodes; + +		fc_modified_inodes = krealloc(state->fc_modified_inodes,  				sizeof(int) * (state->fc_modified_inodes_size +  				EXT4_FC_REPLAY_REALLOC_INCREMENT),  				GFP_KERNEL); -		if (!state->fc_modified_inodes) +		if (!fc_modified_inodes)  			return -ENOMEM; +		state->fc_modified_inodes = fc_modified_inodes;  		state->fc_modified_inodes_size +=  			EXT4_FC_REPLAY_REALLOC_INCREMENT;  	} @@ -1516,7 +1519,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl,  	struct ext4_inode *raw_fc_inode;  	struct inode *inode = NULL;  	struct ext4_iloc iloc; -	int inode_len, ino, ret, tag = le16_to_cpu(tl->fc_tag); +	int inode_len, ino, ret, tag = tl->fc_tag;  	struct ext4_extent_header *eh;  	memcpy(&fc_inode, val, sizeof(fc_inode)); @@ -1541,7 +1544,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl,  	if (ret)  		goto out; -	inode_len = le16_to_cpu(tl->fc_len) - sizeof(struct ext4_fc_inode); +	inode_len = tl->fc_len - sizeof(struct ext4_fc_inode);  	raw_inode = ext4_raw_inode(&iloc);  	memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block)); @@ -1682,15 +1685,18 @@ int ext4_fc_record_regions(struct super_block *sb, int ino,  	if (replay && state->fc_regions_used != state->fc_regions_valid)  		state->fc_regions_used = state->fc_regions_valid;  	if (state->fc_regions_used == state->fc_regions_size) { +		struct ext4_fc_alloc_region *fc_regions; + +		fc_regions = krealloc(state->fc_regions, +				      sizeof(struct ext4_fc_alloc_region) * +				      (state->fc_regions_size + +				       EXT4_FC_REPLAY_REALLOC_INCREMENT), +				      GFP_KERNEL); +		if (!fc_regions) +			return -ENOMEM;  		state->fc_regions_size +=  			EXT4_FC_REPLAY_REALLOC_INCREMENT; -		state->fc_regions = krealloc( -					state->fc_regions, -					state->fc_regions_size * -					sizeof(struct ext4_fc_alloc_region), -					GFP_KERNEL); -		if (!state->fc_regions) -			return -ENOMEM; +		state->fc_regions = fc_regions;  	}  	region = &state->fc_regions[state->fc_regions_used++];  	region->ino = ino; @@ -1770,8 +1776,7 @@ static int ext4_fc_replay_add_range(struct super_block *sb,  			ret = ext4_ext_insert_extent(  				NULL, inode, &path, &newex, 0);  			up_write((&EXT4_I(inode)->i_data_sem)); -			ext4_ext_drop_refs(path); -			kfree(path); +			ext4_free_ext_path(path);  			if (ret)  				goto out;  			goto next; @@ -1926,8 +1931,7 @@ static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)  					for (j = 0; j < path->p_depth; j++)  						ext4_mb_mark_bb(inode->i_sb,  							path[j].p_block, 1, 1); -					ext4_ext_drop_refs(path); -					kfree(path); +					ext4_free_ext_path(path);  				}  				cur += ret;  				ext4_mb_mark_bb(inode->i_sb, map.m_pblk, @@ -1972,6 +1976,34 @@ void ext4_fc_replay_cleanup(struct super_block *sb)  	kfree(sbi->s_fc_replay_state.fc_modified_inodes);  } +static inline bool ext4_fc_tag_len_isvalid(struct ext4_fc_tl *tl, +					   u8 *val, u8 *end) +{ +	if (val + tl->fc_len > end) +		return false; + +	/* Here only check ADD_RANGE/TAIL/HEAD which will read data when do +	 * journal rescan before do CRC check. Other tags length check will +	 * rely on CRC check. +	 */ +	switch (tl->fc_tag) { +	case EXT4_FC_TAG_ADD_RANGE: +		return (sizeof(struct ext4_fc_add_range) == tl->fc_len); +	case EXT4_FC_TAG_TAIL: +		return (sizeof(struct ext4_fc_tail) <= tl->fc_len); +	case EXT4_FC_TAG_HEAD: +		return (sizeof(struct ext4_fc_head) == tl->fc_len); +	case EXT4_FC_TAG_DEL_RANGE: +	case EXT4_FC_TAG_LINK: +	case EXT4_FC_TAG_UNLINK: +	case EXT4_FC_TAG_CREAT: +	case EXT4_FC_TAG_INODE: +	case EXT4_FC_TAG_PAD: +	default: +		return true; +	} +} +  /*   * Recovery Scan phase handler   * @@ -2028,12 +2060,18 @@ static int ext4_fc_replay_scan(journal_t *journal,  	}  	state->fc_replay_expected_off++; -	for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) { -		memcpy(&tl, cur, sizeof(tl)); -		val = cur + sizeof(tl); +	for (cur = start; cur < end - EXT4_FC_TAG_BASE_LEN; +	     cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { +		ext4_fc_get_tl(&tl, cur); +		val = cur + EXT4_FC_TAG_BASE_LEN; +		if (!ext4_fc_tag_len_isvalid(&tl, val, end)) { +			ret = state->fc_replay_num_tags ? +				JBD2_FC_REPLAY_STOP : -ECANCELED; +			goto out_err; +		}  		ext4_debug("Scan phase, tag:%s, blk %lld\n", -			  tag2str(le16_to_cpu(tl.fc_tag)), bh->b_blocknr); -		switch (le16_to_cpu(tl.fc_tag)) { +			   tag2str(tl.fc_tag), bh->b_blocknr); +		switch (tl.fc_tag) {  		case EXT4_FC_TAG_ADD_RANGE:  			memcpy(&ext, val, sizeof(ext));  			ex = (struct ext4_extent *)&ext.fc_ex; @@ -2053,13 +2091,13 @@ static int ext4_fc_replay_scan(journal_t *journal,  		case EXT4_FC_TAG_PAD:  			state->fc_cur_tag++;  			state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, -					sizeof(tl) + le16_to_cpu(tl.fc_len)); +				EXT4_FC_TAG_BASE_LEN + tl.fc_len);  			break;  		case EXT4_FC_TAG_TAIL:  			state->fc_cur_tag++;  			memcpy(&tail, val, sizeof(tail));  			state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, -						sizeof(tl) + +						EXT4_FC_TAG_BASE_LEN +  						offsetof(struct ext4_fc_tail,  						fc_crc));  			if (le32_to_cpu(tail.fc_tid) == expected_tid && @@ -2086,7 +2124,7 @@ static int ext4_fc_replay_scan(journal_t *journal,  			}  			state->fc_cur_tag++;  			state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, -					    sizeof(tl) + le16_to_cpu(tl.fc_len)); +				EXT4_FC_TAG_BASE_LEN + tl.fc_len);  			break;  		default:  			ret = state->fc_replay_num_tags ? @@ -2141,19 +2179,20 @@ static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,  	start = (u8 *)bh->b_data;  	end = (__u8 *)bh->b_data + journal->j_blocksize - 1; -	for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) { -		memcpy(&tl, cur, sizeof(tl)); -		val = cur + sizeof(tl); +	for (cur = start; cur < end - EXT4_FC_TAG_BASE_LEN; +	     cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { +		ext4_fc_get_tl(&tl, cur); +		val = cur + EXT4_FC_TAG_BASE_LEN;  		if (state->fc_replay_num_tags == 0) {  			ret = JBD2_FC_REPLAY_STOP;  			ext4_fc_set_bitmaps_and_counters(sb);  			break;  		} -		ext4_debug("Replay phase, tag:%s\n", -				tag2str(le16_to_cpu(tl.fc_tag))); + +		ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag));  		state->fc_replay_num_tags--; -		switch (le16_to_cpu(tl.fc_tag)) { +		switch (tl.fc_tag) {  		case EXT4_FC_TAG_LINK:  			ret = ext4_fc_replay_link(sb, &tl, val);  			break; @@ -2174,19 +2213,18 @@ static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,  			break;  		case EXT4_FC_TAG_PAD:  			trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0, -					     le16_to_cpu(tl.fc_len), 0); +					     tl.fc_len, 0);  			break;  		case EXT4_FC_TAG_TAIL: -			trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, 0, -					     le16_to_cpu(tl.fc_len), 0); +			trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, +					     0, tl.fc_len, 0);  			memcpy(&tail, val, sizeof(tail));  			WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);  			break;  		case EXT4_FC_TAG_HEAD:  			break;  		default: -			trace_ext4_fc_replay(sb, le16_to_cpu(tl.fc_tag), 0, -					     le16_to_cpu(tl.fc_len), 0); +			trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0);  			ret = -ECANCELED;  			break;  		} | 
