Fix error accounting in forward_io

src/engine/engine_io.c

@@ -20,7 +20,7 @@ void ocf_engine_forward_cache_io(struct ocf_request *req, int dir,
 	uint32_t first_cl = ocf_bytes_2_lines(cache, offset + seek);
 	uint64_t addr;
 
-	req->cache_forward_end = callback;
+	ocf_req_forward_cache_init(req, callback);
 
 	addr = cache->device->metadata_offset;
 	addr += req->map[first_cl].coll_idx * ocf_line_size(cache);
@@ -40,7 +40,7 @@ void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir,
 	uint64_t addr, bytes, total_bytes = 0, addr_next = 0;
 	uint32_t i;
 
-	req->cache_forward_end = callback;
+	ocf_req_forward_cache_init(req, callback);
 
 	if (ocf_engine_is_sequential(req)) {
 		addr = cache->device->metadata_offset;
@@ -113,15 +113,15 @@ void ocf_engine_forward_cache_io_req(struct ocf_request *req, int dir,
 void ocf_engine_forward_cache_flush_req(struct ocf_request *req,
 		ocf_req_end_t callback)
 {
-	req->cache_forward_end = callback;
+	ocf_req_forward_cache_init(req, callback);
 
 	ocf_req_forward_cache_flush(req);
 }
 
 void ocf_engine_forward_cache_discard_req(struct ocf_request *req,
 		ocf_req_end_t callback)
 {
-	req->cache_forward_end = callback;
+	ocf_req_forward_cache_init(req, callback);
 
 	ocf_req_forward_cache_discard(req, req->addr,
 			req->bytes);
@@ -133,7 +133,7 @@ void ocf_engine_forward_core_io_req(struct ocf_request *req,
 	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
 			req->bytes);
 
-	req->core_forward_end = callback;
+	ocf_req_forward_core_init(req, callback);
 
 	ocf_req_forward_core_io(req, req->rw, req->addr,
 			req->bytes, req->offset);
@@ -145,7 +145,7 @@ void ocf_engine_forward_core_flush_req(struct ocf_request *req,
 	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
 			req->bytes);
 
-	req->core_forward_end = callback;
+	ocf_req_forward_core_init(req, callback);
 
 	ocf_req_forward_core_flush(req);
 }
@@ -156,7 +156,7 @@ void ocf_engine_forward_core_discard_req(struct ocf_request *req,
 	ocf_core_stats_core_block_update(req->core, req->part_id, req->rw,
 			req->bytes);
 
-	req->core_forward_end = callback;
+	ocf_req_forward_core_init(req, callback);
 
 	ocf_req_forward_core_discard(req, req->addr, req->bytes);
 }

src/engine/engine_wo.c

@@ -62,7 +62,7 @@ static int ocf_read_wo_cache_do(struct ocf_request *req)
 	uint64_t offset = 0;
 	uint64_t increment = 0;
 
-	req->cache_forward_end = ocf_read_wo_cache_complete;
+	ocf_req_forward_cache_init(req, ocf_read_wo_cache_complete);
 
 	ocf_req_forward_cache_get(req);
 

src/metadata/metadata_io.c

@@ -111,7 +111,7 @@ static int metadata_io_read_i_atomic_step(struct ocf_request *req)
 	/* Reset position in data buffer */
 	ctx_data_seek(cache->owner, req->data, ctx_data_seek_begin, 0);
 
-	req->cache_forward_end = metadata_io_read_i_atomic_step_end;
+	ocf_req_forward_cache_init(req, metadata_io_read_i_atomic_step_end);
 
 	ocf_req_forward_cache_metadata(req, OCF_READ,
 			cache->device->metadata_offset +
@@ -217,7 +217,7 @@ static int metadata_io_do(struct ocf_request *req)
 
 	ctx_data_seek(cache->owner, req->data, ctx_data_seek_begin, 0);
 
-	req->cache_forward_end = metadata_io_end;
+	ocf_req_forward_cache_init(req, metadata_io_end);
 
 	ocf_req_forward_cache_io(req, req->rw, PAGES_TO_BYTES(m_req->page),
 			PAGES_TO_BYTES(m_req->count), 0);

src/metadata/metadata_raw_atomic.c

@@ -118,7 +118,7 @@ int raw_atomic_flush_do_asynch(struct ocf_cache *cache, struct ocf_request *req,
 	}
 
 	req->priv = complete;
-	req->cache_forward_end = _raw_atomic_io_discard_cmpl;
+	ocf_req_forward_cache_init(req, _raw_atomic_io_discard_cmpl);
 
 	if (line_no == 1) {
 		map = &req->map[0];

src/metadata/metadata_raw_dynamic.c

@@ -402,7 +402,7 @@ static int raw_dynamic_load_all_read(struct ocf_request *req)
 
 	count = metadata_io_size(context->i_page, raw->ssd_pages);
 
-	req->cache_forward_end = raw_dynamic_load_all_read_end;
+	ocf_req_forward_cache_init(req, raw_dynamic_load_all_read_end);
 
 	ocf_req_forward_cache_io(req, OCF_READ,
 			PAGES_TO_BYTES(ssd_pages_offset + context->i_page),

src/metadata/metadata_superblock.c

@@ -705,7 +705,7 @@ int ocf_metadata_read_sb(ocf_ctx_t ctx, ocf_volume_t volume,
 		goto err_data;
 	}
 
-	req->volume_forward_end = ocf_metadata_read_sb_complete;
+	ocf_req_forward_volume_init(req, ocf_metadata_read_sb_complete);
 	req->priv = context;
 
 	ocf_req_forward_volume_io_simple(req, volume, OCF_READ, 0,

src/ocf_cache.c

@@ -332,7 +332,7 @@ static void ocf_cache_volume_submit_io(ocf_io_t io)
 
 	env_atomic_set(&req->req_remaining, 3);
 
-	req->cache_forward_end = ocf_cache_io_complete;
+	ocf_req_forward_cache_init(req, ocf_cache_io_complete);
 	ocf_req_forward_cache_io(req, req->rw, req->addr,
 			req->bytes, req->offset);
 
@@ -361,7 +361,7 @@ static void ocf_cache_volume_submit_flush(ocf_io_t io)
 		return;
 	}
 
-	req->cache_forward_end = ocf_cache_volume_io_complete_generic;
+	ocf_req_forward_cache_init(req, ocf_cache_volume_io_complete_generic);
 	ocf_req_forward_cache_flush(req);
 }
 
@@ -380,7 +380,7 @@ static void ocf_cache_volume_submit_discard(ocf_io_t io)
 		return;
 	}
 
-	req->cache_forward_end = ocf_cache_volume_io_complete_generic;
+	ocf_req_forward_cache_init(req, ocf_cache_volume_io_complete_generic);
 	ocf_req_forward_cache_discard(req, req->addr,
 			req->bytes);
 }

src/ocf_request.c

@@ -440,6 +440,16 @@ void ocf_io_end_func(ocf_io_t io, int error)
 		req->io.end(io, req->io.priv1, req->io.priv2, error);
 }
 
+void ocf_req_forward_volume_init(struct ocf_request *req,
+		ocf_req_end_t callback)
+{
+	/* For forward_volume we reuse cache_* fields. This kind of forward
+	   operation should never happen together with forward_cache.
+	*/
+	req->cache_error = 0;
+	req->cache_forward_end = callback;
+}
+
 void ocf_req_forward_volume_io(struct ocf_request *req, ocf_volume_t volume,
 		int dir, uint64_t addr, uint64_t bytes, uint64_t offset)
 {
@@ -471,20 +481,23 @@ void ocf_req_forward_volume_io_simple(struct ocf_request *req,
 {
 	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
 
-	req->cache_error = 0;
-
 	ocf_req_forward_cache_get(req);
 	ocf_volume_forward_io_simple(volume, token, dir, addr, bytes);
 }
 
+void ocf_req_forward_cache_init(struct ocf_request *req,
+		ocf_req_end_t callback)
+{
+	req->cache_error = 0;
+	req->cache_forward_end = callback;
+}
+
 void ocf_req_forward_cache_io(struct ocf_request *req, int dir, uint64_t addr,
 		uint64_t bytes, uint64_t offset)
 {
 	ocf_volume_t volume = ocf_cache_get_volume(req->cache);
 	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
 
-	req->cache_error = 0;
-
 	ocf_req_forward_cache_get(req);
 	ocf_volume_forward_io(volume, token, dir, addr, bytes, offset);
 }
@@ -494,8 +507,6 @@ void ocf_req_forward_cache_flush(struct ocf_request *req)
 	ocf_volume_t volume = ocf_cache_get_volume(req->cache);
 	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
 
-	req->cache_error = 0;
-
 	ocf_req_forward_cache_get(req);
 	ocf_volume_forward_flush(volume, token);
 }
@@ -506,8 +517,6 @@ void ocf_req_forward_cache_discard(struct ocf_request *req, uint64_t addr,
 	ocf_volume_t volume = ocf_cache_get_volume(req->cache);
 	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
 
-	req->cache_error = 0;
-
 	ocf_req_forward_cache_get(req);
 	ocf_volume_forward_discard(volume, token, addr, bytes);
 }
@@ -518,8 +527,6 @@ void ocf_req_forward_cache_write_zeros(struct ocf_request *req, uint64_t addr,
 	ocf_volume_t volume = ocf_cache_get_volume(req->cache);
 	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
 
-	req->cache_error = 0;
-
 	ocf_req_forward_cache_get(req);
 	ocf_volume_forward_write_zeros(volume, token, addr, bytes);
 }
@@ -530,20 +537,23 @@ void ocf_req_forward_cache_metadata(struct ocf_request *req, int dir,
 	ocf_volume_t volume = ocf_cache_get_volume(req->cache);
 	ocf_forward_token_t token = ocf_req_to_cache_forward_token(req);
 
-	req->cache_error = 0;
-
 	ocf_req_forward_cache_get(req);
 	ocf_volume_forward_metadata(volume, token, dir, addr, bytes, offset);
 }
 
+void ocf_req_forward_core_init(struct ocf_request *req,
+		ocf_req_end_t callback)
+{
+	req->core_error = 0;
+	req->core_forward_end = callback;
+}
+
 void ocf_req_forward_core_io(struct ocf_request *req, int dir, uint64_t addr,
 		uint64_t bytes, uint64_t offset)
 {
 	ocf_volume_t volume = ocf_core_get_volume(req->core);
 	ocf_forward_token_t token = ocf_req_to_core_forward_token(req);
 
-	req->core_error = 0;
-
 	ocf_req_forward_core_get(req);
 	ocf_volume_forward_io(volume, token, dir, addr, bytes, offset);
 }
@@ -553,8 +563,6 @@ void ocf_req_forward_core_flush(struct ocf_request *req)
 	ocf_volume_t volume = ocf_core_get_volume(req->core);
 	ocf_forward_token_t token = ocf_req_to_core_forward_token(req);
 
-	req->core_error = 0;
-
 	ocf_req_forward_core_get(req);
 	ocf_volume_forward_flush(volume, token);
 }
@@ -565,8 +573,6 @@ void ocf_req_forward_core_discard(struct ocf_request *req, uint64_t addr,
 	ocf_volume_t volume = ocf_core_get_volume(req->core);
 	ocf_forward_token_t token = ocf_req_to_core_forward_token(req);
 
-	req->core_error = 0;
-
 	ocf_req_forward_core_get(req);
 	ocf_volume_forward_discard(volume, token, addr, bytes);
 }

src/ocf_request.h

@@ -172,10 +172,7 @@ struct ocf_request {
 	/* This struct is temporary. It will be consolidated with ocf_request */
 	struct ocf_request_io io;
 
-	union {
-		ocf_req_end_t cache_forward_end;
-		ocf_req_end_t volume_forward_end;
-	};
+	ocf_req_end_t cache_forward_end;
 	ocf_req_end_t core_forward_end;
 	env_atomic cache_remaining;
 	env_atomic core_remaining;
@@ -602,6 +599,9 @@ static inline struct ocf_request *ocf_req_forward_token_to_req(ocf_forward_token
 	return (struct ocf_request *)(token & ~1);
 }
 
+void ocf_req_forward_volume_init(struct ocf_request *req,
+		ocf_req_end_t callback);
+
 void ocf_req_forward_volume_io(struct ocf_request *req, ocf_volume_t volume,
 		int dir, uint64_t addr, uint64_t bytes, uint64_t offset);
 
@@ -613,6 +613,9 @@ void ocf_req_forward_volume_discard(struct ocf_request *req,
 void ocf_req_forward_volume_io_simple(struct ocf_request *req,
 		ocf_volume_t volume, int dir, uint64_t addr, uint64_t bytes);
 
+void ocf_req_forward_cache_init(struct ocf_request *req,
+		ocf_req_end_t callback);
+
 void ocf_req_forward_cache_io(struct ocf_request *req, int dir, uint64_t addr,
 		uint64_t bytes, uint64_t offset);
 
@@ -627,6 +630,9 @@ void ocf_req_forward_cache_write_zeros(struct ocf_request *req, uint64_t addr,
 void ocf_req_forward_cache_metadata(struct ocf_request *req, int dir,
 		uint64_t addr, uint64_t bytes, uint64_t offset);
 
+void ocf_req_forward_core_init(struct ocf_request *req,
+		ocf_req_end_t callback);
+
 void ocf_req_forward_core_io(struct ocf_request *req, int dir, uint64_t addr,
 		uint64_t bytes, uint64_t offset);
 

src/ocf_volume.c

@@ -281,7 +281,7 @@ ocf_io_t ocf_volume_new_io(ocf_volume_t volume, ocf_queue_t queue,
 	return ocf_io_new(volume, queue, addr, bytes, dir, io_class, flags);
 }
 
-static void ocf_volume_req_forward_complete(struct ocf_request *req, int error)
+static void ocf_volume_req_forward_end(struct ocf_request *req, int error)
 {
 	ocf_io_end_func(req, error);
 }
@@ -299,7 +299,7 @@ void ocf_volume_submit_io(ocf_io_t io)
 	if (likely(volume->type->properties->ops.submit_io)) {
 		volume->type->properties->ops.submit_io(io);
 	} else {
-		req->volume_forward_end = ocf_volume_req_forward_complete;
+		ocf_req_forward_volume_init(req, ocf_volume_req_forward_end);
 		ocf_req_forward_volume_io(req, volume, req->rw, req->addr,
 				req->bytes, req->offset);
 	}
@@ -318,7 +318,7 @@ void ocf_volume_submit_flush(ocf_io_t io)
 	if (likely(volume->type->properties->ops.submit_flush)) {
 		volume->type->properties->ops.submit_flush(io);
 	} else {
-		req->volume_forward_end = ocf_volume_req_forward_complete;
+		ocf_req_forward_volume_init(req, ocf_volume_req_forward_end);
 		ocf_req_forward_volume_flush(req, volume);
 	}
 }
@@ -336,7 +336,7 @@ void ocf_volume_submit_discard(ocf_io_t io)
 	if (likely(volume->type->properties->ops.submit_discard)) {
 		volume->type->properties->ops.submit_discard(io);
 	} else {
-		req->volume_forward_end = ocf_volume_req_forward_complete;
+		ocf_req_forward_volume_init(req, ocf_volume_req_forward_end);
 		ocf_req_forward_volume_discard(req, volume,
 				req->addr, req->bytes);
 	}

src/utils/utils_cleaner.c

@@ -290,7 +290,7 @@ static int _ocf_cleaner_fire_flush_cache(struct ocf_request *req)
 {
 	OCF_DEBUG_TRACE(req->cache);
 
-	req->cache_forward_end = _ocf_cleaner_flush_cache_end;
+	ocf_req_forward_cache_init(req, _ocf_cleaner_flush_cache_end);
 	ocf_req_forward_cache_flush(req);
 
 	return 0;
@@ -390,9 +390,8 @@ static void _ocf_cleaner_flush_core_end(struct ocf_request *req, int error)
 
 static int _ocf_cleaner_fire_flush_core(struct ocf_request *req)
 {
-	req->core_forward_end = _ocf_cleaner_flush_core_end;
-
 	/* Submit flush request */
+	ocf_req_forward_core_init(req, _ocf_cleaner_flush_core_end);
 	ocf_req_forward_core_flush(req);
 
 	return 0;
@@ -494,7 +493,7 @@ static int _ocf_cleaner_fire_core(struct ocf_request *req)
 
 	OCF_DEBUG_TRACE(req->cache);
 
-	req->core_forward_end = _ocf_cleaner_core_io_end;
+	ocf_req_forward_core_init(req, _ocf_cleaner_core_io_end);
 
 	/* Submits writes to the core */
 	ocf_req_forward_core_get(req);
@@ -559,7 +558,7 @@ static int _ocf_cleaner_fire_cache(struct ocf_request *req)
 	uint64_t addr, offset;
 	ocf_part_id_t part_id;
 
-	req->cache_forward_end = _ocf_cleaner_cache_io_end;
+	ocf_req_forward_cache_init(req, _ocf_cleaner_cache_io_end);
 	req->bytes = ocf_line_size(cache);
 
 	ocf_req_forward_cache_get(req);