Skip to content

Commit 536e91e

Browse files
committed
DAOS-18196 object: collective object RPC handler uses deep stack
To avoid potential ULT stack overflow. Signed-off-by: Fan Yong <[email protected]>
1 parent 19c927e commit 536e91e

File tree

15 files changed

+54
-45
lines changed

15 files changed

+54
-45
lines changed

src/container/srv_container.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2245,7 +2245,7 @@ cont_svc_eph_track_leader_start(struct cont_svc *svc)
22452245

22462246
D_ASSERT(svc->cs_cont_ephs_leader_req == NULL);
22472247
uuid_clear(anonym_uuid);
2248-
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, &anonym_uuid);
2248+
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, 0, &anonym_uuid);
22492249
svc->cs_cont_ephs_leader_req = sched_create_ult(&attr, cont_track_eph_leader_ult, svc, 0);
22502250
if (svc->cs_cont_ephs_leader_req == NULL) {
22512251
D_ERROR(DF_UUID" Failed to create EC leader eph ULT.\n",

src/container/srv_target.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -602,7 +602,7 @@ cont_start_agg(struct ds_cont_child *cont)
602602
struct dss_module_info *dmi = dss_get_module_info();
603603
struct sched_req_attr attr;
604604

605-
sched_req_attr_init(&attr, SCHED_REQ_GC, &cont->sc_pool->spc_uuid);
605+
sched_req_attr_init(&attr, SCHED_REQ_GC, 0, &cont->sc_pool->spc_uuid);
606606

607607
if (likely(!ec_agg_disabled)) {
608608
D_ASSERT(cont->sc_ec_agg_req == NULL);
@@ -1277,7 +1277,7 @@ cont_destroy_wait(struct ds_pool_child *child, uuid_t co_uuid)
12771277
DP_CONT(child->spc_uuid, co_uuid));
12781278

12791279
D_ASSERT(child != NULL);
1280-
sched_req_attr_init(&attr, SCHED_REQ_FETCH, &child->spc_uuid);
1280+
sched_req_attr_init(&attr, SCHED_REQ_FETCH, 0, &child->spc_uuid);
12811281
req = sched_req_get(&attr, ABT_THREAD_NULL);
12821282
if (req == NULL) {
12831283
D_CRIT(DF_UUID"[%d]: Failed to get sched req\n",

src/dtx/dtx_common.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ dtx_aggregation_pool(struct dss_module_info *dmi, struct dtx_batched_pool_args *
473473
struct dtx_tls *tls = dtx_tls_get();
474474

475475
D_ASSERT(dbpa->dbpa_pool);
476-
sched_req_attr_init(&attr, SCHED_REQ_GC, &dbpa->dbpa_pool->spc_uuid);
476+
sched_req_attr_init(&attr, SCHED_REQ_GC, 0, &dbpa->dbpa_pool->spc_uuid);
477477

478478
while (!dss_xstream_exiting(dmi->dmi_xstream)) {
479479
struct dtx_stat stat = { 0 };
@@ -585,7 +585,7 @@ dtx_aggregation_main(void *arg)
585585
uuid_t anonym_uuid;
586586

587587
uuid_clear(anonym_uuid);
588-
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, &anonym_uuid);
588+
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, 0, &anonym_uuid);
589589

590590
D_ASSERT(dmi->dmi_dtx_agg_req == NULL);
591591
dmi->dmi_dtx_agg_req = sched_req_get(&attr, ABT_THREAD_NULL);
@@ -705,7 +705,7 @@ dtx_batched_commit(void *arg)
705705
uuid_t anonym_uuid;
706706

707707
uuid_clear(anonym_uuid);
708-
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, &anonym_uuid);
708+
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, 0, &anonym_uuid);
709709

710710
D_ASSERT(dmi->dmi_dtx_cmt_req == NULL);
711711
dmi->dmi_dtx_cmt_req = sched_req_get(&attr, ABT_THREAD_NULL);
@@ -757,7 +757,7 @@ dtx_batched_commit(void *arg)
757757
dtx_get_dbca(dbca);
758758

759759
D_ASSERT(dbca->dbca_cont);
760-
sched_req_attr_init(&attr, SCHED_REQ_GC, &dbca->dbca_cont->sc_pool_uuid);
760+
sched_req_attr_init(&attr, SCHED_REQ_GC, 0, &dbca->dbca_cont->sc_pool_uuid);
761761
dbca->dbca_commit_req = sched_create_ult(&attr, dtx_batched_commit_one,
762762
dbca, 0);
763763
if (dbca->dbca_commit_req == NULL) {
@@ -781,7 +781,7 @@ dtx_batched_commit(void *arg)
781781
dtx_get_dbca(dbca);
782782

783783
D_ASSERT(dbca->dbca_cont);
784-
sched_req_attr_init(&attr, SCHED_REQ_GC, &dbca->dbca_cont->sc_pool_uuid);
784+
sched_req_attr_init(&attr, SCHED_REQ_GC, 0, &dbca->dbca_cont->sc_pool_uuid);
785785
dbca->dbca_cleanup_req = sched_create_ult(&attr, dtx_cleanup, dbca, 0);
786786
if (dbca->dbca_cleanup_req == NULL) {
787787
D_WARN("Fail to start DTX ULT (3) for "DF_UUID"\n",

src/engine/drpc_client.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ dss_drpc_call(int32_t module, int32_t method, void *req, size_t req_size,
116116

117117
/* Initialize sched_req for the backoffs below. */
118118
uuid_clear(anonym_uuid);
119-
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, &anonym_uuid);
119+
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, 0, &anonym_uuid);
120120
sched_req = sched_req_get(&attr, ABT_THREAD_NULL);
121121
if (sched_req == NULL) {
122122
D_ERROR("failed to get sched req\n");

src/engine/sched.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -746,12 +746,17 @@ static inline int
746746
req_kickoff_internal(struct dss_xstream *dx, struct sched_req_attr *attr,
747747
void (*func)(void *), void *arg)
748748
{
749+
uint32_t flags = 0;
750+
749751
D_ASSERT(attr && func && arg);
750752
D_ASSERT(attr->sra_type < SCHED_REQ_TYPE_MAX);
751753

752-
return sched_create_thread(dx, func, arg, ABT_THREAD_ATTR_NULL, NULL,
753-
attr->sra_flags & SCHED_REQ_FL_PERIODIC ?
754-
DSS_ULT_FL_PERIODIC : 0);
754+
if (attr->sra_flags & SCHED_REQ_FL_PERIODIC)
755+
flags |= DSS_ULT_FL_PERIODIC;
756+
if (attr->sra_flags & SCHED_REQ_FL_DEEP_STACK)
757+
flags |= DSS_ULT_DEEP_STACK;
758+
759+
return sched_create_thread(dx, func, arg, ABT_THREAD_ATTR_NULL, NULL, flags);
755760
}
756761

757762
static int

src/engine/srv.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ dss_sleep(uint64_t msec)
223223
uuid_t anonym_uuid;
224224

225225
uuid_clear(anonym_uuid);
226-
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, &anonym_uuid);
226+
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, 0, &anonym_uuid);
227227
req = sched_req_get(&attr, ABT_THREAD_NULL);
228228
if (req == NULL)
229229
return -DER_NOMEM;

src/include/daos_srv/daos_engine.h

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -158,10 +158,11 @@ enum {
158158
};
159159

160160
enum {
161-
SCHED_REQ_FL_NO_DELAY = (1 << 0),
162-
SCHED_REQ_FL_PERIODIC = (1 << 1),
163-
SCHED_REQ_FL_NO_REJECT = (1 << 2),
164-
SCHED_REQ_FL_RESENT = (1 << 3),
161+
SCHED_REQ_FL_NO_DELAY = (1 << 0),
162+
SCHED_REQ_FL_PERIODIC = (1 << 1),
163+
SCHED_REQ_FL_NO_REJECT = (1 << 2),
164+
SCHED_REQ_FL_RESENT = (1 << 3),
165+
SCHED_REQ_FL_DEEP_STACK = (1 << 4),
165166
};
166167

167168
struct sched_req_attr {
@@ -174,11 +175,10 @@ struct sched_req_attr {
174175
};
175176

176177
static inline void
177-
sched_req_attr_init(struct sched_req_attr *attr, unsigned int type,
178-
uuid_t *pool_id)
178+
sched_req_attr_init(struct sched_req_attr *attr, uint32_t type, uint32_t flags, uuid_t *pool_id)
179179
{
180-
attr->sra_type = type;
181-
attr->sra_flags = 0;
180+
attr->sra_type = type;
181+
attr->sra_flags = flags;
182182
uuid_copy(attr->sra_pool_id, *pool_id);
183183
}
184184

src/object/srv_coll.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ obj_coll_local(crt_rpc_t *rpc, struct daos_coll_shard *shards, struct dtx_coll_e
7070
coll_args.ca_tgt_bitmap = dce->dce_bitmap;
7171
coll_args.ca_tgt_bitmap_sz = dce->dce_bitmap_sz;
7272

73-
rc = dss_thread_collective_reduce(&coll_ops, &coll_args, DSS_USE_CURRENT_ULT);
73+
rc = dss_thread_collective_reduce(&coll_ops, &coll_args,
74+
DSS_ULT_DEEP_STACK | DSS_USE_CURRENT_ULT);
7475

7576
out:
7677
if (octa.octa_versions != NULL) {

src/object/srv_mod.c

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -229,14 +229,14 @@ obj_get_req_attr(crt_rpc_t *rpc, struct sched_req_attr *attr)
229229
type = SCHED_REQ_UPDATE;
230230
else
231231
type = SCHED_REQ_FETCH;
232-
sched_req_attr_init(attr, type, &orw->orw_pool_uuid);
232+
sched_req_attr_init(attr, type, 0, &orw->orw_pool_uuid);
233233
break;
234234
}
235235
case DAOS_OBJ_RPC_MIGRATE: {
236236
struct obj_migrate_in *omi = crt_req_get(rpc);
237237

238238
attr->sra_enqueue_id = omi->om_comm_in.req_in_enqueue_id;
239-
sched_req_attr_init(attr, SCHED_REQ_MIGRATE, &omi->om_pool_uuid);
239+
sched_req_attr_init(attr, SCHED_REQ_MIGRATE, 0, &omi->om_pool_uuid);
240240
break;
241241
}
242242
/*
@@ -258,7 +258,7 @@ obj_get_req_attr(crt_rpc_t *rpc, struct sched_req_attr *attr)
258258
attr->sra_enqueue_id = oei_v10->oei_comm_in.req_in_enqueue_id;
259259
}
260260
type = (oei->oei_flags & ORF_FOR_MIGRATION) ? SCHED_REQ_MIGRATE : SCHED_REQ_ANONYM;
261-
sched_req_attr_init(attr, type, &oei->oei_pool_uuid);
261+
sched_req_attr_init(attr, type, 0, &oei->oei_pool_uuid);
262262
break;
263263
}
264264
case DAOS_OBJ_RPC_PUNCH:
@@ -274,7 +274,7 @@ obj_get_req_attr(crt_rpc_t *rpc, struct sched_req_attr *attr)
274274

275275
attr->sra_enqueue_id = opi_v10->opi_comm_in.req_in_enqueue_id;
276276
}
277-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &opi->opi_pool_uuid);
277+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, 0, &opi->opi_pool_uuid);
278278
break;
279279
}
280280
case DAOS_OBJ_RPC_QUERY_KEY: {
@@ -285,7 +285,7 @@ obj_get_req_attr(crt_rpc_t *rpc, struct sched_req_attr *attr)
285285

286286
attr->sra_enqueue_id = okqi_v10->okqi_comm_in.req_in_enqueue_id;
287287
}
288-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &okqi->okqi_pool_uuid);
288+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, 0, &okqi->okqi_pool_uuid);
289289
break;
290290
}
291291
case DAOS_OBJ_RPC_SYNC: {
@@ -296,7 +296,7 @@ obj_get_req_attr(crt_rpc_t *rpc, struct sched_req_attr *attr)
296296

297297
attr->sra_enqueue_id = osi_v10->osi_comm_in.req_in_enqueue_id;
298298
}
299-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &osi->osi_pool_uuid);
299+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, 0, &osi->osi_pool_uuid);
300300
break;
301301
}
302302
case DAOS_OBJ_RPC_KEY2ANCHOR: {
@@ -307,41 +307,43 @@ obj_get_req_attr(crt_rpc_t *rpc, struct sched_req_attr *attr)
307307

308308
attr->sra_enqueue_id = oki_v10->oki_comm_in.req_in_enqueue_id;
309309
}
310-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &oki->oki_pool_uuid);
310+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, 0, &oki->oki_pool_uuid);
311311
break;
312312
}
313313
case DAOS_OBJ_RPC_EC_AGGREGATE: {
314314
struct obj_ec_agg_in *ea = crt_req_get(rpc);
315315

316316
attr->sra_enqueue_id = ea->ea_comm_in.req_in_enqueue_id;
317-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &ea->ea_pool_uuid);
317+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, 0, &ea->ea_pool_uuid);
318318
break;
319319
}
320320
case DAOS_OBJ_RPC_EC_REPLICATE: {
321321
struct obj_ec_rep_in *er = crt_req_get(rpc);
322322

323323
attr->sra_enqueue_id = er->er_comm_in.req_in_enqueue_id;
324-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &er->er_pool_uuid);
324+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, 0, &er->er_pool_uuid);
325325
break;
326326
}
327327
case DAOS_OBJ_RPC_CPD: {
328328
struct obj_cpd_in *oci = crt_req_get(rpc);
329329

330-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &oci->oci_pool_uuid);
330+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, 0, &oci->oci_pool_uuid);
331331
break;
332332
}
333333
case DAOS_OBJ_RPC_COLL_PUNCH: {
334334
struct obj_coll_punch_in *ocpi = crt_req_get(rpc);
335335

336336
attr->sra_enqueue_id = ocpi->ocpi_comm_in.req_in_enqueue_id;
337-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &ocpi->ocpi_po_uuid);
337+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, SCHED_REQ_FL_DEEP_STACK,
338+
&ocpi->ocpi_po_uuid);
338339
break;
339340
}
340341
case DAOS_OBJ_RPC_COLL_QUERY: {
341342
struct obj_coll_query_in *ocqi = crt_req_get(rpc);
342343

343344
attr->sra_enqueue_id = ocqi->ocqi_comm_in.req_in_enqueue_id;
344-
sched_req_attr_init(attr, SCHED_REQ_ANONYM, &ocqi->ocqi_po_uuid);
345+
sched_req_attr_init(attr, SCHED_REQ_ANONYM, SCHED_REQ_FL_DEEP_STACK,
346+
&ocqi->ocqi_po_uuid);
345347
break;
346348
}
347349
default:

src/pool/srv_pool.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1630,7 +1630,7 @@ start_event_timer(struct event_timer_arg *arg)
16301630

16311631
D_ASSERT(events->pse_timer == NULL);
16321632
uuid_clear(uuid);
1633-
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, &uuid);
1633+
sched_req_attr_init(&attr, SCHED_REQ_ANONYM, 0, &uuid);
16341634
events->pse_timer = sched_create_ult(&attr, event_timer, arg, 0);
16351635
if (events->pse_timer == NULL)
16361636
return -DER_NOMEM;

0 commit comments

Comments
 (0)