1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50#ifndef _LUSTRE_NET_H
51#define _LUSTRE_NET_H
52
53
54
55
56
57
58#include "../../include/linux/libcfs/libcfs.h"
59
60#include "../../include/linux/lnet/lnet.h"
61#include "lustre/lustre_idl.h"
62#include "lustre_ha.h"
63#include "lustre_sec.h"
64#include "lustre_import.h"
65#include "lprocfs_status.h"
66#include "lu_object.h"
67#include "lustre_req_layout.h"
68
69#include "obd_support.h"
70#include "lustre_ver.h"
71
72
73#define PTLRPC_MD_OPTIONS 0
74
75
76
77
78
79
80
81#define PTLRPC_BULK_OPS_BITS 2
82#define PTLRPC_BULK_OPS_COUNT (1U << PTLRPC_BULK_OPS_BITS)
83
84
85
86
87
88
89
90#define PTLRPC_BULK_OPS_MASK (~((__u64)PTLRPC_BULK_OPS_COUNT - 1))
91
92
93
94
95
96
97
98
99#define PTLRPC_MAX_BRW_BITS (LNET_MTU_BITS + PTLRPC_BULK_OPS_BITS)
100#define PTLRPC_MAX_BRW_SIZE (1 << PTLRPC_MAX_BRW_BITS)
101#define PTLRPC_MAX_BRW_PAGES (PTLRPC_MAX_BRW_SIZE >> PAGE_CACHE_SHIFT)
102
103#define ONE_MB_BRW_SIZE (1 << LNET_MTU_BITS)
104#define MD_MAX_BRW_SIZE (1 << LNET_MTU_BITS)
105#define MD_MAX_BRW_PAGES (MD_MAX_BRW_SIZE >> PAGE_CACHE_SHIFT)
106#define DT_MAX_BRW_SIZE PTLRPC_MAX_BRW_SIZE
107#define DT_MAX_BRW_PAGES (DT_MAX_BRW_SIZE >> PAGE_CACHE_SHIFT)
108#define OFD_MAX_BRW_SIZE (1 << LNET_MTU_BITS)
109
110
111# if ((PTLRPC_MAX_BRW_PAGES & (PTLRPC_MAX_BRW_PAGES - 1)) != 0)
112# error "PTLRPC_MAX_BRW_PAGES isn't a power of two"
113# endif
114# if (PTLRPC_MAX_BRW_SIZE != (PTLRPC_MAX_BRW_PAGES * PAGE_CACHE_SIZE))
115# error "PTLRPC_MAX_BRW_SIZE isn't PTLRPC_MAX_BRW_PAGES * PAGE_CACHE_SIZE"
116# endif
117# if (PTLRPC_MAX_BRW_SIZE > LNET_MTU * PTLRPC_BULK_OPS_COUNT)
118# error "PTLRPC_MAX_BRW_SIZE too big"
119# endif
120# if (PTLRPC_MAX_BRW_PAGES > LNET_MAX_IOV * PTLRPC_BULK_OPS_COUNT)
121# error "PTLRPC_MAX_BRW_PAGES too big"
122# endif
123
124#define PTLRPC_NTHRS_INIT 2
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253#define LDLM_THR_FACTOR 8
254#define LDLM_NTHRS_INIT PTLRPC_NTHRS_INIT
255#define LDLM_NTHRS_BASE 24
256#define LDLM_NTHRS_MAX (num_online_cpus() == 1 ? 64 : 128)
257
258#define LDLM_BL_THREADS LDLM_NTHRS_AUTO_INIT
259#define LDLM_CLIENT_NBUFS 1
260#define LDLM_SERVER_NBUFS 64
261#define LDLM_BUFSIZE (8 * 1024)
262#define LDLM_MAXREQSIZE (5 * 1024)
263#define LDLM_MAXREPSIZE (1024)
264
265#define MDS_MAXREQSIZE (5 * 1024)
266
267#define OST_MAXREQSIZE (5 * 1024)
268
269
270#define ptlrpc_req_async_args(req) ((void *)&req->rq_async_args)
271
272
273
274
275struct ptlrpc_connection {
276
277 struct hlist_node c_hash;
278
279 lnet_nid_t c_self;
280
281 lnet_process_id_t c_peer;
282
283 struct obd_uuid c_remote_uuid;
284
285 atomic_t c_refcount;
286};
287
288
289struct ptlrpc_client {
290
291 __u32 cli_request_portal;
292
293 __u32 cli_reply_portal;
294
295 char *cli_name;
296};
297
298
299
300#define PTL_RPC_FL_INTR (1 << 0)
301#define PTL_RPC_FL_TIMEOUT (1 << 7)
302
303#define REQ_MAX_ACK_LOCKS 8
304
305union ptlrpc_async_args {
306
307
308
309
310
311
312
313 void *pointer_arg[11];
314 __u64 space[7];
315};
316
317struct ptlrpc_request_set;
318typedef int (*set_interpreter_func)(struct ptlrpc_request_set *, void *, int);
319typedef int (*set_producer_func)(struct ptlrpc_request_set *, void *);
320
321
322
323
324
325
326
327
328
329
330
331
332struct ptlrpc_request_set {
333 atomic_t set_refcount;
334
335 atomic_t set_new_count;
336
337 atomic_t set_remaining;
338
339 wait_queue_head_t set_waitq;
340 wait_queue_head_t *set_wakeup_ptr;
341
342 struct list_head set_requests;
343
344
345
346
347
348 struct list_head set_cblist;
349
350 set_interpreter_func set_interpret;
351
352 void *set_arg;
353
354
355
356
357
358 spinlock_t set_new_req_lock;
359
360 struct list_head set_new_requests;
361
362
363 int set_rc;
364
365
366 int set_max_inflight;
367
368 set_producer_func set_producer;
369
370 void *set_producer_arg;
371};
372
373
374
375
376struct ptlrpc_set_cbdata {
377
378 struct list_head psc_item;
379
380 set_interpreter_func psc_interpret;
381
382 void *psc_data;
383};
384
385struct ptlrpc_bulk_desc;
386struct ptlrpc_service_part;
387struct ptlrpc_service;
388
389
390
391
392struct ptlrpc_cb_id {
393 void (*cbid_fn)(lnet_event_t *ev);
394 void *cbid_arg;
395};
396
397
398#define RS_MAX_LOCKS 8
399#define RS_DEBUG 0
400
401
402
403
404
405
406
407
408struct ptlrpc_reply_state {
409
410 struct ptlrpc_cb_id rs_cb_id;
411
412 struct list_head rs_list;
413
414 struct list_head rs_exp_list;
415
416 struct list_head rs_obd_list;
417#if RS_DEBUG
418 struct list_head rs_debug_list;
419#endif
420
421 spinlock_t rs_lock;
422
423 unsigned long rs_difficult:1;
424 unsigned long rs_no_ack:1;
425
426 unsigned long rs_scheduled:1;
427 unsigned long rs_scheduled_ever:1;
428 unsigned long rs_handled:1;
429 unsigned long rs_on_net:1;
430 unsigned long rs_prealloc:1;
431 unsigned long rs_committed:1;
432
433
434
435 int rs_size;
436
437 __u32 rs_opc;
438
439 __u64 rs_transno;
440
441 __u64 rs_xid;
442 struct obd_export *rs_export;
443 struct ptlrpc_service_part *rs_svcpt;
444
445 lnet_handle_md_t rs_md_h;
446 atomic_t rs_refcount;
447
448
449 struct ptlrpc_svc_ctx *rs_svc_ctx;
450
451 struct lustre_msg *rs_repbuf;
452
453 int rs_repbuf_len;
454
455 int rs_repdata_len;
456
457
458
459
460
461 struct lustre_msg *rs_msg;
462
463
464 int rs_nlocks;
465
466 struct lustre_handle rs_locks[RS_MAX_LOCKS];
467
468 ldlm_mode_t rs_modes[RS_MAX_LOCKS];
469};
470
471struct ptlrpc_thread;
472
473
474enum rq_phase {
475 RQ_PHASE_NEW = 0xebc0de00,
476 RQ_PHASE_RPC = 0xebc0de01,
477 RQ_PHASE_BULK = 0xebc0de02,
478 RQ_PHASE_INTERPRET = 0xebc0de03,
479 RQ_PHASE_COMPLETE = 0xebc0de04,
480 RQ_PHASE_UNREGISTERING = 0xebc0de05,
481 RQ_PHASE_UNDEFINED = 0xebc0de06
482};
483
484
485typedef int (*ptlrpc_interpterer_t)(const struct lu_env *env,
486 struct ptlrpc_request *req,
487 void *arg, int rc);
488
489
490
491
492
493
494
495struct ptlrpc_request_pool {
496
497 spinlock_t prp_lock;
498
499 struct list_head prp_req_list;
500
501 int prp_rq_size;
502
503 void (*prp_populate)(struct ptlrpc_request_pool *, int);
504};
505
506struct lu_context;
507struct lu_env;
508
509struct ldlm_lock;
510
511
512
513
514
515struct ptlrpc_nrs_policy;
516struct ptlrpc_nrs_resource;
517struct ptlrpc_nrs_request;
518
519
520
521
522
523
524enum ptlrpc_nrs_ctl {
525
526
527
528 PTLRPC_NRS_CTL_INVALID,
529
530
531
532 PTLRPC_NRS_CTL_START,
533
534
535
536
537 PTLRPC_NRS_CTL_STOP,
538
539
540
541
542 PTLRPC_NRS_CTL_1ST_POL_SPEC = 0x20,
543};
544
545
546
547
548enum nrs_ctl_orr {
549 NRS_CTL_ORR_RD_QUANTUM = PTLRPC_NRS_CTL_1ST_POL_SPEC,
550 NRS_CTL_ORR_WR_QUANTUM,
551 NRS_CTL_ORR_RD_OFF_TYPE,
552 NRS_CTL_ORR_WR_OFF_TYPE,
553 NRS_CTL_ORR_RD_SUPP_REQ,
554 NRS_CTL_ORR_WR_SUPP_REQ,
555};
556
557
558
559
560
561
562
563struct ptlrpc_nrs_pol_ops {
564
565
566
567
568
569 int (*op_policy_init) (struct ptlrpc_nrs_policy *policy);
570
571
572
573
574
575 void (*op_policy_fini) (struct ptlrpc_nrs_policy *policy);
576
577
578
579
580
581
582
583
584 int (*op_policy_start) (struct ptlrpc_nrs_policy *policy);
585
586
587
588
589
590
591
592
593 void (*op_policy_stop) (struct ptlrpc_nrs_policy *policy);
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609 int (*op_policy_ctl) (struct ptlrpc_nrs_policy *policy,
610 enum ptlrpc_nrs_ctl opc, void *arg);
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648 int (*op_res_get) (struct ptlrpc_nrs_policy *policy,
649 struct ptlrpc_nrs_request *nrq,
650 const struct ptlrpc_nrs_resource *parent,
651 struct ptlrpc_nrs_resource **resp,
652 bool moving_req);
653
654
655
656
657
658
659
660
661
662
663
664 void (*op_res_put) (struct ptlrpc_nrs_policy *policy,
665 const struct ptlrpc_nrs_resource *res);
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683 struct ptlrpc_nrs_request *
684 (*op_req_get) (struct ptlrpc_nrs_policy *policy, bool peek,
685 bool force);
686
687
688
689
690
691
692
693
694
695
696
697
698 int (*op_req_enqueue) (struct ptlrpc_nrs_policy *policy,
699 struct ptlrpc_nrs_request *nrq);
700
701
702
703
704
705
706
707
708
709
710 void (*op_req_dequeue) (struct ptlrpc_nrs_policy *policy,
711 struct ptlrpc_nrs_request *nrq);
712
713
714
715
716
717
718
719
720
721
722
723
724 void (*op_req_stop) (struct ptlrpc_nrs_policy *policy,
725 struct ptlrpc_nrs_request *nrq);
726
727
728
729
730
731
732
733
734 int (*op_lprocfs_init) (struct ptlrpc_service *svc);
735
736
737
738
739
740
741
742
743
744
745
746 void (*op_lprocfs_fini) (struct ptlrpc_service *svc);
747};
748
749
750
751
752enum nrs_policy_flags {
753
754
755
756
757
758 PTLRPC_NRS_FL_FALLBACK = (1 << 0),
759
760
761
762 PTLRPC_NRS_FL_REG_START = (1 << 1),
763
764
765
766
767 PTLRPC_NRS_FL_REG_EXTERN = (1 << 2),
768};
769
770
771
772
773
774
775
776
777enum ptlrpc_nrs_queue_type {
778 PTLRPC_NRS_QUEUE_REG = (1 << 0),
779 PTLRPC_NRS_QUEUE_HP = (1 << 1),
780 PTLRPC_NRS_QUEUE_BOTH = (PTLRPC_NRS_QUEUE_REG | PTLRPC_NRS_QUEUE_HP)
781};
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809struct ptlrpc_nrs {
810 spinlock_t nrs_lock;
811
812
813
814
815 struct list_head nrs_policy_list;
816
817
818
819
820
821
822
823
824 struct list_head nrs_policy_queued;
825
826
827
828 struct ptlrpc_service_part *nrs_svcpt;
829
830
831
832 struct ptlrpc_nrs_policy *nrs_policy_primary;
833
834
835
836 struct ptlrpc_nrs_policy *nrs_policy_fallback;
837
838
839
840 enum ptlrpc_nrs_queue_type nrs_queue_type;
841
842
843
844 unsigned long nrs_req_queued;
845
846
847
848 unsigned long nrs_req_started;
849
850
851
852 unsigned nrs_num_pols;
853
854
855
856 unsigned nrs_policy_starting:1;
857
858
859
860
861 unsigned nrs_stopping:1;
862};
863
864#define NRS_POL_NAME_MAX 16
865
866struct ptlrpc_nrs_pol_desc;
867
868
869
870
871
872
873
874
875
876
877typedef bool (*nrs_pol_desc_compat_t) (const struct ptlrpc_service *svc,
878 const struct ptlrpc_nrs_pol_desc *desc);
879
880struct ptlrpc_nrs_pol_conf {
881
882
883
884 char nc_name[NRS_POL_NAME_MAX];
885
886
887
888 const struct ptlrpc_nrs_pol_ops *nc_ops;
889
890
891
892 nrs_pol_desc_compat_t nc_compat;
893
894
895
896
897
898
899 const char *nc_compat_svc_name;
900
901
902
903
904
905 struct module *nc_owner;
906
907
908
909 unsigned nc_flags;
910};
911
912
913
914
915
916
917
918struct ptlrpc_nrs_pol_desc {
919
920
921
922 char pd_name[NRS_POL_NAME_MAX];
923
924
925
926 struct list_head pd_list;
927
928
929
930 const struct ptlrpc_nrs_pol_ops *pd_ops;
931
932
933
934 nrs_pol_desc_compat_t pd_compat;
935
936
937
938
939
940 const char *pd_compat_svc_name;
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980 struct module *pd_owner;
981
982
983
984 unsigned pd_flags;
985
986
987
988 atomic_t pd_refs;
989};
990
991
992
993
994
995
996enum ptlrpc_nrs_pol_state {
997
998
999
1000 NRS_POL_STATE_INVALID,
1001
1002
1003
1004
1005
1006 NRS_POL_STATE_STOPPED,
1007
1008
1009
1010 NRS_POL_STATE_STOPPING,
1011
1012
1013
1014 NRS_POL_STATE_STARTING,
1015
1016
1017
1018
1019
1020 NRS_POL_STATE_STARTED,
1021};
1022
1023
1024
1025
1026
1027
1028struct ptlrpc_nrs_pol_info {
1029
1030
1031
1032 char pi_name[NRS_POL_NAME_MAX];
1033
1034
1035
1036 enum ptlrpc_nrs_pol_state pi_state;
1037
1038
1039
1040 long pi_req_queued;
1041
1042
1043
1044 long pi_req_started;
1045
1046
1047
1048 unsigned pi_fallback:1;
1049};
1050
1051
1052
1053
1054
1055
1056
1057struct ptlrpc_nrs_policy {
1058
1059
1060
1061
1062 struct list_head pol_list;
1063
1064
1065
1066
1067 struct list_head pol_list_queued;
1068
1069
1070
1071 enum ptlrpc_nrs_pol_state pol_state;
1072
1073
1074
1075 unsigned pol_flags;
1076
1077
1078
1079 long pol_req_queued;
1080
1081
1082
1083 long pol_req_started;
1084
1085
1086
1087 long pol_ref;
1088
1089
1090
1091 struct ptlrpc_nrs *pol_nrs;
1092
1093
1094
1095 void *pol_private;
1096
1097
1098
1099 struct ptlrpc_nrs_pol_desc *pol_desc;
1100};
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129struct ptlrpc_nrs_resource {
1130
1131
1132
1133
1134 struct ptlrpc_nrs_resource *res_parent;
1135
1136
1137
1138 struct ptlrpc_nrs_policy *res_policy;
1139};
1140
1141enum {
1142 NRS_RES_FALLBACK,
1143 NRS_RES_PRIMARY,
1144 NRS_RES_MAX
1145};
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161struct nrs_fifo_head {
1162
1163
1164
1165 struct ptlrpc_nrs_resource fh_res;
1166
1167
1168
1169 struct list_head fh_list;
1170
1171
1172
1173 __u64 fh_sequence;
1174};
1175
1176struct nrs_fifo_req {
1177 struct list_head fr_list;
1178 __u64 fr_sequence;
1179};
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191struct ptlrpc_nrs_request {
1192
1193
1194
1195 struct ptlrpc_nrs_resource *nr_res_ptrs[NRS_RES_MAX];
1196
1197
1198
1199
1200
1201
1202 unsigned nr_res_idx;
1203 unsigned nr_initialized:1;
1204 unsigned nr_enqueued:1;
1205 unsigned nr_started:1;
1206 unsigned nr_finalized:1;
1207
1208
1209
1210
1211
1212 union {
1213
1214
1215
1216 struct nrs_fifo_req fifo;
1217 } nr_u;
1218
1219
1220
1221
1222 void *ext;
1223};
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236struct ptlrpc_hpreq_ops {
1237
1238
1239
1240
1241 int (*hpreq_lock_match)(struct ptlrpc_request *, struct ldlm_lock *);
1242
1243
1244
1245 int (*hpreq_check)(struct ptlrpc_request *);
1246
1247
1248
1249 void (*hpreq_fini)(struct ptlrpc_request *);
1250};
1251
1252
1253
1254
1255
1256
1257
1258struct ptlrpc_request {
1259
1260 int rq_type;
1261
1262 int rq_status;
1263
1264
1265
1266
1267 struct list_head rq_list;
1268
1269
1270
1271
1272
1273
1274
1275 struct list_head rq_timed_list;
1276
1277 struct list_head rq_history_list;
1278
1279 struct list_head rq_exp_list;
1280
1281 struct ptlrpc_hpreq_ops *rq_ops;
1282
1283
1284 struct ptlrpc_thread *rq_svc_thread;
1285
1286
1287 __u64 rq_history_seq;
1288
1289
1290
1291
1292 struct ptlrpc_nrs_request rq_nrq;
1293
1294
1295 time_t rq_at_index;
1296
1297
1298
1299 spinlock_t rq_lock;
1300
1301 unsigned int rq_intr:1, rq_replied:1, rq_err:1,
1302 rq_timedout:1, rq_resend:1, rq_restart:1,
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312 rq_replay:1,
1313 rq_no_resend:1, rq_waiting:1, rq_receiving_reply:1,
1314 rq_no_delay:1, rq_net_err:1, rq_wait_ctx:1,
1315 rq_early:1,
1316 rq_req_unlink:1, rq_reply_unlink:1,
1317 rq_memalloc:1,
1318
1319 rq_packed_final:1,
1320 rq_hp:1,
1321 rq_at_linked:1,
1322 rq_reply_truncate:1,
1323 rq_committed:1,
1324
1325 rq_invalid_rqset:1,
1326 rq_generation_set:1,
1327
1328 rq_no_retry_einprogress:1,
1329
1330
1331 rq_allow_replay:1;
1332
1333 unsigned int rq_nr_resend;
1334
1335 enum rq_phase rq_phase;
1336 enum rq_phase rq_next_phase;
1337 atomic_t rq_refcount;
1338
1339
1340
1341 short rq_request_portal;
1342
1343 short rq_reply_portal;
1344
1345
1346
1347
1348
1349
1350 int rq_nob_received;
1351
1352 int rq_reqlen;
1353
1354 int rq_replen;
1355
1356 struct lustre_msg *rq_reqmsg;
1357
1358 struct lustre_msg *rq_repmsg;
1359
1360 __u64 rq_transno;
1361
1362 __u64 rq_xid;
1363
1364
1365
1366
1367
1368 struct list_head rq_replay_list;
1369
1370
1371
1372
1373 struct ptlrpc_cli_ctx *rq_cli_ctx;
1374 struct ptlrpc_svc_ctx *rq_svc_ctx;
1375 struct list_head rq_ctx_chain;
1376
1377 struct sptlrpc_flavor rq_flvr;
1378 enum lustre_sec_part rq_sp_from;
1379
1380
1381 unsigned int
1382 rq_ctx_init:1,
1383 rq_ctx_fini:1,
1384 rq_bulk_read:1,
1385 rq_bulk_write:1,
1386
1387 rq_auth_gss:1,
1388 rq_auth_remote:1,
1389 rq_auth_usr_root:1,
1390 rq_auth_usr_mdt:1,
1391 rq_auth_usr_ost:1,
1392
1393 rq_pack_udesc:1,
1394 rq_pack_bulk:1,
1395
1396 rq_no_reply:1,
1397 rq_pill_init:1;
1398
1399 uid_t rq_auth_uid;
1400 uid_t rq_auth_mapped_uid;
1401
1402
1403 struct ptlrpc_user_desc *rq_user_desc;
1404
1405
1406 struct lustre_msg *rq_reqbuf;
1407 char *rq_repbuf;
1408 struct lustre_msg *rq_repdata;
1409 struct lustre_msg *rq_clrbuf;
1410 int rq_reqbuf_len;
1411 int rq_reqdata_len;
1412 int rq_repbuf_len;
1413 int rq_repdata_len;
1414 int rq_clrbuf_len;
1415 int rq_clrdata_len;
1416
1417
1418 unsigned int rq_reply_off;
1419
1420
1421
1422
1423 __u32 rq_req_swab_mask;
1424 __u32 rq_rep_swab_mask;
1425
1426
1427 int rq_import_generation;
1428 enum lustre_imp_state rq_send_state;
1429
1430
1431 int rq_early_count;
1432
1433
1434 lnet_handle_md_t rq_req_md_h;
1435 struct ptlrpc_cb_id rq_req_cbid;
1436
1437 long rq_delay_limit;
1438
1439 unsigned long rq_queued_time;
1440
1441
1442
1443 struct timeval rq_arrival_time;
1444
1445 struct ptlrpc_reply_state *rq_reply_state;
1446
1447 struct ptlrpc_request_buffer_desc *rq_rqbd;
1448
1449
1450 lnet_handle_md_t rq_reply_md_h;
1451 wait_queue_head_t rq_reply_waitq;
1452 struct ptlrpc_cb_id rq_reply_cbid;
1453
1454
1455 lnet_nid_t rq_self;
1456
1457 lnet_process_id_t rq_peer;
1458
1459 struct obd_export *rq_export;
1460
1461 struct obd_import *rq_import;
1462
1463
1464 void (*rq_replay_cb)(struct ptlrpc_request *);
1465
1466
1467
1468
1469 void (*rq_commit_cb)(struct ptlrpc_request *);
1470
1471 void *rq_cb_data;
1472
1473
1474 struct ptlrpc_bulk_desc *rq_bulk;
1475
1476
1477
1478
1479
1480 time_t rq_sent;
1481
1482 time_t rq_real_sent;
1483
1484
1485
1486
1487 volatile time_t rq_deadline;
1488
1489 time_t rq_reply_deadline;
1490
1491 time_t rq_bulk_deadline;
1492
1493
1494
1495
1496 int rq_timeout;
1497
1498
1499
1500 wait_queue_head_t rq_set_waitq;
1501
1502 struct list_head rq_set_chain;
1503
1504 struct ptlrpc_request_set *rq_set;
1505
1506 ptlrpc_interpterer_t rq_interpret_reply;
1507
1508 union ptlrpc_async_args rq_async_args;
1509
1510
1511 struct ptlrpc_request_pool *rq_pool;
1512
1513 struct lu_context rq_session;
1514 struct lu_context rq_recov_session;
1515
1516
1517 struct req_capsule rq_pill;
1518};
1519
1520
1521
1522
1523
1524static inline int ptlrpc_req_interpret(const struct lu_env *env,
1525 struct ptlrpc_request *req, int rc)
1526{
1527 if (req->rq_interpret_reply != NULL) {
1528 req->rq_status = req->rq_interpret_reply(env, req,
1529 &req->rq_async_args,
1530 rc);
1531 return req->rq_status;
1532 }
1533 return rc;
1534}
1535
1536
1537
1538
1539int ptlrpc_nrs_policy_register(struct ptlrpc_nrs_pol_conf *conf);
1540int ptlrpc_nrs_policy_unregister(struct ptlrpc_nrs_pol_conf *conf);
1541void ptlrpc_nrs_req_hp_move(struct ptlrpc_request *req);
1542void nrs_policy_get_info_locked(struct ptlrpc_nrs_policy *policy,
1543 struct ptlrpc_nrs_pol_info *info);
1544
1545
1546
1547
1548
1549
1550
1551static inline bool ptlrpc_nrs_req_can_move(struct ptlrpc_request *req)
1552{
1553 struct ptlrpc_nrs_request *nrq = &req->rq_nrq;
1554
1555
1556
1557
1558
1559
1560
1561 return nrq->nr_enqueued && !nrq->nr_started && !req->rq_hp;
1562}
1563
1564
1565
1566
1567
1568static inline int lustre_req_swabbed(struct ptlrpc_request *req, int index)
1569{
1570 LASSERT(index < sizeof(req->rq_req_swab_mask) * 8);
1571 return req->rq_req_swab_mask & (1 << index);
1572}
1573
1574
1575
1576
1577static inline int lustre_rep_swabbed(struct ptlrpc_request *req, int index)
1578{
1579 LASSERT(index < sizeof(req->rq_rep_swab_mask) * 8);
1580 return req->rq_rep_swab_mask & (1 << index);
1581}
1582
1583
1584
1585
1586static inline int ptlrpc_req_need_swab(struct ptlrpc_request *req)
1587{
1588 return lustre_req_swabbed(req, MSG_PTLRPC_HEADER_OFF);
1589}
1590
1591
1592
1593
1594static inline int ptlrpc_rep_need_swab(struct ptlrpc_request *req)
1595{
1596 return lustre_rep_swabbed(req, MSG_PTLRPC_HEADER_OFF);
1597}
1598
1599
1600
1601
1602static inline void lustre_set_req_swabbed(struct ptlrpc_request *req, int index)
1603{
1604 LASSERT(index < sizeof(req->rq_req_swab_mask) * 8);
1605 LASSERT((req->rq_req_swab_mask & (1 << index)) == 0);
1606 req->rq_req_swab_mask |= 1 << index;
1607}
1608
1609
1610
1611
1612static inline void lustre_set_rep_swabbed(struct ptlrpc_request *req, int index)
1613{
1614 LASSERT(index < sizeof(req->rq_rep_swab_mask) * 8);
1615 LASSERT((req->rq_rep_swab_mask & (1 << index)) == 0);
1616 req->rq_rep_swab_mask |= 1 << index;
1617}
1618
1619
1620
1621
1622static inline const char *
1623ptlrpc_phase2str(enum rq_phase phase)
1624{
1625 switch (phase) {
1626 case RQ_PHASE_NEW:
1627 return "New";
1628 case RQ_PHASE_RPC:
1629 return "Rpc";
1630 case RQ_PHASE_BULK:
1631 return "Bulk";
1632 case RQ_PHASE_INTERPRET:
1633 return "Interpret";
1634 case RQ_PHASE_COMPLETE:
1635 return "Complete";
1636 case RQ_PHASE_UNREGISTERING:
1637 return "Unregistering";
1638 default:
1639 return "?Phase?";
1640 }
1641}
1642
1643
1644
1645
1646
1647static inline const char *
1648ptlrpc_rqphase2str(struct ptlrpc_request *req)
1649{
1650 return ptlrpc_phase2str(req->rq_phase);
1651}
1652
1653
1654
1655
1656
1657
1658#define FLAG(field, str) (field ? str : "")
1659
1660
1661#define DEBUG_REQ_FLAGS(req) \
1662 ptlrpc_rqphase2str(req), \
1663 FLAG(req->rq_intr, "I"), FLAG(req->rq_replied, "R"), \
1664 FLAG(req->rq_err, "E"), \
1665 FLAG(req->rq_timedout, "X") , FLAG(req->rq_resend, "S"), \
1666 FLAG(req->rq_restart, "T"), FLAG(req->rq_replay, "P"), \
1667 FLAG(req->rq_no_resend, "N"), \
1668 FLAG(req->rq_waiting, "W"), \
1669 FLAG(req->rq_wait_ctx, "C"), FLAG(req->rq_hp, "H"), \
1670 FLAG(req->rq_committed, "M")
1671
1672#define REQ_FLAGS_FMT "%s:%s%s%s%s%s%s%s%s%s%s%s%s"
1673
1674void _debug_req(struct ptlrpc_request *req,
1675 struct libcfs_debug_msg_data *data, const char *fmt, ...)
1676 __attribute__ ((format (printf, 3, 4)));
1677
1678
1679
1680
1681
1682#define debug_req(msgdata, mask, cdls, req, fmt, a...) \
1683do { \
1684 CFS_CHECK_STACK(msgdata, mask, cdls); \
1685 \
1686 if (((mask) & D_CANTMASK) != 0 || \
1687 ((libcfs_debug & (mask)) != 0 && \
1688 (libcfs_subsystem_debug & DEBUG_SUBSYSTEM) != 0)) \
1689 _debug_req((req), msgdata, fmt, ##a); \
1690} while (0)
1691
1692
1693
1694
1695
1696#define DEBUG_REQ(level, req, fmt, args...) \
1697do { \
1698 if ((level) & (D_ERROR | D_WARNING)) { \
1699 static struct cfs_debug_limit_state cdls; \
1700 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, level, &cdls); \
1701 debug_req(&msgdata, level, &cdls, req, "@@@ "fmt" ", ## args);\
1702 } else { \
1703 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, level, NULL); \
1704 debug_req(&msgdata, level, NULL, req, "@@@ "fmt" ", ## args); \
1705 } \
1706} while (0)
1707
1708
1709
1710
1711
1712struct ptlrpc_bulk_page {
1713
1714 struct list_head bp_link;
1715
1716
1717
1718 int bp_buflen;
1719
1720 int bp_pageoffset;
1721
1722 struct page *bp_page;
1723};
1724
1725#define BULK_GET_SOURCE 0
1726#define BULK_PUT_SINK 1
1727#define BULK_GET_SINK 2
1728#define BULK_PUT_SOURCE 3
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740struct ptlrpc_bulk_desc {
1741
1742 unsigned long bd_failure:1;
1743
1744 unsigned long bd_type:2;
1745
1746 unsigned long bd_registered:1;
1747
1748 spinlock_t bd_lock;
1749
1750 int bd_import_generation;
1751
1752 __u32 bd_portal;
1753
1754 struct obd_export *bd_export;
1755
1756 struct obd_import *bd_import;
1757
1758 struct ptlrpc_request *bd_req;
1759 wait_queue_head_t bd_waitq;
1760 int bd_iov_count;
1761 int bd_max_iov;
1762 int bd_nob;
1763 int bd_nob_transferred;
1764
1765 __u64 bd_last_xid;
1766
1767 struct ptlrpc_cb_id bd_cbid;
1768 lnet_nid_t bd_sender;
1769 int bd_md_count;
1770 int bd_md_max_brw;
1771
1772 lnet_handle_md_t bd_mds[PTLRPC_BULK_OPS_COUNT];
1773
1774
1775
1776
1777 lnet_kiov_t *bd_enc_iov;
1778
1779 lnet_kiov_t bd_iov[0];
1780};
1781
1782enum {
1783 SVC_STOPPED = 1 << 0,
1784 SVC_STOPPING = 1 << 1,
1785 SVC_STARTING = 1 << 2,
1786 SVC_RUNNING = 1 << 3,
1787 SVC_EVENT = 1 << 4,
1788 SVC_SIGNAL = 1 << 5,
1789};
1790
1791#define PTLRPC_THR_NAME_LEN 32
1792
1793
1794
1795struct ptlrpc_thread {
1796
1797
1798
1799 struct list_head t_link;
1800
1801
1802
1803 void *t_data;
1804 __u32 t_flags;
1805
1806
1807
1808 unsigned int t_id;
1809
1810
1811
1812 pid_t t_pid;
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825 struct ptlrpc_service_part *t_svcpt;
1826 wait_queue_head_t t_ctl_waitq;
1827 struct lu_env *t_env;
1828 char t_name[PTLRPC_THR_NAME_LEN];
1829};
1830
1831static inline int thread_is_init(struct ptlrpc_thread *thread)
1832{
1833 return thread->t_flags == 0;
1834}
1835
1836static inline int thread_is_stopped(struct ptlrpc_thread *thread)
1837{
1838 return !!(thread->t_flags & SVC_STOPPED);
1839}
1840
1841static inline int thread_is_stopping(struct ptlrpc_thread *thread)
1842{
1843 return !!(thread->t_flags & SVC_STOPPING);
1844}
1845
1846static inline int thread_is_starting(struct ptlrpc_thread *thread)
1847{
1848 return !!(thread->t_flags & SVC_STARTING);
1849}
1850
1851static inline int thread_is_running(struct ptlrpc_thread *thread)
1852{
1853 return !!(thread->t_flags & SVC_RUNNING);
1854}
1855
1856static inline int thread_is_event(struct ptlrpc_thread *thread)
1857{
1858 return !!(thread->t_flags & SVC_EVENT);
1859}
1860
1861static inline int thread_is_signal(struct ptlrpc_thread *thread)
1862{
1863 return !!(thread->t_flags & SVC_SIGNAL);
1864}
1865
1866static inline void thread_clear_flags(struct ptlrpc_thread *thread, __u32 flags)
1867{
1868 thread->t_flags &= ~flags;
1869}
1870
1871static inline void thread_set_flags(struct ptlrpc_thread *thread, __u32 flags)
1872{
1873 thread->t_flags = flags;
1874}
1875
1876static inline void thread_add_flags(struct ptlrpc_thread *thread, __u32 flags)
1877{
1878 thread->t_flags |= flags;
1879}
1880
1881static inline int thread_test_and_clear_flags(struct ptlrpc_thread *thread,
1882 __u32 flags)
1883{
1884 if (thread->t_flags & flags) {
1885 thread->t_flags &= ~flags;
1886 return 1;
1887 }
1888 return 0;
1889}
1890
1891
1892
1893
1894
1895
1896
1897
1898struct ptlrpc_request_buffer_desc {
1899
1900 struct list_head rqbd_list;
1901
1902 struct list_head rqbd_reqs;
1903
1904 struct ptlrpc_service_part *rqbd_svcpt;
1905
1906 lnet_handle_md_t rqbd_md_h;
1907 int rqbd_refcount;
1908
1909 char *rqbd_buffer;
1910 struct ptlrpc_cb_id rqbd_cbid;
1911
1912
1913
1914
1915 struct ptlrpc_request rqbd_req;
1916};
1917
1918typedef int (*svc_handler_t)(struct ptlrpc_request *req);
1919
1920struct ptlrpc_service_ops {
1921
1922
1923
1924
1925 int (*so_thr_init)(struct ptlrpc_thread *thr);
1926
1927
1928
1929
1930 void (*so_thr_done)(struct ptlrpc_thread *thr);
1931
1932
1933
1934 int (*so_req_handler)(struct ptlrpc_request *req);
1935
1936
1937
1938
1939 int (*so_hpreq_handler)(struct ptlrpc_request *);
1940
1941
1942
1943 void (*so_req_printer)(void *, struct ptlrpc_request *);
1944};
1945
1946#ifndef __cfs_cacheline_aligned
1947
1948# define __cfs_cacheline_aligned
1949#endif
1950
1951
1952
1953
1954
1955#define PTLRPC_SVC_HP_RATIO 10
1956
1957
1958
1959
1960
1961
1962
1963struct ptlrpc_service {
1964
1965 spinlock_t srv_lock;
1966
1967
1968 struct list_head srv_list;
1969
1970 struct ptlrpc_service_ops srv_ops;
1971
1972 char *srv_name;
1973
1974 char *srv_thread_name;
1975
1976 struct list_head srv_threads;
1977
1978 int srv_nthrs_cpt_init;
1979
1980 int srv_nthrs_cpt_limit;
1981
1982 struct proc_dir_entry *srv_procroot;
1983
1984 struct lprocfs_stats *srv_stats;
1985
1986 int srv_hpreq_ratio;
1987
1988 int srv_max_req_size;
1989
1990 int srv_max_reply_size;
1991
1992 int srv_buf_size;
1993
1994 int srv_nbuf_per_group;
1995
1996 __u32 srv_req_portal;
1997
1998 __u32 srv_rep_portal;
1999
2000
2001
2002
2003 __u32 srv_ctx_tags;
2004
2005 int srv_watchdog_factor;
2006
2007 unsigned srv_is_stopping:1;
2008
2009
2010 int srv_hist_nrqbds_cpt_max;
2011
2012 int srv_ncpts;
2013
2014 __u32 *srv_cpts;
2015
2016 int srv_cpt_bits;
2017
2018 struct cfs_cpt_table *srv_cptable;
2019
2020
2021
2022 struct ptlrpc_service_part *srv_parts[0];
2023};
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043struct ptlrpc_service_part {
2044
2045 struct ptlrpc_service *scp_service __cfs_cacheline_aligned;
2046
2047 int scp_cpt;
2048
2049 int scp_thr_nextid;
2050
2051 int scp_nthrs_starting;
2052
2053 int scp_nthrs_stopping;
2054
2055 int scp_nthrs_running;
2056
2057 struct list_head scp_threads;
2058
2059
2060
2061
2062
2063
2064 spinlock_t scp_lock __cfs_cacheline_aligned;
2065
2066 int scp_nrqbds_total;
2067
2068 int scp_nrqbds_posted;
2069
2070 int scp_rqbd_allocating;
2071
2072 int scp_nreqs_incoming;
2073
2074 struct list_head scp_rqbd_idle;
2075
2076 struct list_head scp_rqbd_posted;
2077
2078 struct list_head scp_req_incoming;
2079
2080 long scp_rqbd_timeout;
2081
2082
2083
2084
2085 wait_queue_head_t scp_waitq;
2086
2087
2088 struct list_head scp_hist_reqs;
2089
2090 struct list_head scp_hist_rqbds;
2091
2092 int scp_hist_nrqbds;
2093
2094 __u64 scp_hist_seq;
2095
2096 __u64 scp_hist_seq_culled;
2097
2098
2099
2100
2101
2102 spinlock_t scp_req_lock __cfs_cacheline_aligned;
2103
2104
2105 int scp_nreqs_active;
2106
2107 int scp_nhreqs_active;
2108
2109 int scp_hreq_count;
2110
2111
2112 struct ptlrpc_nrs scp_nrs_reg;
2113
2114
2115 struct ptlrpc_nrs *scp_nrs_hp;
2116
2117
2118
2119
2120
2121
2122
2123 spinlock_t scp_at_lock __cfs_cacheline_aligned;
2124
2125 struct adaptive_timeout scp_at_estimate;
2126
2127 struct ptlrpc_at_array scp_at_array;
2128
2129 struct timer_list scp_at_timer;
2130
2131 unsigned long scp_at_checktime;
2132
2133 unsigned scp_at_check;
2134
2135
2136
2137
2138
2139
2140 spinlock_t scp_rep_lock __cfs_cacheline_aligned;
2141
2142 struct list_head scp_rep_active;
2143
2144 struct list_head scp_rep_idle;
2145
2146 wait_queue_head_t scp_rep_waitq;
2147
2148 atomic_t scp_nreps_difficult;
2149};
2150
2151#define ptlrpc_service_for_each_part(part, i, svc) \
2152 for (i = 0; \
2153 i < (svc)->srv_ncpts && \
2154 (svc)->srv_parts != NULL && \
2155 ((part) = (svc)->srv_parts[i]) != NULL; i++)
2156
2157
2158
2159
2160struct ptlrpcd_ctl {
2161
2162
2163
2164 unsigned long pc_flags;
2165
2166
2167
2168 spinlock_t pc_lock;
2169
2170
2171
2172 struct completion pc_starting;
2173
2174
2175
2176 struct completion pc_finishing;
2177
2178
2179
2180 struct ptlrpc_request_set *pc_set;
2181
2182
2183
2184 char pc_name[16];
2185
2186
2187
2188 struct lu_env pc_env;
2189
2190
2191
2192 int pc_index;
2193
2194
2195
2196 int pc_npartners;
2197
2198
2199
2200 struct ptlrpcd_ctl **pc_partners;
2201
2202
2203
2204 int pc_cursor;
2205};
2206
2207
2208enum ptlrpcd_ctl_flags {
2209
2210
2211
2212 LIOD_START = 1 << 0,
2213
2214
2215
2216 LIOD_STOP = 1 << 1,
2217
2218
2219
2220
2221
2222 LIOD_FORCE = 1 << 2,
2223
2224
2225
2226 LIOD_RECOVERY = 1 << 3,
2227
2228
2229
2230 LIOD_BIND = 1 << 4,
2231};
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246static inline bool nrs_policy_compat_all(const struct ptlrpc_service *svc,
2247 const struct ptlrpc_nrs_pol_desc *desc)
2248{
2249 return true;
2250}
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265static inline bool nrs_policy_compat_one(const struct ptlrpc_service *svc,
2266 const struct ptlrpc_nrs_pol_desc *desc)
2267{
2268 LASSERT(desc->pd_compat_svc_name != NULL);
2269 return strcmp(svc->srv_name, desc->pd_compat_svc_name) == 0;
2270}
2271
2272
2273
2274
2275extern lnet_handle_eq_t ptlrpc_eq_h;
2276extern int ptlrpc_uuid_to_peer(struct obd_uuid *uuid,
2277 lnet_process_id_t *peer, lnet_nid_t *self);
2278
2279
2280
2281
2282
2283extern void request_out_callback(lnet_event_t *ev);
2284extern void reply_in_callback(lnet_event_t *ev);
2285extern void client_bulk_callback(lnet_event_t *ev);
2286extern void request_in_callback(lnet_event_t *ev);
2287extern void reply_out_callback(lnet_event_t *ev);
2288
2289
2290
2291struct ptlrpc_connection *ptlrpc_connection_get(lnet_process_id_t peer,
2292 lnet_nid_t self,
2293 struct obd_uuid *uuid);
2294int ptlrpc_connection_put(struct ptlrpc_connection *c);
2295struct ptlrpc_connection *ptlrpc_connection_addref(struct ptlrpc_connection *);
2296int ptlrpc_connection_init(void);
2297void ptlrpc_connection_fini(void);
2298extern lnet_pid_t ptl_get_pid(void);
2299
2300
2301
2302
2303
2304
2305
2306int ptlrpc_register_bulk(struct ptlrpc_request *req);
2307int ptlrpc_unregister_bulk(struct ptlrpc_request *req, int async);
2308
2309static inline int ptlrpc_client_bulk_active(struct ptlrpc_request *req)
2310{
2311 struct ptlrpc_bulk_desc *desc;
2312 int rc;
2313
2314 LASSERT(req != NULL);
2315 desc = req->rq_bulk;
2316
2317 if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_BULK_UNLINK) &&
2318 req->rq_bulk_deadline > get_seconds())
2319 return 1;
2320
2321 if (!desc)
2322 return 0;
2323
2324 spin_lock(&desc->bd_lock);
2325 rc = desc->bd_md_count;
2326 spin_unlock(&desc->bd_lock);
2327 return rc;
2328}
2329
2330#define PTLRPC_REPLY_MAYBE_DIFFICULT 0x01
2331#define PTLRPC_REPLY_EARLY 0x02
2332int ptlrpc_send_reply(struct ptlrpc_request *req, int flags);
2333int ptlrpc_reply(struct ptlrpc_request *req);
2334int ptlrpc_send_error(struct ptlrpc_request *req, int difficult);
2335int ptlrpc_error(struct ptlrpc_request *req);
2336void ptlrpc_resend_req(struct ptlrpc_request *request);
2337int ptlrpc_at_get_net_latency(struct ptlrpc_request *req);
2338int ptl_send_rpc(struct ptlrpc_request *request, int noreply);
2339int ptlrpc_register_rqbd(struct ptlrpc_request_buffer_desc *rqbd);
2340
2341
2342
2343
2344
2345
2346
2347
2348void ptlrpc_request_committed(struct ptlrpc_request *req, int force);
2349
2350void ptlrpc_init_client(int req_portal, int rep_portal, char *name,
2351 struct ptlrpc_client *);
2352void ptlrpc_cleanup_client(struct obd_import *imp);
2353struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid);
2354
2355int ptlrpc_queue_wait(struct ptlrpc_request *req);
2356int ptlrpc_replay_req(struct ptlrpc_request *req);
2357int ptlrpc_unregister_reply(struct ptlrpc_request *req, int async);
2358void ptlrpc_restart_req(struct ptlrpc_request *req);
2359void ptlrpc_abort_inflight(struct obd_import *imp);
2360void ptlrpc_cleanup_imp(struct obd_import *imp);
2361void ptlrpc_abort_set(struct ptlrpc_request_set *set);
2362
2363struct ptlrpc_request_set *ptlrpc_prep_set(void);
2364struct ptlrpc_request_set *ptlrpc_prep_fcset(int max, set_producer_func func,
2365 void *arg);
2366int ptlrpc_set_add_cb(struct ptlrpc_request_set *set,
2367 set_interpreter_func fn, void *data);
2368int ptlrpc_set_next_timeout(struct ptlrpc_request_set *);
2369int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set);
2370int ptlrpc_set_wait(struct ptlrpc_request_set *);
2371int ptlrpc_expired_set(void *data);
2372void ptlrpc_interrupted_set(void *data);
2373void ptlrpc_mark_interrupted(struct ptlrpc_request *req);
2374void ptlrpc_set_destroy(struct ptlrpc_request_set *);
2375void ptlrpc_set_add_req(struct ptlrpc_request_set *, struct ptlrpc_request *);
2376void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc,
2377 struct ptlrpc_request *req);
2378
2379void ptlrpc_free_rq_pool(struct ptlrpc_request_pool *pool);
2380void ptlrpc_add_rqs_to_pool(struct ptlrpc_request_pool *pool, int num_rq);
2381
2382struct ptlrpc_request_pool *
2383ptlrpc_init_rq_pool(int, int,
2384 void (*populate_pool)(struct ptlrpc_request_pool *, int));
2385
2386void ptlrpc_at_set_req_timeout(struct ptlrpc_request *req);
2387struct ptlrpc_request *ptlrpc_request_alloc(struct obd_import *imp,
2388 const struct req_format *format);
2389struct ptlrpc_request *ptlrpc_request_alloc_pool(struct obd_import *imp,
2390 struct ptlrpc_request_pool *,
2391 const struct req_format *format);
2392void ptlrpc_request_free(struct ptlrpc_request *request);
2393int ptlrpc_request_pack(struct ptlrpc_request *request,
2394 __u32 version, int opcode);
2395struct ptlrpc_request *ptlrpc_request_alloc_pack(struct obd_import *imp,
2396 const struct req_format *format,
2397 __u32 version, int opcode);
2398int ptlrpc_request_bufs_pack(struct ptlrpc_request *request,
2399 __u32 version, int opcode, char **bufs,
2400 struct ptlrpc_cli_ctx *ctx);
2401struct ptlrpc_request *ptlrpc_prep_req(struct obd_import *imp, __u32 version,
2402 int opcode, int count, __u32 *lengths,
2403 char **bufs);
2404struct ptlrpc_request *ptlrpc_prep_req_pool(struct obd_import *imp,
2405 __u32 version, int opcode,
2406 int count, __u32 *lengths, char **bufs,
2407 struct ptlrpc_request_pool *pool);
2408void ptlrpc_req_finished(struct ptlrpc_request *request);
2409void ptlrpc_req_finished_with_imp_lock(struct ptlrpc_request *request);
2410struct ptlrpc_request *ptlrpc_request_addref(struct ptlrpc_request *req);
2411struct ptlrpc_bulk_desc *ptlrpc_prep_bulk_imp(struct ptlrpc_request *req,
2412 unsigned npages, unsigned max_brw,
2413 unsigned type, unsigned portal);
2414void __ptlrpc_free_bulk(struct ptlrpc_bulk_desc *bulk, int pin);
2415static inline void ptlrpc_free_bulk_pin(struct ptlrpc_bulk_desc *bulk)
2416{
2417 __ptlrpc_free_bulk(bulk, 1);
2418}
2419static inline void ptlrpc_free_bulk_nopin(struct ptlrpc_bulk_desc *bulk)
2420{
2421 __ptlrpc_free_bulk(bulk, 0);
2422}
2423void __ptlrpc_prep_bulk_page(struct ptlrpc_bulk_desc *desc,
2424 struct page *page, int pageoffset, int len, int);
2425static inline void ptlrpc_prep_bulk_page_pin(struct ptlrpc_bulk_desc *desc,
2426 struct page *page, int pageoffset,
2427 int len)
2428{
2429 __ptlrpc_prep_bulk_page(desc, page, pageoffset, len, 1);
2430}
2431
2432static inline void ptlrpc_prep_bulk_page_nopin(struct ptlrpc_bulk_desc *desc,
2433 struct page *page, int pageoffset,
2434 int len)
2435{
2436 __ptlrpc_prep_bulk_page(desc, page, pageoffset, len, 0);
2437}
2438
2439void ptlrpc_retain_replayable_request(struct ptlrpc_request *req,
2440 struct obd_import *imp);
2441__u64 ptlrpc_next_xid(void);
2442__u64 ptlrpc_sample_next_xid(void);
2443__u64 ptlrpc_req_xid(struct ptlrpc_request *request);
2444
2445
2446void *ptlrpcd_alloc_work(struct obd_import *imp,
2447 int (*cb)(const struct lu_env *, void *), void *data);
2448void ptlrpcd_destroy_work(void *handler);
2449int ptlrpcd_queue_work(void *handler);
2450
2451
2452struct ptlrpc_service_buf_conf {
2453
2454 unsigned int bc_nbufs;
2455
2456 unsigned int bc_buf_size;
2457
2458 unsigned int bc_req_portal;
2459
2460 unsigned int bc_rep_portal;
2461
2462 unsigned int bc_req_max_size;
2463
2464 unsigned int bc_rep_max_size;
2465};
2466
2467struct ptlrpc_service_thr_conf {
2468
2469 char *tc_thr_name;
2470
2471 unsigned int tc_thr_factor;
2472
2473 unsigned int tc_nthrs_init;
2474
2475
2476
2477
2478
2479
2480 unsigned int tc_nthrs_base;
2481
2482 unsigned int tc_nthrs_max;
2483
2484
2485 unsigned int tc_nthrs_user;
2486
2487 unsigned int tc_cpu_affinity;
2488
2489 __u32 tc_ctx_tags;
2490};
2491
2492struct ptlrpc_service_cpt_conf {
2493 struct cfs_cpt_table *cc_cptable;
2494
2495 char *cc_pattern;
2496};
2497
2498struct ptlrpc_service_conf {
2499
2500 char *psc_name;
2501
2502 unsigned int psc_watchdog_factor;
2503
2504 struct ptlrpc_service_buf_conf psc_buf;
2505
2506 struct ptlrpc_service_thr_conf psc_thr;
2507
2508 struct ptlrpc_service_cpt_conf psc_cpt;
2509
2510 struct ptlrpc_service_ops psc_ops;
2511};
2512
2513
2514
2515
2516
2517
2518
2519
2520void ptlrpc_save_lock(struct ptlrpc_request *req,
2521 struct lustre_handle *lock, int mode, int no_ack);
2522void ptlrpc_commit_replies(struct obd_export *exp);
2523void ptlrpc_dispatch_difficult_reply(struct ptlrpc_reply_state *rs);
2524void ptlrpc_schedule_difficult_reply(struct ptlrpc_reply_state *rs);
2525int ptlrpc_hpreq_handler(struct ptlrpc_request *req);
2526struct ptlrpc_service *ptlrpc_register_service(
2527 struct ptlrpc_service_conf *conf,
2528 struct proc_dir_entry *proc_entry);
2529void ptlrpc_stop_all_threads(struct ptlrpc_service *svc);
2530
2531int ptlrpc_start_threads(struct ptlrpc_service *svc);
2532int ptlrpc_unregister_service(struct ptlrpc_service *service);
2533int liblustre_check_services(void *arg);
2534void ptlrpc_daemonize(char *name);
2535int ptlrpc_service_health_check(struct ptlrpc_service *);
2536void ptlrpc_server_drop_request(struct ptlrpc_request *req);
2537void ptlrpc_request_change_export(struct ptlrpc_request *req,
2538 struct obd_export *export);
2539
2540int ptlrpc_hr_init(void);
2541void ptlrpc_hr_fini(void);
2542
2543
2544
2545
2546
2547
2548
2549
2550int ptlrpc_connect_import(struct obd_import *imp);
2551int ptlrpc_init_import(struct obd_import *imp);
2552int ptlrpc_disconnect_import(struct obd_import *imp, int noclose);
2553int ptlrpc_import_recovery_state_machine(struct obd_import *imp);
2554void deuuidify(char *uuid, const char *prefix, char **uuid_start,
2555 int *uuid_len);
2556
2557
2558int ptlrpc_reconnect_import(struct obd_import *imp);
2559
2560
2561
2562
2563
2564
2565
2566int ptlrpc_buf_need_swab(struct ptlrpc_request *req, const int inout,
2567 int index);
2568void ptlrpc_buf_set_swabbed(struct ptlrpc_request *req, const int inout,
2569 int index);
2570int ptlrpc_unpack_rep_msg(struct ptlrpc_request *req, int len);
2571int ptlrpc_unpack_req_msg(struct ptlrpc_request *req, int len);
2572
2573int lustre_msg_check_version(struct lustre_msg *msg, __u32 version);
2574void lustre_init_msg_v2(struct lustre_msg_v2 *msg, int count, __u32 *lens,
2575 char **bufs);
2576int lustre_pack_request(struct ptlrpc_request *, __u32 magic, int count,
2577 __u32 *lens, char **bufs);
2578int lustre_pack_reply(struct ptlrpc_request *, int count, __u32 *lens,
2579 char **bufs);
2580int lustre_pack_reply_v2(struct ptlrpc_request *req, int count,
2581 __u32 *lens, char **bufs, int flags);
2582#define LPRFL_EARLY_REPLY 1
2583int lustre_pack_reply_flags(struct ptlrpc_request *, int count, __u32 *lens,
2584 char **bufs, int flags);
2585int lustre_shrink_msg(struct lustre_msg *msg, int segment,
2586 unsigned int newlen, int move_data);
2587void lustre_free_reply_state(struct ptlrpc_reply_state *rs);
2588int __lustre_unpack_msg(struct lustre_msg *m, int len);
2589int lustre_msg_hdr_size(__u32 magic, int count);
2590int lustre_msg_size(__u32 magic, int count, __u32 *lengths);
2591int lustre_msg_size_v2(int count, __u32 *lengths);
2592int lustre_packed_msg_size(struct lustre_msg *msg);
2593int lustre_msg_early_size(void);
2594void *lustre_msg_buf_v2(struct lustre_msg_v2 *m, int n, int min_size);
2595void *lustre_msg_buf(struct lustre_msg *m, int n, int minlen);
2596int lustre_msg_buflen(struct lustre_msg *m, int n);
2597void lustre_msg_set_buflen(struct lustre_msg *m, int n, int len);
2598int lustre_msg_bufcount(struct lustre_msg *m);
2599char *lustre_msg_string(struct lustre_msg *m, int n, int max_len);
2600__u32 lustre_msghdr_get_flags(struct lustre_msg *msg);
2601void lustre_msghdr_set_flags(struct lustre_msg *msg, __u32 flags);
2602__u32 lustre_msg_get_flags(struct lustre_msg *msg);
2603void lustre_msg_add_flags(struct lustre_msg *msg, int flags);
2604void lustre_msg_set_flags(struct lustre_msg *msg, int flags);
2605void lustre_msg_clear_flags(struct lustre_msg *msg, int flags);
2606__u32 lustre_msg_get_op_flags(struct lustre_msg *msg);
2607void lustre_msg_add_op_flags(struct lustre_msg *msg, int flags);
2608void lustre_msg_set_op_flags(struct lustre_msg *msg, int flags);
2609struct lustre_handle *lustre_msg_get_handle(struct lustre_msg *msg);
2610__u32 lustre_msg_get_type(struct lustre_msg *msg);
2611__u32 lustre_msg_get_version(struct lustre_msg *msg);
2612void lustre_msg_add_version(struct lustre_msg *msg, int version);
2613__u32 lustre_msg_get_opc(struct lustre_msg *msg);
2614__u64 lustre_msg_get_last_xid(struct lustre_msg *msg);
2615__u64 lustre_msg_get_last_committed(struct lustre_msg *msg);
2616__u64 *lustre_msg_get_versions(struct lustre_msg *msg);
2617__u64 lustre_msg_get_transno(struct lustre_msg *msg);
2618__u64 lustre_msg_get_slv(struct lustre_msg *msg);
2619__u32 lustre_msg_get_limit(struct lustre_msg *msg);
2620void lustre_msg_set_slv(struct lustre_msg *msg, __u64 slv);
2621void lustre_msg_set_limit(struct lustre_msg *msg, __u64 limit);
2622int lustre_msg_get_status(struct lustre_msg *msg);
2623__u32 lustre_msg_get_conn_cnt(struct lustre_msg *msg);
2624int lustre_msg_is_v1(struct lustre_msg *msg);
2625__u32 lustre_msg_get_magic(struct lustre_msg *msg);
2626__u32 lustre_msg_get_timeout(struct lustre_msg *msg);
2627__u32 lustre_msg_get_service_time(struct lustre_msg *msg);
2628char *lustre_msg_get_jobid(struct lustre_msg *msg);
2629__u32 lustre_msg_get_cksum(struct lustre_msg *msg);
2630__u32 lustre_msg_calc_cksum(struct lustre_msg *msg);
2631void lustre_msg_set_handle(struct lustre_msg *msg,
2632 struct lustre_handle *handle);
2633void lustre_msg_set_type(struct lustre_msg *msg, __u32 type);
2634void lustre_msg_set_opc(struct lustre_msg *msg, __u32 opc);
2635void lustre_msg_set_last_xid(struct lustre_msg *msg, __u64 last_xid);
2636void lustre_msg_set_last_committed(struct lustre_msg *msg,
2637 __u64 last_committed);
2638void lustre_msg_set_versions(struct lustre_msg *msg, __u64 *versions);
2639void lustre_msg_set_transno(struct lustre_msg *msg, __u64 transno);
2640void lustre_msg_set_status(struct lustre_msg *msg, __u32 status);
2641void lustre_msg_set_conn_cnt(struct lustre_msg *msg, __u32 conn_cnt);
2642void ptlrpc_req_set_repsize(struct ptlrpc_request *req, int count, __u32 *sizes);
2643void ptlrpc_request_set_replen(struct ptlrpc_request *req);
2644void lustre_msg_set_timeout(struct lustre_msg *msg, __u32 timeout);
2645void lustre_msg_set_service_time(struct lustre_msg *msg, __u32 service_time);
2646void lustre_msg_set_jobid(struct lustre_msg *msg, char *jobid);
2647void lustre_msg_set_cksum(struct lustre_msg *msg, __u32 cksum);
2648
2649static inline void
2650lustre_shrink_reply(struct ptlrpc_request *req, int segment,
2651 unsigned int newlen, int move_data)
2652{
2653 LASSERT(req->rq_reply_state);
2654 LASSERT(req->rq_repmsg);
2655 req->rq_replen = lustre_shrink_msg(req->rq_repmsg, segment,
2656 newlen, move_data);
2657}
2658
2659#ifdef CONFIG_LUSTRE_TRANSLATE_ERRNOS
2660
2661static inline int ptlrpc_status_hton(int h)
2662{
2663
2664
2665
2666
2667 if (h < 0)
2668 return -lustre_errno_hton(-h);
2669 else
2670 return h;
2671}
2672
2673static inline int ptlrpc_status_ntoh(int n)
2674{
2675
2676
2677
2678 if (n < 0)
2679 return -lustre_errno_ntoh(-n);
2680 else
2681 return n;
2682}
2683
2684#else
2685
2686#define ptlrpc_status_hton(h) (h)
2687#define ptlrpc_status_ntoh(n) (n)
2688
2689#endif
2690
2691
2692
2693static inline void
2694ptlrpc_rqphase_move(struct ptlrpc_request *req, enum rq_phase new_phase)
2695{
2696 if (req->rq_phase == new_phase)
2697 return;
2698
2699 if (new_phase == RQ_PHASE_UNREGISTERING) {
2700 req->rq_next_phase = req->rq_phase;
2701 if (req->rq_import)
2702 atomic_inc(&req->rq_import->imp_unregistering);
2703 }
2704
2705 if (req->rq_phase == RQ_PHASE_UNREGISTERING) {
2706 if (req->rq_import)
2707 atomic_dec(&req->rq_import->imp_unregistering);
2708 }
2709
2710 DEBUG_REQ(D_INFO, req, "move req \"%s\" -> \"%s\"",
2711 ptlrpc_rqphase2str(req), ptlrpc_phase2str(new_phase));
2712
2713 req->rq_phase = new_phase;
2714}
2715
2716
2717
2718
2719static inline int
2720ptlrpc_client_early(struct ptlrpc_request *req)
2721{
2722 if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK) &&
2723 req->rq_reply_deadline > get_seconds())
2724 return 0;
2725 return req->rq_early;
2726}
2727
2728
2729
2730
2731static inline int
2732ptlrpc_client_replied(struct ptlrpc_request *req)
2733{
2734 if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK) &&
2735 req->rq_reply_deadline > get_seconds())
2736 return 0;
2737 return req->rq_replied;
2738}
2739
2740
2741static inline int
2742ptlrpc_client_recv(struct ptlrpc_request *req)
2743{
2744 if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK) &&
2745 req->rq_reply_deadline > get_seconds())
2746 return 1;
2747 return req->rq_receiving_reply;
2748}
2749
2750static inline int
2751ptlrpc_client_recv_or_unlink(struct ptlrpc_request *req)
2752{
2753 int rc;
2754
2755 spin_lock(&req->rq_lock);
2756 if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK) &&
2757 req->rq_reply_deadline > get_seconds()) {
2758 spin_unlock(&req->rq_lock);
2759 return 1;
2760 }
2761 rc = req->rq_receiving_reply;
2762 rc = rc || req->rq_req_unlink || req->rq_reply_unlink;
2763 spin_unlock(&req->rq_lock);
2764 return rc;
2765}
2766
2767static inline void
2768ptlrpc_client_wake_req(struct ptlrpc_request *req)
2769{
2770 if (req->rq_set == NULL)
2771 wake_up(&req->rq_reply_waitq);
2772 else
2773 wake_up(&req->rq_set->set_waitq);
2774}
2775
2776static inline void
2777ptlrpc_rs_addref(struct ptlrpc_reply_state *rs)
2778{
2779 LASSERT(atomic_read(&rs->rs_refcount) > 0);
2780 atomic_inc(&rs->rs_refcount);
2781}
2782
2783static inline void
2784ptlrpc_rs_decref(struct ptlrpc_reply_state *rs)
2785{
2786 LASSERT(atomic_read(&rs->rs_refcount) > 0);
2787 if (atomic_dec_and_test(&rs->rs_refcount))
2788 lustre_free_reply_state(rs);
2789}
2790
2791
2792static inline void ptlrpc_req_drop_rs(struct ptlrpc_request *req)
2793{
2794 if (req->rq_reply_state == NULL)
2795 return;
2796 ptlrpc_rs_decref(req->rq_reply_state);
2797 req->rq_reply_state = NULL;
2798 req->rq_repmsg = NULL;
2799}
2800
2801static inline __u32 lustre_request_magic(struct ptlrpc_request *req)
2802{
2803 return lustre_msg_get_magic(req->rq_reqmsg);
2804}
2805
2806static inline int ptlrpc_req_get_repsize(struct ptlrpc_request *req)
2807{
2808 switch (req->rq_reqmsg->lm_magic) {
2809 case LUSTRE_MSG_MAGIC_V2:
2810 return req->rq_reqmsg->lm_repsize;
2811 default:
2812 LASSERTF(0, "incorrect message magic: %08x\n",
2813 req->rq_reqmsg->lm_magic);
2814 return -EFAULT;
2815 }
2816}
2817
2818static inline int ptlrpc_send_limit_expired(struct ptlrpc_request *req)
2819{
2820 if (req->rq_delay_limit != 0 &&
2821 time_before(cfs_time_add(req->rq_queued_time,
2822 cfs_time_seconds(req->rq_delay_limit)),
2823 cfs_time_current())) {
2824 return 1;
2825 }
2826 return 0;
2827}
2828
2829static inline int ptlrpc_no_resend(struct ptlrpc_request *req)
2830{
2831 if (!req->rq_no_resend && ptlrpc_send_limit_expired(req)) {
2832 spin_lock(&req->rq_lock);
2833 req->rq_no_resend = 1;
2834 spin_unlock(&req->rq_lock);
2835 }
2836 return req->rq_no_resend;
2837}
2838
2839static inline int
2840ptlrpc_server_get_timeout(struct ptlrpc_service_part *svcpt)
2841{
2842 int at = AT_OFF ? 0 : at_get(&svcpt->scp_at_estimate);
2843
2844 return svcpt->scp_service->srv_watchdog_factor *
2845 max_t(int, at, obd_timeout);
2846}
2847
2848static inline struct ptlrpc_service *
2849ptlrpc_req2svc(struct ptlrpc_request *req)
2850{
2851 LASSERT(req->rq_rqbd != NULL);
2852 return req->rq_rqbd->rqbd_svcpt->scp_service;
2853}
2854
2855
2856
2857
2858
2859
2860int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg);
2861int client_obd_cleanup(struct obd_device *obddev);
2862int client_connect_import(const struct lu_env *env,
2863 struct obd_export **exp, struct obd_device *obd,
2864 struct obd_uuid *cluuid, struct obd_connect_data *,
2865 void *localdata);
2866int client_disconnect_export(struct obd_export *exp);
2867int client_import_add_conn(struct obd_import *imp, struct obd_uuid *uuid,
2868 int priority);
2869int client_import_del_conn(struct obd_import *imp, struct obd_uuid *uuid);
2870int client_import_find_conn(struct obd_import *imp, lnet_nid_t peer,
2871 struct obd_uuid *uuid);
2872int import_set_conn_priority(struct obd_import *imp, struct obd_uuid *uuid);
2873void client_destroy_import(struct obd_import *imp);
2874
2875
2876
2877
2878
2879
2880
2881
2882enum timeout_event {
2883 TIMEOUT_GRANT = 1
2884};
2885struct timeout_item;
2886typedef int (*timeout_cb_t)(struct timeout_item *, void *);
2887int ptlrpc_pinger_add_import(struct obd_import *imp);
2888int ptlrpc_pinger_del_import(struct obd_import *imp);
2889int ptlrpc_add_timeout_client(int time, enum timeout_event event,
2890 timeout_cb_t cb, void *data,
2891 struct list_head *obd_list);
2892int ptlrpc_del_timeout_client(struct list_head *obd_list,
2893 enum timeout_event event);
2894struct ptlrpc_request *ptlrpc_prep_ping(struct obd_import *imp);
2895int ptlrpc_obd_ping(struct obd_device *obd);
2896void ping_evictor_start(void);
2897void ping_evictor_stop(void);
2898void ptlrpc_pinger_ir_up(void);
2899void ptlrpc_pinger_ir_down(void);
2900
2901int ptlrpc_pinger_suppress_pings(void);
2902
2903
2904typedef enum {
2905
2906 PDB_POLICY_NONE = 1,
2907
2908 PDB_POLICY_FULL = 2,
2909
2910 PDB_POLICY_PAIR = 3,
2911
2912
2913
2914
2915 PDB_POLICY_NEIGHBOR = 4,
2916} pdb_policy_t;
2917
2918
2919
2920
2921
2922
2923
2924typedef enum {
2925
2926 PDL_POLICY_SAME = 1,
2927
2928 PDL_POLICY_LOCAL = 2,
2929
2930 PDL_POLICY_ROUND = 3,
2931
2932 PDL_POLICY_PREFERRED = 4,
2933} pdl_policy_t;
2934
2935
2936void ptlrpcd_stop(struct ptlrpcd_ctl *pc, int force);
2937void ptlrpcd_free(struct ptlrpcd_ctl *pc);
2938void ptlrpcd_wake(struct ptlrpc_request *req);
2939void ptlrpcd_add_req(struct ptlrpc_request *req, pdl_policy_t policy, int idx);
2940void ptlrpcd_add_rqset(struct ptlrpc_request_set *set);
2941int ptlrpcd_addref(void);
2942void ptlrpcd_decref(void);
2943
2944
2945
2946
2947
2948
2949const char *ll_opcode2str(__u32 opcode);
2950#if defined (CONFIG_PROC_FS)
2951void ptlrpc_lprocfs_register_obd(struct obd_device *obd);
2952void ptlrpc_lprocfs_unregister_obd(struct obd_device *obd);
2953void ptlrpc_lprocfs_brw(struct ptlrpc_request *req, int bytes);
2954#else
2955static inline void ptlrpc_lprocfs_register_obd(struct obd_device *obd) {}
2956static inline void ptlrpc_lprocfs_unregister_obd(struct obd_device *obd) {}
2957static inline void ptlrpc_lprocfs_brw(struct ptlrpc_request *req, int bytes) {}
2958#endif
2959
2960
2961
2962extern struct llog_operations llog_client_ops;
2963
2964
2965
2966#endif
2967
2968