1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46#ifndef _LUSTRE_NET_H
47#define _LUSTRE_NET_H
48
49
50
51
52
53
54#include <linux/uio.h>
55#include <linux/libcfs/libcfs.h>
56#include <uapi/linux/lnet/nidstr.h>
57#include <linux/lnet/api.h>
58#include <uapi/linux/lustre/lustre_idl.h>
59#include <lustre_errno.h>
60#include <lustre_ha.h>
61#include <lustre_sec.h>
62#include <lustre_import.h>
63#include <lprocfs_status.h>
64#include <lu_object.h>
65#include <lustre_req_layout.h>
66
67#include <obd_support.h>
68#include <uapi/linux/lustre/lustre_ver.h>
69
70
71#define PTLRPC_MD_OPTIONS 0
72
73
74
75
76
77
78
79
80
81#define PTLRPC_BULK_OPS_BITS 4
82#if PTLRPC_BULK_OPS_BITS > 16
83#error "More than 65536 BRW RPCs not allowed by IOOBJ_MAX_BRW_BITS."
84#endif
85#define PTLRPC_BULK_OPS_COUNT (1U << PTLRPC_BULK_OPS_BITS)
86
87
88
89
90
91
92
93
94#define PTLRPC_BULK_OPS_MASK (~((__u64)PTLRPC_BULK_OPS_COUNT - 1))
95
96
97
98
99
100
101
102
103#define PTLRPC_MAX_BRW_BITS (LNET_MTU_BITS + PTLRPC_BULK_OPS_BITS)
104#define PTLRPC_MAX_BRW_SIZE (1 << PTLRPC_MAX_BRW_BITS)
105#define PTLRPC_MAX_BRW_PAGES (PTLRPC_MAX_BRW_SIZE >> PAGE_SHIFT)
106
107#define ONE_MB_BRW_SIZE (1 << LNET_MTU_BITS)
108#define MD_MAX_BRW_SIZE (1 << LNET_MTU_BITS)
109#define MD_MAX_BRW_PAGES (MD_MAX_BRW_SIZE >> PAGE_SHIFT)
110#define DT_MAX_BRW_SIZE PTLRPC_MAX_BRW_SIZE
111#define DT_MAX_BRW_PAGES (DT_MAX_BRW_SIZE >> PAGE_SHIFT)
112#define OFD_MAX_BRW_SIZE (1 << LNET_MTU_BITS)
113
114
115# if ((PTLRPC_MAX_BRW_PAGES & (PTLRPC_MAX_BRW_PAGES - 1)) != 0)
116# error "PTLRPC_MAX_BRW_PAGES isn't a power of two"
117# endif
118# if (PTLRPC_MAX_BRW_SIZE != (PTLRPC_MAX_BRW_PAGES * PAGE_SIZE))
119# error "PTLRPC_MAX_BRW_SIZE isn't PTLRPC_MAX_BRW_PAGES * PAGE_SIZE"
120# endif
121# if (PTLRPC_MAX_BRW_SIZE > LNET_MTU * PTLRPC_BULK_OPS_COUNT)
122# error "PTLRPC_MAX_BRW_SIZE too big"
123# endif
124# if (PTLRPC_MAX_BRW_PAGES > LNET_MAX_IOV * PTLRPC_BULK_OPS_COUNT)
125# error "PTLRPC_MAX_BRW_PAGES too big"
126# endif
127
128#define PTLRPC_NTHRS_INIT 2
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257#define LDLM_THR_FACTOR 8
258#define LDLM_NTHRS_INIT PTLRPC_NTHRS_INIT
259#define LDLM_NTHRS_BASE 24
260#define LDLM_NTHRS_MAX (num_online_cpus() == 1 ? 64 : 128)
261
262#define LDLM_BL_THREADS LDLM_NTHRS_AUTO_INIT
263#define LDLM_CLIENT_NBUFS 1
264#define LDLM_SERVER_NBUFS 64
265#define LDLM_BUFSIZE (8 * 1024)
266#define LDLM_MAXREQSIZE (5 * 1024)
267#define LDLM_MAXREPSIZE (1024)
268
269#define MDS_MAXREQSIZE (5 * 1024)
270
271
272
273
274#define OST_MAXREQSIZE (16 * 1024)
275
276
277#define ptlrpc_req_async_args(req) ((void *)&req->rq_async_args)
278
279struct ptlrpc_replay_async_args {
280 int praa_old_state;
281 int praa_old_status;
282};
283
284
285
286
287struct ptlrpc_connection {
288
289 struct hlist_node c_hash;
290
291 lnet_nid_t c_self;
292
293 struct lnet_process_id c_peer;
294
295 struct obd_uuid c_remote_uuid;
296
297 atomic_t c_refcount;
298};
299
300
301struct ptlrpc_client {
302
303 __u32 cli_request_portal;
304
305 __u32 cli_reply_portal;
306
307 char *cli_name;
308};
309
310
311
312#define PTL_RPC_FL_INTR (1 << 0)
313#define PTL_RPC_FL_TIMEOUT (1 << 7)
314
315#define REQ_MAX_ACK_LOCKS 8
316
317union ptlrpc_async_args {
318
319
320
321
322
323
324
325 void *pointer_arg[11];
326 __u64 space[7];
327};
328
329struct ptlrpc_request_set;
330typedef int (*set_interpreter_func)(struct ptlrpc_request_set *, void *, int);
331typedef int (*set_producer_func)(struct ptlrpc_request_set *, void *);
332
333
334
335
336
337
338
339
340
341
342
343
344struct ptlrpc_request_set {
345 atomic_t set_refcount;
346
347 atomic_t set_new_count;
348
349 atomic_t set_remaining;
350
351 wait_queue_head_t set_waitq;
352 wait_queue_head_t *set_wakeup_ptr;
353
354 struct list_head set_requests;
355
356
357
358
359
360 struct list_head set_cblist;
361
362 set_interpreter_func set_interpret;
363
364 void *set_arg;
365
366
367
368
369
370 spinlock_t set_new_req_lock;
371
372 struct list_head set_new_requests;
373
374
375 int set_rc;
376
377
378 int set_max_inflight;
379
380 set_producer_func set_producer;
381
382 void *set_producer_arg;
383};
384
385
386
387
388struct ptlrpc_set_cbdata {
389
390 struct list_head psc_item;
391
392 set_interpreter_func psc_interpret;
393
394 void *psc_data;
395};
396
397struct ptlrpc_bulk_desc;
398struct ptlrpc_service_part;
399struct ptlrpc_service;
400
401
402
403
404struct ptlrpc_cb_id {
405 void (*cbid_fn)(struct lnet_event *ev);
406 void *cbid_arg;
407};
408
409
410#define RS_MAX_LOCKS 8
411#define RS_DEBUG 0
412
413
414
415
416
417
418
419
420struct ptlrpc_reply_state {
421
422 struct ptlrpc_cb_id rs_cb_id;
423
424 struct list_head rs_list;
425
426 struct list_head rs_exp_list;
427
428 struct list_head rs_obd_list;
429#if RS_DEBUG
430 struct list_head rs_debug_list;
431#endif
432
433 spinlock_t rs_lock;
434
435 unsigned long rs_difficult:1;
436 unsigned long rs_no_ack:1;
437
438
439 unsigned long rs_scheduled:1;
440 unsigned long rs_scheduled_ever:1;
441 unsigned long rs_handled:1;
442 unsigned long rs_on_net:1;
443 unsigned long rs_prealloc:1;
444 unsigned long rs_committed:1;
445
446
447 atomic_t rs_refcount;
448
449 int rs_nlocks;
450
451
452 int rs_size;
453
454 __u32 rs_opc;
455
456 __u64 rs_transno;
457
458 __u64 rs_xid;
459 struct obd_export *rs_export;
460 struct ptlrpc_service_part *rs_svcpt;
461
462 struct lnet_handle_md rs_md_h;
463
464
465 struct ptlrpc_svc_ctx *rs_svc_ctx;
466
467 struct lustre_msg *rs_repbuf;
468
469 int rs_repbuf_len;
470
471 int rs_repdata_len;
472
473
474
475
476
477 struct lustre_msg *rs_msg;
478
479
480 struct lustre_handle rs_locks[RS_MAX_LOCKS];
481
482 enum ldlm_mode rs_modes[RS_MAX_LOCKS];
483};
484
485struct ptlrpc_thread;
486
487
488enum rq_phase {
489 RQ_PHASE_NEW = 0xebc0de00,
490 RQ_PHASE_RPC = 0xebc0de01,
491 RQ_PHASE_BULK = 0xebc0de02,
492 RQ_PHASE_INTERPRET = 0xebc0de03,
493 RQ_PHASE_COMPLETE = 0xebc0de04,
494 RQ_PHASE_UNREG_RPC = 0xebc0de05,
495 RQ_PHASE_UNREG_BULK = 0xebc0de06,
496 RQ_PHASE_UNDEFINED = 0xebc0de07
497};
498
499
500typedef int (*ptlrpc_interpterer_t)(const struct lu_env *env,
501 struct ptlrpc_request *req,
502 void *arg, int rc);
503
504
505
506
507
508
509
510struct ptlrpc_request_pool {
511
512 spinlock_t prp_lock;
513
514 struct list_head prp_req_list;
515
516 int prp_rq_size;
517
518 int (*prp_populate)(struct ptlrpc_request_pool *, int);
519};
520
521struct lu_context;
522struct lu_env;
523
524struct ldlm_lock;
525
526#include <lustre_nrs.h>
527
528
529
530
531
532
533
534
535
536
537struct ptlrpc_hpreq_ops {
538
539
540
541
542 int (*hpreq_lock_match)(struct ptlrpc_request *, struct ldlm_lock *);
543
544
545
546 int (*hpreq_check)(struct ptlrpc_request *);
547
548
549
550 void (*hpreq_fini)(struct ptlrpc_request *);
551};
552
553struct ptlrpc_cli_req {
554
555 struct ptlrpc_bulk_desc *cr_bulk;
556
557 long cr_delay_limit;
558
559 time_t cr_queued_time;
560
561 struct timespec64 cr_sent_tv;
562
563 time64_t cr_sent_out;
564
565 time64_t cr_reply_deadline;
566
567 time64_t cr_bulk_deadline;
568
569 time64_t cr_req_deadline;
570
571 short cr_req_ptl;
572
573 short cr_rep_ptl;
574
575 unsigned int cr_resend_nr;
576
577 int cr_imp_gen;
578 enum lustre_imp_state cr_send_state;
579
580 wait_queue_head_t cr_set_waitq;
581
582 struct list_head cr_set_chain;
583
584 struct list_head cr_ctx_chain;
585
586
587 struct ptlrpc_cli_ctx *cr_cli_ctx;
588
589 struct ptlrpc_request_set *cr_set;
590
591 struct lnet_handle_md cr_req_md_h;
592
593 struct ptlrpc_cb_id cr_req_cbid;
594
595 struct lnet_handle_md cr_reply_md_h;
596 wait_queue_head_t cr_reply_waitq;
597
598 struct ptlrpc_cb_id cr_reply_cbid;
599
600 ptlrpc_interpterer_t cr_reply_interp;
601
602 union ptlrpc_async_args cr_async_args;
603
604 void *cr_cb_data;
605
606 struct list_head cr_unreplied_list;
607
608
609
610
611 void (*cr_commit_cb)(struct ptlrpc_request *);
612
613 void (*cr_replay_cb)(struct ptlrpc_request *);
614};
615
616
617
618
619
620#define rq_bulk rq_cli.cr_bulk
621#define rq_delay_limit rq_cli.cr_delay_limit
622#define rq_queued_time rq_cli.cr_queued_time
623#define rq_sent_tv rq_cli.cr_sent_tv
624#define rq_real_sent rq_cli.cr_sent_out
625#define rq_reply_deadline rq_cli.cr_reply_deadline
626#define rq_bulk_deadline rq_cli.cr_bulk_deadline
627#define rq_req_deadline rq_cli.cr_req_deadline
628#define rq_nr_resend rq_cli.cr_resend_nr
629#define rq_request_portal rq_cli.cr_req_ptl
630#define rq_reply_portal rq_cli.cr_rep_ptl
631#define rq_import_generation rq_cli.cr_imp_gen
632#define rq_send_state rq_cli.cr_send_state
633#define rq_set_chain rq_cli.cr_set_chain
634#define rq_ctx_chain rq_cli.cr_ctx_chain
635#define rq_set rq_cli.cr_set
636#define rq_set_waitq rq_cli.cr_set_waitq
637#define rq_cli_ctx rq_cli.cr_cli_ctx
638#define rq_req_md_h rq_cli.cr_req_md_h
639#define rq_req_cbid rq_cli.cr_req_cbid
640#define rq_reply_md_h rq_cli.cr_reply_md_h
641#define rq_reply_waitq rq_cli.cr_reply_waitq
642#define rq_reply_cbid rq_cli.cr_reply_cbid
643#define rq_interpret_reply rq_cli.cr_reply_interp
644#define rq_async_args rq_cli.cr_async_args
645#define rq_cb_data rq_cli.cr_cb_data
646#define rq_unreplied_list rq_cli.cr_unreplied_list
647#define rq_commit_cb rq_cli.cr_commit_cb
648#define rq_replay_cb rq_cli.cr_replay_cb
649
650struct ptlrpc_srv_req {
651
652 struct ptlrpc_thread *sr_svc_thread;
653
654
655
656
657
658
659
660 struct list_head sr_timed_list;
661
662 struct list_head sr_exp_list;
663
664 struct list_head sr_hist_list;
665
666 __u64 sr_hist_seq;
667
668 time64_t sr_at_index;
669
670 uid_t sr_auth_uid;
671
672 uid_t sr_auth_mapped_uid;
673
674 enum lustre_sec_part sr_sp_from;
675
676 struct lu_context sr_ses;
677
678
679
680
681 struct ptlrpc_nrs_request sr_nrq;
682
683
684 struct timespec64 sr_arrival_time;
685
686 struct ptlrpc_svc_ctx *sr_svc_ctx;
687
688 struct ptlrpc_user_desc *sr_user_desc;
689
690 struct ptlrpc_reply_state *sr_reply_state;
691
692 struct ptlrpc_hpreq_ops *sr_ops;
693
694 struct ptlrpc_request_buffer_desc *sr_rqbd;
695};
696
697
698
699
700
701#define rq_svc_thread rq_srv.sr_svc_thread
702#define rq_timed_list rq_srv.sr_timed_list
703#define rq_exp_list rq_srv.sr_exp_list
704#define rq_history_list rq_srv.sr_hist_list
705#define rq_history_seq rq_srv.sr_hist_seq
706#define rq_at_index rq_srv.sr_at_index
707#define rq_auth_uid rq_srv.sr_auth_uid
708#define rq_auth_mapped_uid rq_srv.sr_auth_mapped_uid
709#define rq_sp_from rq_srv.sr_sp_from
710#define rq_session rq_srv.sr_ses
711#define rq_nrq rq_srv.sr_nrq
712#define rq_arrival_time rq_srv.sr_arrival_time
713#define rq_reply_state rq_srv.sr_reply_state
714#define rq_svc_ctx rq_srv.sr_svc_ctx
715#define rq_user_desc rq_srv.sr_user_desc
716#define rq_ops rq_srv.sr_ops
717#define rq_rqbd rq_srv.sr_rqbd
718
719
720
721
722
723
724
725struct ptlrpc_request {
726
727 int rq_type;
728
729 int rq_status;
730
731
732
733
734 struct list_head rq_list;
735
736
737
738 spinlock_t rq_lock;
739
740 unsigned int rq_intr:1, rq_replied:1, rq_err:1,
741 rq_timedout:1, rq_resend:1, rq_restart:1,
742
743
744
745
746
747
748
749
750
751 rq_replay:1,
752 rq_no_resend:1, rq_waiting:1, rq_receiving_reply:1,
753 rq_no_delay:1, rq_net_err:1, rq_wait_ctx:1,
754 rq_early:1,
755 rq_req_unlinked:1,
756 rq_reply_unlinked:1,
757 rq_memalloc:1,
758 rq_committed:1,
759 rq_reply_truncated:1,
760
761 rq_invalid_rqset:1,
762 rq_generation_set:1,
763
764 rq_no_retry_einprogress:1,
765
766
767
768 rq_allow_replay:1,
769
770 rq_unstable:1;
771
772
773
774 unsigned int
775 rq_hp:1,
776 rq_at_linked:1,
777 rq_packed_final:1;
778
779
780
781 enum rq_phase rq_phase;
782
783 enum rq_phase rq_next_phase;
784
785
786
787
788 atomic_t rq_refcount;
789
790
791
792
793
794 int rq_nob_received;
795
796 int rq_reqlen;
797
798 int rq_replen;
799
800 struct ptlrpc_request_pool *rq_pool;
801
802 struct lustre_msg *rq_reqmsg;
803
804 struct lustre_msg *rq_repmsg;
805
806 __u64 rq_transno;
807
808 __u64 rq_xid;
809
810 u64 rq_mbits;
811
812
813
814
815
816
817 struct list_head rq_replay_list;
818
819 union {
820 struct ptlrpc_cli_req rq_cli;
821 struct ptlrpc_srv_req rq_srv;
822 };
823
824
825
826
827
828 struct sptlrpc_flavor rq_flvr;
829
830
831 unsigned int
832 rq_ctx_init:1,
833 rq_ctx_fini:1,
834 rq_bulk_read:1,
835 rq_bulk_write:1,
836
837 rq_auth_gss:1,
838 rq_auth_usr_root:1,
839 rq_auth_usr_mdt:1,
840 rq_auth_usr_ost:1,
841
842 rq_pack_udesc:1,
843 rq_pack_bulk:1,
844
845 rq_no_reply:1,
846 rq_pill_init:1,
847 rq_srv_req:1;
848
849
850 struct lustre_msg *rq_reqbuf;
851 char *rq_repbuf;
852 struct lustre_msg *rq_repdata;
853
854 struct lustre_msg *rq_clrbuf;
855 int rq_reqbuf_len;
856 int rq_reqdata_len;
857 int rq_repbuf_len;
858 int rq_repdata_len;
859 int rq_clrbuf_len;
860 int rq_clrdata_len;
861
862
863 unsigned int rq_reply_off;
864
865
866
867
868 __u32 rq_req_swab_mask;
869 __u32 rq_rep_swab_mask;
870
871
872 int rq_early_count;
873
874
875 struct obd_export *rq_export;
876
877 struct obd_import *rq_import;
878
879 lnet_nid_t rq_self;
880
881 struct lnet_process_id rq_peer;
882
883
884
885
886 int rq_timeout;
887
888
889
890 time64_t rq_sent;
891
892 time64_t rq_deadline;
893
894 struct req_capsule rq_pill;
895};
896
897
898
899
900
901static inline int ptlrpc_req_interpret(const struct lu_env *env,
902 struct ptlrpc_request *req, int rc)
903{
904 if (req->rq_interpret_reply) {
905 req->rq_status = req->rq_interpret_reply(env, req,
906 &req->rq_async_args,
907 rc);
908 return req->rq_status;
909 }
910 return rc;
911}
912
913
914
915
916
917
918
919static inline bool ptlrpc_nrs_req_can_move(struct ptlrpc_request *req)
920{
921 struct ptlrpc_nrs_request *nrq = &req->rq_nrq;
922
923
924
925
926
927
928
929 return nrq->nr_enqueued && !nrq->nr_started && !req->rq_hp;
930}
931
932
933
934
935
936
937static inline int lustre_req_swabbed(struct ptlrpc_request *req, size_t index)
938{
939 LASSERT(index < sizeof(req->rq_req_swab_mask) * 8);
940 return req->rq_req_swab_mask & (1 << index);
941}
942
943
944
945
946static inline int lustre_rep_swabbed(struct ptlrpc_request *req, size_t index)
947{
948 LASSERT(index < sizeof(req->rq_rep_swab_mask) * 8);
949 return req->rq_rep_swab_mask & (1 << index);
950}
951
952
953
954
955static inline int ptlrpc_req_need_swab(struct ptlrpc_request *req)
956{
957 return lustre_req_swabbed(req, MSG_PTLRPC_HEADER_OFF);
958}
959
960
961
962
963static inline int ptlrpc_rep_need_swab(struct ptlrpc_request *req)
964{
965 return lustre_rep_swabbed(req, MSG_PTLRPC_HEADER_OFF);
966}
967
968
969
970
971static inline void lustre_set_req_swabbed(struct ptlrpc_request *req,
972 size_t index)
973{
974 LASSERT(index < sizeof(req->rq_req_swab_mask) * 8);
975 LASSERT((req->rq_req_swab_mask & (1 << index)) == 0);
976 req->rq_req_swab_mask |= 1 << index;
977}
978
979
980
981
982static inline void lustre_set_rep_swabbed(struct ptlrpc_request *req,
983 size_t index)
984{
985 LASSERT(index < sizeof(req->rq_rep_swab_mask) * 8);
986 LASSERT((req->rq_rep_swab_mask & (1 << index)) == 0);
987 req->rq_rep_swab_mask |= 1 << index;
988}
989
990
991
992
993static inline const char *
994ptlrpc_phase2str(enum rq_phase phase)
995{
996 switch (phase) {
997 case RQ_PHASE_NEW:
998 return "New";
999 case RQ_PHASE_RPC:
1000 return "Rpc";
1001 case RQ_PHASE_BULK:
1002 return "Bulk";
1003 case RQ_PHASE_INTERPRET:
1004 return "Interpret";
1005 case RQ_PHASE_COMPLETE:
1006 return "Complete";
1007 case RQ_PHASE_UNREG_RPC:
1008 return "UnregRPC";
1009 case RQ_PHASE_UNREG_BULK:
1010 return "UnregBULK";
1011 default:
1012 return "?Phase?";
1013 }
1014}
1015
1016
1017
1018
1019
1020static inline const char *
1021ptlrpc_rqphase2str(struct ptlrpc_request *req)
1022{
1023 return ptlrpc_phase2str(req->rq_phase);
1024}
1025
1026
1027
1028
1029
1030
1031#define FLAG(field, str) (field ? str : "")
1032
1033
1034#define DEBUG_REQ_FLAGS(req) \
1035 ptlrpc_rqphase2str(req), \
1036 FLAG(req->rq_intr, "I"), FLAG(req->rq_replied, "R"), \
1037 FLAG(req->rq_err, "E"), FLAG(req->rq_net_err, "e"), \
1038 FLAG(req->rq_timedout, "X") , FLAG(req->rq_resend, "S"), \
1039 FLAG(req->rq_restart, "T"), FLAG(req->rq_replay, "P"), \
1040 FLAG(req->rq_no_resend, "N"), \
1041 FLAG(req->rq_waiting, "W"), \
1042 FLAG(req->rq_wait_ctx, "C"), FLAG(req->rq_hp, "H"), \
1043 FLAG(req->rq_committed, "M")
1044
1045#define REQ_FLAGS_FMT "%s:%s%s%s%s%s%s%s%s%s%s%s%s%s"
1046
1047void _debug_req(struct ptlrpc_request *req,
1048 struct libcfs_debug_msg_data *data, const char *fmt, ...)
1049 __printf(3, 4);
1050
1051
1052
1053
1054
1055#define debug_req(msgdata, mask, cdls, req, fmt, a...) \
1056do { \
1057 CFS_CHECK_STACK(msgdata, mask, cdls); \
1058 \
1059 if (((mask) & D_CANTMASK) != 0 || \
1060 ((libcfs_debug & (mask)) != 0 && \
1061 (libcfs_subsystem_debug & DEBUG_SUBSYSTEM) != 0)) \
1062 _debug_req((req), msgdata, fmt, ##a); \
1063} while (0)
1064
1065
1066
1067
1068
1069
1070#define DEBUG_REQ(level, req, fmt, args...) \
1071do { \
1072 if ((level) & (D_ERROR | D_WARNING)) { \
1073 static struct cfs_debug_limit_state cdls; \
1074 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, level, &cdls); \
1075 debug_req(&msgdata, level, &cdls, req, "@@@ "fmt" ", ## args);\
1076 } else { \
1077 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, level, NULL); \
1078 debug_req(&msgdata, level, NULL, req, "@@@ "fmt" ", ## args); \
1079 } \
1080} while (0)
1081
1082
1083
1084
1085
1086struct ptlrpc_bulk_page {
1087
1088 struct list_head bp_link;
1089
1090
1091
1092 int bp_buflen;
1093
1094 int bp_pageoffset;
1095
1096 struct page *bp_page;
1097};
1098
1099enum ptlrpc_bulk_op_type {
1100 PTLRPC_BULK_OP_ACTIVE = 0x00000001,
1101 PTLRPC_BULK_OP_PASSIVE = 0x00000002,
1102 PTLRPC_BULK_OP_PUT = 0x00000004,
1103 PTLRPC_BULK_OP_GET = 0x00000008,
1104 PTLRPC_BULK_BUF_KVEC = 0x00000010,
1105 PTLRPC_BULK_BUF_KIOV = 0x00000020,
1106 PTLRPC_BULK_GET_SOURCE = PTLRPC_BULK_OP_PASSIVE | PTLRPC_BULK_OP_GET,
1107 PTLRPC_BULK_PUT_SINK = PTLRPC_BULK_OP_PASSIVE | PTLRPC_BULK_OP_PUT,
1108 PTLRPC_BULK_GET_SINK = PTLRPC_BULK_OP_ACTIVE | PTLRPC_BULK_OP_GET,
1109 PTLRPC_BULK_PUT_SOURCE = PTLRPC_BULK_OP_ACTIVE | PTLRPC_BULK_OP_PUT,
1110};
1111
1112static inline bool ptlrpc_is_bulk_op_get(enum ptlrpc_bulk_op_type type)
1113{
1114 return (type & PTLRPC_BULK_OP_GET) == PTLRPC_BULK_OP_GET;
1115}
1116
1117static inline bool ptlrpc_is_bulk_get_source(enum ptlrpc_bulk_op_type type)
1118{
1119 return (type & PTLRPC_BULK_GET_SOURCE) == PTLRPC_BULK_GET_SOURCE;
1120}
1121
1122static inline bool ptlrpc_is_bulk_put_sink(enum ptlrpc_bulk_op_type type)
1123{
1124 return (type & PTLRPC_BULK_PUT_SINK) == PTLRPC_BULK_PUT_SINK;
1125}
1126
1127static inline bool ptlrpc_is_bulk_get_sink(enum ptlrpc_bulk_op_type type)
1128{
1129 return (type & PTLRPC_BULK_GET_SINK) == PTLRPC_BULK_GET_SINK;
1130}
1131
1132static inline bool ptlrpc_is_bulk_put_source(enum ptlrpc_bulk_op_type type)
1133{
1134 return (type & PTLRPC_BULK_PUT_SOURCE) == PTLRPC_BULK_PUT_SOURCE;
1135}
1136
1137static inline bool ptlrpc_is_bulk_desc_kvec(enum ptlrpc_bulk_op_type type)
1138{
1139 return ((type & PTLRPC_BULK_BUF_KVEC) | (type & PTLRPC_BULK_BUF_KIOV))
1140 == PTLRPC_BULK_BUF_KVEC;
1141}
1142
1143static inline bool ptlrpc_is_bulk_desc_kiov(enum ptlrpc_bulk_op_type type)
1144{
1145 return ((type & PTLRPC_BULK_BUF_KVEC) | (type & PTLRPC_BULK_BUF_KIOV))
1146 == PTLRPC_BULK_BUF_KIOV;
1147}
1148
1149static inline bool ptlrpc_is_bulk_op_active(enum ptlrpc_bulk_op_type type)
1150{
1151 return ((type & PTLRPC_BULK_OP_ACTIVE) |
1152 (type & PTLRPC_BULK_OP_PASSIVE)) == PTLRPC_BULK_OP_ACTIVE;
1153}
1154
1155static inline bool ptlrpc_is_bulk_op_passive(enum ptlrpc_bulk_op_type type)
1156{
1157 return ((type & PTLRPC_BULK_OP_ACTIVE) |
1158 (type & PTLRPC_BULK_OP_PASSIVE)) == PTLRPC_BULK_OP_PASSIVE;
1159}
1160
1161struct ptlrpc_bulk_frag_ops {
1162
1163
1164
1165
1166
1167 void (*add_kiov_frag)(struct ptlrpc_bulk_desc *desc,
1168 struct page *page, int pageoffset, int len);
1169
1170
1171
1172
1173
1174
1175 int (*add_iov_frag)(struct ptlrpc_bulk_desc *desc, void *frag, int len);
1176
1177
1178
1179
1180
1181 void (*release_frags)(struct ptlrpc_bulk_desc *desc);
1182};
1183
1184extern const struct ptlrpc_bulk_frag_ops ptlrpc_bulk_kiov_pin_ops;
1185extern const struct ptlrpc_bulk_frag_ops ptlrpc_bulk_kiov_nopin_ops;
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197struct ptlrpc_bulk_desc {
1198
1199 unsigned long bd_failure:1;
1200
1201 unsigned long bd_registered:1;
1202
1203 spinlock_t bd_lock;
1204
1205 int bd_import_generation;
1206
1207 enum ptlrpc_bulk_op_type bd_type;
1208
1209 __u32 bd_portal;
1210
1211 struct obd_export *bd_export;
1212
1213 struct obd_import *bd_import;
1214
1215 struct ptlrpc_request *bd_req;
1216 struct ptlrpc_bulk_frag_ops *bd_frag_ops;
1217 wait_queue_head_t bd_waitq;
1218 int bd_iov_count;
1219 int bd_max_iov;
1220 int bd_nob;
1221 int bd_nob_transferred;
1222
1223 u64 bd_last_mbits;
1224
1225 struct ptlrpc_cb_id bd_cbid;
1226 lnet_nid_t bd_sender;
1227 int bd_md_count;
1228 int bd_md_max_brw;
1229
1230 struct lnet_handle_md bd_mds[PTLRPC_BULK_OPS_COUNT];
1231
1232 union {
1233 struct {
1234
1235
1236
1237 struct bio_vec *bd_enc_vec;
1238 struct bio_vec *bd_vec;
1239 } bd_kiov;
1240
1241 struct {
1242 struct kvec *bd_enc_kvec;
1243 struct kvec *bd_kvec;
1244 } bd_kvec;
1245 } bd_u;
1246};
1247
1248#define GET_KIOV(desc) ((desc)->bd_u.bd_kiov.bd_vec)
1249#define BD_GET_KIOV(desc, i) ((desc)->bd_u.bd_kiov.bd_vec[i])
1250#define GET_ENC_KIOV(desc) ((desc)->bd_u.bd_kiov.bd_enc_vec)
1251#define BD_GET_ENC_KIOV(desc, i) ((desc)->bd_u.bd_kiov.bd_enc_vec[i])
1252#define GET_KVEC(desc) ((desc)->bd_u.bd_kvec.bd_kvec)
1253#define BD_GET_KVEC(desc, i) ((desc)->bd_u.bd_kvec.bd_kvec[i])
1254#define GET_ENC_KVEC(desc) ((desc)->bd_u.bd_kvec.bd_enc_kvec)
1255#define BD_GET_ENC_KVEC(desc, i) ((desc)->bd_u.bd_kvec.bd_enc_kvec[i])
1256
1257enum {
1258 SVC_STOPPED = 1 << 0,
1259 SVC_STOPPING = 1 << 1,
1260 SVC_STARTING = 1 << 2,
1261 SVC_RUNNING = 1 << 3,
1262};
1263
1264#define PTLRPC_THR_NAME_LEN 32
1265
1266
1267
1268struct ptlrpc_thread {
1269
1270
1271
1272 struct list_head t_link;
1273
1274
1275
1276 void *t_data;
1277 __u32 t_flags;
1278
1279
1280
1281 unsigned int t_id;
1282
1283
1284
1285 pid_t t_pid;
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298 struct ptlrpc_service_part *t_svcpt;
1299 wait_queue_head_t t_ctl_waitq;
1300 struct lu_env *t_env;
1301 char t_name[PTLRPC_THR_NAME_LEN];
1302};
1303
1304static inline int thread_is_stopped(struct ptlrpc_thread *thread)
1305{
1306 return !!(thread->t_flags & SVC_STOPPED);
1307}
1308
1309static inline int thread_is_stopping(struct ptlrpc_thread *thread)
1310{
1311 return !!(thread->t_flags & SVC_STOPPING);
1312}
1313
1314static inline int thread_is_starting(struct ptlrpc_thread *thread)
1315{
1316 return !!(thread->t_flags & SVC_STARTING);
1317}
1318
1319static inline int thread_is_running(struct ptlrpc_thread *thread)
1320{
1321 return !!(thread->t_flags & SVC_RUNNING);
1322}
1323
1324static inline void thread_clear_flags(struct ptlrpc_thread *thread, __u32 flags)
1325{
1326 thread->t_flags &= ~flags;
1327}
1328
1329static inline void thread_set_flags(struct ptlrpc_thread *thread, __u32 flags)
1330{
1331 thread->t_flags = flags;
1332}
1333
1334static inline void thread_add_flags(struct ptlrpc_thread *thread, __u32 flags)
1335{
1336 thread->t_flags |= flags;
1337}
1338
1339static inline int thread_test_and_clear_flags(struct ptlrpc_thread *thread,
1340 __u32 flags)
1341{
1342 if (thread->t_flags & flags) {
1343 thread->t_flags &= ~flags;
1344 return 1;
1345 }
1346 return 0;
1347}
1348
1349
1350
1351
1352
1353
1354
1355
1356struct ptlrpc_request_buffer_desc {
1357
1358 struct list_head rqbd_list;
1359
1360 struct list_head rqbd_reqs;
1361
1362 struct ptlrpc_service_part *rqbd_svcpt;
1363
1364 struct lnet_handle_md rqbd_md_h;
1365 int rqbd_refcount;
1366
1367 char *rqbd_buffer;
1368 struct ptlrpc_cb_id rqbd_cbid;
1369
1370
1371
1372
1373 struct ptlrpc_request rqbd_req;
1374};
1375
1376typedef int (*svc_handler_t)(struct ptlrpc_request *req);
1377
1378struct ptlrpc_service_ops {
1379
1380
1381
1382
1383 int (*so_thr_init)(struct ptlrpc_thread *thr);
1384
1385
1386
1387
1388 void (*so_thr_done)(struct ptlrpc_thread *thr);
1389
1390
1391
1392 int (*so_req_handler)(struct ptlrpc_request *req);
1393
1394
1395
1396
1397 int (*so_hpreq_handler)(struct ptlrpc_request *);
1398
1399
1400
1401 void (*so_req_printer)(void *, struct ptlrpc_request *);
1402};
1403
1404#ifndef __cfs_cacheline_aligned
1405
1406# define __cfs_cacheline_aligned
1407#endif
1408
1409
1410
1411
1412
1413#define PTLRPC_SVC_HP_RATIO 10
1414
1415
1416
1417
1418
1419
1420
1421struct ptlrpc_service {
1422
1423 spinlock_t srv_lock;
1424
1425
1426 struct list_head srv_list;
1427
1428 struct ptlrpc_service_ops srv_ops;
1429
1430 char *srv_name;
1431
1432 char *srv_thread_name;
1433
1434 struct list_head srv_threads;
1435
1436 int srv_nthrs_cpt_init;
1437
1438 int srv_nthrs_cpt_limit;
1439
1440 struct dentry *srv_debugfs_entry;
1441
1442 struct lprocfs_stats *srv_stats;
1443
1444 int srv_hpreq_ratio;
1445
1446 int srv_max_req_size;
1447
1448 int srv_max_reply_size;
1449
1450 int srv_buf_size;
1451
1452 int srv_nbuf_per_group;
1453
1454 __u32 srv_req_portal;
1455
1456 __u32 srv_rep_portal;
1457
1458
1459
1460
1461 __u32 srv_ctx_tags;
1462
1463 int srv_watchdog_factor;
1464
1465 unsigned srv_is_stopping:1;
1466
1467
1468 int srv_hist_nrqbds_cpt_max;
1469
1470 int srv_ncpts;
1471
1472 __u32 *srv_cpts;
1473
1474 int srv_cpt_bits;
1475
1476 struct cfs_cpt_table *srv_cptable;
1477
1478
1479 struct kobject srv_kobj;
1480 struct completion srv_kobj_unregister;
1481
1482
1483
1484 struct ptlrpc_service_part *srv_parts[0];
1485};
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505struct ptlrpc_service_part {
1506
1507 struct ptlrpc_service *scp_service __cfs_cacheline_aligned;
1508
1509 int scp_cpt;
1510
1511 int scp_thr_nextid;
1512
1513 int scp_nthrs_starting;
1514
1515 int scp_nthrs_stopping;
1516
1517 int scp_nthrs_running;
1518
1519 struct list_head scp_threads;
1520
1521
1522
1523
1524
1525
1526 spinlock_t scp_lock __cfs_cacheline_aligned;
1527
1528 int scp_nrqbds_total;
1529
1530 int scp_nrqbds_posted;
1531
1532 int scp_rqbd_allocating;
1533
1534 int scp_nreqs_incoming;
1535
1536 struct list_head scp_rqbd_idle;
1537
1538 struct list_head scp_rqbd_posted;
1539
1540 struct list_head scp_req_incoming;
1541
1542 long scp_rqbd_timeout;
1543
1544
1545
1546
1547 wait_queue_head_t scp_waitq;
1548
1549
1550 struct list_head scp_hist_reqs;
1551
1552 struct list_head scp_hist_rqbds;
1553
1554 int scp_hist_nrqbds;
1555
1556 __u64 scp_hist_seq;
1557
1558 __u64 scp_hist_seq_culled;
1559
1560
1561
1562
1563
1564 spinlock_t scp_req_lock __cfs_cacheline_aligned;
1565
1566
1567 int scp_nreqs_active;
1568
1569 int scp_nhreqs_active;
1570
1571 int scp_hreq_count;
1572
1573
1574 struct ptlrpc_nrs scp_nrs_reg;
1575
1576
1577
1578 struct ptlrpc_nrs *scp_nrs_hp;
1579
1580
1581
1582
1583
1584
1585
1586 spinlock_t scp_at_lock __cfs_cacheline_aligned;
1587
1588 struct adaptive_timeout scp_at_estimate;
1589
1590 struct ptlrpc_at_array scp_at_array;
1591
1592 struct timer_list scp_at_timer;
1593
1594 unsigned long scp_at_checktime;
1595
1596 unsigned scp_at_check;
1597
1598
1599
1600
1601
1602
1603 spinlock_t scp_rep_lock __cfs_cacheline_aligned;
1604
1605 struct list_head scp_rep_active;
1606
1607 struct list_head scp_rep_idle;
1608
1609 wait_queue_head_t scp_rep_waitq;
1610
1611 atomic_t scp_nreps_difficult;
1612};
1613
1614#define ptlrpc_service_for_each_part(part, i, svc) \
1615 for (i = 0; \
1616 i < (svc)->srv_ncpts && \
1617 (svc)->srv_parts && \
1618 ((part) = (svc)->srv_parts[i]); i++)
1619
1620
1621
1622
1623struct ptlrpcd_ctl {
1624
1625
1626
1627 unsigned long pc_flags;
1628
1629
1630
1631 spinlock_t pc_lock;
1632
1633
1634
1635 struct completion pc_starting;
1636
1637
1638
1639 struct completion pc_finishing;
1640
1641
1642
1643 struct ptlrpc_request_set *pc_set;
1644
1645
1646
1647 char pc_name[16];
1648
1649
1650
1651 int pc_cpt;
1652
1653
1654
1655 int pc_index;
1656
1657
1658
1659 struct ptlrpcd_ctl **pc_partners;
1660
1661
1662
1663 int pc_npartners;
1664
1665
1666
1667 int pc_cursor;
1668
1669
1670
1671 int pc_error;
1672};
1673
1674
1675enum ptlrpcd_ctl_flags {
1676
1677
1678
1679 LIOD_START = 1 << 0,
1680
1681
1682
1683 LIOD_STOP = 1 << 1,
1684
1685
1686
1687
1688
1689 LIOD_FORCE = 1 << 2,
1690
1691
1692
1693 LIOD_RECOVERY = 1 << 3,
1694};
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709static inline bool nrs_policy_compat_all(const struct ptlrpc_service *svc,
1710 const struct ptlrpc_nrs_pol_desc *desc)
1711{
1712 return true;
1713}
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728static inline bool nrs_policy_compat_one(const struct ptlrpc_service *svc,
1729 const struct ptlrpc_nrs_pol_desc *desc)
1730{
1731 return strcmp(svc->srv_name, desc->pd_compat_svc_name) == 0;
1732}
1733
1734
1735
1736
1737extern struct lnet_handle_eq ptlrpc_eq_h;
1738int ptlrpc_uuid_to_peer(struct obd_uuid *uuid,
1739 struct lnet_process_id *peer, lnet_nid_t *self);
1740
1741
1742
1743
1744
1745void request_out_callback(struct lnet_event *ev);
1746void reply_in_callback(struct lnet_event *ev);
1747void client_bulk_callback(struct lnet_event *ev);
1748void request_in_callback(struct lnet_event *ev);
1749void reply_out_callback(struct lnet_event *ev);
1750
1751
1752
1753struct ptlrpc_connection *ptlrpc_connection_get(struct lnet_process_id peer,
1754 lnet_nid_t self,
1755 struct obd_uuid *uuid);
1756int ptlrpc_connection_put(struct ptlrpc_connection *c);
1757struct ptlrpc_connection *ptlrpc_connection_addref(struct ptlrpc_connection *);
1758int ptlrpc_connection_init(void);
1759void ptlrpc_connection_fini(void);
1760
1761
1762
1763
1764
1765
1766
1767int ptlrpc_unregister_bulk(struct ptlrpc_request *req, int async);
1768
1769static inline int ptlrpc_client_bulk_active(struct ptlrpc_request *req)
1770{
1771 struct ptlrpc_bulk_desc *desc;
1772 int rc;
1773
1774 desc = req->rq_bulk;
1775
1776 if (req->rq_bulk_deadline > ktime_get_real_seconds())
1777 return 1;
1778
1779 if (!desc)
1780 return 0;
1781
1782 spin_lock(&desc->bd_lock);
1783 rc = desc->bd_md_count;
1784 spin_unlock(&desc->bd_lock);
1785 return rc;
1786}
1787
1788#define PTLRPC_REPLY_MAYBE_DIFFICULT 0x01
1789#define PTLRPC_REPLY_EARLY 0x02
1790int ptlrpc_send_reply(struct ptlrpc_request *req, int flags);
1791int ptlrpc_reply(struct ptlrpc_request *req);
1792int ptlrpc_send_error(struct ptlrpc_request *req, int difficult);
1793int ptlrpc_error(struct ptlrpc_request *req);
1794int ptlrpc_at_get_net_latency(struct ptlrpc_request *req);
1795int ptl_send_rpc(struct ptlrpc_request *request, int noreply);
1796int ptlrpc_register_rqbd(struct ptlrpc_request_buffer_desc *rqbd);
1797
1798
1799
1800
1801
1802
1803
1804
1805void ptlrpc_request_committed(struct ptlrpc_request *req, int force);
1806
1807int ptlrpc_inc_ref(void);
1808void ptlrpc_dec_ref(void);
1809
1810void ptlrpc_init_client(int req_portal, int rep_portal, char *name,
1811 struct ptlrpc_client *);
1812struct ptlrpc_connection *ptlrpc_uuid_to_connection(struct obd_uuid *uuid);
1813
1814int ptlrpc_queue_wait(struct ptlrpc_request *req);
1815int ptlrpc_replay_req(struct ptlrpc_request *req);
1816void ptlrpc_abort_inflight(struct obd_import *imp);
1817void ptlrpc_abort_set(struct ptlrpc_request_set *set);
1818
1819struct ptlrpc_request_set *ptlrpc_prep_set(void);
1820struct ptlrpc_request_set *ptlrpc_prep_fcset(int max, set_producer_func func,
1821 void *arg);
1822int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set);
1823int ptlrpc_set_wait(struct ptlrpc_request_set *);
1824void ptlrpc_mark_interrupted(struct ptlrpc_request *req);
1825void ptlrpc_set_destroy(struct ptlrpc_request_set *);
1826void ptlrpc_set_add_req(struct ptlrpc_request_set *, struct ptlrpc_request *);
1827
1828void ptlrpc_free_rq_pool(struct ptlrpc_request_pool *pool);
1829int ptlrpc_add_rqs_to_pool(struct ptlrpc_request_pool *pool, int num_rq);
1830
1831struct ptlrpc_request_pool *
1832ptlrpc_init_rq_pool(int, int,
1833 int (*populate_pool)(struct ptlrpc_request_pool *, int));
1834
1835void ptlrpc_at_set_req_timeout(struct ptlrpc_request *req);
1836struct ptlrpc_request *ptlrpc_request_alloc(struct obd_import *imp,
1837 const struct req_format *format);
1838struct ptlrpc_request *ptlrpc_request_alloc_pool(struct obd_import *imp,
1839 struct ptlrpc_request_pool *,
1840 const struct req_format *);
1841void ptlrpc_request_free(struct ptlrpc_request *request);
1842int ptlrpc_request_pack(struct ptlrpc_request *request,
1843 __u32 version, int opcode);
1844struct ptlrpc_request *ptlrpc_request_alloc_pack(struct obd_import *,
1845 const struct req_format *,
1846 __u32, int);
1847int ptlrpc_request_bufs_pack(struct ptlrpc_request *request,
1848 __u32 version, int opcode, char **bufs,
1849 struct ptlrpc_cli_ctx *ctx);
1850void ptlrpc_req_finished(struct ptlrpc_request *request);
1851struct ptlrpc_request *ptlrpc_request_addref(struct ptlrpc_request *req);
1852struct ptlrpc_bulk_desc *ptlrpc_prep_bulk_imp(struct ptlrpc_request *req,
1853 unsigned int nfrags,
1854 unsigned int max_brw,
1855 unsigned int type,
1856 unsigned int portal,
1857 const struct ptlrpc_bulk_frag_ops *ops);
1858
1859int ptlrpc_prep_bulk_frag(struct ptlrpc_bulk_desc *desc,
1860 void *frag, int len);
1861void __ptlrpc_prep_bulk_page(struct ptlrpc_bulk_desc *desc,
1862 struct page *page, int pageoffset, int len,
1863 int pin);
1864static inline void ptlrpc_prep_bulk_page_pin(struct ptlrpc_bulk_desc *desc,
1865 struct page *page, int pageoffset,
1866 int len)
1867{
1868 __ptlrpc_prep_bulk_page(desc, page, pageoffset, len, 1);
1869}
1870
1871static inline void ptlrpc_prep_bulk_page_nopin(struct ptlrpc_bulk_desc *desc,
1872 struct page *page, int pageoffset,
1873 int len)
1874{
1875 __ptlrpc_prep_bulk_page(desc, page, pageoffset, len, 0);
1876}
1877
1878void ptlrpc_free_bulk(struct ptlrpc_bulk_desc *bulk);
1879
1880static inline void ptlrpc_release_bulk_page_pin(struct ptlrpc_bulk_desc *desc)
1881{
1882 int i;
1883
1884 for (i = 0; i < desc->bd_iov_count ; i++)
1885 put_page(BD_GET_KIOV(desc, i).bv_page);
1886}
1887
1888void ptlrpc_retain_replayable_request(struct ptlrpc_request *req,
1889 struct obd_import *imp);
1890__u64 ptlrpc_next_xid(void);
1891__u64 ptlrpc_sample_next_xid(void);
1892__u64 ptlrpc_req_xid(struct ptlrpc_request *request);
1893
1894
1895void *ptlrpcd_alloc_work(struct obd_import *imp,
1896 int (*cb)(const struct lu_env *, void *), void *data);
1897void ptlrpcd_destroy_work(void *handler);
1898int ptlrpcd_queue_work(void *handler);
1899
1900
1901struct ptlrpc_service_buf_conf {
1902
1903 unsigned int bc_nbufs;
1904
1905 unsigned int bc_buf_size;
1906
1907 unsigned int bc_req_portal;
1908
1909 unsigned int bc_rep_portal;
1910
1911 unsigned int bc_req_max_size;
1912
1913 unsigned int bc_rep_max_size;
1914};
1915
1916struct ptlrpc_service_thr_conf {
1917
1918 char *tc_thr_name;
1919
1920 unsigned int tc_thr_factor;
1921
1922 unsigned int tc_nthrs_init;
1923
1924
1925
1926
1927
1928
1929 unsigned int tc_nthrs_base;
1930
1931 unsigned int tc_nthrs_max;
1932
1933
1934
1935 unsigned int tc_nthrs_user;
1936
1937 unsigned int tc_cpu_affinity;
1938
1939 __u32 tc_ctx_tags;
1940};
1941
1942struct ptlrpc_service_cpt_conf {
1943 struct cfs_cpt_table *cc_cptable;
1944
1945 char *cc_pattern;
1946};
1947
1948struct ptlrpc_service_conf {
1949
1950 char *psc_name;
1951
1952 unsigned int psc_watchdog_factor;
1953
1954 struct ptlrpc_service_buf_conf psc_buf;
1955
1956 struct ptlrpc_service_thr_conf psc_thr;
1957
1958 struct ptlrpc_service_cpt_conf psc_cpt;
1959
1960 struct ptlrpc_service_ops psc_ops;
1961};
1962
1963
1964
1965
1966
1967
1968
1969
1970void ptlrpc_dispatch_difficult_reply(struct ptlrpc_reply_state *rs);
1971void ptlrpc_schedule_difficult_reply(struct ptlrpc_reply_state *rs);
1972struct ptlrpc_service *ptlrpc_register_service(struct ptlrpc_service_conf *conf,
1973 struct kset *parent,
1974 struct dentry *debugfs_entry);
1975
1976int ptlrpc_start_threads(struct ptlrpc_service *svc);
1977int ptlrpc_unregister_service(struct ptlrpc_service *service);
1978
1979int ptlrpc_hr_init(void);
1980void ptlrpc_hr_fini(void);
1981
1982
1983
1984
1985
1986
1987
1988
1989int ptlrpc_connect_import(struct obd_import *imp);
1990int ptlrpc_init_import(struct obd_import *imp);
1991int ptlrpc_disconnect_import(struct obd_import *imp, int noclose);
1992int ptlrpc_import_recovery_state_machine(struct obd_import *imp);
1993
1994
1995int ptlrpc_reconnect_import(struct obd_import *imp);
1996
1997
1998
1999
2000
2001
2002
2003int ptlrpc_buf_need_swab(struct ptlrpc_request *req, const int inout,
2004 u32 index);
2005void ptlrpc_buf_set_swabbed(struct ptlrpc_request *req, const int inout,
2006 u32 index);
2007int ptlrpc_unpack_rep_msg(struct ptlrpc_request *req, int len);
2008int ptlrpc_unpack_req_msg(struct ptlrpc_request *req, int len);
2009
2010void lustre_init_msg_v2(struct lustre_msg_v2 *msg, int count, __u32 *lens,
2011 char **bufs);
2012int lustre_pack_request(struct ptlrpc_request *, __u32 magic, int count,
2013 __u32 *lens, char **bufs);
2014int lustre_pack_reply(struct ptlrpc_request *, int count, __u32 *lens,
2015 char **bufs);
2016int lustre_pack_reply_v2(struct ptlrpc_request *req, int count,
2017 __u32 *lens, char **bufs, int flags);
2018#define LPRFL_EARLY_REPLY 1
2019int lustre_pack_reply_flags(struct ptlrpc_request *, int count, __u32 *lens,
2020 char **bufs, int flags);
2021int lustre_shrink_msg(struct lustre_msg *msg, int segment,
2022 unsigned int newlen, int move_data);
2023void lustre_free_reply_state(struct ptlrpc_reply_state *rs);
2024int __lustre_unpack_msg(struct lustre_msg *m, int len);
2025u32 lustre_msg_hdr_size(__u32 magic, u32 count);
2026u32 lustre_msg_size(__u32 magic, int count, __u32 *lengths);
2027u32 lustre_msg_size_v2(int count, __u32 *lengths);
2028u32 lustre_packed_msg_size(struct lustre_msg *msg);
2029u32 lustre_msg_early_size(void);
2030void *lustre_msg_buf_v2(struct lustre_msg_v2 *m, u32 n, u32 min_size);
2031void *lustre_msg_buf(struct lustre_msg *m, u32 n, u32 minlen);
2032u32 lustre_msg_buflen(struct lustre_msg *m, u32 n);
2033u32 lustre_msg_bufcount(struct lustre_msg *m);
2034char *lustre_msg_string(struct lustre_msg *m, u32 n, u32 max_len);
2035__u32 lustre_msghdr_get_flags(struct lustre_msg *msg);
2036void lustre_msghdr_set_flags(struct lustre_msg *msg, __u32 flags);
2037__u32 lustre_msg_get_flags(struct lustre_msg *msg);
2038void lustre_msg_add_flags(struct lustre_msg *msg, u32 flags);
2039void lustre_msg_set_flags(struct lustre_msg *msg, u32 flags);
2040void lustre_msg_clear_flags(struct lustre_msg *msg, u32 flags);
2041__u32 lustre_msg_get_op_flags(struct lustre_msg *msg);
2042void lustre_msg_add_op_flags(struct lustre_msg *msg, u32 flags);
2043struct lustre_handle *lustre_msg_get_handle(struct lustre_msg *msg);
2044__u32 lustre_msg_get_type(struct lustre_msg *msg);
2045void lustre_msg_add_version(struct lustre_msg *msg, u32 version);
2046__u32 lustre_msg_get_opc(struct lustre_msg *msg);
2047__u16 lustre_msg_get_tag(struct lustre_msg *msg);
2048__u64 lustre_msg_get_last_committed(struct lustre_msg *msg);
2049__u64 *lustre_msg_get_versions(struct lustre_msg *msg);
2050__u64 lustre_msg_get_transno(struct lustre_msg *msg);
2051__u64 lustre_msg_get_slv(struct lustre_msg *msg);
2052__u32 lustre_msg_get_limit(struct lustre_msg *msg);
2053void lustre_msg_set_slv(struct lustre_msg *msg, __u64 slv);
2054void lustre_msg_set_limit(struct lustre_msg *msg, __u64 limit);
2055int lustre_msg_get_status(struct lustre_msg *msg);
2056__u32 lustre_msg_get_conn_cnt(struct lustre_msg *msg);
2057__u32 lustre_msg_get_magic(struct lustre_msg *msg);
2058__u32 lustre_msg_get_timeout(struct lustre_msg *msg);
2059__u32 lustre_msg_get_service_time(struct lustre_msg *msg);
2060__u32 lustre_msg_get_cksum(struct lustre_msg *msg);
2061__u32 lustre_msg_calc_cksum(struct lustre_msg *msg);
2062void lustre_msg_set_handle(struct lustre_msg *msg,
2063 struct lustre_handle *handle);
2064void lustre_msg_set_type(struct lustre_msg *msg, __u32 type);
2065void lustre_msg_set_opc(struct lustre_msg *msg, __u32 opc);
2066void lustre_msg_set_last_xid(struct lustre_msg *msg, u64 last_xid);
2067void lustre_msg_set_tag(struct lustre_msg *msg, __u16 tag);
2068void lustre_msg_set_versions(struct lustre_msg *msg, __u64 *versions);
2069void lustre_msg_set_transno(struct lustre_msg *msg, __u64 transno);
2070void lustre_msg_set_status(struct lustre_msg *msg, __u32 status);
2071void lustre_msg_set_conn_cnt(struct lustre_msg *msg, __u32 conn_cnt);
2072void ptlrpc_request_set_replen(struct ptlrpc_request *req);
2073void lustre_msg_set_timeout(struct lustre_msg *msg, __u32 timeout);
2074void lustre_msg_set_service_time(struct lustre_msg *msg, __u32 service_time);
2075void lustre_msg_set_jobid(struct lustre_msg *msg, char *jobid);
2076void lustre_msg_set_cksum(struct lustre_msg *msg, __u32 cksum);
2077void lustre_msg_set_mbits(struct lustre_msg *msg, u64 mbits);
2078
2079static inline void
2080lustre_shrink_reply(struct ptlrpc_request *req, int segment,
2081 unsigned int newlen, int move_data)
2082{
2083 LASSERT(req->rq_reply_state);
2084 LASSERT(req->rq_repmsg);
2085 req->rq_replen = lustre_shrink_msg(req->rq_repmsg, segment,
2086 newlen, move_data);
2087}
2088
2089#ifdef CONFIG_LUSTRE_TRANSLATE_ERRNOS
2090
2091static inline int ptlrpc_status_hton(int h)
2092{
2093
2094
2095
2096
2097 if (h < 0)
2098 return -lustre_errno_hton(-h);
2099 else
2100 return h;
2101}
2102
2103static inline int ptlrpc_status_ntoh(int n)
2104{
2105
2106
2107
2108 if (n < 0)
2109 return -lustre_errno_ntoh(-n);
2110 else
2111 return n;
2112}
2113
2114#else
2115
2116#define ptlrpc_status_hton(h) (h)
2117#define ptlrpc_status_ntoh(n) (n)
2118
2119#endif
2120
2121
2122
2123static inline void
2124ptlrpc_rqphase_move(struct ptlrpc_request *req, enum rq_phase new_phase)
2125{
2126 if (req->rq_phase == new_phase)
2127 return;
2128
2129 if (new_phase == RQ_PHASE_UNREG_RPC ||
2130 new_phase == RQ_PHASE_UNREG_BULK) {
2131
2132 if (req->rq_phase == RQ_PHASE_UNREG_RPC ||
2133 req->rq_phase == RQ_PHASE_UNREG_BULK)
2134 return;
2135
2136 req->rq_next_phase = req->rq_phase;
2137 if (req->rq_import)
2138 atomic_inc(&req->rq_import->imp_unregistering);
2139 }
2140
2141 if (req->rq_phase == RQ_PHASE_UNREG_RPC ||
2142 req->rq_phase == RQ_PHASE_UNREG_BULK) {
2143 if (req->rq_import)
2144 atomic_dec(&req->rq_import->imp_unregistering);
2145 }
2146
2147 DEBUG_REQ(D_INFO, req, "move req \"%s\" -> \"%s\"",
2148 ptlrpc_rqphase2str(req), ptlrpc_phase2str(new_phase));
2149
2150 req->rq_phase = new_phase;
2151}
2152
2153
2154
2155
2156static inline int
2157ptlrpc_client_early(struct ptlrpc_request *req)
2158{
2159 return req->rq_early;
2160}
2161
2162
2163
2164
2165static inline int
2166ptlrpc_client_replied(struct ptlrpc_request *req)
2167{
2168 if (req->rq_reply_deadline > ktime_get_real_seconds())
2169 return 0;
2170 return req->rq_replied;
2171}
2172
2173
2174static inline int
2175ptlrpc_client_recv(struct ptlrpc_request *req)
2176{
2177 if (req->rq_reply_deadline > ktime_get_real_seconds())
2178 return 1;
2179 return req->rq_receiving_reply;
2180}
2181
2182static inline int
2183ptlrpc_client_recv_or_unlink(struct ptlrpc_request *req)
2184{
2185 int rc;
2186
2187 spin_lock(&req->rq_lock);
2188 if (req->rq_reply_deadline > ktime_get_real_seconds()) {
2189 spin_unlock(&req->rq_lock);
2190 return 1;
2191 }
2192 if (req->rq_req_deadline > ktime_get_real_seconds()) {
2193 spin_unlock(&req->rq_lock);
2194 return 1;
2195 }
2196 rc = !req->rq_req_unlinked || !req->rq_reply_unlinked ||
2197 req->rq_receiving_reply;
2198 spin_unlock(&req->rq_lock);
2199 return rc;
2200}
2201
2202static inline void
2203ptlrpc_client_wake_req(struct ptlrpc_request *req)
2204{
2205 if (!req->rq_set)
2206 wake_up(&req->rq_reply_waitq);
2207 else
2208 wake_up(&req->rq_set->set_waitq);
2209}
2210
2211static inline void
2212ptlrpc_rs_addref(struct ptlrpc_reply_state *rs)
2213{
2214 LASSERT(atomic_read(&rs->rs_refcount) > 0);
2215 atomic_inc(&rs->rs_refcount);
2216}
2217
2218static inline void
2219ptlrpc_rs_decref(struct ptlrpc_reply_state *rs)
2220{
2221 LASSERT(atomic_read(&rs->rs_refcount) > 0);
2222 if (atomic_dec_and_test(&rs->rs_refcount))
2223 lustre_free_reply_state(rs);
2224}
2225
2226
2227static inline void ptlrpc_req_drop_rs(struct ptlrpc_request *req)
2228{
2229 if (!req->rq_reply_state)
2230 return;
2231 ptlrpc_rs_decref(req->rq_reply_state);
2232 req->rq_reply_state = NULL;
2233 req->rq_repmsg = NULL;
2234}
2235
2236static inline __u32 lustre_request_magic(struct ptlrpc_request *req)
2237{
2238 return lustre_msg_get_magic(req->rq_reqmsg);
2239}
2240
2241static inline int ptlrpc_req_get_repsize(struct ptlrpc_request *req)
2242{
2243 switch (req->rq_reqmsg->lm_magic) {
2244 case LUSTRE_MSG_MAGIC_V2:
2245 return req->rq_reqmsg->lm_repsize;
2246 default:
2247 LASSERTF(0, "incorrect message magic: %08x\n",
2248 req->rq_reqmsg->lm_magic);
2249 return -EFAULT;
2250 }
2251}
2252
2253static inline int ptlrpc_send_limit_expired(struct ptlrpc_request *req)
2254{
2255 if (req->rq_delay_limit != 0 &&
2256 time_before(cfs_time_add(req->rq_queued_time,
2257 req->rq_delay_limit * HZ),
2258 cfs_time_current())) {
2259 return 1;
2260 }
2261 return 0;
2262}
2263
2264static inline int ptlrpc_no_resend(struct ptlrpc_request *req)
2265{
2266 if (!req->rq_no_resend && ptlrpc_send_limit_expired(req)) {
2267 spin_lock(&req->rq_lock);
2268 req->rq_no_resend = 1;
2269 spin_unlock(&req->rq_lock);
2270 }
2271 return req->rq_no_resend;
2272}
2273
2274static inline int
2275ptlrpc_server_get_timeout(struct ptlrpc_service_part *svcpt)
2276{
2277 int at = AT_OFF ? 0 : at_get(&svcpt->scp_at_estimate);
2278
2279 return svcpt->scp_service->srv_watchdog_factor *
2280 max_t(int, at, obd_timeout);
2281}
2282
2283static inline struct ptlrpc_service *
2284ptlrpc_req2svc(struct ptlrpc_request *req)
2285{
2286 return req->rq_rqbd->rqbd_svcpt->scp_service;
2287}
2288
2289
2290
2291
2292
2293
2294int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg);
2295int client_obd_cleanup(struct obd_device *obddev);
2296int client_connect_import(const struct lu_env *env,
2297 struct obd_export **exp, struct obd_device *obd,
2298 struct obd_uuid *cluuid, struct obd_connect_data *,
2299 void *localdata);
2300int client_disconnect_export(struct obd_export *exp);
2301int client_import_add_conn(struct obd_import *imp, struct obd_uuid *uuid,
2302 int priority);
2303int client_import_del_conn(struct obd_import *imp, struct obd_uuid *uuid);
2304int client_import_find_conn(struct obd_import *imp, lnet_nid_t peer,
2305 struct obd_uuid *uuid);
2306int import_set_conn_priority(struct obd_import *imp, struct obd_uuid *uuid);
2307void client_destroy_import(struct obd_import *imp);
2308
2309
2310
2311
2312
2313
2314
2315enum timeout_event {
2316 TIMEOUT_GRANT = 1
2317};
2318
2319struct timeout_item;
2320typedef int (*timeout_cb_t)(struct timeout_item *, void *);
2321int ptlrpc_pinger_add_import(struct obd_import *imp);
2322int ptlrpc_pinger_del_import(struct obd_import *imp);
2323int ptlrpc_add_timeout_client(int time, enum timeout_event event,
2324 timeout_cb_t cb, void *data,
2325 struct list_head *obd_list);
2326int ptlrpc_del_timeout_client(struct list_head *obd_list,
2327 enum timeout_event event);
2328struct ptlrpc_request *ptlrpc_prep_ping(struct obd_import *imp);
2329int ptlrpc_obd_ping(struct obd_device *obd);
2330void ptlrpc_pinger_ir_up(void);
2331void ptlrpc_pinger_ir_down(void);
2332
2333int ptlrpc_pinger_suppress_pings(void);
2334
2335
2336void ptlrpcd_stop(struct ptlrpcd_ctl *pc, int force);
2337void ptlrpcd_free(struct ptlrpcd_ctl *pc);
2338void ptlrpcd_wake(struct ptlrpc_request *req);
2339void ptlrpcd_add_req(struct ptlrpc_request *req);
2340int ptlrpcd_addref(void);
2341void ptlrpcd_decref(void);
2342
2343
2344
2345
2346
2347
2348const char *ll_opcode2str(__u32 opcode);
2349void ptlrpc_lprocfs_register_obd(struct obd_device *obd);
2350void ptlrpc_lprocfs_unregister_obd(struct obd_device *obd);
2351void ptlrpc_lprocfs_brw(struct ptlrpc_request *req, int bytes);
2352
2353
2354
2355extern struct llog_operations llog_client_ops;
2356
2357
2358#endif
2359
2360