linux/drivers/staging/lustre/lnet/selftest/console.c
<<
>>
Prefs
   1/*
   2 * GPL HEADER START
   3 *
   4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 only,
   8 * as published by the Free Software Foundation.
   9 *
  10 * This program is distributed in the hope that it will be useful, but
  11 * WITHOUT ANY WARRANTY; without even the implied warranty of
  12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 * General Public License version 2 for more details (a copy is included
  14 * in the LICENSE file that accompanied this code).
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * version 2 along with this program; If not, see
  18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19 *
  20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21 * CA 95054 USA or visit www.sun.com if you need additional information or
  22 * have any questions.
  23 *
  24 * GPL HEADER END
  25 */
  26/*
  27 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28 * Use is subject to license terms.
  29 *
  30 * Copyright (c) 2012, Intel Corporation.
  31 */
  32/*
  33 * This file is part of Lustre, http://www.lustre.org/
  34 * Lustre is a trademark of Sun Microsystems, Inc.
  35 *
  36 * lnet/selftest/conctl.c
  37 *
  38 * Infrastructure of LST console
  39 *
  40 * Author: Liang Zhen <liangzhen@clusterfs.com>
  41 */
  42
  43#include "../../include/linux/libcfs/libcfs.h"
  44#include "../../include/linux/lnet/lib-lnet.h"
  45#include "console.h"
  46#include "conrpc.h"
  47
  48#define LST_NODE_STATE_COUNTER(nd, p)                   \
  49do {                                                    \
  50        if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  51                (p)->nle_nactive++;                     \
  52        else if ((nd)->nd_state == LST_NODE_BUSY)       \
  53                (p)->nle_nbusy++;                       \
  54        else if ((nd)->nd_state == LST_NODE_DOWN)       \
  55                (p)->nle_ndown++;                       \
  56        else                                            \
  57                (p)->nle_nunknown++;                    \
  58        (p)->nle_nnode++;                               \
  59} while (0)
  60
  61lstcon_session_t console_session;
  62
  63static void
  64lstcon_node_get(lstcon_node_t *nd)
  65{
  66        LASSERT(nd->nd_ref >= 1);
  67
  68        nd->nd_ref++;
  69}
  70
  71static int
  72lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  73{
  74        lstcon_ndlink_t *ndl;
  75        unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  76
  77        LASSERT(id.nid != LNET_NID_ANY);
  78
  79        list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) {
  80                if (ndl->ndl_node->nd_id.nid != id.nid ||
  81                    ndl->ndl_node->nd_id.pid != id.pid)
  82                        continue;
  83
  84                lstcon_node_get(ndl->ndl_node);
  85                *ndpp = ndl->ndl_node;
  86                return 0;
  87        }
  88
  89        if (!create)
  90                return -ENOENT;
  91
  92        LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  93        if (*ndpp == NULL)
  94                return -ENOMEM;
  95
  96        ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  97
  98        ndl->ndl_node = *ndpp;
  99
 100        ndl->ndl_node->nd_ref   = 1;
 101        ndl->ndl_node->nd_id    = id;
 102        ndl->ndl_node->nd_stamp = cfs_time_current();
 103        ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 104        ndl->ndl_node->nd_timeout = 0;
 105        memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
 106
 107        /* queued in global hash & list, no refcount is taken by
 108         * global hash & list, if caller release his refcount,
 109         * node will be released */
 110        list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 111        list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 112
 113        return 0;
 114}
 115
 116static void
 117lstcon_node_put(lstcon_node_t *nd)
 118{
 119        lstcon_ndlink_t *ndl;
 120
 121        LASSERT(nd->nd_ref > 0);
 122
 123        if (--nd->nd_ref > 0)
 124                return;
 125
 126        ndl = (lstcon_ndlink_t *)(nd + 1);
 127
 128        LASSERT(!list_empty(&ndl->ndl_link));
 129        LASSERT(!list_empty(&ndl->ndl_hlink));
 130
 131        /* remove from session */
 132        list_del(&ndl->ndl_link);
 133        list_del(&ndl->ndl_hlink);
 134
 135        LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 136}
 137
 138static int
 139lstcon_ndlink_find(struct list_head *hash,
 140                   lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 141{
 142        unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 143        lstcon_ndlink_t *ndl;
 144        lstcon_node_t *nd;
 145        int rc;
 146
 147        if (id.nid == LNET_NID_ANY)
 148                return -EINVAL;
 149
 150        /* search in hash */
 151        list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 152                if (ndl->ndl_node->nd_id.nid != id.nid ||
 153                    ndl->ndl_node->nd_id.pid != id.pid)
 154                        continue;
 155
 156                *ndlpp = ndl;
 157                return 0;
 158        }
 159
 160        if (create == 0)
 161                return -ENOENT;
 162
 163        /* find or create in session hash */
 164        rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 165        if (rc != 0)
 166                return rc;
 167
 168        LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 169        if (ndl == NULL) {
 170                lstcon_node_put(nd);
 171                return -ENOMEM;
 172        }
 173
 174        *ndlpp = ndl;
 175
 176        ndl->ndl_node = nd;
 177        INIT_LIST_HEAD(&ndl->ndl_link);
 178        list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 179
 180        return  0;
 181}
 182
 183static void
 184lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 185{
 186        LASSERT(list_empty(&ndl->ndl_link));
 187        LASSERT(!list_empty(&ndl->ndl_hlink));
 188
 189        list_del(&ndl->ndl_hlink); /* delete from hash */
 190        lstcon_node_put(ndl->ndl_node);
 191
 192        LIBCFS_FREE(ndl, sizeof(*ndl));
 193}
 194
 195static int
 196lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 197{
 198        lstcon_group_t *grp;
 199        int i;
 200
 201        LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 202                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 203        if (grp == NULL)
 204                return -ENOMEM;
 205
 206        grp->grp_ref = 1;
 207        if (name != NULL)
 208                strcpy(grp->grp_name, name);
 209
 210        INIT_LIST_HEAD(&grp->grp_link);
 211        INIT_LIST_HEAD(&grp->grp_ndl_list);
 212        INIT_LIST_HEAD(&grp->grp_trans_list);
 213
 214        for (i = 0; i < LST_NODE_HASHSIZE; i++)
 215                INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 216
 217        *grpp = grp;
 218
 219        return 0;
 220}
 221
 222static void
 223lstcon_group_addref(lstcon_group_t *grp)
 224{
 225        grp->grp_ref++;
 226}
 227
 228static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 229
 230static void
 231lstcon_group_drain(lstcon_group_t *grp, int keep)
 232{
 233        lstcon_ndlink_t *ndl;
 234        lstcon_ndlink_t *tmp;
 235
 236        list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 237                if ((ndl->ndl_node->nd_state & keep) == 0)
 238                        lstcon_group_ndlink_release(grp, ndl);
 239        }
 240}
 241
 242static void
 243lstcon_group_decref(lstcon_group_t *grp)
 244{
 245        int i;
 246
 247        if (--grp->grp_ref > 0)
 248                return;
 249
 250        if (!list_empty(&grp->grp_link))
 251                list_del(&grp->grp_link);
 252
 253        lstcon_group_drain(grp, 0);
 254
 255        for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 256                LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 257        }
 258
 259        LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 260                                  grp_ndl_hash[LST_NODE_HASHSIZE]));
 261}
 262
 263static int
 264lstcon_group_find(const char *name, lstcon_group_t **grpp)
 265{
 266        lstcon_group_t *grp;
 267
 268        list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 269                if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 270                        continue;
 271
 272                lstcon_group_addref(grp);  /* +1 ref for caller */
 273                *grpp = grp;
 274                return 0;
 275        }
 276
 277        return -ENOENT;
 278}
 279
 280static int
 281lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 282                         lstcon_ndlink_t **ndlpp, int create)
 283{
 284        int rc;
 285
 286        rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 287        if (rc != 0)
 288                return rc;
 289
 290        if (!list_empty(&(*ndlpp)->ndl_link))
 291                return 0;
 292
 293        list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 294        grp->grp_nnode++;
 295
 296        return 0;
 297}
 298
 299static void
 300lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 301{
 302        list_del_init(&ndl->ndl_link);
 303        lstcon_ndlink_release(ndl);
 304        grp->grp_nnode--;
 305}
 306
 307static void
 308lstcon_group_ndlink_move(lstcon_group_t *old,
 309                         lstcon_group_t *new, lstcon_ndlink_t *ndl)
 310{
 311        unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 312                           LST_NODE_HASHSIZE;
 313
 314        list_del(&ndl->ndl_hlink);
 315        list_del(&ndl->ndl_link);
 316        old->grp_nnode--;
 317
 318        list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 319        list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 320        new->grp_nnode++;
 321}
 322
 323static void
 324lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 325{
 326        lstcon_ndlink_t *ndl;
 327
 328        while (!list_empty(&old->grp_ndl_list)) {
 329                ndl = list_entry(old->grp_ndl_list.next,
 330                                     lstcon_ndlink_t, ndl_link);
 331                lstcon_group_ndlink_move(old, new, ndl);
 332        }
 333}
 334
 335static int
 336lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 337{
 338        lstcon_group_t *grp = (lstcon_group_t *)arg;
 339
 340        switch (transop) {
 341        case LST_TRANS_SESNEW:
 342                if (nd->nd_state == LST_NODE_ACTIVE)
 343                        return 0;
 344                break;
 345
 346        case LST_TRANS_SESEND:
 347                if (nd->nd_state != LST_NODE_ACTIVE)
 348                        return 0;
 349
 350                if (grp != NULL && nd->nd_ref > 1)
 351                        return 0;
 352                break;
 353
 354        case LST_TRANS_SESQRY:
 355                break;
 356
 357        default:
 358                LBUG();
 359        }
 360
 361        return 1;
 362}
 363
 364static int
 365lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 366                      lstcon_rpc_ent_t *ent_up)
 367{
 368        srpc_debug_reply_t *rep;
 369
 370        switch (transop) {
 371        case LST_TRANS_SESNEW:
 372        case LST_TRANS_SESEND:
 373                return 0;
 374
 375        case LST_TRANS_SESQRY:
 376                rep = &msg->msg_body.dbg_reply;
 377
 378                if (copy_to_user(&ent_up->rpe_priv[0],
 379                                     &rep->dbg_timeout, sizeof(int)) ||
 380                    copy_to_user(&ent_up->rpe_payload[0],
 381                                     &rep->dbg_name, LST_NAME_SIZE))
 382                        return -EFAULT;
 383
 384                return 0;
 385
 386        default:
 387                LBUG();
 388        }
 389
 390        return 0;
 391}
 392
 393static int
 394lstcon_group_nodes_add(lstcon_group_t *grp,
 395                       int count, lnet_process_id_t *ids_up,
 396                       unsigned *featp, struct list_head *result_up)
 397{
 398        lstcon_rpc_trans_t *trans;
 399        lstcon_ndlink_t *ndl;
 400        lstcon_group_t *tmp;
 401        lnet_process_id_t id;
 402        int i;
 403        int rc;
 404
 405        rc = lstcon_group_alloc(NULL, &tmp);
 406        if (rc != 0) {
 407                CERROR("Out of memory\n");
 408                return -ENOMEM;
 409        }
 410
 411        for (i = 0 ; i < count; i++) {
 412                if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 413                        rc = -EFAULT;
 414                        break;
 415                }
 416
 417                /* skip if it's in this group already */
 418                rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 419                if (rc == 0)
 420                        continue;
 421
 422                /* add to tmp group */
 423                rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 424                if (rc != 0) {
 425                        CERROR("Can't create ndlink, out of memory\n");
 426                        break;
 427                }
 428        }
 429
 430        if (rc != 0) {
 431                lstcon_group_decref(tmp);
 432                return rc;
 433        }
 434
 435        rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 436                                     &tmp->grp_trans_list, LST_TRANS_SESNEW,
 437                                     tmp, lstcon_sesrpc_condition, &trans);
 438        if (rc != 0) {
 439                CERROR("Can't create transaction: %d\n", rc);
 440                lstcon_group_decref(tmp);
 441                return rc;
 442        }
 443
 444        /* post all RPCs */
 445        lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 446
 447        rc = lstcon_rpc_trans_interpreter(trans, result_up,
 448                                          lstcon_sesrpc_readent);
 449        *featp = trans->tas_features;
 450
 451        /* destroy all RPGs */
 452        lstcon_rpc_trans_destroy(trans);
 453
 454        lstcon_group_move(tmp, grp);
 455        lstcon_group_decref(tmp);
 456
 457        return rc;
 458}
 459
 460static int
 461lstcon_group_nodes_remove(lstcon_group_t *grp,
 462                          int count, lnet_process_id_t *ids_up,
 463                          struct list_head *result_up)
 464{
 465        lstcon_rpc_trans_t *trans;
 466        lstcon_ndlink_t *ndl;
 467        lstcon_group_t *tmp;
 468        lnet_process_id_t id;
 469        int rc;
 470        int i;
 471
 472        /* End session and remove node from the group */
 473
 474        rc = lstcon_group_alloc(NULL, &tmp);
 475        if (rc != 0) {
 476                CERROR("Out of memory\n");
 477                return -ENOMEM;
 478        }
 479
 480        for (i = 0; i < count; i++) {
 481                if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 482                        rc = -EFAULT;
 483                        goto error;
 484                }
 485
 486                /* move node to tmp group */
 487                if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 488                        lstcon_group_ndlink_move(grp, tmp, ndl);
 489        }
 490
 491        rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 492                                     &tmp->grp_trans_list, LST_TRANS_SESEND,
 493                                     tmp, lstcon_sesrpc_condition, &trans);
 494        if (rc != 0) {
 495                CERROR("Can't create transaction: %d\n", rc);
 496                goto error;
 497        }
 498
 499        lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 500
 501        rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 502
 503        lstcon_rpc_trans_destroy(trans);
 504        /* release nodes anyway, because we can't rollback status */
 505        lstcon_group_decref(tmp);
 506
 507        return rc;
 508error:
 509        lstcon_group_move(tmp, grp);
 510        lstcon_group_decref(tmp);
 511
 512        return rc;
 513}
 514
 515int
 516lstcon_group_add(char *name)
 517{
 518        lstcon_group_t *grp;
 519        int rc;
 520
 521        rc = (lstcon_group_find(name, &grp) == 0) ? -EEXIST : 0;
 522        if (rc != 0) {
 523                /* find a group with same name */
 524                lstcon_group_decref(grp);
 525                return rc;
 526        }
 527
 528        rc = lstcon_group_alloc(name, &grp);
 529        if (rc != 0) {
 530                CERROR("Can't allocate descriptor for group %s\n", name);
 531                return -ENOMEM;
 532        }
 533
 534        list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 535
 536        return rc;
 537}
 538
 539int
 540lstcon_nodes_add(char *name, int count, lnet_process_id_t *ids_up,
 541                 unsigned *featp, struct list_head *result_up)
 542{
 543        lstcon_group_t *grp;
 544        int rc;
 545
 546        LASSERT(count > 0);
 547        LASSERT(ids_up != NULL);
 548
 549        rc = lstcon_group_find(name, &grp);
 550        if (rc != 0) {
 551                CDEBUG(D_NET, "Can't find group %s\n", name);
 552                return rc;
 553        }
 554
 555        if (grp->grp_ref > 2) {
 556                /* referred by other threads or test */
 557                CDEBUG(D_NET, "Group %s is busy\n", name);
 558                lstcon_group_decref(grp);
 559
 560                return -EBUSY;
 561        }
 562
 563        rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 564
 565        lstcon_group_decref(grp);
 566
 567        return rc;
 568}
 569
 570int
 571lstcon_group_del(char *name)
 572{
 573        lstcon_rpc_trans_t *trans;
 574        lstcon_group_t *grp;
 575        int rc;
 576
 577        rc = lstcon_group_find(name, &grp);
 578        if (rc != 0) {
 579                CDEBUG(D_NET, "Can't find group: %s\n", name);
 580                return rc;
 581        }
 582
 583        if (grp->grp_ref > 2) {
 584                /* referred by others threads or test */
 585                CDEBUG(D_NET, "Group %s is busy\n", name);
 586                lstcon_group_decref(grp);
 587                return -EBUSY;
 588        }
 589
 590        rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 591                                     &grp->grp_trans_list, LST_TRANS_SESEND,
 592                                     grp, lstcon_sesrpc_condition, &trans);
 593        if (rc != 0) {
 594                CERROR("Can't create transaction: %d\n", rc);
 595                lstcon_group_decref(grp);
 596                return rc;
 597        }
 598
 599        lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 600
 601        lstcon_rpc_trans_destroy(trans);
 602
 603        lstcon_group_decref(grp);
 604        /* -ref for session, it's destroyed,
 605         * status can't be rolled back, destroy group anyway */
 606        lstcon_group_decref(grp);
 607
 608        return rc;
 609}
 610
 611int
 612lstcon_group_clean(char *name, int args)
 613{
 614        lstcon_group_t *grp = NULL;
 615        int rc;
 616
 617        rc = lstcon_group_find(name, &grp);
 618        if (rc != 0) {
 619                CDEBUG(D_NET, "Can't find group %s\n", name);
 620                return rc;
 621        }
 622
 623        if (grp->grp_ref > 2) {
 624                /* referred by test */
 625                CDEBUG(D_NET, "Group %s is busy\n", name);
 626                lstcon_group_decref(grp);
 627                return -EBUSY;
 628        }
 629
 630        args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 631                LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 632
 633        lstcon_group_drain(grp, args);
 634
 635        lstcon_group_decref(grp);
 636        /* release empty group */
 637        if (list_empty(&grp->grp_ndl_list))
 638                lstcon_group_decref(grp);
 639
 640        return 0;
 641}
 642
 643int
 644lstcon_nodes_remove(char *name, int count,
 645                    lnet_process_id_t *ids_up, struct list_head *result_up)
 646{
 647        lstcon_group_t *grp = NULL;
 648        int rc;
 649
 650        rc = lstcon_group_find(name, &grp);
 651        if (rc != 0) {
 652                CDEBUG(D_NET, "Can't find group: %s\n", name);
 653                return rc;
 654        }
 655
 656        if (grp->grp_ref > 2) {
 657                /* referred by test */
 658                CDEBUG(D_NET, "Group %s is busy\n", name);
 659                lstcon_group_decref(grp);
 660                return -EBUSY;
 661        }
 662
 663        rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 664
 665        lstcon_group_decref(grp);
 666        /* release empty group */
 667        if (list_empty(&grp->grp_ndl_list))
 668                lstcon_group_decref(grp);
 669
 670        return rc;
 671}
 672
 673int
 674lstcon_group_refresh(char *name, struct list_head *result_up)
 675{
 676        lstcon_rpc_trans_t *trans;
 677        lstcon_group_t *grp;
 678        int rc;
 679
 680        rc = lstcon_group_find(name, &grp);
 681        if (rc != 0) {
 682                CDEBUG(D_NET, "Can't find group: %s\n", name);
 683                return rc;
 684        }
 685
 686        if (grp->grp_ref > 2) {
 687                /* referred by test */
 688                CDEBUG(D_NET, "Group %s is busy\n", name);
 689                lstcon_group_decref(grp);
 690                return -EBUSY;
 691        }
 692
 693        /* re-invite all inactive nodes int the group */
 694        rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 695                                     &grp->grp_trans_list, LST_TRANS_SESNEW,
 696                                     grp, lstcon_sesrpc_condition, &trans);
 697        if (rc != 0) {
 698                /* local error, return */
 699                CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 700                lstcon_group_decref(grp);
 701                return rc;
 702        }
 703
 704        lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 705
 706        rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 707
 708        lstcon_rpc_trans_destroy(trans);
 709        /* -ref for me */
 710        lstcon_group_decref(grp);
 711
 712        return rc;
 713}
 714
 715int
 716lstcon_group_list(int index, int len, char *name_up)
 717{
 718        lstcon_group_t *grp;
 719
 720        LASSERT(index >= 0);
 721        LASSERT(name_up != NULL);
 722
 723        list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 724                if (index-- == 0) {
 725                        return copy_to_user(name_up, grp->grp_name, len) ?
 726                               -EFAULT : 0;
 727                }
 728        }
 729
 730        return -ENOENT;
 731}
 732
 733static int
 734lstcon_nodes_getent(struct list_head *head, int *index_p,
 735                    int *count_p, lstcon_node_ent_t *dents_up)
 736{
 737        lstcon_ndlink_t *ndl;
 738        lstcon_node_t *nd;
 739        int count = 0;
 740        int index = 0;
 741
 742        LASSERT(index_p != NULL && count_p != NULL);
 743        LASSERT(dents_up != NULL);
 744        LASSERT(*index_p >= 0);
 745        LASSERT(*count_p > 0);
 746
 747        list_for_each_entry(ndl, head, ndl_link) {
 748                if (index++ < *index_p)
 749                        continue;
 750
 751                if (count >= *count_p)
 752                        break;
 753
 754                nd = ndl->ndl_node;
 755                if (copy_to_user(&dents_up[count].nde_id,
 756                                     &nd->nd_id, sizeof(nd->nd_id)) ||
 757                    copy_to_user(&dents_up[count].nde_state,
 758                                     &nd->nd_state, sizeof(nd->nd_state)))
 759                        return -EFAULT;
 760
 761                count++;
 762        }
 763
 764        if (index <= *index_p)
 765                return -ENOENT;
 766
 767        *count_p = count;
 768        *index_p = index;
 769
 770        return 0;
 771}
 772
 773int
 774lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p,
 775                  int *index_p, int *count_p, lstcon_node_ent_t *dents_up)
 776{
 777        lstcon_ndlist_ent_t *gentp;
 778        lstcon_group_t *grp;
 779        lstcon_ndlink_t *ndl;
 780        int rc;
 781
 782        rc = lstcon_group_find(name, &grp);
 783        if (rc != 0) {
 784                CDEBUG(D_NET, "Can't find group %s\n", name);
 785                return rc;
 786        }
 787
 788        if (dents_up) {
 789                /* verbose query */
 790                rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 791                                         index_p, count_p, dents_up);
 792                lstcon_group_decref(grp);
 793
 794                return rc;
 795        }
 796
 797        /* non-verbose query */
 798        LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 799        if (gentp == NULL) {
 800                CERROR("Can't allocate ndlist_ent\n");
 801                lstcon_group_decref(grp);
 802
 803                return -ENOMEM;
 804        }
 805
 806        list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 807                LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 808
 809        rc = copy_to_user(gents_p, gentp,
 810                              sizeof(lstcon_ndlist_ent_t)) ? -EFAULT : 0;
 811
 812        LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 813
 814        lstcon_group_decref(grp);
 815
 816        return 0;
 817}
 818
 819static int
 820lstcon_batch_find(const char *name, lstcon_batch_t **batpp)
 821{
 822        lstcon_batch_t *bat;
 823
 824        list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 825                if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 826                        *batpp = bat;
 827                        return 0;
 828                }
 829        }
 830
 831        return -ENOENT;
 832}
 833
 834int
 835lstcon_batch_add(char *name)
 836{
 837        lstcon_batch_t *bat;
 838        int i;
 839        int rc;
 840
 841        rc = (lstcon_batch_find(name, &bat) == 0) ? -EEXIST : 0;
 842        if (rc != 0) {
 843                CDEBUG(D_NET, "Batch %s already exists\n", name);
 844                return rc;
 845        }
 846
 847        LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 848        if (bat == NULL) {
 849                CERROR("Can't allocate descriptor for batch %s\n", name);
 850                return -ENOMEM;
 851        }
 852
 853        LIBCFS_ALLOC(bat->bat_cli_hash,
 854                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
 855        if (bat->bat_cli_hash == NULL) {
 856                CERROR("Can't allocate hash for batch %s\n", name);
 857                LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 858
 859                return -ENOMEM;
 860        }
 861
 862        LIBCFS_ALLOC(bat->bat_srv_hash,
 863                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
 864        if (bat->bat_srv_hash == NULL) {
 865                CERROR("Can't allocate hash for batch %s\n", name);
 866                LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 867                LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 868
 869                return -ENOMEM;
 870        }
 871
 872        strcpy(bat->bat_name, name);
 873        bat->bat_hdr.tsb_index = 0;
 874        bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 875
 876        bat->bat_ntest = 0;
 877        bat->bat_state = LST_BATCH_IDLE;
 878
 879        INIT_LIST_HEAD(&bat->bat_cli_list);
 880        INIT_LIST_HEAD(&bat->bat_srv_list);
 881        INIT_LIST_HEAD(&bat->bat_test_list);
 882        INIT_LIST_HEAD(&bat->bat_trans_list);
 883
 884        for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 885                INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 886                INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 887        }
 888
 889        list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 890
 891        return rc;
 892}
 893
 894int
 895lstcon_batch_list(int index, int len, char *name_up)
 896{
 897        lstcon_batch_t *bat;
 898
 899        LASSERT(name_up != NULL);
 900        LASSERT(index >= 0);
 901
 902        list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 903                if (index-- == 0) {
 904                        return copy_to_user(name_up, bat->bat_name, len) ?
 905                               -EFAULT : 0;
 906                }
 907        }
 908
 909        return -ENOENT;
 910}
 911
 912int
 913lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server,
 914                  int testidx, int *index_p, int *ndent_p,
 915                  lstcon_node_ent_t *dents_up)
 916{
 917        lstcon_test_batch_ent_t *entp;
 918        struct list_head *clilst;
 919        struct list_head *srvlst;
 920        lstcon_test_t *test = NULL;
 921        lstcon_batch_t *bat;
 922        lstcon_ndlink_t *ndl;
 923        int rc;
 924
 925        rc = lstcon_batch_find(name, &bat);
 926        if (rc != 0) {
 927                CDEBUG(D_NET, "Can't find batch %s\n", name);
 928                return -ENOENT;
 929        }
 930
 931        if (testidx > 0) {
 932                /* query test, test index start from 1 */
 933                list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 934                        if (testidx-- == 1)
 935                                break;
 936                }
 937
 938                if (testidx > 0) {
 939                        CDEBUG(D_NET, "Can't find specified test in batch\n");
 940                        return -ENOENT;
 941                }
 942        }
 943
 944        clilst = (test == NULL) ? &bat->bat_cli_list :
 945                                  &test->tes_src_grp->grp_ndl_list;
 946        srvlst = (test == NULL) ? &bat->bat_srv_list :
 947                                  &test->tes_dst_grp->grp_ndl_list;
 948
 949        if (dents_up != NULL) {
 950                rc = lstcon_nodes_getent((server ? srvlst : clilst),
 951                                         index_p, ndent_p, dents_up);
 952                return rc;
 953        }
 954
 955        /* non-verbose query */
 956        LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 957        if (entp == NULL)
 958                return -ENOMEM;
 959
 960        if (test == NULL) {
 961                entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 962                entp->u.tbe_batch.bae_state = bat->bat_state;
 963
 964        } else {
 965
 966                entp->u.tbe_test.tse_type   = test->tes_type;
 967                entp->u.tbe_test.tse_loop   = test->tes_loop;
 968                entp->u.tbe_test.tse_concur = test->tes_concur;
 969        }
 970
 971        list_for_each_entry(ndl, clilst, ndl_link)
 972                LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 973
 974        list_for_each_entry(ndl, srvlst, ndl_link)
 975                LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 976
 977        rc = copy_to_user(ent_up, entp,
 978                              sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 979
 980        LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 981
 982        return rc;
 983}
 984
 985static int
 986lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 987{
 988        switch (transop) {
 989        case LST_TRANS_TSBRUN:
 990                if (nd->nd_state != LST_NODE_ACTIVE)
 991                        return -ENETDOWN;
 992                break;
 993
 994        case LST_TRANS_TSBSTOP:
 995                if (nd->nd_state != LST_NODE_ACTIVE)
 996                        return 0;
 997                break;
 998
 999        case LST_TRANS_TSBCLIQRY:
1000        case LST_TRANS_TSBSRVQRY:
1001                break;
1002        }
1003
1004        return 1;
1005}
1006
1007static int
1008lstcon_batch_op(lstcon_batch_t *bat, int transop,
1009                struct list_head *result_up)
1010{
1011        lstcon_rpc_trans_t *trans;
1012        int rc;
1013
1014        rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1015                                     &bat->bat_trans_list, transop,
1016                                     bat, lstcon_batrpc_condition, &trans);
1017        if (rc != 0) {
1018                CERROR("Can't create transaction: %d\n", rc);
1019                return rc;
1020        }
1021
1022        lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1023
1024        rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1025
1026        lstcon_rpc_trans_destroy(trans);
1027
1028        return rc;
1029}
1030
1031int
1032lstcon_batch_run(char *name, int timeout, struct list_head *result_up)
1033{
1034        lstcon_batch_t *bat;
1035        int rc;
1036
1037        if (lstcon_batch_find(name, &bat) != 0) {
1038                CDEBUG(D_NET, "Can't find batch %s\n", name);
1039                return -ENOENT;
1040        }
1041
1042        bat->bat_arg = timeout;
1043
1044        rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1045
1046        /* mark batch as running if it's started in any node */
1047        if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1048                bat->bat_state = LST_BATCH_RUNNING;
1049
1050        return rc;
1051}
1052
1053int
1054lstcon_batch_stop(char *name, int force, struct list_head *result_up)
1055{
1056        lstcon_batch_t *bat;
1057        int rc;
1058
1059        if (lstcon_batch_find(name, &bat) != 0) {
1060                CDEBUG(D_NET, "Can't find batch %s\n", name);
1061                return -ENOENT;
1062        }
1063
1064        bat->bat_arg = force;
1065
1066        rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1067
1068        /* mark batch as stopped if all RPCs finished */
1069        if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1070                bat->bat_state = LST_BATCH_IDLE;
1071
1072        return rc;
1073}
1074
1075static void
1076lstcon_batch_destroy(lstcon_batch_t *bat)
1077{
1078        lstcon_ndlink_t *ndl;
1079        lstcon_test_t *test;
1080        int i;
1081
1082        list_del(&bat->bat_link);
1083
1084        while (!list_empty(&bat->bat_test_list)) {
1085                test = list_entry(bat->bat_test_list.next,
1086                                      lstcon_test_t, tes_link);
1087                LASSERT(list_empty(&test->tes_trans_list));
1088
1089                list_del(&test->tes_link);
1090
1091                lstcon_group_decref(test->tes_src_grp);
1092                lstcon_group_decref(test->tes_dst_grp);
1093
1094                LIBCFS_FREE(test, offsetof(lstcon_test_t,
1095                                           tes_param[test->tes_paramlen]));
1096        }
1097
1098        LASSERT(list_empty(&bat->bat_trans_list));
1099
1100        while (!list_empty(&bat->bat_cli_list)) {
1101                ndl = list_entry(bat->bat_cli_list.next,
1102                                     lstcon_ndlink_t, ndl_link);
1103                list_del_init(&ndl->ndl_link);
1104
1105                lstcon_ndlink_release(ndl);
1106        }
1107
1108        while (!list_empty(&bat->bat_srv_list)) {
1109                ndl = list_entry(bat->bat_srv_list.next,
1110                                     lstcon_ndlink_t, ndl_link);
1111                list_del_init(&ndl->ndl_link);
1112
1113                lstcon_ndlink_release(ndl);
1114        }
1115
1116        for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1117                LASSERT(list_empty(&bat->bat_cli_hash[i]));
1118                LASSERT(list_empty(&bat->bat_srv_hash[i]));
1119        }
1120
1121        LIBCFS_FREE(bat->bat_cli_hash,
1122                    sizeof(struct list_head) * LST_NODE_HASHSIZE);
1123        LIBCFS_FREE(bat->bat_srv_hash,
1124                    sizeof(struct list_head) * LST_NODE_HASHSIZE);
1125        LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1126}
1127
1128static int
1129lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1130{
1131        lstcon_test_t *test;
1132        lstcon_batch_t *batch;
1133        lstcon_ndlink_t *ndl;
1134        struct list_head *hash;
1135        struct list_head *head;
1136
1137        test = (lstcon_test_t *)arg;
1138        LASSERT(test != NULL);
1139
1140        batch = test->tes_batch;
1141        LASSERT(batch != NULL);
1142
1143        if (test->tes_oneside &&
1144            transop == LST_TRANS_TSBSRVADD)
1145                return 0;
1146
1147        if (nd->nd_state != LST_NODE_ACTIVE)
1148                return -ENETDOWN;
1149
1150        if (transop == LST_TRANS_TSBCLIADD) {
1151                hash = batch->bat_cli_hash;
1152                head = &batch->bat_cli_list;
1153
1154        } else {
1155                LASSERT(transop == LST_TRANS_TSBSRVADD);
1156
1157                hash = batch->bat_srv_hash;
1158                head = &batch->bat_srv_list;
1159        }
1160
1161        LASSERT(nd->nd_id.nid != LNET_NID_ANY);
1162
1163        if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1164                return -ENOMEM;
1165
1166        if (list_empty(&ndl->ndl_link))
1167                list_add_tail(&ndl->ndl_link, head);
1168
1169        return 1;
1170}
1171
1172static int
1173lstcon_test_nodes_add(lstcon_test_t *test, struct list_head *result_up)
1174{
1175        lstcon_rpc_trans_t *trans;
1176        lstcon_group_t *grp;
1177        int transop;
1178        int rc;
1179
1180        LASSERT(test->tes_src_grp != NULL);
1181        LASSERT(test->tes_dst_grp != NULL);
1182
1183        transop = LST_TRANS_TSBSRVADD;
1184        grp  = test->tes_dst_grp;
1185again:
1186        rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1187                                     &test->tes_trans_list, transop,
1188                                     test, lstcon_testrpc_condition, &trans);
1189        if (rc != 0) {
1190                CERROR("Can't create transaction: %d\n", rc);
1191                return rc;
1192        }
1193
1194        lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1195
1196        if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1197            lstcon_trans_stat()->trs_fwk_errno != 0) {
1198                lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1199
1200                lstcon_rpc_trans_destroy(trans);
1201                /* return if any error */
1202                CDEBUG(D_NET, "Failed to add test %s, RPC error %d, framework error %d\n",
1203                       transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1204                       lstcon_trans_stat()->trs_rpc_errno,
1205                       lstcon_trans_stat()->trs_fwk_errno);
1206
1207                return rc;
1208        }
1209
1210        lstcon_rpc_trans_destroy(trans);
1211
1212        if (transop == LST_TRANS_TSBCLIADD)
1213                return rc;
1214
1215        transop = LST_TRANS_TSBCLIADD;
1216        grp = test->tes_src_grp;
1217        test->tes_cliidx = 0;
1218
1219        /* requests to test clients */
1220        goto again;
1221}
1222
1223static int
1224lstcon_verify_batch(const char *name, lstcon_batch_t **batch)
1225{
1226        int rc;
1227
1228        rc = lstcon_batch_find(name, batch);
1229        if (rc != 0) {
1230                CDEBUG(D_NET, "Can't find batch %s\n", name);
1231                return rc;
1232        }
1233
1234        if ((*batch)->bat_state != LST_BATCH_IDLE) {
1235                CDEBUG(D_NET, "Can't change running batch %s\n", name);
1236                return -EINVAL;
1237        }
1238
1239        return 0;
1240}
1241
1242static int
1243lstcon_verify_group(const char *name, lstcon_group_t **grp)
1244{
1245        int rc;
1246        lstcon_ndlink_t *ndl;
1247
1248        rc = lstcon_group_find(name, grp);
1249        if (rc != 0) {
1250                CDEBUG(D_NET, "can't find group %s\n", name);
1251                return rc;
1252        }
1253
1254        list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1255                if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE)
1256                        return 0;
1257        }
1258
1259        CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1260
1261        return -EINVAL;
1262}
1263
1264int
1265lstcon_test_add(char *batch_name, int type, int loop,
1266                int concur, int dist, int span,
1267                char *src_name, char *dst_name,
1268                void *param, int paramlen, int *retp,
1269                struct list_head *result_up)
1270{
1271        lstcon_test_t    *test   = NULL;
1272        int              rc;
1273        lstcon_group_t   *src_grp = NULL;
1274        lstcon_group_t   *dst_grp = NULL;
1275        lstcon_batch_t   *batch = NULL;
1276
1277        /*
1278         * verify that a batch of the given name exists, and the groups
1279         * that will be part of the batch exist and have at least one
1280         * active node
1281         */
1282        rc = lstcon_verify_batch(batch_name, &batch);
1283        if (rc != 0)
1284                goto out;
1285
1286        rc = lstcon_verify_group(src_name, &src_grp);
1287        if (rc != 0)
1288                goto out;
1289
1290        rc = lstcon_verify_group(dst_name, &dst_grp);
1291        if (rc != 0)
1292                goto out;
1293
1294        if (dst_grp->grp_userland)
1295                *retp = 1;
1296
1297        LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1298        if (!test) {
1299                CERROR("Can't allocate test descriptor\n");
1300                rc = -ENOMEM;
1301
1302                goto out;
1303        }
1304
1305        test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1306        test->tes_batch         = batch;
1307        test->tes_type          = type;
1308        test->tes_oneside       = 0; /* TODO */
1309        test->tes_loop          = loop;
1310        test->tes_concur        = concur;
1311        test->tes_stop_onerr    = 1; /* TODO */
1312        test->tes_span          = span;
1313        test->tes_dist          = dist;
1314        test->tes_cliidx        = 0; /* just used for creating RPC */
1315        test->tes_src_grp       = src_grp;
1316        test->tes_dst_grp       = dst_grp;
1317        INIT_LIST_HEAD(&test->tes_trans_list);
1318
1319        if (param != NULL) {
1320                test->tes_paramlen = paramlen;
1321                memcpy(&test->tes_param[0], param, paramlen);
1322        }
1323
1324        rc = lstcon_test_nodes_add(test, result_up);
1325
1326        if (rc != 0)
1327                goto out;
1328
1329        if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1330            lstcon_trans_stat()->trs_fwk_errno != 0)
1331                CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1332                       batch_name);
1333
1334        /* add to test list anyway, so user can check what's going on */
1335        list_add_tail(&test->tes_link, &batch->bat_test_list);
1336
1337        batch->bat_ntest++;
1338        test->tes_hdr.tsb_index = batch->bat_ntest;
1339
1340        /*  hold groups so nobody can change them */
1341        return rc;
1342out:
1343        if (test != NULL)
1344                LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1345
1346        if (dst_grp != NULL)
1347                lstcon_group_decref(dst_grp);
1348
1349        if (src_grp != NULL)
1350                lstcon_group_decref(src_grp);
1351
1352        return rc;
1353}
1354
1355static int
1356lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1357{
1358        lstcon_test_t *test;
1359
1360        list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1361                if (idx == test->tes_hdr.tsb_index) {
1362                        *testpp = test;
1363                        return 0;
1364                }
1365        }
1366
1367        return -ENOENT;
1368}
1369
1370static int
1371lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1372                      lstcon_rpc_ent_t *ent_up)
1373{
1374        srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1375
1376        LASSERT(transop == LST_TRANS_TSBCLIQRY ||
1377                 transop == LST_TRANS_TSBSRVQRY);
1378
1379        /* positive errno, framework error code */
1380        if (copy_to_user(&ent_up->rpe_priv[0],
1381                             &rep->bar_active, sizeof(rep->bar_active)))
1382                return -EFAULT;
1383
1384        return 0;
1385}
1386
1387int
1388lstcon_test_batch_query(char *name, int testidx, int client,
1389                        int timeout, struct list_head *result_up)
1390{
1391        lstcon_rpc_trans_t *trans;
1392        struct list_head *translist;
1393        struct list_head *ndlist;
1394        lstcon_tsb_hdr_t *hdr;
1395        lstcon_batch_t *batch;
1396        lstcon_test_t *test = NULL;
1397        int transop;
1398        int rc;
1399
1400        rc = lstcon_batch_find(name, &batch);
1401        if (rc != 0) {
1402                CDEBUG(D_NET, "Can't find batch: %s\n", name);
1403                return rc;
1404        }
1405
1406        if (testidx == 0) {
1407                translist = &batch->bat_trans_list;
1408                ndlist    = &batch->bat_cli_list;
1409                hdr       = &batch->bat_hdr;
1410
1411        } else {
1412                /* query specified test only */
1413                rc = lstcon_test_find(batch, testidx, &test);
1414                if (rc != 0) {
1415                        CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1416                        return rc;
1417                }
1418
1419                translist = &test->tes_trans_list;
1420                ndlist    = &test->tes_src_grp->grp_ndl_list;
1421                hdr       = &test->tes_hdr;
1422        }
1423
1424        transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1425
1426        rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1427                                     lstcon_batrpc_condition, &trans);
1428        if (rc != 0) {
1429                CERROR("Can't create transaction: %d\n", rc);
1430                return rc;
1431        }
1432
1433        lstcon_rpc_trans_postwait(trans, timeout);
1434
1435        if (testidx == 0 && /* query a batch, not a test */
1436            lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1437            lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1438                /* all RPCs finished, and no active test */
1439                batch->bat_state = LST_BATCH_IDLE;
1440        }
1441
1442        rc = lstcon_rpc_trans_interpreter(trans, result_up,
1443                                          lstcon_tsbrpc_readent);
1444        lstcon_rpc_trans_destroy(trans);
1445
1446        return rc;
1447}
1448
1449static int
1450lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1451                       lstcon_rpc_ent_t *ent_up)
1452{
1453        srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1454        sfw_counters_t *sfwk_stat;
1455        srpc_counters_t *srpc_stat;
1456        lnet_counters_t *lnet_stat;
1457
1458        if (rep->str_status != 0)
1459                return 0;
1460
1461        sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0];
1462        srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat));
1463        lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat));
1464
1465        if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1466            copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1467            copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1468                return -EFAULT;
1469
1470        return 0;
1471}
1472
1473static int
1474lstcon_ndlist_stat(struct list_head *ndlist,
1475                   int timeout, struct list_head *result_up)
1476{
1477        struct list_head head;
1478        lstcon_rpc_trans_t *trans;
1479        int rc;
1480
1481        INIT_LIST_HEAD(&head);
1482
1483        rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1484                                     LST_TRANS_STATQRY, NULL, NULL, &trans);
1485        if (rc != 0) {
1486                CERROR("Can't create transaction: %d\n", rc);
1487                return rc;
1488        }
1489
1490        lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1491
1492        rc = lstcon_rpc_trans_interpreter(trans, result_up,
1493                                          lstcon_statrpc_readent);
1494        lstcon_rpc_trans_destroy(trans);
1495
1496        return rc;
1497}
1498
1499int
1500lstcon_group_stat(char *grp_name, int timeout, struct list_head *result_up)
1501{
1502        lstcon_group_t *grp;
1503        int rc;
1504
1505        rc = lstcon_group_find(grp_name, &grp);
1506        if (rc != 0) {
1507                CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1508                return rc;
1509        }
1510
1511        rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1512
1513        lstcon_group_decref(grp);
1514
1515        return rc;
1516}
1517
1518int
1519lstcon_nodes_stat(int count, lnet_process_id_t *ids_up,
1520                  int timeout, struct list_head *result_up)
1521{
1522        lstcon_ndlink_t *ndl;
1523        lstcon_group_t *tmp;
1524        lnet_process_id_t id;
1525        int i;
1526        int rc;
1527
1528        rc = lstcon_group_alloc(NULL, &tmp);
1529        if (rc != 0) {
1530                CERROR("Out of memory\n");
1531                return -ENOMEM;
1532        }
1533
1534        for (i = 0 ; i < count; i++) {
1535                if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1536                        rc = -EFAULT;
1537                        break;
1538                }
1539
1540                /* add to tmp group */
1541                rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1542                if (rc != 0) {
1543                        CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1544                               "Failed to find or create %s: %d\n",
1545                               libcfs_id2str(id), rc);
1546                        break;
1547                }
1548        }
1549
1550        if (rc != 0) {
1551                lstcon_group_decref(tmp);
1552                return rc;
1553        }
1554
1555        rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1556
1557        lstcon_group_decref(tmp);
1558
1559        return rc;
1560}
1561
1562static int
1563lstcon_debug_ndlist(struct list_head *ndlist,
1564                    struct list_head *translist,
1565                    int timeout, struct list_head *result_up)
1566{
1567        lstcon_rpc_trans_t *trans;
1568        int              rc;
1569
1570        rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1571                                     NULL, lstcon_sesrpc_condition, &trans);
1572        if (rc != 0) {
1573                CERROR("Can't create transaction: %d\n", rc);
1574                return rc;
1575        }
1576
1577        lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1578
1579        rc = lstcon_rpc_trans_interpreter(trans, result_up,
1580                                          lstcon_sesrpc_readent);
1581        lstcon_rpc_trans_destroy(trans);
1582
1583        return rc;
1584}
1585
1586int
1587lstcon_session_debug(int timeout, struct list_head *result_up)
1588{
1589        return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1590                                   NULL, timeout, result_up);
1591}
1592
1593int
1594lstcon_batch_debug(int timeout, char *name,
1595                   int client, struct list_head *result_up)
1596{
1597        lstcon_batch_t *bat;
1598        int rc;
1599
1600        rc = lstcon_batch_find(name, &bat);
1601        if (rc != 0)
1602                return -ENOENT;
1603
1604        rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1605                                          &bat->bat_srv_list,
1606                                 NULL, timeout, result_up);
1607
1608        return rc;
1609}
1610
1611int
1612lstcon_group_debug(int timeout, char *name,
1613                   struct list_head *result_up)
1614{
1615        lstcon_group_t *grp;
1616        int rc;
1617
1618        rc = lstcon_group_find(name, &grp);
1619        if (rc != 0)
1620                return -ENOENT;
1621
1622        rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1623                                 timeout, result_up);
1624        lstcon_group_decref(grp);
1625
1626        return rc;
1627}
1628
1629int
1630lstcon_nodes_debug(int timeout,
1631                   int count, lnet_process_id_t *ids_up,
1632                   struct list_head *result_up)
1633{
1634        lnet_process_id_t id;
1635        lstcon_ndlink_t *ndl;
1636        lstcon_group_t *grp;
1637        int i;
1638        int rc;
1639
1640        rc = lstcon_group_alloc(NULL, &grp);
1641        if (rc != 0) {
1642                CDEBUG(D_NET, "Out of memory\n");
1643                return rc;
1644        }
1645
1646        for (i = 0; i < count; i++) {
1647                if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1648                        rc = -EFAULT;
1649                        break;
1650                }
1651
1652                /* node is added to tmp group */
1653                rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1654                if (rc != 0) {
1655                        CERROR("Can't create node link\n");
1656                        break;
1657                }
1658        }
1659
1660        if (rc != 0) {
1661                lstcon_group_decref(grp);
1662                return rc;
1663        }
1664
1665        rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1666                                 timeout, result_up);
1667
1668        lstcon_group_decref(grp);
1669
1670        return rc;
1671}
1672
1673int
1674lstcon_session_match(lst_sid_t sid)
1675{
1676        return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1677                console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1 : 0;
1678}
1679
1680static void
1681lstcon_new_session_id(lst_sid_t *sid)
1682{
1683        lnet_process_id_t id;
1684
1685        LASSERT(console_session.ses_state == LST_SESSION_NONE);
1686
1687        LNetGetId(1, &id);
1688        sid->ses_nid   = id.nid;
1689        sid->ses_stamp = cfs_time_current();
1690}
1691
1692extern srpc_service_t lstcon_acceptor_service;
1693
1694int
1695lstcon_session_new(char *name, int key, unsigned feats,
1696                   int timeout, int force, lst_sid_t *sid_up)
1697{
1698        int rc = 0;
1699        int i;
1700
1701        if (console_session.ses_state != LST_SESSION_NONE) {
1702                /* session exists */
1703                if (!force) {
1704                        CNETERR("Session %s already exists\n",
1705                                console_session.ses_name);
1706                        return -EEXIST;
1707                }
1708
1709                rc = lstcon_session_end();
1710
1711                /* lstcon_session_end() only return local error */
1712                if  (rc != 0)
1713                        return rc;
1714        }
1715
1716        if ((feats & ~LST_FEATS_MASK) != 0) {
1717                CNETERR("Unknown session features %x\n",
1718                        (feats & ~LST_FEATS_MASK));
1719                return -EINVAL;
1720        }
1721
1722        for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1723                LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1724
1725        lstcon_new_session_id(&console_session.ses_id);
1726
1727        console_session.ses_key = key;
1728        console_session.ses_state = LST_SESSION_ACTIVE;
1729        console_session.ses_force = !!force;
1730        console_session.ses_features = feats;
1731        console_session.ses_feats_updated = 0;
1732        console_session.ses_timeout = (timeout <= 0) ?
1733                                      LST_CONSOLE_TIMEOUT : timeout;
1734        strlcpy(console_session.ses_name, name,
1735                sizeof(console_session.ses_name));
1736
1737        rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1738        if (rc != 0)
1739                return rc;
1740
1741        rc = lstcon_rpc_pinger_start();
1742        if (rc != 0) {
1743                lstcon_batch_t *bat = NULL;
1744
1745                lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1746                lstcon_batch_destroy(bat);
1747
1748                return rc;
1749        }
1750
1751        if (copy_to_user(sid_up, &console_session.ses_id,
1752                             sizeof(lst_sid_t)) == 0)
1753                return rc;
1754
1755        lstcon_session_end();
1756
1757        return -EFAULT;
1758}
1759
1760int
1761lstcon_session_info(lst_sid_t *sid_up, int *key_up, unsigned *featp,
1762                    lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len)
1763{
1764        lstcon_ndlist_ent_t *entp;
1765        lstcon_ndlink_t *ndl;
1766        int rc = 0;
1767
1768        if (console_session.ses_state != LST_SESSION_ACTIVE)
1769                return -ESRCH;
1770
1771        LIBCFS_ALLOC(entp, sizeof(*entp));
1772        if (entp == NULL)
1773                return -ENOMEM;
1774
1775        list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1776                LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1777
1778        if (copy_to_user(sid_up, &console_session.ses_id,
1779                             sizeof(lst_sid_t)) ||
1780            copy_to_user(key_up, &console_session.ses_key,
1781                             sizeof(*key_up)) ||
1782            copy_to_user(featp, &console_session.ses_features,
1783                             sizeof(*featp)) ||
1784            copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1785            copy_to_user(name_up, console_session.ses_name, len))
1786                rc = -EFAULT;
1787
1788        LIBCFS_FREE(entp, sizeof(*entp));
1789
1790        return rc;
1791}
1792
1793int
1794lstcon_session_end(void)
1795{
1796        lstcon_rpc_trans_t *trans;
1797        lstcon_group_t *grp;
1798        lstcon_batch_t *bat;
1799        int rc = 0;
1800
1801        LASSERT(console_session.ses_state == LST_SESSION_ACTIVE);
1802
1803        rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1804                                     NULL, LST_TRANS_SESEND, NULL,
1805                                     lstcon_sesrpc_condition, &trans);
1806        if (rc != 0) {
1807                CERROR("Can't create transaction: %d\n", rc);
1808                return rc;
1809        }
1810
1811        console_session.ses_shutdown = 1;
1812
1813        lstcon_rpc_pinger_stop();
1814
1815        lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1816
1817        lstcon_rpc_trans_destroy(trans);
1818        /* User can do nothing even rpc failed, so go on */
1819
1820        /* waiting for orphan rpcs to die */
1821        lstcon_rpc_cleanup_wait();
1822
1823        console_session.ses_id    = LST_INVALID_SID;
1824        console_session.ses_state = LST_SESSION_NONE;
1825        console_session.ses_key   = 0;
1826        console_session.ses_force = 0;
1827        console_session.ses_feats_updated = 0;
1828
1829        /* destroy all batches */
1830        while (!list_empty(&console_session.ses_bat_list)) {
1831                bat = list_entry(console_session.ses_bat_list.next,
1832                                     lstcon_batch_t, bat_link);
1833
1834                lstcon_batch_destroy(bat);
1835        }
1836
1837        /* destroy all groups */
1838        while (!list_empty(&console_session.ses_grp_list)) {
1839                grp = list_entry(console_session.ses_grp_list.next,
1840                                     lstcon_group_t, grp_link);
1841                LASSERT(grp->grp_ref == 1);
1842
1843                lstcon_group_decref(grp);
1844        }
1845
1846        /* all nodes should be released */
1847        LASSERT(list_empty(&console_session.ses_ndl_list));
1848
1849        console_session.ses_shutdown = 0;
1850        console_session.ses_expired  = 0;
1851
1852        return rc;
1853}
1854
1855int
1856lstcon_session_feats_check(unsigned feats)
1857{
1858        int rc = 0;
1859
1860        if ((feats & ~LST_FEATS_MASK) != 0) {
1861                CERROR("Can't support these features: %x\n",
1862                       (feats & ~LST_FEATS_MASK));
1863                return -EPROTO;
1864        }
1865
1866        spin_lock(&console_session.ses_rpc_lock);
1867
1868        if (!console_session.ses_feats_updated) {
1869                console_session.ses_feats_updated = 1;
1870                console_session.ses_features = feats;
1871        }
1872
1873        if (console_session.ses_features != feats)
1874                rc = -EPROTO;
1875
1876        spin_unlock(&console_session.ses_rpc_lock);
1877
1878        if (rc != 0) {
1879                CERROR("remote features %x do not match with session features %x of console\n",
1880                       feats, console_session.ses_features);
1881        }
1882
1883        return rc;
1884}
1885
1886static int
1887lstcon_acceptor_handle(struct srpc_server_rpc *rpc)
1888{
1889        srpc_msg_t *rep  = &rpc->srpc_replymsg;
1890        srpc_msg_t *req  = &rpc->srpc_reqstbuf->buf_msg;
1891        srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1892        srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1893        lstcon_group_t *grp  = NULL;
1894        lstcon_ndlink_t *ndl;
1895        int rc   = 0;
1896
1897        sfw_unpack_message(req);
1898
1899        mutex_lock(&console_session.ses_mutex);
1900
1901        jrep->join_sid = console_session.ses_id;
1902
1903        if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1904                jrep->join_status = ESRCH;
1905                goto out;
1906        }
1907
1908        if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1909                jrep->join_status = EPROTO;
1910                goto out;
1911        }
1912
1913        if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1914             !lstcon_session_match(jreq->join_sid)) {
1915                jrep->join_status = EBUSY;
1916                goto out;
1917        }
1918
1919        if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1920                rc = lstcon_group_alloc(jreq->join_group, &grp);
1921                if (rc != 0) {
1922                        CERROR("Out of memory\n");
1923                        goto out;
1924                }
1925
1926                list_add_tail(&grp->grp_link,
1927                                  &console_session.ses_grp_list);
1928                lstcon_group_addref(grp);
1929        }
1930
1931        if (grp->grp_ref > 2) {
1932                /* Group in using */
1933                jrep->join_status = EBUSY;
1934                goto out;
1935        }
1936
1937        rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1938        if (rc == 0) {
1939                jrep->join_status = EEXIST;
1940                goto out;
1941        }
1942
1943        rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1944        if (rc != 0) {
1945                CERROR("Out of memory\n");
1946                goto out;
1947        }
1948
1949        ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1950        ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1951
1952        if (grp->grp_userland == 0)
1953                grp->grp_userland = 1;
1954
1955        strlcpy(jrep->join_session, console_session.ses_name,
1956                sizeof(jrep->join_session));
1957        jrep->join_timeout = console_session.ses_timeout;
1958        jrep->join_status  = 0;
1959
1960out:
1961        rep->msg_ses_feats = console_session.ses_features;
1962        if (grp != NULL)
1963                lstcon_group_decref(grp);
1964
1965        mutex_unlock(&console_session.ses_mutex);
1966
1967        return rc;
1968}
1969
1970srpc_service_t lstcon_acceptor_service;
1971static void lstcon_init_acceptor_service(void)
1972{
1973        /* initialize selftest console acceptor service table */
1974        lstcon_acceptor_service.sv_name     = "join session";
1975        lstcon_acceptor_service.sv_handler  = lstcon_acceptor_handle;
1976        lstcon_acceptor_service.sv_id       = SRPC_SERVICE_JOIN;
1977        lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1978}
1979
1980extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data);
1981
1982static DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1983
1984/* initialize console */
1985int
1986lstcon_console_init(void)
1987{
1988        int i;
1989        int rc;
1990
1991        memset(&console_session, 0, sizeof(lstcon_session_t));
1992
1993        console_session.ses_id            = LST_INVALID_SID;
1994        console_session.ses_state         = LST_SESSION_NONE;
1995        console_session.ses_timeout       = 0;
1996        console_session.ses_force         = 0;
1997        console_session.ses_expired       = 0;
1998        console_session.ses_feats_updated = 0;
1999        console_session.ses_features      = LST_FEATS_MASK;
2000        console_session.ses_laststamp     = ktime_get_real_seconds();
2001
2002        mutex_init(&console_session.ses_mutex);
2003
2004        INIT_LIST_HEAD(&console_session.ses_ndl_list);
2005        INIT_LIST_HEAD(&console_session.ses_grp_list);
2006        INIT_LIST_HEAD(&console_session.ses_bat_list);
2007        INIT_LIST_HEAD(&console_session.ses_trans_list);
2008
2009        LIBCFS_ALLOC(console_session.ses_ndl_hash,
2010                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2011        if (console_session.ses_ndl_hash == NULL)
2012                return -ENOMEM;
2013
2014        for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2015                INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2016
2017        /* initialize acceptor service table */
2018        lstcon_init_acceptor_service();
2019
2020        rc = srpc_add_service(&lstcon_acceptor_service);
2021        LASSERT(rc != -EBUSY);
2022        if (rc != 0) {
2023                LIBCFS_FREE(console_session.ses_ndl_hash,
2024                            sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2025                return rc;
2026        }
2027
2028        rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2029                                      lstcon_acceptor_service.sv_wi_total);
2030        if (rc != 0) {
2031                rc = -ENOMEM;
2032                goto out;
2033        }
2034
2035        rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2036
2037        if (rc == 0) {
2038                lstcon_rpc_module_init();
2039                return 0;
2040        }
2041
2042out:
2043        srpc_shutdown_service(&lstcon_acceptor_service);
2044        srpc_remove_service(&lstcon_acceptor_service);
2045
2046        LIBCFS_FREE(console_session.ses_ndl_hash,
2047                    sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2048
2049        srpc_wait_service_shutdown(&lstcon_acceptor_service);
2050
2051        return rc;
2052}
2053
2054int
2055lstcon_console_fini(void)
2056{
2057        int i;
2058
2059        libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2060
2061        mutex_lock(&console_session.ses_mutex);
2062
2063        srpc_shutdown_service(&lstcon_acceptor_service);
2064        srpc_remove_service(&lstcon_acceptor_service);
2065
2066        if (console_session.ses_state != LST_SESSION_NONE)
2067                lstcon_session_end();
2068
2069        lstcon_rpc_module_fini();
2070
2071        mutex_unlock(&console_session.ses_mutex);
2072
2073        LASSERT(list_empty(&console_session.ses_ndl_list));
2074        LASSERT(list_empty(&console_session.ses_grp_list));
2075        LASSERT(list_empty(&console_session.ses_bat_list));
2076        LASSERT(list_empty(&console_session.ses_trans_list));
2077
2078        for (i = 0; i < LST_NODE_HASHSIZE; i++) {
2079                LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2080        }
2081
2082        LIBCFS_FREE(console_session.ses_ndl_hash,
2083                    sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2084
2085        srpc_wait_service_shutdown(&lstcon_acceptor_service);
2086
2087        return 0;
2088}
2089