1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/kernel.h>
25#include <linux/module.h>
26#include <linux/nx842.h>
27#include <linux/of.h>
28#include <linux/slab.h>
29
30#include <asm/page.h>
31#include <asm/vio.h>
32
33#include "nx_csbcpb.h"
34
35#define MODULE_NAME "nx-compress"
36MODULE_LICENSE("GPL");
37MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
38MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
39
40#define SHIFT_4K 12
41#define SHIFT_64K 16
42#define SIZE_4K (1UL << SHIFT_4K)
43#define SIZE_64K (1UL << SHIFT_64K)
44
45
46#define IO_BUFFER_ALIGN 128
47
48struct nx842_header {
49 int blocks_nr;
50 int offset;
51 int sizes[0];
52};
53
54static inline int nx842_header_size(const struct nx842_header *hdr)
55{
56 return sizeof(struct nx842_header) +
57 hdr->blocks_nr * sizeof(hdr->sizes[0]);
58}
59
60
61
62#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
63
64
65
66
67
68#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
69#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
70#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
71
72
73#define NX842_HW_PAGE_SHIFT SHIFT_4K
74#define NX842_HW_PAGE_SIZE (ASM_CONST(1) << NX842_HW_PAGE_SHIFT)
75#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
76
77enum nx842_status {
78 UNAVAILABLE,
79 AVAILABLE
80};
81
82struct ibm_nx842_counters {
83 atomic64_t comp_complete;
84 atomic64_t comp_failed;
85 atomic64_t decomp_complete;
86 atomic64_t decomp_failed;
87 atomic64_t swdecomp;
88 atomic64_t comp_times[32];
89 atomic64_t decomp_times[32];
90};
91
92static struct nx842_devdata {
93 struct vio_dev *vdev;
94 struct device *dev;
95 struct ibm_nx842_counters *counters;
96 unsigned int max_sg_len;
97 unsigned int max_sync_size;
98 unsigned int max_sync_sg;
99 enum nx842_status status;
100} __rcu *devdata;
101static DEFINE_SPINLOCK(devdata_mutex);
102
103#define NX842_COUNTER_INC(_x) \
104static inline void nx842_inc_##_x( \
105 const struct nx842_devdata *dev) { \
106 if (dev) \
107 atomic64_inc(&dev->counters->_x); \
108}
109NX842_COUNTER_INC(comp_complete);
110NX842_COUNTER_INC(comp_failed);
111NX842_COUNTER_INC(decomp_complete);
112NX842_COUNTER_INC(decomp_failed);
113NX842_COUNTER_INC(swdecomp);
114
115#define NX842_HIST_SLOTS 16
116
117static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
118{
119 int bucket = fls(time);
120
121 if (bucket)
122 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
123
124 atomic64_inc(×[bucket]);
125}
126
127
128#define NX842_OP_COMPRESS 0x0
129#define NX842_OP_CRC 0x1
130#define NX842_OP_DECOMPRESS 0x2
131#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
132#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
133#define NX842_OP_ASYNC (1<<23)
134#define NX842_OP_NOTIFY (1<<22)
135#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
136
137static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
138{
139
140 return 0;
141}
142
143struct nx842_slentry {
144 unsigned long ptr;
145 unsigned long len;
146};
147
148
149struct nx842_scatterlist {
150 int entry_nr;
151 struct nx842_slentry *entries;
152};
153
154
155static inline unsigned long nx842_get_scatterlist_size(
156 struct nx842_scatterlist *sl)
157{
158 return sl->entry_nr * sizeof(struct nx842_slentry);
159}
160
161static int nx842_build_scatterlist(unsigned long buf, int len,
162 struct nx842_scatterlist *sl)
163{
164 unsigned long nextpage;
165 struct nx842_slentry *entry;
166
167 sl->entry_nr = 0;
168
169 entry = sl->entries;
170 while (len) {
171 entry->ptr = __pa(buf);
172 nextpage = ALIGN(buf + 1, NX842_HW_PAGE_SIZE);
173 if (nextpage < buf + len) {
174
175 if (IS_ALIGNED(buf, NX842_HW_PAGE_SIZE))
176
177 entry->len = NX842_HW_PAGE_SIZE;
178 else
179
180 entry->len = nextpage - buf;
181 } else {
182
183 entry->len = len;
184 }
185
186 len -= entry->len;
187 buf += entry->len;
188 sl->entry_nr++;
189 entry++;
190 }
191
192 return 0;
193}
194
195
196
197
198struct sw842_fifo {
199 union {
200 char f8[256][8];
201 char f4[512][4];
202 };
203 char f2[256][2];
204 unsigned char f84_full;
205 unsigned char f2_full;
206 unsigned char f8_count;
207 unsigned char f2_count;
208 unsigned int f4_count;
209};
210
211
212
213
214struct nx842_workmem {
215 char bounce[PAGE_SIZE];
216 union {
217
218 struct {
219
220 char slin[SIZE_4K];
221 char slout[SIZE_4K];
222
223 struct nx_csbcpb csbcpb;
224 };
225
226 struct sw842_fifo swfifo;
227 };
228};
229
230int nx842_get_workmem_size(void)
231{
232 return sizeof(struct nx842_workmem) + NX842_HW_PAGE_SIZE;
233}
234EXPORT_SYMBOL_GPL(nx842_get_workmem_size);
235
236int nx842_get_workmem_size_aligned(void)
237{
238 return sizeof(struct nx842_workmem);
239}
240EXPORT_SYMBOL_GPL(nx842_get_workmem_size_aligned);
241
242static int nx842_validate_result(struct device *dev,
243 struct cop_status_block *csb)
244{
245
246 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
247 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
248 __func__);
249 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
250 csb->valid,
251 csb->crb_seq_number,
252 csb->completion_code,
253 csb->completion_extension);
254 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
255 csb->processed_byte_count,
256 (unsigned long)csb->address);
257 return -EIO;
258 }
259
260
261 switch (csb->completion_code) {
262 case 0:
263 break;
264 case 64:
265 case 13:
266 dev_dbg(dev, "%s: Compression output larger than input\n",
267 __func__);
268 return -ENOSPC;
269 case 66:
270 case 67:
271 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
272 __func__, csb->completion_code);
273 return -EINVAL;
274 default:
275 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
276 __func__, csb->completion_code);
277 return -EIO;
278 }
279
280
281 if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
282 dev_err(dev, "%s: No error returned by hardware, but "
283 "data returned is unusable, contact support.\n"
284 "(Additional info: csbcbp->processed bytes "
285 "does not specify processed bytes for the "
286 "target buffer.)\n", __func__);
287 return -EIO;
288 }
289
290 return 0;
291}
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319int nx842_compress(const unsigned char *in, unsigned int inlen,
320 unsigned char *out, unsigned int *outlen, void *wmem)
321{
322 struct nx842_header *hdr;
323 struct nx842_devdata *local_devdata;
324 struct device *dev = NULL;
325 struct nx842_workmem *workmem;
326 struct nx842_scatterlist slin, slout;
327 struct nx_csbcpb *csbcpb;
328 int ret = 0, max_sync_size, i, bytesleft, size, hdrsize;
329 unsigned long inbuf, outbuf, padding;
330 struct vio_pfo_op op = {
331 .done = NULL,
332 .handle = 0,
333 .timeout = 0,
334 };
335 unsigned long start_time = get_tb();
336
337
338
339
340
341
342
343 inbuf = (unsigned long)in;
344 if (!IS_ALIGNED(inbuf, PAGE_SIZE) || inlen != PAGE_SIZE)
345 return -EINVAL;
346
347 rcu_read_lock();
348 local_devdata = rcu_dereference(devdata);
349 if (!local_devdata || !local_devdata->dev) {
350 rcu_read_unlock();
351 return -ENODEV;
352 }
353 max_sync_size = local_devdata->max_sync_size;
354 dev = local_devdata->dev;
355
356
357 hdr = (struct nx842_header *)out;
358 hdr->blocks_nr = PAGE_SIZE / max_sync_size;
359 hdrsize = nx842_header_size(hdr);
360 outbuf = (unsigned long)out + hdrsize;
361 bytesleft = *outlen - hdrsize;
362
363
364 workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
365 NX842_HW_PAGE_SIZE);
366 slin.entries = (struct nx842_slentry *)workmem->slin;
367 slout.entries = (struct nx842_slentry *)workmem->slout;
368
369
370 op.flags = NX842_OP_COMPRESS;
371 csbcpb = &workmem->csbcpb;
372 memset(csbcpb, 0, sizeof(*csbcpb));
373 op.csbcpb = __pa(csbcpb);
374 op.out = __pa(slout.entries);
375
376 for (i = 0; i < hdr->blocks_nr; i++) {
377
378
379
380
381
382
383
384
385 padding = ALIGN(outbuf, IO_BUFFER_ALIGN) - outbuf;
386 outbuf += padding;
387 bytesleft -= padding;
388 if (i == 0)
389
390 hdr->offset = padding + hdrsize;
391
392 if (bytesleft <= 0) {
393 ret = -ENOSPC;
394 goto unlock;
395 }
396
397
398
399
400
401
402 if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
403
404 op.in = __pa(inbuf);
405 op.inlen = max_sync_size;
406
407 } else {
408
409 nx842_build_scatterlist(inbuf, max_sync_size, &slin);
410 op.in = __pa(slin.entries);
411 op.inlen = -nx842_get_scatterlist_size(&slin);
412 }
413
414
415
416
417
418
419
420
421
422 nx842_build_scatterlist(outbuf,
423 min(bytesleft, max_sync_size), &slout);
424
425 op.outlen = -nx842_get_scatterlist_size(&slout);
426
427
428 ret = vio_h_cop_sync(local_devdata->vdev, &op);
429
430
431 if (ret) {
432 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
433 __func__, ret, op.hcall_err);
434 ret = -EIO;
435 goto unlock;
436 }
437
438
439 ret = nx842_validate_result(dev, &csbcpb->csb);
440 if (ret && ret != -ENOSPC)
441 goto unlock;
442
443
444 if (unlikely(ret == -ENOSPC)) {
445 if (bytesleft < max_sync_size) {
446
447
448
449
450 goto unlock;
451 } else {
452
453 memcpy((void *)outbuf, (void *)inbuf,
454 max_sync_size);
455 hdr->sizes[i] = -max_sync_size;
456 outbuf += max_sync_size;
457 bytesleft -= max_sync_size;
458
459 ret = 0;
460 }
461 } else {
462
463 size = csbcpb->csb.processed_byte_count;
464 dev_dbg(dev, "%s: processed_bytes=%d\n",
465 __func__, size);
466 hdr->sizes[i] = size;
467 outbuf += size;
468 bytesleft -= size;
469 }
470
471 inbuf += max_sync_size;
472 }
473
474 *outlen = (unsigned int)(outbuf - (unsigned long)out);
475
476unlock:
477 if (ret)
478 nx842_inc_comp_failed(local_devdata);
479 else {
480 nx842_inc_comp_complete(local_devdata);
481 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
482 (get_tb() - start_time) / tb_ticks_per_usec);
483 }
484 rcu_read_unlock();
485 return ret;
486}
487EXPORT_SYMBOL_GPL(nx842_compress);
488
489static int sw842_decompress(const unsigned char *, int, unsigned char *, int *,
490 const void *);
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519int nx842_decompress(const unsigned char *in, unsigned int inlen,
520 unsigned char *out, unsigned int *outlen, void *wmem)
521{
522 struct nx842_header *hdr;
523 struct nx842_devdata *local_devdata;
524 struct device *dev = NULL;
525 struct nx842_workmem *workmem;
526 struct nx842_scatterlist slin, slout;
527 struct nx_csbcpb *csbcpb;
528 int ret = 0, i, size, max_sync_size;
529 unsigned long inbuf, outbuf;
530 struct vio_pfo_op op = {
531 .done = NULL,
532 .handle = 0,
533 .timeout = 0,
534 };
535 unsigned long start_time = get_tb();
536
537
538 outbuf = (unsigned long)out;
539 if (!IS_ALIGNED(outbuf, PAGE_SIZE) || *outlen != PAGE_SIZE)
540 return -EINVAL;
541
542 rcu_read_lock();
543 local_devdata = rcu_dereference(devdata);
544 if (local_devdata)
545 dev = local_devdata->dev;
546
547
548 hdr = (struct nx842_header *)in;
549
550 workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
551 NX842_HW_PAGE_SIZE);
552
553 inbuf = (unsigned long)in + hdr->offset;
554 if (likely(!IS_ALIGNED(inbuf, IO_BUFFER_ALIGN))) {
555
556 memcpy(workmem->bounce, in + hdr->offset, inlen - hdr->offset);
557 inbuf = (unsigned long)workmem->bounce;
558 }
559
560
561 slin.entries = (struct nx842_slentry *)workmem->slin;
562 slout.entries = (struct nx842_slentry *)workmem->slout;
563
564
565 op.flags = NX842_OP_DECOMPRESS;
566 csbcpb = &workmem->csbcpb;
567 memset(csbcpb, 0, sizeof(*csbcpb));
568 op.csbcpb = __pa(csbcpb);
569
570
571
572
573
574
575 max_sync_size = PAGE_SIZE / hdr->blocks_nr;
576
577 for (i = 0; i < hdr->blocks_nr; i++) {
578
579 inbuf = ALIGN(inbuf, IO_BUFFER_ALIGN);
580
581 if (hdr->sizes[i] < 0) {
582
583 size = abs(hdr->sizes[i]);
584 memcpy((void *)outbuf, (void *)inbuf, size);
585 outbuf += size;
586 inbuf += size;
587 continue;
588 }
589
590 if (!dev)
591 goto sw;
592
593
594
595
596
597 if (likely((inbuf & NX842_HW_PAGE_MASK) ==
598 ((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) {
599
600 op.in = __pa(inbuf);
601 op.inlen = hdr->sizes[i];
602 } else {
603
604 nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin);
605 op.in = __pa(slin.entries);
606 op.inlen = -nx842_get_scatterlist_size(&slin);
607 }
608
609
610
611
612
613
614 if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
615
616 op.out = __pa(outbuf);
617 op.outlen = max_sync_size;
618 } else {
619
620 nx842_build_scatterlist(outbuf, max_sync_size, &slout);
621 op.out = __pa(slout.entries);
622 op.outlen = -nx842_get_scatterlist_size(&slout);
623 }
624
625
626 ret = vio_h_cop_sync(local_devdata->vdev, &op);
627
628
629 if (ret) {
630 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
631 __func__, ret, op.hcall_err);
632 dev = NULL;
633 goto sw;
634 }
635
636
637 ret = nx842_validate_result(dev, &csbcpb->csb);
638 if (ret) {
639 dev = NULL;
640 goto sw;
641 }
642
643
644 inbuf += hdr->sizes[i];
645 outbuf += csbcpb->csb.processed_byte_count;
646 continue;
647
648sw:
649
650 size = max_sync_size;
651 ret = sw842_decompress(
652 (unsigned char *)inbuf, hdr->sizes[i],
653 (unsigned char *)outbuf, &size, wmem);
654 if (ret)
655 pr_debug("%s: sw842_decompress failed with %d\n",
656 __func__, ret);
657
658 if (ret) {
659 if (ret != -ENOSPC && ret != -EINVAL &&
660 ret != -EMSGSIZE)
661 ret = -EIO;
662 goto unlock;
663 }
664
665
666 inbuf += hdr->sizes[i];
667 outbuf += size;
668 }
669
670 *outlen = (unsigned int)(outbuf - (unsigned long)out);
671
672unlock:
673 if (ret)
674
675 nx842_inc_decomp_failed(local_devdata);
676 else {
677 if (!dev)
678
679 nx842_inc_swdecomp(local_devdata);
680 nx842_inc_decomp_complete(local_devdata);
681 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
682 (get_tb() - start_time) / tb_ticks_per_usec);
683 }
684
685 rcu_read_unlock();
686 return ret;
687}
688EXPORT_SYMBOL_GPL(nx842_decompress);
689
690
691
692
693
694
695
696
697
698
699static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
700{
701 if (devdata) {
702 devdata->max_sync_size = 0;
703 devdata->max_sync_sg = 0;
704 devdata->max_sg_len = 0;
705 devdata->status = UNAVAILABLE;
706 return 0;
707 } else
708 return -ENOENT;
709}
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726static int nx842_OF_upd_status(struct nx842_devdata *devdata,
727 struct property *prop) {
728 int ret = 0;
729 const char *status = (const char *)prop->value;
730
731 if (!strncmp(status, "okay", (size_t)prop->length)) {
732 devdata->status = AVAILABLE;
733 } else {
734 dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
735 __func__, status);
736 devdata->status = UNAVAILABLE;
737 }
738
739 return ret;
740}
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
764 struct property *prop) {
765 int ret = 0;
766 const int *maxsglen = prop->value;
767
768 if (prop->length != sizeof(*maxsglen)) {
769 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
770 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
771 prop->length, sizeof(*maxsglen));
772 ret = -EINVAL;
773 } else {
774 devdata->max_sg_len = (unsigned int)min(*maxsglen,
775 (int)NX842_HW_PAGE_SIZE);
776 }
777
778 return ret;
779}
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
812 struct property *prop) {
813 int ret = 0;
814 const struct maxsynccop_t {
815 int comp_elements;
816 int comp_data_limit;
817 int comp_sg_limit;
818 int decomp_elements;
819 int decomp_data_limit;
820 int decomp_sg_limit;
821 } *maxsynccop;
822
823 if (prop->length != sizeof(*maxsynccop)) {
824 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
825 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
826 sizeof(*maxsynccop));
827 ret = -EINVAL;
828 goto out;
829 }
830
831 maxsynccop = (const struct maxsynccop_t *)prop->value;
832
833
834
835
836
837 devdata->max_sync_size =
838 (unsigned int)min(maxsynccop->comp_data_limit,
839 maxsynccop->decomp_data_limit);
840
841 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
842 SIZE_64K);
843
844 if (devdata->max_sync_size < SIZE_4K) {
845 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
846 "less than the driver minimum, unable to use "
847 "the hardware device\n",
848 __func__, devdata->max_sync_size);
849 ret = -EINVAL;
850 goto out;
851 }
852
853 devdata->max_sync_sg = (unsigned int)min(maxsynccop->comp_sg_limit,
854 maxsynccop->decomp_sg_limit);
855 if (devdata->max_sync_sg < 1) {
856 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
857 "less than the driver minimum, unable to use "
858 "the hardware device\n",
859 __func__, devdata->max_sync_sg);
860 ret = -EINVAL;
861 goto out;
862 }
863
864out:
865 return ret;
866}
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887static int nx842_OF_upd(struct property *new_prop)
888{
889 struct nx842_devdata *old_devdata = NULL;
890 struct nx842_devdata *new_devdata = NULL;
891 struct device_node *of_node = NULL;
892 struct property *status = NULL;
893 struct property *maxsglen = NULL;
894 struct property *maxsyncop = NULL;
895 int ret = 0;
896 unsigned long flags;
897
898 spin_lock_irqsave(&devdata_mutex, flags);
899 old_devdata = rcu_dereference_check(devdata,
900 lockdep_is_held(&devdata_mutex));
901 if (old_devdata)
902 of_node = old_devdata->dev->of_node;
903
904 if (!old_devdata || !of_node) {
905 pr_err("%s: device is not available\n", __func__);
906 spin_unlock_irqrestore(&devdata_mutex, flags);
907 return -ENODEV;
908 }
909
910 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
911 if (!new_devdata) {
912 dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__);
913 ret = -ENOMEM;
914 goto error_out;
915 }
916
917 memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
918 new_devdata->counters = old_devdata->counters;
919
920
921 status = of_find_property(of_node, "status", NULL);
922 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
923 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
924 if (!status || !maxsglen || !maxsyncop) {
925 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
926 ret = -EINVAL;
927 goto error_out;
928 }
929
930
931 if (new_prop) {
932
933 if (!strncmp(new_prop->name, "status", new_prop->length)) {
934 status = new_prop;
935
936 } else if (!strncmp(new_prop->name, "ibm,max-sg-len",
937 new_prop->length)) {
938 maxsglen = new_prop;
939
940 } else if (!strncmp(new_prop->name, "ibm,max-sync-cop",
941 new_prop->length)) {
942 maxsyncop = new_prop;
943
944 } else {
945
946
947
948
949 goto out;
950 }
951 }
952
953
954 ret = nx842_OF_upd_status(new_devdata, status);
955 if (ret)
956 goto error_out;
957
958 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
959 if (ret)
960 goto error_out;
961
962 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
963 if (ret)
964 goto error_out;
965
966out:
967 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
968 __func__, new_devdata->max_sync_size,
969 old_devdata->max_sync_size);
970 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
971 __func__, new_devdata->max_sync_sg,
972 old_devdata->max_sync_sg);
973 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
974 __func__, new_devdata->max_sg_len,
975 old_devdata->max_sg_len);
976
977 rcu_assign_pointer(devdata, new_devdata);
978 spin_unlock_irqrestore(&devdata_mutex, flags);
979 synchronize_rcu();
980 dev_set_drvdata(new_devdata->dev, new_devdata);
981 kfree(old_devdata);
982 return 0;
983
984error_out:
985 if (new_devdata) {
986 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
987 nx842_OF_set_defaults(new_devdata);
988 rcu_assign_pointer(devdata, new_devdata);
989 spin_unlock_irqrestore(&devdata_mutex, flags);
990 synchronize_rcu();
991 dev_set_drvdata(new_devdata->dev, new_devdata);
992 kfree(old_devdata);
993 } else {
994 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
995 spin_unlock_irqrestore(&devdata_mutex, flags);
996 }
997
998 if (!ret)
999 ret = -EINVAL;
1000 return ret;
1001}
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
1017 void *update)
1018{
1019 struct of_prop_reconfig *upd = update;
1020 struct nx842_devdata *local_devdata;
1021 struct device_node *node = NULL;
1022
1023 rcu_read_lock();
1024 local_devdata = rcu_dereference(devdata);
1025 if (local_devdata)
1026 node = local_devdata->dev->of_node;
1027
1028 if (local_devdata &&
1029 action == OF_RECONFIG_UPDATE_PROPERTY &&
1030 !strcmp(upd->dn->name, node->name)) {
1031 rcu_read_unlock();
1032 nx842_OF_upd(upd->prop);
1033 } else
1034 rcu_read_unlock();
1035
1036 return NOTIFY_OK;
1037}
1038
1039static struct notifier_block nx842_of_nb = {
1040 .notifier_call = nx842_OF_notifier,
1041};
1042
1043#define nx842_counter_read(_name) \
1044static ssize_t nx842_##_name##_show(struct device *dev, \
1045 struct device_attribute *attr, \
1046 char *buf) { \
1047 struct nx842_devdata *local_devdata; \
1048 int p = 0; \
1049 rcu_read_lock(); \
1050 local_devdata = rcu_dereference(devdata); \
1051 if (local_devdata) \
1052 p = snprintf(buf, PAGE_SIZE, "%ld\n", \
1053 atomic64_read(&local_devdata->counters->_name)); \
1054 rcu_read_unlock(); \
1055 return p; \
1056}
1057
1058#define NX842DEV_COUNTER_ATTR_RO(_name) \
1059 nx842_counter_read(_name); \
1060 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
1061 0444, \
1062 nx842_##_name##_show,\
1063 NULL);
1064
1065NX842DEV_COUNTER_ATTR_RO(comp_complete);
1066NX842DEV_COUNTER_ATTR_RO(comp_failed);
1067NX842DEV_COUNTER_ATTR_RO(decomp_complete);
1068NX842DEV_COUNTER_ATTR_RO(decomp_failed);
1069NX842DEV_COUNTER_ATTR_RO(swdecomp);
1070
1071static ssize_t nx842_timehist_show(struct device *,
1072 struct device_attribute *, char *);
1073
1074static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
1075 nx842_timehist_show, NULL);
1076static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
1077 0444, nx842_timehist_show, NULL);
1078
1079static ssize_t nx842_timehist_show(struct device *dev,
1080 struct device_attribute *attr, char *buf) {
1081 char *p = buf;
1082 struct nx842_devdata *local_devdata;
1083 atomic64_t *times;
1084 int bytes_remain = PAGE_SIZE;
1085 int bytes;
1086 int i;
1087
1088 rcu_read_lock();
1089 local_devdata = rcu_dereference(devdata);
1090 if (!local_devdata) {
1091 rcu_read_unlock();
1092 return 0;
1093 }
1094
1095 if (attr == &dev_attr_comp_times)
1096 times = local_devdata->counters->comp_times;
1097 else if (attr == &dev_attr_decomp_times)
1098 times = local_devdata->counters->decomp_times;
1099 else {
1100 rcu_read_unlock();
1101 return 0;
1102 }
1103
1104 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
1105 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
1106 i ? (2<<(i-1)) : 0, (2<<i)-1,
1107 atomic64_read(×[i]));
1108 bytes_remain -= bytes;
1109 p += bytes;
1110 }
1111
1112
1113 bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
1114 2<<(NX842_HIST_SLOTS - 2),
1115 atomic64_read(×[(NX842_HIST_SLOTS - 1)]));
1116 p += bytes;
1117
1118 rcu_read_unlock();
1119 return p - buf;
1120}
1121
1122static struct attribute *nx842_sysfs_entries[] = {
1123 &dev_attr_comp_complete.attr,
1124 &dev_attr_comp_failed.attr,
1125 &dev_attr_decomp_complete.attr,
1126 &dev_attr_decomp_failed.attr,
1127 &dev_attr_swdecomp.attr,
1128 &dev_attr_comp_times.attr,
1129 &dev_attr_decomp_times.attr,
1130 NULL,
1131};
1132
1133static struct attribute_group nx842_attribute_group = {
1134 .name = NULL,
1135 .attrs = nx842_sysfs_entries,
1136};
1137
1138static int __init nx842_probe(struct vio_dev *viodev,
1139 const struct vio_device_id *id)
1140{
1141 struct nx842_devdata *old_devdata, *new_devdata = NULL;
1142 unsigned long flags;
1143 int ret = 0;
1144
1145 spin_lock_irqsave(&devdata_mutex, flags);
1146 old_devdata = rcu_dereference_check(devdata,
1147 lockdep_is_held(&devdata_mutex));
1148
1149 if (old_devdata && old_devdata->vdev != NULL) {
1150 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
1151 ret = -1;
1152 goto error_unlock;
1153 }
1154
1155 dev_set_drvdata(&viodev->dev, NULL);
1156
1157 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
1158 if (!new_devdata) {
1159 dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__);
1160 ret = -ENOMEM;
1161 goto error_unlock;
1162 }
1163
1164 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
1165 GFP_NOFS);
1166 if (!new_devdata->counters) {
1167 dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__);
1168 ret = -ENOMEM;
1169 goto error_unlock;
1170 }
1171
1172 new_devdata->vdev = viodev;
1173 new_devdata->dev = &viodev->dev;
1174 nx842_OF_set_defaults(new_devdata);
1175
1176 rcu_assign_pointer(devdata, new_devdata);
1177 spin_unlock_irqrestore(&devdata_mutex, flags);
1178 synchronize_rcu();
1179 kfree(old_devdata);
1180
1181 of_reconfig_notifier_register(&nx842_of_nb);
1182
1183 ret = nx842_OF_upd(NULL);
1184 if (ret && ret != -ENODEV) {
1185 dev_err(&viodev->dev, "could not parse device tree. %d\n", ret);
1186 ret = -1;
1187 goto error;
1188 }
1189
1190 rcu_read_lock();
1191 if (dev_set_drvdata(&viodev->dev, rcu_dereference(devdata))) {
1192 rcu_read_unlock();
1193 dev_err(&viodev->dev, "failed to set driver data for device\n");
1194 ret = -1;
1195 goto error;
1196 }
1197 rcu_read_unlock();
1198
1199 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1200 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1201 ret = -1;
1202 goto error;
1203 }
1204
1205 return 0;
1206
1207error_unlock:
1208 spin_unlock_irqrestore(&devdata_mutex, flags);
1209 if (new_devdata)
1210 kfree(new_devdata->counters);
1211 kfree(new_devdata);
1212error:
1213 return ret;
1214}
1215
1216static int __exit nx842_remove(struct vio_dev *viodev)
1217{
1218 struct nx842_devdata *old_devdata;
1219 unsigned long flags;
1220
1221 pr_info("Removing IBM Power 842 compression device\n");
1222 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1223
1224 spin_lock_irqsave(&devdata_mutex, flags);
1225 old_devdata = rcu_dereference_check(devdata,
1226 lockdep_is_held(&devdata_mutex));
1227 of_reconfig_notifier_unregister(&nx842_of_nb);
1228 rcu_assign_pointer(devdata, NULL);
1229 spin_unlock_irqrestore(&devdata_mutex, flags);
1230 synchronize_rcu();
1231 dev_set_drvdata(&viodev->dev, NULL);
1232 if (old_devdata)
1233 kfree(old_devdata->counters);
1234 kfree(old_devdata);
1235 return 0;
1236}
1237
1238static struct vio_device_id nx842_driver_ids[] = {
1239 {"ibm,compression-v1", "ibm,compression"},
1240 {"", ""},
1241};
1242
1243static struct vio_driver nx842_driver = {
1244 .name = MODULE_NAME,
1245 .probe = nx842_probe,
1246 .remove = nx842_remove,
1247 .get_desired_dma = nx842_get_desired_dma,
1248 .id_table = nx842_driver_ids,
1249};
1250
1251static int __init nx842_init(void)
1252{
1253 struct nx842_devdata *new_devdata;
1254 pr_info("Registering IBM Power 842 compression driver\n");
1255
1256 RCU_INIT_POINTER(devdata, NULL);
1257 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1258 if (!new_devdata) {
1259 pr_err("Could not allocate memory for device data\n");
1260 return -ENOMEM;
1261 }
1262 new_devdata->status = UNAVAILABLE;
1263 RCU_INIT_POINTER(devdata, new_devdata);
1264
1265 return vio_register_driver(&nx842_driver);
1266}
1267
1268module_init(nx842_init);
1269
1270static void __exit nx842_exit(void)
1271{
1272 struct nx842_devdata *old_devdata;
1273 unsigned long flags;
1274
1275 pr_info("Exiting IBM Power 842 compression driver\n");
1276 spin_lock_irqsave(&devdata_mutex, flags);
1277 old_devdata = rcu_dereference_check(devdata,
1278 lockdep_is_held(&devdata_mutex));
1279 rcu_assign_pointer(devdata, NULL);
1280 spin_unlock_irqrestore(&devdata_mutex, flags);
1281 synchronize_rcu();
1282 if (old_devdata)
1283 dev_set_drvdata(old_devdata->dev, NULL);
1284 kfree(old_devdata);
1285 vio_unregister_driver(&nx842_driver);
1286}
1287
1288module_exit(nx842_exit);
1289
1290
1291
1292
1293typedef int (*sw842_template_op)(const char **, int *, unsigned char **,
1294 struct sw842_fifo *);
1295
1296static int sw842_data8(const char **, int *, unsigned char **,
1297 struct sw842_fifo *);
1298static int sw842_data4(const char **, int *, unsigned char **,
1299 struct sw842_fifo *);
1300static int sw842_data2(const char **, int *, unsigned char **,
1301 struct sw842_fifo *);
1302static int sw842_ptr8(const char **, int *, unsigned char **,
1303 struct sw842_fifo *);
1304static int sw842_ptr4(const char **, int *, unsigned char **,
1305 struct sw842_fifo *);
1306static int sw842_ptr2(const char **, int *, unsigned char **,
1307 struct sw842_fifo *);
1308
1309
1310#define SW842_TMPL_REPEAT 0x1B
1311#define SW842_TMPL_ZEROS 0x1C
1312#define SW842_TMPL_EOF 0x1E
1313
1314static sw842_template_op sw842_tmpl_ops[26][4] = {
1315 { sw842_data8, NULL},
1316 { sw842_data4, sw842_data2, sw842_ptr2, NULL},
1317 { sw842_data4, sw842_ptr2, sw842_data2, NULL},
1318 { sw842_data4, sw842_ptr2, sw842_ptr2, NULL},
1319 { sw842_data4, sw842_ptr4, NULL},
1320 { sw842_data2, sw842_ptr2, sw842_data4, NULL},
1321 { sw842_data2, sw842_ptr2, sw842_data2, sw842_ptr2},
1322 { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_data2},
1323 { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_ptr2,},
1324 { sw842_data2, sw842_ptr2, sw842_ptr4, NULL},
1325 { sw842_ptr2, sw842_data2, sw842_data4, NULL},
1326 { sw842_ptr2, sw842_data4, sw842_ptr2, NULL},
1327 { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_data2},
1328 { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_ptr2},
1329 { sw842_ptr2, sw842_data2, sw842_ptr4, NULL},
1330 { sw842_ptr2, sw842_ptr2, sw842_data4, NULL},
1331 { sw842_ptr2, sw842_ptr2, sw842_data2, sw842_ptr2},
1332 { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_data2},
1333 { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_ptr2},
1334 { sw842_ptr2, sw842_ptr2, sw842_ptr4, NULL},
1335 { sw842_ptr4, sw842_data4, NULL},
1336 { sw842_ptr4, sw842_data2, sw842_ptr2, NULL},
1337 { sw842_ptr4, sw842_ptr2, sw842_data2, NULL},
1338 { sw842_ptr4, sw842_ptr2, sw842_ptr2, NULL},
1339 { sw842_ptr4, sw842_ptr4, NULL},
1340 { sw842_ptr8, NULL}
1341};
1342
1343
1344
1345static uint8_t sw842_get_byte(const char *buf, int bit)
1346{
1347 uint8_t tmpl;
1348 uint16_t tmp;
1349 tmp = htons(*(uint16_t *)(buf));
1350 tmp = (uint16_t)(tmp << bit);
1351 tmp = ntohs(tmp);
1352 memcpy(&tmpl, &tmp, 1);
1353 return tmpl;
1354}
1355
1356static uint8_t sw842_get_template(const char **buf, int *bit)
1357{
1358 uint8_t byte;
1359 byte = sw842_get_byte(*buf, *bit);
1360 byte = byte >> 3;
1361 byte &= 0x1F;
1362 *buf += (*bit + 5) / 8;
1363 *bit = (*bit + 5) % 8;
1364 return byte;
1365}
1366
1367
1368static uint8_t sw842_get_repeat_count(const char **buf, int *bit)
1369{
1370 uint8_t byte;
1371 byte = sw842_get_byte(*buf, *bit);
1372 byte = byte >> 2;
1373 byte &= 0x3F;
1374 *buf += (*bit + 6) / 8;
1375 *bit = (*bit + 6) % 8;
1376 return byte;
1377}
1378
1379static uint8_t sw842_get_ptr2(const char **buf, int *bit)
1380{
1381 uint8_t ptr;
1382 ptr = sw842_get_byte(*buf, *bit);
1383 (*buf)++;
1384 return ptr;
1385}
1386
1387static uint16_t sw842_get_ptr4(const char **buf, int *bit,
1388 struct sw842_fifo *fifo)
1389{
1390 uint16_t ptr;
1391 ptr = htons(*(uint16_t *)(*buf));
1392 ptr = (uint16_t)(ptr << *bit);
1393 ptr = ptr >> 7;
1394 ptr &= 0x01FF;
1395 *buf += (*bit + 9) / 8;
1396 *bit = (*bit + 9) % 8;
1397 return ptr;
1398}
1399
1400static uint8_t sw842_get_ptr8(const char **buf, int *bit,
1401 struct sw842_fifo *fifo)
1402{
1403 return sw842_get_ptr2(buf, bit);
1404}
1405
1406
1407
1408static int sw842_data8(const char **inbuf, int *inbit,
1409 unsigned char **outbuf, struct sw842_fifo *fifo)
1410{
1411 int ret;
1412
1413 ret = sw842_data4(inbuf, inbit, outbuf, fifo);
1414 if (ret)
1415 return ret;
1416 ret = sw842_data4(inbuf, inbit, outbuf, fifo);
1417 return ret;
1418}
1419
1420static int sw842_data4(const char **inbuf, int *inbit,
1421 unsigned char **outbuf, struct sw842_fifo *fifo)
1422{
1423 int ret;
1424
1425 ret = sw842_data2(inbuf, inbit, outbuf, fifo);
1426 if (ret)
1427 return ret;
1428 ret = sw842_data2(inbuf, inbit, outbuf, fifo);
1429 return ret;
1430}
1431
1432static int sw842_data2(const char **inbuf, int *inbit,
1433 unsigned char **outbuf, struct sw842_fifo *fifo)
1434{
1435 **outbuf = sw842_get_byte(*inbuf, *inbit);
1436 (*inbuf)++;
1437 (*outbuf)++;
1438 **outbuf = sw842_get_byte(*inbuf, *inbit);
1439 (*inbuf)++;
1440 (*outbuf)++;
1441 return 0;
1442}
1443
1444static int sw842_ptr8(const char **inbuf, int *inbit,
1445 unsigned char **outbuf, struct sw842_fifo *fifo)
1446{
1447 uint8_t ptr;
1448 ptr = sw842_get_ptr8(inbuf, inbit, fifo);
1449 if (!fifo->f84_full && (ptr >= fifo->f8_count))
1450 return 1;
1451 memcpy(*outbuf, fifo->f8[ptr], 8);
1452 *outbuf += 8;
1453 return 0;
1454}
1455
1456static int sw842_ptr4(const char **inbuf, int *inbit,
1457 unsigned char **outbuf, struct sw842_fifo *fifo)
1458{
1459 uint16_t ptr;
1460 ptr = sw842_get_ptr4(inbuf, inbit, fifo);
1461 if (!fifo->f84_full && (ptr >= fifo->f4_count))
1462 return 1;
1463 memcpy(*outbuf, fifo->f4[ptr], 4);
1464 *outbuf += 4;
1465 return 0;
1466}
1467
1468static int sw842_ptr2(const char **inbuf, int *inbit,
1469 unsigned char **outbuf, struct sw842_fifo *fifo)
1470{
1471 uint8_t ptr;
1472 ptr = sw842_get_ptr2(inbuf, inbit);
1473 if (!fifo->f2_full && (ptr >= fifo->f2_count))
1474 return 1;
1475 memcpy(*outbuf, fifo->f2[ptr], 2);
1476 *outbuf += 2;
1477 return 0;
1478}
1479
1480static void sw842_copy_to_fifo(const char *buf, struct sw842_fifo *fifo)
1481{
1482 unsigned char initial_f2count = fifo->f2_count;
1483
1484 memcpy(fifo->f8[fifo->f8_count], buf, 8);
1485 fifo->f4_count += 2;
1486 fifo->f8_count += 1;
1487
1488 if (!fifo->f84_full && fifo->f4_count >= 512) {
1489 fifo->f84_full = 1;
1490 fifo->f4_count /= 512;
1491 }
1492
1493 memcpy(fifo->f2[fifo->f2_count++], buf, 2);
1494 memcpy(fifo->f2[fifo->f2_count++], buf + 2, 2);
1495 memcpy(fifo->f2[fifo->f2_count++], buf + 4, 2);
1496 memcpy(fifo->f2[fifo->f2_count++], buf + 6, 2);
1497 if (fifo->f2_count < initial_f2count)
1498 fifo->f2_full = 1;
1499}
1500
1501static int sw842_decompress(const unsigned char *src, int srclen,
1502 unsigned char *dst, int *destlen,
1503 const void *wrkmem)
1504{
1505 uint8_t tmpl;
1506 const char *inbuf;
1507 int inbit = 0;
1508 unsigned char *outbuf, *outbuf_end, *origbuf, *prevbuf;
1509 const char *inbuf_end;
1510 sw842_template_op op;
1511 int opindex;
1512 int i, repeat_count;
1513 struct sw842_fifo *fifo;
1514 int ret = 0;
1515
1516 fifo = &((struct nx842_workmem *)(wrkmem))->swfifo;
1517 memset(fifo, 0, sizeof(*fifo));
1518
1519 origbuf = NULL;
1520 inbuf = src;
1521 inbuf_end = src + srclen;
1522 outbuf = dst;
1523 outbuf_end = dst + *destlen;
1524
1525 while ((tmpl = sw842_get_template(&inbuf, &inbit)) != SW842_TMPL_EOF) {
1526 if (inbuf >= inbuf_end) {
1527 ret = -EINVAL;
1528 goto out;
1529 }
1530
1531 opindex = 0;
1532 prevbuf = origbuf;
1533 origbuf = outbuf;
1534 switch (tmpl) {
1535 case SW842_TMPL_REPEAT:
1536 if (prevbuf == NULL) {
1537 ret = -EINVAL;
1538 goto out;
1539 }
1540
1541 repeat_count = sw842_get_repeat_count(&inbuf,
1542 &inbit) + 1;
1543
1544
1545 if (inbuf > inbuf_end) {
1546 ret = -EINVAL;
1547 goto out;
1548 }
1549
1550 for (i = 0; i < repeat_count; i++) {
1551
1552 if ((outbuf + 8) > outbuf_end) {
1553 ret = -ENOSPC;
1554 goto out;
1555 }
1556
1557 memcpy(outbuf, prevbuf, 8);
1558 sw842_copy_to_fifo(outbuf, fifo);
1559 outbuf += 8;
1560 }
1561 break;
1562
1563 case SW842_TMPL_ZEROS:
1564
1565 if ((outbuf + 8) > outbuf_end) {
1566 ret = -ENOSPC;
1567 goto out;
1568 }
1569
1570 memset(outbuf, 0, 8);
1571 sw842_copy_to_fifo(outbuf, fifo);
1572 outbuf += 8;
1573 break;
1574
1575 default:
1576 if (tmpl > 25) {
1577 ret = -EINVAL;
1578 goto out;
1579 }
1580
1581
1582 if ((inbuf + 2) > inbuf_end) {
1583 ret = -EINVAL;
1584 goto out;
1585 }
1586
1587
1588 if ((outbuf + 8) > outbuf_end) {
1589 ret = -ENOSPC;
1590 goto out;
1591 }
1592
1593 while (opindex < 4 &&
1594 (op = sw842_tmpl_ops[tmpl][opindex++])
1595 != NULL) {
1596 ret = (*op)(&inbuf, &inbit, &outbuf, fifo);
1597 if (ret) {
1598 ret = -EINVAL;
1599 goto out;
1600 }
1601 sw842_copy_to_fifo(origbuf, fifo);
1602 }
1603 }
1604 }
1605
1606out:
1607 if (!ret)
1608 *destlen = (unsigned int)(outbuf - dst);
1609 else
1610 *destlen = 0;
1611
1612 return ret;
1613}
1614