1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include <linux/kernel.h>
20#include <linux/bio.h>
21#include <linux/buffer_head.h>
22#include <linux/file.h>
23#include <linux/fs.h>
24#include <linux/pagemap.h>
25#include <linux/highmem.h>
26#include <linux/time.h>
27#include <linux/init.h>
28#include <linux/string.h>
29#include <linux/backing-dev.h>
30#include <linux/mpage.h>
31#include <linux/swap.h>
32#include <linux/writeback.h>
33#include <linux/bit_spinlock.h>
34#include <linux/slab.h>
35#include "compat.h"
36#include "ctree.h"
37#include "disk-io.h"
38#include "transaction.h"
39#include "btrfs_inode.h"
40#include "volumes.h"
41#include "ordered-data.h"
42#include "compression.h"
43#include "extent_io.h"
44#include "extent_map.h"
45
46struct compressed_bio {
47
48 atomic_t pending_bios;
49
50
51 struct page **compressed_pages;
52
53
54 struct inode *inode;
55
56
57 u64 start;
58
59
60 unsigned long len;
61
62
63 unsigned long compressed_len;
64
65
66 int compress_type;
67
68
69 unsigned long nr_pages;
70
71
72 int errors;
73 int mirror_num;
74
75
76 struct bio *orig_bio;
77
78
79
80
81
82 u32 sums;
83};
84
85static inline int compressed_bio_size(struct btrfs_root *root,
86 unsigned long disk_size)
87{
88 u16 csum_size = btrfs_super_csum_size(&root->fs_info->super_copy);
89 return sizeof(struct compressed_bio) +
90 ((disk_size + root->sectorsize - 1) / root->sectorsize) *
91 csum_size;
92}
93
94static struct bio *compressed_bio_alloc(struct block_device *bdev,
95 u64 first_byte, gfp_t gfp_flags)
96{
97 int nr_vecs;
98
99 nr_vecs = bio_get_nr_vecs(bdev);
100 return btrfs_bio_alloc(bdev, first_byte >> 9, nr_vecs, gfp_flags);
101}
102
103static int check_compressed_csum(struct inode *inode,
104 struct compressed_bio *cb,
105 u64 disk_start)
106{
107 int ret;
108 struct btrfs_root *root = BTRFS_I(inode)->root;
109 struct page *page;
110 unsigned long i;
111 char *kaddr;
112 u32 csum;
113 u32 *cb_sum = &cb->sums;
114
115 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
116 return 0;
117
118 for (i = 0; i < cb->nr_pages; i++) {
119 page = cb->compressed_pages[i];
120 csum = ~(u32)0;
121
122 kaddr = kmap_atomic(page, KM_USER0);
123 csum = btrfs_csum_data(root, kaddr, csum, PAGE_CACHE_SIZE);
124 btrfs_csum_final(csum, (char *)&csum);
125 kunmap_atomic(kaddr, KM_USER0);
126
127 if (csum != *cb_sum) {
128 printk(KERN_INFO "btrfs csum failed ino %lu "
129 "extent %llu csum %u "
130 "wanted %u mirror %d\n", inode->i_ino,
131 (unsigned long long)disk_start,
132 csum, *cb_sum, cb->mirror_num);
133 ret = -EIO;
134 goto fail;
135 }
136 cb_sum++;
137
138 }
139 ret = 0;
140fail:
141 return ret;
142}
143
144
145
146
147
148
149
150
151
152
153
154static void end_compressed_bio_read(struct bio *bio, int err)
155{
156 struct compressed_bio *cb = bio->bi_private;
157 struct inode *inode;
158 struct page *page;
159 unsigned long index;
160 int ret;
161
162 if (err)
163 cb->errors = 1;
164
165
166
167
168 if (!atomic_dec_and_test(&cb->pending_bios))
169 goto out;
170
171 inode = cb->inode;
172 ret = check_compressed_csum(inode, cb, (u64)bio->bi_sector << 9);
173 if (ret)
174 goto csum_failed;
175
176
177
178
179 ret = btrfs_decompress_biovec(cb->compress_type,
180 cb->compressed_pages,
181 cb->start,
182 cb->orig_bio->bi_io_vec,
183 cb->orig_bio->bi_vcnt,
184 cb->compressed_len);
185csum_failed:
186 if (ret)
187 cb->errors = 1;
188
189
190 index = 0;
191 for (index = 0; index < cb->nr_pages; index++) {
192 page = cb->compressed_pages[index];
193 page->mapping = NULL;
194 page_cache_release(page);
195 }
196
197
198 if (cb->errors) {
199 bio_io_error(cb->orig_bio);
200 } else {
201 int bio_index = 0;
202 struct bio_vec *bvec = cb->orig_bio->bi_io_vec;
203
204
205
206
207
208 while (bio_index < cb->orig_bio->bi_vcnt) {
209 SetPageChecked(bvec->bv_page);
210 bvec++;
211 bio_index++;
212 }
213 bio_endio(cb->orig_bio, 0);
214 }
215
216
217 kfree(cb->compressed_pages);
218 kfree(cb);
219out:
220 bio_put(bio);
221}
222
223
224
225
226
227static noinline int end_compressed_writeback(struct inode *inode, u64 start,
228 unsigned long ram_size)
229{
230 unsigned long index = start >> PAGE_CACHE_SHIFT;
231 unsigned long end_index = (start + ram_size - 1) >> PAGE_CACHE_SHIFT;
232 struct page *pages[16];
233 unsigned long nr_pages = end_index - index + 1;
234 int i;
235 int ret;
236
237 while (nr_pages > 0) {
238 ret = find_get_pages_contig(inode->i_mapping, index,
239 min_t(unsigned long,
240 nr_pages, ARRAY_SIZE(pages)), pages);
241 if (ret == 0) {
242 nr_pages -= 1;
243 index += 1;
244 continue;
245 }
246 for (i = 0; i < ret; i++) {
247 end_page_writeback(pages[i]);
248 page_cache_release(pages[i]);
249 }
250 nr_pages -= ret;
251 index += ret;
252 }
253
254 return 0;
255}
256
257
258
259
260
261
262
263
264
265static void end_compressed_bio_write(struct bio *bio, int err)
266{
267 struct extent_io_tree *tree;
268 struct compressed_bio *cb = bio->bi_private;
269 struct inode *inode;
270 struct page *page;
271 unsigned long index;
272
273 if (err)
274 cb->errors = 1;
275
276
277
278
279 if (!atomic_dec_and_test(&cb->pending_bios))
280 goto out;
281
282
283
284
285 inode = cb->inode;
286 tree = &BTRFS_I(inode)->io_tree;
287 cb->compressed_pages[0]->mapping = cb->inode->i_mapping;
288 tree->ops->writepage_end_io_hook(cb->compressed_pages[0],
289 cb->start,
290 cb->start + cb->len - 1,
291 NULL, 1);
292 cb->compressed_pages[0]->mapping = NULL;
293
294 end_compressed_writeback(inode, cb->start, cb->len);
295
296
297
298
299
300
301 index = 0;
302 for (index = 0; index < cb->nr_pages; index++) {
303 page = cb->compressed_pages[index];
304 page->mapping = NULL;
305 page_cache_release(page);
306 }
307
308
309 kfree(cb->compressed_pages);
310 kfree(cb);
311out:
312 bio_put(bio);
313}
314
315
316
317
318
319
320
321
322
323
324int btrfs_submit_compressed_write(struct inode *inode, u64 start,
325 unsigned long len, u64 disk_start,
326 unsigned long compressed_len,
327 struct page **compressed_pages,
328 unsigned long nr_pages)
329{
330 struct bio *bio = NULL;
331 struct btrfs_root *root = BTRFS_I(inode)->root;
332 struct compressed_bio *cb;
333 unsigned long bytes_left;
334 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
335 int page_index = 0;
336 struct page *page;
337 u64 first_byte = disk_start;
338 struct block_device *bdev;
339 int ret;
340
341 WARN_ON(start & ((u64)PAGE_CACHE_SIZE - 1));
342 cb = kmalloc(compressed_bio_size(root, compressed_len), GFP_NOFS);
343 atomic_set(&cb->pending_bios, 0);
344 cb->errors = 0;
345 cb->inode = inode;
346 cb->start = start;
347 cb->len = len;
348 cb->mirror_num = 0;
349 cb->compressed_pages = compressed_pages;
350 cb->compressed_len = compressed_len;
351 cb->orig_bio = NULL;
352 cb->nr_pages = nr_pages;
353
354 bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
355
356 bio = compressed_bio_alloc(bdev, first_byte, GFP_NOFS);
357 bio->bi_private = cb;
358 bio->bi_end_io = end_compressed_bio_write;
359 atomic_inc(&cb->pending_bios);
360
361
362 bytes_left = compressed_len;
363 for (page_index = 0; page_index < cb->nr_pages; page_index++) {
364 page = compressed_pages[page_index];
365 page->mapping = inode->i_mapping;
366 if (bio->bi_size)
367 ret = io_tree->ops->merge_bio_hook(page, 0,
368 PAGE_CACHE_SIZE,
369 bio, 0);
370 else
371 ret = 0;
372
373 page->mapping = NULL;
374 if (ret || bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) <
375 PAGE_CACHE_SIZE) {
376 bio_get(bio);
377
378
379
380
381
382
383
384 atomic_inc(&cb->pending_bios);
385 ret = btrfs_bio_wq_end_io(root->fs_info, bio, 0);
386 BUG_ON(ret);
387
388 ret = btrfs_csum_one_bio(root, inode, bio, start, 1);
389 BUG_ON(ret);
390
391 ret = btrfs_map_bio(root, WRITE, bio, 0, 1);
392 BUG_ON(ret);
393
394 bio_put(bio);
395
396 bio = compressed_bio_alloc(bdev, first_byte, GFP_NOFS);
397 bio->bi_private = cb;
398 bio->bi_end_io = end_compressed_bio_write;
399 bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
400 }
401 if (bytes_left < PAGE_CACHE_SIZE) {
402 printk("bytes left %lu compress len %lu nr %lu\n",
403 bytes_left, cb->compressed_len, cb->nr_pages);
404 }
405 bytes_left -= PAGE_CACHE_SIZE;
406 first_byte += PAGE_CACHE_SIZE;
407 cond_resched();
408 }
409 bio_get(bio);
410
411 ret = btrfs_bio_wq_end_io(root->fs_info, bio, 0);
412 BUG_ON(ret);
413
414 ret = btrfs_csum_one_bio(root, inode, bio, start, 1);
415 BUG_ON(ret);
416
417 ret = btrfs_map_bio(root, WRITE, bio, 0, 1);
418 BUG_ON(ret);
419
420 bio_put(bio);
421 return 0;
422}
423
424static noinline int add_ra_bio_pages(struct inode *inode,
425 u64 compressed_end,
426 struct compressed_bio *cb)
427{
428 unsigned long end_index;
429 unsigned long page_index;
430 u64 last_offset;
431 u64 isize = i_size_read(inode);
432 int ret;
433 struct page *page;
434 unsigned long nr_pages = 0;
435 struct extent_map *em;
436 struct address_space *mapping = inode->i_mapping;
437 struct extent_map_tree *em_tree;
438 struct extent_io_tree *tree;
439 u64 end;
440 int misses = 0;
441
442 page = cb->orig_bio->bi_io_vec[cb->orig_bio->bi_vcnt - 1].bv_page;
443 last_offset = (page_offset(page) + PAGE_CACHE_SIZE);
444 em_tree = &BTRFS_I(inode)->extent_tree;
445 tree = &BTRFS_I(inode)->io_tree;
446
447 if (isize == 0)
448 return 0;
449
450 end_index = (i_size_read(inode) - 1) >> PAGE_CACHE_SHIFT;
451
452 while (last_offset < compressed_end) {
453 page_index = last_offset >> PAGE_CACHE_SHIFT;
454
455 if (page_index > end_index)
456 break;
457
458 rcu_read_lock();
459 page = radix_tree_lookup(&mapping->page_tree, page_index);
460 rcu_read_unlock();
461 if (page) {
462 misses++;
463 if (misses > 4)
464 break;
465 goto next;
466 }
467
468 page = __page_cache_alloc(mapping_gfp_mask(mapping) &
469 ~__GFP_FS);
470 if (!page)
471 break;
472
473 if (add_to_page_cache_lru(page, mapping, page_index,
474 GFP_NOFS)) {
475 page_cache_release(page);
476 goto next;
477 }
478
479 end = last_offset + PAGE_CACHE_SIZE - 1;
480
481
482
483
484
485 set_page_extent_mapped(page);
486 lock_extent(tree, last_offset, end, GFP_NOFS);
487 read_lock(&em_tree->lock);
488 em = lookup_extent_mapping(em_tree, last_offset,
489 PAGE_CACHE_SIZE);
490 read_unlock(&em_tree->lock);
491
492 if (!em || last_offset < em->start ||
493 (last_offset + PAGE_CACHE_SIZE > extent_map_end(em)) ||
494 (em->block_start >> 9) != cb->orig_bio->bi_sector) {
495 free_extent_map(em);
496 unlock_extent(tree, last_offset, end, GFP_NOFS);
497 unlock_page(page);
498 page_cache_release(page);
499 break;
500 }
501 free_extent_map(em);
502
503 if (page->index == end_index) {
504 char *userpage;
505 size_t zero_offset = isize & (PAGE_CACHE_SIZE - 1);
506
507 if (zero_offset) {
508 int zeros;
509 zeros = PAGE_CACHE_SIZE - zero_offset;
510 userpage = kmap_atomic(page, KM_USER0);
511 memset(userpage + zero_offset, 0, zeros);
512 flush_dcache_page(page);
513 kunmap_atomic(userpage, KM_USER0);
514 }
515 }
516
517 ret = bio_add_page(cb->orig_bio, page,
518 PAGE_CACHE_SIZE, 0);
519
520 if (ret == PAGE_CACHE_SIZE) {
521 nr_pages++;
522 page_cache_release(page);
523 } else {
524 unlock_extent(tree, last_offset, end, GFP_NOFS);
525 unlock_page(page);
526 page_cache_release(page);
527 break;
528 }
529next:
530 last_offset += PAGE_CACHE_SIZE;
531 }
532 return 0;
533}
534
535
536
537
538
539
540
541
542
543
544
545
546
547int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
548 int mirror_num, unsigned long bio_flags)
549{
550 struct extent_io_tree *tree;
551 struct extent_map_tree *em_tree;
552 struct compressed_bio *cb;
553 struct btrfs_root *root = BTRFS_I(inode)->root;
554 unsigned long uncompressed_len = bio->bi_vcnt * PAGE_CACHE_SIZE;
555 unsigned long compressed_len;
556 unsigned long nr_pages;
557 unsigned long page_index;
558 struct page *page;
559 struct block_device *bdev;
560 struct bio *comp_bio;
561 u64 cur_disk_byte = (u64)bio->bi_sector << 9;
562 u64 em_len;
563 u64 em_start;
564 struct extent_map *em;
565 int ret = -ENOMEM;
566 u32 *sums;
567
568 tree = &BTRFS_I(inode)->io_tree;
569 em_tree = &BTRFS_I(inode)->extent_tree;
570
571
572 read_lock(&em_tree->lock);
573 em = lookup_extent_mapping(em_tree,
574 page_offset(bio->bi_io_vec->bv_page),
575 PAGE_CACHE_SIZE);
576 read_unlock(&em_tree->lock);
577
578 compressed_len = em->block_len;
579 cb = kmalloc(compressed_bio_size(root, compressed_len), GFP_NOFS);
580 if (!cb)
581 goto out;
582
583 atomic_set(&cb->pending_bios, 0);
584 cb->errors = 0;
585 cb->inode = inode;
586 cb->mirror_num = mirror_num;
587 sums = &cb->sums;
588
589 cb->start = em->orig_start;
590 em_len = em->len;
591 em_start = em->start;
592
593 free_extent_map(em);
594 em = NULL;
595
596 cb->len = uncompressed_len;
597 cb->compressed_len = compressed_len;
598 cb->compress_type = extent_compress_type(bio_flags);
599 cb->orig_bio = bio;
600
601 nr_pages = (compressed_len + PAGE_CACHE_SIZE - 1) /
602 PAGE_CACHE_SIZE;
603 cb->compressed_pages = kzalloc(sizeof(struct page *) * nr_pages,
604 GFP_NOFS);
605 if (!cb->compressed_pages)
606 goto fail1;
607
608 bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
609
610 for (page_index = 0; page_index < nr_pages; page_index++) {
611 cb->compressed_pages[page_index] = alloc_page(GFP_NOFS |
612 __GFP_HIGHMEM);
613 if (!cb->compressed_pages[page_index])
614 goto fail2;
615 }
616 cb->nr_pages = nr_pages;
617
618 add_ra_bio_pages(inode, em_start + em_len, cb);
619
620
621 uncompressed_len = bio->bi_vcnt * PAGE_CACHE_SIZE;
622 cb->len = uncompressed_len;
623
624 comp_bio = compressed_bio_alloc(bdev, cur_disk_byte, GFP_NOFS);
625 if (!comp_bio)
626 goto fail2;
627 comp_bio->bi_private = cb;
628 comp_bio->bi_end_io = end_compressed_bio_read;
629 atomic_inc(&cb->pending_bios);
630
631 for (page_index = 0; page_index < nr_pages; page_index++) {
632 page = cb->compressed_pages[page_index];
633 page->mapping = inode->i_mapping;
634 page->index = em_start >> PAGE_CACHE_SHIFT;
635
636 if (comp_bio->bi_size)
637 ret = tree->ops->merge_bio_hook(page, 0,
638 PAGE_CACHE_SIZE,
639 comp_bio, 0);
640 else
641 ret = 0;
642
643 page->mapping = NULL;
644 if (ret || bio_add_page(comp_bio, page, PAGE_CACHE_SIZE, 0) <
645 PAGE_CACHE_SIZE) {
646 bio_get(comp_bio);
647
648 ret = btrfs_bio_wq_end_io(root->fs_info, comp_bio, 0);
649 BUG_ON(ret);
650
651
652
653
654
655
656
657 atomic_inc(&cb->pending_bios);
658
659 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
660 btrfs_lookup_bio_sums(root, inode, comp_bio,
661 sums);
662 }
663 sums += (comp_bio->bi_size + root->sectorsize - 1) /
664 root->sectorsize;
665
666 ret = btrfs_map_bio(root, READ, comp_bio,
667 mirror_num, 0);
668 BUG_ON(ret);
669
670 bio_put(comp_bio);
671
672 comp_bio = compressed_bio_alloc(bdev, cur_disk_byte,
673 GFP_NOFS);
674 comp_bio->bi_private = cb;
675 comp_bio->bi_end_io = end_compressed_bio_read;
676
677 bio_add_page(comp_bio, page, PAGE_CACHE_SIZE, 0);
678 }
679 cur_disk_byte += PAGE_CACHE_SIZE;
680 }
681 bio_get(comp_bio);
682
683 ret = btrfs_bio_wq_end_io(root->fs_info, comp_bio, 0);
684 BUG_ON(ret);
685
686 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM))
687 btrfs_lookup_bio_sums(root, inode, comp_bio, sums);
688
689 ret = btrfs_map_bio(root, READ, comp_bio, mirror_num, 0);
690 BUG_ON(ret);
691
692 bio_put(comp_bio);
693 return 0;
694
695fail2:
696 for (page_index = 0; page_index < nr_pages; page_index++)
697 free_page((unsigned long)cb->compressed_pages[page_index]);
698
699 kfree(cb->compressed_pages);
700fail1:
701 kfree(cb);
702out:
703 free_extent_map(em);
704 return ret;
705}
706
707static struct list_head comp_idle_workspace[BTRFS_COMPRESS_TYPES];
708static spinlock_t comp_workspace_lock[BTRFS_COMPRESS_TYPES];
709static int comp_num_workspace[BTRFS_COMPRESS_TYPES];
710static atomic_t comp_alloc_workspace[BTRFS_COMPRESS_TYPES];
711static wait_queue_head_t comp_workspace_wait[BTRFS_COMPRESS_TYPES];
712
713struct btrfs_compress_op *btrfs_compress_op[] = {
714 &btrfs_zlib_compress,
715 &btrfs_lzo_compress,
716};
717
718int __init btrfs_init_compress(void)
719{
720 int i;
721
722 for (i = 0; i < BTRFS_COMPRESS_TYPES; i++) {
723 INIT_LIST_HEAD(&comp_idle_workspace[i]);
724 spin_lock_init(&comp_workspace_lock[i]);
725 atomic_set(&comp_alloc_workspace[i], 0);
726 init_waitqueue_head(&comp_workspace_wait[i]);
727 }
728 return 0;
729}
730
731
732
733
734
735static struct list_head *find_workspace(int type)
736{
737 struct list_head *workspace;
738 int cpus = num_online_cpus();
739 int idx = type - 1;
740
741 struct list_head *idle_workspace = &comp_idle_workspace[idx];
742 spinlock_t *workspace_lock = &comp_workspace_lock[idx];
743 atomic_t *alloc_workspace = &comp_alloc_workspace[idx];
744 wait_queue_head_t *workspace_wait = &comp_workspace_wait[idx];
745 int *num_workspace = &comp_num_workspace[idx];
746again:
747 spin_lock(workspace_lock);
748 if (!list_empty(idle_workspace)) {
749 workspace = idle_workspace->next;
750 list_del(workspace);
751 (*num_workspace)--;
752 spin_unlock(workspace_lock);
753 return workspace;
754
755 }
756 if (atomic_read(alloc_workspace) > cpus) {
757 DEFINE_WAIT(wait);
758
759 spin_unlock(workspace_lock);
760 prepare_to_wait(workspace_wait, &wait, TASK_UNINTERRUPTIBLE);
761 if (atomic_read(alloc_workspace) > cpus && !*num_workspace)
762 schedule();
763 finish_wait(workspace_wait, &wait);
764 goto again;
765 }
766 atomic_inc(alloc_workspace);
767 spin_unlock(workspace_lock);
768
769 workspace = btrfs_compress_op[idx]->alloc_workspace();
770 if (IS_ERR(workspace)) {
771 atomic_dec(alloc_workspace);
772 wake_up(workspace_wait);
773 }
774 return workspace;
775}
776
777
778
779
780
781static void free_workspace(int type, struct list_head *workspace)
782{
783 int idx = type - 1;
784 struct list_head *idle_workspace = &comp_idle_workspace[idx];
785 spinlock_t *workspace_lock = &comp_workspace_lock[idx];
786 atomic_t *alloc_workspace = &comp_alloc_workspace[idx];
787 wait_queue_head_t *workspace_wait = &comp_workspace_wait[idx];
788 int *num_workspace = &comp_num_workspace[idx];
789
790 spin_lock(workspace_lock);
791 if (*num_workspace < num_online_cpus()) {
792 list_add_tail(workspace, idle_workspace);
793 (*num_workspace)++;
794 spin_unlock(workspace_lock);
795 goto wake;
796 }
797 spin_unlock(workspace_lock);
798
799 btrfs_compress_op[idx]->free_workspace(workspace);
800 atomic_dec(alloc_workspace);
801wake:
802 if (waitqueue_active(workspace_wait))
803 wake_up(workspace_wait);
804}
805
806
807
808
809static void free_workspaces(void)
810{
811 struct list_head *workspace;
812 int i;
813
814 for (i = 0; i < BTRFS_COMPRESS_TYPES; i++) {
815 while (!list_empty(&comp_idle_workspace[i])) {
816 workspace = comp_idle_workspace[i].next;
817 list_del(workspace);
818 btrfs_compress_op[i]->free_workspace(workspace);
819 atomic_dec(&comp_alloc_workspace[i]);
820 }
821 }
822}
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843int btrfs_compress_pages(int type, struct address_space *mapping,
844 u64 start, unsigned long len,
845 struct page **pages,
846 unsigned long nr_dest_pages,
847 unsigned long *out_pages,
848 unsigned long *total_in,
849 unsigned long *total_out,
850 unsigned long max_out)
851{
852 struct list_head *workspace;
853 int ret;
854
855 workspace = find_workspace(type);
856 if (IS_ERR(workspace))
857 return -1;
858
859 ret = btrfs_compress_op[type-1]->compress_pages(workspace, mapping,
860 start, len, pages,
861 nr_dest_pages, out_pages,
862 total_in, total_out,
863 max_out);
864 free_workspace(type, workspace);
865 return ret;
866}
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884int btrfs_decompress_biovec(int type, struct page **pages_in, u64 disk_start,
885 struct bio_vec *bvec, int vcnt, size_t srclen)
886{
887 struct list_head *workspace;
888 int ret;
889
890 workspace = find_workspace(type);
891 if (IS_ERR(workspace))
892 return -ENOMEM;
893
894 ret = btrfs_compress_op[type-1]->decompress_biovec(workspace, pages_in,
895 disk_start,
896 bvec, vcnt, srclen);
897 free_workspace(type, workspace);
898 return ret;
899}
900
901
902
903
904
905
906int btrfs_decompress(int type, unsigned char *data_in, struct page *dest_page,
907 unsigned long start_byte, size_t srclen, size_t destlen)
908{
909 struct list_head *workspace;
910 int ret;
911
912 workspace = find_workspace(type);
913 if (IS_ERR(workspace))
914 return -ENOMEM;
915
916 ret = btrfs_compress_op[type-1]->decompress(workspace, data_in,
917 dest_page, start_byte,
918 srclen, destlen);
919
920 free_workspace(type, workspace);
921 return ret;
922}
923
924void btrfs_exit_compress(void)
925{
926 free_workspaces();
927}
928
929
930
931
932
933
934
935
936int btrfs_decompress_buf2page(char *buf, unsigned long buf_start,
937 unsigned long total_out, u64 disk_start,
938 struct bio_vec *bvec, int vcnt,
939 unsigned long *page_index,
940 unsigned long *pg_offset)
941{
942 unsigned long buf_offset;
943 unsigned long current_buf_start;
944 unsigned long start_byte;
945 unsigned long working_bytes = total_out - buf_start;
946 unsigned long bytes;
947 char *kaddr;
948 struct page *page_out = bvec[*page_index].bv_page;
949
950
951
952
953
954 start_byte = page_offset(page_out) - disk_start;
955
956
957 if (total_out <= start_byte)
958 return 1;
959
960
961
962
963
964 if (total_out > start_byte && buf_start < start_byte) {
965 buf_offset = start_byte - buf_start;
966 working_bytes -= buf_offset;
967 } else {
968 buf_offset = 0;
969 }
970 current_buf_start = buf_start;
971
972
973 while (working_bytes > 0) {
974 bytes = min(PAGE_CACHE_SIZE - *pg_offset,
975 PAGE_CACHE_SIZE - buf_offset);
976 bytes = min(bytes, working_bytes);
977 kaddr = kmap_atomic(page_out, KM_USER0);
978 memcpy(kaddr + *pg_offset, buf + buf_offset, bytes);
979 kunmap_atomic(kaddr, KM_USER0);
980 flush_dcache_page(page_out);
981
982 *pg_offset += bytes;
983 buf_offset += bytes;
984 working_bytes -= bytes;
985 current_buf_start += bytes;
986
987
988 if (*pg_offset == PAGE_CACHE_SIZE) {
989 (*page_index)++;
990 if (*page_index >= vcnt)
991 return 0;
992
993 page_out = bvec[*page_index].bv_page;
994 *pg_offset = 0;
995 start_byte = page_offset(page_out) - disk_start;
996
997
998
999
1000
1001 if (total_out <= start_byte)
1002 return 1;
1003
1004
1005
1006
1007
1008
1009 if (total_out > start_byte &&
1010 current_buf_start < start_byte) {
1011 buf_offset = start_byte - buf_start;
1012 working_bytes = total_out - start_byte;
1013 current_buf_start = buf_start + buf_offset;
1014 }
1015 }
1016 }
1017
1018 return 1;
1019}
1020