1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/kernel.h>
24#include <linux/slab.h>
25#include <linux/zlib.h>
26#include <linux/zutil.h>
27#include <linux/mm.h>
28#include <linux/init.h>
29#include <linux/err.h>
30#include <linux/sched.h>
31#include <linux/pagemap.h>
32#include <linux/bio.h>
33#include <linux/refcount.h>
34#include "compression.h"
35
36struct workspace {
37 z_stream strm;
38 char *buf;
39 struct list_head list;
40};
41
42static void zlib_free_workspace(struct list_head *ws)
43{
44 struct workspace *workspace = list_entry(ws, struct workspace, list);
45
46 kvfree(workspace->strm.workspace);
47 kfree(workspace->buf);
48 kfree(workspace);
49}
50
51static struct list_head *zlib_alloc_workspace(void)
52{
53 struct workspace *workspace;
54 int workspacesize;
55
56 workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
57 if (!workspace)
58 return ERR_PTR(-ENOMEM);
59
60 workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
61 zlib_inflate_workspacesize());
62 workspace->strm.workspace = kvmalloc(workspacesize, GFP_KERNEL);
63 workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
64 if (!workspace->strm.workspace || !workspace->buf)
65 goto fail;
66
67 INIT_LIST_HEAD(&workspace->list);
68
69 return &workspace->list;
70fail:
71 zlib_free_workspace(&workspace->list);
72 return ERR_PTR(-ENOMEM);
73}
74
75static int zlib_compress_pages(struct list_head *ws,
76 struct address_space *mapping,
77 u64 start,
78 struct page **pages,
79 unsigned long *out_pages,
80 unsigned long *total_in,
81 unsigned long *total_out)
82{
83 struct workspace *workspace = list_entry(ws, struct workspace, list);
84 int ret;
85 char *data_in;
86 char *cpage_out;
87 int nr_pages = 0;
88 struct page *in_page = NULL;
89 struct page *out_page = NULL;
90 unsigned long bytes_left;
91 unsigned long len = *total_out;
92 unsigned long nr_dest_pages = *out_pages;
93 const unsigned long max_out = nr_dest_pages * PAGE_SIZE;
94
95 *out_pages = 0;
96 *total_out = 0;
97 *total_in = 0;
98
99 if (Z_OK != zlib_deflateInit(&workspace->strm, 3)) {
100 pr_warn("BTRFS: deflateInit failed\n");
101 ret = -EIO;
102 goto out;
103 }
104
105 workspace->strm.total_in = 0;
106 workspace->strm.total_out = 0;
107
108 in_page = find_get_page(mapping, start >> PAGE_SHIFT);
109 data_in = kmap(in_page);
110
111 out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
112 if (out_page == NULL) {
113 ret = -ENOMEM;
114 goto out;
115 }
116 cpage_out = kmap(out_page);
117 pages[0] = out_page;
118 nr_pages = 1;
119
120 workspace->strm.next_in = data_in;
121 workspace->strm.next_out = cpage_out;
122 workspace->strm.avail_out = PAGE_SIZE;
123 workspace->strm.avail_in = min(len, PAGE_SIZE);
124
125 while (workspace->strm.total_in < len) {
126 ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH);
127 if (ret != Z_OK) {
128 pr_debug("BTRFS: deflate in loop returned %d\n",
129 ret);
130 zlib_deflateEnd(&workspace->strm);
131 ret = -EIO;
132 goto out;
133 }
134
135
136 if (workspace->strm.total_in > 8192 &&
137 workspace->strm.total_in <
138 workspace->strm.total_out) {
139 ret = -E2BIG;
140 goto out;
141 }
142
143
144
145
146 if (workspace->strm.avail_out == 0) {
147 kunmap(out_page);
148 if (nr_pages == nr_dest_pages) {
149 out_page = NULL;
150 ret = -E2BIG;
151 goto out;
152 }
153 out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
154 if (out_page == NULL) {
155 ret = -ENOMEM;
156 goto out;
157 }
158 cpage_out = kmap(out_page);
159 pages[nr_pages] = out_page;
160 nr_pages++;
161 workspace->strm.avail_out = PAGE_SIZE;
162 workspace->strm.next_out = cpage_out;
163 }
164
165 if (workspace->strm.total_in >= len)
166 break;
167
168
169 if (workspace->strm.avail_in == 0) {
170 if (workspace->strm.total_out > max_out)
171 break;
172
173 bytes_left = len - workspace->strm.total_in;
174 kunmap(in_page);
175 put_page(in_page);
176
177 start += PAGE_SIZE;
178 in_page = find_get_page(mapping,
179 start >> PAGE_SHIFT);
180 data_in = kmap(in_page);
181 workspace->strm.avail_in = min(bytes_left,
182 PAGE_SIZE);
183 workspace->strm.next_in = data_in;
184 }
185 }
186 workspace->strm.avail_in = 0;
187 ret = zlib_deflate(&workspace->strm, Z_FINISH);
188 zlib_deflateEnd(&workspace->strm);
189
190 if (ret != Z_STREAM_END) {
191 ret = -EIO;
192 goto out;
193 }
194
195 if (workspace->strm.total_out >= workspace->strm.total_in) {
196 ret = -E2BIG;
197 goto out;
198 }
199
200 ret = 0;
201 *total_out = workspace->strm.total_out;
202 *total_in = workspace->strm.total_in;
203out:
204 *out_pages = nr_pages;
205 if (out_page)
206 kunmap(out_page);
207
208 if (in_page) {
209 kunmap(in_page);
210 put_page(in_page);
211 }
212 return ret;
213}
214
215static int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
216{
217 struct workspace *workspace = list_entry(ws, struct workspace, list);
218 int ret = 0, ret2;
219 int wbits = MAX_WBITS;
220 char *data_in;
221 size_t total_out = 0;
222 unsigned long page_in_index = 0;
223 size_t srclen = cb->compressed_len;
224 unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
225 unsigned long buf_start;
226 struct page **pages_in = cb->compressed_pages;
227 u64 disk_start = cb->start;
228 struct bio *orig_bio = cb->orig_bio;
229
230 data_in = kmap(pages_in[page_in_index]);
231 workspace->strm.next_in = data_in;
232 workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE);
233 workspace->strm.total_in = 0;
234
235 workspace->strm.total_out = 0;
236 workspace->strm.next_out = workspace->buf;
237 workspace->strm.avail_out = PAGE_SIZE;
238
239
240
241 if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
242 ((data_in[0] & 0x0f) == Z_DEFLATED) &&
243 !(((data_in[0]<<8) + data_in[1]) % 31)) {
244
245 wbits = -((data_in[0] >> 4) + 8);
246 workspace->strm.next_in += 2;
247 workspace->strm.avail_in -= 2;
248 }
249
250 if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
251 pr_warn("BTRFS: inflateInit failed\n");
252 kunmap(pages_in[page_in_index]);
253 return -EIO;
254 }
255 while (workspace->strm.total_in < srclen) {
256 ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);
257 if (ret != Z_OK && ret != Z_STREAM_END)
258 break;
259
260 buf_start = total_out;
261 total_out = workspace->strm.total_out;
262
263
264 if (buf_start == total_out)
265 break;
266
267 ret2 = btrfs_decompress_buf2page(workspace->buf, buf_start,
268 total_out, disk_start,
269 orig_bio);
270 if (ret2 == 0) {
271 ret = 0;
272 goto done;
273 }
274
275 workspace->strm.next_out = workspace->buf;
276 workspace->strm.avail_out = PAGE_SIZE;
277
278 if (workspace->strm.avail_in == 0) {
279 unsigned long tmp;
280 kunmap(pages_in[page_in_index]);
281 page_in_index++;
282 if (page_in_index >= total_pages_in) {
283 data_in = NULL;
284 break;
285 }
286 data_in = kmap(pages_in[page_in_index]);
287 workspace->strm.next_in = data_in;
288 tmp = srclen - workspace->strm.total_in;
289 workspace->strm.avail_in = min(tmp,
290 PAGE_SIZE);
291 }
292 }
293 if (ret != Z_STREAM_END)
294 ret = -EIO;
295 else
296 ret = 0;
297done:
298 zlib_inflateEnd(&workspace->strm);
299 if (data_in)
300 kunmap(pages_in[page_in_index]);
301 if (!ret)
302 zero_fill_bio(orig_bio);
303 return ret;
304}
305
306static int zlib_decompress(struct list_head *ws, unsigned char *data_in,
307 struct page *dest_page,
308 unsigned long start_byte,
309 size_t srclen, size_t destlen)
310{
311 struct workspace *workspace = list_entry(ws, struct workspace, list);
312 int ret = 0;
313 int wbits = MAX_WBITS;
314 unsigned long bytes_left;
315 unsigned long total_out = 0;
316 unsigned long pg_offset = 0;
317 char *kaddr;
318
319 destlen = min_t(unsigned long, destlen, PAGE_SIZE);
320 bytes_left = destlen;
321
322 workspace->strm.next_in = data_in;
323 workspace->strm.avail_in = srclen;
324 workspace->strm.total_in = 0;
325
326 workspace->strm.next_out = workspace->buf;
327 workspace->strm.avail_out = PAGE_SIZE;
328 workspace->strm.total_out = 0;
329
330
331 if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
332 ((data_in[0] & 0x0f) == Z_DEFLATED) &&
333 !(((data_in[0]<<8) + data_in[1]) % 31)) {
334
335 wbits = -((data_in[0] >> 4) + 8);
336 workspace->strm.next_in += 2;
337 workspace->strm.avail_in -= 2;
338 }
339
340 if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) {
341 pr_warn("BTRFS: inflateInit failed\n");
342 return -EIO;
343 }
344
345 while (bytes_left > 0) {
346 unsigned long buf_start;
347 unsigned long buf_offset;
348 unsigned long bytes;
349
350 ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);
351 if (ret != Z_OK && ret != Z_STREAM_END)
352 break;
353
354 buf_start = total_out;
355 total_out = workspace->strm.total_out;
356
357 if (total_out == buf_start) {
358 ret = -EIO;
359 break;
360 }
361
362 if (total_out <= start_byte)
363 goto next;
364
365 if (total_out > start_byte && buf_start < start_byte)
366 buf_offset = start_byte - buf_start;
367 else
368 buf_offset = 0;
369
370 bytes = min(PAGE_SIZE - pg_offset,
371 PAGE_SIZE - buf_offset);
372 bytes = min(bytes, bytes_left);
373
374 kaddr = kmap_atomic(dest_page);
375 memcpy(kaddr + pg_offset, workspace->buf + buf_offset, bytes);
376 kunmap_atomic(kaddr);
377
378 pg_offset += bytes;
379 bytes_left -= bytes;
380next:
381 workspace->strm.next_out = workspace->buf;
382 workspace->strm.avail_out = PAGE_SIZE;
383 }
384
385 if (ret != Z_STREAM_END && bytes_left != 0)
386 ret = -EIO;
387 else
388 ret = 0;
389
390 zlib_inflateEnd(&workspace->strm);
391
392
393
394
395
396
397 if (pg_offset < destlen) {
398 kaddr = kmap_atomic(dest_page);
399 memset(kaddr + pg_offset, 0, destlen - pg_offset);
400 kunmap_atomic(kaddr);
401 }
402 return ret;
403}
404
405const struct btrfs_compress_op btrfs_zlib_compress = {
406 .alloc_workspace = zlib_alloc_workspace,
407 .free_workspace = zlib_free_workspace,
408 .compress_pages = zlib_compress_pages,
409 .decompress_bio = zlib_decompress_bio,
410 .decompress = zlib_decompress,
411};
412