1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/kernel.h> 4 #include <linux/string.h> 5 #include <linux/err.h> 6 #include <linux/slab.h> 7 #include <linux/wait.h> 8 #include <linux/sched.h> 9 #include <linux/cpuhotplug.h> 10 #include <linux/vmalloc.h> 11 12 #include "zcomp.h" 13 14 #include "backend_lzo.h" 15 #include "backend_lzorle.h" 16 #include "backend_lz4.h" 17 #include "backend_lz4hc.h" 18 #include "backend_zstd.h" 19 #include "backend_deflate.h" 20 #include "backend_842.h" 21 22 static const struct zcomp_ops *backends[] = { 23 #if IS_ENABLED(CONFIG_ZRAM_BACKEND_LZO) 24 &backend_lzorle, 25 &backend_lzo, 26 #endif 27 #if IS_ENABLED(CONFIG_ZRAM_BACKEND_LZ4) 28 &backend_lz4, 29 #endif 30 #if IS_ENABLED(CONFIG_ZRAM_BACKEND_LZ4HC) 31 &backend_lz4hc, 32 #endif 33 #if IS_ENABLED(CONFIG_ZRAM_BACKEND_ZSTD) 34 &backend_zstd, 35 #endif 36 #if IS_ENABLED(CONFIG_ZRAM_BACKEND_DEFLATE) 37 &backend_deflate, 38 #endif 39 #if IS_ENABLED(CONFIG_ZRAM_BACKEND_842) 40 &backend_842, 41 #endif 42 NULL 43 }; 44 45 static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm) 46 { 47 comp->ops->destroy_ctx(&zstrm->ctx); 48 vfree(zstrm->local_copy); 49 vfree(zstrm->buffer); 50 zstrm->buffer = NULL; 51 } 52 53 static int zcomp_strm_init(struct zcomp *comp, struct zcomp_strm *zstrm) 54 { 55 int ret; 56 57 ret = comp->ops->create_ctx(comp->params, &zstrm->ctx); 58 if (ret) 59 return ret; 60 61 zstrm->local_copy = vzalloc(PAGE_SIZE); 62 /* 63 * allocate 2 pages. 1 for compressed data, plus 1 extra for the 64 * case when compressed size is larger than the original one 65 */ 66 zstrm->buffer = vzalloc(2 * PAGE_SIZE); 67 if (!zstrm->buffer || !zstrm->local_copy) { 68 zcomp_strm_free(comp, zstrm); 69 return -ENOMEM; 70 } 71 return 0; 72 } 73 74 static const struct zcomp_ops *lookup_backend_ops(const char *comp) 75 { 76 int i = 0; 77 78 while (backends[i]) { 79 if (sysfs_streq(comp, backends[i]->name)) 80 break; 81 i++; 82 } 83 return backends[i]; 84 } 85 86 bool zcomp_available_algorithm(const char *comp) 87 { 88 return lookup_backend_ops(comp) != NULL; 89 } 90 91 /* show available compressors */ 92 ssize_t zcomp_available_show(const char *comp, char *buf) 93 { 94 ssize_t sz = 0; 95 int i; 96 97 for (i = 0; i < ARRAY_SIZE(backends) - 1; i++) { 98 if (!strcmp(comp, backends[i]->name)) { 99 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 100 "[%s] ", backends[i]->name); 101 } else { 102 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, 103 "%s ", backends[i]->name); 104 } 105 } 106 107 sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); 108 return sz; 109 } 110 111 struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) 112 { 113 for (;;) { 114 struct zcomp_strm *zstrm = raw_cpu_ptr(comp->stream); 115 116 /* 117 * Inspired by zswap 118 * 119 * stream is returned with ->mutex locked which prevents 120 * cpu_dead() from releasing this stream under us, however 121 * there is still a race window between raw_cpu_ptr() and 122 * mutex_lock(), during which we could have been migrated 123 * from a CPU that has already destroyed its stream. If 124 * so then unlock and re-try on the current CPU. 125 */ 126 mutex_lock(&zstrm->lock); 127 if (likely(zstrm->buffer)) 128 return zstrm; 129 mutex_unlock(&zstrm->lock); 130 } 131 } 132 133 void zcomp_stream_put(struct zcomp_strm *zstrm) 134 { 135 mutex_unlock(&zstrm->lock); 136 } 137 138 int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, 139 const void *src, unsigned int *dst_len) 140 { 141 struct zcomp_req req = { 142 .src = src, 143 .dst = zstrm->buffer, 144 .src_len = PAGE_SIZE, 145 .dst_len = 2 * PAGE_SIZE, 146 }; 147 int ret; 148 149 might_sleep(); 150 ret = comp->ops->compress(comp->params, &zstrm->ctx, &req); 151 if (!ret) 152 *dst_len = req.dst_len; 153 return ret; 154 } 155 156 int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm, 157 const void *src, unsigned int src_len, void *dst) 158 { 159 struct zcomp_req req = { 160 .src = src, 161 .dst = dst, 162 .src_len = src_len, 163 .dst_len = PAGE_SIZE, 164 }; 165 166 might_sleep(); 167 return comp->ops->decompress(comp->params, &zstrm->ctx, &req); 168 } 169 170 int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) 171 { 172 struct zcomp *comp = hlist_entry(node, struct zcomp, node); 173 struct zcomp_strm *zstrm = per_cpu_ptr(comp->stream, cpu); 174 int ret; 175 176 ret = zcomp_strm_init(comp, zstrm); 177 if (ret) 178 pr_err("Can't allocate a compression stream\n"); 179 return ret; 180 } 181 182 int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node) 183 { 184 struct zcomp *comp = hlist_entry(node, struct zcomp, node); 185 struct zcomp_strm *zstrm = per_cpu_ptr(comp->stream, cpu); 186 187 mutex_lock(&zstrm->lock); 188 zcomp_strm_free(comp, zstrm); 189 mutex_unlock(&zstrm->lock); 190 return 0; 191 } 192 193 static int zcomp_init(struct zcomp *comp, struct zcomp_params *params) 194 { 195 int ret, cpu; 196 197 comp->stream = alloc_percpu(struct zcomp_strm); 198 if (!comp->stream) 199 return -ENOMEM; 200 201 comp->params = params; 202 ret = comp->ops->setup_params(comp->params); 203 if (ret) 204 goto cleanup; 205 206 for_each_possible_cpu(cpu) 207 mutex_init(&per_cpu_ptr(comp->stream, cpu)->lock); 208 209 ret = cpuhp_state_add_instance(CPUHP_ZCOMP_PREPARE, &comp->node); 210 if (ret < 0) 211 goto cleanup; 212 213 return 0; 214 215 cleanup: 216 comp->ops->release_params(comp->params); 217 free_percpu(comp->stream); 218 return ret; 219 } 220 221 void zcomp_destroy(struct zcomp *comp) 222 { 223 cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node); 224 comp->ops->release_params(comp->params); 225 free_percpu(comp->stream); 226 kfree(comp); 227 } 228 229 struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params) 230 { 231 struct zcomp *comp; 232 int error; 233 234 /* 235 * The backends array has a sentinel NULL value, so the minimum 236 * size is 1. In order to be valid the array, apart from the 237 * sentinel NULL element, should have at least one compression 238 * backend selected. 239 */ 240 BUILD_BUG_ON(ARRAY_SIZE(backends) <= 1); 241 242 comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); 243 if (!comp) 244 return ERR_PTR(-ENOMEM); 245 246 comp->ops = lookup_backend_ops(alg); 247 if (!comp->ops) { 248 kfree(comp); 249 return ERR_PTR(-EINVAL); 250 } 251 252 error = zcomp_init(comp, params); 253 if (error) { 254 kfree(comp); 255 return ERR_PTR(error); 256 } 257 return comp; 258 } 259