Compare commits
No commits in common. 'c9' and 'i8c' have entirely different histories.
@ -1 +1 @@
|
||||
SOURCES/4.4-git.1.tar.gz
|
||||
SOURCES/squashfs4.3.tar.gz
|
||||
|
||||
@ -1 +1 @@
|
||||
ccdbdb36be907de767019f2f35e985c6fad1bc2c SOURCES/4.4-git.1.tar.gz
|
||||
a615979db9cee82e4a934a1455577f597d290b41 SOURCES/squashfs4.3.tar.gz
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
--- squashfs-tools/mksquashfs.c.orig 2014-09-13 11:08:27.352318167 -0500
|
||||
+++ squashfs-tools/mksquashfs.c 2014-09-13 11:09:36.701132044 -0500
|
||||
@@ -2055,7 +2055,7 @@
|
||||
|
||||
inline int is_fragment(struct inode_info *inode)
|
||||
{
|
||||
- int file_size = inode->buf.st_size;
|
||||
+ off_t file_size = inode->buf.st_size;
|
||||
|
||||
/*
|
||||
* If this block is to be compressed differently to the
|
||||
@ -0,0 +1,159 @@
|
||||
From 55f7ba830d40d438f0b0663a505e0c227fc68b6b Mon Sep 17 00:00:00 2001
|
||||
From: Phillip Lougher <phillip@squashfs.org.uk>
|
||||
Date: Tue, 10 Jun 2014 21:51:52 +0100
|
||||
Subject: mksquashfs: fix phys mem calculation for 32-bit processes on
|
||||
PAE/64-bit kernels
|
||||
|
||||
When adding the code to base default memory usage on physical memory
|
||||
(by default use 25% of physical memory), I made an oversight. I assumed
|
||||
the process would be able to address 25% of physical memory.
|
||||
|
||||
However, for 32-bit processes running on a PAE kernel or 64-bit kernel,
|
||||
25% of physical memory can easily exceed the addressible memory for a
|
||||
32-bit process, e.g. if a machine has 24 GB of physical memory, the
|
||||
code would asume the process could easily use 6 GB.
|
||||
|
||||
A 32-bit process by definition can only address 4 GB (32-bit pointers).
|
||||
But, due to the typical kernel/user-space split (1GB/3GB, or 2GB/2GB)
|
||||
on PAE kernels, a 32-bit process may only be able to address 2 GB.
|
||||
|
||||
So, if Mksquashfs is a 32-bit application running on a PAE/64-bit kernel,
|
||||
the code assumes it can address much more memory than it really can, which
|
||||
means it runs out of memory.
|
||||
|
||||
The fix is to impose a maximum default limit on 32-bit kernels, or
|
||||
otherwise to never use a value more than 25% of the address space. If
|
||||
we assume the maximum address space is 2 GB, then the maximum becomes
|
||||
512 MB. But, given most kernels used the 1GB/3GB split, that may be
|
||||
unduely conservative, and 25% of 3 GB (756 MB) may be better. This
|
||||
patch compromises on 640 MB, which is mid-way between the 512 MB and 756 MB
|
||||
values. It is also the fixed default value previously used by Mksquashfs.
|
||||
|
||||
This patch also alters the code which imposes a maximum size. Previously
|
||||
it was believed limiting to the physical memory size was adequate. But
|
||||
obviously this needs to be updated to take into account a 32-bit process
|
||||
may only be able to address 2 GB. In the process I've also taken the
|
||||
opportunity to limit all requests to no more than 75% of physical memory.
|
||||
|
||||
Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
|
||||
|
||||
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
index 86f82bb..5370ecf 100644
|
||||
--- a/squashfs-tools/mksquashfs.c
|
||||
+++ b/squashfs-tools/mksquashfs.c
|
||||
@@ -304,7 +304,7 @@ void restorefs();
|
||||
struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
|
||||
void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
|
||||
unsigned short get_checksum_mem(char *buff, int bytes);
|
||||
-int get_physical_memory();
|
||||
+void check_usable_phys_mem(int total_mem);
|
||||
|
||||
|
||||
void prep_exit()
|
||||
@@ -4053,11 +4053,7 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
|
||||
BAD_ERROR("Queue sizes rediculously too large\n");
|
||||
total_mem += fwriteq;
|
||||
|
||||
- if(total_mem > get_physical_memory()) {
|
||||
- ERROR("Total queue sizes larger than physical memory.\n");
|
||||
- ERROR("Mksquashfs will exhaust physical memory and thrash.\n");
|
||||
- BAD_ERROR("Queues too large\n");
|
||||
- }
|
||||
+ check_usable_phys_mem(total_mem);
|
||||
|
||||
/*
|
||||
* convert from queue size in Mbytes to queue size in
|
||||
@@ -4879,6 +4875,72 @@ int get_physical_memory()
|
||||
}
|
||||
|
||||
|
||||
+void check_usable_phys_mem(int total_mem)
|
||||
+{
|
||||
+ /*
|
||||
+ * We want to allow users to use as much of their physical
|
||||
+ * memory as they wish. However, for practical reasons there are
|
||||
+ * limits which need to be imposed, to protect users from themselves
|
||||
+ * and to prevent people from using Mksquashfs as a DOS attack by using
|
||||
+ * all physical memory. Mksquashfs uses memory to cache data from disk
|
||||
+ * to optimise performance. It is pointless to ask it to use more
|
||||
+ * than 75% of physical memory, as this causes thrashing and it is thus
|
||||
+ * self-defeating.
|
||||
+ */
|
||||
+ int mem = get_physical_memory();
|
||||
+
|
||||
+ mem = (mem >> 1) + (mem >> 2); /* 75% */
|
||||
+
|
||||
+ if(total_mem > mem) {
|
||||
+ ERROR("Total memory requested is more than 75%% of physical "
|
||||
+ "memory.\n");
|
||||
+ ERROR("Mksquashfs uses memory to cache data from disk to "
|
||||
+ "optimise performance.\n");
|
||||
+ ERROR("It is pointless to ask it to use more than this amount "
|
||||
+ "of memory, as this\n");
|
||||
+ ERROR("causes thrashing and it is thus self-defeating.\n");
|
||||
+ BAD_ERROR("Requested memory size too large\n");
|
||||
+ }
|
||||
+
|
||||
+ if(sizeof(void *) == 4 && total_mem > 2048) {
|
||||
+ /*
|
||||
+ * If we're running on a kernel with PAE or on a 64-bit kernel,
|
||||
+ * then the 75% physical memory limit can still easily exceed
|
||||
+ * the addressable memory by this process.
|
||||
+ *
|
||||
+ * Due to the typical kernel/user-space split (1GB/3GB, or
|
||||
+ * 2GB/2GB), we have to conservatively assume the 32-bit
|
||||
+ * processes can only address 2-3GB. So refuse if the user
|
||||
+ * tries to allocate more than 2GB.
|
||||
+ */
|
||||
+ ERROR("Total memory requested may exceed maximum "
|
||||
+ "addressable memory by this process\n");
|
||||
+ BAD_ERROR("Requested memory size too large\n");
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+
|
||||
+int get_default_phys_mem()
|
||||
+{
|
||||
+ int mem = get_physical_memory() / SQUASHFS_TAKE;
|
||||
+
|
||||
+ if(sizeof(void *) == 4 && mem > 640) {
|
||||
+ /*
|
||||
+ * If we're running on a kernel with PAE or on a 64-bit kernel,
|
||||
+ * the default memory usage can exceed the addressable
|
||||
+ * memory by this process.
|
||||
+ * Due to the typical kernel/user-space split (1GB/3GB, or
|
||||
+ * 2GB/2GB), we have to conservatively assume the 32-bit
|
||||
+ * processes can only address 2-3GB. So limit the default
|
||||
+ * usage to 640M, which gives room for other data.
|
||||
+ */
|
||||
+ mem = 640;
|
||||
+ }
|
||||
+
|
||||
+ return mem;
|
||||
+}
|
||||
+
|
||||
+
|
||||
void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
|
||||
int *fwriteq)
|
||||
{
|
||||
@@ -4890,7 +4952,7 @@ void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
|
||||
|
||||
|
||||
#define VERSION() \
|
||||
- printf("mksquashfs version 4.3 (2014/05/12)\n");\
|
||||
+ printf("mksquashfs version 4.3-git (2014/06/09)\n");\
|
||||
printf("copyright (C) 2014 Phillip Lougher "\
|
||||
"<phillip@squashfs.org.uk>\n\n"); \
|
||||
printf("This program is free software; you can redistribute it and/or"\
|
||||
@@ -4918,7 +4980,7 @@ int main(int argc, char *argv[])
|
||||
int fragq;
|
||||
int bwriteq;
|
||||
int fwriteq;
|
||||
- int total_mem = get_physical_memory() / SQUASHFS_TAKE;
|
||||
+ int total_mem = get_default_phys_mem();
|
||||
int progress = TRUE;
|
||||
int force_progress = FALSE;
|
||||
struct file_buffer **fragment = NULL;
|
||||
--
|
||||
cgit v0.10.1
|
||||
|
||||
@ -0,0 +1,95 @@
|
||||
diff -Nupr a/squashfs-tools/action.c b/squashfs-tools/action.c
|
||||
--- a/squashfs-tools/action.c 2014-05-09 23:54:13.000000000 -0500
|
||||
+++ b/squashfs-tools/action.c 2019-04-18 10:59:53.140496887 -0500
|
||||
@@ -1094,8 +1094,14 @@ static int parse_sym_mode_args(struct ac
|
||||
for (i = 0; i < args; i++) {
|
||||
struct mode_data *entry = parse_sym_mode_arg(argv[i]);
|
||||
|
||||
- if (entry == NULL)
|
||||
+ if (entry == NULL) {
|
||||
+ while (head) {
|
||||
+ cur = head;
|
||||
+ head = head->next;
|
||||
+ free(cur);
|
||||
+ }
|
||||
return 0;
|
||||
+ }
|
||||
|
||||
if (cur) {
|
||||
cur->next = entry;
|
||||
diff -Nupr a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
--- a/squashfs-tools/mksquashfs.c 2019-04-18 10:51:05.975460126 -0500
|
||||
+++ b/squashfs-tools/mksquashfs.c 2019-04-18 11:04:16.860682497 -0500
|
||||
@@ -3537,11 +3537,11 @@ void dir_scan2(struct dir_info *dir, str
|
||||
char *subpath = strdup(subpathname(dir_ent));
|
||||
struct dir_info *sub_dir = scan1_opendir("", subpath,
|
||||
dir->depth + 1);
|
||||
+ free(subpath);
|
||||
if(sub_dir == NULL) {
|
||||
ERROR_START("Could not create pseudo directory "
|
||||
"\"%s\"", pseudo_ent->pathname);
|
||||
ERROR_EXIT(", skipping...\n");
|
||||
- free(subpath);
|
||||
pseudo_ino --;
|
||||
continue;
|
||||
}
|
||||
diff -Nupr a/squashfs-tools/read_fs.c b/squashfs-tools/read_fs.c
|
||||
--- a/squashfs-tools/read_fs.c 2014-05-09 23:54:13.000000000 -0500
|
||||
+++ b/squashfs-tools/read_fs.c 2019-04-18 11:06:32.499233451 -0500
|
||||
@@ -974,7 +974,8 @@ long long read_filesystem(char *root_nam
|
||||
|
||||
error:
|
||||
free(id_table);
|
||||
- free(inode_table);
|
||||
+ if (inode_table)
|
||||
+ free(inode_table);
|
||||
free(directory_table);
|
||||
return 0;
|
||||
}
|
||||
diff -Nupr a/squashfs-tools/unsquash-4.c b/squashfs-tools/unsquash-4.c
|
||||
--- a/squashfs-tools/unsquash-4.c 2019-04-18 10:51:05.972460124 -0500
|
||||
+++ b/squashfs-tools/unsquash-4.c 2019-04-18 11:32:54.600160166 -0500
|
||||
@@ -50,9 +50,11 @@ int read_fragment_table_4(long long *dir
|
||||
"fragment table index\n");
|
||||
|
||||
fragment_table = malloc(bytes);
|
||||
- if(fragment_table == NULL)
|
||||
+ if(fragment_table == NULL) {
|
||||
+ free(fragment_table_index);
|
||||
EXIT_UNSQUASH("read_fragment_table: failed to allocate "
|
||||
"fragment table\n");
|
||||
+ }
|
||||
|
||||
res = read_fs_bytes(fd, sBlk.s.fragment_table_start,
|
||||
SQUASHFS_FRAGMENT_INDEX_BYTES(sBlk.s.fragments),
|
||||
@@ -60,7 +62,7 @@ int read_fragment_table_4(long long *dir
|
||||
if(res == FALSE) {
|
||||
ERROR("read_fragment_table: failed to read fragment table "
|
||||
"index\n");
|
||||
- return FALSE;
|
||||
+ goto out;
|
||||
}
|
||||
SQUASHFS_INSWAP_FRAGMENT_INDEXES(fragment_table_index, indexes);
|
||||
|
||||
@@ -75,7 +77,8 @@ int read_fragment_table_4(long long *dir
|
||||
if(length == FALSE) {
|
||||
ERROR("read_fragment_table: failed to read fragment "
|
||||
"table index\n");
|
||||
- return FALSE;
|
||||
+ res = FALSE;
|
||||
+ goto out;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +86,10 @@ int read_fragment_table_4(long long *dir
|
||||
SQUASHFS_INSWAP_FRAGMENT_ENTRY(&fragment_table[i]);
|
||||
|
||||
*directory_table_end = fragment_table_index[0];
|
||||
- return TRUE;
|
||||
+ res = TRUE;
|
||||
+out:
|
||||
+ free(fragment_table_index);
|
||||
+ return res;
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
--- a/squashfs-tools/Makefile 2019-07-24 13:52:09.552920269 -0400
|
||||
+++ b/squashfs-tools/Makefile 2019-07-24 23:52:47.839665687 -0400
|
||||
@@ -107,6 +107,8 @@ XATTR_DEFAULT = 1
|
||||
# End of BUILD options section #
|
||||
###############################################
|
||||
|
||||
+EXTRA_LDFLAGS += -z now -pie
|
||||
+
|
||||
INCLUDEDIR = -I.
|
||||
INSTALL_DIR = /usr/local/bin
|
||||
|
||||
@ -1,15 +1,14 @@
|
||||
diff -Nupr a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
--- a/squashfs-tools/mksquashfs.c 2020-10-30 06:13:56.000000000 -0500
|
||||
+++ b/squashfs-tools/mksquashfs.c 2022-01-12 10:58:37.704782953 -0600
|
||||
@@ -290,6 +290,7 @@ int all_time_opt = FALSE;
|
||||
int clamping = TRUE;
|
||||
--- a/squashfs-tools/mksquashfs.c 2019-08-06 11:03:34.071402614 -0400
|
||||
+++ b/squashfs-tools/mksquashfs.c 2019-08-06 11:09:54.063340318 -0400
|
||||
@@ -267,6 +267,7 @@ pthread_mutex_t pos_mutex = PTHREAD_MUTE
|
||||
pthread_mutex_t dup_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
/* user options that control parallelisation */
|
||||
+#define MAX_CPUS 256
|
||||
int processors = -1;
|
||||
int bwriter_size;
|
||||
|
||||
@@ -4385,6 +4386,10 @@ void initialise_threads(int readq, int f
|
||||
@@ -4124,6 +4125,10 @@ void initialise_threads(int readq, int f
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -0,0 +1,13 @@
|
||||
--- a/squashfs-tools/mksquashfs.c
|
||||
+++ b/squashfs-tools/mksquashfs.c
|
||||
@@ -2447,8 +2447,8 @@ void *frag_deflator(void *arg)
|
||||
write_buffer->block = bytes;
|
||||
bytes += compressed_size;
|
||||
fragments_outstanding --;
|
||||
- pthread_mutex_unlock(&fragment_mutex);
|
||||
queue_put(to_writer, write_buffer);
|
||||
+ pthread_mutex_unlock(&fragment_mutex);
|
||||
TRACE("Writing fragment %lld, uncompressed size %d, "
|
||||
"compressed size %d\n", file_buffer->block,
|
||||
file_buffer->size, compressed_size);
|
||||
|
||||
@ -0,0 +1,20 @@
|
||||
--- a/squashfs-tools/unsquashfs.c 2021-01-14 14:34:49.658184361 -0600
|
||||
+++ b/squashfs-tools/unsquashfs.c 2021-01-14 14:50:19.604100949 -0600
|
||||
@@ -822,8 +822,6 @@ int set_attributes(char *pathname, int m
|
||||
{
|
||||
struct utimbuf times = { time, time };
|
||||
|
||||
- write_xattr(pathname, xattr);
|
||||
-
|
||||
if(utime(pathname, ×) == -1) {
|
||||
ERROR("set_attributes: failed to set time on %s, because %s\n",
|
||||
pathname, strerror(errno));
|
||||
@@ -846,6 +844,8 @@ int set_attributes(char *pathname, int m
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
+ write_xattr(pathname, xattr);
|
||||
+
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
@ -0,0 +1,29 @@
|
||||
diff --git a/squashfs-tools/unsquash-4.c b/squashfs-tools/unsquash-4.c
|
||||
index ecdaac796f09..2c0cf63daf67 100644
|
||||
--- a/squashfs-tools/unsquash-4.c
|
||||
+++ b/squashfs-tools/unsquash-4.c
|
||||
@@ -31,9 +31,9 @@ static unsigned int *id_table;
|
||||
int read_fragment_table_4(long long *directory_table_end)
|
||||
{
|
||||
int res, i;
|
||||
- int bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
|
||||
- int indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
|
||||
- long long fragment_table_index[indexes];
|
||||
+ size_t bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
|
||||
+ size_t indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
|
||||
+ long long *fragment_table_index;
|
||||
|
||||
TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
|
||||
"from 0x%llx\n", sBlk.s.fragments, indexes,
|
||||
@@ -44,6 +44,11 @@ int read_fragment_table_4(long long *directory_table_end)
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
+ fragment_table_index = malloc(indexes*sizeof(long long));
|
||||
+ if(fragment_table_index == NULL)
|
||||
+ EXIT_UNSQUASH("read_fragment_table: failed to allocate "
|
||||
+ "fragment table index\n");
|
||||
+
|
||||
fragment_table = malloc(bytes);
|
||||
if(fragment_table == NULL)
|
||||
EXIT_UNSQUASH("read_fragment_table: failed to allocate "
|
||||
@ -0,0 +1,11 @@
|
||||
--- squashfs-tools/unsquash-4.c.orig 2015-06-24 14:23:22.270710744 -0500
|
||||
+++ squashfs-tools/unsquash-4.c 2015-06-24 14:24:13.671243487 -0500
|
||||
@@ -35,7 +35,7 @@
|
||||
size_t indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
|
||||
long long *fragment_table_index;
|
||||
|
||||
- TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
|
||||
+ TRACE("read_fragment_table: %u fragments, reading %zu fragment indexes "
|
||||
"from 0x%llx\n", sBlk.s.fragments, indexes,
|
||||
sBlk.s.fragment_table_start);
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
From 604b607d8ac91eb8afc0b6e3d917d5c073096103 Mon Sep 17 00:00:00 2001
|
||||
From: Phillip Lougher <phillip@squashfs.org.uk>
|
||||
Date: Wed, 11 Jun 2014 04:51:37 +0100
|
||||
Subject: mksquashfs: ensure value does not overflow a signed int in -mem
|
||||
option
|
||||
|
||||
Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
|
||||
|
||||
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
index 5370ecf..9676dc8 100644
|
||||
--- a/squashfs-tools/mksquashfs.c
|
||||
+++ b/squashfs-tools/mksquashfs.c
|
||||
@@ -5193,7 +5193,16 @@ print_compressor_options:
|
||||
argv[0]);
|
||||
exit(1);
|
||||
}
|
||||
- /* convert from bytes to Mbytes */
|
||||
+
|
||||
+ /*
|
||||
+ * convert from bytes to Mbytes, ensuring the value
|
||||
+ * does not overflow a signed int
|
||||
+ */
|
||||
+ if(number >= (1LL << 51)) {
|
||||
+ ERROR("%s: -mem invalid mem size\n", argv[0]);
|
||||
+ exit(1);
|
||||
+ }
|
||||
+
|
||||
total_mem = number / 1048576;
|
||||
if(total_mem < (SQUASHFS_LOWMEM / SQUASHFS_TAKE)) {
|
||||
ERROR("%s: -mem should be %d Mbytes or "
|
||||
--
|
||||
cgit v0.10.1
|
||||
|
||||
@ -0,0 +1,22 @@
|
||||
diff -Nupr squashfs4.3.orig/squashfs-tools/mksquashfs.c squashfs4.3/squashfs-tools/mksquashfs.c
|
||||
--- squashfs4.3.orig/squashfs-tools/mksquashfs.c 2018-08-20 10:47:49.286289155 -0400
|
||||
+++ squashfs4.3/squashfs-tools/mksquashfs.c 2018-08-20 10:49:28.348541210 -0400
|
||||
@@ -35,6 +35,7 @@
|
||||
#include <stddef.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
+#include <sys/sysmacros.h>
|
||||
#include <fcntl.h>
|
||||
#include <errno.h>
|
||||
#include <dirent.h>
|
||||
diff -Nupr squashfs4.3.orig/squashfs-tools/unsquashfs.c squashfs4.3/squashfs-tools/unsquashfs.c
|
||||
--- squashfs4.3.orig/squashfs-tools/unsquashfs.c 2018-08-20 10:47:49.283289208 -0400
|
||||
+++ squashfs4.3/squashfs-tools/unsquashfs.c 2018-08-20 10:58:44.238732579 -0400
|
||||
@@ -35,6 +35,7 @@
|
||||
#include <sys/types.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/resource.h>
|
||||
+#include <sys/sysmacros.h>
|
||||
#include <limits.h>
|
||||
#include <ctype.h>
|
||||
|
||||
Loading…
Reference in new issue