mirror
/
openwrt-packages
mirror of https://git.openwrt.org/feed/packages.git
1
0
Fork 0
Code Releases Activity
openwrt-packages/net/rsync/patches/002-Add-checksum-choice-opt...

651 lines
20 KiB
Diff
Raw Blame History

commit a5a7d3a297b836387b0ac677383bdddaf2ac3598
Author: Wayne Davison <wayned@samba.org>
Date: Sun May 1 16:32:45 2016 -0700
Add --checksum-choice option to choose the checksum algorithms.
diff --git a/authenticate.c b/authenticate.c
index 5f125de..d60ee20 100644
--- a/authenticate.c
+++ b/authenticate.c
@@ -71,7 +71,7 @@ static void gen_challenge(const char *addr, char *challenge)
SIVAL(input, 20, tv.tv_usec);
SIVAL(input, 24, getpid());
- sum_init(0);
+ sum_init(-1, 0);
sum_update(input, sizeof input);
len = sum_end(digest);
@@ -85,7 +85,7 @@ static void generate_hash(const char *in, const char *challenge, char *out)
char buf[MAX_DIGEST_LEN];
int len;
- sum_init(0);
+ sum_init(-1, 0);
sum_update(in, strlen(in));
sum_update(challenge, strlen(challenge));
len = sum_end(buf);
diff --git a/checksum.c b/checksum.c
index bac775d..8b38833 100644
--- a/checksum.c
+++ b/checksum.c
@@ -24,6 +24,76 @@
extern int checksum_seed;
extern int protocol_version;
extern int proper_seed_order;
+extern char *checksum_choice;
+
+#define CSUM_NONE 0
+#define CSUM_ARCHAIC 1
+#define CSUM_MD4_BUSTED 2
+#define CSUM_MD4_OLD 3
+#define CSUM_MD4 4
+#define CSUM_MD5 5
+
+int xfersum_type = 0; /* used for the file transfer checksums */
+int checksum_type = 0; /* used for the pre-transfer (--checksum) checksums */
+
+/* Returns 1 if --whole-file must be enabled. */
+int parse_checksum_choice(void)
+{
+ char *cp = checksum_choice ? strchr(checksum_choice, ',') : NULL;
+ if (cp) {
+ xfersum_type = parse_csum_name(checksum_choice, cp - checksum_choice);
+ checksum_type = parse_csum_name(cp+1, -1);
+ } else
+ xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1);
+ return xfersum_type == CSUM_NONE;
+}
+
+int parse_csum_name(const char *name, int len)
+{
+ if (len < 0 && name)
+ len = strlen(name);
+
+ if (!name || (len == 4 && strncasecmp(name, "auto", 4) == 0)) {
+ if (protocol_version >= 30)
+ return CSUM_MD5;
+ if (protocol_version >= 27)
+ return CSUM_MD4_OLD;
+ if (protocol_version >= 21)
+ return CSUM_MD4_BUSTED;
+ return CSUM_ARCHAIC;
+ }
+ if (len == 3 && strncasecmp(name, "md4", 3) == 0)
+ return CSUM_MD4;
+ if (len == 3 && strncasecmp(name, "md5", 3) == 0)
+ return CSUM_MD5;
+ if (len == 4 && strncasecmp(name, "none", 4) == 0)
+ return CSUM_NONE;
+
+ rprintf(FERROR, "unknown checksum name: %s\n", name);
+ exit_cleanup(RERR_UNSUPPORTED);
+}
+
+int csum_len_for_type(int cst)
+{
+ switch (cst) {
+ case CSUM_NONE:
+ return 1;
+ case CSUM_ARCHAIC:
+ return 2;
+ case CSUM_MD4:
+ case CSUM_MD4_OLD:
+ case CSUM_MD4_BUSTED:
+ return MD4_DIGEST_LEN;
+ case CSUM_MD5:
+ return MD5_DIGEST_LEN;
+ }
+ return 0;
+}
+
+int canonical_checksum(int csum_type)
+{
+ return csum_type >= CSUM_MD4 ? 1 : 0;
+}
/*
a simple 32 bit checksum that can be upadted from either end
@@ -47,12 +117,12 @@ uint32 get_checksum1(char *buf1, int32 len)
return (s1 & 0xffff) + (s2 << 16);
}
-
void get_checksum2(char *buf, int32 len, char *sum)
{
md_context m;
- if (protocol_version >= 30) {
+ switch (xfersum_type) {
+ case CSUM_MD5: {
uchar seedbuf[4];
md5_begin(&m);
if (proper_seed_order) {
@@ -69,7 +139,11 @@ void get_checksum2(char *buf, int32 len, char *sum)
}
}
md5_result(&m, (uchar *)sum);
- } else {
+ break;
+ }
+ case CSUM_MD4:
+ case CSUM_MD4_OLD:
+ case CSUM_MD4_BUSTED: {
int32 i;
static char *buf1;
static int32 len1;
@@ -100,10 +174,12 @@ void get_checksum2(char *buf, int32 len, char *sum)
* are multiples of 64. This is fixed by calling mdfour_update()
* even when there are no more bytes.
*/
- if (len - i > 0 || protocol_version >= 27)
+ if (len - i > 0 || xfersum_type != CSUM_MD4_BUSTED)
mdfour_update(&m, (uchar *)(buf1+i), len-i);
mdfour_result(&m, (uchar *)sum);
+ break;
+ }
}
}
@@ -123,7 +199,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
- if (protocol_version >= 30) {
+ switch (checksum_type) {
+ case CSUM_MD5:
md5_begin(&m);
for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
@@ -136,7 +213,10 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
md5_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
md5_result(&m, (uchar *)sum);
- } else {
+ break;
+ case CSUM_MD4:
+ case CSUM_MD4_OLD:
+ case CSUM_MD4_BUSTED:
mdfour_begin(&m);
for (i = 0; i + CSUM_CHUNK <= len; i += CSUM_CHUNK) {
@@ -149,10 +229,14 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
* are multiples of 64. This is fixed by calling mdfour_update()
* even when there are no more bytes. */
remainder = (int32)(len - i);
- if (remainder > 0 || protocol_version >= 27)
+ if (remainder > 0 || checksum_type != CSUM_MD4_BUSTED)
mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
mdfour_result(&m, (uchar *)sum);
+ break;
+ default:
+ rprintf(FERROR, "invalid checksum-choice for the --checksum option (%d)\n", checksum_type);
+ exit_cleanup(RERR_UNSUPPORTED);
}
close(fd);
@@ -161,18 +245,33 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
static int32 sumresidue;
static md_context md;
+static int cursum_type;
-void sum_init(int seed)
+void sum_init(int csum_type, int seed)
{
char s[4];
- if (protocol_version >= 30)
+ if (csum_type < 0)
+ csum_type = parse_csum_name(NULL, 0);
+ cursum_type = csum_type;
+
+ switch (csum_type) {
+ case CSUM_MD5:
md5_begin(&md);
- else {
+ break;
+ case CSUM_MD4:
+ mdfour_begin(&md);
+ sumresidue = 0;
+ break;
+ case CSUM_MD4_OLD:
+ case CSUM_MD4_BUSTED:
mdfour_begin(&md);
sumresidue = 0;
SIVAL(s, 0, seed);
sum_update(s, 4);
+ break;
+ case CSUM_NONE:
+ break;
}
}
@@ -186,13 +285,17 @@ void sum_init(int seed)
**/
void sum_update(const char *p, int32 len)
{
- if (protocol_version >= 30) {
+ switch (cursum_type) {
+ case CSUM_MD5:
md5_update(&md, (uchar *)p, len);
- } else {
+ break;
+ case CSUM_MD4:
+ case CSUM_MD4_OLD:
+ case CSUM_MD4_BUSTED:
if (len + sumresidue < CSUM_CHUNK) {
memcpy(md.buffer + sumresidue, p, len);
sumresidue += len;
- return;
+ break;
}
if (sumresidue) {
@@ -212,20 +315,32 @@ void sum_update(const char *p, int32 len)
sumresidue = len;
if (sumresidue)
memcpy(md.buffer, p, sumresidue);
+ break;
+ case CSUM_NONE:
+ break;
}
}
int sum_end(char *sum)
{
- if (protocol_version >= 30) {
+ switch (cursum_type) {
+ case CSUM_MD5:
md5_result(&md, (uchar *)sum);
- return MD5_DIGEST_LEN;
- } else {
- if (sumresidue || protocol_version >= 27)
+ break;
+ case CSUM_MD4:
+ case CSUM_MD4_OLD:
+ mdfour_update(&md, (uchar *)md.buffer, sumresidue);
+ mdfour_result(&md, (uchar *)sum);
+ break;
+ case CSUM_MD4_BUSTED:
+ if (sumresidue)
mdfour_update(&md, (uchar *)md.buffer, sumresidue);
-
mdfour_result(&md, (uchar *)sum);
-
- return MD4_DIGEST_LEN;
+ break;
+ case CSUM_NONE:
+ *sum = '\0';
+ break;
}
+
+ return csum_len_for_type(cursum_type);
}
diff --git a/compat.c b/compat.c
index c792312..505cb7f 100644
--- a/compat.c
+++ b/compat.c
@@ -338,4 +338,6 @@ void setup_protocol(int f_out,int f_in)
} else {
checksum_seed = read_int(f_in);
}
+
+ init_flist();
}
diff --git a/flist.c b/flist.c
index c1e48b3..acb95f7 100644
--- a/flist.c
+++ b/flist.c
@@ -33,6 +33,7 @@ extern int am_sender;
extern int am_generator;
extern int inc_recurse;
extern int always_checksum;
+extern int checksum_type;
extern int module_id;
extern int ignore_errors;
extern int numeric_ids;
@@ -137,9 +138,8 @@ void init_flist(void)
rprintf(FINFO, "FILE_STRUCT_LEN=%d, EXTRA_LEN=%d\n",
(int)FILE_STRUCT_LEN, (int)EXTRA_LEN);
}
- checksum_len = protocol_version < 21 ? 2
- : protocol_version < 30 ? MD4_DIGEST_LEN
- : MD5_DIGEST_LEN;
+ parse_checksum_choice(); /* Sets checksum_type && xfersum_type */
+ checksum_len = csum_len_for_type(checksum_type);
}
static int show_filelist_p(void)
diff --git a/log.c b/log.c
index 24256de..f7da1e5 100644
--- a/log.c
+++ b/log.c
@@ -31,12 +31,13 @@ extern int am_generator;
extern int local_server;
extern int quiet;
extern int module_id;
-extern int checksum_len;
extern int allow_8bit_chars;
extern int protocol_version;
extern int always_checksum;
extern int preserve_times;
extern int msgs2stderr;
+extern int xfersum_type;
+extern int checksum_type;
extern int stdout_format_has_i;
extern int stdout_format_has_o_or_i;
extern int logfile_format_has_i;
@@ -46,6 +47,7 @@ extern int64 total_data_written;
extern int64 total_data_read;
extern mode_t orig_umask;
extern char *auth_user;
+extern char *checksum_choice;
extern char *stdout_format;
extern char *logfile_format;
extern char *logfile_name;
@@ -669,13 +671,15 @@ static void log_formatted(enum logcode code, const char *format, const char *op,
n = buf2;
break;
case 'C':
- if (protocol_version >= 30
- && (iflags & ITEM_TRANSFER
- || (always_checksum && S_ISREG(file->mode)))) {
- const char *sum = iflags & ITEM_TRANSFER
- ? sender_file_sum : F_SUM(file);
- n = sum_as_hex(sum);
- } else {
+ n = NULL;
+ if (S_ISREG(file->mode)) {
+ if (always_checksum && canonical_checksum(checksum_type))
+ n = sum_as_hex(checksum_type, F_SUM(file));
+ else if (iflags & ITEM_TRANSFER && canonical_checksum(xfersum_type))
+ n = sum_as_hex(xfersum_type, sender_file_sum);
+ }
+ if (!n) {
+ int checksum_len = csum_len_for_type(always_checksum ? checksum_type : xfersum_type);
memset(buf2, ' ', checksum_len*2);
buf2[checksum_len*2] = '\0';
n = buf2;
diff --git a/main.c b/main.c
index 3132aa9..3908ccf 100644
--- a/main.c
+++ b/main.c
@@ -1595,8 +1595,6 @@ int main(int argc,char *argv[])
* that implement getcwd that way "pwd" can't be found after chroot. */
change_dir(NULL, CD_NORMAL);
- init_flist();
-
if ((write_batch || read_batch) && !am_server) {
if (write_batch)
write_batch_shell_file(orig_argc, orig_argv, argc);
diff --git a/match.c b/match.c
index b15f2eb..ff10310 100644
--- a/match.c
+++ b/match.c
@@ -24,7 +24,7 @@
extern int checksum_seed;
extern int append_mode;
-extern int checksum_len;
+extern int xfersum_type;
int updating_basis_file;
char sender_file_sum[MAX_DIGEST_LEN];
@@ -360,13 +360,15 @@ static void hash_search(int f,struct sum_struct *s,
**/
void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
{
+ int checksum_len;
+
last_match = 0;
false_alarms = 0;
hash_hits = 0;
matches = 0;
data_transfer = 0;
- sum_init(checksum_seed);
+ sum_init(xfersum_type, checksum_seed);
if (append_mode > 0) {
if (append_mode == 2) {
@@ -407,8 +409,7 @@ void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len)
matched(f, s, buf, len, -1);
}
- if (sum_end(sender_file_sum) != checksum_len)
- overflow_exit("checksum_len"); /* Impossible... */
+ checksum_len = sum_end(sender_file_sum);
/* If we had a read error, send a bad checksum. We use all bits
* off as long as the checksum doesn't happen to be that, in
diff --git a/options.c b/options.c
index 4a5cdc8..308443b 100644
--- a/options.c
+++ b/options.c
@@ -182,6 +182,7 @@ char *dest_option = NULL;
static int remote_option_alloc = 0;
int remote_option_cnt = 0;
const char **remote_options = NULL;
+const char *checksum_choice = NULL;
int quiet = 0;
int output_motd = 1;
@@ -721,6 +722,7 @@ void usage(enum logcode F)
#endif
rprintf(F," -n, --dry-run perform a trial run with no changes made\n");
rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n");
+ rprintf(F," --checksum-choice=STR choose the checksum algorithms\n");
rprintf(F," -x, --one-file-system don't cross filesystem boundaries\n");
rprintf(F," -B, --block-size=SIZE force a fixed checksum block-size\n");
rprintf(F," -e, --rsh=COMMAND specify the remote shell to use\n");
@@ -953,6 +955,7 @@ static struct poptOption long_options[] = {
{"cvs-exclude", 'C', POPT_ARG_NONE, &cvs_exclude, 0, 0, 0 },
{"whole-file", 'W', POPT_ARG_VAL, &whole_file, 1, 0, 0 },
{"no-whole-file", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 },
+ {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
{"no-W", 0, POPT_ARG_VAL, &whole_file, 0, 0, 0 },
{"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 },
{"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
@@ -1814,6 +1817,15 @@ int parse_arguments(int *argc_p, const char ***argv_p)
}
}
+ if (checksum_choice && strcmp(checksum_choice, "auto") != 0 && strcmp(checksum_choice, "auto,auto") != 0) {
+ /* Call this early to verify the args and figure out if we need to force
+ * --whole-file. Note that the parse function will get called again later,
+ * just in case an "auto" choice needs to know the protocol_version. */
+ if (parse_checksum_choice())
+ whole_file = 1;
+ } else
+ checksum_choice = NULL;
+
if (human_readable > 1 && argc == 2 && !am_server) {
/* Allow the old meaning of 'h' (--help) on its own. */
usage(FINFO);
@@ -2597,6 +2609,12 @@ void server_options(char **args, int *argc_p)
args[ac++] = arg;
}
+ if (checksum_choice) {
+ if (asprintf(&arg, "--checksum-choice=%s", checksum_choice) < 0)
+ goto oom;
+ args[ac++] = arg;
+ }
+
if (am_sender) {
if (max_delete > 0) {
if (asprintf(&arg, "--max-delete=%d", max_delete) < 0)
diff --git a/receiver.c b/receiver.c
index 4ea4c09..f9b97dd 100644
--- a/receiver.c
+++ b/receiver.c
@@ -48,11 +48,11 @@ extern int append_mode;
extern int sparse_files;
extern int preallocate_files;
extern int keep_partial;
-extern int checksum_len;
extern int checksum_seed;
extern int inplace;
extern int allowed_lull;
extern int delay_updates;
+extern int xfersum_type;
extern mode_t orig_umask;
extern struct stats stats;
extern char *tmpdir;
@@ -234,6 +234,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
static char file_sum1[MAX_DIGEST_LEN];
struct map_struct *mapbuf;
struct sum_struct sum;
+ int checksum_len;
int32 len;
OFF_T offset = 0;
OFF_T offset2;
@@ -269,7 +270,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
} else
mapbuf = NULL;
- sum_init(checksum_seed);
+ sum_init(xfersum_type, checksum_seed);
if (append_mode > 0) {
OFF_T j;
@@ -393,8 +394,7 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
exit_cleanup(RERR_FILEIO);
}
- if (sum_end(file_sum1) != checksum_len)
- overflow_exit("checksum_len"); /* Impossible... */
+ checksum_len = sum_end(file_sum1);
if (mapbuf)
unmap_file(mapbuf);
diff --git a/rsync.yo b/rsync.yo
index 8971828..0ec5e55 100644
--- a/rsync.yo
+++ b/rsync.yo
@@ -380,6 +380,7 @@ to the detailed description below for a complete description. verb(
--preallocate allocate dest files before writing
-n, --dry-run perform a trial run with no changes made
-W, --whole-file copy files whole (w/o delta-xfer algorithm)
+ --checksum-choice=STR choose the checksum algorithms
-x, --one-file-system don't cross filesystem boundaries
-B, --block-size=SIZE force a fixed checksum block-size
-e, --rsh=COMMAND specify the remote shell to use
@@ -1280,14 +1281,27 @@ the "bytes sent", "bytes received", "literal data", and "matched data"
statistics are too small, and the "speedup" value is equivalent to a run
where no file transfers were needed.
-dit(bf(-W, --whole-file)) With this option rsync's delta-transfer algorithm
-is not used and the whole file is sent as-is instead. The transfer may be
+dit(bf(-W, --whole-file)) This option disables rsync's delta-transfer algorithm,
+which causes all transferred files to be sent whole. The transfer may be
faster if this option is used when the bandwidth between the source and
destination machines is higher than the bandwidth to disk (especially when the
"disk" is actually a networked filesystem). This is the default when both
the source and destination are specified as local paths, but only if no
batch-writing option is in effect.
+dit(bf(--checksum-choice=STR)) This option overrides the checksum algoriths.
+If one algorithm name is specified, it is used for both the transfer checksums
+and (assuming bf(--checksum) is specifed) the pre-transfer checksumming. If two
+comma-separated names are supplied, the first name affects the transfer
+checksums, and the second name affects the pre-transfer checksumming.
+
+The algorithm choices are "auto", "md4", "md5", and "none". If "none" is
+specified for the first name, the bf(--whole-file) option is forced on and no
+checksum verification is performed on the transferred data. If "none" is
+specified for the second name, the bf(--checksum) option cannot be used. The
+"auto" option is the default, where rsync bases its algorithm choice on the
+protocol version (for backward compatibility with older rsync versions).
+
dit(bf(-x, --one-file-system)) This tells rsync to avoid crossing a
filesystem boundary when recursing. This does not limit the user's ability
to specify items to copy from multiple filesystems, just rsync's recursion
diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
index 1813354..64156ae 100644
--- a/rsyncd.conf.yo
+++ b/rsyncd.conf.yo
@@ -656,7 +656,7 @@ quote(itemization(
it() %b the number of bytes actually transferred
it() %B the permission bits of the file (e.g. rwxrwxrwt)
it() %c the total size of the block checksums received for the basis file (only when sending)
- it() %C the full-file MD5 checksum if bf(--checksum) is enabled or a file was transferred (only for protocol 30 or above).
+ it() %C the full-file checksum if it is known for the file. For older rsync protocols/versions, the checksum was salted, and is thus not a useful value (and is not displayed when that is the case). For the checksum to output for a file, either the bf(--checksum) option must be in-effect or the file must have been transferred without a salted checksum being used. See the bf(--checksum-choice) option for a way to choose the algorithm.
it() %f the filename (long form on sender; no trailing "/")
it() %G the gid of the file (decimal) or "DEFAULT"
it() %h the remote host name (only available for a daemon)
diff --git a/t_stub.c b/t_stub.c
index 6002250..26951a6 100644
--- a/t_stub.c
+++ b/t_stub.c
@@ -25,7 +25,6 @@ int modify_window = 0;
int preallocate_files = 0;
int protect_args = 0;
int module_id = -1;
-int checksum_len = 0;
int relative_paths = 0;
int module_dirlen = 0;
int preserve_acls = 0;
@@ -97,3 +96,8 @@ filter_rule_list daemon_filter_list;
{
return "tester";
}
+
+ int csum_len_for_type(int cst)
+{
+ return cst ? 16 : 1;
+}
diff --git a/util2.c b/util2.c
index cc368af..a892e51 100644
--- a/util2.c
+++ b/util2.c
@@ -25,8 +25,6 @@
#include "itypes.h"
#include "inums.h"
-extern int checksum_len;
-
/**
* Sleep for a specified number of milliseconds.
*
@@ -79,10 +77,11 @@ void *_realloc_array(void *ptr, unsigned int size, size_t num)
return realloc(ptr, size * num);
}
-const char *sum_as_hex(const char *sum)
+const char *sum_as_hex(int csum_type, const char *sum)
{
static char buf[MAX_DIGEST_LEN*2+1];
int i, x1, x2;
+ int checksum_len = csum_len_for_type(csum_type);
char *c = buf + checksum_len*2;
assert(c - buf < (int)sizeof buf);
diff --git a/xattrs.c b/xattrs.c
index 57833e5..6a77a0b 100644
--- a/xattrs.c
+++ b/xattrs.c
@@ -258,7 +258,7 @@ static int rsync_xal_get(const char *fname, item_list *xalp)
if (datum_len > MAX_FULL_DATUM) {
/* For large datums, we store a flag and a checksum. */
name_offset = 1 + MAX_DIGEST_LEN;
- sum_init(checksum_seed);
+ sum_init(-1, checksum_seed);
sum_update(ptr, datum_len);
free(ptr);
@@ -821,7 +821,7 @@ static int rsync_xal_set(const char *fname, item_list *xalp,
goto still_abbrev;
}
- sum_init(checksum_seed);
+ sum_init(-1, checksum_seed);
sum_update(ptr, len);
sum_end(sum);
if (memcmp(sum, rxas[i].datum + 1, MAX_DIGEST_LEN) != 0) {
View Git Blame Copy Permalink