agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH] Reorganize pglz compression code 35+ messages / 10 participants [nested] [flat]
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH] Reorganize pglz compression code @ 2019-06-27 18:18 Andrey <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Andrey @ 2019-06-27 18:18 UTC (permalink / raw) This patch accumulates several changes: 1. Convert macro-functions to regular functions for readability 2. Use more compact hash table with uint16 indexes instead of pointers 3. Avoid prev pointer in hash table 4. Use 4-byte comparisons in a search instead of 1-byte Total speedup about x1.4 --- src/common/pg_lzcompress.c | 475 ++++++++++++++++++++++--------------- 1 file changed, 288 insertions(+), 187 deletions(-) diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index fdd527f757..ffa4fe549e 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -118,13 +118,13 @@ * our 4K maximum look-back distance is too small. * * The compressor creates a table for lists of positions. - * For each input position (except the last 3), a hash key is + * For each input position (except the last 4), a hash key is * built from the 4 next input bytes and the position remembered * in the appropriate list. Thus, the table points to linked * lists of likely to be at least in the first 4 characters * matching strings. This is done on the fly while the input * is compressed into the output area. Table entries are only - * kept for the last 4096 input positions, since we cannot use + * kept for the last 4094 input positions, since we cannot use * back-pointers larger than that anyway. The size of the hash * table is chosen based on the size of the input - a larger table * has a larger startup cost, as it needs to be initialized to @@ -146,17 +146,15 @@ * used for the next tag in the output. * * For each subsequent entry in the history list, the "good_match" - * is lowered by 10%. So the compressor will be more happy with - * short matches the farer it has to go back in the history. + * is lowered roughly by 10%. So the compressor will be more happy + * with short matches the further it has to go back in the history. * Another "speed against ratio" preference characteristic of * the algorithm. * - * Thus there are 3 stop conditions for the lookup of matches: + * Thus there are 2 stop conditions for the lookup of matches: * * - a match >= good_match is found * - there are no more history entries to look at - * - the next history entry is already too far back - * to be coded into a tag. * * Finally the match algorithm checks that at least a match * of 3 or more bytes has been found, because that is the smallest @@ -173,6 +171,10 @@ * Jan Wieck * * Copyright (c) 1999-2021, PostgreSQL Global Development Group + * Many thanks to Yann Collet for his article about unaligned + * memory access. + * + * Leskov Vladimir * * src/common/pg_lzcompress.c * ---------- @@ -188,12 +190,57 @@ #include "common/pg_lzcompress.h" +/************************************** + * CPU Feature Detection * + **************************************/ +/* PGLZ_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1) + */ +#ifndef PGLZ_FORCE_MEMORY_ACCESS /* can be defined externally */ +#if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +#define PGLZ_FORCE_MEMORY_ACCESS 1 +#endif +#endif + +#if defined(PGLZ_FORCE_MEMORY_ACCESS) && (PGLZ_FORCE_MEMORY_ACCESS==1) +/* lie to the compiler about data alignment; use with caution */ + +static uint32 +pglz_read32(const void *ptr) +{ + return *(const uint32 *) ptr; +} + +#else /* safe and portable access using memcpy() */ + +static uint32 +pglz_read32(const void *ptr) +{ + uint32 val; + + memcpy(&val, ptr, sizeof(val)); + return val; +} + +#endif /* PGLZ_FORCE_MEMORY_ACCESS */ + + /* ---------- * Local definitions * ---------- */ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ -#define PGLZ_HISTORY_SIZE 4096 +#define PGLZ_HISTORY_SIZE 0x0fff - 1 /* to avoid compare in iteration */ #define PGLZ_MAX_MATCH 273 @@ -202,17 +249,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked in a singly linked list. + * Links are not changed during insertion in order to speed it up. + * Instead more complicated stop condition is used during list iteration. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + int16 next_id; /* links for my hash key's list */ + uint16 hist_idx; /* my current hash key */ + const unsigned char *pos; /* my input position */ } PGLZ_HistEntry; @@ -256,11 +302,9 @@ static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -274,117 +318,111 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * hash keys more. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint16 +pglz_hist_idx(const unsigned char *s, uint16 mask) +{ + /* + * Note that we only calculate function at the beginning of compressed data. + * Further hash valued are computed by subtracting prev byte and adding next. + * For details see pglz_hist_add(). + */ + return ((s[0] << 6) ^ (s[1] << 4) ^ (s[2] << 2) ^ s[3]) & mask; +} /* ---------- * pglz_hist_add - * - * Adds a new entry to the history table. - * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. + * Adds a new entry to the history table + * and recalculates hash value. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline int16 +pglz_hist_add(int16 hist_next, uint16 *hist_idx, const unsigned char *s, uint16 mask) +{ + int16 *my_hist_start = &hist_start[*hist_idx]; + PGLZ_HistEntry *entry = &(hist_entries)[hist_next]; + /* + * Initialize entry with a new value. + */ + entry->next_id = *my_hist_start; + entry->hist_idx = *hist_idx; + entry->pos = s; -/* ---------- - * pglz_out_ctrl - - * - * Outputs the last and allocates a new control byte if needed. - * ---------- - */ -#define pglz_out_ctrl(__ctrlp,__ctrlb,__ctrl,__buf) \ -do { \ - if ((__ctrl & 0xff) == 0) \ - { \ - *(__ctrlp) = __ctrlb; \ - __ctrlp = (__buf)++; \ - __ctrlb = 0; \ - __ctrl = 1; \ - } \ -} while (0) + /* + * Update linked list head pointer. + */ + *my_hist_start = hist_next; + + /* + * Recalculate hash value for next position. Remove current byte and add next. + */ + *hist_idx = ((((*hist_idx) ^ (s[0] << 6)) << 2) ^ s[4]) & mask; + + /* + * Shift history pointer. + */ + hist_next++; + if (hist_next == PGLZ_HISTORY_SIZE + 1) + { + hist_next = 1; + } + return hist_next; +} /* ---------- - * pglz_out_literal - + * pglz_out_tag - * - * Outputs a literal byte to the destination buffer including the + * Outputs a backward reference tag of 2-4 bytes (depending on + * offset and length) to the destination buffer including the * appropriate control bit. * ---------- */ -#define pglz_out_literal(_ctrlp,_ctrlb,_ctrl,_buf,_byte) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - *(_buf)++ = (unsigned char)(_byte); \ - _ctrl <<= 1; \ -} while (0) +static inline unsigned char * +pglz_out_tag(unsigned char *dest_ptr, int32 match_len, int32 match_offset) +{ + if (match_len > 17) + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | 0x0f); + *(dest_ptr++) = (unsigned char) (((match_offset) & 0xff)); + *(dest_ptr++) = (unsigned char) ((match_len) - 18); + } + else + { + *(dest_ptr++) = (unsigned char) ((((match_offset) & 0xf00) >> 4) | ((match_len) - 3)); + *(dest_ptr++) = (unsigned char) ((match_offset) & 0xff); + } + return dest_ptr; +} /* ---------- - * pglz_out_tag - + * pglz_compare - * - * Outputs a backward reference tag of 2-4 bytes (depending on - * offset and length) to the destination buffer including the - * appropriate control bit. + * Compares two sequences of bytes + * and returns position of first mismatch. * ---------- */ -#define pglz_out_tag(_ctrlp,_ctrlb,_ctrl,_buf,_len,_off) \ -do { \ - pglz_out_ctrl(_ctrlp,_ctrlb,_ctrl,_buf); \ - _ctrlb |= _ctrl; \ - _ctrl <<= 1; \ - if (_len > 17) \ - { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | 0x0f); \ - (_buf)[1] = (unsigned char)(((_off) & 0xff)); \ - (_buf)[2] = (unsigned char)((_len) - 18); \ - (_buf) += 3; \ - } else { \ - (_buf)[0] = (unsigned char)((((_off) & 0xf00) >> 4) | ((_len) - 3)); \ - (_buf)[1] = (unsigned char)((_off) & 0xff); \ - (_buf) += 2; \ - } \ -} while (0) +static inline int32 +pglz_compare(int32 len, int32 len_bound, const unsigned char *input_pos, + const unsigned char *hist_pos) +{ + while (len <= len_bound - 4 && pglz_read32(input_pos) == pglz_read32(hist_pos)) + { + len += 4; + input_pos += 4; + hist_pos += 4; + } + while (len < len_bound && *input_pos == *hist_pos) + { + len++; + input_pos++; + hist_pos++; + } + return len; +} /* ---------- @@ -396,32 +434,40 @@ do { \ * ---------- */ static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +pglz_find_match(uint16 hist_idx, const unsigned char *input, const unsigned char *end, + int *lenp, int *offp, int good_match, int good_drop) { - PGLZ_HistEntry *hent; - int16 hentno; + PGLZ_HistEntry *hist_entry; + int16 *hist_entry_number; int32 len = 0; - int32 off = 0; + int32 offset = 0; + int32 cur_len = 0; + int32 len_bound = Min(end - input, PGLZ_MAX_MATCH); /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) - { - const char *ip = input; - const char *hp = hent->pos; - int32 thisoff; - int32 thislen; + hist_entry_number = &hist_start[hist_idx]; + if (*hist_entry_number == INVALID_ENTRY) + return 0; + hist_entry = &hist_entries[*hist_entry_number]; + if (hist_idx != hist_entry->hist_idx) + { /* - * Stop if the offset does not fit into our tag anymore. + * If current linked list head points to invalid entry then clear it + * to reduce the number of comparisons in future. */ - thisoff = ip - hp; - if (thisoff >= 0x0fff) - break; + *hist_entry_number = INVALID_ENTRY; + return 0; + } + + while (true) + { + const unsigned char *input_pos = input; + const unsigned char *hist_pos = hist_entry->pos; + const unsigned char *my_pos; + int32 cur_offset = input_pos - hist_pos; /* * Determine length of match. A better match must be larger than the @@ -431,58 +477,60 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, * character by character comparison to know the exact position where * the diff occurred. */ - thislen = 0; if (len >= 16) { - if (memcmp(ip, hp, len) == 0) + if (memcmp(input_pos, hist_pos, len) == 0) { - thislen = len; - ip += len; - hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + offset = cur_offset; + len = pglz_compare(len, len_bound, input_pos + len, hist_pos + len); } } else { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) + /* + * Start search for short matches by comparing 4 bytes + */ + if (pglz_read32(input_pos) == pglz_read32(hist_pos)) { - thislen++; - ip++; - hp++; + cur_len = pglz_compare(4, len_bound, input_pos + 4, hist_pos + 4); + if (cur_len > len) + { + len = cur_len; + offset = cur_offset; + } } } - /* - * Remember this match as the best (if it is) - */ - if (thislen > len) - { - len = thislen; - off = thisoff; - } - /* * Advance to the next history entry */ - hent = hent->next; + my_pos = hist_entry->pos; + hist_entry = &hist_entries[hist_entry->next_id]; /* - * Be happy with lesser good matches the more entries we visited. But - * no point in doing calculation if we're at end of list. + * If current match length is ok then stop iteration. As outdated list + * links are not updated during insertion process then additional stop + * condition should be introduced to avoid following them. If recycled + * entry has another hash, then iteration stops. If it has the same + * hash then recycled cell would break input stream position + * monotonicity, which is checked after. */ - if (hent != INVALID_ENTRY_PTR) + if (len >= good_match || hist_idx != hist_entry->hist_idx || my_pos <= hist_entry->pos) { - if (len >= good_match) - break; - good_match -= (good_match * good_drop) / 100; + break; } + + /* + * Be happy with less good matches the more entries we visited. + */ + good_match -= (good_match * good_drop) >> 7; } + /* + * found match can be slightly more than necessary, bound it with len_bound + */ + len = Min(len, len_bound); + /* * Return match information only if it results at least in one byte * reduction. @@ -490,7 +538,7 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, if (len > 2) { *lenp = len; - *offp = off; + *offp = offset; return 1; } @@ -509,26 +557,28 @@ int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = (unsigned char *) dest; - unsigned char *bstart = bp; - int hist_next = 1; - bool hist_recycle = false; - const char *dp = source; - const char *dend = source + slen; - unsigned char ctrl_dummy = 0; - unsigned char *ctrlp = &ctrl_dummy; - unsigned char ctrlb = 0; - unsigned char ctrl = 0; + unsigned char *dest_ptr = (unsigned char *) dest; + unsigned char *dest_start = dest_ptr; + uint16 hist_next = 1; + uint16 hist_idx; + const unsigned char *src_ptr = (const unsigned char *) source; + const unsigned char *src_end = (const unsigned char *) source + slen; + const unsigned char *compress_src_end = src_end - 4; + unsigned char control_dummy = 0; + unsigned char *control_ptr = &control_dummy; + unsigned char control_byte = 0; + unsigned char control_pos = 0; bool found_match = false; int32 match_len; - int32 match_off; + int32 match_offset; int32 good_match; int32 good_drop; int32 result_size; int32 result_max; int32 need_rate; int hashsz; - int mask; + uint16 mask; + /* * Our fallback strategy is the default. @@ -560,6 +610,9 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (good_drop > 100) good_drop = 100; + /* We use <<7 later to calculate actual drop, so align percents to 128 */ + good_drop = good_drop * 128 / 100; + need_rate = strategy->min_comp_rate; if (need_rate < 0) need_rate = 0; @@ -577,7 +630,9 @@ pglz_compress(const char *source, int32 slen, char *dest, result_max = (slen / 100) * (100 - need_rate); } else + { result_max = (slen * (100 - need_rate)) / 100; + } /* * Experiments suggest that these hash sizes work pretty well. A large @@ -603,10 +658,21 @@ pglz_compress(const char *source, int32 slen, char *dest, */ memset(hist_start, 0, hashsz * sizeof(int16)); + /* + * Initialize INVALID_ENTRY for stopping during lookup. + */ + hist_entries[INVALID_ENTRY].pos = src_end; + hist_entries[INVALID_ENTRY].hist_idx = hashsz; + + /* + * Calculate initial hash value. + */ + hist_idx = pglz_hist_idx(src_ptr, mask); + /* * Compress the source directly into the output buffer. */ - while (dp < dend) + while (src_ptr < compress_src_end) { /* * If we already exceeded the maximum result size, fail. @@ -615,7 +681,7 @@ pglz_compress(const char *source, int32 slen, char *dest, * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better * allow 4 slop bytes. */ - if (bp - bstart >= result_max) + if (dest_ptr - dest_start >= result_max) return -1; /* @@ -624,27 +690,36 @@ pglz_compress(const char *source, int32 slen, char *dest, * reasonably quickly when looking at incompressible input (such as * pre-compressed data). */ - if (!found_match && bp - bstart >= strategy->first_success_by) + if (!found_match && dest_ptr - dest_start >= strategy->first_success_by) return -1; + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; + } + /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + if (pglz_find_match(hist_idx, src_ptr, compress_src_end, &match_len, + &match_offset, good_match, good_drop)) { /* * Create the tag and add history entries for all matched * characters. */ - pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); + control_byte |= control_pos; + dest_ptr = pglz_out_tag(dest_ptr, match_len, match_offset); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + src_ptr++; } found_match = true; } @@ -653,21 +728,47 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * No match found. Copy one literal byte. */ - pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + hist_next = pglz_hist_add(hist_next, &hist_idx, src_ptr, mask); + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + } + control_pos <<= 1; + } + + + while (src_ptr < src_end) + { + /* + * If we already exceeded the maximum result size, fail. + * + * We check once per loop; since the loop body could emit as many as 4 + * bytes (a control byte and 3-byte tag), PGLZ_MAX_OUTPUT() had better + * allow 4 slop bytes. + */ + if (dest_ptr - dest_start >= result_max) + return -1; + + /* + * Refresh control byte if needed. + */ + if ((control_pos & 0xff) == 0) + { + *(control_ptr) = control_byte; + control_ptr = (dest_ptr)++; + control_byte = 0; + control_pos = 1; } + *(dest_ptr)++ = (unsigned char) (*src_ptr); + src_ptr++; + control_pos <<= 1; } /* * Write out the last control byte and check that we haven't overrun the * output size allowed by the strategy. */ - *ctrlp = ctrlb; - result_size = bp - bstart; + *control_ptr = control_byte; + result_size = dest_ptr - dest_start; if (result_size >= result_max) return -1; -- 2.17.0 --/9DWx/yDrRhgMJTb-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* code contributions for 2024, WIP version @ 2024-12-02 21:10 Robert Haas <[email protected]> 0 siblings, 2 replies; 35+ messages in thread From: Robert Haas @ 2024-12-02 21:10 UTC (permalink / raw) To: pgsql-hackers Hi, As many of you are probably aware, I have been doing an annual blog post on who contributes to PostgreSQL development for some years now. It includes information on lines of code committed to PostgreSQL, and also emails sent to the list. This year, I got a jump on analyzing the commit log, and a draft of the data covering January-November of 2024 has been uploaded in pg_dump format to here: https://sites.google.com/site/robertmhaas/contributions I'm sending this message to invite anyone who is interested to review the data in the commits2024 table and send me corrections. For example, it's possible that there are cases where I've failed to pick out the correct primary author for a commit; or where somebody's name is spelled in two different ways; or where somebody's name is not spelled the way that they prefer. You'll notice that the table has columns "lines" and "xlines". I have set xlines=0 in cases where (a) I considered the commit to be a large, mechanical commit such as a pgindent run or translation updates; or (b) the commit was reverting some other commit that occurred earlier in 2024; or (c) the commit was subsequently reverted. When I run the final statistics, those commits will still count for the statistics that count the number of commits, but the lines they inserted will not be counted as lines of code contributed in 2024. Also for clarity, please be aware that the "ncauthor" column is not used in the final reporting; that is just there so that I can set author=coalesce(ncauthor,committer) at a certain phase of the data preparation. Corrections should be made to the author column, not ncauthor. If you would like to correct the data, please send me your corrections off-list, as a reply to this email, ideally in the form of one or more UPDATE statements. If you would like to complain about the methodology, I can't stop you, but please bear in mind that (1) this is already a lot of work and (2) I've always been upfront in my blog post about what the limitations of the methodology are and I do my best not to suggest that this method is somehow perfect or unimpeachable and (3) you're welcome to publish your own blog post where you compute things differently. I'm open to reasonable suggestions for improvement, but if your overall view is that this sucks or that I suck for doing it, I'm sorry that you feel that way but giving me that feedback probably will not induce me to do anything differently. Donning my asbestos underwear, I remain yours faithfully, -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-03 01:16 Michael Paquier <[email protected]> parent: Robert Haas <[email protected]> 1 sibling, 1 reply; 35+ messages in thread From: Michael Paquier @ 2024-12-03 01:16 UTC (permalink / raw) To: Robert Haas <[email protected]>; +Cc: pgsql-hackers On Mon, Dec 02, 2024 at 04:10:22PM -0500, Robert Haas wrote: > Donning my asbestos underwear, I remain yours faithfully, Thanks for taking the time to compile all that. That's really nice. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-03 15:37 Nathan Bossart <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 35+ messages in thread From: Nathan Bossart @ 2024-12-03 15:37 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Robert Haas <[email protected]>; pgsql-hackers On Tue, Dec 03, 2024 at 10:16:35AM +0900, Michael Paquier wrote: > On Mon, Dec 02, 2024 at 04:10:22PM -0500, Robert Haas wrote: >> Donning my asbestos underwear, I remain yours faithfully, > > Thanks for taking the time to compile all that. That's really nice. +1, I always look forward to the blog post. -- nathan ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-03 15:44 Robert Haas <[email protected]> parent: Nathan Bossart <[email protected]> 0 siblings, 1 reply; 35+ messages in thread From: Robert Haas @ 2024-12-03 15:44 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers On Tue, Dec 3, 2024 at 10:37 AM Nathan Bossart <[email protected]> wrote: > On Tue, Dec 03, 2024 at 10:16:35AM +0900, Michael Paquier wrote: > > On Mon, Dec 02, 2024 at 04:10:22PM -0500, Robert Haas wrote: > >> Donning my asbestos underwear, I remain yours faithfully, > > > > Thanks for taking the time to compile all that. That's really nice. > > +1, I always look forward to the blog post. Thanks, glad it's appreciated. -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-03 16:19 Joe Conway <[email protected]> parent: Robert Haas <[email protected]> 0 siblings, 2 replies; 35+ messages in thread From: Joe Conway @ 2024-12-03 16:19 UTC (permalink / raw) To: Robert Haas <[email protected]>; Nathan Bossart <[email protected]>; +Cc: Michael Paquier <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Christoph Berg <[email protected]>; Melanie Plageman <[email protected]>; Claire Giordano <[email protected]> On 12/3/24 10:44, Robert Haas wrote: > On Tue, Dec 3, 2024 at 10:37 AM Nathan Bossart <[email protected]> wrote: >> On Tue, Dec 03, 2024 at 10:16:35AM +0900, Michael Paquier wrote: >> > On Mon, Dec 02, 2024 at 04:10:22PM -0500, Robert Haas wrote: >> >> Donning my asbestos underwear, I remain yours faithfully, >> > >> > Thanks for taking the time to compile all that. That's really nice. >> >> +1, I always look forward to the blog post. > > Thanks, glad it's appreciated. It is definitely appreciated. While I know you said "you will do you" when it comes to your annual blog, there are a number of similar efforts -- top of mind is the analysis done (as I understand it) by Daniel Gustafsson and Claire Giordano [1], as well as ongoing/recurring analysis done by the contributor committee. And there is the adjacent related discussion around commit messages/authors. It makes me wonder if there isn't a way to make all of our lives easier going forward. [1] https://speakerdeck.com/clairegiordano/whats-in-a-postgres-major-release-an-analysis-of-contribution... -- Joe Conway PostgreSQL Contributors Team RDS Open Source Databases Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-03 16:41 Robert Haas <[email protected]> parent: Joe Conway <[email protected]> 1 sibling, 1 reply; 35+ messages in thread From: Robert Haas @ 2024-12-03 16:41 UTC (permalink / raw) To: Joe Conway <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Christoph Berg <[email protected]>; Melanie Plageman <[email protected]>; Claire Giordano <[email protected]> On Tue, Dec 3, 2024 at 11:19 AM Joe Conway <[email protected]> wrote: > While I know you said "you will do you" when it comes to your annual > blog, there are a number of similar efforts -- top of mind is the > analysis done (as I understand it) by Daniel Gustafsson and Claire > Giordano [1], as well as ongoing/recurring analysis done by the > contributor committee. And there is the adjacent related discussion > around commit messages/authors. It makes me wonder if there isn't a way > to make all of our lives easier going forward. Yes, I'm game to try to figure out how to combine our efforts. I don't think it's a bad thing that different people have different takes; this is complicated and looking at it through just one lens is limiting. But people duplicating work is, well, not so good. -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-03 21:07 Daniel Gustafsson <[email protected]> parent: Robert Haas <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Daniel Gustafsson @ 2024-12-03 21:07 UTC (permalink / raw) To: Robert Haas <[email protected]>; +Cc: Joe Conway <[email protected]>; Nathan Bossart <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers; Christoph Berg <[email protected]>; Melanie Plageman <[email protected]>; Claire Giordano <[email protected]> > On 3 Dec 2024, at 17:41, Robert Haas <[email protected]> wrote: > > On Tue, Dec 3, 2024 at 11:19 AM Joe Conway <[email protected]> wrote: >> While I know you said "you will do you" when it comes to your annual >> blog, there are a number of similar efforts -- top of mind is the >> analysis done (as I understand it) by Daniel Gustafsson and Claire >> Giordano [1], as well as ongoing/recurring analysis done by the >> contributor committee. And there is the adjacent related discussion >> around commit messages/authors. It makes me wonder if there isn't a way >> to make all of our lives easier going forward. > > Yes, I'm game to try to figure out how to combine our efforts. I don't > think it's a bad thing that different people have different takes; > this is complicated and looking at it through just one lens is > limiting. But people duplicating work is, well, not so good. If we settled on a meta-data standard for how to identify authors, reviewers, backpatches etc I think that would go a very long way to lower the complexity of getting to the data and keep folks focused on doing interesting analysis. -- Daniel Gustafsson ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-05 12:46 Alvaro Herrera <[email protected]> parent: Robert Haas <[email protected]> 1 sibling, 2 replies; 35+ messages in thread From: Alvaro Herrera @ 2024-12-05 12:46 UTC (permalink / raw) To: Robert Haas <[email protected]>; +Cc: pgsql-hackers Hello Robert, On 2024-Dec-02, Robert Haas wrote: > As many of you are probably aware, I have been doing an annual blog > post on who contributes to PostgreSQL development for some years now. > It includes information on lines of code committed to PostgreSQL, and > also emails sent to the list. This year, I got a jump on analyzing the > commit log, and a draft of the data covering January-November of 2024 > has been uploaded in pg_dump format to here: > > https://sites.google.com/site/robertmhaas/contributions > > I'm sending this message to invite anyone who is interested to review > the data in the commits2024 table and send me corrections. No corrections here -- I noticed nothing wrong with the commits I am involved with, in a quick read. I did notice that for patches with multiple authors, only the first one is listed. For instance, 53c2a97a926's author ("Improve performance of subsystems on top of SLRU") is listed as Andrey Borodin, leaving Dilip Kumar out. I realize that addressing this would complicate the schema and queries, but maybe it's worth thinking about for next time. We have plenty of patches with multiple authors, after all. Hmm, maybe UPDATE commits2024 SET xlines = 0 WHERE commitid in ('43ce181059d', '4632e5cf4bc', '6377e12a5a5', 'ff9f72c68f6', '21ef4d4d897', '592a2283721'); How did you come up with the 'lines' number for each commit anyway? Judging by 592a2283721 it's not just the number of lines added, since that commit added 3 lines and you have lines=2. An unrelated (and possibly useless) thing is that some committers seem firmly in the camp of ending commit titles with a period, others are firmly in the other camp; only two people seem not to have made up their minds about that: committer │ with end period │ without end period │ fraction with end period ────────────────────┼─────────────────┼────────────────────┼────────────────────────── Etsuro Fujita │ 6 │ 0 │ 100.00 Peter Geoghegan │ 39 │ 0 │ 100.00 Tatsuo Ishii │ 8 │ 0 │ 100.00 Amit Kapila │ 87 │ 0 │ 100.00 Fujii Masao │ 35 │ 0 │ 100.00 Tom Lane │ 296 │ 1 │ 99.66 Nathan Bossart │ 131 │ 1 │ 99.24 Jeff Davis │ 88 │ 1 │ 98.88 Noah Misch │ 61 │ 1 │ 98.39 Thomas Munro │ 59 │ 1 │ 98.33 Masahiko Sawada │ 39 │ 1 │ 97.50 Dean Rasheed │ 23 │ 1 │ 95.83 Robert Haas │ 77 │ 10 │ 88.51 Joe Conway │ 1 │ 2 │ 33.33 Alexander Korotkov │ 4 │ 153 │ 2.55 Andrew Dunstan │ 1 │ 40 │ 2.44 Bruce Momjian │ 2 │ 82 │ 2.38 Heikki Linnakangas │ 4 │ 174 │ 2.25 Peter Eisentraut │ 6 │ 309 │ 1.90 Amit Langote │ 1 │ 54 │ 1.82 Álvaro Herrera │ 1 │ 118 │ 0.84 Michael Paquier │ 1 │ 275 │ 0.36 Andres Freund │ 0 │ 26 │ 0.00 Richard Guo │ 0 │ 27 │ 0.00 Daniel Gustafsson │ 0 │ 99 │ 0.00 Magnus Hagander │ 0 │ 4 │ 0.00 John Naylor │ 0 │ 33 │ 0.00 Melanie Plageman │ 0 │ 6 │ 0.00 David Rowley │ 0 │ 106 │ 0.00 Tomas Vondra │ 0 │ 33 │ 0.00 Query was: select committer, count(*) filter (where subject like '%.') as "with end period", count(*) filter (where subject not like '%.') "without end period", ((count(*) filter (where subject like '%.'))::numeric / count(*) * 100)::numeric(5,2) as "fraction with end period" from commits2024 group by committer order by 4 desc, split_part(committer, ' ', 2); Thanks! -- Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/ "The problem with the facetime model is not just that it's demoralizing, but that the people pretending to work interrupt the ones actually working." -- Paul Graham, http://www.paulgraham.com/opensource.html ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-05 13:43 Andrey M. Borodin <[email protected]> parent: Alvaro Herrera <[email protected]> 1 sibling, 0 replies; 35+ messages in thread From: Andrey M. Borodin @ 2024-12-05 13:43 UTC (permalink / raw) To: Alvaro Herrera <[email protected]>; +Cc: Robert Haas <[email protected]>; pgsql-hackers > On 5 Dec 2024, at 17:46, Alvaro Herrera <[email protected]> wrote: > > We have plenty of patches with > multiple authors, after all. +1, thanks for raising this. A lot of stuff is actually joint work. It’s much more fun to develop something in a group of co-authors. Best regards, Andrey Borodin. ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-05 15:23 Robert Haas <[email protected]> parent: Alvaro Herrera <[email protected]> 1 sibling, 1 reply; 35+ messages in thread From: Robert Haas @ 2024-12-05 15:23 UTC (permalink / raw) To: Alvaro Herrera <[email protected]>; +Cc: pgsql-hackers On Thu, Dec 5, 2024 at 7:46 AM Alvaro Herrera <[email protected]> wrote: > No corrections here -- I noticed nothing wrong with the commits I am > involved with, in a quick read. I did notice that for patches with > multiple authors, only the first one is listed. For instance, > 53c2a97a926's author ("Improve performance of subsystems on top of > SLRU") is listed as Andrey Borodin, leaving Dilip Kumar out. I realize > that addressing this would complicate the schema and queries, but maybe > it's worth thinking about for next time. We have plenty of patches with > multiple authors, after all. I agree, but I don't know how to apportion the work between the authors. I think dividing credit equally between two or three authors would often be very unfair to the first author. If we want to annotate commit messages in a way that allows me to apportion credit more fairly, I'm totally game to do that, but otherwise I think that giving the credit to the first author is probably more fair on average. > Hmm, maybe > UPDATE commits2024 SET xlines = 0 WHERE commitid in > ('43ce181059d', '4632e5cf4bc', '6377e12a5a5', 'ff9f72c68f6', > '21ef4d4d897', '592a2283721'); Thanks. > How did you come up with the 'lines' number for each commit anyway? > Judging by 592a2283721 it's not just the number of lines added, since > that commit added 3 lines and you have lines=2. git log --before=${YEAR}-12-31 --after=${YEAR}-01-01 --shortstat -w -M -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-05 15:39 Tom Kincaid <[email protected]> parent: Joe Conway <[email protected]> 1 sibling, 1 reply; 35+ messages in thread From: Tom Kincaid @ 2024-12-05 15:39 UTC (permalink / raw) To: Joe Conway <[email protected]>; +Cc: Robert Haas <[email protected]>; Nathan Bossart <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Christoph Berg <[email protected]>; Melanie Plageman <[email protected]>; Claire Giordano <[email protected]> > > > While I know you said "you will do you" when it comes to your annual > blog, there are a number of similar efforts -- top of mind is the > analysis done (as I understand it) by Daniel Gustafsson and Claire > Giordano [1], as well as ongoing/recurring analysis done by the > contributor committee. And there is the adjacent related discussion > around commit messages/authors. It makes me wonder if there isn't a way > to make all of our lives easier going forward. > Perhaps slightly off topic, so how does one provide input to the contributor committee? > > [1] > > https://speakerdeck.com/clairegiordano/whats-in-a-postgres-major-release-an-analysis-of-contribution... > -- > Joe Conway > PostgreSQL Contributors Team > RDS Open Source Databases > Amazon Web Services: https://aws.amazon.com > > > -- Thomas John Kincaid ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-05 16:19 Alvaro Herrera <[email protected]> parent: Robert Haas <[email protected]> 0 siblings, 1 reply; 35+ messages in thread From: Alvaro Herrera @ 2024-12-05 16:19 UTC (permalink / raw) To: Robert Haas <[email protected]>; +Cc: pgsql-hackers On 2024-Dec-05, Robert Haas wrote: > On Thu, Dec 5, 2024 at 7:46 AM Alvaro Herrera <[email protected]> wrote: > > No corrections here -- I noticed nothing wrong with the commits I am > > involved with, in a quick read. I did notice that for patches with > > multiple authors, only the first one is listed. For instance, > > 53c2a97a926's author ("Improve performance of subsystems on top of > > SLRU") is listed as Andrey Borodin, leaving Dilip Kumar out. I realize > > that addressing this would complicate the schema and queries, but maybe > > it's worth thinking about for next time. We have plenty of patches with > > multiple authors, after all. > > I agree, but I don't know how to apportion the work between the > authors. I think dividing credit equally between two or three authors > would often be very unfair to the first author. If we want to annotate > commit messages in a way that allows me to apportion credit more > fairly, I'm totally game to do that, but otherwise I think that giving > the credit to the first author is probably more fair on average. Just give credit to all lines for all authors, would be my approach. Is that unfair? Perhaps, but I'd rather err on the side of giving too much credit, than on not giving enough. > > How did you come up with the 'lines' number for each commit anyway? > > Judging by 592a2283721 it's not just the number of lines added, since > > that commit added 3 lines and you have lines=2. > > git log --before=${YEAR}-12-31 --after=${YEAR}-01-01 --shortstat -w -M Ah, it's -w that makes the difference, got it. -- Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/ "Right now the sectors on the hard disk run clockwise, but I heard a rumor that you can squeeze 0.2% more throughput by running them counterclockwise. It's worth the effort. Recommended." (Gerry Pourwelle) ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-05 16:42 Bruce Momjian <[email protected]> parent: Tom Kincaid <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Bruce Momjian @ 2024-12-05 16:42 UTC (permalink / raw) To: Tom Kincaid <[email protected]>; +Cc: Joe Conway <[email protected]>; Robert Haas <[email protected]>; Nathan Bossart <[email protected]>; Michael Paquier <[email protected]>; pgsql-hackers; Daniel Gustafsson <[email protected]>; Christoph Berg <[email protected]>; Melanie Plageman <[email protected]>; Claire Giordano <[email protected]> On Thu, Dec 5, 2024 at 10:39:38AM -0500, Tom Kincaid wrote: > While I know you said "you will do you" when it comes to your annual > blog, there are a number of similar efforts -- top of mind is the > analysis done (as I understand it) by Daniel Gustafsson and Claire > Giordano [1], as well as ongoing/recurring analysis done by the > contributor committee. And there is the adjacent related discussion > around commit messages/authors. It makes me wonder if there isn't a way > to make all of our lives easier going forward. > > Perhaps slightly off topic, so how does one provide input to the contributor > committee? The committee is responsible for updating the contributors list web page: https://www.postgresql.org/community/contributors/ and does analysis of contributions to the Postgres community to help update the list. Their email address at the bottom: [email protected] -- Bruce Momjian <[email protected]> https://momjian.us EDB https://enterprisedb.com Do not let urgent matters crowd out time for investment in the future. ^ permalink raw reply [nested|flat] 35+ messages in thread
* Re: code contributions for 2024, WIP version @ 2024-12-05 16:43 Robert Haas <[email protected]> parent: Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Robert Haas @ 2024-12-05 16:43 UTC (permalink / raw) To: Alvaro Herrera <[email protected]>; +Cc: pgsql-hackers On Thu, Dec 5, 2024 at 11:19 AM Alvaro Herrera <[email protected]> wrote: > Just give credit to all lines for all authors, would be my approach. Is > that unfair? Perhaps, but I'd rather err on the side of giving too much > credit, than on not giving enough. I'm not against somebody putting that together, but I don't think it would be useful for me. I think it would inflate the numbers for committers by quite a lot more than what is fair, because if I commit a 1000 line patch and I add 50 lines of code, I'm going to get an awful lot more credit than I deserve. It would probably also inflate or distort the numbers for some other people as well. But what I would say is -- if you think it's a useful thing, try doing it. -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 35+ messages in thread
end of thread, other threads:[~2024-12-05 16:43 UTC | newest] Thread overview: 35+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2019-06-27 18:18 [PATCH] Reorganize pglz compression code Andrey <[email protected]> 2024-12-02 21:10 code contributions for 2024, WIP version Robert Haas <[email protected]> 2024-12-03 01:16 ` Re: code contributions for 2024, WIP version Michael Paquier <[email protected]> 2024-12-03 15:37 ` Re: code contributions for 2024, WIP version Nathan Bossart <[email protected]> 2024-12-03 15:44 ` Re: code contributions for 2024, WIP version Robert Haas <[email protected]> 2024-12-03 16:19 ` Re: code contributions for 2024, WIP version Joe Conway <[email protected]> 2024-12-03 16:41 ` Re: code contributions for 2024, WIP version Robert Haas <[email protected]> 2024-12-03 21:07 ` Re: code contributions for 2024, WIP version Daniel Gustafsson <[email protected]> 2024-12-05 15:39 ` Re: code contributions for 2024, WIP version Tom Kincaid <[email protected]> 2024-12-05 16:42 ` Re: code contributions for 2024, WIP version Bruce Momjian <[email protected]> 2024-12-05 12:46 ` Re: code contributions for 2024, WIP version Alvaro Herrera <[email protected]> 2024-12-05 13:43 ` Re: code contributions for 2024, WIP version Andrey M. Borodin <[email protected]> 2024-12-05 15:23 ` Re: code contributions for 2024, WIP version Robert Haas <[email protected]> 2024-12-05 16:19 ` Re: code contributions for 2024, WIP version Alvaro Herrera <[email protected]> 2024-12-05 16:43 ` Re: code contributions for 2024, WIP version Robert Haas <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox